From 65cdf52061be28e11033bf893b6ee65bb10e4a94 Mon Sep 17 00:00:00 2001 From: Bryan Helmkamp Date: Thu, 23 Jul 2026 10:12:25 -0400 Subject: [PATCH 01/16] feat: make model aliases provider-aware --- .../fabro-web/app/lib/model-offerings.test.ts | 32 + apps/fabro-web/app/lib/model-offerings.ts | 13 + apps/fabro-web/app/routes/settings-models.tsx | 48 +- docs/public/api-reference/fabro-api.yaml | 68 +- docs/public/changelog/2026-05-12.mdx | 2 +- docs/public/changelog/2026-05-13.mdx | 3 +- docs/public/core-concepts/models.mdx | 62 +- docs/public/execution/failures.mdx | 8 +- docs/public/execution/run-configuration.mdx | 6 +- docs/public/integrations/bedrock.mdx | 2 +- docs/public/integrations/litellm.mdx | 16 +- docs/public/integrations/openrouter.mdx | 39 +- docs/public/reference/user-configuration.mdx | 57 +- lib/crates/fabro-agent/src/agent_profile.rs | 6 +- lib/crates/fabro-agent/src/cli.rs | 21 +- lib/crates/fabro-agent/src/session.rs | 4 +- lib/crates/fabro-agent/tests/it/guardrails.rs | 8 +- .../fabro-api/tests/model_round_trip.rs | 2 +- .../fabro-api/tests/provider_id_round_trip.rs | 2 +- .../tests/session_contract_round_trip.rs | 2 + lib/crates/fabro-cli/Cargo.toml | 1 + lib/crates/fabro-cli/src/commands/model.rs | 170 ++- lib/crates/fabro-cli/src/commands/run/ask.rs | 5 +- .../src/commands/run/run_progress/event.rs | 2 +- .../fabro-cli/src/shared/provider_auth.rs | 7 +- .../fabro-cli/tests/it/cmd/model_test.rs | 5 + .../tests/it/support/auth_harness.rs | 6 +- lib/crates/fabro-client/src/client.rs | 4 + lib/crates/fabro-config/src/builders.rs | 9 +- lib/crates/fabro-config/src/layers/llm.rs | 292 +++- lib/crates/fabro-config/src/layers/mod.rs | 6 +- lib/crates/fabro-config/src/lib.rs | 25 +- lib/crates/fabro-config/src/parse.rs | 203 ++- .../src/commands/docs_options_reference.rs | 46 +- lib/crates/fabro-hooks/src/executor.rs | 13 +- lib/crates/fabro-llm/src/adapter_registry.rs | 33 +- lib/crates/fabro-llm/src/client.rs | 431 +++++- .../src/codec/anthropic_messages/encode.rs | 8 +- .../src/codec/bedrock_converse/encode.rs | 4 +- .../src/codec/openai_compatible/request.rs | 10 +- lib/crates/fabro-llm/src/cost.rs | 11 +- lib/crates/fabro-llm/src/model_test.rs | 6 +- .../fabro-llm/src/providers/anthropic.rs | 27 +- .../fabro-llm/src/providers/bedrock/mod.rs | 18 +- lib/crates/fabro-llm/src/providers/common.rs | 14 +- lib/crates/fabro-llm/src/providers/gemini.rs | 24 +- lib/crates/fabro-llm/src/providers/openai.rs | 24 +- .../src/providers/openai_compatible.rs | 8 +- lib/crates/fabro-llm/tests/integration.rs | 4 +- lib/crates/fabro-model/src/billing.rs | 46 +- lib/crates/fabro-model/src/catalog.rs | 1290 ++++++++++++++--- .../src/catalog/providers/anthropic.toml | 84 +- .../src/catalog/providers/bedrock-openai.toml | 20 +- .../src/catalog/providers/bedrock.toml | 142 +- .../src/catalog/providers/gemini.toml | 50 +- .../src/catalog/providers/inception.toml | 10 +- .../src/catalog/providers/kimi.toml | 22 +- .../src/catalog/providers/litellm.toml | 7 +- .../src/catalog/providers/minimax.toml | 10 +- .../src/catalog/providers/ollama.toml | 7 +- .../src/catalog/providers/openai.toml | 86 +- .../src/catalog/providers/openrouter.toml | 248 ++-- .../src/catalog/providers/poolside.toml | 18 +- .../src/catalog/providers/venice.toml | 20 +- .../src/catalog/providers/zai.toml | 22 +- lib/crates/fabro-model/src/ids.rs | 35 +- lib/crates/fabro-model/src/lib.rs | 2 +- lib/crates/fabro-model/src/model_ref.rs | 2 +- lib/crates/fabro-model/src/types.rs | 8 +- lib/crates/fabro-server/src/diagnostics.rs | 11 +- lib/crates/fabro-server/src/install.rs | 2 +- lib/crates/fabro-server/src/run_manifest.rs | 76 +- lib/crates/fabro-server/src/server.rs | 2 +- .../src/server/automation_scheduler.rs | 1 + .../src/server/handler/completions.rs | 141 +- .../fabro-server/src/server/handler/models.rs | 48 +- .../fabro-server/src/server/handler/pair.rs | 4 +- .../src/server/handler/playground.rs | 58 +- .../src/server/handler/pull_requests.rs | 5 +- .../fabro-server/src/server/handler/runs.rs | 22 +- .../src/server/handler/sessions.rs | 379 ++++- lib/crates/fabro-server/src/server/tests.rs | 325 ++++- lib/crates/fabro-server/src/spawn_env.rs | 2 + lib/crates/fabro-server/src/test_support.rs | 48 +- .../fabro-server/tests/it/api/automations.rs | 2 + .../fabro-server/tests/it/api/events.rs | 2 + .../fabro-server/tests/it/api/sessions.rs | 54 +- .../fabro-server/tests/it/scenario/dry_run.rs | 19 +- .../tests/it/scenario/run_completion.rs | 4 +- lib/crates/fabro-store/src/run_sessions.rs | 33 +- lib/crates/fabro-store/src/run_state.rs | 2 +- lib/crates/fabro-test/src/lib.rs | 3 + lib/crates/fabro-types/src/run_event/agent.rs | 2 +- lib/crates/fabro-types/src/run_event/mod.rs | 4 +- .../fabro-types/src/run_event/session.rs | 7 +- lib/crates/fabro-types/src/session.rs | 7 + .../fabro-types/src/settings/model_ref.rs | 20 +- .../fabro-validate/src/rules/model_support.rs | 2 +- lib/crates/fabro-workflow/Cargo.toml | 3 + .../fabro-workflow/src/billing_rollup.rs | 2 +- lib/crates/fabro-workflow/src/error.rs | 12 + .../fabro-workflow/src/event/convert.rs | 6 +- .../fabro-workflow/src/event/stored_fields.rs | 2 +- .../fabro-workflow/src/handler/llm/api.rs | 16 +- .../src/handler/llm/preamble.rs | 2 +- .../fabro-workflow/src/handler/llm/routing.rs | 10 +- .../fabro-workflow/src/operations/create.rs | 179 ++- .../fabro-workflow/src/operations/start.rs | 394 ++++- .../fabro-workflow/src/operations/validate.rs | 14 + lib/crates/fabro-workflow/src/outcome.rs | 2 +- .../src/pipeline/pull_request.rs | 21 +- .../fabro-workflow/src/pipeline/transform.rs | 77 +- .../fabro-workflow/src/pipeline/types.rs | 18 +- .../fabro-workflow/src/pipeline/validate.rs | 16 +- .../fabro-workflow/src/run_materialization.rs | 89 +- lib/crates/fabro-workflow/src/test_support.rs | 15 + .../src/transforms/model_resolution.rs | 163 ++- .../fabro-workflow/tests/it/integration.rs | 16 +- .../fabro-workflow/tests/materialize_run.rs | 8 +- .../src/.openapi-generator/FILES | 2 + .../fabro-api-client/src/api/models-api.ts | 124 +- .../src/models/completion-response.ts | 7 + .../src/models/create-completion-request.ts | 4 +- .../models/create-playground-chat-request.ts | 6 +- .../src/models/create-run-session-request.ts | 6 +- .../src/models/hook-definition.ts | 3 + .../fabro-api-client/src/models/index.ts | 2 + .../src/models/model-test-result.ts | 6 +- .../fabro-api-client/src/models/model.ts | 4 +- .../provider-credential-test-request.ts | 22 + .../provider-credential-test-response.ts | 22 + .../src/models/session-detail.ts | 7 + .../src/models/session-record.ts | 7 + .../src/models/session-summary.ts | 7 + 134 files changed, 5065 insertions(+), 1460 deletions(-) create mode 100644 apps/fabro-web/app/lib/model-offerings.test.ts create mode 100644 apps/fabro-web/app/lib/model-offerings.ts create mode 100644 lib/packages/fabro-api-client/src/models/provider-credential-test-request.ts create mode 100644 lib/packages/fabro-api-client/src/models/provider-credential-test-response.ts diff --git a/apps/fabro-web/app/lib/model-offerings.test.ts b/apps/fabro-web/app/lib/model-offerings.test.ts new file mode 100644 index 000000000..a54a1b0e8 --- /dev/null +++ b/apps/fabro-web/app/lib/model-offerings.test.ts @@ -0,0 +1,32 @@ +import { describe, expect, test } from "bun:test"; +import { + modelOfferingKey, + modelOfferingTestArgs, +} from "./model-offerings"; + +describe("model offering identity", () => { + const openai = { id: "portable-model", provider: "openai" }; + const openrouter = { id: "portable-model", provider: "openrouter" }; + + test("keeps duplicate model IDs in independent row state", () => { + expect(modelOfferingKey(openai)).not.toBe(modelOfferingKey(openrouter)); + + const state = new Map([ + [modelOfferingKey(openai), "ok"], + [modelOfferingKey(openrouter), "error"], + ]); + expect(state.get(modelOfferingKey(openai))).toBe("ok"); + expect(state.get(modelOfferingKey(openrouter))).toBe("error"); + }); + + test("includes the row provider in model-test request arguments", () => { + expect(modelOfferingTestArgs(openai)).toEqual([ + "portable-model", + "openai", + ]); + expect(modelOfferingTestArgs(openrouter)).toEqual([ + "portable-model", + "openrouter", + ]); + }); +}); diff --git a/apps/fabro-web/app/lib/model-offerings.ts b/apps/fabro-web/app/lib/model-offerings.ts new file mode 100644 index 000000000..e2bc94d2d --- /dev/null +++ b/apps/fabro-web/app/lib/model-offerings.ts @@ -0,0 +1,13 @@ +import type { Model } from "@qltysh/fabro-api-client"; + +type ModelOfferingIdentity = Pick; + +export function modelOfferingKey(model: ModelOfferingIdentity): string { + return `${model.provider}\u0000${model.id}`; +} + +export function modelOfferingTestArgs( + model: ModelOfferingIdentity, +): [id: string, provider: string] { + return [model.id, model.provider]; +} diff --git a/apps/fabro-web/app/routes/settings-models.tsx b/apps/fabro-web/app/routes/settings-models.tsx index 57b7429ed..f6e3905df 100644 --- a/apps/fabro-web/app/routes/settings-models.tsx +++ b/apps/fabro-web/app/routes/settings-models.tsx @@ -28,6 +28,10 @@ import { } from "../components/runs-list/sort-header"; import { Tooltip } from "../components/ui"; import { formatContextWindow, formatTokensPerSecond } from "../lib/format"; +import { + modelOfferingKey, + modelOfferingTestArgs, +} from "../lib/model-offerings"; import { useDebouncedValue } from "../hooks/effects"; export function meta() { @@ -277,30 +281,46 @@ function ModelsSection({ providers }: { providers: Provider[] }) { const runSweep = useCallback(async () => { if (running) return; - const ids = rows.map((r) => r.id); - if (ids.length === 0) return; + const offerings = rows.map((model) => ({ + id: model.id, + provider: model.provider, + key: modelOfferingKey(model), + })); + if (offerings.length === 0) return; const seed = new Map(); - for (const id of ids) seed.set(id, { phase: "queued" }); + for (const offering of offerings) { + seed.set(offering.key, { phase: "queued" }); + } setResults(seed); - setSweep({ done: 0, total: ids.length, ok: 0, failed: 0 }); + setSweep({ done: 0, total: offerings.length, ok: 0, failed: 0 }); let cursor = 0; const worker = async () => { - while (cursor < ids.length) { + while (cursor < offerings.length) { const i = cursor; cursor += 1; - const id = ids[i]; + const offering = offerings[i]; setResults((prev) => { const next = new Map(prev); - next.set(id, { phase: "running" }); + next.set(offering.key, { phase: "running" }); return next; }); let outcome: RowState; try { - const result = await apiData(() => modelsApi.testModel(id)); - if (result.status === "ok") { + const result = await apiData(() => + modelsApi.testModel(...modelOfferingTestArgs(offering)), + ); + if ( + result.provider !== offering.provider || + result.model_id !== offering.id + ) { + outcome = { + phase: "error", + message: `Server tested unexpected offering ${result.provider}/${result.model_id}`, + }; + } else if (result.status === "ok") { outcome = { phase: "ok" }; } else if (result.status === "error") { outcome = { @@ -319,7 +339,7 @@ function ModelsSection({ providers }: { providers: Provider[] }) { setResults((prev) => { const next = new Map(prev); - next.set(id, outcome); + next.set(offering.key, outcome); return next; }); setSweep((prev) => @@ -336,7 +356,7 @@ function ModelsSection({ providers }: { providers: Provider[] }) { }; await Promise.all( - Array.from({ length: Math.min(TEST_CONCURRENCY, ids.length) }, () => + Array.from({ length: Math.min(TEST_CONCURRENCY, offerings.length) }, () => worker(), ), ); @@ -435,12 +455,12 @@ function ModelsSection({ providers }: { providers: Provider[] }) { {rows.map((model) => ( ))} @@ -468,7 +488,7 @@ function ModelTableRow({ state: RowState | undefined; }) { return ( - + {providerLabel} diff --git a/docs/public/api-reference/fabro-api.yaml b/docs/public/api-reference/fabro-api.yaml index d6e384405..ca80fd423 100644 --- a/docs/public/api-reference/fabro-api.yaml +++ b/docs/public/api-reference/fabro-api.yaml @@ -5452,7 +5452,9 @@ paths: operationId: listModels tags: [Models] summary: List Models - description: Returns a paginated list of available LLM models from the built-in catalog. + description: | + Returns one row per provider/model offering from the catalog. Model IDs + are unique within a provider; `(provider, id)` is the resource identity. parameters: - $ref: "#/components/parameters/ModelProviderFilter" - $ref: "#/components/parameters/ModelQueryFilter" @@ -5487,7 +5489,8 @@ paths: required: true schema: type: string - description: The model identifier. + description: The canonical model ID or an alias. + - $ref: "#/components/parameters/ModelTestProviderParam" - $ref: "#/components/parameters/ModelTestModeParam" responses: "200": @@ -5996,6 +5999,17 @@ components: $ref: "#/components/schemas/ModelTestMode" example: basic + ModelTestProviderParam: + name: provider + in: query + required: false + description: | + Pin the test to this provider's offering. When omitted, the server + selects among ready providers by catalog priority. + schema: + $ref: "#/components/schemas/ProviderId" + example: openrouter + headers: XRequestId: description: > @@ -7766,6 +7780,12 @@ components: $ref: "#/components/schemas/SessionStatus" model: type: ["string", "null"] + description: Canonical model ID selected when the session was created. + provider: + oneOf: + - $ref: "#/components/schemas/ProviderId" + - type: "null" + description: Provider selected when the session was created. active_turn: oneOf: - $ref: "#/components/schemas/SessionTurn" @@ -7798,6 +7818,12 @@ components: $ref: "#/components/schemas/SessionStatus" model: type: ["string", "null"] + description: Canonical model ID selected when the session was created. + provider: + oneOf: + - $ref: "#/components/schemas/ProviderId" + - type: "null" + description: Provider selected when the session was created. active_turn: oneOf: - $ref: "#/components/schemas/SessionTurn" @@ -7832,6 +7858,12 @@ components: $ref: "#/components/schemas/SessionStatus" model: type: ["string", "null"] + description: Canonical model ID selected when the session was created. + provider: + oneOf: + - $ref: "#/components/schemas/ProviderId" + - type: "null" + description: Provider selected when the session was created. active_turn: oneOf: - $ref: "#/components/schemas/SessionTurn" @@ -7857,7 +7889,12 @@ components: type: string model: type: string - description: Catalog model ID or alias, or provider-qualified provider/model reference. Stored as the canonical catalog model ID. + description: | + Catalog model ID or alias. The server selects among ready + providers and stores the canonical model ID. + provider: + $ref: "#/components/schemas/ProviderId" + description: Optional provider pin. Provider-qualified model references remain accepted for compatibility. SubmitTurnRequest: type: object @@ -8235,7 +8272,9 @@ components: example: 1.50 Model: - description: An available LLM model from the built-in catalog. + description: | + One provider's offering of an LLM model. The `id` is unique within + `provider`; `(provider, id)` is the stable resource identity. type: object required: - id @@ -8255,7 +8294,7 @@ components: properties: id: type: string - description: Unique model identifier. + description: Canonical human-facing model ID, unique within the provider. example: "claude-opus-4-6" provider: $ref: "#/components/schemas/ProviderId" @@ -8310,12 +8349,15 @@ components: type: object required: - model_id + - provider - status properties: model_id: type: string - description: The model identifier that was tested. + description: The canonical model ID that was tested. example: "claude-opus-4-6" + provider: + $ref: "#/components/schemas/ProviderId" status: type: string enum: @@ -8406,7 +8448,7 @@ components: $ref: "#/components/schemas/CompletionMessage" model: type: string - description: Model ID or alias. Server picks default if omitted. + description: Model ID or alias. Server picks a ready-provider default if omitted. system: type: string description: System prompt (convenience; prepended as a system message). @@ -8442,7 +8484,7 @@ components: description: Reasoning effort level. provider: type: string - description: Provider to route to. + description: Optional provider pin. provider_options: description: Provider-specific options. @@ -8459,12 +8501,15 @@ components: CompletionResponse: type: object - required: [id, model, message, stop_reason, usage] + required: [id, model, provider, message, stop_reason, usage] properties: id: type: string model: type: string + description: Canonical model ID selected for the request. + provider: + $ref: "#/components/schemas/ProviderId" message: $ref: "#/components/schemas/CompletionMessage" stop_reason: @@ -8518,7 +8563,10 @@ components: same format the model writes back via `write_workflow_file`. model: type: string - description: Model id or alias. Server picks the default if omitted. + description: Model ID or alias. Server picks a ready-provider default if omitted. + provider: + $ref: "#/components/schemas/ProviderId" + description: Optional provider pin. PaginatedSavedQueryList: description: Paginated list of saved queries. diff --git a/docs/public/changelog/2026-05-12.mdx b/docs/public/changelog/2026-05-12.mdx index c7d47f285..96387b4e7 100644 --- a/docs/public/changelog/2026-05-12.mdx +++ b/docs/public/changelog/2026-05-12.mdx @@ -24,7 +24,7 @@ This is useful for workflows that prepare their own workspace, run against alrea - Added `[run.clone]`, `[run.run_branch]`, and `[run.meta_branch]` settings for clone, branch, and metadata branch behavior -- Added `[llm.providers]` and `[llm.models]` settings foundations for configurable provider and model catalog data +- Added provider-scoped `[llm.providers..models.]` settings foundations for configurable provider and model catalog data - Added typed provider `extra_headers` settings for gateway-backed LLM providers diff --git a/docs/public/changelog/2026-05-13.mdx b/docs/public/changelog/2026-05-13.mdx index 3cb981f80..8659dafaa 100644 --- a/docs/public/changelog/2026-05-13.mdx +++ b/docs/public/changelog/2026-05-13.mdx @@ -28,8 +28,7 @@ credentials = ["env:ACME_GATEWAY_API_KEY"] x-portkey-api-key = { env = "PORTKEY_API_KEY" } x-portkey-config = { literal = "@bedrock-prod" } -[llm.models."team-code-large"] -provider = "proxy" +[llm.providers.proxy.models."team-code-large"] api_id = "provider-wire-model-name" default = true ``` diff --git a/docs/public/core-concepts/models.mdx b/docs/public/core-concepts/models.mdx index 7bda6217b..efa60f8cf 100644 --- a/docs/public/core-concepts/models.mdx +++ b/docs/public/core-concepts/models.mdx @@ -11,6 +11,29 @@ No single model is best at everything. Fabro lets you assign the right model to ## Model catalog +Fabro keeps five related concepts separate: + +- A **provider** serves requests, such as `openai` or `openrouter`. +- A **model slug** is Fabro's canonical, human-facing model ID, such as `gpt-5.6-sol`. +- An **alias** is another user-facing selector, such as `gpt-56-sol`. +- A **family** is display and matching metadata. It does not affect routing identity. +- An **API ID** is the opaque model string sent on the provider wire. Workflows should never reference it. + +One provider's route to one model slug is an **offering**, identified by `(provider, model slug)`. The same slug and alias may appear on several providers. Within one provider, however, every slug or alias must identify exactly one offering. + +For an unqualified selector, Fabro checks canonical slugs before aliases, filters the candidates to providers whose adapters are ready, and then chooses the highest provider priority. Equal priorities use canonical provider ID order. An explicit provider restricts selection to that provider and is a pin: if it is unavailable, Fabro reports the error instead of silently switching. + +For example, the shared `gpt-56-sol` alias can be portable across direct OpenAI and OpenRouter offerings: + +| Ready providers | Selector | Selected offering | +|---|---|---| +| OpenAI only | `gpt-56-sol` | `openai/gpt-5.6-sol` | +| OpenRouter only | `gpt-56-sol` | `openrouter/gpt-5.6-sol` | +| OpenAI and OpenRouter | `gpt-56-sol` | OpenAI, because it has higher priority | +| Both, with `provider = "openrouter"` | `gpt-56-sol` | OpenRouter, because the provider is pinned | + +Fabro performs this selection once when creating a run and persists the chosen provider and canonical slug in the run settings and graph. Resuming that run does not reconsider provider priority when credentials change. Runtime [model fallbacks](/execution/failures#model-fallbacks) are the separate mechanism for handling a later provider failure. + | Model | Provider | Aliases | Context | Cost (in/out per Mtok) | Speed | |---|---|---|---|---|---| | `claude-fable-5` | anthropic | `fable`, `claude-fable` | 1M | $10.00 / $50.00 | n/a | @@ -46,7 +69,7 @@ Claude Fable 5 is available as an explicit model but is not the default Anthropi ## Configuring providers and models -Fabro's catalog starts with the built-in providers and models, then merges any `[llm]` entries from settings. Provider and model IDs are strings, so a server or project can add an OpenAI-compatible provider without a Fabro release. +Fabro's catalog starts with the built-in providers and models, then merges any `[llm]` entries from settings. Models are nested under their provider, so two providers can expose the same model slug without overwriting each other. ```toml title="settings.toml" [llm.providers.proxy] @@ -62,8 +85,7 @@ credentials = ["env:ACME_GATEWAY_API_KEY", "vault:ACME_GATEWAY_API_KEY"] x-portkey-api-key = "{{ env.PORTKEY_API_KEY }}" x-portkey-config = "@bedrock-prod" -[llm.models."team-code-large"] -provider = "proxy" +[llm.providers.proxy.models."team-code-large"] api_id = "provider-wire-model-name" agent_profile = "anthropic" display_name = "Team Code Large" @@ -73,27 +95,26 @@ small_default = true aliases = ["team-code"] estimated_output_tps = 80 -[llm.models."team-code-large".limits] +[llm.providers.proxy.models."team-code-large".limits] context_window = 200000 max_output = 32000 -[llm.models."team-code-large".features] +[llm.providers.proxy.models."team-code-large".features] tools = true reasoning = true reasoning_effort = "levels" prompt_cache = true -effort = true -[llm.models."team-code-large".controls] +[llm.providers.proxy.models."team-code-large".controls] reasoning_effort = ["low", "medium", "high"] speed = ["fast"] -[llm.models."team-code-large".costs] +[llm.providers.proxy.models."team-code-large".costs] input_cost_per_mtok = 1.50 output_cost_per_mtok = 8.00 cache_input_cost_per_mtok = 0.30 -[llm.models."team-code-large".costs.speed.fast] +[llm.providers.proxy.models."team-code-large".costs.speed.fast] input_cost_per_mtok = 3.00 output_cost_per_mtok = 16.00 cache_input_cost_per_mtok = 0.60 @@ -106,24 +127,27 @@ For [LiteLLM](/integrations/litellm), Fabro ships a disabled provider entry. Ena enabled = true base_url = "http://localhost:4000/v1" -[llm.models."litellm-gpt-5"] -provider = "litellm" +[llm.providers.litellm.models."litellm-gpt-5"] api_id = "gpt-5" display_name = "LiteLLM GPT-5" family = "litellm" default = true -[llm.models."litellm-gpt-5".limits] +[llm.providers.litellm.models."litellm-gpt-5".limits] context_window = 128000 max_output = 8192 -[llm.models."litellm-gpt-5".features] +[llm.providers.litellm.models."litellm-gpt-5".features] tools = true vision = false reasoning = false ``` -`api_id` is the model name sent to the provider API. Omit it when the Fabro model ID and provider model ID are the same. +`api_id` is the opaque model name sent to that provider's API. It defaults to the exact model slug, so omit it when the two strings match. Fabro does not infer vendor prefixes or rewrite the value. + + +Built-in catalog keys that formerly exposed an aggregator's wire API ID are retired selectors. Fabro rejects them with their replacement address instead of silently routing them. Some contain a slash and look provider-qualified: for example, the old OpenRouter key `openai/gpt-5.6-sol` is retired in favor of `openrouter/gpt-5.6-sol`. To pin the direct OpenAI offering, set `provider = "openai"` and `name = "gpt-5.6-sol"` as separate fields. + Model roles are separate: `default = true` controls normal model selection for workflow execution, while `small_default = true` marks the provider's small/cheap utility model for metadata tasks such as generated run titles. If a provider has no small default, Fabro falls back to that provider's normal default. @@ -139,7 +163,7 @@ Provider fields in configuration, APIs, and model routing are provider ID string ### Poolside -Fabro ships a built-in [Poolside](/integrations/poolside) provider for Laguna S 2.1 and Laguna XS 2.1 over Poolside's OpenAI-compatible API. Store a direct API key with `fabro provider login --provider poolside`. The same models are also available through the opt-in OpenRouter provider under vendor-namespaced IDs. +Fabro ships a built-in [Poolside](/integrations/poolside) provider for Laguna S 2.1 and Laguna XS 2.1 over Poolside's OpenAI-compatible API. Store a direct API key with `fabro provider login --provider poolside`. The same model slugs are also available through the opt-in OpenRouter provider; its vendor-namespaced strings remain provider-only `api_id` values. ### OpenRouter @@ -169,11 +193,11 @@ Fabro ships an Ollama provider definition that is disabled by default. Enable it enabled = true ``` -Enabling the provider alone does not expose any models — until #267 adds auto-discovery, add explicit `[llm.models.]` blocks for each Ollama model you have pulled locally. Ollama's OpenAI-compatible endpoint accepts any bearer token, so local users can set `OLLAMA_API_KEY=ollama`. +Enabling the provider alone does not expose any models — until #267 adds auto-discovery, add explicit `[llm.providers.ollama.models.""]` blocks for each Ollama model you have pulled locally. Ollama's OpenAI-compatible endpoint accepts any bearer token, so local users can set `OLLAMA_API_KEY=ollama`. ## Default models -When no model or provider is specified, Fabro checks configured provider credentials and chooses the first configured provider by catalog priority. If no provider credentials are configured, it uses the catalog's global default model. Each provider has a default model: +When no model or provider is specified, Fabro chooses the default offering on the highest-priority ready provider. If no provider adapter is ready, run creation reports that no eligible offering is available. Each provider has its own default model: | Provider | Default model | |---|---| @@ -184,7 +208,7 @@ When no model or provider is specified, Fabro checks configured provider credent | `poolside` | `laguna-s-2.1` | | `zai` | `glm-4.7` | | `minimax` | `minimax-m2.5` | -| `inception` | `mercury` | +| `inception` | `mercury-2` | ## Using models in workflows @@ -221,7 +245,7 @@ fabro run docs/internal/demo/01-hello.fabro --model claude-opus-4-6 fabro run docs/internal/demo/04-pipeline.fabro --model gemini-3.1-pro-preview ``` -These flags set the default model for all nodes that don't have an explicit model assigned via a stylesheet. The provider is automatically inferred from the model catalog — you only need `--provider` for models not in the catalog or to force a specific provider. +These flags set the default model for all nodes that don't have an explicit model assigned via a stylesheet. Without `--provider`, Fabro selects among ready offerings by priority. Add `--provider` to pin an exact provider, including for an uncatalogued provider model string. ### Run config TOML diff --git a/docs/public/execution/failures.mdx b/docs/public/execution/failures.mdx index 6884d5708..ef91658be 100644 --- a/docs/public/execution/failures.mdx +++ b/docs/public/execution/failures.mdx @@ -121,7 +121,13 @@ provider = "anthropic" fallbacks = ["gemini", "openai"] ``` -When Anthropic is unavailable, Fabro tries Gemini first, then OpenAI. Each fallback entry may be a bare provider token (like `"gemini"`), a bare model alias (like `"gpt-5.4"`), or a qualified `"provider/model"` reference. For each fallback provider, Fabro selects the closest model by matching required capabilities (tool use, vision, reasoning) and minimizing cost difference. +When Anthropic fails, Fabro tries Gemini first, then OpenAI. Fallback resolution is provider-aware: + +- A bare provider token such as `"gemini"` selects that provider's closest compatible model. +- A qualified selector such as `"openrouter/gpt-56-sol"` resolves only within that provider. +- A bare model slug or alias considers ready providers and uses provider priority. + +The primary provider and model were already resolved and persisted when the run was created; resuming does not re-run primary selection. Fallbacks are only considered after an eligible runtime failure. ### What triggers failover diff --git a/docs/public/execution/run-configuration.mdx b/docs/public/execution/run-configuration.mdx index 1b51a9161..43bb9a3d1 100644 --- a/docs/public/execution/run-configuration.mdx +++ b/docs/public/execution/run-configuration.mdx @@ -138,12 +138,14 @@ name = "claude-sonnet-4-5" | Field | Description | |---|---| -| `name` | Model ID or alias (e.g. `claude-sonnet-4-5`, `opus`, `gemini-pro`). See [Models](/core-concepts/models). | -| `provider` | Provider name (optional — auto-inferred from the model catalog). Only needed for models not in the catalog or to force a specific provider. | +| `name` | Canonical model slug or alias (e.g. `claude-sonnet-4-5`, `opus`, `gemini-pro`). See [Models](/core-concepts/models). | +| `provider` | Optional provider pin. When omitted, Fabro selects among ready offerings by provider priority. When present, an unavailable provider is an error rather than permission to switch. | | `fallbacks` | Ordered list of model references to try when the primary is unavailable. Entries can be bare provider tokens (`"openai"`), bare model aliases, or qualified `"provider/model"` references. | Provider values are catalog provider ID strings. Built-in IDs like `anthropic` and `openai` work, and settings-defined IDs like `proxy` work after they are added under `[llm.providers.]`. +At run creation, Fabro resolves the primary selector and every node selector against the ready-provider snapshot. It persists the selected canonical model slug and provider, so resuming the run does not choose a different provider just because credentials or priorities changed. The configured fallback chain remains available for failures that occur while the materialized run is executing. + #### `[run.model.controls]` Set default model controls for all nodes that do not override them in the workflow stylesheet: diff --git a/docs/public/integrations/bedrock.mdx b/docs/public/integrations/bedrock.mdx index 66a03c91c..34781fe59 100644 --- a/docs/public/integrations/bedrock.mdx +++ b/docs/public/integrations/bedrock.mdx @@ -157,6 +157,6 @@ Bedrock-specific request fields pass through verbatim via `provider_options.bedr How Fabro routes model IDs, providers, and fallbacks. - Full reference for `[llm.providers.]` and `[llm.models.]`. + Full reference for provider settings and provider-scoped model offerings. diff --git a/docs/public/integrations/litellm.mdx b/docs/public/integrations/litellm.mdx index 7f1891f8d..5fb1b6955 100644 --- a/docs/public/integrations/litellm.mdx +++ b/docs/public/integrations/litellm.mdx @@ -24,18 +24,17 @@ _version = 1 enabled = true base_url = "http://localhost:4000/v1" -[llm.models."litellm-gpt-5"] -provider = "litellm" +[llm.providers.litellm.models."litellm-gpt-5"] api_id = "gpt-5" display_name = "LiteLLM GPT-5" family = "litellm" default = true -[llm.models."litellm-gpt-5".limits] +[llm.providers.litellm.models."litellm-gpt-5".limits] context_window = 128000 max_output = 8192 -[llm.models."litellm-gpt-5".features] +[llm.providers.litellm.models."litellm-gpt-5".features] tools = true vision = false reasoning = false @@ -94,18 +93,17 @@ digraph Example { Declare each LiteLLM-routed model explicitly so Fabro knows its provider, context window, tool support, and routing defaults: ```toml title="settings.toml" -[llm.models."litellm-fast"] -provider = "litellm" +[llm.providers.litellm.models."litellm-fast"] api_id = "fast-model" display_name = "LiteLLM Fast" family = "litellm" aliases = ["fast"] -[llm.models."litellm-fast".limits] +[llm.providers.litellm.models."litellm-fast".limits] context_window = 64000 max_output = 4096 -[llm.models."litellm-fast".features] +[llm.providers.litellm.models."litellm-fast".features] tools = true vision = false reasoning = false @@ -128,6 +126,6 @@ Only one model for a provider should set `default = true`. You may also mark one How Fabro routes model IDs, providers, and fallbacks. - Full reference for `[llm.providers.]` and `[llm.models.]`. + Full reference for provider settings and provider-scoped model offerings. diff --git a/docs/public/integrations/openrouter.mdx b/docs/public/integrations/openrouter.mdx index b1b6f2534..6f1f046b5 100644 --- a/docs/public/integrations/openrouter.mdx +++ b/docs/public/integrations/openrouter.mdx @@ -44,34 +44,33 @@ export OPENROUTER_API_KEY=sk-or-v1-... ## Included models -The built-in catalog curates frontier and open-weights models under vendor-namespaced IDs: +The built-in catalog gives OpenRouter offerings the same human-facing model slugs used by direct providers. Vendor-namespaced OpenRouter IDs remain opaque `api_id` values: -| Fabro model ID | Notes | +| Fabro model slug | OpenRouter API ID / notes | | --- | --- | -| `anthropic/claude-opus-4-7` | Claude via OpenRouter, Anthropic-style cache billing | -| `anthropic/claude-sonnet-4-6` | Provider default | -| `anthropic/claude-haiku-4-5` | Provider small default | -| `openai/gpt-5.4`, `openai/gpt-5.5` | | -| `google/gemini-3.1-pro-preview`, `google/gemini-3.5-flash` | | -| `deepseek/deepseek-v4-pro`, `deepseek/deepseek-v4-flash` | | -| `moonshotai/kimi-k2.6`, `qwen/qwen3-coder`, `qwen/qwen3.6-flash` | | -| `poolside/laguna-s-2.1`, `poolside/laguna-xs-2.1` | Poolside Laguna coding models with native reasoning and tool use | -| `z-ai/glm-4.6`, `minimax/minimax-m2.7`, `xiaomi/mimo-v2.5-pro` | | -| `nvidia/nemotron-3-super-120b-a12b`, `mistralai/devstral-2512` | | +| `claude-opus-4-7` | `anthropic/claude-opus-4.7`; Anthropic-style cache billing | +| `claude-sonnet-4-6` | `anthropic/claude-sonnet-4.6`; provider default | +| `claude-haiku-4-5` | `anthropic/claude-haiku-4.5`; provider small default | +| `gpt-5.4`, `gpt-5.5` | `openai/gpt-5.4`, `openai/gpt-5.5` | +| `gemini-3.1-pro-preview`, `gemini-3.5-flash` | `google/...` API IDs | +| `deepseek-v4-pro`, `deepseek-v4-flash` | `deepseek/...` API IDs | +| `kimi-k2.6`, `qwen3-coder`, `qwen3.6-flash` | Vendor-prefixed API IDs | +| `laguna-s-2.1`, `laguna-xs-2.1` | `poolside/...`; native reasoning and tool use | +| `glm-4.6`, `minimax-m2.7`, `mimo-v2.5-pro` | Vendor-prefixed API IDs | +| `nemotron-3-super-120b-a12b`, `devstral-2512` | Vendor-prefixed API IDs | -Any other OpenRouter model can be added as a settings model entry with `provider = "openrouter"` and the OpenRouter slug as `api_id`: +Any other OpenRouter model can be added under the provider. Choose a stable Fabro model slug as the table key and put OpenRouter's exact vendor/model string in `api_id`: ```toml title="settings.toml" -[llm.models."meta-llama/llama-4-maverick"] -provider = "openrouter" +[llm.providers.openrouter.models."llama-4-maverick"] api_id = "meta-llama/llama-4-maverick" display_name = "Llama 4 Maverick" family = "llama-4" -[llm.models."meta-llama/llama-4-maverick".limits] +[llm.providers.openrouter.models."llama-4-maverick".limits] context_window = 1000000 -[llm.models."meta-llama/llama-4-maverick".features] +[llm.providers.openrouter.models."llama-4-maverick".features] tools = true vision = false reasoning = false @@ -81,8 +80,8 @@ reasoning = false ```bash fabro model list --provider openrouter -fabro model test --model anthropic/claude-sonnet-4-6 -fabro run workflow.fabro --model deepseek/deepseek-v4-flash +fabro model test --provider openrouter --model claude-sonnet-4-6 +fabro run workflow.fabro --provider openrouter --model deepseek-v4-flash ``` When targeting a non-default remote server, pass the same `--server` value to verification commands: @@ -159,6 +158,6 @@ Fabro does not send OpenRouter's optional attribution headers (`HTTP-Referer`, ` How Fabro routes model IDs, providers, and fallbacks. - Full reference for `[llm.providers.]` and `[llm.models.]`. + Full reference for provider settings and provider-scoped model offerings. diff --git a/docs/public/reference/user-configuration.mdx b/docs/public/reference/user-configuration.mdx index fe62e4ee7..dd6afd83f 100644 --- a/docs/public/reference/user-configuration.mdx +++ b/docs/public/reference/user-configuration.mdx @@ -35,7 +35,7 @@ Files that omit `_version` are treated as version `1`. The legacy top-level `ver |---|---| | CLI-only | `[cli.target]`, `[cli.auth]`, `[cli.exec]`, `[cli.output]`, `[cli.updates]`, `[cli.logging]` | | Shared run defaults | `[run.model]`, `[run.environment]`, `[environments.]`, `[run.checkpoint]`, `[run.inputs]`, `[run.prepare]`, `[run.pull_request]`, `[run.integrations.github.permissions]`, `[run.hooks]`, `[run.agent.mcps]` | -| Shared LLM catalog | `[llm.providers.]`, `[llm.models.]`, model limits, features, controls, and costs | +| Shared LLM catalog | `[llm.providers.]`, provider-scoped `[llm.providers..models.]` offerings, limits, features, controls, and costs | | Server-only | `[server.listen]`, `[server.api]`, `[server.web]`, `[server.auth]`, `[server.storage]`, `[server.artifacts]`, `[server.slatedb]`, `[server.scheduler]`, `[server.logging]`, `[server.integrations]` | `[cli.*]` and `[server.*]` stanzas are owner-specific: they are only consumed from `~/.fabro/settings.toml` (plus process-local flags and env overrides). The same stanzas in `.fabro/project.toml` or `workflow.toml` remain schema-valid but runtime-inert. @@ -97,23 +97,22 @@ credentials = ["env:ACME_GATEWAY_API_KEY", "vault:ACME_GATEWAY_API_KEY"] x-portkey-api-key = "{{ env.PORTKEY_API_KEY }}" x-portkey-config = "@bedrock-prod" -[llm.models."team-code-large"] -provider = "proxy" +[llm.providers.proxy.models."team-code-large"] api_id = "provider-wire-model-name" agent_profile = "anthropic" display_name = "Team Code Large" default = true aliases = ["team-code"] -[llm.models."team-code-large".controls] +[llm.providers.proxy.models."team-code-large".controls] reasoning_effort = ["low", "medium", "high"] speed = ["fast"] -[llm.models."team-code-large".costs] +[llm.providers.proxy.models."team-code-large".costs] input_cost_per_mtok = 1.50 output_cost_per_mtok = 8.00 -[llm.models."team-code-large".costs.speed.fast] +[llm.providers.proxy.models."team-code-large".costs.speed.fast] input_cost_per_mtok = 3.00 output_cost_per_mtok = 16.00 @@ -201,19 +200,20 @@ x-team-secret = "{{ secrets.gateway_team_secret }}" | `auth.credentials` | array | required when `auth` present | Ordered credential refs. Accepted forms are `vault:`, `env:`, and `aws_sigv4` (sign requests from the AWS default credential chain — Bedrock). Literal secret strings are rejected. | | `auth.header` | `"bearer"` or `{ custom = "Header-Name" }` | `"bearer"` | Primary API-key header policy. Omit when the provider uses a standard bearer token. | | `extra_headers` | table | `{}` | Additional headers attached to provider requests. Values are interpolation strings: literal text, an `{{ env.NAME }}` token, or a `{{ secrets.NAME }}` token. Put credentials in a secret and reference them with a `{{ secrets.NAME }}` token, not a bare literal. | -| `priority` | integer | `0` | Higher-priority configured providers win default selection; ties use canonical provider ID. | +| `priority` | integer | `0` | Higher-priority ready providers win unqualified model and default selection; ties use canonical provider ID. | | `enabled` | boolean | `true` | Set `false` to disable a provider after lower-precedence layers define it. | | `aliases` | array | `[]` | Additional provider names accepted by model routing and fallback config. | -## `[llm.models.]` +## `[llm.providers..models.]` -Define or override a model in the catalog. The table key is the canonical -model ID Fabro users reference; `api_id` is the model string sent to the -provider API. +Define or override one provider's offering of a model. The table key is the +canonical model slug Fabro users reference. An offering's identity is the +pair `(provider, model slug)`, so different providers may use the same slug +and aliases. `api_id` is the opaque model string sent to this provider's API +and defaults to the exact model slug. ```toml title="settings.toml" -[llm.models."team-code-large"] -provider = "proxy" +[llm.providers.proxy.models."team-code-large"] api_id = "provider-wire-model-name" agent_profile = "anthropic" display_name = "Team Code Large" @@ -224,27 +224,27 @@ enabled = true aliases = ["team-code"] estimated_output_tps = 80 -[llm.models."team-code-large".limits] +[llm.providers.proxy.models."team-code-large".limits] context_window = 200000 max_output = 32000 -[llm.models."team-code-large".features] +[llm.providers.proxy.models."team-code-large".features] tools = true vision = false reasoning = true reasoning_effort = "levels" prompt_cache = true -[llm.models."team-code-large".controls] +[llm.providers.proxy.models."team-code-large".controls] reasoning_effort = ["low", "medium", "high"] speed = ["fast"] -[llm.models."team-code-large".costs] +[llm.providers.proxy.models."team-code-large".costs] input_cost_per_mtok = 1.50 output_cost_per_mtok = 8.00 cache_input_cost_per_mtok = 0.30 -[llm.models."team-code-large".costs.speed.fast] +[llm.providers.proxy.models."team-code-large".costs.speed.fast] input_cost_per_mtok = 3.00 output_cost_per_mtok = 16.00 cache_input_cost_per_mtok = 0.60 @@ -252,8 +252,7 @@ cache_input_cost_per_mtok = 0.60 | Key | Type / values | Default | Description | |---|---|---|---| -| `provider` | string | None | Provider ID this model belongs to. | -| `api_id` | string | model ID | Identifier sent to the provider API. | +| `api_id` | string | model slug | Opaque identifier sent to this provider's API. An explicitly empty value is invalid. | | `agent_profile` | `"anthropic"` \| `"openai"` \| `"gemini"` | provider profile | Agent profile override for this model. Model overrides take precedence over provider overrides. | | `billing_policy` | `"openai"` \| `"anthropic"` \| `"gemini"` \| `"none"` | provider policy | Billing algorithm override for this model — for models whose billing family differs from their provider's (e.g. Claude served through OpenRouter bills Anthropic-style cache reads/writes). | | `display_name` | string | model ID | Human-readable model name. | @@ -263,17 +262,17 @@ cache_input_cost_per_mtok = 0.60 | `default` | boolean | `false` | Whether this is the provider default model. | | `probe` | boolean | `false` | Whether this model should be preferred for provider connectivity probes. Set `false` in a higher-precedence layer to clear an inherited probe marker. | | `enabled` | boolean | `true` | Set `false` to disable a model after lower-precedence layers define it. | -| `aliases` | array | `[]` | Additional model names accepted by routing and fallback config. | +| `aliases` | array | `[]` | Additional model selectors accepted by routing and fallback config. Aliases may repeat across providers, but one selector cannot identify two models within the same provider. | | `estimated_output_tps` | number | None | Estimated output tokens per second for catalog display and planning. | -## `[llm.models..limits]` +## `[llm.providers..models..limits]` | Key | Type / values | Default | Description | |---|---|---|---| | `context_window` | integer | None | Maximum context window size in tokens. | | `max_output` | integer | None | Maximum output tokens, if known. | -## `[llm.models..features]` +## `[llm.providers..models..features]` | Key | Type / values | Default | Description | |---|---|---|---| @@ -284,14 +283,14 @@ cache_input_cost_per_mtok = 0.60 | `prompt_cache` | boolean | `false` | Whether prompt cache pricing/usage applies. | | `sampling_params` | boolean | `true` | Whether the model accepts classic sampling parameters (`temperature`, `top_p`). | -## `[llm.models..controls]` +## `[llm.providers..models..controls]` | Key | Type / values | Default | Description | |---|---|---|---| | `reasoning_effort` | array | all standard levels when feature is `"levels"` or `"always_adaptive"` | User-facing reasoning effort values Fabro may send for this model. Can be set explicitly for reasoning models whose provider adapter maps effort to a non-native API shape. | | `speed` | array | `[]` | Additional speeds beyond implicit `standard`; do not list `standard`. | -## `[llm.models..costs]` +## `[llm.providers..models..costs]` | Key | Type / values | Default | Description | |---|---|---|---| @@ -299,10 +298,12 @@ cache_input_cost_per_mtok = 0.60 | `output_cost_per_mtok` | number | None | Output cost in USD per million tokens. | | `cache_input_cost_per_mtok` | number | None | Cached input/read cost in USD per million tokens. | -## `[llm.models..costs.speed.]` +## `[llm.providers..models..costs.speed.]` -Per-speed cost overrides use the same keys as `[llm.models..costs]`. -Each `` key must be declared in `[llm.models..controls].speed`. +Per-speed cost overrides use the same keys as +`[llm.providers..models..costs]`. Each `` key +must be declared in +`[llm.providers..models..controls].speed`. The `standard` speed is implicit and always uses the base cost table. ## `[cli.updates]` diff --git a/lib/crates/fabro-agent/src/agent_profile.rs b/lib/crates/fabro-agent/src/agent_profile.rs index 632ef4655..607d14fa0 100644 --- a/lib/crates/fabro-agent/src/agent_profile.rs +++ b/lib/crates/fabro-agent/src/agent_profile.rs @@ -36,13 +36,13 @@ pub trait AgentProfile: Send + Sync { } fn knowledge_cutoff(&self) -> Option { - self.catalog() - .and_then(|catalog| catalog.get(self.model())) + self.catalog_model() .and_then(|m| m.knowledge_cutoff().map(str::to_string)) } fn catalog_model(&self) -> Option<&Model> { - self.catalog().and_then(|catalog| catalog.get(self.model())) + let catalog = self.catalog()?; + catalog.get_on_provider(&self.provider_id(), self.model()) } fn context_window_size(&self) -> usize { diff --git a/lib/crates/fabro-agent/src/cli.rs b/lib/crates/fabro-agent/src/cli.rs index b32ef10dd..749f38b0a 100644 --- a/lib/crates/fabro-agent/src/cli.rs +++ b/lib/crates/fabro-agent/src/cli.rs @@ -20,7 +20,7 @@ use fabro_llm::types::{Request, Response}; use fabro_mcp::config::McpServerSettings; #[cfg(test)] use fabro_model::catalog::LlmCatalogSettings; -use fabro_model::{AgentProfileKind, Catalog, ModelHandle, ProviderId}; +use fabro_model::{AgentProfileKind, Catalog, ModelHandle, ModelSelectionError, ProviderId}; use fabro_static::EnvVars; use fabro_util::terminal::Styles; use fabro_vault::SecretStore; @@ -255,7 +255,11 @@ fn parse_provider(args: &AgentArgs) -> anyhow::Result { Ok(provider_str.parse()?) } -fn resolve_provider_id(catalog: &Catalog, args: &AgentArgs) -> anyhow::Result { +fn resolve_provider_id( + catalog: &Catalog, + args: &AgentArgs, + eligible_providers: &std::collections::HashSet, +) -> anyhow::Result { if args.provider.is_some() { let requested = parse_provider(args)?; return Ok(catalog @@ -263,8 +267,10 @@ fn resolve_provider_id(catalog: &Catalog, args: &AgentArgs) -> anyhow::Result return Ok(model.provider.clone()), + Err(ModelSelectionError::UnknownSelector { .. }) => {} + Err(error) => return Err(error.into()), } } let requested = parse_provider(args)?; @@ -530,7 +536,7 @@ pub async fn run_with_args_and_client_and_catalog( // threads let styles: &'static Styles = Box::leak(Box::new(Styles::detect_stderr())); - let provider_id = resolve_provider_id(&catalog, &args)?; + let provider_id = resolve_provider_id(&catalog, &args, &client.provider_ids())?; ensure_provider_registered(&client, &provider_id)?; if args.verbose { @@ -550,6 +556,7 @@ pub async fn run_with_args_and_client_and_catalog( "provider '{provider_id}' has no default model in the catalog; pass --model explicitly" ) })? + .to_string() }; let profile_kind = profile_kind_for_provider(&catalog, &provider_id, Some(&model))?; eprintln!("{}", styles.dim.apply_to(format!("Using model: {model}"))); @@ -1089,7 +1096,7 @@ mod tests { }; assert_eq!( - resolve_provider_id(&catalog, &args).unwrap(), + resolve_provider_id(&catalog, &args, &catalog.all_provider_ids()).unwrap(), ProviderId::new("acme-aws") ); } @@ -1121,7 +1128,7 @@ mod tests { }; assert_eq!( - resolve_provider_id(&catalog, &args).unwrap(), + resolve_provider_id(&catalog, &args, &catalog.all_provider_ids()).unwrap(), ProviderId::new("acme-aws") ); } diff --git a/lib/crates/fabro-agent/src/session.rs b/lib/crates/fabro-agent/src/session.rs index 2bd840bda..dfc754d1f 100644 --- a/lib/crates/fabro-agent/src/session.rs +++ b/lib/crates/fabro-agent/src/session.rs @@ -1718,9 +1718,9 @@ impl Session { let model = ModelRef { provider: self.provider_profile.provider_id(), model_id: if response.model.is_empty() { - self.provider_profile.model().to_string() + self.provider_profile.model().into() } else { - response.model.clone() + response.model.clone().into() }, speed: self.config.speed, }; diff --git a/lib/crates/fabro-agent/tests/it/guardrails.rs b/lib/crates/fabro-agent/tests/it/guardrails.rs index 8e1f898fe..cae5461e3 100644 --- a/lib/crates/fabro-agent/tests/it/guardrails.rs +++ b/lib/crates/fabro-agent/tests/it/guardrails.rs @@ -17,18 +17,18 @@ fn profile_context_window_matches_catalog_for_default_models() { let profile: Box = match provider.agent_profile { fabro_model::AgentProfileKind::OpenAi if provider.id == ProviderId::openai() => { - Box::new(OpenAiProfile::new(model).with_catalog(Arc::clone(&catalog))) + Box::new(OpenAiProfile::new(model.as_str()).with_catalog(Arc::clone(&catalog))) } fabro_model::AgentProfileKind::OpenAi => Box::new( - OpenAiProfile::new(model) + OpenAiProfile::new(model.as_str()) .with_provider_id(provider.id.clone()) .with_catalog(Arc::clone(&catalog)), ), fabro_model::AgentProfileKind::Gemini => { - Box::new(GeminiProfile::new(model).with_catalog(Arc::clone(&catalog))) + Box::new(GeminiProfile::new(model.as_str()).with_catalog(Arc::clone(&catalog))) } fabro_model::AgentProfileKind::Anthropic => { - Box::new(AnthropicProfile::new(model).with_catalog(Arc::clone(&catalog))) + Box::new(AnthropicProfile::new(model.as_str()).with_catalog(Arc::clone(&catalog))) } }; diff --git a/lib/crates/fabro-api/tests/model_round_trip.rs b/lib/crates/fabro-api/tests/model_round_trip.rs index 727cd41da..93d4e7692 100644 --- a/lib/crates/fabro-api/tests/model_round_trip.rs +++ b/lib/crates/fabro-api/tests/model_round_trip.rs @@ -13,7 +13,7 @@ fn model_reuses_canonical_type() { #[test] fn model_json_matches_openapi_shape() { let model = Model { - id: "claude-opus-4-7".to_string(), + id: "claude-opus-4-7".into(), provider: ProviderId::anthropic(), family: "claude-4".to_string(), display_name: "Claude Opus 4.7".to_string(), diff --git a/lib/crates/fabro-api/tests/provider_id_round_trip.rs b/lib/crates/fabro-api/tests/provider_id_round_trip.rs index 47a1a2ce3..e648b008f 100644 --- a/lib/crates/fabro-api/tests/provider_id_round_trip.rs +++ b/lib/crates/fabro-api/tests/provider_id_round_trip.rs @@ -23,7 +23,7 @@ fn provider_id_json_matches_openapi_shape_through_model() { ); let model = Model { - id: "venice-custom".to_string(), + id: "venice-custom".into(), provider: ProviderId::new("venice"), family: "venice".to_string(), display_name: "Venice Custom".to_string(), diff --git a/lib/crates/fabro-api/tests/session_contract_round_trip.rs b/lib/crates/fabro-api/tests/session_contract_round_trip.rs index c8b26fb7a..f987f73f8 100644 --- a/lib/crates/fabro-api/tests/session_contract_round_trip.rs +++ b/lib/crates/fabro-api/tests/session_contract_round_trip.rs @@ -5,6 +5,7 @@ use fabro_api::types::{ SessionDetail as ApiSessionDetail, SessionRecord as ApiSessionRecord, SessionSummary as ApiSessionSummary, SessionTurn as ApiSessionTurn, SubmitTurnRequest, }; +use fabro_model::ProviderId; use fabro_types::{ SessionDetail, SessionId, SessionMessage, SessionRecord, SessionStatus, SessionSummary, SessionTurn, TurnId, fixtures, @@ -33,6 +34,7 @@ fn session_detail_round_trips_messages_active_turn_and_last_seq() { title: Some("Ask Fabro".to_string()), status: SessionStatus::Running, model: Some("gpt-5.4".to_string()), + provider: Some(ProviderId::openai()), active_turn: Some(SessionTurn { id: turn_id, started_at: turn_started_at, diff --git a/lib/crates/fabro-cli/Cargo.toml b/lib/crates/fabro-cli/Cargo.toml index 6f14e385e..4c84def39 100644 --- a/lib/crates/fabro-cli/Cargo.toml +++ b/lib/crates/fabro-cli/Cargo.toml @@ -119,6 +119,7 @@ assert_cmd = "2" fabro-acp = { path = "../fabro-acp", features = ["test-support"] } fabro-build-support = { path = "../build-support" } fabro-server = { path = "../fabro-server", features = ["test-support"] } +fabro-workflow = { path = "../fabro-workflow", features = ["test-support"] } fabro-types = { path = "../fabro-types", features = ["clap", "test-support"] } insta = { workspace = true, features = ["filters"] } paste = "1" diff --git a/lib/crates/fabro-cli/src/commands/model.rs b/lib/crates/fabro-cli/src/commands/model.rs index e63b926f4..d5d41bb20 100644 --- a/lib/crates/fabro-cli/src/commands/model.rs +++ b/lib/crates/fabro-cli/src/commands/model.rs @@ -45,10 +45,17 @@ struct CompletedModelTest { status: String, } -fn find_model_by_id_or_alias(models: &[Model], id: &str) -> Option { +fn find_model_by_id_or_alias( + models: &[Model], + id: &str, + provider: Option<&ProviderId>, +) -> Option { models .iter() - .find(|model| model.id == id || model.aliases.iter().any(|alias| alias == id)) + .find(|model| { + provider.is_none_or(|provider| &model.provider == provider) + && (model.id == id || model.aliases.iter().any(|alias| alias == id)) + }) .cloned() } @@ -116,7 +123,7 @@ fn model_row(model: &Model, use_color: bool) -> Vec { format_cost(model.costs.output_cost_per_mtok), ); vec![ - model.id.clone().cell().bold(use_color), + model.id.as_str().cell().bold(use_color), model .provider .as_str() @@ -172,9 +179,20 @@ fn print_models_table(models: &[Model], styles: &Styles) { } fn configured_model_test_status( + expected: &Model, result: Result, ) -> (Color, String, bool) { match result { + Ok(resp) if resp.provider != expected.provider || resp.model_id != expected.id.as_str() => { + ( + Color::Red, + format!( + "error: server tested unexpected offering {}/{}", + resp.provider, resp.model_id + ), + true, + ) + } Ok(resp) if resp.status == api_types::ModelTestResultStatus::Ok => { (Color::Green, "ok".to_string(), false) } @@ -197,21 +215,21 @@ fn model_test_row_from_status(model: &Model, status: &str, result_color: Color) let trimmed = status.trim(); match result_color { Color::Green => ModelTestRow { - model: model.id.clone(), + model: model.id.to_string(), provider: model.provider.clone(), result: ModelTestResultKind::Pass, detail: None, error: None, }, Color::Yellow => ModelTestRow { - model: model.id.clone(), + model: model.id.to_string(), provider: model.provider.clone(), result: ModelTestResultKind::Skip, detail: Some(trimmed.to_string()), error: None, }, _ => ModelTestRow { - model: model.id.clone(), + model: model.id.to_string(), provider: model.provider.clone(), result: ModelTestResultKind::Fail, detail: None, @@ -251,21 +269,46 @@ async fn test_models_via_server( let mut skipped = 0u32; let mut skipped_providers: Vec = Vec::new(); if let Some(model_id) = model { - let listed_models = client.list_models(None, Some(model_id)).await?; - let listed_info = find_model_by_id_or_alias(&listed_models, model_id); + let requested_provider = provider.map(ProviderId::new); + let listed_models = client.list_models(provider, Some(model_id)).await?; + let listed_info = find_model_by_id_or_alias(&listed_models, model_id, None); if !json_output { eprint!("Testing {model_id}..."); } - let result = client.test_model(model_id, request_mode).await; + let has_configured_match = listed_models.iter().any(|model| { + model.configured + && (model.id == model_id || model.aliases.iter().any(|alias| alias == model_id)) + }); + let result = + if requested_provider.is_none() && listed_info.is_some() && !has_configured_match { + None + } else { + Some( + client + .test_model(model_id, requested_provider.as_ref(), request_mode) + .await, + ) + }; if !json_output { eprintln!(" done"); } let (info, result_color, status) = match result { - Ok(resp) => { - let info = find_model_by_id_or_alias(&listed_models, &resp.model_id).with_context( - || format!("Unknown model returned by server: {}", resp.model_id), - )?; + None => { + let info = listed_info.with_context(|| format!("Unknown model: {model_id}"))?; + failures += 1; + skipped += 1; + (info, Color::Yellow, "not configured".to_string()) + } + Some(Ok(resp)) => { + let info = + find_model_by_id_or_alias(&listed_models, &resp.model_id, Some(&resp.provider)) + .with_context(|| { + format!( + "Unknown model returned by server: {}/{}", + resp.provider, resp.model_id + ) + })?; if resp.status == api_types::ModelTestResultStatus::Ok { (info, Color::Green, "ok".to_string()) } else if resp.status == api_types::ModelTestResultStatus::Skip { @@ -280,10 +323,10 @@ async fn test_models_via_server( (info, Color::Red, format!("error: {message}")) } } - Err(err) if err.to_string().contains("Model not found") => { + Some(Err(err)) if err.to_string().contains("Model not found") => { bail!("Unknown model: {model_id}"); } - Err(err) => { + Some(Err(err)) => { let info = listed_info.with_context(|| format!("Unknown model: {model_id}"))?; failures += 1; (info, Color::Red, format!("error: {err}")) @@ -328,11 +371,14 @@ async fn test_models_via_server( .map(|(index, info)| { let client = client.clone(); async move { - let result = client.test_model(&info.id, request_mode).await; + let result = client + .test_model(info.id.as_str(), Some(&info.provider), request_mode) + .await; if !json_output { eprintln!("Testing {}... done", info.id); } - let (result_color, status, failed) = configured_model_test_status(result); + let (result_color, status, failed) = + configured_model_test_status(&info, result); ( CompletedModelTest { index, @@ -478,7 +524,7 @@ mod tests { fn test_model_json(id: &str, provider: ProviderId) -> serde_json::Value { serde_json::to_value(Model { - id: id.to_string(), + id: id.into(), provider, family: "test".to_string(), display_name: format!("{id} display"), @@ -512,7 +558,7 @@ mod tests { fn custom_model_json(id: &str, provider: &str) -> serde_json::Value { serde_json::to_value(Model { - id: id.to_string(), + id: id.into(), provider: ProviderId::new(provider), family: "test".to_string(), display_name: format!("{id} display"), @@ -610,6 +656,7 @@ mod tests { .body( serde_json::json!({ "model_id": "test-model", + "provider": "anthropic", "status": "ok" }) .to_string(), @@ -618,7 +665,7 @@ mod tests { .await; let client = test_client(&server.url("")); - let response = client.test_model("test-model", None).await.unwrap(); + let response = client.test_model("test-model", None, None).await.unwrap(); assert_eq!(response.status, api_types::ModelTestResultStatus::Ok); assert!(response.error_message.is_none()); @@ -637,6 +684,7 @@ mod tests { .body( serde_json::json!({ "model_id": "test-model", + "provider": "anthropic", "status": "error", "error_message": "timeout" }) @@ -647,7 +695,7 @@ mod tests { let client = test_client(&server.url("")); let response = client - .test_model("test-model", Some(ModelTestMode::Deep)) + .test_model("test-model", None, Some(ModelTestMode::Deep)) .await .unwrap(); @@ -666,6 +714,7 @@ mod tests { .body( serde_json::json!({ "model_id": "kimi-k2.5", + "provider": "kimi", "status": "skip" }) .to_string(), @@ -674,7 +723,7 @@ mod tests { .await; let client = test_client(&server.url("")); - let response = client.test_model("kimi-k2.5", None).await.unwrap(); + let response = client.test_model("kimi-k2.5", None, None).await.unwrap(); assert_eq!(response.status, api_types::ModelTestResultStatus::Skip); assert!(response.error_message.is_none()); @@ -698,7 +747,7 @@ mod tests { .await; let client = test_client(&server.url("")); - let result = client.test_model("bad-model", None).await; + let result = client.test_model("bad-model", None, None).await; assert!(result.is_err()); assert!(result.unwrap_err().to_string().contains("Model not found")); } @@ -732,6 +781,7 @@ mod tests { .body( serde_json::json!({ "model_id": "venice-large", + "provider": "venice", "status": "ok" }) .to_string(), @@ -754,6 +804,80 @@ mod tests { .unwrap(); } + #[tokio::test] + async fn bulk_model_test_keeps_duplicate_ids_scoped_by_provider() { + let server = httpmock::MockServer::start_async().await; + server + .mock_async(|when, then| { + when.method("GET") + .path("/api/v1/models") + .query_param("page[limit]", "100") + .query_param("page[offset]", "0"); + then.status(200) + .header("Content-Type", "application/json") + .body( + serde_json::json!({ + "data": [ + custom_model_json("portable-model", "openai"), + custom_model_json("portable-model", "openrouter") + ], + "meta": { "has_more": false } + }) + .to_string(), + ); + }) + .await; + let openai = server + .mock_async(|when, then| { + when.method("POST") + .path("/api/v1/models/portable-model/test") + .query_param("provider", "openai"); + then.status(200) + .header("Content-Type", "application/json") + .body( + serde_json::json!({ + "model_id": "portable-model", + "provider": "openai", + "status": "ok" + }) + .to_string(), + ); + }) + .await; + let openrouter = server + .mock_async(|when, then| { + when.method("POST") + .path("/api/v1/models/portable-model/test") + .query_param("provider", "openrouter"); + then.status(200) + .header("Content-Type", "application/json") + .body( + serde_json::json!({ + "model_id": "portable-model", + "provider": "openrouter", + "status": "ok" + }) + .to_string(), + ); + }) + .await; + + test_models_via_server( + &test_client(&server.url("")), + None, + None, + false, + 2, + &Styles::new(false), + true, + ) + .await + .unwrap(); + + openai.assert_async().await; + openrouter.assert_async().await; + } + #[tokio::test] async fn fetch_models_from_server_parses_response() { let server = httpmock::MockServer::start_async().await; diff --git a/lib/crates/fabro-cli/src/commands/run/ask.rs b/lib/crates/fabro-cli/src/commands/run/ask.rs index 1b38ad734..2c7819186 100644 --- a/lib/crates/fabro-cli/src/commands/run/ask.rs +++ b/lib/crates/fabro-cli/src/commands/run/ask.rs @@ -11,8 +11,9 @@ pub(crate) async fn run(args: AskArgs, base_ctx: &CommandContext) -> Result<()> let run_id = client.resolve_run(&args.run).await?.id; let session = client .create_run_session(run_id, CreateRunSessionRequest { - title: Some(session_title(&args.prompt)), - model: args.model, + title: Some(session_title(&args.prompt)), + model: args.model, + provider: None, }) .await?; let mut stream = client diff --git a/lib/crates/fabro-cli/src/commands/run/run_progress/event.rs b/lib/crates/fabro-cli/src/commands/run/run_progress/event.rs index a4a72f0bc..4cdc36056 100644 --- a/lib/crates/fabro-cli/src/commands/run/run_progress/event.rs +++ b/lib/crates/fabro-cli/src/commands/run/run_progress/event.rs @@ -323,7 +323,7 @@ pub(super) fn from_run_event(stored: &RunEvent) -> Option { EventBody::ParallelCompleted(_) => Some(ProgressEvent::ParallelCompleted), EventBody::AgentMessage(props) => Some(ProgressEvent::AssistantMessage { stage_node_id: node_id, - model: props.model.model_id.clone(), + model: props.model.model_id.to_string(), }), EventBody::AgentToolStarted(props) => Some(ProgressEvent::ToolCallStarted { stage_node_id: node_id, diff --git a/lib/crates/fabro-cli/src/shared/provider_auth.rs b/lib/crates/fabro-cli/src/shared/provider_auth.rs index 70e258393..1d3992f1f 100644 --- a/lib/crates/fabro-cli/src/shared/provider_auth.rs +++ b/lib/crates/fabro-cli/src/shared/provider_auth.rs @@ -100,9 +100,10 @@ pub(crate) async fn validate_api_key( .await .context("failed to create LLM client")?; - let probe_model = catalog - .probe_for_provider(provider) - .map_or_else(|| format!("unknown-{provider}"), |model| model.id.clone()); + let probe_model = catalog.probe_for_provider(provider).map_or_else( + || format!("unknown-{provider}"), + |model| model.id.to_string(), + ); let params = GenerateParams::new(probe_model, Arc::new(client)) .provider(provider.to_string()) diff --git a/lib/crates/fabro-cli/tests/it/cmd/model_test.rs b/lib/crates/fabro-cli/tests/it/cmd/model_test.rs index f7d731d2d..70e64ac77 100644 --- a/lib/crates/fabro-cli/tests/it/cmd/model_test.rs +++ b/lib/crates/fabro-cli/tests/it/cmd/model_test.rs @@ -202,6 +202,7 @@ fn model_test_does_not_announce_unconfigured() { .header("Content-Type", "application/json") .json_body(serde_json::json!({ "model_id": "claude-opus-4-7", + "provider": "anthropic", "status": "ok" })); }); @@ -251,6 +252,7 @@ fn model_test_skipped_footer_sources_from_listing() { .header("Content-Type", "application/json") .json_body(serde_json::json!({ "model_id": "claude-opus-4-7", + "provider": "anthropic", "status": "ok" })); }); @@ -282,6 +284,7 @@ fn model_test_post_list_race_is_a_failure() { .header("Content-Type", "application/json") .json_body(serde_json::json!({ "model_id": "claude-opus-4-7", + "provider": "anthropic", "status": "skip" })); }); @@ -323,6 +326,7 @@ fn model_test_json_partitions_skip_and_fail() { .header("Content-Type", "application/json") .json_body(serde_json::json!({ "model_id": "claude-opus-4-7", + "provider": "anthropic", "status": "skip" })); }); @@ -529,6 +533,7 @@ async fn concurrent_test_model( Json(serde_json::json!({ "model_id": id, + "provider": "anthropic", "status": "ok" })) } diff --git a/lib/crates/fabro-cli/tests/it/support/auth_harness.rs b/lib/crates/fabro-cli/tests/it/support/auth_harness.rs index cfaf06fd0..315465096 100644 --- a/lib/crates/fabro-cli/tests/it/support/auth_harness.rs +++ b/lib/crates/fabro-cli/tests/it/support/auth_harness.rs @@ -24,6 +24,7 @@ use fabro_server::auth::GithubEndpoints; use fabro_server::jwt_auth::resolve_auth_mode_with_lookup; use fabro_server::server::{RouterOptions, build_router_with_options}; use fabro_server::test_support::TestAppStateBuilder; +use fabro_static::EnvVars; use fabro_test::{GitHubAppState, TestContext, apply_test_isolation}; use serde_json::Value; use tokio::net::TcpListener; @@ -88,7 +89,10 @@ impl RealAuthHarness { .max_concurrent_runs(5) .env_lookup(|_| None) .server_secret_env(secrets) - .vault_entries([("GITHUB_APP_CLIENT_SECRET", github_client_secret.as_str())]) + .vault_entries([ + ("GITHUB_APP_CLIENT_SECRET", github_client_secret.as_str()), + (EnvVars::OPENAI_API_KEY, "test-openai-api-key"), + ]) .build(); let github_base = github_base_url(&twin.base_url); let router = build_router_with_options(state, &auth_mode, RouterOptions { diff --git a/lib/crates/fabro-client/src/client.rs b/lib/crates/fabro-client/src/client.rs index 71f185104..be21fe2dd 100644 --- a/lib/crates/fabro-client/src/client.rs +++ b/lib/crates/fabro-client/src/client.rs @@ -833,11 +833,15 @@ impl Client { pub async fn test_model( &self, id: &str, + provider: Option<&ProviderId>, mode: Option, ) -> Result { let response = self .send_api(|client| async move { let mut request = client.test_model().id(id.to_string()); + if let Some(provider) = provider { + request = request.provider(provider.clone()); + } if let Some(mode) = mode { request = request.mode(mode); } diff --git a/lib/crates/fabro-config/src/builders.rs b/lib/crates/fabro-config/src/builders.rs index 1458ba0b9..1ffdc650d 100644 --- a/lib/crates/fabro-config/src/builders.rs +++ b/lib/crates/fabro-config/src/builders.rs @@ -340,6 +340,12 @@ fn provider_settings_to_catalog( .map(|(name, value)| (name, value.as_source())) .collect() }); + let models = settings + .models + .into_inner() + .into_iter() + .map(|(id, settings)| (id, model_settings_to_catalog(settings))) + .collect(); model_catalog::ProviderCatalogSettings { display_name: settings.display_name, adapter: settings.adapter, @@ -353,6 +359,7 @@ fn provider_settings_to_catalog( priority: settings.priority, enabled: settings.enabled, aliases: settings.aliases, + models, } } @@ -863,7 +870,7 @@ reasoning = false assert_eq!( catalog - .get("acme-large") + .get_on_provider(&fabro_model::ProviderId::new("acme"), "acme-large") .map(|model| model.provider.clone()), Some(fabro_model::ProviderId::new("acme")) ); diff --git a/lib/crates/fabro-config/src/layers/llm.rs b/lib/crates/fabro-config/src/layers/llm.rs index 5fb8a0a64..32169b980 100644 --- a/lib/crates/fabro-config/src/layers/llm.rs +++ b/lib/crates/fabro-config/src/layers/llm.rs @@ -12,8 +12,7 @@ //! enabled = true //! aliases = ["moonshot"] //! -//! [llm.models."kimi-k2.5"] -//! provider = "kimi" +//! [llm.providers.kimi.models."kimi-k2.5"] //! ... //! ``` //! @@ -26,10 +25,12 @@ //! Resolution against the static adapter registry happens in `fabro-model` //! when the resolved [`Catalog`](fabro_model::Catalog) is built. -use std::collections::{BTreeMap, HashMap}; +use std::collections::{BTreeMap, BTreeSet, HashMap}; use fabro_model::catalog::deserialize_knowledge_cutoff; -use fabro_model::{AgentProfileKind, BillingPolicy, CodecKind, ProviderAuthConfig}; +use fabro_model::{ + AgentProfileKind, BillingPolicy, CodecKind, ModelId, ProviderAuthConfig, ProviderId, catalog, +}; pub use fabro_model::{CredentialRef, CredentialRefParseError, ReasoningEffortFeature}; use fabro_types::settings::InterpString; use serde::{Deserialize, Serialize}; @@ -43,7 +44,8 @@ pub struct LlmLayer { /// Provider definitions keyed by provider ID. #[serde(default, skip_serializing_if = "MergeMap::is_empty")] pub providers: MergeMap, - /// Model definitions keyed by canonical model ID. + /// Legacy top-level model definitions. New settings put models below + /// their provider; parsing normalizes this map before layers combine. #[serde(default, skip_serializing_if = "MergeMap::is_empty")] pub models: MergeMap, } @@ -87,13 +89,16 @@ pub struct ProviderSettings { pub enabled: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub aliases: Option>, + /// Model offerings served by this provider, keyed by canonical model ID. + #[serde(default, skip_serializing_if = "MergeMap::is_empty")] + pub models: MergeMap, } -/// One entry in `[llm.models.]`. +/// One entry in `[llm.providers..models.]`. #[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize, fabro_macros::Combine)] #[serde(deny_unknown_fields)] pub struct ModelSettings { - /// Provider ID this model belongs to. + /// Compatibility-only provider for legacy `[llm.models.]` rows. #[serde(default, skip_serializing_if = "Option::is_none")] pub provider: Option, /// Identifier sent to the provider API. Defaults to the catalog model ID @@ -155,6 +160,279 @@ pub struct ModelSettings { pub costs: Option, } +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum LlmNormalizationError { + BuiltinCatalog { + message: String, + }, + RetiredModel { + identifier: String, + provider: ProviderId, + model: ModelId, + }, + UnknownLegacyModel { + model: String, + }, + AmbiguousLegacyModel { + model: String, + candidates: Vec<(ProviderId, ModelId)>, + }, + AmbiguousProviderModel { + provider: ProviderId, + selector: String, + models: Vec, + }, + DuplicateModelDefinition { + provider: ProviderId, + model: ModelId, + }, + ScopedModelDeclaresProvider { + provider: ProviderId, + model: ModelId, + }, +} + +impl std::fmt::Display for LlmNormalizationError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::BuiltinCatalog { message } => { + write!(f, "failed to inspect the built-in model catalog: {message}") + } + Self::RetiredModel { + identifier, + provider, + model, + } => write!( + f, + "model identifier '{identifier}' has been retired; use '{provider}/{model}' instead" + ), + Self::UnknownLegacyModel { model } => write!( + f, + "legacy model row '{model}' omits provider and does not match a unique known offering" + ), + Self::AmbiguousLegacyModel { model, candidates } => write!( + f, + "legacy model row '{model}' omits provider and matches multiple offerings: {candidates:?}" + ), + Self::AmbiguousProviderModel { + provider, + selector, + models, + } => write!( + f, + "legacy model selector '{selector}' is ambiguous on provider '{provider}': {models:?}" + ), + Self::DuplicateModelDefinition { provider, model } => write!( + f, + "provider '{provider}' model '{model}' is defined through both provider-scoped and legacy top-level syntax" + ), + Self::ScopedModelDeclaresProvider { provider, model } => write!( + f, + "provider-scoped model '{provider}/{model}' must not declare a provider field" + ), + } + } +} + +impl std::error::Error for LlmNormalizationError {} + +impl LlmLayer { + /// Normalize the temporary legacy model table before this source is + /// combined with any other settings source. + pub(crate) fn normalize_legacy_models(&mut self) -> Result<(), LlmNormalizationError> { + for (provider, settings) in self.providers.iter() { + for (model, settings) in settings.models.iter() { + if settings.provider.is_some() { + return Err(LlmNormalizationError::ScopedModelDeclaresProvider { + provider: ProviderId::new(provider.clone()), + model: ModelId::new(model.clone()), + }); + } + } + } + + let mut legacy_models = std::mem::take(&mut self.models.0) + .into_iter() + .collect::>(); + legacy_models.sort_by(|(left, _), (right, _)| left.cmp(right)); + + for (legacy_id, mut settings) in legacy_models { + if let Some((provider, model)) = catalog::retired_model_replacement(&legacy_id) { + return Err(LlmNormalizationError::RetiredModel { + identifier: legacy_id, + provider, + model, + }); + } + + let explicit_provider = settings.provider.take(); + let (provider, model) = if let Some(provider) = explicit_provider { + let provider = self.canonical_provider(&provider)?; + let model = self + .canonical_model_on_provider(&provider, &legacy_id)? + .unwrap_or_else(|| ModelId::new(legacy_id.clone())); + (provider, model) + } else { + let candidates = self.model_candidates(&legacy_id)?; + match candidates.as_slice() { + [(provider, model)] => (provider.clone(), model.clone()), + [] => { + return Err(LlmNormalizationError::UnknownLegacyModel { model: legacy_id }); + } + _ => { + return Err(LlmNormalizationError::AmbiguousLegacyModel { + model: legacy_id, + candidates, + }); + } + } + }; + + let provider_settings = self.providers.entry(provider.to_string()).or_default(); + if provider_settings.models.contains_key(model.as_str()) { + return Err(LlmNormalizationError::DuplicateModelDefinition { provider, model }); + } + provider_settings + .models + .insert(model.into_inner(), settings); + } + Ok(()) + } + + fn canonical_provider(&self, selector: &str) -> Result { + if self.providers.contains_key(selector) { + return Ok(ProviderId::new(selector)); + } + let mut aliases = self + .providers + .iter() + .filter(|(_, settings)| { + settings + .aliases + .as_ref() + .is_some_and(|aliases| aliases.iter().any(|alias| alias == selector)) + }) + .map(|(provider, _)| ProviderId::new(provider.clone())) + .collect::>(); + aliases.sort(); + if let Some(provider) = aliases.into_iter().next() { + return Ok(provider); + } + catalog::builtin_provider_id(selector) + .map_err(|error| normalization_catalog_error(&error)) + .map(|provider| provider.unwrap_or_else(|| ProviderId::new(selector))) + } + + fn canonical_model_on_provider( + &self, + provider: &ProviderId, + selector: &str, + ) -> Result, LlmNormalizationError> { + let current = self + .providers + .get(provider.as_str()) + .map(|settings| &settings.models); + if current.is_some_and(|models| models.contains_key(selector)) { + return Ok(Some(ModelId::new(selector))); + } + + let builtin_canonical = catalog::builtin_canonical_model_offerings(selector) + .map_err(|error| normalization_catalog_error(&error))? + .into_iter() + .find_map(|(candidate_provider, model)| { + (candidate_provider == *provider).then_some(model) + }); + if builtin_canonical.is_some() { + return Ok(builtin_canonical); + } + + let mut aliases = current + .into_iter() + .flat_map(|models| models.iter()) + .filter(|(_, settings)| { + settings + .aliases + .as_ref() + .is_some_and(|aliases| aliases.iter().any(|alias| alias == selector)) + }) + .map(|(model, _)| ModelId::new(model.clone())) + .collect::>(); + aliases.extend( + catalog::builtin_alias_model_offerings(selector) + .map_err(|error| normalization_catalog_error(&error))? + .into_iter() + .filter_map(|(candidate_provider, model)| { + (candidate_provider == *provider).then_some(model) + }), + ); + match aliases.len() { + 0 => Ok(None), + 1 => Ok(aliases.into_iter().next()), + _ => Err(LlmNormalizationError::AmbiguousProviderModel { + provider: provider.clone(), + selector: selector.to_string(), + models: aliases.into_iter().collect(), + }), + } + } + + fn model_candidates( + &self, + selector: &str, + ) -> Result, LlmNormalizationError> { + let mut canonical = self + .providers + .iter() + .filter(|(_, settings)| settings.models.contains_key(selector)) + .map(|(provider, _)| { + ( + ProviderId::new(provider.clone()), + ModelId::new(selector.to_string()), + ) + }) + .collect::>(); + canonical.extend( + catalog::builtin_canonical_model_offerings(selector) + .map_err(|error| normalization_catalog_error(&error))?, + ); + if !canonical.is_empty() { + return Ok(canonical.into_iter().collect()); + } + + let mut aliases = + self.providers + .iter() + .flat_map(|(provider, settings)| { + settings + .models + .iter() + .filter(move |(_, settings)| { + settings.aliases.as_ref().is_some_and(|aliases| { + aliases.iter().any(|alias| alias == selector) + }) + }) + .map(move |(model, _)| { + ( + ProviderId::new(provider.clone()), + ModelId::new(model.clone()), + ) + }) + }) + .collect::>(); + aliases.extend( + catalog::builtin_alias_model_offerings(selector) + .map_err(|error| normalization_catalog_error(&error))?, + ); + Ok(aliases.into_iter().collect()) + } +} + +fn normalization_catalog_error(error: &catalog::CatalogBuildError) -> LlmNormalizationError { + LlmNormalizationError::BuiltinCatalog { + message: error.to_string(), + } +} + #[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize, fabro_macros::Combine)] #[serde(deny_unknown_fields)] pub struct ModelLimits { diff --git a/lib/crates/fabro-config/src/layers/mod.rs b/lib/crates/fabro-config/src/layers/mod.rs index c3fa1632c..c25838781 100644 --- a/lib/crates/fabro-config/src/layers/mod.rs +++ b/lib/crates/fabro-config/src/layers/mod.rs @@ -21,9 +21,9 @@ pub use environment::{ EnvironmentNetworkLayer, EnvironmentResourcesLayer, RunEnvironmentLayer, }; pub use llm::{ - CostRates, CredentialRef, CredentialRefParseError, LlmLayer, ModelControls, ModelCostTable, - ModelFeatures as LlmModelFeatures, ModelLimits as LlmModelLimits, ModelSettings, - ProviderSettings, ReasoningEffortFeature, + CostRates, CredentialRef, CredentialRefParseError, LlmLayer, LlmNormalizationError, + ModelControls, ModelCostTable, ModelFeatures as LlmModelFeatures, + ModelLimits as LlmModelLimits, ModelSettings, ProviderSettings, ReasoningEffortFeature, }; pub use log_filter::LogFilter; pub use maps::{MergeMap, ReplaceMap, StickyMap}; diff --git a/lib/crates/fabro-config/src/lib.rs b/lib/crates/fabro-config/src/lib.rs index f6097b9be..694ab1773 100644 --- a/lib/crates/fabro-config/src/lib.rs +++ b/lib/crates/fabro-config/src/lib.rs @@ -46,18 +46,19 @@ pub use layers::{ CredentialRefParseError, EnvironmentDockerfileLayer, EnvironmentImageLayer, EnvironmentLayer, EnvironmentLifecycleLayer, EnvironmentNetworkLayer, EnvironmentResourcesLayer, GitAuthorLayer, GithubIntegrationLayer, HookAgentMarker, HookEntry, HookTlsMode, IntegrationWebhooksLayer, - InterviewProviderLayer, InterviewsLayer, LlmLayer, LlmModelFeatures, LlmModelLimits, LogFilter, - McpEntryLayer, MergeMap, ModelControls, ModelCostTable, ModelRefOrSplice, ModelSettings, - NotificationProviderLayer, NotificationRouteLayer, ObjectStoreLocalLayer, ObjectStoreS3Layer, - PrepareStep, ProjectLayer, ProviderSettings, ReasoningEffortFeature, ReplaceMap, RunAgentLayer, - RunArtifactsLayer, RunCheckpointLayer, RunCloneLayer, RunEnvironmentLayer, RunExecutionLayer, - RunGitLayer, RunGoalLayer, RunIntegrationsGithubLayer, RunIntegrationsLayer, RunLayer, - RunMetaBranchLayer, RunModelControlsLayer, RunModelLayer, RunPrepareLayer, RunPullRequestLayer, - RunRunBranchLayer, RunScmLayer, ScmGitHubLayer, ServerApiLayer, ServerArtifactsLayer, - ServerAuthGithubLayer, ServerAuthLayer, ServerIntegrationsLayer, ServerLayer, - ServerListenLayer, ServerLoggingLayer, ServerSandboxLayer, ServerSandboxProviderLayer, - ServerSandboxProvidersLayer, ServerSchedulerLayer, ServerSlateDbLayer, ServerStorageLayer, - ServerWebLayer, SettingsLayer, SlackIntegrationLayer, StickyMap, StringOrSplice, WorkflowLayer, + InterviewProviderLayer, InterviewsLayer, LlmLayer, LlmModelFeatures, LlmModelLimits, + LlmNormalizationError, LogFilter, McpEntryLayer, MergeMap, ModelControls, ModelCostTable, + ModelRefOrSplice, ModelSettings, NotificationProviderLayer, NotificationRouteLayer, + ObjectStoreLocalLayer, ObjectStoreS3Layer, PrepareStep, ProjectLayer, ProviderSettings, + ReasoningEffortFeature, ReplaceMap, RunAgentLayer, RunArtifactsLayer, RunCheckpointLayer, + RunCloneLayer, RunEnvironmentLayer, RunExecutionLayer, RunGitLayer, RunGoalLayer, + RunIntegrationsGithubLayer, RunIntegrationsLayer, RunLayer, RunMetaBranchLayer, + RunModelControlsLayer, RunModelLayer, RunPrepareLayer, RunPullRequestLayer, RunRunBranchLayer, + RunScmLayer, ScmGitHubLayer, ServerApiLayer, ServerArtifactsLayer, ServerAuthGithubLayer, + ServerAuthLayer, ServerIntegrationsLayer, ServerLayer, ServerListenLayer, ServerLoggingLayer, + ServerSandboxLayer, ServerSandboxProviderLayer, ServerSandboxProvidersLayer, + ServerSchedulerLayer, ServerSlateDbLayer, ServerStorageLayer, ServerWebLayer, SettingsLayer, + SlackIntegrationLayer, StickyMap, StringOrSplice, WorkflowLayer, }; pub use logging::{resolve_log_destination, resolve_log_destination_with_env}; pub use parse::ParseError; diff --git a/lib/crates/fabro-config/src/parse.rs b/lib/crates/fabro-config/src/parse.rs index b4e008791..025e0b130 100644 --- a/lib/crates/fabro-config/src/parse.rs +++ b/lib/crates/fabro-config/src/parse.rs @@ -1,6 +1,7 @@ use std::fmt; use crate::SettingsLayer; +use crate::layers::LlmNormalizationError; const CURRENT_VERSION: u32 = 1; @@ -30,6 +31,7 @@ const LEGACY_LLM_KEYS: &[&str] = &[ #[derive(Debug, Clone, PartialEq, Eq)] pub enum ParseError { Toml(String), + LlmCatalog(LlmNormalizationError), Version(VersionError), UnknownTopLevelKey { key: String, @@ -45,6 +47,7 @@ impl fmt::Display for ParseError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { Self::Toml(msg) => write!(f, "settings file is not valid TOML: {msg}"), + Self::LlmCatalog(err) => fmt::Display::fmt(err, f), Self::Version(err) => fmt::Display::fmt(err, f), Self::UnknownTopLevelKey { key, hint } => { if let Some(hint) = hint { @@ -118,8 +121,14 @@ pub(crate) fn parse_settings(input: &str) -> Result { } } - raw.try_into::() - .map_err(|e| ParseError::Toml(e.to_string())) + let mut layer = raw + .try_into::() + .map_err(|e| ParseError::Toml(e.to_string()))?; + if let Some(llm) = layer.llm.as_mut() { + llm.normalize_legacy_models() + .map_err(ParseError::LlmCatalog)?; + } + Ok(layer) } #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -290,10 +299,196 @@ mod tests { #[test] fn accepts_new_llm_models_subtree() { - let parsed = "[llm.models.\"foo\"]\nprovider = \"kimi\"\n" + let parsed = "[llm.providers.kimi.models.\"foo\"]\n" .parse::() .unwrap(); - assert!(parsed.llm.unwrap().models.contains_key("foo")); + assert!( + parsed + .llm + .unwrap() + .providers + .get("kimi") + .unwrap() + .models + .contains_key("foo") + ); + } + + #[test] + fn provider_scoped_model_rejects_redundant_provider_field() { + let error = r#" +[llm.providers.openai.models."gpt-5.4"] +provider = "openai" +"# + .parse::() + .unwrap_err(); + + assert!(matches!( + error, + ParseError::LlmCatalog(LlmNormalizationError::ScopedModelDeclaresProvider { + provider, + model, + }) if provider.as_str() == "openai" && model.as_str() == "gpt-5.4" + )); + } + + #[test] + fn legacy_model_row_with_provider_normalizes_before_merge() { + use crate::layers::Combine as _; + + let higher = r#" +[llm.models."gpt-5.4"] +provider = "openai" +display_name = "Configured display name" +"# + .parse::() + .unwrap(); + let fallback = r#" +[llm.providers.openai.models."gpt-5.4"] +family = "gpt-5" +"# + .parse::() + .unwrap(); + + let merged = higher.combine(fallback); + let llm = merged.llm.unwrap(); + assert!(llm.models.is_empty()); + let model = llm + .providers + .get("openai") + .unwrap() + .models + .get("gpt-5.4") + .unwrap(); + assert_eq!( + model.display_name.as_deref(), + Some("Configured display name") + ); + assert_eq!(model.family.as_deref(), Some("gpt-5")); + } + + #[test] + fn provider_less_legacy_row_adopts_unique_builtin_offering() { + let parsed = r#" +[llm.models.mercury] +display_name = "Configured Mercury" +"# + .parse::() + .unwrap(); + let llm = parsed.llm.unwrap(); + + assert!(llm.models.is_empty()); + assert!( + llm.providers + .get("inception") + .unwrap() + .models + .contains_key("mercury-2") + ); + } + + #[test] + fn provider_less_legacy_row_rejects_ambiguous_builtin_offering() { + let error = r#" +[llm.models."gpt-5.6-sol"] +display_name = "Ambiguous" +"# + .parse::() + .unwrap_err(); + + assert!(matches!( + error, + ParseError::LlmCatalog(LlmNormalizationError::AmbiguousLegacyModel { + model, + candidates, + }) if model == "gpt-5.6-sol" && candidates.len() >= 2 + )); + } + + #[test] + fn same_source_legacy_and_provider_scoped_rows_conflict() { + let error = r#" +[llm.providers.openai.models."gpt-5.4"] +display_name = "Canonical" + +[llm.models."gpt-5.4"] +provider = "openai" +display_name = "Legacy" +"# + .parse::() + .unwrap_err(); + + assert!(matches!( + error, + ParseError::LlmCatalog(LlmNormalizationError::DuplicateModelDefinition { + provider, + model, + }) if provider.as_str() == "openai" && model.as_str() == "gpt-5.4" + )); + } + + #[test] + fn retired_legacy_model_id_names_replacement() { + let error = r#" +[llm.models."openai/gpt-5.6-sol"] +provider = "openrouter" +"# + .parse::() + .unwrap_err(); + + assert!(matches!( + error, + ParseError::LlmCatalog(LlmNormalizationError::RetiredModel { + identifier, + provider, + model, + }) if identifier == "openai/gpt-5.6-sol" + && provider.as_str() == "openrouter" + && model.as_str() == "gpt-5.6-sol" + )); + } + + #[test] + fn same_model_slug_on_different_providers_merges_independently() { + use crate::layers::Combine as _; + + let direct = r#" +[llm.providers.openai.models.shared] +display_name = "Direct" +"# + .parse::() + .unwrap(); + let aggregator = r#" +[llm.providers.openrouter.models.shared] +display_name = "Aggregator" +"# + .parse::() + .unwrap(); + + let merged = direct.combine(aggregator); + let providers = merged.llm.unwrap().providers; + assert_eq!( + providers + .get("openai") + .unwrap() + .models + .get("shared") + .unwrap() + .display_name + .as_deref(), + Some("Direct") + ); + assert_eq!( + providers + .get("openrouter") + .unwrap() + .models + .get("shared") + .unwrap() + .display_name + .as_deref(), + Some("Aggregator") + ); } #[test] diff --git a/lib/crates/fabro-dev/src/commands/docs_options_reference.rs b/lib/crates/fabro-dev/src/commands/docs_options_reference.rs index aacc08575..9a2414e0a 100644 --- a/lib/crates/fabro-dev/src/commands/docs_options_reference.rs +++ b/lib/crates/fabro-dev/src/commands/docs_options_reference.rs @@ -247,19 +247,20 @@ x-team-secret = "{{ secrets.gateway_team_secret }}" | `auth.credentials` | array | required when `auth` present | Ordered credential refs. Accepted forms are `vault:`, `env:`, and `aws_sigv4` (sign requests from the AWS default credential chain — Bedrock). Literal secret strings are rejected. | | `auth.header` | `"bearer"` or `{ custom = "Header-Name" }` | `"bearer"` | Primary API-key header policy. Omit when the provider uses a standard bearer token. | | `extra_headers` | table | `{}` | Additional headers attached to provider requests. Values are interpolation strings: literal text, an `{{ env.NAME }}` token, or a `{{ secrets.NAME }}` token. Put credentials in a secret and reference them with a `{{ secrets.NAME }}` token, not a bare literal. | -| `priority` | integer | `0` | Higher-priority configured providers win default selection; ties use canonical provider ID. | +| `priority` | integer | `0` | Higher-priority ready providers win unqualified model and default selection; ties use canonical provider ID. | | `enabled` | boolean | `true` | Set `false` to disable a provider after lower-precedence layers define it. | | `aliases` | array | `[]` | Additional provider names accepted by model routing and fallback config. | -## `[llm.models.]` +## `[llm.providers..models.]` -Define or override a model in the catalog. The table key is the canonical -model ID Fabro users reference; `api_id` is the model string sent to the -provider API. +Define or override one provider's offering of a model. The table key is the +canonical model slug Fabro users reference. An offering's identity is the +pair `(provider, model slug)`, so different providers may use the same slug +and aliases. `api_id` is the opaque model string sent to this provider's API +and defaults to the exact model slug. ```toml title="settings.toml" -[llm.models."team-code-large"] -provider = "proxy" +[llm.providers.proxy.models."team-code-large"] api_id = "provider-wire-model-name" agent_profile = "anthropic" display_name = "Team Code Large" @@ -270,27 +271,27 @@ enabled = true aliases = ["team-code"] estimated_output_tps = 80 -[llm.models."team-code-large".limits] +[llm.providers.proxy.models."team-code-large".limits] context_window = 200000 max_output = 32000 -[llm.models."team-code-large".features] +[llm.providers.proxy.models."team-code-large".features] tools = true vision = false reasoning = true reasoning_effort = "levels" prompt_cache = true -[llm.models."team-code-large".controls] +[llm.providers.proxy.models."team-code-large".controls] reasoning_effort = ["low", "medium", "high"] speed = ["fast"] -[llm.models."team-code-large".costs] +[llm.providers.proxy.models."team-code-large".costs] input_cost_per_mtok = 1.50 output_cost_per_mtok = 8.00 cache_input_cost_per_mtok = 0.30 -[llm.models."team-code-large".costs.speed.fast] +[llm.providers.proxy.models."team-code-large".costs.speed.fast] input_cost_per_mtok = 3.00 output_cost_per_mtok = 16.00 cache_input_cost_per_mtok = 0.60 @@ -298,8 +299,7 @@ cache_input_cost_per_mtok = 0.60 | Key | Type / values | Default | Description | |---|---|---|---| -| `provider` | string | None | Provider ID this model belongs to. | -| `api_id` | string | model ID | Identifier sent to the provider API. | +| `api_id` | string | model slug | Opaque identifier sent to this provider's API. An explicitly empty value is invalid. | | `agent_profile` | `"anthropic"` \| `"openai"` \| `"gemini"` | provider profile | Agent profile override for this model. Model overrides take precedence over provider overrides. | | `billing_policy` | `"openai"` \| `"anthropic"` \| `"gemini"` \| `"none"` | provider policy | Billing algorithm override for this model — for models whose billing family differs from their provider's (e.g. Claude served through OpenRouter bills Anthropic-style cache reads/writes). | | `display_name` | string | model ID | Human-readable model name. | @@ -309,17 +309,17 @@ cache_input_cost_per_mtok = 0.60 | `default` | boolean | `false` | Whether this is the provider default model. | | `probe` | boolean | `false` | Whether this model should be preferred for provider connectivity probes. Set `false` in a higher-precedence layer to clear an inherited probe marker. | | `enabled` | boolean | `true` | Set `false` to disable a model after lower-precedence layers define it. | -| `aliases` | array | `[]` | Additional model names accepted by routing and fallback config. | +| `aliases` | array | `[]` | Additional model selectors accepted by routing and fallback config. Aliases may repeat across providers, but one selector cannot identify two models within the same provider. | | `estimated_output_tps` | number | None | Estimated output tokens per second for catalog display and planning. | -## `[llm.models..limits]` +## `[llm.providers..models..limits]` | Key | Type / values | Default | Description | |---|---|---|---| | `context_window` | integer | None | Maximum context window size in tokens. | | `max_output` | integer | None | Maximum output tokens, if known. | -## `[llm.models..features]` +## `[llm.providers..models..features]` | Key | Type / values | Default | Description | |---|---|---|---| @@ -330,14 +330,14 @@ cache_input_cost_per_mtok = 0.60 | `prompt_cache` | boolean | `false` | Whether prompt cache pricing/usage applies. | | `sampling_params` | boolean | `true` | Whether the model accepts classic sampling parameters (`temperature`, `top_p`). | -## `[llm.models..controls]` +## `[llm.providers..models..controls]` | Key | Type / values | Default | Description | |---|---|---|---| | `reasoning_effort` | array | all standard levels when feature is `"levels"` or `"always_adaptive"` | User-facing reasoning effort values Fabro may send for this model. Can be set explicitly for reasoning models whose provider adapter maps effort to a non-native API shape. | | `speed` | array | `[]` | Additional speeds beyond implicit `standard`; do not list `standard`. | -## `[llm.models..costs]` +## `[llm.providers..models..costs]` | Key | Type / values | Default | Description | |---|---|---|---| @@ -345,10 +345,12 @@ cache_input_cost_per_mtok = 0.60 | `output_cost_per_mtok` | number | None | Output cost in USD per million tokens. | | `cache_input_cost_per_mtok` | number | None | Cached input/read cost in USD per million tokens. | -## `[llm.models..costs.speed.]` +## `[llm.providers..models..costs.speed.]` -Per-speed cost overrides use the same keys as `[llm.models..costs]`. -Each `` key must be declared in `[llm.models..controls].speed`. +Per-speed cost overrides use the same keys as +`[llm.providers..models..costs]`. Each `` key +must be declared in +`[llm.providers..models..controls].speed`. The `standard` speed is implicit and always uses the base cost table. "#, diff --git a/lib/crates/fabro-hooks/src/executor.rs b/lib/crates/fabro-hooks/src/executor.rs index a409a49b7..052776af1 100644 --- a/lib/crates/fabro-hooks/src/executor.rs +++ b/lib/crates/fabro-hooks/src/executor.rs @@ -323,11 +323,10 @@ impl HookExecutorImpl { } } - /// Resolve a model alias (e.g. "haiku") to a concrete model ID. - fn resolve_model(model: Option<&str>, catalog: &Catalog) -> String { - let model_id = model.unwrap_or("haiku"); - let model_info = catalog.get(model_id); - model_info.map_or(model_id, |m| m.id.as_str()).to_string() + /// Keep the requested selector intact so the ready-provider-aware LLM + /// client can resolve aliases at dispatch time. + fn resolve_model(model: Option<&str>) -> String { + model.unwrap_or("haiku").to_string() } /// Build the user message for prompt/agent hooks. @@ -377,7 +376,7 @@ impl HookExecutorImpl { } }; - let resolved_model = Self::resolve_model(model.as_deref(), catalog.as_ref()); + let resolved_model = Self::resolve_model(model.as_deref()); let user_msg = Self::build_hook_user_message(&prompt, context); Self::execute_llm_with_timeout(definition.timeout(), "prompt", || async move { @@ -446,7 +445,7 @@ impl HookExecutorImpl { } }; - let resolved_model = Self::resolve_model(model.as_deref(), catalog.as_ref()); + let resolved_model = Self::resolve_model(model.as_deref()); let user_msg = Self::build_hook_user_message(&prompt, context); Self::execute_llm_with_timeout(definition.timeout(), "agent", || async move { diff --git a/lib/crates/fabro-llm/src/adapter_registry.rs b/lib/crates/fabro-llm/src/adapter_registry.rs index 7a5638c93..810f6d6f4 100644 --- a/lib/crates/fabro-llm/src/adapter_registry.rs +++ b/lib/crates/fabro-llm/src/adapter_registry.rs @@ -10,7 +10,9 @@ use std::collections::HashMap; use std::sync::Arc; use fabro_auth::ApiKeyHeader; -use fabro_model::{AdapterKind, AgentProfileKind, BillingPolicy, Catalog, CodecKind, ProviderId}; +use fabro_model::{ + AdapterKind, AgentProfileKind, BillingPolicy, Catalog, CodecKind, Model, ProviderId, +}; use crate::error::Error; use crate::provider::ProviderAdapter; @@ -242,13 +244,11 @@ pub struct Route { pub agent_profile: AgentProfileKind, } -/// Resolve the route for `model_id_or_alias` from the catalog's provider and -/// model rows. Returns `None` when the model or its provider is unknown. +/// Resolve the route for one already-selected catalog offering. #[must_use] -pub fn resolve_route(catalog: &Catalog, model_id_or_alias: &str) -> Option { - let model = catalog.get(model_id_or_alias)?; +pub fn resolve_route(catalog: &Catalog, model: &Model) -> Option { let provider = catalog.provider(&model.provider)?; - let settings = catalog.model_settings(&model.id)?; + let settings = catalog.settings_for(model)?; Some(Route { provider: provider.id.clone(), transport: provider.adapter, @@ -263,6 +263,12 @@ pub fn resolve_route(catalog: &Catalog, model_id_or_alias: &str) -> Option(catalog: &'a Catalog, selector: &str) -> &'a Model { + catalog + .select(selector, None, &catalog.all_provider_ids()) + .unwrap_or_else(|error| panic!("built-in model '{selector}' should resolve: {error}")) + } + /// One row of the route-equivalence table: model id plus the /// `(deployment_id, transport, codec, billing_policy, agent_profile)` /// tuple it must resolve to. @@ -336,7 +342,8 @@ mod tests { ); for (model_id, deployment_id, transport, codec, billing_policy, agent_profile) in expected { - let route = resolve_route(catalog, model_id) + let model = select_from_all(catalog, model_id); + let route = resolve_route(catalog, model) .unwrap_or_else(|| panic!("built-in model '{model_id}' should resolve")); assert_eq!(route.deployment_id, *deployment_id, "{model_id}"); assert_eq!(route.transport, *transport, "{model_id}"); @@ -350,16 +357,20 @@ mod tests { fn resolve_route_follows_model_aliases() { let catalog = Catalog::builtin(); - let by_alias = resolve_route(catalog, "sonnet").expect("alias should resolve"); - let by_id = resolve_route(catalog, "claude-sonnet-4-6").expect("id should resolve"); + let by_alias = resolve_route(catalog, select_from_all(catalog, "sonnet")) + .expect("alias should resolve"); + let by_id = resolve_route(catalog, select_from_all(catalog, "claude-sonnet-4-6")) + .expect("id should resolve"); assert_eq!(by_alias, by_id); assert_eq!(by_alias.provider, ProviderId::anthropic()); } #[test] - fn resolve_route_returns_none_for_unknown_models() { - assert_eq!(resolve_route(Catalog::builtin(), "not-a-model"), None); + fn resolve_route_returns_none_for_model_from_another_catalog() { + let other = Catalog::from_builtin().unwrap(); + let model = select_from_all(&other, "gpt-5.4"); + assert!(resolve_route(Catalog::builtin(), model).is_some()); } #[test] diff --git a/lib/crates/fabro-llm/src/client.rs b/lib/crates/fabro-llm/src/client.rs index c72e9ea50..e40a8ede2 100644 --- a/lib/crates/fabro-llm/src/client.rs +++ b/lib/crates/fabro-llm/src/client.rs @@ -1,12 +1,12 @@ -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::sync::Arc; use fabro_auth::{ApiCredential, CredentialSource}; -use fabro_model::{AdapterKind, Catalog, ProviderId}; +use fabro_model::{AdapterKind, Catalog, ModelSelectionError, ProviderId}; use tracing::debug; use crate::adapter_registry::{ - self, AdapterConfig, AdapterKindOptions, OpenAiAdapterOptions, factory_for, + AdapterConfig, AdapterKindOptions, OpenAiAdapterOptions, factory_for, }; use crate::cost; use crate::error::{Error, ProviderErrorKind}; @@ -44,6 +44,11 @@ enum RegistrationMode { CollectIssues, } +struct ResolvedRequest { + provider: Arc, + request: Request, +} + impl Client { /// Create a new Client with explicit configuration. #[must_use] @@ -259,44 +264,150 @@ impl Client { ) } - /// Resolve the provider for a request: an explicit `request.provider` - /// wins, then the model's catalog route, then the default provider. - fn resolve_provider(&self, request: &Request) -> Result, Error> { - let route = self - .catalog - .as_ref() - .and_then(|catalog| adapter_registry::resolve_route(catalog, &request.model)); - - let provider_name = request - .provider - .as_deref() - .or_else(|| route.as_ref().map(|route| route.provider.as_str())) - .or(self.default_provider.as_deref()) - .ok_or_else(|| Error::Configuration { - message: "No provider specified and no default provider set".into(), - source: None, - })?; - let provider_name = self.canonical_provider_name(provider_name); - - self.providers - .get(&provider_name) - .cloned() - .ok_or_else(|| Error::Configuration { - message: format!("Provider '{provider_name}' not registered"), - source: None, + fn provider_adapter(&self, provider_name: &str) -> Option> { + let canonical = self.canonical_provider_name(provider_name); + self.providers.get(&canonical).cloned().or_else(|| { + self.providers.iter().find_map(|(name, adapter)| { + (self.canonical_provider_name(name) == canonical).then(|| Arc::clone(adapter)) }) + }) + } + + fn eligible_provider_ids(&self) -> HashSet { + self.providers + .keys() + .map(|provider| ProviderId::new(self.canonical_provider_name(provider))) + .collect() + } + + /// Resolve one concrete provider/model offering and canonicalize a cloned + /// request. Explicit-provider unknown models remain passthrough values. + fn resolve_request_with_adapter(&self, request: &Request) -> Result { + let mut resolved = request.clone(); + let Some(catalog) = &self.catalog else { + let provider_name = request + .provider + .as_deref() + .or(self.default_provider.as_deref()) + .ok_or_else(|| Error::Configuration { + message: "No provider specified and no default provider set".into(), + source: None, + })?; + let provider = + self.provider_adapter(provider_name) + .ok_or_else(|| Error::Configuration { + message: format!("Provider '{provider_name}' not registered"), + source: None, + })?; + resolved.provider = Some(provider.name().to_string()); + return Ok(ResolvedRequest { + provider, + request: resolved, + }); + }; + + let eligible = self.eligible_provider_ids(); + if let Some(explicit) = request.provider.as_deref() { + let explicit = ProviderId::new(explicit); + if let Some(catalog_provider) = catalog.provider(&explicit) { + let provider = self + .provider_adapter(catalog_provider.id.as_str()) + .ok_or_else(|| { + selection_error(ModelSelectionError::ProviderUnavailable { + provider: catalog_provider.id.clone(), + }) + })?; + match catalog.resolve_on_provider(&catalog_provider.id, &request.model) { + Ok(model) => resolved.model = model.id.to_string(), + Err(ModelSelectionError::UnknownSelectorOnProvider { .. }) => {} + Err(error) => return Err(selection_error(error)), + } + resolved.provider = Some(catalog_provider.id.to_string()); + return Ok(ResolvedRequest { + provider, + request: resolved, + }); + } + + let provider = + self.provider_adapter(explicit.as_str()) + .ok_or_else(|| Error::Configuration { + message: format!("Provider '{explicit}' not registered"), + source: None, + })?; + resolved.provider = Some(provider.name().to_string()); + return Ok(ResolvedRequest { + provider, + request: resolved, + }); + } + + match catalog.select(&request.model, None, &eligible) { + Ok(model) => { + let provider = self + .provider_adapter(model.provider.as_str()) + .ok_or_else(|| { + selection_error(ModelSelectionError::ProviderUnavailable { + provider: model.provider.clone(), + }) + })?; + resolved.model = model.id.to_string(); + resolved.provider = Some(model.provider.to_string()); + Ok(ResolvedRequest { + provider, + request: resolved, + }) + } + Err(ModelSelectionError::UnknownSelector { .. }) => { + let provider_name = + self.default_provider + .as_deref() + .ok_or_else(|| Error::Configuration { + message: "No provider specified and no default provider set".into(), + source: None, + })?; + let provider = + self.provider_adapter(provider_name) + .ok_or_else(|| Error::Configuration { + message: format!("Provider '{provider_name}' not registered"), + source: None, + })?; + resolved.provider = Some(self.canonical_provider_name(provider.name())); + Ok(ResolvedRequest { + provider, + request: resolved, + }) + } + Err(error) => Err(selection_error(error)), + } + } + + /// Resolve the concrete provider/model route and return a canonicalized + /// clone of the request without dispatching it. + /// + /// This is useful at persistence and API boundaries that must expose the + /// selected provider alongside the canonical model ID. The caller-owned + /// request is never modified. + pub fn resolve_request(&self, request: &Request) -> Result { + self.resolve_request_with_adapter(request) + .map(|resolved| resolved.request) } fn validate_request_controls(&self, request: &Request) -> Result<(), Error> { let Some(catalog) = &self.catalog else { return Ok(()); }; - let Some(settings) = catalog.model_settings(&request.model) else { + let Some(provider) = request.provider.as_deref() else { return Ok(()); }; - let model_id = catalog - .get(&request.model) - .map_or(request.model.as_str(), |model| model.id.as_str()); + let Some(model) = catalog.get_on_provider(&ProviderId::new(provider), &request.model) + else { + return Ok(()); + }; + let Some(settings) = catalog.settings_for(model) else { + return Ok(()); + }; + let model_id = model.id.as_str(); if let Some(effort) = request.reasoning_effort { if !settings.controls.reasoning_effort.contains(&effort) { @@ -333,11 +444,11 @@ impl Client { /// registered, or any provider/middleware error encountered during the /// request. pub async fn complete(&self, request: &Request) -> Result { - self.validate_request_controls(request)?; - let provider = self.resolve_provider(request)?; + let ResolvedRequest { provider, request } = self.resolve_request_with_adapter(request)?; + self.validate_request_controls(&request)?; if self.middleware.is_empty() { - return complete_stamped(&provider, self.catalog.as_deref(), request).await; + return complete_stamped(&provider, self.catalog.as_deref(), &request).await; } // Build middleware chain. Cost is stamped at the base so middleware @@ -358,7 +469,7 @@ impl Client { }) }); - chain(request.clone()).await + chain(request).await } /// Send a streaming request (Section 4.2). @@ -369,11 +480,11 @@ impl Client { /// registered, or any provider/middleware error encountered during the /// request. pub async fn stream(&self, request: &Request) -> Result { - self.validate_request_controls(request)?; - let provider = self.resolve_provider(request)?; + let ResolvedRequest { provider, request } = self.resolve_request_with_adapter(request)?; + self.validate_request_controls(&request)?; if self.middleware.is_empty() { - return stream_stamped(&provider, self.catalog.clone(), request).await; + return stream_stamped(&provider, self.catalog.clone(), &request).await; } // Build streaming middleware chain. Cost is stamped at the base so @@ -394,7 +505,7 @@ impl Client { }) }); - chain(request.clone()).await + chain(request).await } /// Count the model-visible input/context tokens for a request without @@ -410,19 +521,19 @@ impl Client { request: &Request, preference: InputTokenCountPreference, ) -> Result { - self.validate_request_controls(request)?; - let provider = self.resolve_provider(request)?; - provider.validate_request(request)?; + let ResolvedRequest { provider, request } = self.resolve_request_with_adapter(request)?; + self.validate_request_controls(&request)?; + provider.validate_request(&request)?; if preference == InputTokenCountPreference::EstimateOnly { - return Ok(estimate_input_tokens(request, provider.name())); + return Ok(estimate_input_tokens(&request, provider.name())); } - match provider.count_input_tokens(request).await { + match provider.count_input_tokens(&request).await { Ok(Some(count)) => Ok(count), Ok(None) if preference == InputTokenCountPreference::PreferProvider => { Ok(fallback_estimate( - request, + &request, provider.name(), "provider_token_count_unsupported", "provider does not support input token counting; returned local estimate", @@ -440,7 +551,7 @@ impl Client { && token_count_fallback_eligible(&error) => { Ok(fallback_estimate( - request, + &request, provider.name(), "provider_token_count_failed", "provider input token counting failed; returned local estimate", @@ -471,6 +582,12 @@ impl Client { .collect() } + /// Canonical IDs for provider adapters that registered successfully. + #[must_use] + pub fn provider_ids(&self) -> HashSet { + self.eligible_provider_ids() + } + /// Check whether a provider adapter is registered. #[must_use] pub fn has_provider(&self, name: &str) -> bool { @@ -489,6 +606,10 @@ impl Client { } } +fn selection_error(error: ModelSelectionError) -> Error { + Error::configuration_error(error.to_string(), error) +} + /// Validate, run, and cost-stamp a blocking request. Shared by /// [`Client::complete`]'s direct path and its middleware-chain base so cost /// stamping stays single-sited. @@ -499,7 +620,19 @@ async fn complete_stamped( ) -> Result { provider.validate_request(request)?; let mut response = provider.complete(request).await?; - cost::apply_estimated_cost(catalog, &request.model, request.speed, &mut response); + let selected_provider = request + .provider + .as_deref() + .unwrap_or_else(|| provider.name()); + response.model.clone_from(&request.model); + response.provider = selected_provider.to_string(); + cost::apply_estimated_cost( + catalog, + selected_provider, + &request.model, + request.speed, + &mut response, + ); Ok(response) } @@ -515,6 +648,10 @@ async fn stream_stamped( let stream = provider.stream(request).await?; Ok(stamp_stream_costs( catalog, + request + .provider + .clone() + .unwrap_or_else(|| provider.name().to_string()), request.model.clone(), request.speed, stream, @@ -526,6 +663,7 @@ async fn stream_stamped( /// [`Client::complete`] stamps on blocking responses. fn stamp_stream_costs( catalog: Option>, + provider: String, model: String, speed: Option, stream: StreamEventStream, @@ -534,8 +672,12 @@ fn stamp_stream_costs( Box::pin(stream.map(move |event| { event.map(|mut event| { - if let StreamEvent::Finish { response, .. } = &mut event { - cost::apply_estimated_cost(catalog.as_deref(), &model, speed, response); + if let StreamEvent::Finish { response, .. } | StreamEvent::StepFinish { response, .. } = + &mut event + { + response.model.clone_from(&model); + response.provider.clone_from(&provider); + cost::apply_estimated_cost(catalog.as_deref(), &provider, &model, speed, response); } event }) @@ -606,7 +748,9 @@ mod tests { use futures::stream; use super::*; + use crate::adapter_registry; use crate::error::ProviderErrorDetail; + use crate::providers::openai_compatible; use crate::types::*; /// A mock provider for testing. @@ -863,6 +1007,169 @@ output_cost_per_mtok = 2.0 Arc::new(Catalog::from_settings(&settings).unwrap()) } + fn portable_model_catalog(openrouter_base_url: &str) -> Arc { + let settings: LlmCatalogSettings = toml::from_str(&format!( + r#" +[providers.openai] +display_name = "OpenAI" +adapter = "openai_compatible" +agent_profile = "openai" +base_url = "https://openai.invalid/v1" +priority = 90 + +[providers.openai.models."gpt-5.6-sol"] +display_name = "GPT-5.6 Sol" +family = "gpt-5" +aliases = ["gpt-56-sol"] +default = true + +[providers.openai.models."gpt-5.6-sol".limits] +context_window = 1000 + +[providers.openai.models."gpt-5.6-sol".features] +tools = true +vision = false +reasoning = false + +[providers.openai.models."gpt-5.6-sol".costs] +input_cost_per_mtok = 1.0 +output_cost_per_mtok = 2.0 + +[providers.openrouter] +display_name = "OpenRouter" +adapter = "openai_compatible" +agent_profile = "openai" +base_url = "{openrouter_base_url}" +priority = 25 + +[providers.openrouter.models."gpt-5.6-sol"] +api_id = "openai/gpt-5.6-sol" +display_name = "GPT-5.6 Sol (via OpenRouter)" +family = "gpt-5" +aliases = ["gpt-56-sol"] +default = true + +[providers.openrouter.models."gpt-5.6-sol".limits] +context_window = 1000 + +[providers.openrouter.models."gpt-5.6-sol".features] +tools = true +vision = false +reasoning = false + +[providers.openrouter.models."gpt-5.6-sol".costs] +input_cost_per_mtok = 10.0 +output_cost_per_mtok = 20.0 +"#, + )) + .unwrap(); + Arc::new(Catalog::from_settings(&settings).unwrap()) + } + + async fn portable_mock_client(catalog: &Arc, providers: &[&str]) -> Client { + let mut client = Client::new(HashMap::new(), None, vec![]); + for provider in providers { + client + .register_provider(Arc::new(MockProvider::new(provider, provider))) + .await + .unwrap(); + } + client.catalog = Some(Arc::clone(catalog)); + client + } + + #[tokio::test] + async fn shared_alias_selects_by_ready_providers_and_priority_without_mutating_request() { + let catalog = portable_model_catalog("https://openrouter.invalid/v1"); + let mut original = test_request(); + original.model = "gpt-56-sol".to_string(); + + let direct = portable_mock_client(&catalog, &["openai"]).await; + let direct_request = direct.resolve_request(&original).unwrap(); + assert_eq!(direct_request.model, "gpt-5.6-sol"); + assert_eq!(direct_request.provider.as_deref(), Some("openai")); + + let aggregator = portable_mock_client(&catalog, &["openrouter"]).await; + let aggregator_request = aggregator.resolve_request(&original).unwrap(); + assert_eq!(aggregator_request.model, "gpt-5.6-sol"); + assert_eq!(aggregator_request.provider.as_deref(), Some("openrouter")); + + let both = portable_mock_client(&catalog, &["openrouter", "openai"]).await; + let both_request = both.resolve_request(&original).unwrap(); + assert_eq!(both_request.provider.as_deref(), Some("openai")); + + assert_eq!(original.model, "gpt-56-sol"); + assert_eq!(original.provider, None); + + let response = aggregator.complete(&original).await.unwrap(); + assert_eq!(response.model, "gpt-5.6-sol"); + assert_eq!(response.provider, "openrouter"); + assert_eq!(response.cost_source, Some(CostSource::Estimated)); + assert_eq!(response.cost_usd, Some(0.0005)); + } + + #[tokio::test] + async fn explicit_provider_pins_shared_alias_and_preserves_unknown_passthrough() { + let catalog = portable_model_catalog("https://openrouter.invalid/v1"); + let client = portable_mock_client(&catalog, &["openai", "openrouter"]).await; + + let mut aliased = test_request(); + aliased.model = "gpt-56-sol".to_string(); + aliased.provider = Some("openrouter".to_string()); + let resolved = client.resolve_request(&aliased).unwrap(); + assert_eq!(resolved.model, "gpt-5.6-sol"); + assert_eq!(resolved.provider.as_deref(), Some("openrouter")); + + let mut unknown = test_request(); + unknown.model = "provider-private-preview".to_string(); + unknown.provider = Some("openrouter".to_string()); + let resolved = client.resolve_request(&unknown).unwrap(); + assert_eq!(resolved.model, "provider-private-preview"); + assert_eq!(resolved.provider.as_deref(), Some("openrouter")); + } + + #[tokio::test] + async fn selected_offering_api_id_reaches_openai_compatible_wire_request() { + let upstream = httpmock::MockServer::start_async().await; + let completion = upstream + .mock_async(|when, then| { + when.method(httpmock::Method::POST) + .path("/chat/completions") + .json_body_includes(r#"{"model":"openai/gpt-5.6-sol"}"#); + then.status(200) + .header("content-type", "application/json") + .json_body(serde_json::json!({ + "id": "chatcmpl-portable", + "model": "openai/gpt-5.6-sol", + "choices": [{ + "message": {"role": "assistant", "content": "OK"}, + "finish_reason": "stop" + }], + "usage": { + "prompt_tokens": 1, + "completion_tokens": 1, + "total_tokens": 2 + } + })); + }) + .await; + let catalog = portable_model_catalog(&upstream.base_url()); + let adapter = openai_compatible::Adapter::new("test-key", upstream.base_url()) + .with_name("openrouter") + .with_catalog(Arc::clone(&catalog)); + let mut client = Client::new(HashMap::new(), None, vec![]); + client.register_provider(Arc::new(adapter)).await.unwrap(); + client.catalog = Some(catalog); + let mut request = test_request(); + request.model = "gpt-56-sol".to_string(); + + let response = client.complete(&request).await.unwrap(); + + assert_eq!(response.model, "gpt-5.6-sol"); + assert_eq!(response.provider, "openrouter"); + completion.assert_async().await; + } + #[tokio::test] async fn complete_stamps_estimated_cost_from_catalog() { let mut client = Client::new(HashMap::new(), None, vec![]); @@ -1592,7 +1899,10 @@ reasoning = false let mut request = test_request(); request.provider = Some("acme-ai".to_string()); - let provider = client.resolve_provider(&request).unwrap(); + let provider = client + .resolve_request_with_adapter(&request) + .unwrap() + .provider; assert_eq!(provider.name(), "acme"); } @@ -1620,12 +1930,15 @@ reasoning = false let client = client_with_all_catalog_providers(&catalog).await; for model in catalog.list(None) { - let route = adapter_registry::resolve_route(&catalog, &model.id) + let route = adapter_registry::resolve_route(&catalog, model) .expect("built-in model should resolve to a route"); let mut request = test_request(); - request.model = model.id.clone(); + request.model = model.id.to_string(); - let provider = client.resolve_provider(&request).unwrap(); + let provider = client + .resolve_request_with_adapter(&request) + .unwrap() + .provider; assert_eq!(provider.name(), route.provider.as_str(), "{}", model.id); } @@ -1640,7 +1953,10 @@ reasoning = false request.model = "gpt-5.4-mini".to_string(); request.provider = Some("anthropic".to_string()); - let provider = client.resolve_provider(&request).unwrap(); + let provider = client + .resolve_request_with_adapter(&request) + .unwrap() + .provider; assert_eq!(provider.name(), "anthropic"); } @@ -1654,7 +1970,10 @@ reasoning = false let mut request = test_request(); request.model = "model-not-in-any-catalog".to_string(); - let provider = client.resolve_provider(&request).unwrap(); + let provider = client + .resolve_request_with_adapter(&request) + .unwrap() + .provider; assert_eq!(provider.name(), default); } diff --git a/lib/crates/fabro-llm/src/codec/anthropic_messages/encode.rs b/lib/crates/fabro-llm/src/codec/anthropic_messages/encode.rs index 360f4210a..2803c221c 100644 --- a/lib/crates/fabro-llm/src/codec/anthropic_messages/encode.rs +++ b/lib/crates/fabro-llm/src/codec/anthropic_messages/encode.rs @@ -618,26 +618,26 @@ reasoning = true /// Encode `request` on the direct-Anthropic route, optionally with a /// catalog (for capability-driven behavior like prompt-cache/effort). fn encode_direct(request: &Request, catalog: Option<&Catalog>, stream: bool) -> EncodedRequest { - let deployment_id = common::api_model_id(catalog, &request.model); + let deployment_id = common::api_model_id(catalog, "anthropic", &request.model); let params = direct_params(); let ctx = CodecCtx { request, provider_name: "anthropic", deployment_id: &deployment_id, - model: common::catalog_model(catalog, &request.model), + model: common::catalog_model(catalog, "anthropic", &request.model), params: ¶ms, }; encode(&ctx, stream) } fn encode_count_direct(request: &Request, catalog: Option<&Catalog>) -> EncodedRequest { - let deployment_id = common::api_model_id(catalog, &request.model); + let deployment_id = common::api_model_id(catalog, "anthropic", &request.model); let params = direct_params(); let ctx = CodecCtx { request, provider_name: "anthropic", deployment_id: &deployment_id, - model: common::catalog_model(catalog, &request.model), + model: common::catalog_model(catalog, "anthropic", &request.model), params: ¶ms, }; encode_count_tokens(&ctx) diff --git a/lib/crates/fabro-llm/src/codec/bedrock_converse/encode.rs b/lib/crates/fabro-llm/src/codec/bedrock_converse/encode.rs index a328434de..54a38314e 100644 --- a/lib/crates/fabro-llm/src/codec/bedrock_converse/encode.rs +++ b/lib/crates/fabro-llm/src/codec/bedrock_converse/encode.rs @@ -347,8 +347,8 @@ fn merge_provider_options(body: &mut Value, provider_options: Option<&Value>, pr #[cfg(test)] mod tests { - use fabro_model::Catalog; use fabro_model::catalog::LlmCatalogSettings; + use fabro_model::{Catalog, ProviderId}; use serde_json::json; use super::*; @@ -617,7 +617,7 @@ sampling_params = false request: &request, provider_name: "bedrock", deployment_id: "pinned-model", - model: catalog.get("pinned-model"), + model: catalog.get_on_provider(&ProviderId::new("bedrock"), "pinned-model"), params: ¶ms, }; let encoded = encode(&ctx, false).unwrap(); diff --git a/lib/crates/fabro-llm/src/codec/openai_compatible/request.rs b/lib/crates/fabro-llm/src/codec/openai_compatible/request.rs index 154013725..72920b34c 100644 --- a/lib/crates/fabro-llm/src/codec/openai_compatible/request.rs +++ b/lib/crates/fabro-llm/src/codec/openai_compatible/request.rs @@ -79,7 +79,7 @@ pub(super) fn merge_provider_options( #[cfg(test)] mod tests { - use fabro_model::Catalog; + use fabro_model::{Catalog, ProviderId}; use super::super::wire::ApiRequest; use super::*; @@ -183,16 +183,18 @@ mod tests { #[test] fn encode_omits_sampling_params_for_models_that_reject_them() { - let model = Catalog::builtin().get("kimi-k3").unwrap(); + let model = Catalog::builtin() + .get_on_provider(&ProviderId::new("kimi"), "kimi-k3") + .unwrap(); let mut request = minimal_request(); - request.model = model.id.clone(); + request.model = model.id.to_string(); request.temperature = Some(0.7); request.top_p = Some(0.9); let params = CodecParams::default(); let ctx = CodecCtx { request: &request, provider_name: "kimi", - deployment_id: &model.id, + deployment_id: model.id.as_str(), model: Some(model), params: ¶ms, }; diff --git a/lib/crates/fabro-llm/src/cost.rs b/lib/crates/fabro-llm/src/cost.rs index e03ead5de..898dcb362 100644 --- a/lib/crates/fabro-llm/src/cost.rs +++ b/lib/crates/fabro-llm/src/cost.rs @@ -25,8 +25,8 @@ pub(crate) fn estimate_cost_usd( let catalog = catalog?; // The billing machinery compares ModelRefs against the catalog's // canonical identity, so resolve model aliases and provider names first. - let model = catalog.get(model)?; let provider = catalog.provider(&ProviderId::new(provider))?; + let model = catalog.get_on_provider(&provider.id, model)?; let model_ref = ModelRef { provider: provider.id.clone(), model_id: model.id.clone(), @@ -47,6 +47,7 @@ pub(crate) fn estimate_cost_usd( /// aliases); the response's provider name selects the billing policy. pub(crate) fn apply_estimated_cost( catalog: Option<&Catalog>, + provider: &str, model: &str, speed: Option, response: &mut Response, @@ -54,7 +55,7 @@ pub(crate) fn apply_estimated_cost( if response.cost_usd.is_some() { return; } - let estimate = estimate_cost_usd(catalog, &response.provider, model, &response.usage, speed); + let estimate = estimate_cost_usd(catalog, provider, model, &response.usage, speed); response.cost_usd = estimate; response.cost_source = estimate.map(|_| CostSource::Estimated); } @@ -214,7 +215,7 @@ output_cost_per_mtok = {output_cost_per_mtok} ..TokenCounts::default() }); - apply_estimated_cost(Some(&catalog), "gpt-test", None, &mut response); + apply_estimated_cost(Some(&catalog), "openai", "gpt-test", None, &mut response); assert_eq!(response.cost_source, Some(CostSource::Estimated)); assert!(response.cost_usd.is_some()); @@ -224,7 +225,7 @@ output_cost_per_mtok = {output_cost_per_mtok} fn apply_estimated_cost_leaves_source_unset_without_estimate() { let mut response = response_with_usage(TokenCounts::default()); - apply_estimated_cost(None, "gpt-test", None, &mut response); + apply_estimated_cost(None, "openai", "gpt-test", None, &mut response); assert_eq!(response.cost_usd, None); assert_eq!(response.cost_source, None); @@ -241,7 +242,7 @@ output_cost_per_mtok = {output_cost_per_mtok} response.cost_usd = Some(0.42); response.cost_source = Some(CostSource::Authoritative); - apply_estimated_cost(Some(&catalog), "gpt-test", None, &mut response); + apply_estimated_cost(Some(&catalog), "openai", "gpt-test", None, &mut response); assert_eq!(response.cost_usd, Some(0.42)); assert_eq!(response.cost_source, Some(CostSource::Authoritative)); diff --git a/lib/crates/fabro-llm/src/model_test.rs b/lib/crates/fabro-llm/src/model_test.rs index 47bca353c..3d8ea8eb0 100644 --- a/lib/crates/fabro-llm/src/model_test.rs +++ b/lib/crates/fabro-llm/src/model_test.rs @@ -54,7 +54,7 @@ pub async fn run_model_test( } async fn run_basic_test(info: &Model, client: Arc) -> ModelTestOutcome { - run_basic_model_probe(&info.id, &info.provider, client).await + run_basic_model_probe(info.id.as_str(), &info.provider, client).await } /// Run the cheap single-prompt model availability probe without requiring a @@ -132,7 +132,7 @@ fn build_deep_test_params(info: &Model, client: Arc) -> Option Model { Model { - id: "test-model".to_string(), + id: "test-model".into(), provider: ProviderId::anthropic(), family: "test".to_string(), display_name: "Test Model".to_string(), diff --git a/lib/crates/fabro-llm/src/providers/anthropic.rs b/lib/crates/fabro-llm/src/providers/anthropic.rs index d65494b34..539b2b0fc 100644 --- a/lib/crates/fabro-llm/src/providers/anthropic.rs +++ b/lib/crates/fabro-llm/src/providers/anthropic.rs @@ -118,7 +118,11 @@ impl Adapter { request, provider_name: &self.provider_name, deployment_id, - model: common::catalog_model(self.catalog.as_deref(), &request.model), + model: common::catalog_model( + self.catalog.as_deref(), + &self.provider_name, + &request.model, + ), params, } } @@ -228,7 +232,11 @@ impl ProviderAdapter for Adapter { self.validate_request(request)?; let resolved = self.resolve_request(request).await; let codec = AnthropicMessages; - let deployment_id = common::api_model_id(self.catalog.as_deref(), &resolved.model); + let deployment_id = common::api_model_id( + self.catalog.as_deref(), + &self.provider_name, + &resolved.model, + ); let ctx = self.codec_ctx(&resolved, &deployment_id, &route.codec_params); let Some(encoded) = codec.encode_count_tokens(&ctx).transpose()? else { @@ -264,7 +272,11 @@ impl ProviderAdapter for Adapter { let resolved = self.resolve_request(request).await; let codec = AnthropicMessages; - let deployment_id = common::api_model_id(self.catalog.as_deref(), &resolved.model); + let deployment_id = common::api_model_id( + self.catalog.as_deref(), + &self.provider_name, + &resolved.model, + ); let ctx = self.codec_ctx(&resolved, &deployment_id, &route.codec_params); let encoded = codec.encode(&ctx, false)?; @@ -281,7 +293,11 @@ impl ProviderAdapter for Adapter { let route = self.route_config(); let resolved = self.resolve_request(request).await; let codec = AnthropicMessages; - let deployment_id = common::api_model_id(self.catalog.as_deref(), &resolved.model); + let deployment_id = common::api_model_id( + self.catalog.as_deref(), + &self.provider_name, + &resolved.model, + ); let ctx = self.codec_ctx(&resolved, &deployment_id, &route.codec_params); let encoded = codec.encode(&ctx, true)?; @@ -307,7 +323,8 @@ impl ProviderAdapter for Adapter { // Always-adaptive models reject manual enabled/disabled thinking // configs at the API, so fail them locally with a clear message // instead. - let model_info = common::catalog_model(self.catalog.as_deref(), &request.model); + let model_info = + common::catalog_model(self.catalog.as_deref(), &self.provider_name, &request.model); if let Some(model) = model_info .filter(|m| m.features.reasoning_effort == ReasoningEffortFeature::AlwaysAdaptive) { diff --git a/lib/crates/fabro-llm/src/providers/bedrock/mod.rs b/lib/crates/fabro-llm/src/providers/bedrock/mod.rs index e2cffd49d..97c15dc02 100644 --- a/lib/crates/fabro-llm/src/providers/bedrock/mod.rs +++ b/lib/crates/fabro-llm/src/providers/bedrock/mod.rs @@ -169,7 +169,11 @@ impl Adapter { request, provider_name: &self.provider_name, deployment_id, - model: common::catalog_model(self.catalog.as_deref(), &request.model), + model: common::catalog_model( + self.catalog.as_deref(), + &self.provider_name, + &request.model, + ), params, } } @@ -239,7 +243,11 @@ impl ProviderAdapter for Adapter { let resolved = self.resolve_request(request).await; let codec = BedrockConverse; - let deployment_id = common::api_model_id(self.catalog.as_deref(), &resolved.model); + let deployment_id = common::api_model_id( + self.catalog.as_deref(), + &self.provider_name, + &resolved.model, + ); let params = CodecParams::default(); let ctx = self.codec_ctx(&resolved, &deployment_id, ¶ms); @@ -253,7 +261,11 @@ impl ProviderAdapter for Adapter { let resolved = self.resolve_request(request).await; let codec = BedrockConverse; - let deployment_id = common::api_model_id(self.catalog.as_deref(), &resolved.model); + let deployment_id = common::api_model_id( + self.catalog.as_deref(), + &self.provider_name, + &resolved.model, + ); let params = CodecParams::default(); let ctx = self.codec_ctx(&resolved, &deployment_id, ¶ms); diff --git a/lib/crates/fabro-llm/src/providers/common.rs b/lib/crates/fabro-llm/src/providers/common.rs index 8871ff3f7..f85397935 100644 --- a/lib/crates/fabro-llm/src/providers/common.rs +++ b/lib/crates/fabro-llm/src/providers/common.rs @@ -1,18 +1,22 @@ use base64::Engine; use base64::engine::general_purpose::STANDARD as BASE64_STANDARD; -use fabro_model::{Catalog, Model}; +use fabro_model::{Catalog, Model, ProviderId}; use fabro_static::EnvVars; use tokio::fs; #[must_use] -pub fn catalog_model<'a>(catalog: Option<&'a Catalog>, model: &str) -> Option<&'a Model> { - catalog.and_then(|catalog| catalog.get(model)) +pub fn catalog_model<'a>( + catalog: Option<&'a Catalog>, + provider: &str, + model: &str, +) -> Option<&'a Model> { + catalog.and_then(|catalog| catalog.get_on_provider(&ProviderId::new(provider), model)) } #[must_use] -pub fn api_model_id(catalog: Option<&Catalog>, model: &str) -> String { +pub fn api_model_id(catalog: Option<&Catalog>, provider: &str, model: &str) -> String { catalog - .and_then(|catalog| catalog.model_settings(model)) + .and_then(|catalog| catalog.model_settings_on_provider(&ProviderId::new(provider), model)) .map_or_else(|| model.to_string(), |settings| settings.api_id.clone()) } diff --git a/lib/crates/fabro-llm/src/providers/gemini.rs b/lib/crates/fabro-llm/src/providers/gemini.rs index 2b1f9ee4b..a9e2fab0f 100644 --- a/lib/crates/fabro-llm/src/providers/gemini.rs +++ b/lib/crates/fabro-llm/src/providers/gemini.rs @@ -102,7 +102,11 @@ impl Adapter { request, provider_name: &self.provider_name, deployment_id, - model: common::catalog_model(self.catalog.as_deref(), &request.model), + model: common::catalog_model( + self.catalog.as_deref(), + &self.provider_name, + &request.model, + ), params, } } @@ -147,7 +151,11 @@ impl ProviderAdapter for Adapter { let resolved = self.resolve_request(request).await; let codec = GeminiGenerate; - let deployment_id = common::api_model_id(self.catalog.as_deref(), &resolved.model); + let deployment_id = common::api_model_id( + self.catalog.as_deref(), + &self.provider_name, + &resolved.model, + ); let params = CodecParams::default(); let ctx = self.codec_ctx(&resolved, &deployment_id, ¶ms); @@ -177,7 +185,11 @@ impl ProviderAdapter for Adapter { let resolved = self.resolve_request(request).await; let codec = GeminiGenerate; - let deployment_id = common::api_model_id(self.catalog.as_deref(), &resolved.model); + let deployment_id = common::api_model_id( + self.catalog.as_deref(), + &self.provider_name, + &resolved.model, + ); let params = CodecParams::default(); let ctx = self.codec_ctx(&resolved, &deployment_id, ¶ms); @@ -194,7 +206,11 @@ impl ProviderAdapter for Adapter { let resolved = self.resolve_request(request).await; let codec = GeminiGenerate; - let deployment_id = common::api_model_id(self.catalog.as_deref(), &resolved.model); + let deployment_id = common::api_model_id( + self.catalog.as_deref(), + &self.provider_name, + &resolved.model, + ); let params = CodecParams::default(); let ctx = self.codec_ctx(&resolved, &deployment_id, ¶ms); diff --git a/lib/crates/fabro-llm/src/providers/openai.rs b/lib/crates/fabro-llm/src/providers/openai.rs index ad25d01a9..185af1714 100644 --- a/lib/crates/fabro-llm/src/providers/openai.rs +++ b/lib/crates/fabro-llm/src/providers/openai.rs @@ -131,7 +131,11 @@ impl Adapter { request, provider_name: &self.provider_name, deployment_id, - model: common::catalog_model(self.catalog.as_deref(), &request.model), + model: common::catalog_model( + self.catalog.as_deref(), + &self.provider_name, + &request.model, + ), params, } } @@ -214,7 +218,11 @@ impl ProviderAdapter for Adapter { let resolved = self.resolve_request(request).await; let codec = OpenAiResponses; - let deployment_id = common::api_model_id(self.catalog.as_deref(), &resolved.model); + let deployment_id = common::api_model_id( + self.catalog.as_deref(), + &self.provider_name, + &resolved.model, + ); let params = self.codec_params(); let ctx = self.codec_ctx(&resolved, &deployment_id, ¶ms); @@ -250,7 +258,11 @@ impl ProviderAdapter for Adapter { let resolved = self.resolve_request(request).await; let codec = OpenAiResponses; - let deployment_id = common::api_model_id(self.catalog.as_deref(), &resolved.model); + let deployment_id = common::api_model_id( + self.catalog.as_deref(), + &self.provider_name, + &resolved.model, + ); let params = self.codec_params(); let ctx = self.codec_ctx(&resolved, &deployment_id, ¶ms); @@ -267,7 +279,11 @@ impl ProviderAdapter for Adapter { let resolved = self.resolve_request(request).await; let codec = OpenAiResponses; - let deployment_id = common::api_model_id(self.catalog.as_deref(), &resolved.model); + let deployment_id = common::api_model_id( + self.catalog.as_deref(), + &self.provider_name, + &resolved.model, + ); let params = self.codec_params(); let ctx = self.codec_ctx(&resolved, &deployment_id, ¶ms); diff --git a/lib/crates/fabro-llm/src/providers/openai_compatible.rs b/lib/crates/fabro-llm/src/providers/openai_compatible.rs index 9169287b7..3feb0b577 100644 --- a/lib/crates/fabro-llm/src/providers/openai_compatible.rs +++ b/lib/crates/fabro-llm/src/providers/openai_compatible.rs @@ -88,7 +88,7 @@ impl Adapter { /// Resolve the wire model id (catalog `api_id`, falling back to the /// requested model). fn deployment_id(&self, request: &Request) -> String { - common::api_model_id(self.catalog.as_deref(), &request.model) + common::api_model_id(self.catalog.as_deref(), &self.provider_name, &request.model) } /// Build the borrowed codec context. `deployment_id` and `params` are @@ -103,7 +103,11 @@ impl Adapter { request, provider_name: &self.provider_name, deployment_id, - model: common::catalog_model(self.catalog.as_deref(), &request.model), + model: common::catalog_model( + self.catalog.as_deref(), + &self.provider_name, + &request.model, + ), params, } } diff --git a/lib/crates/fabro-llm/tests/integration.rs b/lib/crates/fabro-llm/tests/integration.rs index a7a421afe..9eb558fd5 100644 --- a/lib/crates/fabro-llm/tests/integration.rs +++ b/lib/crates/fabro-llm/tests/integration.rs @@ -326,7 +326,7 @@ async fn poolside_laguna_xs_deep_tool_round_trip() { .expect("Poolside client should build from the catalog"), ); let model = catalog - .get("laguna-xs-2.1") + .get_on_provider(&ProviderId::new("poolside"), "laguna-xs-2.1") .expect("direct Poolside Laguna XS should be present"); let outcome = run_model_test(model, ModelTestMode::Deep, client).await; @@ -572,7 +572,7 @@ async fn openrouter_kimi_k3_deep_tool_round_trip() { .expect("OpenRouter client should build from the catalog"), ); let model = catalog - .get("moonshotai/kimi-k3") + .get_on_provider(&ProviderId::new("openrouter"), "kimi-k3") .expect("OpenRouter Kimi K3 should be present"); let outcome = run_model_test(model, ModelTestMode::Deep, client).await; diff --git a/lib/crates/fabro-model/src/billing.rs b/lib/crates/fabro-model/src/billing.rs index d75992d7a..0f571bfb3 100644 --- a/lib/crates/fabro-model/src/billing.rs +++ b/lib/crates/fabro-model/src/billing.rs @@ -2,7 +2,7 @@ use serde::{Deserialize, Serialize}; use strum::{Display, EnumString, IntoStaticStr}; use crate::catalog::{BillingPolicy, Catalog, CatalogModelSettings}; -use crate::{Model, ModelCosts, ProviderId}; +use crate::{Model, ModelCosts, ModelId, ProviderId}; const TOKENS_PER_MTOK: i128 = 1_000_000; const ANTHROPIC_CACHE_WRITE_5M_NUMERATOR: i64 = 5; @@ -138,7 +138,7 @@ pub enum CostSource { #[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] pub struct ModelRef { pub provider: ProviderId, - pub model_id: String, + pub model_id: ModelId, #[serde(default, skip_serializing_if = "Option::is_none")] pub speed: Option, } @@ -321,7 +321,7 @@ impl BilledModelUsage { #[must_use] pub fn model_id(&self) -> &str { - &self.input.usage.model.model_id + self.input.usage.model.model_id.as_str() } #[must_use] @@ -435,13 +435,9 @@ fn anthropic_billing_facts(tokens: &TokenCounts) -> AnthropicBillingFacts { impl Catalog { #[must_use] pub fn pricing_for(&self, model_ref: &ModelRef) -> Option { - let model = self.get(&model_ref.model_id)?; + let model = self.offering(&model_ref.provider, &model_ref.model_id)?; let provider = self.provider(&model_ref.provider)?; - if model.provider != provider.id { - return None; - } - - let settings = self.model_settings(&model.id)?; + let settings = self.settings_for(model)?; let costs = costs_for_speed(model, settings, model_ref.speed)?; pricing_for_model_costs( model, @@ -459,7 +455,7 @@ impl Catalog { tokens: &TokenCounts, ) -> Option { let policy = - self.effective_billing_policy(&model_ref.provider, Some(&model_ref.model_id))?; + self.effective_billing_policy(&model_ref.provider, Some(model_ref.model_id.as_str()))?; ModelBillingFacts::for_policy(policy, tokens) } @@ -713,7 +709,7 @@ mod tests { usage: ModelUsage { model: ModelRef { provider: ProviderId::openai(), - model_id: "gpt-5.4".to_string(), + model_id: ModelId::new("gpt-5.4"), speed: None, }, tokens: TokenCounts { @@ -785,12 +781,12 @@ cache_input_cost_per_mtok = 0.3 }; let claude = ModelRef { provider: ProviderId::new("aggregator"), - model_id: "claude-via-aggregator".to_string(), + model_id: ModelId::new("claude-via-aggregator"), speed: None, }; let plain = ModelRef { provider: ProviderId::new("aggregator"), - model_id: "plain-model".to_string(), + model_id: ModelId::new("plain-model"), speed: None, }; @@ -919,7 +915,7 @@ cache_input_cost_per_mtok = 0.3 let pricing = ModelPricing { model: ModelRef { provider: ProviderId::openai(), - model_id: "gpt-5.4".to_string(), + model_id: ModelId::new("gpt-5.4"), speed: None, }, policy: ModelPricingPolicy::OpenAi(OpenAiModelPricing { @@ -956,7 +952,7 @@ cache_input_cost_per_mtok = 0.3 let pricing = Catalog::builtin() .pricing_for(&ModelRef { provider: ProviderId::anthropic(), - model_id: "claude-opus-4-6".to_string(), + model_id: ModelId::new("claude-opus-4-6"), speed: Some(Speed::Fast), }) .unwrap(); @@ -980,7 +976,7 @@ cache_input_cost_per_mtok = 0.3 let pricing = Catalog::builtin() .pricing_for(&ModelRef { provider: ProviderId::anthropic(), - model_id: "claude-opus-4-6".to_string(), + model_id: ModelId::new("claude-opus-4-6"), speed: Some(Speed::Standard), }) .unwrap(); @@ -1033,7 +1029,7 @@ cache_input_cost_per_mtok = 0.25 let pricing = catalog .pricing_for(&ModelRef { provider: ProviderId::new("test_anthropic"), - model_id: "test-opus".to_string(), + model_id: ModelId::new("test-opus"), speed: Some(Speed::Fast), }) .unwrap(); @@ -1081,7 +1077,7 @@ cache_input_cost_per_mtok = 0.1 let pricing = catalog .pricing_for(&ModelRef { provider: ProviderId::new("proxy"), - model_id: "proxy-model".to_string(), + model_id: ModelId::new("proxy-model"), speed: None, }) .unwrap(); @@ -1131,7 +1127,7 @@ output_cost_per_mtok = 2.0 catalog .pricing_for(&ModelRef { provider: ProviderId::new("proxy"), - model_id: "canonical-model".to_string(), + model_id: ModelId::new("canonical-model"), speed: None, }) .is_some() @@ -1140,7 +1136,7 @@ output_cost_per_mtok = 2.0 catalog .pricing_for(&ModelRef { provider: ProviderId::new("proxy"), - model_id: "wire-model".to_string(), + model_id: ModelId::new("wire-model"), speed: None, }) .is_none() @@ -1153,7 +1149,7 @@ output_cost_per_mtok = 2.0 Catalog::builtin() .pricing_for(&ModelRef { provider: ProviderId::new("unknown"), - model_id: "claude-opus-4-6".to_string(), + model_id: ModelId::new("claude-opus-4-6"), speed: None, }) .is_none() @@ -1162,7 +1158,7 @@ output_cost_per_mtok = 2.0 Catalog::builtin() .pricing_for(&ModelRef { provider: ProviderId::anthropic(), - model_id: "unknown".to_string(), + model_id: ModelId::new("unknown"), speed: None, }) .is_none() @@ -1171,7 +1167,7 @@ output_cost_per_mtok = 2.0 Catalog::builtin() .pricing_for(&ModelRef { provider: ProviderId::openai(), - model_id: "gpt-5.4".to_string(), + model_id: ModelId::new("gpt-5.4"), speed: Some(Speed::Fast), }) .is_none() @@ -1183,7 +1179,7 @@ output_cost_per_mtok = 2.0 let pricing = ModelPricing { model: ModelRef { provider: ProviderId::anthropic(), - model_id: "claude-opus-4-6".to_string(), + model_id: ModelId::new("claude-opus-4-6"), speed: Some(Speed::Fast), }, policy: ModelPricingPolicy::Anthropic(AnthropicModelPricing { @@ -1229,7 +1225,7 @@ output_cost_per_mtok = 2.0 let pricing = ModelPricing { model: ModelRef { provider: ProviderId::gemini(), - model_id: "gemini-3.1-pro-preview".to_string(), + model_id: ModelId::new("gemini-3.1-pro-preview"), speed: None, }, policy: ModelPricingPolicy::Gemini(GeminiModelPricing { diff --git a/lib/crates/fabro-model/src/catalog.rs b/lib/crates/fabro-model/src/catalog.rs index ce586679e..ee362d414 100644 --- a/lib/crates/fabro-model/src/catalog.rs +++ b/lib/crates/fabro-model/src/catalog.rs @@ -1,5 +1,5 @@ use std::borrow::Cow; -use std::collections::{BTreeMap, HashMap, HashSet}; +use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; use std::str::FromStr; use std::sync::LazyLock; @@ -12,7 +12,7 @@ use tracing::warn; use crate::Speed; use crate::adapter::{AdapterKind, AgentProfileKind}; use crate::codec::CodecKind; -use crate::ids::ProviderId; +use crate::ids::{ModelId, ProviderId}; use crate::provider::Provider; use crate::reasoning::ReasoningEffort; use crate::types::{Model, ModelCosts, ModelFeatures, ModelLimits, ReasoningEffortFeature}; @@ -32,6 +32,8 @@ struct BuiltinCatalogToml; pub struct LlmCatalogSettings { #[serde(default)] pub providers: HashMap, + /// Legacy `[models.""]` input. Canonical settings place model rows + /// under their provider; this map is normalized before layers merge. #[serde(default)] pub models: HashMap, } @@ -68,11 +70,16 @@ pub struct ProviderCatalogSettings { pub enabled: Option, #[serde(default)] pub aliases: Option>, + /// Model declarations keyed by Fabro's canonical model slug. + #[serde(default)] + pub models: HashMap, } #[derive(Debug, Clone, Default, PartialEq, Deserialize)] #[serde(deny_unknown_fields)] pub struct ModelCatalogSettings { + /// Provider used only by the temporary legacy top-level `[models]` + /// compatibility shape. Canonical provider-scoped rows leave this unset. #[serde(default)] pub provider: Option, #[serde(default)] @@ -528,11 +535,46 @@ pub enum CatalogBuildError { model: String, provider: ProviderId, }, - #[error("model identifier '{identifier}' is declared by both '{first}' and '{second}'")] - DuplicateModelIdentifier { + #[error( + "provider '{provider}' model selector '{selector}' is declared by both '{first}' and '{second}'" + )] + DuplicateProviderModelSelector { + provider: ProviderId, + selector: String, + first: ModelId, + second: ModelId, + }, + #[error( + "provider '{provider}' model '{model}' is defined through both provider-scoped and legacy top-level syntax" + )] + DuplicateProviderModelDefinition { + provider: ProviderId, + model: ModelId, + }, + #[error("provider-scoped model '{provider}/{model}' must not declare a provider field")] + ScopedModelDeclaresProvider { + provider: ProviderId, + model: ModelId, + }, + #[error("legacy model row '{model}' omits provider and does not match a unique known offering")] + LegacyModelProviderUnknown { model: String }, + #[error( + "legacy model row '{model}' omits provider and matches multiple providers: {providers:?}" + )] + LegacyModelProviderAmbiguous { + model: String, + providers: Vec, + }, + #[error("model identifier '{identifier}' has been retired; use '{provider}/{model}' instead")] + RetiredModelIdentifier { identifier: String, - first: String, - second: String, + provider: ProviderId, + model: ModelId, + }, + #[error("provider '{provider}' model '{model}' has an empty api_id")] + EmptyModelApiId { + provider: ProviderId, + model: ModelId, }, #[error("provider '{provider}' has multiple default models: {models:?}")] MultipleProviderDefaults { @@ -574,17 +616,49 @@ pub enum CatalogBuildError { UndeclaredSpeedCost { model: String, speed: Speed }, } +/// Failure to select one concrete provider/model offering. +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +pub enum ModelSelectionError { + #[error("unknown model provider '{provider}'")] + UnknownProvider { provider: ProviderId }, + #[error("model provider '{provider}' is unavailable")] + ProviderUnavailable { provider: ProviderId }, + #[error("unknown model selector '{selector}'")] + UnknownSelector { selector: String }, + #[error("model selector '{selector}' is unknown on provider '{provider}'")] + UnknownSelectorOnProvider { + selector: String, + provider: ProviderId, + }, + #[error( + "model selector '{selector}' is known but has no offering on an eligible provider; available providers: {providers:?}" + )] + NoEligibleOffering { + selector: String, + providers: Vec, + }, + #[error("model identifier '{identifier}' has been retired; use '{provider}/{model}' instead")] + RetiredModelIdentifier { + identifier: String, + provider: ProviderId, + model: ModelId, + }, +} + /// Typed model catalog backed by a `Vec`. /// /// Use [`Catalog::builtin()`] for the embedded settings-backed catalog. #[derive(Debug)] pub struct Catalog { - models: Vec, - providers: Vec, - model_settings: HashMap, - model_index: HashMap, - provider_aliases: HashMap, - provider_index: HashMap, + models: Vec, + providers: Vec, + model_settings: HashMap<(ProviderId, ModelId), CatalogModelSettings>, + offering_index: HashMap<(ProviderId, ModelId), usize>, + provider_selector_index: HashMap<(ProviderId, String), usize>, + canonical_candidates: HashMap>, + alias_candidates: HashMap>, + provider_aliases: HashMap, + provider_index: HashMap, } impl Catalog { @@ -596,7 +670,8 @@ impl Catalog { } pub fn from_settings(settings: &LlmCatalogSettings) -> Result { - let mut providers = build_providers(settings)?; + let settings = normalize_catalog_settings(settings.clone(), None)?; + let mut providers = build_providers(&settings)?; providers.sort_by(provider_order); let mut provider_index = HashMap::new(); @@ -617,63 +692,84 @@ impl Catalog { .collect(); let mut models_with_settings = Vec::new(); - let mut model_identifiers = BTreeMap::::new(); - let mut defaults_by_provider = HashMap::>::new(); - let mut small_defaults_by_provider = HashMap::>::new(); + let mut model_identifiers = HashMap::>::new(); + let mut defaults_by_provider = HashMap::>::new(); + let mut small_defaults_by_provider = HashMap::>::new(); - let mut model_ids = settings.models.keys().cloned().collect::>(); - model_ids.sort_unstable(); - for model_id in model_ids { - let model_settings = settings - .models - .get(&model_id) - .expect("model ID came from settings map keys"); - if model_settings.enabled == Some(false) { + let mut provider_ids = settings.providers.keys().cloned().collect::>(); + provider_ids.sort_unstable(); + for provider_id in provider_ids { + if !known_providers.contains(provider_id.as_str()) + || !enabled_providers.contains(provider_id.as_str()) + { continue; } - - let provider_id = - required_model_string(&model_id, model_settings.provider.as_ref(), "provider")?; - if !known_providers.contains(provider_id.as_str()) { - return Err(CatalogBuildError::UnknownModelProvider { - model: model_id, - provider: ProviderId::from(provider_id), - }); - } - if !enabled_providers.contains(provider_id.as_str()) { - continue; - } - let provider = provider_by_id .get(provider_id.as_str()) .expect("enabled provider ID should have provider metadata"); - let (model, resolved_settings) = build_model(&model_id, model_settings, provider)?; + let provider_settings = settings + .providers + .get(&provider_id) + .expect("provider ID came from settings map keys"); + let identifiers = model_identifiers.entry(provider.id.clone()).or_default(); + let mut model_ids = provider_settings.models.keys().cloned().collect::>(); + model_ids.sort_unstable(); + for model_id in model_ids { + let model_settings = provider_settings + .models + .get(&model_id) + .expect("model ID came from provider model map keys"); + if model_settings.enabled == Some(false) { + continue; + } - register_model_identifier(&mut model_identifiers, model.id.clone(), model.id.clone())?; - for alias in &model.aliases { - register_model_identifier(&mut model_identifiers, alias.clone(), model.id.clone())?; - } + if let Some((replacement_provider, replacement_model)) = + retired_model_address(&model_id) + { + return Err(CatalogBuildError::RetiredModelIdentifier { + identifier: model_id, + provider: replacement_provider, + model: replacement_model, + }); + } - if model.default { - defaults_by_provider - .entry(model.provider.clone()) - .or_default() - .push(model.id.clone()); + let (model, resolved_settings) = build_model(&model_id, model_settings, provider)?; + register_model_identifier( + identifiers, + model.id.as_str().to_string(), + model.id.clone(), + &model.provider, + )?; + for alias in &model.aliases { + register_model_identifier( + identifiers, + alias.clone(), + model.id.clone(), + &model.provider, + )?; + } + + if model.default { + defaults_by_provider + .entry(model.provider.clone()) + .or_default() + .push(model.id.clone()); + } + if model.small_default { + small_defaults_by_provider + .entry(model.provider.clone()) + .or_default() + .push(model.id.clone()); + } + models_with_settings.push((model, resolved_settings)); } - if model.small_default { - small_defaults_by_provider - .entry(model.provider.clone()) - .or_default() - .push(model.id.clone()); - } - models_with_settings.push((model, resolved_settings)); } for (provider, defaults) in defaults_by_provider { if defaults.len() > 1 { return Err(CatalogBuildError::MultipleProviderDefaults { provider, - models: defaults, + models: defaults.into_iter().map(ModelId::into_inner).collect(), }); } } @@ -681,7 +777,10 @@ impl Catalog { if small_defaults.len() > 1 { return Err(CatalogBuildError::MultipleProviderSmallDefaults { provider, - models: small_defaults, + models: small_defaults + .into_iter() + .map(ModelId::into_inner) + .collect(), }); } } @@ -689,21 +788,29 @@ impl Catalog { return Err(CatalogBuildError::NoDefaultModel); } - models_with_settings.sort_by(|(left, _), (right, _)| model_order(left, right)); + models_with_settings.sort_by(|(left, _), (right, _)| { + provider_index[&left.provider] + .cmp(&provider_index[&right.provider]) + .then_with(|| left.id.cmp(&right.id)) + }); warn_multiple_probe_models(&models_with_settings); - let mut model_settings_by_id = HashMap::new(); + let mut model_settings_by_offering = HashMap::new(); let mut models = Vec::new(); for (model, settings) in models_with_settings { - model_settings_by_id.insert(model.id.clone(), settings); + model_settings_by_offering.insert((model.provider.clone(), model.id.clone()), settings); models.push(model); } - let model_index = build_model_index(&models); + let (offering_index, provider_selector_index, canonical_candidates, alias_candidates) = + build_model_indexes(&models); Ok(Self { models, providers, - model_settings: model_settings_by_id, - model_index, + model_settings: model_settings_by_offering, + offering_index, + provider_selector_index, + canonical_candidates, + alias_candidates, provider_aliases, provider_index, }) @@ -712,8 +819,9 @@ impl Catalog { pub fn from_builtin_with_overrides( overrides: &LlmCatalogSettings, ) -> Result { - let builtins = Self::builtin_settings()?; - let settings = merge_catalog_settings(overrides.clone(), builtins); + let builtins = normalize_catalog_settings(Self::builtin_settings()?, None)?; + let overrides = normalize_catalog_settings(overrides.clone(), Some(&builtins))?; + let settings = merge_catalog_settings(overrides, builtins); Self::from_settings(&settings) } @@ -753,21 +861,204 @@ impl Catalog { layer.models.extend(fragment.models); } - Ok(layer) + normalize_catalog_settings(layer, None) } fn from_builtin_toml() -> Result { Self::from_settings(&Self::builtin_settings()?) } - /// Look up a model by ID or alias. + /// Test-only shorthand for selecting from every enabled catalog provider. + /// + /// Production callers must supply an explicit ready-provider snapshot to + /// [`Catalog::select`] or use a provider-scoped lookup. + #[cfg(test)] #[must_use] - pub fn get(&self, id: &str) -> Option<&Model> { - self.model_index - .get(id) + pub(crate) fn get(&self, selector: &str) -> Option<&Model> { + self.candidate_indices(selector) + .and_then(|indices| indices.first()) .and_then(|idx| self.models.get(*idx)) } + /// Look up a selector on exactly one provider, without considering + /// provider availability. + #[must_use] + pub fn get_on_provider(&self, provider: &ProviderId, selector: &str) -> Option<&Model> { + let provider = self.provider(provider)?; + self.provider_selector_index + .get(&(provider.id.clone(), selector.to_string())) + .and_then(|idx| self.models.get(*idx)) + } + + /// Look up a canonical offering by its composite identity. + #[must_use] + pub fn offering(&self, provider: &ProviderId, model: &ModelId) -> Option<&Model> { + let provider = self.provider(provider)?; + self.offering_index + .get(&(provider.id.clone(), model.clone())) + .and_then(|idx| self.models.get(*idx)) + } + + /// Resolve a selector on exactly one provider. + pub fn resolve_on_provider( + &self, + provider: &ProviderId, + selector: &str, + ) -> Result<&Model, ModelSelectionError> { + if let Some((replacement_provider, replacement_model)) = retired_model_address(selector) { + return Err(ModelSelectionError::RetiredModelIdentifier { + identifier: selector.to_string(), + provider: replacement_provider, + model: replacement_model, + }); + } + let provider = + self.provider(provider) + .ok_or_else(|| ModelSelectionError::UnknownProvider { + provider: provider.clone(), + })?; + if let Some(model) = self.get_on_provider(&provider.id, selector) { + return Ok(model); + } + Err(ModelSelectionError::UnknownSelectorOnProvider { + selector: selector.to_string(), + provider: provider.id.clone(), + }) + } + + /// Select one concrete offering for a selector and ready-provider + /// snapshot. + /// + /// An explicit provider is a pin. Unqualified selection checks canonical + /// IDs before aliases and uses the catalog's provider priority ordering. + pub fn select<'a>( + &'a self, + selector: &str, + explicit_provider: Option<&ProviderId>, + eligible_providers: &HashSet, + ) -> Result<&'a Model, ModelSelectionError> { + if let Some((replacement_provider, replacement_model)) = retired_model_address(selector) { + return Err(ModelSelectionError::RetiredModelIdentifier { + identifier: selector.to_string(), + provider: replacement_provider, + model: replacement_model, + }); + } + let eligible = eligible_providers + .iter() + .filter_map(|provider| self.provider(provider).map(|provider| provider.id.clone())) + .collect::>(); + + if let Some(explicit_provider) = explicit_provider { + let provider = self.provider(explicit_provider).ok_or_else(|| { + ModelSelectionError::UnknownProvider { + provider: explicit_provider.clone(), + } + })?; + if !eligible.contains(&provider.id) { + return Err(ModelSelectionError::ProviderUnavailable { + provider: provider.id.clone(), + }); + } + return self.resolve_on_provider(&provider.id, selector); + } + + let canonical = self.canonical_candidates.get(&ModelId::new(selector)); + if let Some(indices) = canonical { + if let Some(model) = indices + .iter() + .filter_map(|idx| self.models.get(*idx)) + .find(|model| eligible.contains(&model.provider)) + { + return Ok(model); + } + } + + let aliases = self.alias_candidates.get(selector); + if let Some(indices) = aliases { + if let Some(model) = indices + .iter() + .filter_map(|idx| self.models.get(*idx)) + .find(|model| eligible.contains(&model.provider)) + { + return Ok(model); + } + } + + let mut providers = Vec::new(); + for index in canonical + .into_iter() + .flatten() + .chain(aliases.into_iter().flatten()) + { + let Some(model) = self.models.get(*index) else { + continue; + }; + if !providers.contains(&model.provider) { + providers.push(model.provider.clone()); + } + } + if !providers.is_empty() { + return Err(ModelSelectionError::NoEligibleOffering { + selector: selector.to_string(), + providers, + }); + } + + Err(ModelSelectionError::UnknownSelector { + selector: selector.to_string(), + }) + } + + #[must_use] + pub fn all_provider_ids(&self) -> HashSet { + self.providers + .iter() + .map(|provider| provider.id.clone()) + .collect() + } + + /// Select the highest-priority default model on an eligible provider. + pub fn select_default( + &self, + eligible_providers: &HashSet, + ) -> Result<&Model, ModelSelectionError> { + let eligible = eligible_providers + .iter() + .filter_map(|provider| self.provider(provider).map(|provider| provider.id.clone())) + .collect::>(); + if let Some(model) = self + .models + .iter() + .find(|model| model.default && eligible.contains(&model.provider)) + { + return Ok(model); + } + let mut providers = self + .models + .iter() + .filter(|model| model.default) + .map(|model| model.provider.clone()) + .collect::>(); + providers.sort(); + providers.dedup(); + Err(ModelSelectionError::NoEligibleOffering { + selector: "".to_string(), + providers, + }) + } + + #[must_use] + pub fn is_model_selector(&self, selector: &str) -> bool { + self.candidate_indices(selector).is_some() + } + + fn candidate_indices(&self, selector: &str) -> Option<&Vec> { + self.canonical_candidates + .get(&ModelId::new(selector)) + .or_else(|| self.alias_candidates.get(selector)) + } + #[must_use] pub fn providers(&self) -> &[CatalogProvider] { &self.providers @@ -786,7 +1077,7 @@ impl Catalog { let stats = stats_by_provider.entry(model.provider.clone()).or_default(); stats.model_count = stats.model_count.saturating_add(1); if model.default { - stats.default_model = Some(model.id.clone()); + stats.default_model = Some(model.id.to_string()); } } @@ -818,9 +1109,32 @@ impl Catalog { } #[must_use] - pub fn model_settings(&self, id: &str) -> Option<&CatalogModelSettings> { - let model = self.get(id)?; - self.model_settings.get(&model.id) + pub fn settings_for(&self, model: &Model) -> Option<&CatalogModelSettings> { + self.model_settings + .get(&(model.provider.clone(), model.id.clone())) + } + + /// Test-only shorthand for settings on the highest-priority enabled + /// offering. Production callers must retain the resolved offering and use + /// [`Catalog::settings_for`]. + #[cfg(test)] + #[must_use] + pub(crate) fn model_settings( + &self, + selector: impl AsRef, + ) -> Option<&CatalogModelSettings> { + self.get(selector.as_ref()) + .and_then(|model| self.settings_for(model)) + } + + #[must_use] + pub fn model_settings_on_provider( + &self, + provider: &ProviderId, + selector: &str, + ) -> Option<&CatalogModelSettings> { + let model = self.get_on_provider(provider, selector)?; + self.settings_for(model) } #[must_use] @@ -831,9 +1145,8 @@ impl Catalog { ) -> Option { let provider = self.provider(provider_id)?; let model_profile = model_id_or_alias - .and_then(|model_id| self.get(model_id)) - .filter(|model| model.provider == provider.id) - .and_then(|model| self.model_settings.get(&model.id)) + .and_then(|model_id| self.get_on_provider(&provider.id, model_id)) + .and_then(|model| self.settings_for(model)) .map(|settings| settings.agent_profile); Some(model_profile.unwrap_or(provider.agent_profile)) } @@ -849,9 +1162,8 @@ impl Catalog { ) -> Option { let provider = self.provider(provider_id)?; let model_codec = model_id_or_alias - .and_then(|model_id| self.get(model_id)) - .filter(|model| model.provider == provider.id) - .and_then(|model| self.model_settings.get(&model.id)) + .and_then(|model_id| self.get_on_provider(&provider.id, model_id)) + .and_then(|model| self.settings_for(model)) .map(|settings| settings.codec); Some(model_codec.unwrap_or(provider.codec)) } @@ -867,9 +1179,8 @@ impl Catalog { ) -> Option { let provider = self.provider(provider_id)?; let model_policy = model_id_or_alias - .and_then(|model_id| self.get(model_id)) - .filter(|model| model.provider == provider.id) - .and_then(|model| self.model_settings.get(&model.id)) + .and_then(|model_id| self.get_on_provider(&provider.id, model_id)) + .and_then(|model| self.settings_for(model)) .map(|settings| settings.billing_policy); Some(model_policy.unwrap_or(provider.billing_policy)) } @@ -993,8 +1304,7 @@ impl Catalog { if let Some(model) = self.models.iter().find(|model| { &model.provider == provider_id && self - .model_settings - .get(&model.id) + .settings_for(model) .is_some_and(|settings| settings.probe) }) { return Some(model); @@ -1043,7 +1353,7 @@ impl Catalog { model: &str, fallbacks: &HashMap>, ) -> Vec { - let Some(reference) = self.get(model) else { + let Some(reference) = self.get_on_provider(primary, model) else { return Vec::new(); }; @@ -1057,22 +1367,359 @@ impl Catalog { let provider = ProviderId::from(provider_str.clone()); self.closest(&provider, reference).map(|m| FallbackTarget { provider: provider_str.clone(), - model: m.id.clone(), + model: m.id.to_string(), }) }) .collect() } } -fn build_model_index(models: &[Model]) -> HashMap { - let mut index = HashMap::new(); +type ModelIndexes = ( + HashMap<(ProviderId, ModelId), usize>, + HashMap<(ProviderId, String), usize>, + HashMap>, + HashMap>, +); + +fn build_model_indexes(models: &[Model]) -> ModelIndexes { + let mut offering_index = HashMap::new(); + let mut provider_selector_index = HashMap::new(); + let mut canonical_candidates = HashMap::>::new(); + let mut alias_candidates = HashMap::>::new(); for (idx, model) in models.iter().enumerate() { - index.insert(model.id.clone(), idx); + offering_index.insert((model.provider.clone(), model.id.clone()), idx); + provider_selector_index + .insert((model.provider.clone(), model.id.as_str().to_string()), idx); + canonical_candidates + .entry(model.id.clone()) + .or_default() + .push(idx); for alias in &model.aliases { - index.insert(alias.clone(), idx); + provider_selector_index.insert((model.provider.clone(), alias.clone()), idx); + alias_candidates.entry(alias.clone()).or_default().push(idx); } } - index + ( + offering_index, + provider_selector_index, + canonical_candidates, + alias_candidates, + ) +} + +fn normalize_catalog_settings( + mut settings: LlmCatalogSettings, + known: Option<&LlmCatalogSettings>, +) -> Result { + for (provider, settings) in &settings.providers { + for (model, settings) in &settings.models { + if settings.provider.is_some() { + return Err(CatalogBuildError::ScopedModelDeclaresProvider { + provider: ProviderId::new(provider.clone()), + model: ModelId::new(model.clone()), + }); + } + } + } + + let legacy_models = std::mem::take(&mut settings.models); + for (legacy_id, mut model_settings) in legacy_models { + if let Some((provider, model)) = retired_model_address(&legacy_id) { + return Err(CatalogBuildError::RetiredModelIdentifier { + identifier: legacy_id, + provider, + model, + }); + } + + let explicit_provider = model_settings.provider.take(); + let (provider, model_id) = if let Some(provider) = explicit_provider { + let provider = canonical_settings_provider(&provider, &settings, known) + .unwrap_or_else(|| ProviderId::new(provider)); + let model_id = canonical_settings_model(&provider, &legacy_id, &settings, known) + .unwrap_or_else(|| ModelId::new(legacy_id.clone())); + (provider, model_id) + } else { + let candidates = settings_model_candidates(&legacy_id, &settings, known); + match candidates.as_slice() { + [(provider, model)] => (provider.clone(), model.clone()), + [] => { + return Err(CatalogBuildError::LegacyModelProviderUnknown { model: legacy_id }); + } + _ => { + return Err(CatalogBuildError::LegacyModelProviderAmbiguous { + model: legacy_id, + providers: candidates + .into_iter() + .map(|(provider, _)| provider) + .collect(), + }); + } + } + }; + + if !settings.providers.contains_key(provider.as_str()) + && !known.is_some_and(|known| known.providers.contains_key(provider.as_str())) + { + return Err(CatalogBuildError::UnknownModelProvider { + model: legacy_id, + provider, + }); + } + + let provider_settings = settings.providers.entry(provider.to_string()).or_default(); + if provider_settings.models.contains_key(model_id.as_str()) { + return Err(CatalogBuildError::DuplicateProviderModelDefinition { + provider, + model: model_id, + }); + } + provider_settings + .models + .insert(model_id.into_inner(), model_settings); + } + Ok(settings) +} + +fn canonical_settings_provider( + selector: &str, + settings: &LlmCatalogSettings, + known: Option<&LlmCatalogSettings>, +) -> Option { + let providers = || { + settings + .providers + .iter() + .chain(known.into_iter().flat_map(|known| known.providers.iter())) + }; + providers() + .find(|(id, _)| id.as_str() == selector) + .or_else(|| { + providers().find(|(_, provider)| { + provider + .aliases + .as_ref() + .is_some_and(|aliases| aliases.iter().any(|alias| alias == selector)) + }) + }) + .map(|(id, _)| ProviderId::new(id.clone())) +} + +fn canonical_settings_model( + provider: &ProviderId, + selector: &str, + settings: &LlmCatalogSettings, + known: Option<&LlmCatalogSettings>, +) -> Option { + let models = || { + settings + .providers + .get(provider.as_str()) + .into_iter() + .chain(known.and_then(|known| known.providers.get(provider.as_str()))) + .flat_map(|provider| provider.models.iter()) + }; + models() + .find(|(id, _)| id.as_str() == selector) + .or_else(|| { + models().find(|(_, model)| { + model + .aliases + .as_ref() + .is_some_and(|aliases| aliases.iter().any(|alias| alias == selector)) + }) + }) + .map(|(id, _)| ModelId::new(id.clone())) +} + +fn settings_model_candidates( + selector: &str, + settings: &LlmCatalogSettings, + known: Option<&LlmCatalogSettings>, +) -> Vec<(ProviderId, ModelId)> { + let collect = |matches: &dyn Fn(&str, &ModelCatalogSettings) -> bool| { + let mut candidates = BTreeSet::<(ProviderId, ModelId)>::new(); + for (provider_id, provider) in settings + .providers + .iter() + .chain(known.into_iter().flat_map(|known| known.providers.iter())) + { + for (model_id, model) in &provider.models { + if matches(model_id, model) { + candidates.insert(( + ProviderId::new(provider_id.clone()), + ModelId::new(model_id.clone()), + )); + } + } + } + candidates.into_iter().collect::>() + }; + + let canonical = collect(&|model_id, _| model_id == selector); + if canonical.is_empty() { + collect(&|_, model| { + model + .aliases + .as_ref() + .is_some_and(|aliases| aliases.iter().any(|alias| alias == selector)) + }) + } else { + canonical + } +} + +/// Return every built-in offering whose canonical model ID is `selector`. +/// +/// This includes disabled providers because config compatibility +/// normalization happens before runtime availability is known. +pub fn builtin_canonical_model_offerings( + selector: &str, +) -> Result, CatalogBuildError> { + let settings = Catalog::builtin_settings()?; + Ok(settings + .providers + .iter() + .filter_map(|(provider, settings)| { + settings.models.get_key_value(selector).map(|(model, _)| { + ( + ProviderId::new(provider.clone()), + ModelId::new(model.clone()), + ) + }) + }) + .collect()) +} + +/// Return every built-in offering that declares `selector` as an alias. +/// +/// This includes disabled providers because config compatibility +/// normalization happens before runtime availability is known. +pub fn builtin_alias_model_offerings( + selector: &str, +) -> Result, CatalogBuildError> { + let settings = Catalog::builtin_settings()?; + let mut offerings = settings + .providers + .iter() + .flat_map(|(provider, settings)| { + settings + .models + .iter() + .filter(move |(_, settings)| { + settings + .aliases + .as_ref() + .is_some_and(|aliases| aliases.iter().any(|alias| alias == selector)) + }) + .map(move |(model, _)| { + ( + ProviderId::new(provider.clone()), + ModelId::new(model.clone()), + ) + }) + }) + .collect::>(); + offerings.sort(); + offerings.dedup(); + Ok(offerings) +} + +/// Resolve a built-in provider ID or alias without filtering disabled rows. +pub fn builtin_provider_id(selector: &str) -> Result, CatalogBuildError> { + let settings = Catalog::builtin_settings()?; + Ok(canonical_settings_provider(selector, &settings, None)) +} + +/// Built-in catalog keys retired when provider API identifiers stopped being +/// Fabro model IDs. Keep this list explicit so old workflow and persisted-run +/// references fail with an actionable replacement instead of silently +/// selecting another route. +const RETIRED_MODEL_IDENTIFIERS: &[(&str, &str, &str)] = &[ + ("openai.gpt-5.5", "bedrock-openai", "gpt-5.5"), + ("openai.gpt-5.4", "bedrock-openai", "gpt-5.4"), + ( + "us.anthropic.claude-sonnet-4-6", + "bedrock", + "claude-sonnet-4-6", + ), + ("us.anthropic.claude-opus-4-8", "bedrock", "claude-opus-4-8"), + ( + "us.anthropic.claude-haiku-4-5", + "bedrock", + "claude-haiku-4-5", + ), + ("openai.gpt-oss-120b", "bedrock", "gpt-oss-120b"), + ("openai.gpt-oss-20b", "bedrock", "gpt-oss-20b"), + ("amazon.nova-2-lite", "bedrock", "nova-2-lite"), + ("meta.llama4-maverick", "bedrock", "llama-4-maverick"), + ("mistral.mistral-large-3", "bedrock", "mistral-large-3"), + ("mistral.devstral-2", "bedrock", "devstral-2"), + ("deepseek.v3-2", "bedrock", "deepseek-v3.2"), + ("moonshotai.kimi-k2.5", "bedrock", "kimi-k2.5"), + ("zai.glm-5", "bedrock", "glm-5"), + ("minimax.minimax-m2.5", "bedrock", "minimax-m2.5"), + ("nvidia.nemotron-3-super", "bedrock", "nemotron-3-super"), + ("us.anthropic.claude-fable-5", "bedrock", "claude-fable-5"), + ("anthropic/claude-fable-5", "openrouter", "claude-fable-5"), + ("anthropic/claude-opus-4-8", "openrouter", "claude-opus-4-8"), + ("anthropic/claude-opus-4-7", "openrouter", "claude-opus-4-7"), + ( + "anthropic/claude-sonnet-4-6", + "openrouter", + "claude-sonnet-4-6", + ), + ( + "anthropic/claude-haiku-4-5", + "openrouter", + "claude-haiku-4-5", + ), + ("openai/gpt-5.6-sol", "openrouter", "gpt-5.6-sol"), + ("openai/gpt-5.6-terra", "openrouter", "gpt-5.6-terra"), + ("openai/gpt-5.6-luna", "openrouter", "gpt-5.6-luna"), + ("openai/gpt-5.4", "openrouter", "gpt-5.4"), + ("openai/gpt-5.5", "openrouter", "gpt-5.5"), + ( + "google/gemini-3.1-pro-preview", + "openrouter", + "gemini-3.1-pro-preview", + ), + ("google/gemini-3.5-flash", "openrouter", "gemini-3.5-flash"), + ("xiaomi/mimo-v2.5-pro", "openrouter", "mimo-v2.5-pro"), + ("minimax/minimax-m2.7", "openrouter", "minimax-m2.7"), + ("deepseek/deepseek-v4-pro", "openrouter", "deepseek-v4-pro"), + ( + "deepseek/deepseek-v4-flash", + "openrouter", + "deepseek-v4-flash", + ), + ("moonshotai/kimi-k2.6", "openrouter", "kimi-k2.6"), + ("moonshotai/kimi-k3", "openrouter", "kimi-k3"), + ("poolside/laguna-s-2.1", "openrouter", "laguna-s-2.1"), + ("poolside/laguna-xs-2.1", "openrouter", "laguna-xs-2.1"), + ("qwen/qwen3-coder", "openrouter", "qwen3-coder"), + ("qwen/qwen3.6-flash", "openrouter", "qwen3.6-flash"), + ("z-ai/glm-5.2", "openrouter", "glm-5.2"), + ("z-ai/glm-4.6", "openrouter", "glm-4.6"), + ( + "nvidia/nemotron-3-super-120b-a12b", + "openrouter", + "nemotron-3-super-120b-a12b", + ), + ("mistralai/devstral-2512", "openrouter", "devstral-2512"), +]; + +/// Return the replacement address for a retired built-in catalog key. +#[must_use] +pub fn retired_model_replacement(identifier: &str) -> Option<(ProviderId, ModelId)> { + RETIRED_MODEL_IDENTIFIERS + .iter() + .find(|(retired, _, _)| *retired == identifier) + .map(|(_, provider, model)| (ProviderId::new(*provider), ModelId::new(*model))) +} + +fn retired_model_address(identifier: &str) -> Option<(ProviderId, ModelId)> { + retired_model_replacement(identifier) } fn merge_catalog_settings( @@ -1087,21 +1734,20 @@ fn merge_catalog_settings( fallback.providers.insert(id, provider); } - for (id, model) in higher.models { + fallback +} + +fn merge_provider_settings( + mut higher: ProviderCatalogSettings, + mut fallback: ProviderCatalogSettings, +) -> ProviderCatalogSettings { + for (id, model) in higher.models.drain() { let model = match fallback.models.remove(&id) { Some(fallback_model) => merge_model_settings(model, fallback_model), None => model, }; fallback.models.insert(id, model); } - - fallback -} - -fn merge_provider_settings( - higher: ProviderCatalogSettings, - fallback: ProviderCatalogSettings, -) -> ProviderCatalogSettings { ProviderCatalogSettings { display_name: higher.display_name.or(fallback.display_name), adapter: higher.adapter.or(fallback.adapter), @@ -1115,6 +1761,7 @@ fn merge_provider_settings( priority: higher.priority.or(fallback.priority), enabled: higher.enabled.or(fallback.enabled), aliases: higher.aliases.or(fallback.aliases), + models: fallback.models, } } @@ -1418,7 +2065,7 @@ fn build_model( let speed_costs = build_speed_costs(model_id, settings.costs.as_ref(), &controls)?; let model = Model { - id: model_id.to_string(), + id: ModelId::new(model_id), provider: provider.id.clone(), family, display_name, @@ -1436,11 +2083,18 @@ fn build_model( small_default: settings.small_default.unwrap_or_default(), configured: false, }; + let api_id = match settings.api_id.as_ref() { + Some(api_id) if api_id.is_empty() => { + return Err(CatalogBuildError::EmptyModelApiId { + provider: provider.id.clone(), + model: ModelId::new(model_id), + }); + } + Some(api_id) => api_id.clone(), + None => model_id.to_string(), + }; let catalog_settings = CatalogModelSettings { - api_id: settings - .api_id - .clone() - .unwrap_or_else(|| model_id.to_string()), + api_id, codec: resolve_model_codec(model_id, provider, settings.codec)?, billing_policy: settings.billing_policy.unwrap_or(provider.billing_policy), agent_profile: settings.agent_profile.unwrap_or(provider.agent_profile), @@ -1458,7 +2112,7 @@ fn warn_multiple_probe_models(models_with_settings: &[(Model, CatalogModelSettin probes_by_provider .entry(model.provider.clone()) .or_default() - .push(model.id.clone()); + .push(model.id.to_string()); } } @@ -1675,16 +2329,20 @@ fn register_provider_identifier( } fn register_model_identifier( - identifiers: &mut BTreeMap, + identifiers: &mut BTreeMap, identifier: String, - owner: String, + owner: ModelId, + provider: &ProviderId, ) -> Result<(), CatalogBuildError> { match identifiers.get(&identifier) { - Some(existing) if existing != &owner => Err(CatalogBuildError::DuplicateModelIdentifier { - identifier, - first: existing.clone(), - second: owner, - }), + Some(existing) if existing != &owner => { + Err(CatalogBuildError::DuplicateProviderModelSelector { + provider: provider.clone(), + selector: identifier, + first: existing.clone(), + second: owner, + }) + } _ => { identifiers.insert(identifier, owner); Ok(()) @@ -1743,12 +2401,6 @@ fn provider_order(left: &CatalogProvider, right: &CatalogProvider) -> std::cmp:: .then_with(|| left.id.cmp(&right.id)) } -fn model_order(left: &Model, right: &Model) -> std::cmp::Ordering { - left.provider - .cmp(&right.provider) - .then_with(|| left.id.cmp(&right.id)) -} - #[cfg(test)] mod tests { use strum::VariantArray; @@ -1762,6 +2414,54 @@ mod tests { toml::from_str(source).expect("fixture should parse as an LLM settings layer") } + fn portable_model_catalog() -> Catalog { + Catalog::from_settings(&minimal_settings( + r#" +[providers.openai] +display_name = "OpenAI" +adapter = "openai" +agent_profile = "openai" +priority = 90 + +[providers.openai.models."gpt-5.6-sol"] +display_name = "GPT-5.6 Sol" +family = "gpt-5" +aliases = ["gpt-56-sol", "portable"] +default = true + +[providers.openai.models."gpt-5.6-sol".limits] +context_window = 1000 + +[providers.openai.models."gpt-5.6-sol".features] +tools = true +vision = false +reasoning = true + +[providers.openrouter] +display_name = "OpenRouter" +adapter = "openai_compatible" +agent_profile = "openai" +priority = 25 + +[providers.openrouter.models."gpt-5.6-sol"] +api_id = "openai/gpt-5.6-sol" +display_name = "GPT-5.6 Sol (via OpenRouter)" +family = "gpt-5" +aliases = ["gpt-56-sol", "portable"] +default = true + +[providers.openrouter.models."gpt-5.6-sol".limits] +context_window = 1000 + +[providers.openrouter.models."gpt-5.6-sol".features] +tools = true +vision = false +reasoning = true +"#, + )) + .expect("portable model fixture should build") + } + const BEDROCK_SIGV4_LAYER: &str = r#" [providers.bedrock] adapter = "bedrock" @@ -1934,18 +2634,21 @@ enabled = true // provider's Anthropic defaults the other way. assert_eq!( catalog - .model_settings("us.anthropic.claude-sonnet-4-6") + .model_settings_on_provider(&bedrock, "claude-sonnet-4-6") .unwrap() .billing_policy, BillingPolicy::Anthropic ); assert_eq!( - catalog.model_settings("zai.glm-5").unwrap().billing_policy, + catalog + .model_settings_on_provider(&bedrock, "glm-5") + .unwrap() + .billing_policy, BillingPolicy::OpenAi ); assert_eq!( catalog - .model_settings("us.anthropic.claude-haiku-4-5") + .model_settings_on_provider(&bedrock, "claude-haiku-4-5") .unwrap() .api_id, "us.anthropic.claude-haiku-4-5-20251001-v1:0" @@ -1954,17 +2657,17 @@ enabled = true catalog .default_for_provider(&bedrock) .map(|model| model.id.as_str()), - Some("us.anthropic.claude-sonnet-4-6") + Some("claude-sonnet-4-6") ); // Fable 5 ships with sampling params pinned off (the Converse // encoder drops temperature/top_p for it). let fable = catalog - .get("us.anthropic.claude-fable-5") + .get_on_provider(&bedrock, "claude-fable-5") .expect("fable row should be present"); assert!(!fable.features.sampling_params); assert_eq!( catalog - .model_settings("us.anthropic.claude-fable-5") + .model_settings_on_provider(&bedrock, "claude-fable-5") .unwrap() .billing_policy, BillingPolicy::Anthropic @@ -2001,7 +2704,7 @@ enabled = true catalog .default_for_provider(&provider_id) .map(|model| model.id.as_str()), - Some("openai.gpt-5.5") + Some("gpt-5.5") ); } @@ -2115,14 +2818,14 @@ enabled = true // open-weights rows inherit it. assert_eq!( catalog - .model_settings("anthropic/claude-sonnet-4-6") + .model_settings_on_provider(&openrouter, "claude-sonnet-4-6") .unwrap() .billing_policy, BillingPolicy::Anthropic ); assert_eq!( catalog - .model_settings("deepseek/deepseek-v4-flash") + .model_settings_on_provider(&openrouter, "deepseek-v4-flash") .unwrap() .billing_policy, BillingPolicy::OpenAi @@ -2131,7 +2834,7 @@ enabled = true catalog .default_for_provider(&openrouter) .map(|model| model.id.as_str()), - Some("anthropic/claude-sonnet-4-6") + Some("claude-sonnet-4-6") ); } @@ -2147,7 +2850,7 @@ enabled = true let expected = [ ( - "openai/gpt-5.6-sol", + "gpt-5.6-sol", "openai/gpt-5.6-sol", "gpt-5", 1_050_000, @@ -2159,7 +2862,7 @@ enabled = true BillingPolicy::OpenAi, ), ( - "openai/gpt-5.6-terra", + "gpt-5.6-terra", "openai/gpt-5.6-terra", "gpt-5", 1_050_000, @@ -2171,7 +2874,7 @@ enabled = true BillingPolicy::OpenAi, ), ( - "openai/gpt-5.6-luna", + "gpt-5.6-luna", "openai/gpt-5.6-luna", "gpt-5", 1_050_000, @@ -2183,7 +2886,7 @@ enabled = true BillingPolicy::OpenAi, ), ( - "anthropic/claude-opus-4-8", + "claude-opus-4-8", "anthropic/claude-opus-4.8", "claude-4", 1_000_000, @@ -2195,7 +2898,7 @@ enabled = true BillingPolicy::Anthropic, ), ( - "anthropic/claude-fable-5", + "claude-fable-5", "anthropic/claude-fable-5", "claude-5", 1_000_000, @@ -2222,7 +2925,7 @@ enabled = true ) in expected { let model = catalog - .get(id) + .get_on_provider(&ProviderId::new("openrouter"), id) .unwrap_or_else(|| panic!("OpenRouter model '{id}' should be present")); assert_eq!(model.provider, ProviderId::new("openrouter"), "{id}"); assert_eq!(model.family, family, "{id}"); @@ -2243,15 +2946,10 @@ enabled = true ); let settings = catalog - .model_settings(id) + .model_settings_on_provider(&ProviderId::new("openrouter"), id) .unwrap_or_else(|| panic!("OpenRouter settings for '{id}' should be present")); assert_eq!(settings.api_id, api_id, "{id}"); assert_eq!(settings.billing_policy, billing_policy, "{id}"); - assert_eq!( - catalog.get(api_id).map(|model| model.id.as_str()), - Some(id), - "{id}" - ); assert_eq!( settings.controls.reasoning_effort, ReasoningEffort::VARIANTS, @@ -2260,6 +2958,33 @@ enabled = true } } + #[test] + fn builtin_gpt_5_6_short_aliases_are_portable() { + let catalog = Catalog::from_builtin_with_overrides(&minimal_settings( + r" +[providers.openrouter] +enabled = true +", + )) + .expect("enabled OpenRouter override should build from the built-in provider settings"); + + for provider in [ProviderId::openai(), ProviderId::new("openrouter")] { + for (alias, canonical_id) in [ + ("sol", "gpt-5.6-sol"), + ("terra", "gpt-5.6-terra"), + ("luna", "gpt-5.6-luna"), + ] { + let model = catalog + .resolve_on_provider(&provider, alias) + .unwrap_or_else(|error| { + panic!("{alias} should resolve on {provider}: {error}") + }); + assert_eq!(model.provider, provider, "{alias}"); + assert_eq!(model.id, canonical_id, "{alias}"); + } + } + } + #[test] fn builtin_openrouter_includes_glm_5_2_when_enabled() { let catalog = Catalog::from_builtin_with_overrides(&minimal_settings( @@ -2271,11 +2996,11 @@ enabled = true .expect("enabled OpenRouter override should build from the built-in provider settings"); let model = catalog - .get("z-ai/glm-5.2") + .get_on_provider(&ProviderId::new("openrouter"), "glm-5.2") .expect("OpenRouter GLM 5.2 should be present"); insta::assert_debug_snapshot!(model, @r#" Model { - id: "z-ai/glm-5.2", + id: "glm-5.2", provider: openrouter, family: "glm-5", display_name: "GLM 5.2 (via OpenRouter)", @@ -2315,7 +3040,7 @@ enabled = true "#); let settings = catalog - .model_settings("z-ai/glm-5.2") + .model_settings_on_provider(&ProviderId::new("openrouter"), "glm-5.2") .expect("OpenRouter GLM 5.2 settings should be present"); assert_eq!(settings.api_id, "z-ai/glm-5.2"); assert_eq!(settings.controls.reasoning_effort, vec![ @@ -2335,11 +3060,11 @@ enabled = true .expect("enabled OpenRouter override should build from the built-in provider settings"); let model = catalog - .get("moonshotai/kimi-k3") + .get_on_provider(&ProviderId::new("openrouter"), "kimi-k3") .expect("OpenRouter Kimi K3 should be present"); insta::assert_debug_snapshot!(model, @r#" Model { - id: "moonshotai/kimi-k3", + id: "kimi-k3", provider: openrouter, family: "kimi-k3", display_name: "Kimi K3 (via OpenRouter)", @@ -2379,7 +3104,7 @@ enabled = true "#); let settings = catalog - .model_settings("moonshotai/kimi-k3") + .model_settings_on_provider(&ProviderId::new("openrouter"), "kimi-k3") .expect("OpenRouter Kimi K3 settings should be present"); assert_eq!(settings.api_id, "moonshotai/kimi-k3"); assert_eq!(settings.controls.reasoning_effort, vec![ @@ -2400,20 +3125,13 @@ enabled = true .expect("enabled OpenRouter override should build from the built-in provider settings"); let expected = [ - ( - "poolside/laguna-s-2.1", - 1_048_576, - 131_072, - 0.10, - 0.20, - 0.01, - ), - ("poolside/laguna-xs-2.1", 262_144, 32_768, 0.06, 0.12, 0.03), + ("laguna-s-2.1", 1_048_576, 131_072, 0.10, 0.20, 0.01), + ("laguna-xs-2.1", 262_144, 32_768, 0.06, 0.12, 0.03), ]; for (id, context, max_output, input, output, cache_read) in expected { let model = catalog - .get(id) + .get_on_provider(&ProviderId::new("openrouter"), id) .unwrap_or_else(|| panic!("OpenRouter model '{id}' should be present")); assert_eq!(model.provider, ProviderId::new("openrouter"), "{id}"); assert_eq!(model.family, "laguna-2", "{id}"); @@ -2434,9 +3152,9 @@ enabled = true ); let settings = catalog - .model_settings(id) + .model_settings_on_provider(&ProviderId::new("openrouter"), id) .unwrap_or_else(|| panic!("OpenRouter settings for '{id}' should be present")); - assert_eq!(settings.api_id, id, "{id}"); + assert_eq!(settings.api_id, format!("poolside/{id}"), "{id}"); assert!(settings.controls.reasoning_effort.is_empty(), "{id}"); } } @@ -2898,30 +3616,28 @@ adapter = "openai" agent_profile = "openai" enabled = true -[models.one] -provider = "test" +[providers.test.models.one] display_name = "One" family = "test" aliases = ["shared"] -[models.one.limits] +[providers.test.models.one.limits] context_window = 1000 -[models.one.features] +[providers.test.models.one.features] tools = false vision = false reasoning = false -[models.two] -provider = "test" +[providers.test.models.two] display_name = "Two" family = "test" aliases = ["shared"] -[models.two.limits] +[providers.test.models.two.limits] context_window = 1000 -[models.two.features] +[providers.test.models.two.features] tools = false vision = false reasoning = false @@ -2932,8 +3648,238 @@ reasoning = false assert!(matches!( err, - CatalogBuildError::DuplicateModelIdentifier { identifier, first, second } - if identifier == "shared" && first == "one" && second == "two" + CatalogBuildError::DuplicateProviderModelSelector { + provider, + selector, + first, + second, + } if provider == ProviderId::new("test") + && selector == "shared" + && first == "one" + && second == "two" + )); + } + + #[test] + fn provider_scoped_model_rejects_redundant_provider_field() { + let error = Catalog::from_settings(&minimal_settings( + r#" +[providers.test] +display_name = "Test" +adapter = "openai" + +[providers.test.models.one] +provider = "test" +"#, + )) + .unwrap_err(); + + assert!(matches!( + error, + CatalogBuildError::ScopedModelDeclaresProvider { provider, model } + if provider == ProviderId::new("test") && model == "one" + )); + } + + #[test] + fn provider_aware_selection_uses_readiness_priority_and_api_ids() { + let catalog = portable_model_catalog(); + let openai = ProviderId::openai(); + let openrouter = ProviderId::new("openrouter"); + + let offerings = catalog + .list(None) + .into_iter() + .filter(|model| model.id.as_str() == "gpt-5.6-sol") + .collect::>(); + assert_eq!(offerings.len(), 2); + + let direct = catalog + .select("gpt-56-sol", None, &HashSet::from([openai.clone()])) + .unwrap(); + assert_eq!(direct.provider, openai); + assert_eq!(direct.id, "gpt-5.6-sol"); + assert_eq!(catalog.settings_for(direct).unwrap().api_id, "gpt-5.6-sol"); + + let aggregator = catalog + .select("gpt-56-sol", None, &HashSet::from([openrouter.clone()])) + .unwrap(); + assert_eq!(aggregator.provider, openrouter); + assert_eq!(aggregator.id, "gpt-5.6-sol"); + assert_eq!( + catalog.settings_for(aggregator).unwrap().api_id, + "openai/gpt-5.6-sol" + ); + + let both = HashSet::from([ProviderId::openai(), ProviderId::new("openrouter")]); + assert_eq!( + catalog.select("portable", None, &both).unwrap().provider, + ProviderId::openai() + ); + assert_eq!( + catalog + .select("portable", Some(&ProviderId::new("openrouter")), &both,) + .unwrap() + .provider, + ProviderId::new("openrouter") + ); + assert!(matches!( + catalog.select( + "portable", + Some(&ProviderId::new("openrouter")), + &HashSet::from([ProviderId::openai()]), + ), + Err(ModelSelectionError::ProviderUnavailable { provider }) + if provider == ProviderId::new("openrouter") + )); + } + + #[test] + fn equal_provider_priorities_use_canonical_provider_id_as_tie_breaker() { + let catalog = Catalog::from_settings(&minimal_settings( + r#" +[providers.zeta] +display_name = "Zeta" +adapter = "openai" +agent_profile = "openai" +priority = 10 + +[providers.zeta.models.zeta] +display_name = "Zeta" +family = "test" +aliases = ["shared"] +default = true + +[providers.zeta.models.zeta.limits] +context_window = 1000 + +[providers.zeta.models.zeta.features] +tools = false +vision = false +reasoning = false + +[providers.alpha] +display_name = "Alpha" +adapter = "openai" +agent_profile = "openai" +priority = 10 + +[providers.alpha.models.alpha] +display_name = "Alpha" +family = "test" +aliases = ["shared"] +default = true + +[providers.alpha.models.alpha.limits] +context_window = 1000 + +[providers.alpha.models.alpha.features] +tools = false +vision = false +reasoning = false +"#, + )) + .unwrap(); + + let eligible = HashSet::from([ProviderId::new("zeta"), ProviderId::new("alpha")]); + assert_eq!( + catalog.select("shared", None, &eligible).unwrap().provider, + ProviderId::new("alpha") + ); + } + + #[test] + fn canonical_id_wins_over_cross_provider_alias() { + let catalog = Catalog::from_settings(&minimal_settings( + r#" +[providers.direct] +display_name = "Direct" +adapter = "openai" +agent_profile = "openai" +priority = 1 + +[providers.direct.models.shared] +display_name = "Canonical Shared" +family = "test" +default = true + +[providers.direct.models.shared.limits] +context_window = 1000 + +[providers.direct.models.shared.features] +tools = false +vision = false +reasoning = false + +[providers.aggregator] +display_name = "Aggregator" +adapter = "openai" +agent_profile = "openai" +priority = 100 + +[providers.aggregator.models.other] +display_name = "Alias Shared" +family = "test" +aliases = ["shared"] +default = true + +[providers.aggregator.models.other.limits] +context_window = 1000 + +[providers.aggregator.models.other.features] +tools = false +vision = false +reasoning = false +"#, + )) + .unwrap(); + let eligible = HashSet::from([ProviderId::new("direct"), ProviderId::new("aggregator")]); + + let unqualified = catalog.select("shared", None, &eligible).unwrap(); + assert_eq!(unqualified.provider, ProviderId::new("direct")); + assert_eq!(unqualified.id, "shared"); + + let qualified = catalog + .resolve_on_provider(&ProviderId::new("aggregator"), "shared") + .unwrap(); + assert_eq!(qualified.id, "other"); + + let aggregator_only = HashSet::from([ProviderId::new("aggregator")]); + let portable_alias = catalog.select("shared", None, &aggregator_only).unwrap(); + assert_eq!(portable_alias.provider, ProviderId::new("aggregator")); + assert_eq!(portable_alias.id, "other"); + } + + #[test] + fn empty_api_id_is_rejected() { + let error = Catalog::from_settings(&minimal_settings( + r#" +[providers.test] +display_name = "Test" +adapter = "openai" +agent_profile = "openai" + +[providers.test.models.model] +api_id = "" +display_name = "Model" +family = "test" +default = true + +[providers.test.models.model.limits] +context_window = 1000 + +[providers.test.models.model.features] +tools = false +vision = false +reasoning = false +"#, + )) + .unwrap_err(); + + assert!(matches!( + error, + CatalogBuildError::EmptyModelApiId { provider, model } + if provider == ProviderId::new("test") && model == "model" )); } @@ -3068,7 +4014,7 @@ reasoning = false } #[test] - fn catalog_lists_models_by_provider_then_model_id() { + fn catalog_lists_models_by_provider_priority_then_model_id() { let layer = minimal_settings( r#" [providers.zeta] @@ -3133,7 +4079,7 @@ reasoning = false .map(|model| model.id.as_str()) .collect::>(); - assert_eq!(ids, ["alpha_one", "zeta_one", "zeta_two"]); + assert_eq!(ids, ["zeta_one", "zeta_two", "alpha_one"]); assert_eq!(catalog.default_model().id, "zeta_two"); } @@ -4964,7 +5910,7 @@ sampling_params = false fn openai_context_window_can_be_overridden_for_direct_api_usage() { let catalog = Catalog::from_builtin_with_overrides(&minimal_settings( r#" -[models."gpt-5.5".limits] +[providers.openai.models."gpt-5.5".limits] context_window = 1050000 "#, )) diff --git a/lib/crates/fabro-model/src/catalog/providers/anthropic.toml b/lib/crates/fabro-model/src/catalog/providers/anthropic.toml index 0a1587197..95649e214 100644 --- a/lib/crates/fabro-model/src/catalog/providers/anthropic.toml +++ b/lib/crates/fabro-model/src/catalog/providers/anthropic.toml @@ -9,18 +9,16 @@ priority = 100 credentials = ["env:ANTHROPIC_API_KEY", "vault:ANTHROPIC_API_KEY"] header = { custom = "x-api-key" } -[models."claude-fable-5"] -provider = "anthropic" -api_id = "claude-fable-5" +[providers.anthropic.models."claude-fable-5"] display_name = "Claude Fable 5" family = "claude-5" aliases = ["fable", "claude-fable"] -[models."claude-fable-5".limits] +[providers.anthropic.models."claude-fable-5".limits] context_window = 1000000 max_output = 128000 -[models."claude-fable-5".features] +[providers.anthropic.models."claude-fable-5".features] tools = true vision = true reasoning = true @@ -28,14 +26,12 @@ reasoning_effort = "always_adaptive" prompt_cache = true sampling_params = false -[models."claude-fable-5".costs] +[providers.anthropic.models."claude-fable-5".costs] input_cost_per_mtok = 10.0 output_cost_per_mtok = 50.0 cache_input_cost_per_mtok = 1.0 -[models."claude-opus-4-8"] -provider = "anthropic" -api_id = "claude-opus-4-8" +[providers.anthropic.models."claude-opus-4-8"] display_name = "Claude Opus 4.8" family = "claude-4" training = "2026-01-01" @@ -43,11 +39,11 @@ knowledge_cutoff = "Jan 2026" estimated_output_tps = 25 aliases = ["opus", "claude-opus"] -[models."claude-opus-4-8".limits] +[providers.anthropic.models."claude-opus-4-8".limits] context_window = 1000000 max_output = 128000 -[models."claude-opus-4-8".features] +[providers.anthropic.models."claude-opus-4-8".features] tools = true vision = true reasoning = true @@ -55,33 +51,31 @@ reasoning_effort = "levels" prompt_cache = true sampling_params = false -[models."claude-opus-4-8".controls] +[providers.anthropic.models."claude-opus-4-8".controls] speed = ["fast"] -[models."claude-opus-4-8".costs] +[providers.anthropic.models."claude-opus-4-8".costs] input_cost_per_mtok = 5.0 output_cost_per_mtok = 25.0 cache_input_cost_per_mtok = 0.5 -[models."claude-opus-4-8".costs.speed.fast] +[providers.anthropic.models."claude-opus-4-8".costs.speed.fast] input_cost_per_mtok = 10.0 output_cost_per_mtok = 50.0 cache_input_cost_per_mtok = 1.0 -[models."claude-opus-4-7"] -provider = "anthropic" -api_id = "claude-opus-4-7" +[providers.anthropic.models."claude-opus-4-7"] display_name = "Claude Opus 4.7" family = "claude-4" training = "2025-08-01" knowledge_cutoff = "May 2025" estimated_output_tps = 25 -[models."claude-opus-4-7".limits] +[providers.anthropic.models."claude-opus-4-7".limits] context_window = 1000000 max_output = 128000 -[models."claude-opus-4-7".features] +[providers.anthropic.models."claude-opus-4-7".features] tools = true vision = true reasoning = true @@ -89,82 +83,76 @@ reasoning_effort = "levels" prompt_cache = true sampling_params = false -[models."claude-opus-4-7".controls] +[providers.anthropic.models."claude-opus-4-7".controls] speed = ["fast"] -[models."claude-opus-4-7".costs] +[providers.anthropic.models."claude-opus-4-7".costs] input_cost_per_mtok = 5.0 output_cost_per_mtok = 25.0 cache_input_cost_per_mtok = 0.5 -[models."claude-opus-4-7".costs.speed.fast] +[providers.anthropic.models."claude-opus-4-7".costs.speed.fast] input_cost_per_mtok = 30.0 output_cost_per_mtok = 150.0 cache_input_cost_per_mtok = 3.0 -[models."claude-opus-4-6"] -provider = "anthropic" -api_id = "claude-opus-4-6" +[providers.anthropic.models."claude-opus-4-6"] display_name = "Claude Opus 4.6" family = "claude-4" training = "2025-08-01" knowledge_cutoff = "May 2025" estimated_output_tps = 25 -[models."claude-opus-4-6".limits] +[providers.anthropic.models."claude-opus-4-6".limits] context_window = 1000000 max_output = 128000 -[models."claude-opus-4-6".features] +[providers.anthropic.models."claude-opus-4-6".features] tools = true vision = true reasoning = true reasoning_effort = "levels" prompt_cache = true -[models."claude-opus-4-6".controls] +[providers.anthropic.models."claude-opus-4-6".controls] speed = ["fast"] -[models."claude-opus-4-6".costs] +[providers.anthropic.models."claude-opus-4-6".costs] input_cost_per_mtok = 5.0 output_cost_per_mtok = 25.0 cache_input_cost_per_mtok = 0.5 -[models."claude-opus-4-6".costs.speed.fast] +[providers.anthropic.models."claude-opus-4-6".costs.speed.fast] input_cost_per_mtok = 30.0 output_cost_per_mtok = 150.0 cache_input_cost_per_mtok = 3.0 -[models."claude-sonnet-4-5"] -provider = "anthropic" -api_id = "claude-sonnet-4-5" +[providers.anthropic.models."claude-sonnet-4-5"] display_name = "Claude Sonnet 4.5" family = "claude-4" training = "2025-08-01" knowledge_cutoff = "May 2025" estimated_output_tps = 50 -[models."claude-sonnet-4-5".limits] +[providers.anthropic.models."claude-sonnet-4-5".limits] context_window = 200000 max_output = 64000 -[models."claude-sonnet-4-5".features] +[providers.anthropic.models."claude-sonnet-4-5".features] tools = true vision = true reasoning = true prompt_cache = true -[models."claude-sonnet-4-5".controls] +[providers.anthropic.models."claude-sonnet-4-5".controls] reasoning_effort = ["low", "medium", "high", "xhigh", "max"] -[models."claude-sonnet-4-5".costs] +[providers.anthropic.models."claude-sonnet-4-5".costs] input_cost_per_mtok = 3.0 output_cost_per_mtok = 15.0 cache_input_cost_per_mtok = 0.3 -[models."claude-sonnet-4-6"] -provider = "anthropic" -api_id = "claude-sonnet-4-6" +[providers.anthropic.models."claude-sonnet-4-6"] display_name = "Claude Sonnet 4.6" family = "claude-4" training = "2025-08-01" @@ -173,25 +161,23 @@ default = true estimated_output_tps = 50 aliases = ["sonnet", "claude-sonnet"] -[models."claude-sonnet-4-6".limits] +[providers.anthropic.models."claude-sonnet-4-6".limits] context_window = 200000 max_output = 64000 -[models."claude-sonnet-4-6".features] +[providers.anthropic.models."claude-sonnet-4-6".features] tools = true vision = true reasoning = true reasoning_effort = "levels" prompt_cache = true -[models."claude-sonnet-4-6".costs] +[providers.anthropic.models."claude-sonnet-4-6".costs] input_cost_per_mtok = 3.0 output_cost_per_mtok = 15.0 cache_input_cost_per_mtok = 0.3 -[models."claude-haiku-4-5"] -provider = "anthropic" -api_id = "claude-haiku-4-5" +[providers.anthropic.models."claude-haiku-4-5"] display_name = "Claude Haiku 4.5" family = "claude-4" training = "2025-08-01" @@ -201,17 +187,17 @@ aliases = ["haiku", "claude-haiku"] probe = true small_default = true -[models."claude-haiku-4-5".limits] +[providers.anthropic.models."claude-haiku-4-5".limits] context_window = 200000 max_output = 8192 -[models."claude-haiku-4-5".features] +[providers.anthropic.models."claude-haiku-4-5".features] tools = true vision = true reasoning = false prompt_cache = true -[models."claude-haiku-4-5".costs] +[providers.anthropic.models."claude-haiku-4-5".costs] input_cost_per_mtok = 0.8 output_cost_per_mtok = 4.0 cache_input_cost_per_mtok = 0.08 diff --git a/lib/crates/fabro-model/src/catalog/providers/bedrock-openai.toml b/lib/crates/fabro-model/src/catalog/providers/bedrock-openai.toml index 20b011b9c..21384d9b2 100644 --- a/lib/crates/fabro-model/src/catalog/providers/bedrock-openai.toml +++ b/lib/crates/fabro-model/src/catalog/providers/bedrock-openai.toml @@ -35,41 +35,41 @@ credentials = [ # [llm.providers.bedrock-openai] # enabled = true -[models."openai.gpt-5.5"] -provider = "bedrock-openai" +[providers.bedrock-openai.models."gpt-5.5"] +api_id = "openai.gpt-5.5" display_name = "GPT-5.5 (Bedrock)" family = "gpt-5" default = true -[models."openai.gpt-5.5".limits] +[providers.bedrock-openai.models."gpt-5.5".limits] context_window = 272000 max_output = 128000 -[models."openai.gpt-5.5".features] +[providers.bedrock-openai.models."gpt-5.5".features] tools = true vision = true reasoning = true reasoning_effort = "levels" -[models."openai.gpt-5.5".costs] +[providers.bedrock-openai.models."gpt-5.5".costs] input_cost_per_mtok = 5.5 output_cost_per_mtok = 33.0 -[models."openai.gpt-5.4"] -provider = "bedrock-openai" +[providers.bedrock-openai.models."gpt-5.4"] +api_id = "openai.gpt-5.4" display_name = "GPT-5.4 (Bedrock)" family = "gpt-5" -[models."openai.gpt-5.4".limits] +[providers.bedrock-openai.models."gpt-5.4".limits] context_window = 272000 max_output = 128000 -[models."openai.gpt-5.4".features] +[providers.bedrock-openai.models."gpt-5.4".features] tools = true vision = true reasoning = true reasoning_effort = "levels" -[models."openai.gpt-5.4".costs] +[providers.bedrock-openai.models."gpt-5.4".costs] input_cost_per_mtok = 2.75 output_cost_per_mtok = 16.5 diff --git a/lib/crates/fabro-model/src/catalog/providers/bedrock.toml b/lib/crates/fabro-model/src/catalog/providers/bedrock.toml index f6f217a4f..aeed22e46 100644 --- a/lib/crates/fabro-model/src/catalog/providers/bedrock.toml +++ b/lib/crates/fabro-model/src/catalog/providers/bedrock.toml @@ -45,68 +45,67 @@ credentials = [ # file because its Bedrock deployment pins sampling parameters and requires an # extra data-sharing opt-in. -[models."us.anthropic.claude-sonnet-4-6"] -provider = "bedrock" +[providers.bedrock.models."claude-sonnet-4-6"] +api_id = "us.anthropic.claude-sonnet-4-6" display_name = "Claude Sonnet 4.6 (Bedrock)" family = "claude-4" billing_policy = "anthropic" default = true -[models."us.anthropic.claude-sonnet-4-6".limits] +[providers.bedrock.models."claude-sonnet-4-6".limits] context_window = 1000000 max_output = 64000 -[models."us.anthropic.claude-sonnet-4-6".features] +[providers.bedrock.models."claude-sonnet-4-6".features] tools = true vision = true reasoning = true prompt_cache = true -[models."us.anthropic.claude-sonnet-4-6".costs] +[providers.bedrock.models."claude-sonnet-4-6".costs] input_cost_per_mtok = 3.0 output_cost_per_mtok = 15.0 cache_input_cost_per_mtok = 0.3 -[models."us.anthropic.claude-opus-4-8"] -provider = "bedrock" +[providers.bedrock.models."claude-opus-4-8"] +api_id = "us.anthropic.claude-opus-4-8" display_name = "Claude Opus 4.8 (Bedrock)" family = "claude-4" billing_policy = "anthropic" -[models."us.anthropic.claude-opus-4-8".limits] +[providers.bedrock.models."claude-opus-4-8".limits] context_window = 1000000 max_output = 128000 -[models."us.anthropic.claude-opus-4-8".features] +[providers.bedrock.models."claude-opus-4-8".features] tools = true vision = true reasoning = true prompt_cache = true -[models."us.anthropic.claude-opus-4-8".costs] +[providers.bedrock.models."claude-opus-4-8".costs] input_cost_per_mtok = 5.0 output_cost_per_mtok = 25.0 cache_input_cost_per_mtok = 0.5 -[models."us.anthropic.claude-haiku-4-5"] -provider = "bedrock" +[providers.bedrock.models."claude-haiku-4-5"] api_id = "us.anthropic.claude-haiku-4-5-20251001-v1:0" display_name = "Claude Haiku 4.5 (Bedrock)" family = "claude-4" billing_policy = "anthropic" small_default = true -[models."us.anthropic.claude-haiku-4-5".limits] +[providers.bedrock.models."claude-haiku-4-5".limits] context_window = 200000 max_output = 64000 -[models."us.anthropic.claude-haiku-4-5".features] +[providers.bedrock.models."claude-haiku-4-5".features] tools = true vision = true reasoning = false prompt_cache = true -[models."us.anthropic.claude-haiku-4-5".costs] +[providers.bedrock.models."claude-haiku-4-5".costs] input_cost_per_mtok = 1.0 output_cost_per_mtok = 5.0 cache_input_cost_per_mtok = 0.1 @@ -116,149 +115,142 @@ cache_input_cost_per_mtok = 0.1 # GPT-5.5/5.4 are NOT here: on Bedrock they are Responses-API-only on the # bedrock-mantle endpoint (no Converse), a named follow-up route. -[models."openai.gpt-oss-120b"] -provider = "bedrock" +[providers.bedrock.models."gpt-oss-120b"] api_id = "openai.gpt-oss-120b-1:0" display_name = "GPT-OSS 120B (Bedrock)" family = "gpt-oss" billing_policy = "openai" agent_profile = "openai" -[models."openai.gpt-oss-120b".limits] +[providers.bedrock.models."gpt-oss-120b".limits] context_window = 128000 max_output = 16384 -[models."openai.gpt-oss-120b".features] +[providers.bedrock.models."gpt-oss-120b".features] tools = true vision = false reasoning = true -[models."openai.gpt-oss-120b".costs] +[providers.bedrock.models."gpt-oss-120b".costs] input_cost_per_mtok = 0.15 output_cost_per_mtok = 0.60 -[models."openai.gpt-oss-20b"] -provider = "bedrock" +[providers.bedrock.models."gpt-oss-20b"] api_id = "openai.gpt-oss-20b-1:0" display_name = "GPT-OSS 20B (Bedrock)" family = "gpt-oss" billing_policy = "openai" agent_profile = "openai" -[models."openai.gpt-oss-20b".limits] +[providers.bedrock.models."gpt-oss-20b".limits] context_window = 128000 max_output = 16384 -[models."openai.gpt-oss-20b".features] +[providers.bedrock.models."gpt-oss-20b".features] tools = true vision = false reasoning = true -[models."openai.gpt-oss-20b".costs] +[providers.bedrock.models."gpt-oss-20b".costs] input_cost_per_mtok = 0.07 output_cost_per_mtok = 0.30 # ---------- Amazon Nova ---------- -[models."amazon.nova-2-lite"] -provider = "bedrock" +[providers.bedrock.models."nova-2-lite"] api_id = "global.amazon.nova-2-lite-v1:0" display_name = "Nova 2 Lite (Bedrock)" family = "nova-2" billing_policy = "openai" agent_profile = "openai" -[models."amazon.nova-2-lite".limits] +[providers.bedrock.models."nova-2-lite".limits] context_window = 1000000 # Bedrock caps Nova output at 65535 (2^16 - 1); 65536 trips # "maximum tokens exceeds the model limit of 65535" since the prompt handler # defaults max_tokens to max_output. max_output = 65535 -[models."amazon.nova-2-lite".features] +[providers.bedrock.models."nova-2-lite".features] tools = true vision = true reasoning = false -[models."amazon.nova-2-lite".costs] +[providers.bedrock.models."nova-2-lite".costs] input_cost_per_mtok = 0.30 output_cost_per_mtok = 2.50 # ---------- Open-weights ---------- -[models."meta.llama4-maverick"] -provider = "bedrock" +[providers.bedrock.models."llama-4-maverick"] api_id = "us.meta.llama4-maverick-17b-instruct-v1:0" display_name = "Llama 4 Maverick (Bedrock)" family = "llama-4" billing_policy = "openai" agent_profile = "openai" -[models."meta.llama4-maverick".limits] +[providers.bedrock.models."llama-4-maverick".limits] context_window = 1000000 max_output = 8192 -[models."meta.llama4-maverick".features] +[providers.bedrock.models."llama-4-maverick".features] tools = true vision = true reasoning = false -[models."mistral.mistral-large-3"] -provider = "bedrock" +[providers.bedrock.models."mistral-large-3"] api_id = "mistral.mistral-large-3-675b-instruct" display_name = "Mistral Large 3 (Bedrock)" family = "mistral-large" billing_policy = "openai" agent_profile = "openai" -[models."mistral.mistral-large-3".limits] +[providers.bedrock.models."mistral-large-3".limits] context_window = 256000 max_output = 32768 -[models."mistral.mistral-large-3".features] +[providers.bedrock.models."mistral-large-3".features] tools = true vision = true reasoning = false -[models."mistral.mistral-large-3".costs] +[providers.bedrock.models."mistral-large-3".costs] input_cost_per_mtok = 0.50 output_cost_per_mtok = 1.50 -[models."mistral.devstral-2"] -provider = "bedrock" +[providers.bedrock.models."devstral-2"] api_id = "mistral.devstral-2-123b" display_name = "Devstral 2 (Bedrock)" family = "devstral" billing_policy = "openai" agent_profile = "openai" -[models."mistral.devstral-2".limits] +[providers.bedrock.models."devstral-2".limits] context_window = 256000 max_output = 32768 -[models."mistral.devstral-2".features] +[providers.bedrock.models."devstral-2".features] tools = true vision = false reasoning = false -[models."deepseek.v3-2"] -provider = "bedrock" +[providers.bedrock.models."deepseek-v3.2"] api_id = "deepseek.v3.2" display_name = "DeepSeek V3.2 (Bedrock)" family = "deepseek-v3" billing_policy = "openai" agent_profile = "openai" -[models."deepseek.v3-2".limits] +[providers.bedrock.models."deepseek-v3.2".limits] context_window = 164000 max_output = 8192 -[models."deepseek.v3-2".features] +[providers.bedrock.models."deepseek-v3.2".features] tools = true vision = false reasoning = true -[models."deepseek.v3-2".costs] +[providers.bedrock.models."deepseek-v3.2".costs] input_cost_per_mtok = 0.62 output_cost_per_mtok = 1.85 @@ -267,92 +259,90 @@ output_cost_per_mtok = 1.85 # "The provided model identifier is invalid"), so this row needs an explicit # `api_id` confirmed against `aws bedrock list-inference-profiles` before it # ships. Re-add with: -# [models."qwen.qwen3-coder-next"] -# provider = "bedrock" +# [providers.bedrock.models."qwen3-coder-next"] # api_id = "" # display_name = "Qwen3 Coder Next (Bedrock)" # family = "qwen3" # billing_policy = "openai" # agent_profile = "openai" -# [models."qwen.qwen3-coder-next".limits] +# [providers.bedrock.models."qwen3-coder-next".limits] # context_window = 256000 # max_output = 16384 -# [models."qwen.qwen3-coder-next".features] +# [providers.bedrock.models."qwen3-coder-next".features] # tools = true -[models."moonshotai.kimi-k2.5"] -provider = "bedrock" +[providers.bedrock.models."kimi-k2.5"] +api_id = "moonshotai.kimi-k2.5" display_name = "Kimi K2.5 (Bedrock)" family = "kimi-k2" billing_policy = "openai" agent_profile = "openai" -[models."moonshotai.kimi-k2.5".limits] +[providers.bedrock.models."kimi-k2.5".limits] context_window = 262144 max_output = 16384 -[models."moonshotai.kimi-k2.5".features] +[providers.bedrock.models."kimi-k2.5".features] tools = true vision = true reasoning = false -[models."moonshotai.kimi-k2.5".costs] +[providers.bedrock.models."kimi-k2.5".costs] input_cost_per_mtok = 0.60 output_cost_per_mtok = 3.00 -[models."zai.glm-5"] -provider = "bedrock" +[providers.bedrock.models."glm-5"] +api_id = "zai.glm-5" display_name = "GLM 5 (Bedrock)" family = "glm" billing_policy = "openai" agent_profile = "openai" -[models."zai.glm-5".limits] +[providers.bedrock.models."glm-5".limits] context_window = 200000 max_output = 128000 -[models."zai.glm-5".features] +[providers.bedrock.models."glm-5".features] tools = true vision = false reasoning = false -[models."zai.glm-5".costs] +[providers.bedrock.models."glm-5".costs] input_cost_per_mtok = 1.00 output_cost_per_mtok = 3.20 -[models."minimax.minimax-m2.5"] -provider = "bedrock" +[providers.bedrock.models."minimax-m2.5"] +api_id = "minimax.minimax-m2.5" display_name = "MiniMax M2.5 (Bedrock)" family = "minimax-m2" billing_policy = "openai" agent_profile = "openai" -[models."minimax.minimax-m2.5".limits] +[providers.bedrock.models."minimax-m2.5".limits] context_window = 196000 max_output = 8192 -[models."minimax.minimax-m2.5".features] +[providers.bedrock.models."minimax-m2.5".features] tools = true vision = false reasoning = false -[models."minimax.minimax-m2.5".costs] +[providers.bedrock.models."minimax-m2.5".costs] input_cost_per_mtok = 0.30 output_cost_per_mtok = 1.20 -[models."nvidia.nemotron-3-super"] -provider = "bedrock" +[providers.bedrock.models."nemotron-3-super"] api_id = "nvidia.nemotron-super-3-120b" display_name = "Nemotron 3 Super (Bedrock)" family = "nemotron-3" billing_policy = "openai" agent_profile = "openai" -[models."nvidia.nemotron-3-super".limits] +[providers.bedrock.models."nemotron-3-super".limits] context_window = 256000 max_output = 32768 -[models."nvidia.nemotron-3-super".features] +[providers.bedrock.models."nemotron-3-super".features] tools = true vision = false reasoning = false @@ -365,24 +355,24 @@ reasoning = false # reasoning_effort stays undeclared here (requests carrying one are # rejected up front rather than silently dropped). -[models."us.anthropic.claude-fable-5"] -provider = "bedrock" +[providers.bedrock.models."claude-fable-5"] +api_id = "us.anthropic.claude-fable-5" display_name = "Claude Fable 5 (Bedrock)" family = "claude-5" billing_policy = "anthropic" -[models."us.anthropic.claude-fable-5".limits] +[providers.bedrock.models."claude-fable-5".limits] context_window = 1000000 max_output = 128000 -[models."us.anthropic.claude-fable-5".features] +[providers.bedrock.models."claude-fable-5".features] tools = true vision = true reasoning = true prompt_cache = true sampling_params = false -[models."us.anthropic.claude-fable-5".costs] +[providers.bedrock.models."claude-fable-5".costs] input_cost_per_mtok = 10.0 output_cost_per_mtok = 50.0 cache_input_cost_per_mtok = 1.0 diff --git a/lib/crates/fabro-model/src/catalog/providers/gemini.toml b/lib/crates/fabro-model/src/catalog/providers/gemini.toml index 74ffc91de..a03c2a249 100644 --- a/lib/crates/fabro-model/src/catalog/providers/gemini.toml +++ b/lib/crates/fabro-model/src/catalog/providers/gemini.toml @@ -9,9 +9,7 @@ priority = 80 credentials = ["env:GEMINI_API_KEY", "env:GOOGLE_API_KEY", "vault:GEMINI_API_KEY"] header = { custom = "x-goog-api-key" } -[models."gemini-3.1-pro-preview"] -provider = "gemini" -api_id = "gemini-3.1-pro-preview" +[providers.gemini.models."gemini-3.1-pro-preview"] display_name = "Gemini 3.1 Pro (Preview)" family = "gemini-3" training = "2025-01-01" @@ -19,24 +17,22 @@ knowledge_cutoff = "January 2025" estimated_output_tps = 85 aliases = ["gemini-pro"] -[models."gemini-3.1-pro-preview".limits] +[providers.gemini.models."gemini-3.1-pro-preview".limits] context_window = 1048576 max_output = 65536 -[models."gemini-3.1-pro-preview".features] +[providers.gemini.models."gemini-3.1-pro-preview".features] tools = true vision = true reasoning = true reasoning_effort = "levels" -[models."gemini-3.1-pro-preview".costs] +[providers.gemini.models."gemini-3.1-pro-preview".costs] input_cost_per_mtok = 2.0 output_cost_per_mtok = 12.0 cache_input_cost_per_mtok = 0.5 -[models."gemini-3.1-pro-preview-customtools"] -provider = "gemini" -api_id = "gemini-3.1-pro-preview-customtools" +[providers.gemini.models."gemini-3.1-pro-preview-customtools"] display_name = "Gemini 3.1 Pro Custom Tools (Preview)" family = "gemini-3" training = "2025-01-01" @@ -44,24 +40,22 @@ knowledge_cutoff = "January 2025" estimated_output_tps = 85 aliases = ["gemini-customtools"] -[models."gemini-3.1-pro-preview-customtools".limits] +[providers.gemini.models."gemini-3.1-pro-preview-customtools".limits] context_window = 1048576 max_output = 65536 -[models."gemini-3.1-pro-preview-customtools".features] +[providers.gemini.models."gemini-3.1-pro-preview-customtools".features] tools = true vision = true reasoning = true reasoning_effort = "levels" -[models."gemini-3.1-pro-preview-customtools".costs] +[providers.gemini.models."gemini-3.1-pro-preview-customtools".costs] input_cost_per_mtok = 2.0 output_cost_per_mtok = 12.0 cache_input_cost_per_mtok = 0.5 -[models."gemini-3.5-flash"] -provider = "gemini" -api_id = "gemini-3.5-flash" +[providers.gemini.models."gemini-3.5-flash"] display_name = "Gemini 3.5 Flash" family = "gemini-3" training = "2025-01-01" @@ -70,24 +64,22 @@ default = true estimated_output_tps = 150 aliases = ["gemini-35-flash"] -[models."gemini-3.5-flash".limits] +[providers.gemini.models."gemini-3.5-flash".limits] context_window = 1048576 max_output = 65536 -[models."gemini-3.5-flash".features] +[providers.gemini.models."gemini-3.5-flash".features] tools = true vision = true reasoning = true reasoning_effort = "levels" -[models."gemini-3.5-flash".costs] +[providers.gemini.models."gemini-3.5-flash".costs] input_cost_per_mtok = 1.5 output_cost_per_mtok = 9.0 cache_input_cost_per_mtok = 0.15 -[models."gemini-3-flash-preview"] -provider = "gemini" -api_id = "gemini-3-flash-preview" +[providers.gemini.models."gemini-3-flash-preview"] display_name = "Gemini 3 Flash (Preview)" family = "gemini-3" training = "2025-01-01" @@ -95,24 +87,22 @@ knowledge_cutoff = "January 2025" estimated_output_tps = 150 aliases = ["gemini-flash"] -[models."gemini-3-flash-preview".limits] +[providers.gemini.models."gemini-3-flash-preview".limits] context_window = 1048576 max_output = 65536 -[models."gemini-3-flash-preview".features] +[providers.gemini.models."gemini-3-flash-preview".features] tools = true vision = true reasoning = true reasoning_effort = "levels" -[models."gemini-3-flash-preview".costs] +[providers.gemini.models."gemini-3-flash-preview".costs] input_cost_per_mtok = 0.5 output_cost_per_mtok = 3.0 cache_input_cost_per_mtok = 0.125 -[models."gemini-3.1-flash-lite"] -provider = "gemini" -api_id = "gemini-3.1-flash-lite" +[providers.gemini.models."gemini-3.1-flash-lite"] display_name = "Gemini 3.1 Flash Lite" family = "gemini-3" training = "2025-01-01" @@ -121,17 +111,17 @@ estimated_output_tps = 200 aliases = ["gemini-flash-lite", "gemini-3.1-flash-lite-preview"] small_default = true -[models."gemini-3.1-flash-lite".limits] +[providers.gemini.models."gemini-3.1-flash-lite".limits] context_window = 1048576 max_output = 65536 -[models."gemini-3.1-flash-lite".features] +[providers.gemini.models."gemini-3.1-flash-lite".features] tools = true vision = true reasoning = true reasoning_effort = "levels" -[models."gemini-3.1-flash-lite".costs] +[providers.gemini.models."gemini-3.1-flash-lite".costs] input_cost_per_mtok = 0.25 output_cost_per_mtok = 1.5 cache_input_cost_per_mtok = 0.025 diff --git a/lib/crates/fabro-model/src/catalog/providers/inception.toml b/lib/crates/fabro-model/src/catalog/providers/inception.toml index 1d2b08a64..965120f27 100644 --- a/lib/crates/fabro-model/src/catalog/providers/inception.toml +++ b/lib/crates/fabro-model/src/catalog/providers/inception.toml @@ -8,25 +8,23 @@ priority = 40 [providers.inception.auth] credentials = ["env:INCEPTION_API_KEY", "vault:INCEPTION_API_KEY"] -[models."mercury-2"] -provider = "inception" -api_id = "mercury-2" +[providers.inception.models."mercury-2"] display_name = "Mercury 2" family = "mercury" default = true estimated_output_tps = 1000 aliases = ["mercury"] -[models."mercury-2".limits] +[providers.inception.models."mercury-2".limits] context_window = 131072 max_output = 50000 -[models."mercury-2".features] +[providers.inception.models."mercury-2".features] tools = true vision = false reasoning = true reasoning_effort = "levels" -[models."mercury-2".costs] +[providers.inception.models."mercury-2".costs] input_cost_per_mtok = 0.25 output_cost_per_mtok = 0.75 diff --git a/lib/crates/fabro-model/src/catalog/providers/kimi.toml b/lib/crates/fabro-model/src/catalog/providers/kimi.toml index c57779116..daa4b20c2 100644 --- a/lib/crates/fabro-model/src/catalog/providers/kimi.toml +++ b/lib/crates/fabro-model/src/catalog/providers/kimi.toml @@ -8,46 +8,42 @@ priority = 70 [providers.kimi.auth] credentials = ["env:KIMI_API_KEY", "vault:KIMI_API_KEY"] -[models."kimi-k2.5"] -provider = "kimi" -api_id = "kimi-k2.5" +[providers.kimi.models."kimi-k2.5"] display_name = "Kimi K2.5" family = "kimi-k2" training = "2025-10-01" knowledge_cutoff = "October 2025" estimated_output_tps = 50 -[models."kimi-k2.5".limits] +[providers.kimi.models."kimi-k2.5".limits] context_window = 262144 max_output = 32768 -[models."kimi-k2.5".features] +[providers.kimi.models."kimi-k2.5".features] tools = true vision = true reasoning = true prompt_cache = true sampling_params = false -[models."kimi-k2.5".costs] +[providers.kimi.models."kimi-k2.5".costs] input_cost_per_mtok = 0.6 output_cost_per_mtok = 3.0 cache_input_cost_per_mtok = 0.1 -[models."kimi-k3"] -provider = "kimi" -api_id = "kimi-k3" +[providers.kimi.models."kimi-k3"] display_name = "Kimi K3" family = "kimi-k3" default = true aliases = ["kimi"] -[models."kimi-k3".limits] +[providers.kimi.models."kimi-k3".limits] context_window = 1048576 # K3 accepts explicit completion budgets up to 1048576, but Fabro also uses # max_output as the default request budget. Match Kimi's 131072-token default. max_output = 131072 -[models."kimi-k3".features] +[providers.kimi.models."kimi-k3".features] tools = true vision = true reasoning = true @@ -55,10 +51,10 @@ reasoning_effort = "always_adaptive" prompt_cache = true sampling_params = false -[models."kimi-k3".controls] +[providers.kimi.models."kimi-k3".controls] reasoning_effort = ["low", "high", "max"] -[models."kimi-k3".costs] +[providers.kimi.models."kimi-k3".costs] input_cost_per_mtok = 3.0 output_cost_per_mtok = 15.0 cache_input_cost_per_mtok = 0.3 diff --git a/lib/crates/fabro-model/src/catalog/providers/litellm.toml b/lib/crates/fabro-model/src/catalog/providers/litellm.toml index 55f5aef15..1307378c1 100644 --- a/lib/crates/fabro-model/src/catalog/providers/litellm.toml +++ b/lib/crates/fabro-model/src/catalog/providers/litellm.toml @@ -14,18 +14,17 @@ credentials = ["env:LITELLM_API_KEY", "vault:LITELLM_API_KEY"] # enabled = true # base_url = "http://localhost:4000/v1" # -# [llm.models."litellm-gpt-5"] -# provider = "litellm" +# [llm.providers.litellm.models."litellm-gpt-5"] # api_id = "gpt-5" # display_name = "LiteLLM GPT-5" # family = "litellm" # default = true # -# [llm.models."litellm-gpt-5".limits] +# [llm.providers.litellm.models."litellm-gpt-5".limits] # context_window = 128000 # max_output = 8192 # -# [llm.models."litellm-gpt-5".features] +# [llm.providers.litellm.models."litellm-gpt-5".features] # tools = true # vision = false # reasoning = false diff --git a/lib/crates/fabro-model/src/catalog/providers/minimax.toml b/lib/crates/fabro-model/src/catalog/providers/minimax.toml index 172a6fd6a..e68dfc290 100644 --- a/lib/crates/fabro-model/src/catalog/providers/minimax.toml +++ b/lib/crates/fabro-model/src/catalog/providers/minimax.toml @@ -8,24 +8,22 @@ priority = 50 [providers.minimax.auth] credentials = ["env:MINIMAX_API_KEY", "vault:MINIMAX_API_KEY"] -[models."minimax-m2.5"] -provider = "minimax" -api_id = "minimax-m2.5" +[providers.minimax.models."minimax-m2.5"] display_name = "Minimax M2.5" family = "minimax-m2" default = true estimated_output_tps = 45 aliases = ["minimax"] -[models."minimax-m2.5".limits] +[providers.minimax.models."minimax-m2.5".limits] context_window = 196608 max_output = 16384 -[models."minimax-m2.5".features] +[providers.minimax.models."minimax-m2.5".features] tools = true vision = false reasoning = false -[models."minimax-m2.5".costs] +[providers.minimax.models."minimax-m2.5".costs] input_cost_per_mtok = 0.3 output_cost_per_mtok = 1.2 diff --git a/lib/crates/fabro-model/src/catalog/providers/ollama.toml b/lib/crates/fabro-model/src/catalog/providers/ollama.toml index bf3b16f5e..78dc5db69 100644 --- a/lib/crates/fabro-model/src/catalog/providers/ollama.toml +++ b/lib/crates/fabro-model/src/catalog/providers/ollama.toml @@ -9,18 +9,17 @@ enabled = false # Example model. Uncomment after `ollama pull qwen3.5` (and `enabled = true` # above) to expose it through the OpenAI-compatible adapter. # -# [models."qwen3.5"] -# provider = "ollama" +# [providers.ollama.models."qwen3.5"] # api_id = "qwen3.5:latest" # display_name = "Qwen3.5" # family = "qwen3.5" # default = true # aliases = ["ollama-qwen3.5"] # -# [models."qwen3.5".limits] +# [providers.ollama.models."qwen3.5".limits] # context_window = 32768 # -# [models."qwen3.5".features] +# [providers.ollama.models."qwen3.5".features] # tools = true # vision = false # reasoning = false diff --git a/lib/crates/fabro-model/src/catalog/providers/openai.toml b/lib/crates/fabro-model/src/catalog/providers/openai.toml index 56fe7a03c..10b7b2193 100644 --- a/lib/crates/fabro-model/src/catalog/providers/openai.toml +++ b/lib/crates/fabro-model/src/catalog/providers/openai.toml @@ -8,85 +8,77 @@ priority = 90 [providers.openai.auth] credentials = ["env:OPENAI_API_KEY", "vault:OPENAI_API_KEY", "vault:OPENAI_CODEX"] -[models."gpt-5.6-sol"] -provider = "openai" -api_id = "gpt-5.6-sol" +[providers.openai.models."gpt-5.6-sol"] display_name = "GPT-5.6 Sol" family = "gpt-5" training = "2026-02-16" knowledge_cutoff = "February 16, 2026" default = true -aliases = ["gpt56-sol", "gpt-56-sol", "gpt-5.6", "gpt56", "gpt-56"] +aliases = ["sol", "gpt56-sol", "gpt-56-sol", "gpt-5.6", "gpt56", "gpt-56"] -[models."gpt-5.6-sol".limits] +[providers.openai.models."gpt-5.6-sol".limits] context_window = 272000 max_output = 128000 -[models."gpt-5.6-sol".features] +[providers.openai.models."gpt-5.6-sol".features] tools = true vision = true reasoning = true reasoning_effort = "levels" prompt_cache = true -[models."gpt-5.6-sol".costs] +[providers.openai.models."gpt-5.6-sol".costs] input_cost_per_mtok = 5.0 output_cost_per_mtok = 30.0 cache_input_cost_per_mtok = 0.5 -[models."gpt-5.6-terra"] -provider = "openai" -api_id = "gpt-5.6-terra" +[providers.openai.models."gpt-5.6-terra"] display_name = "GPT-5.6 Terra" family = "gpt-5" training = "2026-02-16" knowledge_cutoff = "February 16, 2026" -aliases = ["gpt56-terra", "gpt-56-terra"] +aliases = ["terra", "gpt56-terra", "gpt-56-terra"] -[models."gpt-5.6-terra".limits] +[providers.openai.models."gpt-5.6-terra".limits] context_window = 272000 max_output = 128000 -[models."gpt-5.6-terra".features] +[providers.openai.models."gpt-5.6-terra".features] tools = true vision = true reasoning = true reasoning_effort = "levels" prompt_cache = true -[models."gpt-5.6-terra".costs] +[providers.openai.models."gpt-5.6-terra".costs] input_cost_per_mtok = 2.5 output_cost_per_mtok = 15.0 cache_input_cost_per_mtok = 0.25 -[models."gpt-5.6-luna"] -provider = "openai" -api_id = "gpt-5.6-luna" +[providers.openai.models."gpt-5.6-luna"] display_name = "GPT-5.6 Luna" family = "gpt-5" training = "2026-02-16" knowledge_cutoff = "February 16, 2026" -aliases = ["gpt56-luna", "gpt-56-luna"] +aliases = ["luna", "gpt56-luna", "gpt-56-luna"] -[models."gpt-5.6-luna".limits] +[providers.openai.models."gpt-5.6-luna".limits] context_window = 272000 max_output = 128000 -[models."gpt-5.6-luna".features] +[providers.openai.models."gpt-5.6-luna".features] tools = true vision = true reasoning = true reasoning_effort = "levels" prompt_cache = true -[models."gpt-5.6-luna".costs] +[providers.openai.models."gpt-5.6-luna".costs] input_cost_per_mtok = 1.0 output_cost_per_mtok = 6.0 cache_input_cost_per_mtok = 0.1 -[models."gpt-5.4"] -provider = "openai" -api_id = "gpt-5.4" +[providers.openai.models."gpt-5.4"] display_name = "GPT-5.4" family = "gpt-5" training = "2025-08-31" @@ -94,24 +86,22 @@ knowledge_cutoff = "April 2025" estimated_output_tps = 70 aliases = ["gpt54", "gpt-54", "gpt-5.2", "gpt5", "gpt-5.3-codex", "codex"] -[models."gpt-5.4".limits] +[providers.openai.models."gpt-5.4".limits] context_window = 272000 max_output = 128000 -[models."gpt-5.4".features] +[providers.openai.models."gpt-5.4".features] tools = true vision = true reasoning = true reasoning_effort = "levels" -[models."gpt-5.4".costs] +[providers.openai.models."gpt-5.4".costs] input_cost_per_mtok = 2.5 output_cost_per_mtok = 15.0 cache_input_cost_per_mtok = 0.25 -[models."gpt-5.5"] -provider = "openai" -api_id = "gpt-5.5" +[providers.openai.models."gpt-5.5"] display_name = "GPT-5.5" family = "gpt-5" training = "2025-12-01" @@ -119,24 +109,22 @@ knowledge_cutoff = "December 2025" estimated_output_tps = 70 aliases = ["gpt55", "gpt-55"] -[models."gpt-5.5".limits] +[providers.openai.models."gpt-5.5".limits] context_window = 272000 max_output = 128000 -[models."gpt-5.5".features] +[providers.openai.models."gpt-5.5".features] tools = true vision = true reasoning = true reasoning_effort = "levels" -[models."gpt-5.5".costs] +[providers.openai.models."gpt-5.5".costs] input_cost_per_mtok = 5.0 output_cost_per_mtok = 30.0 cache_input_cost_per_mtok = 0.5 -[models."gpt-5.5-pro"] -provider = "openai" -api_id = "gpt-5.5-pro" +[providers.openai.models."gpt-5.5-pro"] display_name = "GPT-5.5 Pro" family = "gpt-5" training = "2025-12-01" @@ -144,24 +132,22 @@ knowledge_cutoff = "December 2025" estimated_output_tps = 20 aliases = ["gpt55-pro", "gpt-55-pro"] -[models."gpt-5.5-pro".limits] +[providers.openai.models."gpt-5.5-pro".limits] context_window = 1050000 max_output = 128000 -[models."gpt-5.5-pro".features] +[providers.openai.models."gpt-5.5-pro".features] tools = true vision = true reasoning = true reasoning_effort = "levels" -[models."gpt-5.5-pro".costs] +[providers.openai.models."gpt-5.5-pro".costs] input_cost_per_mtok = 30.0 output_cost_per_mtok = 180.0 cache_input_cost_per_mtok = 3.0 -[models."gpt-5.4-pro"] -provider = "openai" -api_id = "gpt-5.4-pro" +[providers.openai.models."gpt-5.4-pro"] display_name = "GPT-5.4 Pro" family = "gpt-5" training = "2025-08-31" @@ -169,24 +155,22 @@ knowledge_cutoff = "April 2025" estimated_output_tps = 20 aliases = ["gpt54-pro", "gpt-54-pro"] -[models."gpt-5.4-pro".limits] +[providers.openai.models."gpt-5.4-pro".limits] context_window = 1047576 max_output = 128000 -[models."gpt-5.4-pro".features] +[providers.openai.models."gpt-5.4-pro".features] tools = true vision = true reasoning = true reasoning_effort = "levels" -[models."gpt-5.4-pro".costs] +[providers.openai.models."gpt-5.4-pro".costs] input_cost_per_mtok = 30.0 output_cost_per_mtok = 180.0 cache_input_cost_per_mtok = 3.0 -[models."gpt-5.4-mini"] -provider = "openai" -api_id = "gpt-5.4-mini" +[providers.openai.models."gpt-5.4-mini"] display_name = "GPT-5.4 Mini" family = "gpt-5" training = "2025-08-31" @@ -196,17 +180,17 @@ aliases = ["gpt54-mini", "gpt-54-mini", "gpt-5.3-codex-spark", "codex-spark"] probe = true small_default = true -[models."gpt-5.4-mini".limits] +[providers.openai.models."gpt-5.4-mini".limits] context_window = 272000 max_output = 128000 -[models."gpt-5.4-mini".features] +[providers.openai.models."gpt-5.4-mini".features] tools = true vision = true reasoning = true reasoning_effort = "levels" -[models."gpt-5.4-mini".costs] +[providers.openai.models."gpt-5.4-mini".costs] input_cost_per_mtok = 0.75 output_cost_per_mtok = 4.5 cache_input_cost_per_mtok = 0.075 diff --git a/lib/crates/fabro-model/src/catalog/providers/openrouter.toml b/lib/crates/fabro-model/src/catalog/providers/openrouter.toml index d01fcb3dc..4ef7d7c63 100644 --- a/lib/crates/fabro-model/src/catalog/providers/openrouter.toml +++ b/lib/crates/fabro-model/src/catalog/providers/openrouter.toml @@ -33,18 +33,18 @@ credentials = ["env:OPENROUTER_API_KEY", "vault:OPENROUTER_API_KEY"] # best-effort estimates; OpenRouter returns the authoritative usage.cost # in-band on every response. -[models."anthropic/claude-fable-5"] -provider = "openrouter" +[providers.openrouter.models."claude-fable-5"] api_id = "anthropic/claude-fable-5" display_name = "Claude Fable 5 (via OpenRouter)" family = "claude-5" billing_policy = "anthropic" +aliases = ["fable", "claude-fable"] -[models."anthropic/claude-fable-5".limits] +[providers.openrouter.models."claude-fable-5".limits] context_window = 1000000 max_output = 128000 -[models."anthropic/claude-fable-5".features] +[providers.openrouter.models."claude-fable-5".features] tools = true vision = true reasoning = true @@ -52,26 +52,25 @@ reasoning_effort = "always_adaptive" prompt_cache = true sampling_params = false -[models."anthropic/claude-fable-5".costs] +[providers.openrouter.models."claude-fable-5".costs] input_cost_per_mtok = 10.0 output_cost_per_mtok = 50.0 cache_input_cost_per_mtok = 1.0 -[models."anthropic/claude-opus-4-8"] -provider = "openrouter" +[providers.openrouter.models."claude-opus-4-8"] api_id = "anthropic/claude-opus-4.8" display_name = "Claude Opus 4.8 (via OpenRouter)" family = "claude-4" billing_policy = "anthropic" training = "2026-01-01" knowledge_cutoff = "Jan 2026" -aliases = ["anthropic/claude-opus-4.8"] +aliases = ["opus", "claude-opus"] -[models."anthropic/claude-opus-4-8".limits] +[providers.openrouter.models."claude-opus-4-8".limits] context_window = 1000000 max_output = 128000 -[models."anthropic/claude-opus-4-8".features] +[providers.openrouter.models."claude-opus-4-8".features] tools = true vision = true reasoning = true @@ -79,75 +78,74 @@ reasoning_effort = "levels" prompt_cache = true sampling_params = false -[models."anthropic/claude-opus-4-8".costs] +[providers.openrouter.models."claude-opus-4-8".costs] input_cost_per_mtok = 5.0 output_cost_per_mtok = 25.0 cache_input_cost_per_mtok = 0.5 -[models."anthropic/claude-opus-4-7"] -provider = "openrouter" +[providers.openrouter.models."claude-opus-4-7"] api_id = "anthropic/claude-opus-4.7" display_name = "Claude Opus 4.7 (via OpenRouter)" family = "claude-4" billing_policy = "anthropic" -[models."anthropic/claude-opus-4-7".limits] +[providers.openrouter.models."claude-opus-4-7".limits] context_window = 1000000 max_output = 128000 -[models."anthropic/claude-opus-4-7".features] +[providers.openrouter.models."claude-opus-4-7".features] tools = true vision = true reasoning = true prompt_cache = true -[models."anthropic/claude-opus-4-7".costs] +[providers.openrouter.models."claude-opus-4-7".costs] input_cost_per_mtok = 5.0 output_cost_per_mtok = 25.0 cache_input_cost_per_mtok = 0.5 -[models."anthropic/claude-sonnet-4-6"] -provider = "openrouter" +[providers.openrouter.models."claude-sonnet-4-6"] api_id = "anthropic/claude-sonnet-4.6" display_name = "Claude Sonnet 4.6 (via OpenRouter)" family = "claude-4" billing_policy = "anthropic" default = true +aliases = ["sonnet", "claude-sonnet"] -[models."anthropic/claude-sonnet-4-6".limits] +[providers.openrouter.models."claude-sonnet-4-6".limits] context_window = 1000000 max_output = 64000 -[models."anthropic/claude-sonnet-4-6".features] +[providers.openrouter.models."claude-sonnet-4-6".features] tools = true vision = true reasoning = true prompt_cache = true -[models."anthropic/claude-sonnet-4-6".costs] +[providers.openrouter.models."claude-sonnet-4-6".costs] input_cost_per_mtok = 3.0 output_cost_per_mtok = 15.0 cache_input_cost_per_mtok = 0.3 -[models."anthropic/claude-haiku-4-5"] -provider = "openrouter" +[providers.openrouter.models."claude-haiku-4-5"] api_id = "anthropic/claude-haiku-4.5" display_name = "Claude Haiku 4.5 (via OpenRouter)" family = "claude-4" billing_policy = "anthropic" small_default = true +aliases = ["haiku", "claude-haiku"] -[models."anthropic/claude-haiku-4-5".limits] +[providers.openrouter.models."claude-haiku-4-5".limits] context_window = 200000 max_output = 8192 -[models."anthropic/claude-haiku-4-5".features] +[providers.openrouter.models."claude-haiku-4-5".features] tools = true vision = true reasoning = false prompt_cache = true -[models."anthropic/claude-haiku-4-5".costs] +[providers.openrouter.models."claude-haiku-4-5".costs] input_cost_per_mtok = 1.0 output_cost_per_mtok = 5.0 cache_input_cost_per_mtok = 0.1 @@ -158,19 +156,19 @@ cache_input_cost_per_mtok = 0.1 # long-context threshold. Authoritative in-band usage.cost covers # long-context and cache-write pricing. -[models."openai/gpt-5.6-sol"] -provider = "openrouter" +[providers.openrouter.models."gpt-5.6-sol"] api_id = "openai/gpt-5.6-sol" display_name = "GPT-5.6 Sol (via OpenRouter)" family = "gpt-5" training = "2026-02-16" knowledge_cutoff = "February 16, 2026" +aliases = ["sol", "gpt56-sol", "gpt-56-sol", "gpt-5.6", "gpt56", "gpt-56"] -[models."openai/gpt-5.6-sol".limits] +[providers.openrouter.models."gpt-5.6-sol".limits] context_window = 1050000 max_output = 128000 -[models."openai/gpt-5.6-sol".features] +[providers.openrouter.models."gpt-5.6-sol".features] tools = true vision = true reasoning = true @@ -178,24 +176,24 @@ reasoning_effort = "levels" prompt_cache = true sampling_params = false -[models."openai/gpt-5.6-sol".costs] +[providers.openrouter.models."gpt-5.6-sol".costs] input_cost_per_mtok = 5.0 output_cost_per_mtok = 30.0 cache_input_cost_per_mtok = 0.5 -[models."openai/gpt-5.6-terra"] -provider = "openrouter" +[providers.openrouter.models."gpt-5.6-terra"] api_id = "openai/gpt-5.6-terra" display_name = "GPT-5.6 Terra (via OpenRouter)" family = "gpt-5" training = "2026-02-16" knowledge_cutoff = "February 16, 2026" +aliases = ["terra", "gpt56-terra", "gpt-56-terra"] -[models."openai/gpt-5.6-terra".limits] +[providers.openrouter.models."gpt-5.6-terra".limits] context_window = 1050000 max_output = 128000 -[models."openai/gpt-5.6-terra".features] +[providers.openrouter.models."gpt-5.6-terra".features] tools = true vision = true reasoning = true @@ -203,24 +201,24 @@ reasoning_effort = "levels" prompt_cache = true sampling_params = false -[models."openai/gpt-5.6-terra".costs] +[providers.openrouter.models."gpt-5.6-terra".costs] input_cost_per_mtok = 2.5 output_cost_per_mtok = 15.0 cache_input_cost_per_mtok = 0.25 -[models."openai/gpt-5.6-luna"] -provider = "openrouter" +[providers.openrouter.models."gpt-5.6-luna"] api_id = "openai/gpt-5.6-luna" display_name = "GPT-5.6 Luna (via OpenRouter)" family = "gpt-5" training = "2026-02-16" knowledge_cutoff = "February 16, 2026" +aliases = ["luna", "gpt56-luna", "gpt-56-luna"] -[models."openai/gpt-5.6-luna".limits] +[providers.openrouter.models."gpt-5.6-luna".limits] context_window = 1050000 max_output = 128000 -[models."openai/gpt-5.6-luna".features] +[providers.openrouter.models."gpt-5.6-luna".features] tools = true vision = true reasoning = true @@ -228,197 +226,189 @@ reasoning_effort = "levels" prompt_cache = true sampling_params = false -[models."openai/gpt-5.6-luna".costs] +[providers.openrouter.models."gpt-5.6-luna".costs] input_cost_per_mtok = 1.0 output_cost_per_mtok = 6.0 cache_input_cost_per_mtok = 0.1 -[models."openai/gpt-5.4"] -provider = "openrouter" +[providers.openrouter.models."gpt-5.4"] api_id = "openai/gpt-5.4" display_name = "GPT-5.4 (via OpenRouter)" family = "gpt-5" +aliases = ["gpt54", "gpt-54", "gpt-5.2", "gpt5", "gpt-5.3-codex", "codex"] -[models."openai/gpt-5.4".limits] +[providers.openrouter.models."gpt-5.4".limits] context_window = 1050000 max_output = 32768 -[models."openai/gpt-5.4".features] +[providers.openrouter.models."gpt-5.4".features] tools = true vision = true reasoning = true -[models."openai/gpt-5.4".costs] +[providers.openrouter.models."gpt-5.4".costs] input_cost_per_mtok = 2.5 output_cost_per_mtok = 15.0 -[models."openai/gpt-5.5"] -provider = "openrouter" +[providers.openrouter.models."gpt-5.5"] api_id = "openai/gpt-5.5" display_name = "GPT-5.5 (via OpenRouter)" family = "gpt-5" +aliases = ["gpt55", "gpt-55"] -[models."openai/gpt-5.5".limits] +[providers.openrouter.models."gpt-5.5".limits] context_window = 1050000 max_output = 32768 -[models."openai/gpt-5.5".features] +[providers.openrouter.models."gpt-5.5".features] tools = true vision = true reasoning = true -[models."openai/gpt-5.5".costs] +[providers.openrouter.models."gpt-5.5".costs] input_cost_per_mtok = 5.0 output_cost_per_mtok = 30.0 # ---------- Google Gemini via OpenRouter ---------- -[models."google/gemini-3.1-pro-preview"] -provider = "openrouter" +[providers.openrouter.models."gemini-3.1-pro-preview"] api_id = "google/gemini-3.1-pro-preview" display_name = "Gemini 3.1 Pro Preview (via OpenRouter)" family = "gemini-3" -[models."google/gemini-3.1-pro-preview".limits] +[providers.openrouter.models."gemini-3.1-pro-preview".limits] context_window = 1048576 max_output = 65536 -[models."google/gemini-3.1-pro-preview".features] +[providers.openrouter.models."gemini-3.1-pro-preview".features] tools = true vision = true reasoning = true -[models."google/gemini-3.1-pro-preview".costs] +[providers.openrouter.models."gemini-3.1-pro-preview".costs] input_cost_per_mtok = 2.0 output_cost_per_mtok = 12.0 -[models."google/gemini-3.5-flash"] -provider = "openrouter" +[providers.openrouter.models."gemini-3.5-flash"] api_id = "google/gemini-3.5-flash" display_name = "Gemini 3.5 Flash (via OpenRouter)" family = "gemini-3" -[models."google/gemini-3.5-flash".limits] +[providers.openrouter.models."gemini-3.5-flash".limits] context_window = 1048576 max_output = 65536 -[models."google/gemini-3.5-flash".features] +[providers.openrouter.models."gemini-3.5-flash".features] tools = true vision = true reasoning = false -[models."google/gemini-3.5-flash".costs] +[providers.openrouter.models."gemini-3.5-flash".costs] input_cost_per_mtok = 1.5 output_cost_per_mtok = 9.0 # ---------- Open-weights models ---------- -[models."xiaomi/mimo-v2.5-pro"] -provider = "openrouter" +[providers.openrouter.models."mimo-v2.5-pro"] api_id = "xiaomi/mimo-v2.5-pro" display_name = "Xiaomi MiMo v2.5 Pro" family = "mimo-v2" -[models."xiaomi/mimo-v2.5-pro".limits] +[providers.openrouter.models."mimo-v2.5-pro".limits] context_window = 1050000 max_output = 16384 -[models."xiaomi/mimo-v2.5-pro".features] +[providers.openrouter.models."mimo-v2.5-pro".features] tools = true vision = false reasoning = false -[models."xiaomi/mimo-v2.5-pro".costs] +[providers.openrouter.models."mimo-v2.5-pro".costs] input_cost_per_mtok = 0.435 output_cost_per_mtok = 0.87 -[models."minimax/minimax-m2.7"] -provider = "openrouter" +[providers.openrouter.models."minimax-m2.7"] api_id = "minimax/minimax-m2.7" display_name = "MiniMax M2.7" family = "minimax-m2" -[models."minimax/minimax-m2.7".limits] +[providers.openrouter.models."minimax-m2.7".limits] context_window = 200000 max_output = 16384 -[models."minimax/minimax-m2.7".features] +[providers.openrouter.models."minimax-m2.7".features] tools = true vision = false reasoning = false -[models."minimax/minimax-m2.7".costs] +[providers.openrouter.models."minimax-m2.7".costs] input_cost_per_mtok = 0.28 output_cost_per_mtok = 1.20 -[models."deepseek/deepseek-v4-pro"] -provider = "openrouter" +[providers.openrouter.models."deepseek-v4-pro"] api_id = "deepseek/deepseek-v4-pro" display_name = "DeepSeek V4 Pro" family = "deepseek-v4" -[models."deepseek/deepseek-v4-pro".limits] +[providers.openrouter.models."deepseek-v4-pro".limits] context_window = 1050000 max_output = 16384 -[models."deepseek/deepseek-v4-pro".features] +[providers.openrouter.models."deepseek-v4-pro".features] tools = true vision = false reasoning = true -[models."deepseek/deepseek-v4-pro".costs] +[providers.openrouter.models."deepseek-v4-pro".costs] input_cost_per_mtok = 0.435 output_cost_per_mtok = 0.87 -[models."deepseek/deepseek-v4-flash"] -provider = "openrouter" +[providers.openrouter.models."deepseek-v4-flash"] api_id = "deepseek/deepseek-v4-flash" display_name = "DeepSeek V4 Flash" family = "deepseek-v4" -[models."deepseek/deepseek-v4-flash".limits] +[providers.openrouter.models."deepseek-v4-flash".limits] context_window = 1050000 max_output = 16384 -[models."deepseek/deepseek-v4-flash".features] +[providers.openrouter.models."deepseek-v4-flash".features] tools = true vision = false reasoning = false -[models."deepseek/deepseek-v4-flash".costs] +[providers.openrouter.models."deepseek-v4-flash".costs] input_cost_per_mtok = 0.10 output_cost_per_mtok = 0.20 -[models."moonshotai/kimi-k2.6"] -provider = "openrouter" +[providers.openrouter.models."kimi-k2.6"] api_id = "moonshotai/kimi-k2.6" display_name = "Kimi K2.6" family = "kimi-k2" -[models."moonshotai/kimi-k2.6".limits] +[providers.openrouter.models."kimi-k2.6".limits] context_window = 262144 max_output = 16384 -[models."moonshotai/kimi-k2.6".features] +[providers.openrouter.models."kimi-k2.6".features] tools = true vision = false reasoning = false -[models."moonshotai/kimi-k2.6".costs] +[providers.openrouter.models."kimi-k2.6".costs] input_cost_per_mtok = 0.73 output_cost_per_mtok = 3.49 -[models."moonshotai/kimi-k3"] -provider = "openrouter" +[providers.openrouter.models."kimi-k3"] api_id = "moonshotai/kimi-k3" display_name = "Kimi K3 (via OpenRouter)" family = "kimi-k3" -[models."moonshotai/kimi-k3".limits] +[providers.openrouter.models."kimi-k3".limits] context_window = 1048576 max_output = 131072 -[models."moonshotai/kimi-k3".features] +[providers.openrouter.models."kimi-k3".features] tools = true vision = true reasoning = true @@ -426,47 +416,45 @@ reasoning_effort = "always_adaptive" prompt_cache = true sampling_params = false -[models."moonshotai/kimi-k3".controls] +[providers.openrouter.models."kimi-k3".controls] reasoning_effort = ["low", "high", "max"] -[models."moonshotai/kimi-k3".costs] +[providers.openrouter.models."kimi-k3".costs] input_cost_per_mtok = 3.0 output_cost_per_mtok = 15.0 cache_input_cost_per_mtok = 0.3 -[models."poolside/laguna-s-2.1"] -provider = "openrouter" +[providers.openrouter.models."laguna-s-2.1"] api_id = "poolside/laguna-s-2.1" display_name = "Laguna S 2.1 (via OpenRouter)" family = "laguna-2" -[models."poolside/laguna-s-2.1".limits] +[providers.openrouter.models."laguna-s-2.1".limits] context_window = 1048576 max_output = 131072 -[models."poolside/laguna-s-2.1".features] +[providers.openrouter.models."laguna-s-2.1".features] tools = true vision = false reasoning = true prompt_cache = true sampling_params = true -[models."poolside/laguna-s-2.1".costs] +[providers.openrouter.models."laguna-s-2.1".costs] input_cost_per_mtok = 0.10 output_cost_per_mtok = 0.20 cache_input_cost_per_mtok = 0.01 -[models."poolside/laguna-xs-2.1"] -provider = "openrouter" +[providers.openrouter.models."laguna-xs-2.1"] api_id = "poolside/laguna-xs-2.1" display_name = "Laguna XS 2.1 (via OpenRouter)" family = "laguna-2" -[models."poolside/laguna-xs-2.1".limits] +[providers.openrouter.models."laguna-xs-2.1".limits] context_window = 262144 max_output = 32768 -[models."poolside/laguna-xs-2.1".features] +[providers.openrouter.models."laguna-xs-2.1".features] tools = true vision = false reasoning = true @@ -475,127 +463,121 @@ sampling_params = true # Current promotional rate. OpenRouter's authoritative in-band usage.cost # supersedes this estimate on completed responses. -[models."poolside/laguna-xs-2.1".costs] +[providers.openrouter.models."laguna-xs-2.1".costs] input_cost_per_mtok = 0.06 output_cost_per_mtok = 0.12 cache_input_cost_per_mtok = 0.03 -[models."qwen/qwen3-coder"] -provider = "openrouter" +[providers.openrouter.models."qwen3-coder"] api_id = "qwen/qwen3-coder" display_name = "Qwen3 Coder" family = "qwen3" -[models."qwen/qwen3-coder".limits] +[providers.openrouter.models."qwen3-coder".limits] context_window = 1050000 max_output = 16384 -[models."qwen/qwen3-coder".features] +[providers.openrouter.models."qwen3-coder".features] tools = true vision = false reasoning = false -[models."qwen/qwen3-coder".costs] +[providers.openrouter.models."qwen3-coder".costs] input_cost_per_mtok = 0.22 output_cost_per_mtok = 1.80 -[models."qwen/qwen3.6-flash"] -provider = "openrouter" +[providers.openrouter.models."qwen3.6-flash"] api_id = "qwen/qwen3.6-flash" display_name = "Qwen3.6 Flash" family = "qwen3" -[models."qwen/qwen3.6-flash".limits] +[providers.openrouter.models."qwen3.6-flash".limits] context_window = 1000000 max_output = 16384 -[models."qwen/qwen3.6-flash".features] +[providers.openrouter.models."qwen3.6-flash".features] tools = true vision = false reasoning = false -[models."qwen/qwen3.6-flash".costs] +[providers.openrouter.models."qwen3.6-flash".costs] input_cost_per_mtok = 0.1875 output_cost_per_mtok = 1.125 -[models."z-ai/glm-5.2"] -provider = "openrouter" +[providers.openrouter.models."glm-5.2"] api_id = "z-ai/glm-5.2" display_name = "GLM 5.2 (via OpenRouter)" family = "glm-5" -[models."z-ai/glm-5.2".limits] +[providers.openrouter.models."glm-5.2".limits] context_window = 1048576 max_output = 131072 -[models."z-ai/glm-5.2".features] +[providers.openrouter.models."glm-5.2".features] tools = true vision = false reasoning = true reasoning_effort = "levels" prompt_cache = true -[models."z-ai/glm-5.2".controls] +[providers.openrouter.models."glm-5.2".controls] reasoning_effort = ["high", "xhigh"] -[models."z-ai/glm-5.2".costs] +[providers.openrouter.models."glm-5.2".costs] input_cost_per_mtok = 0.784 output_cost_per_mtok = 2.464 cache_input_cost_per_mtok = 0.1456 -[models."z-ai/glm-4.6"] -provider = "openrouter" +[providers.openrouter.models."glm-4.6"] api_id = "z-ai/glm-4.6" display_name = "GLM 4.6" family = "glm-4" -[models."z-ai/glm-4.6".limits] +[providers.openrouter.models."glm-4.6".limits] context_window = 203000 max_output = 16384 -[models."z-ai/glm-4.6".features] +[providers.openrouter.models."glm-4.6".features] tools = true vision = false reasoning = false -[models."z-ai/glm-4.6".costs] +[providers.openrouter.models."glm-4.6".costs] input_cost_per_mtok = 0.43 output_cost_per_mtok = 1.74 -[models."nvidia/nemotron-3-super-120b-a12b"] -provider = "openrouter" +[providers.openrouter.models."nemotron-3-super-120b-a12b"] api_id = "nvidia/nemotron-3-super-120b-a12b" display_name = "NVIDIA Nemotron 3 Super 120B" family = "nemotron-3" -[models."nvidia/nemotron-3-super-120b-a12b".limits] +[providers.openrouter.models."nemotron-3-super-120b-a12b".limits] context_window = 1000000 max_output = 16384 -[models."nvidia/nemotron-3-super-120b-a12b".features] +[providers.openrouter.models."nemotron-3-super-120b-a12b".features] tools = true vision = false reasoning = false -[models."nvidia/nemotron-3-super-120b-a12b".costs] +[providers.openrouter.models."nemotron-3-super-120b-a12b".costs] input_cost_per_mtok = 0.09 output_cost_per_mtok = 0.45 -[models."mistralai/devstral-2512"] -provider = "openrouter" +[providers.openrouter.models."devstral-2512"] api_id = "mistralai/devstral-2512" display_name = "Devstral 2512" family = "devstral" -[models."mistralai/devstral-2512".limits] +[providers.openrouter.models."devstral-2512".limits] context_window = 262144 max_output = 16384 -[models."mistralai/devstral-2512".features] +[providers.openrouter.models."devstral-2512".features] tools = true vision = false reasoning = false -[models."mistralai/devstral-2512".costs] +[providers.openrouter.models."devstral-2512".costs] input_cost_per_mtok = 0.40 output_cost_per_mtok = 2.00 diff --git a/lib/crates/fabro-model/src/catalog/providers/poolside.toml b/lib/crates/fabro-model/src/catalog/providers/poolside.toml index 8fd1b6855..65ce87246 100644 --- a/lib/crates/fabro-model/src/catalog/providers/poolside.toml +++ b/lib/crates/fabro-model/src/catalog/providers/poolside.toml @@ -8,19 +8,18 @@ priority = 65 [providers.poolside.auth] credentials = ["env:POOLSIDE_API_KEY", "vault:POOLSIDE_API_KEY"] -[models."laguna-s-2.1"] -provider = "poolside" +[providers.poolside.models."laguna-s-2.1"] api_id = "poolside/laguna-s-2.1" display_name = "Laguna S 2.1" family = "laguna-2" default = true aliases = ["laguna", "laguna-s"] -[models."laguna-s-2.1".limits] +[providers.poolside.models."laguna-s-2.1".limits] context_window = 1048576 max_output = 131072 -[models."laguna-s-2.1".features] +[providers.poolside.models."laguna-s-2.1".features] tools = true vision = false reasoning = true @@ -30,13 +29,12 @@ sampling_params = true # Poolside Platform is free for a limited preview period. Keep the published # paid hosted rate as Fabro's durable estimate for paid access and post-preview # usage. -[models."laguna-s-2.1".costs] +[providers.poolside.models."laguna-s-2.1".costs] input_cost_per_mtok = 0.10 output_cost_per_mtok = 0.20 cache_input_cost_per_mtok = 0.01 -[models."laguna-xs-2.1"] -provider = "poolside" +[providers.poolside.models."laguna-xs-2.1"] api_id = "poolside/laguna-xs-2.1" display_name = "Laguna XS 2.1" family = "laguna-2" @@ -44,11 +42,11 @@ small_default = true probe = true aliases = ["laguna-xs"] -[models."laguna-xs-2.1".limits] +[providers.poolside.models."laguna-xs-2.1".limits] context_window = 262144 max_output = 32768 -[models."laguna-xs-2.1".features] +[providers.poolside.models."laguna-xs-2.1".features] tools = true vision = false reasoning = true @@ -57,7 +55,7 @@ sampling_params = true # Poolside Platform is free for a limited preview period. These are Poolside's # published paid endpoint rates. -[models."laguna-xs-2.1".costs] +[providers.poolside.models."laguna-xs-2.1".costs] input_cost_per_mtok = 0.10 output_cost_per_mtok = 0.20 cache_input_cost_per_mtok = 0.05 diff --git a/lib/crates/fabro-model/src/catalog/providers/venice.toml b/lib/crates/fabro-model/src/catalog/providers/venice.toml index bb91aeada..dc97c4ab4 100644 --- a/lib/crates/fabro-model/src/catalog/providers/venice.toml +++ b/lib/crates/fabro-model/src/catalog/providers/venice.toml @@ -8,43 +8,39 @@ aliases = ["venice-ai"] [providers.venice.auth] credentials = ["env:VENICE_API_KEY", "vault:VENICE_API_KEY"] -[models."venice-uncensored-1-2"] -provider = "venice" -api_id = "venice-uncensored-1-2" +[providers.venice.models."venice-uncensored-1-2"] display_name = "Venice Uncensored 1.2" family = "venice-uncensored" default = true aliases = ["venice-uncensored", "vu"] -[models."venice-uncensored-1-2".limits] +[providers.venice.models."venice-uncensored-1-2".limits] context_window = 128000 max_output = 8192 -[models."venice-uncensored-1-2".features] +[providers.venice.models."venice-uncensored-1-2".features] tools = true vision = true reasoning = false -[models."venice-uncensored-1-2".costs] +[providers.venice.models."venice-uncensored-1-2".costs] input_cost_per_mtok = 0.2 output_cost_per_mtok = 0.9 -[models."venice-uncensored-role-play"] -provider = "venice" -api_id = "venice-uncensored-role-play" +[providers.venice.models."venice-uncensored-role-play"] display_name = "Venice Uncensored Role Play" family = "venice-uncensored" aliases = ["venice-roleplay", "vrp"] -[models."venice-uncensored-role-play".limits] +[providers.venice.models."venice-uncensored-role-play".limits] context_window = 128000 max_output = 4096 -[models."venice-uncensored-role-play".features] +[providers.venice.models."venice-uncensored-role-play".features] tools = true vision = true reasoning = false -[models."venice-uncensored-role-play".costs] +[providers.venice.models."venice-uncensored-role-play".costs] input_cost_per_mtok = 0.5 output_cost_per_mtok = 2.0 diff --git a/lib/crates/fabro-model/src/catalog/providers/zai.toml b/lib/crates/fabro-model/src/catalog/providers/zai.toml index e720df4da..c6d70cd64 100644 --- a/lib/crates/fabro-model/src/catalog/providers/zai.toml +++ b/lib/crates/fabro-model/src/catalog/providers/zai.toml @@ -8,50 +8,46 @@ priority = 60 [providers.zai.auth] credentials = ["env:ZAI_API_KEY", "vault:ZAI_API_KEY"] -[models."glm-5.2"] -provider = "zai" -api_id = "glm-5.2" +[providers.zai.models."glm-5.2"] display_name = "GLM 5.2" family = "glm-5" default = true aliases = ["glm", "glm5"] -[models."glm-5.2".limits] +[providers.zai.models."glm-5.2".limits] context_window = 1048576 max_output = 131072 -[models."glm-5.2".features] +[providers.zai.models."glm-5.2".features] tools = true vision = false reasoning = true reasoning_effort = "levels" prompt_cache = true -[models."glm-5.2".controls] +[providers.zai.models."glm-5.2".controls] reasoning_effort = ["high", "max"] -[models."glm-5.2".costs] +[providers.zai.models."glm-5.2".costs] input_cost_per_mtok = 1.4 output_cost_per_mtok = 4.4 cache_input_cost_per_mtok = 0.26 -[models."glm-4.7"] -provider = "zai" -api_id = "glm-4.7" +[providers.zai.models."glm-4.7"] display_name = "GLM 4.7" family = "glm-4" estimated_output_tps = 100 aliases = ["glm4"] -[models."glm-4.7".limits] +[providers.zai.models."glm-4.7".limits] context_window = 202752 max_output = 16384 -[models."glm-4.7".features] +[providers.zai.models."glm-4.7".features] tools = true vision = false reasoning = false -[models."glm-4.7".costs] +[providers.zai.models."glm-4.7".costs] input_cost_per_mtok = 0.6 output_cost_per_mtok = 2.2 diff --git a/lib/crates/fabro-model/src/ids.rs b/lib/crates/fabro-model/src/ids.rs index 18d1bdd86..3429ae094 100644 --- a/lib/crates/fabro-model/src/ids.rs +++ b/lib/crates/fabro-model/src/ids.rs @@ -101,9 +101,12 @@ impl AsRef for ProviderId { } } -/// Stable model identifier — either the canonical catalog ID or one of its -/// declared aliases. -#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)] +/// Stable canonical, human-facing model identifier. +/// +/// Aliases are alternate selectors for a model offering; they are not model +/// IDs. The same `ModelId` may be offered by more than one provider, so a +/// concrete catalog offering is identified by `(ProviderId, ModelId)`. +#[derive(Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)] #[serde(transparent)] pub struct ModelId(String); @@ -129,6 +132,12 @@ impl fmt::Display for ModelId { } } +impl fmt::Debug for ModelId { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.0.fmt(f) + } +} + impl From<&str> for ModelId { fn from(s: &str) -> Self { Self(s.to_string()) @@ -141,12 +150,32 @@ impl From for ModelId { } } +impl FromStr for ModelId { + type Err = std::convert::Infallible; + + fn from_str(s: &str) -> Result { + Ok(Self::from(s)) + } +} + impl AsRef for ModelId { fn as_ref(&self) -> &str { &self.0 } } +impl PartialEq for ModelId { + fn eq(&self, other: &str) -> bool { + self.0 == other + } +} + +impl PartialEq<&str> for ModelId { + fn eq(&self, other: &&str) -> bool { + self.0 == *other + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/lib/crates/fabro-model/src/lib.rs b/lib/crates/fabro-model/src/lib.rs index 8dfd96f23..337c00fdd 100644 --- a/lib/crates/fabro-model/src/lib.rs +++ b/lib/crates/fabro-model/src/lib.rs @@ -19,7 +19,7 @@ pub use billing::{ }; pub use catalog::{ ApiKeyHeaderPolicy, BillingPolicy, Catalog, CredentialRef, CredentialRefParseError, - FallbackTarget, ProviderAuthConfig, + FallbackTarget, ModelSelectionError, ProviderAuthConfig, }; pub use codec::CodecKind; pub use ids::{ModelId, ProviderId}; diff --git a/lib/crates/fabro-model/src/model_ref.rs b/lib/crates/fabro-model/src/model_ref.rs index c9560a049..f8d1f0a8f 100644 --- a/lib/crates/fabro-model/src/model_ref.rs +++ b/lib/crates/fabro-model/src/model_ref.rs @@ -22,7 +22,7 @@ impl ModelHandle { #[must_use] pub fn model_id(&self) -> &str { match self { - Self::Resolved(m) => &m.id, + Self::Resolved(m) => m.id.as_str(), Self::ByName { model, .. } => model, } } diff --git a/lib/crates/fabro-model/src/types.rs b/lib/crates/fabro-model/src/types.rs index 8dcbd3270..fb0164804 100644 --- a/lib/crates/fabro-model/src/types.rs +++ b/lib/crates/fabro-model/src/types.rs @@ -1,6 +1,6 @@ use serde::{Deserialize, Serialize}; -use crate::ids::ProviderId; +use crate::ids::{ModelId, ProviderId}; // --- 2.9 Model --- @@ -78,7 +78,7 @@ pub struct ModelCosts { #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct Model { - pub id: String, + pub id: ModelId, pub provider: ProviderId, pub family: String, pub display_name: String, @@ -102,7 +102,7 @@ pub struct Model { impl Model { pub fn id(&self) -> &str { - &self.id + self.id.as_str() } pub fn provider(&self) -> &ProviderId { @@ -210,7 +210,7 @@ mod tests { #[test] fn inherent_methods_return_correct_values() { let info = Model { - id: "model-id".to_string(), + id: ModelId::new("model-id"), provider: ProviderId::new("provider-id"), family: "family".to_string(), display_name: "Display Name".to_string(), diff --git a/lib/crates/fabro-server/src/diagnostics.rs b/lib/crates/fabro-server/src/diagnostics.rs index b88f93716..988f84404 100644 --- a/lib/crates/fabro-server/src/diagnostics.rs +++ b/lib/crates/fabro-server/src/diagnostics.rs @@ -254,11 +254,11 @@ async fn probe_single_provider( }; let model_id = model.id.clone(); - let outcome = run_basic_model_probe(&model_id, &provider, client).await; + let outcome = run_basic_model_probe(model_id.as_str(), &provider, client).await; match outcome.status { ModelTestStatus::Ok => ProviderProbeResult { provider, - model_id: Some(model_id), + model_id: Some(model_id.to_string()), status: ProviderProbeStatus::Ok, error_message: None, diagnostic_detail: None, @@ -267,7 +267,12 @@ async fn probe_single_provider( let raw = outcome .error_message .unwrap_or_else(|| "provider probe failed".to_string()); - provider_probe_error(provider, Some(model_id), redact_string(&raw), None) + provider_probe_error( + provider, + Some(model_id.to_string()), + redact_string(&raw), + None, + ) } } } diff --git a/lib/crates/fabro-server/src/install.rs b/lib/crates/fabro-server/src/install.rs index 841f405b5..12bb5fbb7 100644 --- a/lib/crates/fabro-server/src/install.rs +++ b/lib/crates/fabro-server/src/install.rs @@ -2167,7 +2167,7 @@ async fn validate_llm_provider( })? .id .clone(); - let params = GenerateParams::new(probe_model, Arc::new(client)) + let params = GenerateParams::new(probe_model.to_string(), Arc::new(client)) .provider(input.provider.to_string()) .prompt("Say OK") .max_tokens(16); diff --git a/lib/crates/fabro-server/src/run_manifest.rs b/lib/crates/fabro-server/src/run_manifest.rs index 93c6660c6..259e3d2b5 100644 --- a/lib/crates/fabro-server/src/run_manifest.rs +++ b/lib/crates/fabro-server/src/run_manifest.rs @@ -476,19 +476,20 @@ async fn build_preflight_report( let catalog = state.catalog(); let llm_result = state.resolve_llm_client().await; - let configured_providers = match &llm_result { - Ok(result) => result.provider_ids(), - Err(err) => { - warn!(error = ?err, "Failed to resolve LLM client while checking ready providers"); - Vec::new() - } - }; + if let Err(err) = &llm_result { + warn!(error = ?err, "Failed to resolve LLM client while checking ready providers"); + } + // Preflight is credential-independent static validation. Materialize + // against every enabled catalog provider so aliases and defaults can be + // inspected even when the corresponding adapter is not currently ready; + // `run_llm_check` below reports actual credential/registration readiness. + let enabled_providers = catalog.all_provider_ids().into_iter().collect::>(); let materialized = materialize_run( prepared.settings.clone(), graph, catalog.as_ref(), - &configured_providers, - ); + &enabled_providers, + )?; let resolved_run = materialized.run; let server_settings = state.server_settings(); let github_integration = &server_settings.server.integrations.github; @@ -1049,8 +1050,15 @@ async fn run_llm_check( has_llm_nodes = true; let node_model = node.model().unwrap_or(model); let node_provider = node.provider().unwrap_or(default_provider); - let (resolved_model, resolved_provider) = if let Some(info) = catalog.get(node_model) { - (info.id.clone(), info.provider.to_string()) + let resolved = if node.provider().is_some() { + catalog.get_on_provider(&ProviderId::new(node_provider), node_model) + } else { + catalog + .select(node_model, None, &catalog.all_provider_ids()) + .ok() + }; + let (resolved_model, resolved_provider) = if let Some(info) = resolved { + (info.id.to_string(), info.provider.to_string()) } else { (node_model.to_string(), node_provider.to_string()) }; @@ -1534,6 +1542,7 @@ enabled = {clone_enabled} Catalog::builtin(), &[ProviderId::anthropic()], ) + .unwrap() .run; (prepared, resolved) @@ -2381,9 +2390,8 @@ digraph Demo { assert!(response_mock.calls_async().await >= 1); } - #[tokio::test] - async fn preflight_unknown_llm_provider_reports_not_configured() { - let state = crate::test_support::test_app_state(); + #[test] + fn static_validation_rejects_unknown_llm_provider() { let mut manifest = minimal_manifest(); manifest.workflows.get_mut("workflow.fabro").unwrap().source = r#" digraph Demo { @@ -2399,29 +2407,16 @@ digraph Demo { &manifest, ) .unwrap(); - let validated = validate_prepared_manifest(&prepared, test_catalog()).unwrap(); + let Err(error) = validate_prepared_manifest(&prepared, test_catalog()) else { + panic!("unknown provider should fail static validation"); + }; - let (response, ok) = run_preflight(state.as_ref(), &prepared, &validated) - .await - .unwrap(); - - assert!(!ok); - let llm_check = response.checks.sections[0] - .checks - .iter() - .find(|check| check.name == "LLM" && check.summary == "missing-model") - .expect("preflight should include the requested custom LLM provider"); - assert_eq!(llm_check.status, types::PreflightCheckResultStatus::Warning); - assert_eq!( - llm_check.remediation.as_deref(), - Some("Provider \"missing-provider\" is not configured") - ); - assert!( - llm_check - .details - .iter() - .any(|detail| detail.text == "Provider: missing-provider") - ); + assert!(matches!( + error, + WorkflowError::ModelSelection(fabro_model::ModelSelectionError::UnknownProvider { + provider + }) if provider.as_str() == "missing-provider" + )); } #[tokio::test] @@ -2437,17 +2432,16 @@ base_url = "https://api.acme.test/v1" [providers.acme.auth] credentials = ["env:ACME_API_KEY"] -[models."acme-large"] -provider = "acme" +[providers.acme.models."acme-large"] display_name = "Acme Large" family = "acme" default = true aliases = ["vl"] -[models."acme-large".limits] +[providers.acme.models."acme-large".limits] context_window = 128000 -[models."acme-large".features] +[providers.acme.models."acme-large".features] tools = true vision = false reasoning = false @@ -2472,7 +2466,7 @@ digraph Demo { &manifest, ) .unwrap(); - let validated = validate_prepared_manifest(&prepared, test_catalog()).unwrap(); + let validated = validate_prepared_manifest(&prepared, state.catalog()).unwrap(); let (response, ok) = run_preflight(state.as_ref(), &prepared, &validated) .await diff --git a/lib/crates/fabro-server/src/server.rs b/lib/crates/fabro-server/src/server.rs index ab4c0a726..98dcc4a62 100644 --- a/lib/crates/fabro-server/src/server.rs +++ b/lib/crates/fabro-server/src/server.rs @@ -1453,7 +1453,7 @@ impl AppState { self.catalog() .default_for_configured_ids(&provider_ids) .id - .clone(), + .to_string(), ) }; AskFabroReadiness { default_model } diff --git a/lib/crates/fabro-server/src/server/automation_scheduler.rs b/lib/crates/fabro-server/src/server/automation_scheduler.rs index 4eb802fab..9499d4170 100644 --- a/lib/crates/fabro-server/src/server/automation_scheduler.rs +++ b/lib/crates/fabro-server/src/server/automation_scheduler.rs @@ -428,6 +428,7 @@ mod tests { fn test_state_with_materializer(materializer: TestAutomationRunMaterializer) -> Arc { TestAppStateBuilder::new() .env_lookup(|_| None) + .vault_entries([("OPENAI_API_KEY", "test-openai-api-key")]) .automation_materializer(materializer) .build() } diff --git a/lib/crates/fabro-server/src/server/handler/completions.rs b/lib/crates/fabro-server/src/server/handler/completions.rs index d5df7299a..3d36f9036 100644 --- a/lib/crates/fabro-server/src/server/handler/completions.rs +++ b/lib/crates/fabro-server/src/server/handler/completions.rs @@ -1,10 +1,13 @@ +use std::collections::HashSet; use std::sync::Arc; +use fabro_model::{Catalog, ModelSelectionError}; + use super::super::{ ApiError, AppState, CompletionResponse, CompletionToolChoiceMode, CompletionUsage, CreateCompletionRequest, FinishReason, GenerateParams, IntoResponse, Json, LlmMessage, - LlmRequest, RequiredUser, Response, Router, State, StatusCode, ToolChoice, ToolDefinition, - Ulid, error, generate_object, info, post, warn, + LlmRequest, ProviderId, RequiredUser, Response, Router, State, StatusCode, ToolChoice, + ToolDefinition, Ulid, error, generate_object, info, post, warn, }; use super::llm_sse; @@ -28,21 +31,34 @@ async fn create_completion( State(state): State>, Json(req): Json, ) -> Response { - // Resolve model let catalog = state.catalog(); - let model_id = req - .model - .unwrap_or_else(|| catalog.default_model().id.clone()); - - let catalog_info = catalog.get(&model_id); - - // Resolve provider: explicit request > catalog > None - let explicit_provider = req.provider; - let provider_name = explicit_provider - .clone() - .or_else(|| catalog_info.map(|i| i.provider.to_string())); - - info!(model = %model_id, provider = ?provider_name, "Completion request received"); + let llm_result = match state.resolve_llm_client().await { + Ok(result) => result, + Err(err) => { + error!(error = ?err, "Failed to create LLM client"); + return ApiError::new( + StatusCode::INTERNAL_SERVER_ERROR, + format!("Failed to resolve LLM providers: {err}"), + ) + .into_response(); + } + }; + for (provider, issue) in &llm_result.auth_issues { + warn!(provider = %provider, error = %issue, "LLM provider unavailable due to auth issue"); + } + for issue in &llm_result.registration_issues { + warn!(provider = %issue.provider, error = %issue.error, "LLM provider unavailable due to registration issue"); + } + let client = llm_result.client; + let (model_id, selected_provider) = match resolve_request_model( + catalog.as_ref(), + &client.provider_ids(), + req.model, + req.provider, + ) { + Ok(selection) => selection, + Err(error) => return ApiError::bad_request(error.to_string()).into_response(), + }; // Build messages list. Request messages are already the canonical // `fabro_types::Message` — the API schema reuses it via build.rs @@ -81,7 +97,7 @@ async fn create_completion( let request = LlmRequest { model: model_id.clone(), messages, - provider: provider_name.clone(), + provider: Some(selected_provider.to_string()), tools, tool_choice, response_format: None, @@ -98,35 +114,27 @@ async fn create_completion( metadata: None, provider_options: req.provider_options, }; + let request = match client.resolve_request(&request) { + Ok(request) => request, + Err(error) => return ApiError::bad_request(error.to_string()).into_response(), + }; + let selected_provider = ProviderId::new( + request + .provider + .as_deref() + .expect("resolved LLM request always has a provider"), + ); + let selected_model = request.model.clone(); + + info!( + model = %selected_model, + provider = %selected_provider, + "Completion request received" + ); // Force non-streaming for structured output let use_stream = req.stream && req.schema.is_none(); - let llm_result = match state.resolve_llm_client().await { - Ok(result) => result, - Err(err) => { - error!(error = ?err, "Failed to create LLM client"); - return ApiError::new( - StatusCode::INTERNAL_SERVER_ERROR, - format!("Failed to create LLM client: {err}"), - ) - .into_response(); - } - }; - for (provider, issue) in &llm_result.auth_issues { - warn!(provider = %provider, error = %issue, "LLM provider unavailable due to auth issue"); - } - for issue in &llm_result.registration_issues { - warn!(provider = %issue.provider, error = %issue.error, "LLM provider unavailable due to registration issue"); - } - let client = llm_result.client; - if let Some(provider) = explicit_provider.as_deref() { - if !client.has_provider(provider) { - return ApiError::bad_request(format!("Provider \"{provider}\" is not configured")) - .into_response(); - } - } - if use_stream { // Streaming path: forward all StreamEvents as SSE let stream_result = match client.stream(&request).await { @@ -169,7 +177,8 @@ async fn create_completion( let stop_reason = finish_reason_to_api_stop_reason(&response.finish_reason); Json(CompletionResponse { id: msg_id, - model: model_id, + model: selected_model, + provider: selected_provider, message: response.message, stop_reason, usage: CompletionUsage { @@ -192,6 +201,7 @@ async fn create_completion( Json(CompletionResponse { id: response.id, model: response.model, + provider: ProviderId::new(response.provider), message: response.message, stop_reason, usage: CompletionUsage { @@ -210,3 +220,48 @@ async fn create_completion( } } } + +pub(super) fn resolve_request_model( + catalog: &Catalog, + eligible: &HashSet, + requested_model: Option, + explicit_provider: Option, +) -> Result<(String, ProviderId), ModelSelectionError> { + let explicit_provider = explicit_provider.map(ProviderId::new); + if let Some(model) = requested_model { + return match catalog.select(&model, explicit_provider.as_ref(), eligible) { + Ok(offering) => Ok((offering.id.to_string(), offering.provider.clone())), + Err(ModelSelectionError::UnknownSelectorOnProvider { provider, .. }) => { + Ok((model, provider)) + } + Err(ModelSelectionError::UnknownSelector { .. }) => { + let default = catalog.select_default(eligible)?; + Ok((model, default.provider.clone())) + } + Err(error) => Err(error), + }; + } + + let eligible = if let Some(requested_provider) = explicit_provider { + let provider = catalog.provider(&requested_provider).ok_or_else(|| { + ModelSelectionError::UnknownProvider { + provider: requested_provider.clone(), + } + })?; + let provider_is_ready = eligible.iter().any(|eligible_provider| { + catalog + .provider(eligible_provider) + .is_some_and(|eligible_provider| eligible_provider.id == provider.id) + }); + if !provider_is_ready { + return Err(ModelSelectionError::ProviderUnavailable { + provider: provider.id.clone(), + }); + } + HashSet::from([provider.id.clone()]) + } else { + eligible.clone() + }; + let default = catalog.select_default(&eligible)?; + Ok((default.id.to_string(), default.provider.clone())) +} diff --git a/lib/crates/fabro-server/src/server/handler/models.rs b/lib/crates/fabro-server/src/server/handler/models.rs index 3d5341270..9b639af99 100644 --- a/lib/crates/fabro-server/src/server/handler/models.rs +++ b/lib/crates/fabro-server/src/server/handler/models.rs @@ -3,6 +3,7 @@ use std::sync::Arc; use fabro_auth::ApiCredential; use fabro_llm::client::Client as LlmClient; use fabro_llm::model_test::{ModelTestStatus, run_basic_model_probe}; +use fabro_model::ModelSelectionError; use fabro_redact::redact_string; use super::super::{ @@ -40,7 +41,9 @@ struct ModelListParams { #[derive(serde::Deserialize)] struct ModelTestParams { #[serde(default)] - mode: Option, + mode: Option, + #[serde(default)] + provider: Option, } async fn list_models( @@ -62,7 +65,7 @@ async fn list_models( .into_iter() .filter(|model| match &query { Some(query) => { - model.id.to_lowercase().contains(query) + model.id.as_str().to_lowercase().contains(query) || model.display_name.to_lowercase().contains(query) || model .aliases @@ -155,7 +158,7 @@ async fn test_provider_credentials( .into_response(); }; - let outcome = run_basic_model_probe(&model.id, &provider_id, client).await; + let outcome = run_basic_model_probe(model.id.as_str(), &provider_id, client).await; match outcome.status { ModelTestStatus::Ok => ( StatusCode::OK, @@ -204,11 +207,6 @@ async fn test_model( }, None => ModelTestMode::Basic, }; - let catalog = state.catalog(); - let Some(info) = catalog.get(&id) else { - return ApiError::not_found(format!("Model not found: {id}")).into_response(); - }; - let llm_result = match state.resolve_llm_client().await { Ok(result) => result, Err(err) => { @@ -220,6 +218,23 @@ async fn test_model( .into_response(); } }; + let catalog = state.catalog(); + let eligible = llm_result + .provider_ids() + .into_iter() + .collect::>(); + let explicit_provider = params.provider.map(ProviderId::new); + let info = if let Some(provider) = explicit_provider.as_ref() { + match catalog.resolve_on_provider(provider, &id) { + Ok(info) => info, + Err(error) => return model_selection_response(&error), + } + } else { + match catalog.select(&id, None, &eligible) { + Ok(info) => info, + Err(error) => return model_selection_response(&error), + } + }; if let Some((_, issue)) = llm_result .auth_issues .iter() @@ -231,6 +246,7 @@ async fn test_model( if !llm_result.client.has_provider(provider_name) { return Json(serde_json::json!({ "model_id": info.id, + "provider": info.provider, "status": "skip", })) .into_response(); @@ -240,8 +256,24 @@ async fn test_model( let outcome = run_model_test(info, mode, client).await; Json(serde_json::json!({ "model_id": info.id, + "provider": info.provider, "status": <&'static str>::from(outcome.status), "error_message": outcome.error_message, })) .into_response() } + +fn model_selection_response(error: &ModelSelectionError) -> Response { + match error { + ModelSelectionError::UnknownProvider { .. } + | ModelSelectionError::UnknownSelector { .. } + | ModelSelectionError::UnknownSelectorOnProvider { .. } => { + ApiError::not_found(error.to_string()).into_response() + } + ModelSelectionError::RetiredModelIdentifier { .. } + | ModelSelectionError::ProviderUnavailable { .. } + | ModelSelectionError::NoEligibleOffering { .. } => { + ApiError::bad_request(error.to_string()).into_response() + } + } +} diff --git a/lib/crates/fabro-server/src/server/handler/pair.rs b/lib/crates/fabro-server/src/server/handler/pair.rs index 8304bf1ec..8367e34ad 100644 --- a/lib/crates/fabro-server/src/server/handler/pair.rs +++ b/lib/crates/fabro-server/src/server/handler/pair.rs @@ -883,7 +883,7 @@ mod tests { text: "I found the issue.".to_string(), model: ModelRef { provider: ProviderId::new("openai"), - model_id: "gpt-5.4".to_string(), + model_id: "gpt-5.4".into(), speed: None, }, billing: BilledTokenCounts::default(), @@ -915,7 +915,7 @@ mod tests { text: "wrong stage".to_string(), model: ModelRef { provider: ProviderId::new("openai"), - model_id: "gpt-5.4".to_string(), + model_id: "gpt-5.4".into(), speed: None, }, billing: BilledTokenCounts::default(), diff --git a/lib/crates/fabro-server/src/server/handler/playground.rs b/lib/crates/fabro-server/src/server/handler/playground.rs index 0b3111d90..046cf2ef3 100644 --- a/lib/crates/fabro-server/src/server/handler/playground.rs +++ b/lib/crates/fabro-server/src/server/handler/playground.rs @@ -109,11 +109,33 @@ async fn create_playground_chat( } let catalog = state.catalog(); - let model_id = req - .model - .unwrap_or_else(|| catalog.default_model().id.clone()); - - info!(model = %model_id, "Playground chat turn"); + let llm_result = match state.resolve_llm_client().await { + Ok(result) => result, + Err(err) => { + error!(error = ?err, "playground: failed to create LLM client"); + return ApiError::new( + StatusCode::INTERNAL_SERVER_ERROR, + format!("Failed to resolve LLM providers: {err}"), + ) + .into_response(); + } + }; + for (provider, issue) in &llm_result.auth_issues { + warn!(provider = %provider, error = %issue, "playground: provider auth issue"); + } + for issue in &llm_result.registration_issues { + warn!(provider = %issue.provider, error = %issue.error, "playground: provider registration issue"); + } + let client = llm_result.client; + let (model_id, selected_provider) = match super::completions::resolve_request_model( + catalog.as_ref(), + &client.provider_ids(), + req.model, + req.provider.map(|provider| provider.to_string()), + ) { + Ok(selection) => selection, + Err(error) => return ApiError::bad_request(error.to_string()).into_response(), + }; // Request messages are already the canonical `fabro_types::Message` — // the API schema reuses it via build.rs `with_replacement`. @@ -124,7 +146,7 @@ async fn create_playground_chat( let request = LlmRequest { model: model_id, messages, - provider: None, + provider: Some(selected_provider.to_string()), tools: Some(playground_tools()), tool_choice: Some(ToolChoice::Auto), response_format: None, @@ -137,22 +159,15 @@ async fn create_playground_chat( metadata: None, provider_options: None, }; - - let llm_result = match state.resolve_llm_client().await { - Ok(r) => r, - Err(err) => { - error!(error = ?err, "playground: failed to create LLM client"); - return ApiError::new( - StatusCode::INTERNAL_SERVER_ERROR, - format!("Failed to create LLM client: {err}"), - ) - .into_response(); - } + let request = match client.resolve_request(&request) { + Ok(request) => request, + Err(error) => return ApiError::bad_request(error.to_string()).into_response(), }; - for (provider, issue) in &llm_result.auth_issues { - warn!(provider = %provider, error = %issue, "playground: provider auth issue"); - } - let client = llm_result.client; + info!( + model = %request.model, + provider = request.provider.as_deref().unwrap_or(""), + "Playground chat turn" + ); let stream_result = match client.stream(&request).await { Ok(s) => s, @@ -238,6 +253,7 @@ mod tests { messages, workflow_fabro, model: None, + provider: None, } } diff --git a/lib/crates/fabro-server/src/server/handler/pull_requests.rs b/lib/crates/fabro-server/src/server/handler/pull_requests.rs index b23af4d82..7acd8fb9d 100644 --- a/lib/crates/fabro-server/src/server/handler/pull_requests.rs +++ b/lib/crates/fabro-server/src/server/handler/pull_requests.rs @@ -321,7 +321,10 @@ async fn create_run_pull_request( } else { let catalog = state.catalog(); let configured = state.ready_llm_provider_ids().await; - catalog.default_for_configured_ids(&configured).id.clone() + catalog + .default_for_configured_ids(&configured) + .id + .to_string() }; let catalog = state.catalog(); diff --git a/lib/crates/fabro-server/src/server/handler/runs.rs b/lib/crates/fabro-server/src/server/handler/runs.rs index 6ea48942f..e3e7f3dd2 100644 --- a/lib/crates/fabro-server/src/server/handler/runs.rs +++ b/lib/crates/fabro-server/src/server/handler/runs.rs @@ -52,6 +52,8 @@ use crate::run_manifest; use crate::run_selector::{ResolveRunError, resolve_run_by_selector}; use crate::run_title_generation::{self, GenerateTitleInput, TitlePromptInput, WorkflowSummary}; use crate::server_secrets::LlmClientResult; +#[cfg(any(test, feature = "test-support"))] +use crate::test_support as server_test_support; pub(super) fn manifest_routes() -> Router> { Router::new() @@ -600,10 +602,23 @@ pub(crate) async fn create_run_from_manifest( .as_ref() .map(LlmClientResult::provider_ids) .unwrap_or_default(); + let run_materialization_provider_ids = { + #[cfg(any(test, feature = "test-support"))] + { + server_test_support::test_run_materialization_provider_ids( + catalog.as_ref(), + &ready_provider_ids, + ) + } + #[cfg(not(any(test, feature = "test-support")))] + { + ready_provider_ids.clone() + } + }; let provenance = run_provenance(&headers, &actor); let mut create_input = run_manifest::create_run_input( prepared.clone(), - ready_provider_ids.clone(), + run_materialization_provider_ids, provenance, web_url.clone(), vars, @@ -625,6 +640,9 @@ pub(crate) async fn create_run_from_manifest( Err(WorkflowError::ValidationFailed { .. } | WorkflowError::Parse(_)) => { return ApiError::bad_request("Validation failed").into_response(); } + Err(err @ WorkflowError::ModelSelection(_)) => { + return ApiError::bad_request(err.to_string()).into_response(); + } Err(err) => { return ApiError::new( StatusCode::INTERNAL_SERVER_ERROR, @@ -681,7 +699,7 @@ pub(crate) async fn create_run_from_manifest( workflow, run_inputs, client: llm_result.client, - model_id: title_model_id, + model_id: title_model_id.to_string(), provider_id: title_provider_id, }); } diff --git a/lib/crates/fabro-server/src/server/handler/sessions.rs b/lib/crates/fabro-server/src/server/handler/sessions.rs index 3e24a1f9d..eb72bd386 100644 --- a/lib/crates/fabro-server/src/server/handler/sessions.rs +++ b/lib/crates/fabro-server/src/server/handler/sessions.rs @@ -22,7 +22,9 @@ use fabro_api::types::{ }; use fabro_llm::client::Client as LlmClient; use fabro_llm::types::ToolDefinition; -use fabro_model::{AgentProfileKind, Catalog, ModelHandle, ProviderId}; +use fabro_model::{ + AgentProfileKind, Catalog, ModelHandle, ModelSelectionError, ProviderId, catalog, +}; use fabro_sandbox::reconnect::reconnect_for_run; use fabro_static::EnvVars; use fabro_store::{ @@ -35,7 +37,7 @@ use fabro_types::run_event::{ RunSessionTurnFailedProps, RunSessionTurnInterruptedProps, RunSessionTurnStartedProps, RunSessionTurnSucceededProps, RunSessionUserMessageProps, }; -use fabro_types::settings::{ModelRef as SettingsModelRef, ModelRegistry, ResolvedModelRef}; +use fabro_types::settings::ModelRef as SettingsModelRef; use fabro_types::{EventBody, EventEnvelope, RunEvent, RunId, SessionDetail, SessionId, TurnId}; use fabro_workflow::handler::llm::api::register_named_fabro_run_tools; use fabro_workflow::services::FabroRunToolServices; @@ -160,8 +162,24 @@ async fn create_run_session( Ok(store) => store, Err(response) => return response, }; - let model = match canonical_session_model(state.catalog().as_ref(), request.model.as_deref()) { - Ok(model) => model, + let llm_result = match state.resolve_llm_client().await { + Ok(result) => result, + Err(err) => { + return ApiError::new( + StatusCode::INTERNAL_SERVER_ERROR, + format!("Failed to resolve LLM providers: {err}"), + ) + .into_response(); + } + }; + let eligible = llm_result.provider_ids().into_iter().collect(); + let (provider, model) = match canonical_session_model( + state.catalog().as_ref(), + &eligible, + request.model.as_deref(), + request.provider.as_ref(), + ) { + Ok(selection) => selection, Err(err) => return err.into_response(), }; @@ -180,8 +198,9 @@ async fn create_run_session( run_id, session_id, EventBody::RunSessionCreated(RunSessionCreatedProps { - title: request.title, - model, + title: request.title, + model: Some(model), + provider: Some(provider), }), now, ) @@ -782,17 +801,41 @@ fn selected_session_model( llm_result: &LlmClientResult, session: &ProjectedRunSession, ) -> Result<(ProviderId, String, AgentProfileKind), AskFabroBuildError> { - let configured_provider_ids = llm_result.provider_ids(); - let selected = match session.record.model.as_deref() { - Some(model_id) => catalog.get(model_id).ok_or_else(|| { - AskFabroBuildError::ModelUnavailable(format!( - "session model '{model_id}' is not in the catalog" - )) - })?, - None => catalog.default_for_configured_ids(&configured_provider_ids), + let eligible = llm_result + .provider_ids() + .into_iter() + .collect::>(); + let (provider_id, model) = if let Some(model_id) = session.record.model.as_deref() { + match catalog.select(model_id, session.record.provider.as_ref(), &eligible) { + Ok(selected) => (selected.provider.clone(), selected.id.to_string()), + Err(ModelSelectionError::UnknownSelectorOnProvider { provider, .. }) => { + (provider, model_id.to_string()) + } + Err(ModelSelectionError::UnknownSelector { .. }) + if session.record.provider.is_none() => + { + let selected = catalog + .select_default(&eligible) + .map_err(|error| AskFabroBuildError::LlmUnconfigured(error.to_string()))?; + (selected.provider.clone(), model_id.to_string()) + } + Err(error) => { + return Err(AskFabroBuildError::ModelUnavailable(error.to_string())); + } + } + } else { + let selected = if let Some(provider) = session.record.provider.as_ref() { + let provider_only = std::collections::HashSet::from([provider.clone()]); + catalog + .select_default(&provider_only) + .map_err(|error| AskFabroBuildError::ModelUnavailable(error.to_string()))? + } else { + catalog + .select_default(&eligible) + .map_err(|error| AskFabroBuildError::LlmUnconfigured(error.to_string()))? + }; + (selected.provider.clone(), selected.id.to_string()) }; - let provider_id = selected.provider.clone(); - let model = selected.id.clone(); let profile_kind = catalog .effective_agent_profile(&provider_id, Some(&model)) .ok_or_else(|| { @@ -805,79 +848,117 @@ fn selected_session_model( fn canonical_session_model( catalog: &Catalog, + eligible: &std::collections::HashSet, requested: Option<&str>, -) -> Result, ApiError> { + explicit_provider: Option<&ProviderId>, +) -> Result<(ProviderId, String), ApiError> { + let explicit_provider = explicit_provider + .map(|provider| { + catalog + .provider(provider) + .map(|provider| provider.id.clone()) + .ok_or_else(|| { + session_selection_error(&ModelSelectionError::UnknownProvider { + provider: provider.clone(), + }) + }) + }) + .transpose()?; let Some(requested) = requested else { - return Ok(None); + let eligible = if let Some(provider) = explicit_provider.as_ref() { + let provider_is_ready = eligible.iter().any(|eligible_provider| { + catalog + .provider(eligible_provider) + .is_some_and(|eligible_provider| eligible_provider.id == *provider) + }); + if !provider_is_ready { + return Err(session_selection_error( + &ModelSelectionError::ProviderUnavailable { + provider: provider.clone(), + }, + )); + } + std::collections::HashSet::from([provider.clone()]) + } else { + eligible.clone() + }; + let model = catalog + .select_default(&eligible) + .map_err(|error| session_selection_error(&error))?; + return Ok((model.provider.clone(), model.id.to_string())); }; let requested = requested.trim(); if requested.is_empty() { return Err(ApiError::bad_request("Session model must not be empty.")); } + if let Some((provider, model)) = catalog::retired_model_replacement(requested) { + return Err(session_selection_error( + &ModelSelectionError::RetiredModelIdentifier { + identifier: requested.to_string(), + provider, + model, + }, + )); + } let model_ref = requested .parse::() .map_err(|err| ApiError::bad_request(err.to_string()))?; - let registry = CatalogModelRegistry { catalog }; - match model_ref - .resolve(®istry) - .map_err(|err| ApiError::bad_request(err.to_string()))? - { - ResolvedModelRef::Provider(provider) => Err(ApiError::bad_request(format!( - "Session model reference '{provider}' names a provider; include a model ID." - ))), - ResolvedModelRef::Model { - provider: Some(provider), - model, - } => resolve_provider_qualified_session_model(catalog, &provider, &model).map(Some), - ResolvedModelRef::Model { - provider: None, - model, - } => catalog - .get(&model) - .map(|model| Some(model.id.clone())) - .ok_or_else(|| ApiError::bad_request(format!("Unknown session model '{model}'."))), + let (qualified_provider, model) = match model_ref { + SettingsModelRef::Qualified { provider, model } => { + let requested_provider = ProviderId::new(provider); + let provider = catalog + .provider(&requested_provider) + .map(|provider| provider.id.clone()) + .ok_or_else(|| { + session_selection_error(&ModelSelectionError::UnknownProvider { + provider: requested_provider, + }) + })?; + if let Some(explicit) = explicit_provider.as_ref() { + if explicit != &provider { + return Err(ApiError::bad_request(format!( + "Session provider pin '{explicit}' conflicts with model reference provider \ + '{provider}'." + ))); + } + } + (Some(provider), model) + } + SettingsModelRef::Bare(model) => { + if explicit_provider.is_none() && catalog.provider(&ProviderId::new(&model)).is_some() { + let detail = if catalog.is_model_selector(&model) { + format!( + "Session model reference '{model}' is ambiguous between a provider and a \ + model selector; supply `provider` or use `provider/model`." + ) + } else { + format!( + "Session model reference '{model}' names a provider; include a model ID." + ) + }; + return Err(ApiError::bad_request(detail)); + } + (None, model) + } + }; + let provider = qualified_provider.as_ref().or(explicit_provider.as_ref()); + match catalog.select(&model, provider, eligible) { + Ok(selected) => Ok((selected.provider.clone(), selected.id.to_string())), + Err(ModelSelectionError::UnknownSelectorOnProvider { provider, .. }) => { + Ok((provider, model)) + } + Err(ModelSelectionError::UnknownSelector { .. }) if provider.is_none() => { + let selected = catalog + .select_default(eligible) + .map_err(|error| session_selection_error(&error))?; + Ok((selected.provider.clone(), model)) + } + Err(error) => Err(session_selection_error(&error)), } } -fn resolve_provider_qualified_session_model( - catalog: &Catalog, - provider_ref: &str, - model_ref: &str, -) -> Result { - let provider_id = ProviderId::new(provider_ref); - let provider = catalog.provider(&provider_id).ok_or_else(|| { - ApiError::bad_request(format!("Unknown session model provider '{provider_ref}'.")) - })?; - let model = catalog - .get(model_ref) - .ok_or_else(|| ApiError::bad_request(format!("Unknown session model '{model_ref}'.")))?; - if model.provider != provider.id { - return Err(ApiError::bad_request(format!( - "Session model '{model_ref}' belongs to provider '{}', not '{}'.", - model.provider, provider.id - ))); - } - Ok(model.id.clone()) -} - -struct CatalogModelRegistry<'a> { - catalog: &'a Catalog, -} - -impl ModelRegistry for CatalogModelRegistry<'_> { - fn is_provider(&self, token: &str) -> bool { - self.catalog.provider(&ProviderId::new(token)).is_some() - } - - fn is_model(&self, token: &str) -> bool { - self.catalog.get(token).is_some() - } - - fn provider_of(&self, token: &str) -> Option { - self.catalog - .get(token) - .map(|model| model.provider.to_string()) - } +fn session_selection_error(error: &ModelSelectionError) -> ApiError { + ApiError::bad_request(error.to_string()) } fn build_profile( @@ -1275,7 +1356,7 @@ fn agent_event_payload(event_turn_id: TurnId, event: AgentEvent) -> Option Catalog { + let settings: LlmCatalogSettings = toml::from_str( + r#" +[providers.openai] +display_name = "OpenAI" +adapter = "openai" +agent_profile = "openai" +priority = 90 + +[providers.openai.models."gpt-5.6-sol"] +display_name = "GPT-5.6 Sol" +family = "gpt-5" +aliases = ["gpt-56-sol"] +default = true + +[providers.openai.models."gpt-5.6-sol".limits] +context_window = 1000 + +[providers.openai.models."gpt-5.6-sol".features] +tools = true +vision = false +reasoning = false + +[providers.openrouter] +display_name = "OpenRouter" +adapter = "openai_compatible" +agent_profile = "openai" +priority = 25 + +[providers.openrouter.models."gpt-5.6-sol"] +api_id = "openai/gpt-5.6-sol" +display_name = "GPT-5.6 Sol (via OpenRouter)" +family = "gpt-5" +aliases = ["gpt-56-sol"] +default = true + +[providers.openrouter.models."gpt-5.6-sol".limits] +context_window = 1000 + +[providers.openrouter.models."gpt-5.6-sol".features] +tools = true +vision = false +reasoning = false +"#, + ) + .unwrap(); + Catalog::from_settings(&settings).unwrap() + } + + #[test] + fn canonical_session_model_uses_readiness_priority_and_explicit_pins() { + let catalog = portable_session_catalog(); + let openai = ProviderId::openai(); + let openrouter = ProviderId::new("openrouter"); + + assert_eq!( + canonical_session_model( + &catalog, + &std::collections::HashSet::from([openai.clone()]), + Some("gpt-56-sol"), + None, + ) + .unwrap(), + (openai.clone(), "gpt-5.6-sol".to_string()) + ); + assert_eq!( + canonical_session_model( + &catalog, + &std::collections::HashSet::from([openrouter.clone()]), + Some("gpt-56-sol"), + None, + ) + .unwrap(), + (openrouter.clone(), "gpt-5.6-sol".to_string()) + ); + let both = std::collections::HashSet::from([openai.clone(), openrouter.clone()]); + assert_eq!( + canonical_session_model(&catalog, &both, Some("gpt-56-sol"), None).unwrap(), + (openai, "gpt-5.6-sol".to_string()) + ); + assert_eq!( + canonical_session_model(&catalog, &both, Some("gpt-56-sol"), Some(&openrouter),) + .unwrap(), + (openrouter.clone(), "gpt-5.6-sol".to_string()) + ); + assert_eq!( + canonical_session_model(&catalog, &both, Some("openrouter/gpt-56-sol"), None,).unwrap(), + (openrouter, "gpt-5.6-sol".to_string()) + ); + } + + #[test] + fn canonical_session_model_preserves_unknown_passthrough_on_selected_provider() { + let catalog = portable_session_catalog(); + let openai = ProviderId::openai(); + let openrouter = ProviderId::new("openrouter"); + let both = std::collections::HashSet::from([openai.clone(), openrouter.clone()]); + + assert_eq!( + canonical_session_model(&catalog, &both, Some("future-model"), Some(&openrouter),) + .unwrap(), + (openrouter, "future-model".to_string()) + ); + assert_eq!( + canonical_session_model(&catalog, &both, Some("future-model"), None).unwrap(), + (openai, "future-model".to_string()) + ); + } + + #[test] + fn canonical_session_model_rejects_an_unavailable_explicit_provider() { + let catalog = portable_session_catalog(); + let error = canonical_session_model( + &catalog, + &std::collections::HashSet::from([ProviderId::openai()]), + Some("gpt-56-sol"), + Some(&ProviderId::new("openrouter")), + ) + .unwrap_err(); + + assert_eq!(error.status(), StatusCode::BAD_REQUEST); + } + + #[test] + fn canonical_session_model_rejects_retired_wire_identifier_before_qualification() { + let catalog = Catalog::from_builtin().unwrap(); + let error = canonical_session_model( + &catalog, + &catalog.all_provider_ids(), + Some("openai/gpt-5.6-sol"), + None, + ) + .unwrap_err(); + + assert_eq!(error.status(), StatusCode::BAD_REQUEST); + assert!( + error + .into_response_entry() + .detail + .contains("openrouter/gpt-5.6-sol") + ); + } + #[test] fn agent_event_payload_maps_text_delta_to_session_assistant_delta() { let turn_id = TurnId::new(); diff --git a/lib/crates/fabro-server/src/server/tests.rs b/lib/crates/fabro-server/src/server/tests.rs index f7fed5606..ca29053a9 100644 --- a/lib/crates/fabro-server/src/server/tests.rs +++ b/lib/crates/fabro-server/src/server/tests.rs @@ -1553,7 +1553,7 @@ async fn github_webhook_accepts_valid_signature_with_wrong_bearer_token() { #[tokio::test] async fn create_secret_stores_valid_oauth_entries() { - let state = test_app_state(); + let state = TestAppStateBuilder::new().build(); let app = crate::test_support::build_test_router(Arc::clone(&state)); let req = Request::builder() @@ -1993,7 +1993,7 @@ async fn create_secret_rejects_invalid_oauth_name() { #[tokio::test] async fn delete_secret_by_name_removes_file_secret() { - let state = test_app_state(); + let state = TestAppStateBuilder::new().build(); let app = crate::test_support::build_test_router(Arc::clone(&state)); let create_req = Request::builder() @@ -3394,25 +3394,30 @@ async fn create_run_with_explicit_title_skips_generated_title_work() { } #[tokio::test] -async fn create_run_without_ready_llm_provider_skips_generated_title_work() { +async fn create_run_without_ready_llm_provider_rejects_implicit_model_selection() { let state = TestAppStateBuilder::new().env_lookup(|_| None).build(); let app = crate::test_support::build_test_router(Arc::clone(&state)); - let body = post_run_manifest(&app, minimal_manifest_json(MINIMAL_DOT)).await; - let run_id: RunId = body["id"].as_str().unwrap().parse().unwrap(); + let response = app + .oneshot( + Request::builder() + .method("POST") + .uri(api("/runs")) + .header("content-type", "application/json") + .body(Body::from(minimal_manifest_json(MINIMAL_DOT).to_string())) + .unwrap(), + ) + .await + .unwrap(); + let body = response_json!(response, StatusCode::BAD_REQUEST).await; - assert_eq!( - state - .stores - .runs - .get_cached_summary(&run_id, Utc::now()) - .await - .unwrap() - .unwrap() - .title, - "Test" + assert!( + body["errors"][0]["detail"] + .as_str() + .is_some_and(|detail| detail.contains("no offering on an eligible provider")), + "unexpected response: {body}" ); - assert_eq!(title_update_event_count(&state, run_id).await, 0); + assert!(state.runs.lock().expect("runs lock poisoned").is_empty()); } #[tokio::test] @@ -3524,7 +3529,10 @@ async fn post_run_manifest(app: &Router, manifest: serde_json::Value) -> serde_j #[tokio::test] async fn post_runs_create_regression_keeps_api_behavior_without_automation_metadata() { - let state = TestAppStateBuilder::new().env_lookup(|_| None).build(); + let state = TestAppStateBuilder::new() + .env_lookup(|_| None) + .vault_entries([(EnvVars::OPENAI_API_KEY, "test-openai-api-key")]) + .build(); let app = crate::test_support::build_test_router(Arc::clone(&state)); let mut manifest = minimal_manifest_json(MINIMAL_DOT); manifest["title"] = json!("API title"); @@ -3547,7 +3555,10 @@ async fn post_runs_create_regression_keeps_api_behavior_without_automation_metad #[tokio::test] async fn create_run_from_manifest_helper_persists_without_automation_metadata() { - let state = TestAppStateBuilder::new().env_lookup(|_| None).build(); + let state = TestAppStateBuilder::new() + .env_lookup(|_| None) + .vault_entries([(EnvVars::OPENAI_API_KEY, "test-openai-api-key")]) + .build(); let manifest: RunManifest = serde_json::from_value(minimal_manifest_json(MINIMAL_DOT)).unwrap(); let submitted_manifest_bytes = serde_json::to_vec(&manifest).unwrap(); let run_id = RunId::new(); @@ -3583,7 +3594,10 @@ async fn create_run_from_manifest_helper_persists_without_automation_metadata() #[tokio::test] async fn create_run_from_manifest_helper_persists_automation_metadata() { - let state = TestAppStateBuilder::new().env_lookup(|_| None).build(); + let state = TestAppStateBuilder::new() + .env_lookup(|_| None) + .vault_entries([(EnvVars::OPENAI_API_KEY, "test-openai-api-key")]) + .build(); let manifest: RunManifest = serde_json::from_value(minimal_manifest_json(MINIMAL_DOT)).unwrap(); let submitted_manifest_bytes = serde_json::to_vec(&manifest).unwrap(); let run_id = RunId::new(); @@ -4130,7 +4144,7 @@ fn context_window_event( text: "assistant response".to_string(), model: ModelRef { provider: ProviderId::openai(), - model_id: "gpt-5.4".to_string(), + model_id: "gpt-5.4".into(), speed: None, }, usage: TokenCounts::default(), @@ -6092,6 +6106,15 @@ fn create_github_token_app_state_with_env_lookup_and_llm_catalog_settings( .set("GITHUB_TOKEN", token, SecretType::Token, None) .expect("test github token should be writable"); } + Vault::load(vault_path.clone()) + .expect("test vault should load") + .set( + EnvVars::OPENAI_API_KEY, + "test-openai-api-key", + SecretType::Token, + None, + ) + .expect("test OpenAI credential should be writable"); let db_pool = test_db_pool_for_vault_path(&vault_path).expect("test db pool should build"); let preloaded_vault = crate::test_support::test_secret_snapshot(db_pool.clone()) .expect("test secret snapshot should build"); @@ -6390,7 +6413,7 @@ async fn test_model_unknown_returns_404() { } #[tokio::test] -async fn test_model_alias_returns_canonical_model_id() { +async fn test_model_explicit_provider_alias_returns_canonical_model_id_when_unavailable() { let state = test_app_state_with_env_lookup( default_test_server_settings(), RunLayer::default(), @@ -6401,7 +6424,7 @@ async fn test_model_alias_returns_canonical_model_id() { let req = Request::builder() .method("POST") - .uri(api("/models/sonnet/test")) + .uri(api("/models/sonnet/test?provider=anthropic")) .header("content-type", "application/json") .body(Body::empty()) .unwrap(); @@ -6409,9 +6432,208 @@ async fn test_model_alias_returns_canonical_model_id() { let response = app.oneshot(req).await.unwrap(); let body = response_json!(response, StatusCode::OK).await; assert_eq!(body["model_id"], "claude-sonnet-4-6"); + assert_eq!(body["provider"], "anthropic"); assert_eq!(body["status"], "skip"); } +#[tokio::test] +async fn test_model_unqualified_known_alias_requires_a_ready_provider() { + let state = test_app_state_with_env_lookup( + default_test_server_settings(), + RunLayer::default(), + 5, + |_| None, + ); + let app = crate::test_support::build_test_router(state); + + let response = app + .oneshot( + Request::builder() + .method("POST") + .uri(api("/models/sonnet/test")) + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + + assert_status!(response, StatusCode::BAD_REQUEST).await; +} + +#[tokio::test] +async fn model_api_keeps_duplicate_ids_provider_scoped_and_selects_ready_priority() { + let direct_upstream = MockServer::start(); + let aggregator_upstream = MockServer::start(); + let direct_probe = direct_upstream.mock(|when, then| { + when.method(POST) + .path("/chat/completions") + .json_body_includes(r#"{"model":"portable-model"}"#); + then.status(200) + .header("content-type", "application/json") + .json_body(json!({ + "id": "chatcmpl-direct", + "model": "portable-model", + "choices": [{ + "message": {"role": "assistant", "content": "OK"}, + "finish_reason": "stop" + }], + "usage": { + "prompt_tokens": 1, + "completion_tokens": 1, + "total_tokens": 2 + } + })); + }); + let aggregator_probe = aggregator_upstream.mock(|when, then| { + when.method(POST) + .path("/chat/completions") + .json_body_includes(r#"{"model":"vendor/portable-model"}"#); + then.status(200) + .header("content-type", "application/json") + .json_body(json!({ + "id": "chatcmpl-aggregator", + "model": "vendor/portable-model", + "choices": [{ + "message": {"role": "assistant", "content": "OK"}, + "finish_reason": "stop" + }], + "usage": { + "prompt_tokens": 1, + "completion_tokens": 1, + "total_tokens": 2 + } + })); + }); + let settings: LlmCatalogSettings = toml::from_str(&format!( + r#" +[providers.direct] +display_name = "Direct" +adapter = "openai_compatible" +agent_profile = "openai" +base_url = "{}" +priority = 120 + +[providers.direct.auth] +credentials = ["vault:DIRECT_API_KEY"] + +[providers.direct.models.portable-model] +display_name = "Portable (direct)" +family = "portable" +aliases = ["portable"] +default = true + +[providers.direct.models.portable-model.limits] +context_window = 1000 + +[providers.direct.models.portable-model.features] +tools = false +vision = false +reasoning = false + +[providers.aggregator] +display_name = "Aggregator" +adapter = "openai_compatible" +agent_profile = "openai" +base_url = "{}" +priority = 110 + +[providers.aggregator.auth] +credentials = ["vault:AGGREGATOR_API_KEY"] + +[providers.aggregator.models.portable-model] +api_id = "vendor/portable-model" +display_name = "Portable (aggregator)" +family = "portable" +aliases = ["portable"] +default = true + +[providers.aggregator.models.portable-model.limits] +context_window = 1000 + +[providers.aggregator.models.portable-model.features] +tools = false +vision = false +reasoning = false +"#, + direct_upstream.base_url(), + aggregator_upstream.base_url(), + )) + .unwrap(); + let state = TestAppStateBuilder::new() + .llm_catalog_settings(settings) + .vault_entries([ + ("DIRECT_API_KEY", "direct-test-key"), + ("AGGREGATOR_API_KEY", "aggregator-test-key"), + ]) + .build(); + let app = crate::test_support::build_test_router(state); + + let list = app + .clone() + .oneshot( + Request::builder() + .method("GET") + .uri(api("/models?query=portable-model")) + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + let list = response_json!(list, StatusCode::OK).await; + let rows = list["data"].as_array().unwrap(); + assert_eq!(rows.len(), 2); + assert_eq!( + rows.iter() + .map(|row| row["provider"].as_str().unwrap()) + .collect::>(), + std::collections::BTreeSet::from(["aggregator", "direct"]) + ); + assert!( + rows.iter() + .all(|row| row["id"] == "portable-model" && row["configured"] == true) + ); + + let filtered = app + .clone() + .oneshot( + Request::builder() + .method("GET") + .uri(api("/models?provider=aggregator&query=portable-model")) + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + let filtered = response_json!(filtered, StatusCode::OK).await; + assert_eq!(filtered["data"].as_array().unwrap().len(), 1); + assert_eq!(filtered["data"][0]["provider"], "aggregator"); + + for (query, expected_provider) in [ + ("?provider=direct", "direct"), + ("?provider=aggregator", "aggregator"), + ("", "direct"), + ] { + let response = app + .clone() + .oneshot( + Request::builder() + .method("POST") + .uri(api(&format!("/models/portable/test{query}"))) + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + let body = response_json!(response, StatusCode::OK).await; + assert_eq!(body["model_id"], "portable-model"); + assert_eq!(body["provider"], expected_provider); + assert_eq!(body["status"], "ok"); + } + + direct_probe.assert_calls(2); + aggregator_probe.assert_calls(1); +} + #[tokio::test] async fn test_model_invalid_mode_returns_400() { let state = test_app_state_with_env_lookup( @@ -13481,7 +13703,7 @@ async fn get_aggregate_billing_returns_provider_model_speed_identity() { agg.by_model.insert( ModelRef { provider: ProviderId::anthropic(), - model_id: "claude-opus-4-6".to_string(), + model_id: "claude-opus-4-6".into(), speed: None, }, ModelBillingTotals { @@ -13500,7 +13722,7 @@ async fn get_aggregate_billing_returns_provider_model_speed_identity() { agg.by_model.insert( ModelRef { provider: ProviderId::anthropic(), - model_id: "claude-opus-4-6".to_string(), + model_id: "claude-opus-4-6".into(), speed: Some(Speed::Fast), }, ModelBillingTotals { @@ -13567,7 +13789,7 @@ fn aggregate_billing_counts_projection_rollup_usage_visits() { fabro_workflow::ProjectionBillingByModel { model: ModelRef { provider: ProviderId::openai(), - model_id: "gpt-5.4".to_string(), + model_id: "gpt-5.4".into(), speed: None, }, stages: 1, @@ -13584,7 +13806,7 @@ fn aggregate_billing_counts_projection_rollup_usage_visits() { fabro_workflow::ProjectionBillingByModel { model: ModelRef { provider: ProviderId::openai(), - model_id: "gpt-5.4".to_string(), + model_id: "gpt-5.4".into(), speed: Some(Speed::Fast), }, stages: 1, @@ -13611,7 +13833,7 @@ fn aggregate_billing_counts_projection_rollup_usage_visits() { assert_eq!( accumulator.by_model[&ModelRef { provider: ProviderId::openai(), - model_id: "gpt-5.4".to_string(), + model_id: "gpt-5.4".into(), speed: None, }] .stages, @@ -13620,7 +13842,7 @@ fn aggregate_billing_counts_projection_rollup_usage_visits() { assert_eq!( accumulator.by_model[&ModelRef { provider: ProviderId::openai(), - model_id: "gpt-5.4".to_string(), + model_id: "gpt-5.4".into(), speed: None, }] .billing @@ -13630,7 +13852,7 @@ fn aggregate_billing_counts_projection_rollup_usage_visits() { assert_eq!( accumulator.by_model[&ModelRef { provider: ProviderId::openai(), - model_id: "gpt-5.4".to_string(), + model_id: "gpt-5.4".into(), speed: Some(Speed::Fast), }] .stages, @@ -13639,7 +13861,7 @@ fn aggregate_billing_counts_projection_rollup_usage_visits() { assert_eq!( accumulator.by_model[&ModelRef { provider: ProviderId::openai(), - model_id: "gpt-5.4".to_string(), + model_id: "gpt-5.4".into(), speed: Some(Speed::Fast), }] .billing @@ -13727,6 +13949,17 @@ level = "debug" manifest_run_defaults_from_toml(source), 5, ); + state + .stores + .vault + .set( + EnvVars::ANTHROPIC_API_KEY, + "test-anthropic-api-key", + SecretType::Token, + None, + ) + .await + .unwrap(); let app = crate::test_support::build_test_router(Arc::clone(&state)); let req = Request::builder() @@ -13892,6 +14125,7 @@ async fn cancel_run_overwrites_pending_pause_request() { async fn cancel_run_requests_worker_runtime_stop_when_control_unavailable() { let runtime = StdArc::new(RecordingWorkerRuntime::default()); let state = TestAppStateBuilder::new() + .vault_entries([(EnvVars::OPENAI_API_KEY, "test-openai-api-key")]) .worker_runtime(runtime.clone()) .build(); let app = crate::test_support::build_test_router(Arc::clone(&state)); @@ -14771,42 +15005,52 @@ async fn create_completion_unknown_provider_returns_clear_error() { let body = response_json!(response, StatusCode::BAD_REQUEST).await; assert_eq!( body["errors"][0]["detail"], - "Provider \"missing-provider\" is not configured" + "unknown model provider 'missing-provider'" ); } #[tokio::test] async fn create_completion_default_model_uses_app_state_catalog() { - let llm_catalog_settings: LlmCatalogSettings = toml::from_str( + let upstream = MockServer::start(); + let completion = upstream.mock(|when, then| { + when.method(POST) + .path("/chat/completions") + .json_body_includes(r#"{"model":"acme-large"}"#); + then.status(500) + .header("content-type", "application/json") + .json_body(json!({"error": {"message": "expected test failure"}})); + }); + let llm_catalog_settings: LlmCatalogSettings = toml::from_str(&format!( r#" [providers.acme] display_name = "Acme" adapter = "openai_compatible" agent_profile = "openai" -base_url = "https://api.acme.test/v1" +base_url = "{}" priority = 120 [providers.acme.auth] -credentials = ["env:ACME_API_KEY"] +credentials = ["vault:ACME_API_KEY"] -[models."acme-large"] -provider = "acme" +[providers.acme.models."acme-large"] display_name = "Acme Large" family = "acme" default = true -[models."acme-large".limits] +[providers.acme.models."acme-large".limits] context_window = 128000 -[models."acme-large".features] +[providers.acme.models."acme-large".features] tools = true vision = false reasoning = false "#, - ) + upstream.base_url() + )) .expect("catalog fixture should parse"); let state = TestAppStateBuilder::new() .llm_catalog_settings(llm_catalog_settings) + .vault_entries([("ACME_API_KEY", "acme-test-key")]) .build(); let app = crate::test_support::build_test_router(state); @@ -14834,9 +15078,10 @@ reasoning = false body["errors"][0]["detail"] .as_str() .unwrap() - .contains("Provider 'acme' not registered"), + .contains("expected test failure"), "unexpected error body: {body:?}" ); + completion.assert(); } #[tokio::test] diff --git a/lib/crates/fabro-server/src/spawn_env.rs b/lib/crates/fabro-server/src/spawn_env.rs index e4c58f2e1..bfea6b038 100644 --- a/lib/crates/fabro-server/src/spawn_env.rs +++ b/lib/crates/fabro-server/src/spawn_env.rs @@ -18,6 +18,8 @@ const WORKER_ENV_ALLOWLIST: &[&str] = &[ // reach the refresh-ahead loop in the ACP handler. EnvVars::FABRO_PUSH_CRED_REFRESH_AHEAD, EnvVars::FABRO_PUSH_CRED_REFRESH_INTERVAL_SECONDS, + #[cfg(feature = "test-support")] + "FABRO_TEST_ASSUME_LLM_READY", EnvVars::TERM, EnvVars::NO_COLOR, EnvVars::CLICOLOR, diff --git a/lib/crates/fabro-server/src/test_support.rs b/lib/crates/fabro-server/src/test_support.rs index 4394c02a9..8fe368fff 100644 --- a/lib/crates/fabro-server/src/test_support.rs +++ b/lib/crates/fabro-server/src/test_support.rs @@ -17,6 +17,7 @@ use fabro_config::{RunLayer, ServerSettingsBuilder, Storage, envfile}; use fabro_db::DbPool; use fabro_interview::Interviewer; use fabro_model::catalog::{LlmCatalogSettings, ProviderCatalogSettings}; +use fabro_model::{Catalog, ProviderId}; use fabro_sandbox::SandboxProviderRegistry; use fabro_static::EnvVars; use fabro_store::{ArtifactStore, Database}; @@ -49,6 +50,27 @@ pub const TEST_DEV_TOKEN: &str = "fabro_dev_abababababababababababababababababababababababababababababababab"; pub const TEST_SESSION_SECRET: &str = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; +const TEST_OPENAI_API_KEY: &str = "test-openai-api-key"; +const FABRO_TEST_ASSUME_LLM_READY: &str = "FABRO_TEST_ASSUME_LLM_READY"; + +/// Supply enabled catalog providers to CLI fixture run materialization. +/// +/// This is scoped to the `test-support` feature and an explicit child-process +/// flag. The CLI suite shares a server whose credential state can change +/// between tests, so the flag deliberately ignores that mutable state. It does +/// not register adapters or make model execution available. +pub(crate) fn test_run_materialization_provider_ids( + catalog: &Catalog, + ready_provider_ids: &[ProviderId], +) -> Vec { + let assume_ready = process_env_var(FABRO_TEST_ASSUME_LLM_READY) + .is_some_and(|value| !matches!(value.as_str(), "" | "0" | "false" | "no")); + if assume_ready { + catalog.all_provider_ids().into_iter().collect() + } else { + ready_provider_ids.to_vec() + } +} pub fn default_test_server_settings() -> ServerSettings { ServerSettingsBuilder::from_toml( @@ -320,13 +342,13 @@ pub fn llm_catalog_settings_with_provider_base_url( } pub fn test_app_state() -> Arc { - TestAppStateBuilder::new().build() + ready_test_app_state_builder().build() } pub fn test_app_state_with_registry_factory( registry_factory_override: impl Fn(Arc) -> HandlerRegistry + Send + Sync + 'static, ) -> Arc { - TestAppStateBuilder::new() + ready_test_app_state_builder() .registry_factory(registry_factory_override) .build() } @@ -336,7 +358,7 @@ pub fn test_app_state_with_settings_and_registry_factory( manifest_run_defaults: RunLayer, registry_factory_override: impl Fn(Arc) -> HandlerRegistry + Send + Sync + 'static, ) -> Arc { - TestAppStateBuilder::new() + ready_test_app_state_builder() .runtime_settings(server_settings, manifest_run_defaults) .registry_factory(registry_factory_override) .build() @@ -348,7 +370,7 @@ pub fn test_app_state_with_options_and_registry_factory( max_concurrent_runs: usize, registry_factory_override: impl Fn(Arc) -> HandlerRegistry + Send + Sync + 'static, ) -> Arc { - TestAppStateBuilder::new() + ready_test_app_state_builder() .runtime_settings(server_settings, manifest_run_defaults) .max_concurrent_runs(max_concurrent_runs) .registry_factory(registry_factory_override) @@ -360,12 +382,16 @@ pub fn test_app_state_with_options( manifest_run_defaults: RunLayer, max_concurrent_runs: usize, ) -> Arc { - TestAppStateBuilder::new() + ready_test_app_state_builder() .runtime_settings(server_settings, manifest_run_defaults) .max_concurrent_runs(max_concurrent_runs) .build() } +fn ready_test_app_state_builder() -> TestAppStateBuilder { + TestAppStateBuilder::new().vault_entries([(EnvVars::OPENAI_API_KEY, TEST_OPENAI_API_KEY)]) +} + pub(crate) fn resolved_runtime_settings_for_tests( server_settings: ServerSettings, manifest_run_defaults: RunLayer, @@ -383,7 +409,7 @@ pub fn test_app_state_with_runtime_settings_and_registry_factory( manifest_run_defaults: RunLayer, registry_factory_override: impl Fn(Arc) -> HandlerRegistry + Send + Sync + 'static, ) -> Arc { - TestAppStateBuilder::new() + ready_test_app_state_builder() .runtime_settings(server_settings, manifest_run_defaults) .registry_factory(registry_factory_override) .build() @@ -395,7 +421,7 @@ pub fn test_app_state_with_runtime_settings_and_options_and_registry_factory( max_concurrent_runs: usize, registry_factory_override: impl Fn(Arc) -> HandlerRegistry + Send + Sync + 'static, ) -> Arc { - TestAppStateBuilder::new() + ready_test_app_state_builder() .runtime_settings(server_settings, manifest_run_defaults) .max_concurrent_runs(max_concurrent_runs) .registry_factory(registry_factory_override) @@ -407,7 +433,7 @@ pub fn test_app_state_with_runtime_settings_and_options( manifest_run_defaults: RunLayer, max_concurrent_runs: usize, ) -> Arc { - TestAppStateBuilder::new() + ready_test_app_state_builder() .runtime_settings(server_settings, manifest_run_defaults) .max_concurrent_runs(max_concurrent_runs) .build() @@ -475,7 +501,7 @@ pub fn test_app_state_with_runtime_settings_and_session_key( ) .expect("test server env should be writable"); } - TestAppStateBuilder::new() + ready_test_app_state_builder() .runtime_settings(server_settings, manifest_run_defaults) .vault_path(vault_path) .server_env_path(server_env_path) @@ -501,7 +527,7 @@ pub fn test_app_state_with_store( store: Arc, artifact_store: ArtifactStore, ) -> Arc { - TestAppStateBuilder::new() + ready_test_app_state_builder() .runtime_settings(server_settings, manifest_run_defaults) .max_concurrent_runs(max_concurrent_runs) .store_bundle(store, artifact_store) @@ -612,7 +638,7 @@ pub fn test_app_state_with_store_and_runtime_settings( store: Arc, artifact_store: ArtifactStore, ) -> Arc { - TestAppStateBuilder::new() + ready_test_app_state_builder() .runtime_settings(server_settings, manifest_run_defaults) .max_concurrent_runs(max_concurrent_runs) .store_bundle(store, artifact_store) diff --git a/lib/crates/fabro-server/tests/it/api/automations.rs b/lib/crates/fabro-server/tests/it/api/automations.rs index 4a2b96de0..f17bab0e8 100644 --- a/lib/crates/fabro-server/tests/it/api/automations.rs +++ b/lib/crates/fabro-server/tests/it/api/automations.rs @@ -7,6 +7,7 @@ use fabro_server::server::build_router; use fabro_server::test_support::{ TestAppStateBuilder, TestAutomationRunMaterializer, build_test_router, test_auth_mode, }; +use fabro_static::EnvVars; use serde_json::{Value, json}; use sqlx::Row as _; use tower::ServiceExt; @@ -86,6 +87,7 @@ fn automation_app_with_fake_materializer() -> (axum::Router, tempfile::TempDir, let state = TestAppStateBuilder::new() .active_config_path(active_config_path) .vault_path(vault_path) + .vault_entries([(EnvVars::OPENAI_API_KEY, "test-openai-api-key")]) .automation_materializer(TestAutomationRunMaterializer::succeed( materialized_manifest, submitted_manifest_bytes, diff --git a/lib/crates/fabro-server/tests/it/api/events.rs b/lib/crates/fabro-server/tests/it/api/events.rs index e29b51b21..a5c8153c7 100644 --- a/lib/crates/fabro-server/tests/it/api/events.rs +++ b/lib/crates/fabro-server/tests/it/api/events.rs @@ -4,6 +4,7 @@ use std::time::Duration; use axum::body::{Body, to_bytes}; use axum::http::{Request, StatusCode}; use chrono::{SecondsFormat, Utc}; +use fabro_static::EnvVars; use object_store::ObjectStore; use object_store::memory::InMemory; use tokio::sync::Barrier; @@ -23,6 +24,7 @@ fn app_with_store(object_store: Arc) -> axum::Router { let state = fabro_server::test_support::TestAppStateBuilder::new() .runtime_settings(settings.server_settings, settings.manifest_run_defaults) .env_lookup(|_| None) + .vault_entries([(EnvVars::OPENAI_API_KEY, "test-openai-api-key")]) .store_bundle(store, artifact_store) .build(); fabro_server::test_support::build_test_router(state) diff --git a/lib/crates/fabro-server/tests/it/api/sessions.rs b/lib/crates/fabro-server/tests/it/api/sessions.rs index dc8217198..c989aa4c3 100644 --- a/lib/crates/fabro-server/tests/it/api/sessions.rs +++ b/lib/crates/fabro-server/tests/it/api/sessions.rs @@ -1,10 +1,10 @@ use axum::body::{Body, to_bytes}; use axum::http::{Request, StatusCode}; +use fabro_static::EnvVars; use tower::ServiceExt; use crate::helpers::{ MINIMAL_DOT, api, minimal_manifest_json, response_json, response_status, test_app_state, - test_app_with_no_providers, }; async fn create_run(app: &axum::Router) -> String { @@ -196,6 +196,7 @@ async fn supplied_session_model_alias_is_canonicalized() { .find(|event| event["event"] == "run.session.created") .expect("session creation event should be recorded"); assert_eq!(created_event["properties"]["model"], "gpt-5.4"); + assert_eq!(created_event["properties"]["provider"], "openai"); } #[tokio::test] @@ -203,9 +204,23 @@ async fn provider_qualified_session_model_is_canonicalized() { let app = fabro_server::test_support::build_test_router(test_app_state()); let run_id = create_run(&app).await; - let created = create_session_with_model(&app, &run_id, "Ask Fabro", "openai/gpt-5.4").await; + let created = + create_session_with_model(&app, &run_id, "Ask Fabro", "openai/gpt-5.4-mini").await; - assert_eq!(created["model"], "gpt-5.4"); + assert_eq!(created["model"], "gpt-5.4-mini"); + assert_eq!(created["provider"], "openai"); +} + +#[tokio::test] +async fn unknown_session_models_preserve_passthrough_on_the_selected_provider() { + let app = fabro_server::test_support::build_test_router(test_app_state()); + let run_id = create_run(&app).await; + + for model in ["not-a-real-model", "openai/not-a-real-model"] { + let created = create_session_with_model(&app, &run_id, "Ask Fabro", model).await; + assert_eq!(created["model"], "not-a-real-model"); + assert_eq!(created["provider"], "openai"); + } } #[tokio::test] @@ -213,7 +228,7 @@ async fn invalid_session_model_refs_are_rejected_at_creation() { let app = fabro_server::test_support::build_test_router(test_app_state()); let run_id = create_run(&app).await; - for model in ["not-a-real-model", "openai", "openai/", "anthropic/gpt-5.4"] { + for model in ["openai", "openai/", "anthropic/gpt-5.4"] { let response = create_session_response( &app, &run_id, @@ -241,6 +256,7 @@ async fn ambiguous_session_model_refs_are_rejected_at_creation() { ); let state = fabro_server::test_support::TestAppStateBuilder::new() .llm_catalog_settings(catalog_settings) + .vault_entries([(EnvVars::OPENAI_API_KEY, "test-openai-api-key")]) .build(); let app = fabro_server::test_support::build_test_router(state); let run_id = create_run(&app).await; @@ -260,13 +276,14 @@ async fn ambiguous_session_model_refs_are_rejected_at_creation() { } #[tokio::test] -async fn session_turn_fails_when_selected_model_provider_is_unconfigured() { - let app = test_app_with_no_providers(); +async fn session_turn_fails_when_selected_model_provider_becomes_unconfigured() { + let app = fabro_server::test_support::build_test_router(test_app_state()); let run_id = create_run(&app).await; let created = create_session_with_model(&app, &run_id, "Ask Fabro", "gpt54").await; let session_id = created["id"] .as_str() .expect("session response should include an id"); + delete_openai_credential(&app).await; let request = Request::builder() .method("POST") @@ -347,7 +364,7 @@ async fn unsupported_derived_turn_read_routes_are_removed() { #[tokio::test] async fn session_events_are_filtered_by_session_and_paginated_by_run_sequence() { - let app = test_app_with_no_providers(); + let app = fabro_server::test_support::build_test_router(test_app_state()); let run_id = create_run(&app).await; let first = create_session(&app, &run_id, "First").await; let second = create_session(&app, &run_id, "Second").await; @@ -365,6 +382,7 @@ async fn session_events_are_filtered_by_session_and_paginated_by_run_sequence() ) .await; let after_first_created_seq = first_detail["last_seq"].as_u64().unwrap() + 1; + delete_openai_credential(&app).await; let turn_id = fabro_types::TurnId::new(); let submit = Request::builder() @@ -443,13 +461,33 @@ async fn session_sse_events(response: axum::response::Response) -> Vec = Arc::new( EnvCredentialSource::with_env_lookup(Arc::new(move |name| match name { "OPENAI_API_KEY" => Some(source_api_key.clone()), @@ -52,6 +53,7 @@ fn test_app_with_openai_agent_backend(openai_base_url: String, api_key: String) .runtime_settings(settings.server_settings, settings.manifest_run_defaults) .max_concurrent_runs(5) .llm_catalog_settings(llm_catalog_settings) + .vault_entries([(EnvVars::OPENAI_API_KEY, api_key)]) .registry_factory(move |interviewer| { let catalog = Arc::clone(&catalog); let llm_source = Arc::clone(&llm_source); diff --git a/lib/crates/fabro-store/src/run_sessions.rs b/lib/crates/fabro-store/src/run_sessions.rs index 4fe396c72..a5998baed 100644 --- a/lib/crates/fabro-store/src/run_sessions.rs +++ b/lib/crates/fabro-store/src/run_sessions.rs @@ -77,6 +77,7 @@ impl RunSessionProjection { let mut record = SessionRecord::new(session_id, run_id, envelope.event.ts); record.title.clone_from(&props.title); record.model.clone_from(&props.model); + record.provider.clone_from(&props.provider); let projected = ProjectedRunSession { record, runtime_context: Vec::new(), @@ -254,8 +255,9 @@ mod tests { 1, session_id, EventBody::RunSessionCreated(RunSessionCreatedProps { - title: Some("Ask".to_string()), - model: Some("test-model".to_string()), + title: Some("Ask".to_string()), + model: Some("test-model".to_string()), + provider: None, }), ), event( @@ -318,8 +320,9 @@ mod tests { 1, session_id, EventBody::RunSessionCreated(RunSessionCreatedProps { - title: None, - model: None, + title: None, + model: None, + provider: None, }), ), event( @@ -387,8 +390,9 @@ mod tests { 1, session_id, EventBody::RunSessionCreated(RunSessionCreatedProps { - title: Some("Ask".to_string()), - model: Some("test-model".to_string()), + title: Some("Ask".to_string()), + model: Some("test-model".to_string()), + provider: None, }), ), event( @@ -408,7 +412,7 @@ mod tests { assert!(value.get("runtime_context").is_none()); assert!(value.get("working_dir").is_none()); - assert!(value.get("provider").is_none()); + assert!(value["provider"].is_null()); assert!(value.get("permissions").is_none()); assert!(value.get("deleted_at").is_none()); } @@ -423,8 +427,9 @@ mod tests { 1, session_id, EventBody::RunSessionCreated(RunSessionCreatedProps { - title: None, - model: None, + title: None, + model: None, + provider: None, }), ), event( @@ -439,8 +444,9 @@ mod tests { 3, other_session_id, EventBody::RunSessionCreated(RunSessionCreatedProps { - title: Some("Other".to_string()), - model: None, + title: Some("Other".to_string()), + model: None, + provider: None, }), ), ]; @@ -478,8 +484,9 @@ mod tests { 1, session_id, EventBody::RunSessionCreated(RunSessionCreatedProps { - title: None, - model: None, + title: None, + model: None, + provider: None, }), ), event( diff --git a/lib/crates/fabro-store/src/run_state.rs b/lib/crates/fabro-store/src/run_state.rs index ec67aa4f0..0d21ba098 100644 --- a/lib/crates/fabro-store/src/run_state.rs +++ b/lib/crates/fabro-store/src/run_state.rs @@ -1036,7 +1036,7 @@ fn run_models(state: &RunProjection) -> Vec { .filter_map(|(_, stage)| stage.model.as_ref()) .map(|model| RunModel { provider: Some(model.provider.to_string()), - name: model.model_id.clone(), + name: model.model_id.to_string(), }) .collect::>(); models.sort_by(|left, right| { diff --git a/lib/crates/fabro-test/src/lib.rs b/lib/crates/fabro-test/src/lib.rs index bfa267320..9a3e9bc86 100644 --- a/lib/crates/fabro-test/src/lib.rs +++ b/lib/crates/fabro-test/src/lib.rs @@ -85,6 +85,7 @@ const TEST_SESSION_SECRET: &str = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; const TEST_DEV_TOKEN: &str = "fabro_dev_abababababababababababababababababababababababababababababababab"; +const FABRO_TEST_ASSUME_LLM_READY: &str = "FABRO_TEST_ASSUME_LLM_READY"; #[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] pub enum TestMode { @@ -187,6 +188,7 @@ pub fn isolated_env(home_dir: &Path) -> HashMap { EnvVars::FABRO_TEST_IN_MEMORY_STORE.to_string(), "1".to_string(), ); + env.insert(FABRO_TEST_ASSUME_LLM_READY.to_string(), "1".to_string()); env } @@ -210,6 +212,7 @@ fn apply_test_isolation_with_lookup( .env(EnvVars::FABRO_SUPPRESS_OPEN_BROWSER, "1"); cmd.env(EnvVars::FABRO_SERVER_MAX_CONCURRENT_RUNS, "64"); cmd.env(EnvVars::FABRO_TEST_IN_MEMORY_STORE, "1"); + cmd.env(FABRO_TEST_ASSUME_LLM_READY, "1"); } /// Create a fresh tempdir containing an empty `storage/` subdirectory, for diff --git a/lib/crates/fabro-types/src/run_event/agent.rs b/lib/crates/fabro-types/src/run_event/agent.rs index 64b2ede3d..ada131b09 100644 --- a/lib/crates/fabro-types/src/run_event/agent.rs +++ b/lib/crates/fabro-types/src/run_event/agent.rs @@ -382,7 +382,7 @@ mod tests { fn sample_model_ref() -> ModelRef { ModelRef { provider: fabro_model::ProviderId::openai(), - model_id: "gpt-5".to_string(), + model_id: "gpt-5".into(), speed: None, } } diff --git a/lib/crates/fabro-types/src/run_event/mod.rs b/lib/crates/fabro-types/src/run_event/mod.rs index e76558f3e..bfb396663 100644 --- a/lib/crates/fabro-types/src/run_event/mod.rs +++ b/lib/crates/fabro-types/src/run_event/mod.rs @@ -2135,7 +2135,7 @@ mod tests { text: "ok".to_string(), model: crate::ModelRef { provider: fabro_model::ProviderId::openai(), - model_id: "gpt-5.4".to_string(), + model_id: "gpt-5.4".into(), speed: None, }, billing: BilledTokenCounts::default(), @@ -2187,7 +2187,7 @@ mod tests { text: "ok".to_string(), model: crate::ModelRef { provider: fabro_model::ProviderId::openai(), - model_id: "gpt-5.4".to_string(), + model_id: "gpt-5.4".into(), speed: None, }, billing: BilledTokenCounts::default(), diff --git a/lib/crates/fabro-types/src/run_event/session.rs b/lib/crates/fabro-types/src/run_event/session.rs index a44c04003..504d4f653 100644 --- a/lib/crates/fabro-types/src/run_event/session.rs +++ b/lib/crates/fabro-types/src/run_event/session.rs @@ -1,3 +1,4 @@ +use fabro_model::ProviderId; use serde::{Deserialize, Serialize}; use serde_json::Value; @@ -6,9 +7,11 @@ use crate::TurnId; #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct RunSessionCreatedProps { #[serde(default, skip_serializing_if = "Option::is_none")] - pub title: Option, + pub title: Option, #[serde(default, skip_serializing_if = "Option::is_none")] - pub model: Option, + pub model: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub provider: Option, } #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] diff --git a/lib/crates/fabro-types/src/session.rs b/lib/crates/fabro-types/src/session.rs index 4e704e239..eaad05181 100644 --- a/lib/crates/fabro-types/src/session.rs +++ b/lib/crates/fabro-types/src/session.rs @@ -1,4 +1,5 @@ use chrono::{DateTime, Utc}; +use fabro_model::ProviderId; use serde::{Deserialize, Serialize}; use strum::{Display, EnumString, IntoStaticStr}; @@ -63,6 +64,8 @@ pub struct SessionRecord { pub status: SessionStatus, pub model: Option, #[serde(default)] + pub provider: Option, + #[serde(default)] pub active_turn: Option, pub created_at: DateTime, pub updated_at: DateTime, @@ -76,6 +79,7 @@ impl SessionRecord { title: None, status: SessionStatus::Idle, model: None, + provider: None, active_turn: None, created_at: now, updated_at: now, @@ -91,6 +95,8 @@ pub struct SessionSummary { pub status: SessionStatus, pub model: Option, #[serde(default)] + pub provider: Option, + #[serde(default)] pub active_turn: Option, pub created_at: DateTime, pub updated_at: DateTime, @@ -104,6 +110,7 @@ impl From<&SessionRecord> for SessionSummary { title: record.title.clone(), status: record.status, model: record.model.clone(), + provider: record.provider.clone(), active_turn: record.active_turn.clone(), created_at: record.created_at, updated_at: record.updated_at, diff --git a/lib/crates/fabro-types/src/settings/model_ref.rs b/lib/crates/fabro-types/src/settings/model_ref.rs index ee912ff51..eb12263da 100644 --- a/lib/crates/fabro-types/src/settings/model_ref.rs +++ b/lib/crates/fabro-types/src/settings/model_ref.rs @@ -142,9 +142,6 @@ pub enum ResolvedModelRef { pub trait ModelRegistry { fn is_provider(&self, token: &str) -> bool; fn is_model(&self, token: &str) -> bool; - /// Returns the canonical provider for a bare model token, when the registry - /// knows of a unique provider for that model. - fn provider_of(&self, token: &str) -> Option; } impl ModelRef { @@ -169,17 +166,13 @@ impl ModelRef { let is_model = registry.is_model(token); match (is_provider, is_model) { (true, false) => Ok(ResolvedModelRef::Provider(token.clone())), - (false, true) => Ok(ResolvedModelRef::Model { - provider: registry.provider_of(token), - model: token.clone(), - }), (true, true) => Err(AmbiguousModelRef { input: token.clone(), providers: vec![token.clone()], models: vec![token.clone()], }), - // Unknown tokens flow through as bare models — provider TBD at runtime. - (false, false) => Ok(ResolvedModelRef::Model { + // Known and unknown bare models leave provider selection to the runtime. + (false, _) => Ok(ResolvedModelRef::Model { provider: None, model: token.clone(), }), @@ -237,13 +230,6 @@ mod tests { fn is_model(&self, token: &str) -> bool { self.models.contains(&token) } - fn provider_of(&self, token: &str) -> Option { - if self.models.contains(&token) { - Some("test".to_owned()) - } else { - None - } - } } #[test] @@ -309,7 +295,7 @@ mod tests { }; let resolved = ModelRef::Bare("gpt-5.4".into()).resolve(®).unwrap(); assert_eq!(resolved, ResolvedModelRef::Model { - provider: Some("test".into()), + provider: None, model: "gpt-5.4".into(), }); } diff --git a/lib/crates/fabro-validate/src/rules/model_support.rs b/lib/crates/fabro-validate/src/rules/model_support.rs index 91df02991..7bab56a87 100644 --- a/lib/crates/fabro-validate/src/rules/model_support.rs +++ b/lib/crates/fabro-validate/src/rules/model_support.rs @@ -7,7 +7,7 @@ pub(super) fn check_model_known( context: &str, node_id: Option, ) -> Option { - if catalog.get(model).is_some() { + if catalog.is_model_selector(model) { return None; } Some(Diagnostic { diff --git a/lib/crates/fabro-workflow/Cargo.toml b/lib/crates/fabro-workflow/Cargo.toml index ca74a4878..fc7166d7a 100644 --- a/lib/crates/fabro-workflow/Cargo.toml +++ b/lib/crates/fabro-workflow/Cargo.toml @@ -13,6 +13,9 @@ readme = "README.md" [lib] doctest = false +[features] +test-support = [] + [lints] workspace = true diff --git a/lib/crates/fabro-workflow/src/billing_rollup.rs b/lib/crates/fabro-workflow/src/billing_rollup.rs index 9762b27f4..4ba8976bc 100644 --- a/lib/crates/fabro-workflow/src/billing_rollup.rs +++ b/lib/crates/fabro-workflow/src/billing_rollup.rs @@ -294,7 +294,7 @@ mod tests { let mut projection = test_projection(); let model = ModelRef { provider: ProviderId::openai(), - model_id: "gpt-5.4".to_string(), + model_id: "gpt-5.4".into(), speed: None, }; let stage = projection.stage_entry("agent", 1, first_event_seq(1)); diff --git a/lib/crates/fabro-workflow/src/error.rs b/lib/crates/fabro-workflow/src/error.rs index 241787868..71ded36ac 100644 --- a/lib/crates/fabro-workflow/src/error.rs +++ b/lib/crates/fabro-workflow/src/error.rs @@ -3,9 +3,11 @@ use std::sync::Arc; use fabro_graphviz::Error as GraphvizError; use fabro_llm::{Error as LlmError, ProviderErrorKind}; +use fabro_model::ModelSelectionError; use fabro_template::TemplateError; pub use fabro_types::failure_signature::FailureSignature; pub use fabro_types::outcome::FailureCategory; +use fabro_types::settings::AmbiguousModelRef; use fabro_types::{ExecOutputTail, FailureReason, RunFailure}; use fabro_util::error::{SharedError, collect_causes, collect_chain, render_with_causes}; use fabro_validate::Diagnostic; @@ -262,6 +264,12 @@ pub enum Error { #[error("Validation failed")] ValidationFailed { diagnostics: Vec }, + #[error("Model selection failed: {0}")] + ModelSelection(#[from] ModelSelectionError), + + #[error("Model reference failed: {0}")] + ModelReference(#[from] AmbiguousModelRef), + #[error("{message}")] Template { message: String, @@ -445,6 +453,8 @@ impl Error { Self::Parse(_) | Self::Validation(_) | Self::ValidationFailed { .. } + | Self::ModelSelection(_) + | Self::ModelReference(_) | Self::Template { .. } | Self::Stylesheet(_) | Self::Checkpoint(_) @@ -466,6 +476,8 @@ impl Error { Self::Parse(_) | Self::Validation(_) | Self::ValidationFailed { .. } + | Self::ModelSelection(_) + | Self::ModelReference(_) | Self::Template { .. } | Self::Stylesheet(_) | Self::Checkpoint(_) diff --git a/lib/crates/fabro-workflow/src/event/convert.rs b/lib/crates/fabro-workflow/src/event/convert.rs index 5f0fc9959..8aa10c200 100644 --- a/lib/crates/fabro-workflow/src/event/convert.rs +++ b/lib/crates/fabro-workflow/src/event/convert.rs @@ -2112,7 +2112,7 @@ mod tests { text: "ok".to_string(), model: ModelRef { provider: ProviderId::anthropic(), - model_id: "claude-sonnet".to_string(), + model_id: "claude-sonnet".into(), speed: None, }, usage: LlmTokenCounts::default(), @@ -2140,7 +2140,7 @@ mod tests { text: "ok".to_string(), model: ModelRef { provider: ProviderId::new("custom_proxy"), - model_id: "proxy-model".to_string(), + model_id: "proxy-model".into(), speed: None, }, usage: LlmTokenCounts { @@ -2192,7 +2192,7 @@ mod tests { text: "ok".to_string(), model: ModelRef { provider: ProviderId::openai(), - model_id: "gpt-5.4".to_string(), + model_id: "gpt-5.4".into(), speed: None, }, usage: LlmTokenCounts::default(), diff --git a/lib/crates/fabro-workflow/src/event/stored_fields.rs b/lib/crates/fabro-workflow/src/event/stored_fields.rs index 4ada17b67..285bd9c89 100644 --- a/lib/crates/fabro-workflow/src/event/stored_fields.rs +++ b/lib/crates/fabro-workflow/src/event/stored_fields.rs @@ -324,7 +324,7 @@ fn agent_actor_for_event( AgentEvent::AssistantMessage { model, .. } => Some(Principal::Agent { session_id: session_id.map(str::to_string), parent_session_id: parent_session_id.map(str::to_string), - model: Some(model.model_id.clone()), + model: Some(model.model_id.to_string()), }), AgentEvent::ToolCallStarted { .. } | AgentEvent::ToolCallOutputDelta { .. } diff --git a/lib/crates/fabro-workflow/src/handler/llm/api.rs b/lib/crates/fabro-workflow/src/handler/llm/api.rs index c1f27a7a3..9cb5469bb 100644 --- a/lib/crates/fabro-workflow/src/handler/llm/api.rs +++ b/lib/crates/fabro-workflow/src/handler/llm/api.rs @@ -935,7 +935,7 @@ impl AgentApiBackend { .clone() .unwrap_or_else(|| default_provider.clone()), ), - model_id: request.model.clone(), + model_id: request.model.clone().into(), speed: controls.speed, }), Err(sdk_err) if sdk_err.failover_eligible() && !fallback_chain.is_empty() => { @@ -964,7 +964,7 @@ impl AgentApiBackend { let max_tokens = node.max_tokens().or_else(|| { self.catalog - .get(&target.model) + .get_on_provider(&ProviderId::new(&target.provider), &target.model) .and_then(|model| model.limits.max_output) }); @@ -983,7 +983,7 @@ impl AgentApiBackend { response: resp, model: ModelRef { provider: ProviderId::from(target.provider.clone()), - model_id: target.model.clone(), + model_id: target.model.clone().into(), speed: controls.speed, }, }); @@ -1034,9 +1034,11 @@ impl CodergenBackend for AgentApiBackend { let provider_id = provider.provider_id.to_string(); let controls = self.resolve_effective_request_controls(node)?; - let max_tokens = node - .max_tokens() - .or_else(|| self.catalog.get(model).and_then(|m| m.limits.max_output)); + let max_tokens = node.max_tokens().or_else(|| { + self.catalog + .get_on_provider(&provider.provider_id, model) + .and_then(|model| model.limits.max_output) + }); let mut messages = Vec::new(); if let Some(sys) = system_prompt { @@ -1513,7 +1515,7 @@ impl CodergenBackend for AgentApiBackend { self.catalog.as_ref(), &ModelRef { provider: session.provider_id(), - model_id: session.model().to_string(), + model_id: session.model().into(), speed: billing_controls.speed, }, &total_usage, diff --git a/lib/crates/fabro-workflow/src/handler/llm/preamble.rs b/lib/crates/fabro-workflow/src/handler/llm/preamble.rs index 6d1bd106d..fbceff0d7 100644 --- a/lib/crates/fabro-workflow/src/handler/llm/preamble.rs +++ b/lib/crates/fabro-workflow/src/handler/llm/preamble.rs @@ -597,7 +597,7 @@ mod tests { Catalog::builtin(), &ModelRef { provider: ProviderId::anthropic(), - model_id: model.to_string(), + model_id: model.into(), speed: None, }, &TokenCounts { diff --git a/lib/crates/fabro-workflow/src/handler/llm/routing.rs b/lib/crates/fabro-workflow/src/handler/llm/routing.rs index 952c31939..0bb05b216 100644 --- a/lib/crates/fabro-workflow/src/handler/llm/routing.rs +++ b/lib/crates/fabro-workflow/src/handler/llm/routing.rs @@ -58,10 +58,14 @@ pub(crate) fn resolve_provider_context( })? .id .clone() - } else if let Some(model) = catalog.get(model) { - model.provider.clone() } else { - default_provider_id.clone() + match catalog.select(model, None, &catalog.all_provider_ids()) { + Ok(model) => model.provider.clone(), + Err(fabro_model::ModelSelectionError::UnknownSelector { .. }) => { + default_provider_id.clone() + } + Err(error) => return Err(error.into()), + } }; let provider = catalog.provider(&provider_id).ok_or_else(|| { diff --git a/lib/crates/fabro-workflow/src/operations/create.rs b/lib/crates/fabro-workflow/src/operations/create.rs index 7d42f2c35..488bb5c74 100644 --- a/lib/crates/fabro-workflow/src/operations/create.rs +++ b/lib/crates/fabro-workflow/src/operations/create.rs @@ -303,6 +303,15 @@ fn create_from_source( template_context, goal_override, RenderMode::Structural, + options + .settings + .run + .model + .provider + .as_deref() + .filter(|provider| !provider.is_empty()) + .map(ProviderId::new), + &options.configured_providers, &options.catalog, )?; @@ -325,6 +334,8 @@ pub(super) fn preprocess_and_validate( template_context: TemplateContext, goal_override: Option<&str>, render_mode: RenderMode, + default_provider: Option, + eligible_providers: &[ProviderId], catalog: &Arc, ) -> Result { let mut parsed = pipeline::parse(dot_source)?; @@ -338,6 +349,8 @@ pub(super) fn preprocess_and_validate( render_mode, custom_transforms, catalog: Arc::clone(catalog), + default_provider, + eligible_providers: eligible_providers.iter().cloned().collect(), })?; Ok(pipeline::validate(transformed, catalog.as_ref(), &[])) } @@ -391,7 +404,7 @@ fn persist_validated( validated.graph(), catalog.as_ref(), &configured_providers, - ); + )?; let run_id = run_id.unwrap_or_else(RunId::new); let run_dir = run_dir.unwrap_or_else(|| default_run_dir(&run_id)); @@ -483,6 +496,59 @@ mod tests { Arc::new(Catalog::from_builtin().unwrap()) } + fn portable_model_catalog() -> Arc { + let settings: fabro_model::catalog::LlmCatalogSettings = toml::from_str( + r#" +[providers.openai] +display_name = "OpenAI" +adapter = "openai" +agent_profile = "openai" +priority = 90 + +[providers.openai.models."gpt-5.6-sol"] +display_name = "GPT-5.6 Sol" +family = "gpt-5" +aliases = ["gpt-56-sol"] +default = true + +[providers.openai.models."gpt-5.6-sol".limits] +context_window = 1000 + +[providers.openai.models."gpt-5.6-sol".features] +tools = true +vision = false +reasoning = false + +[providers.openrouter] +display_name = "OpenRouter" +adapter = "openai_compatible" +agent_profile = "openai" +priority = 25 + +[providers.openrouter.models."gpt-5.6-sol"] +api_id = "openai/gpt-5.6-sol" +display_name = "GPT-5.6 Sol (via OpenRouter)" +family = "gpt-5" +aliases = ["gpt-56-sol"] +default = true + +[providers.openrouter.models."gpt-5.6-sol".limits] +context_window = 1000 + +[providers.openrouter.models."gpt-5.6-sol".features] +tools = true +vision = false +reasoning = false +"#, + ) + .unwrap(); + Arc::new(Catalog::from_settings(&settings).unwrap()) + } + + fn test_provider_ids() -> Vec { + Catalog::builtin().all_provider_ids().into_iter().collect() + } + fn validate_dot(dot_source: &str, settings: WorkflowSettings) -> Validated { validate(ValidateInput { workflow: WorkflowInput::DotSource { @@ -511,6 +577,8 @@ mod tests { template_context(Some(&WorkflowSettings::default()), vars), None, RenderMode::Structural, + None, + &test_provider_ids(), &test_catalog(), ) .unwrap() @@ -679,6 +747,8 @@ mod tests { template_context(Some(&WorkflowSettings::default()), HashMap::new()), None, RenderMode::Strict, + None, + &test_provider_ids(), &test_catalog(), ); let Err(err) = result else { @@ -715,6 +785,8 @@ mod tests { template_context(Some(&WorkflowSettings::default()), HashMap::new()), None, RenderMode::Strict, + None, + &test_provider_ids(), &test_catalog(), ); let Err(err) = result else { @@ -1245,7 +1317,7 @@ mod tests { fork_source_ref: None, parent_id: None, provenance: test_support::test_run_provenance(), - configured_providers: Vec::new(), + configured_providers: test_provider_ids(), web_url: None, }, storage_root, @@ -1318,7 +1390,7 @@ mod tests { fork_source_ref: None, parent_id: None, provenance: test_support::test_run_provenance(), - configured_providers: Vec::new(), + configured_providers: test_provider_ids(), web_url: None, }, storage_root.clone(), @@ -1389,6 +1461,97 @@ mod tests { assert!(created.run_dir.is_dir()); } + #[tokio::test] + async fn create_materializes_shared_alias_for_ready_provider_snapshot_and_pin() { + const ALIAS_DOT: &str = r#"digraph Test { + graph [goal="Test"] + start [shape=Mdiamond] + work [prompt="Do work", model="gpt-56-sol"] + exit [shape=Msquare] + start -> work -> exit + }"#; + let catalog = portable_model_catalog(); + let cases = [ + (vec![ProviderId::openai()], None, ProviderId::openai()), + ( + vec![ProviderId::new("openrouter")], + None, + ProviderId::new("openrouter"), + ), + ( + vec![ProviderId::openai(), ProviderId::new("openrouter")], + None, + ProviderId::openai(), + ), + ( + vec![ProviderId::openai(), ProviderId::new("openrouter")], + Some("openrouter"), + ProviderId::new("openrouter"), + ), + ]; + + for (ready, explicit_provider, expected_provider) in cases { + let dir = tempfile::tempdir().unwrap(); + let mut settings = test_default_settings(); + settings.run.model.name = Some("gpt-56-sol".to_string()); + settings.run.model.provider = explicit_provider.map(str::to_string); + let store = memory_store(); + let created = create( + store.as_ref(), + CreateRunInput { + workflow: WorkflowInput::DotSource { + source: ALIAS_DOT.to_string(), + base_dir: None, + }, + settings, + vars: HashMap::new(), + cwd: dir.path().to_path_buf(), + workflow_slug: None, + workflow_path: None, + workflow_bundle: None, + submitted_manifest_bytes: None, + run_id: None, + title: None, + automation: None, + git: None, + fork_source_ref: None, + parent_id: None, + provenance: test_support::test_run_provenance(), + configured_providers: ready, + web_url: None, + }, + dir.path().join("storage"), + Arc::clone(&catalog), + ) + .await + .unwrap(); + let run_spec = created.persisted.run_spec(); + + assert_eq!( + run_spec.settings.run.model.name.as_deref(), + Some("gpt-5.6-sol") + ); + assert_eq!( + run_spec.settings.run.model.provider.as_deref(), + Some(expected_provider.as_str()) + ); + assert_eq!( + run_spec.graph.nodes["work"] + .attrs + .get("model") + .and_then(AttrValue::as_str), + Some("gpt-5.6-sol") + ); + assert_eq!( + run_spec.graph.nodes["work"] + .attrs + .get("provider") + .and_then(AttrValue::as_str), + Some(expected_provider.as_str()) + ); + } + } + #[tokio::test] async fn create_persists_secret_tokens_in_run_created_settings_source_form() { let dir = tempfile::tempdir().unwrap(); @@ -1435,7 +1598,7 @@ mod tests { fork_source_ref: None, parent_id: None, provenance: test_support::test_run_provenance(), - configured_providers: Vec::new(), + configured_providers: test_provider_ids(), web_url: None, }, storage_root, @@ -1512,7 +1675,7 @@ mod tests { fork_source_ref: None, parent_id: None, provenance: test_support::test_run_provenance(), - configured_providers: Vec::new(), + configured_providers: test_provider_ids(), web_url: None, }, storage_root, @@ -1559,7 +1722,7 @@ mod tests { fork_source_ref: None, parent_id: None, provenance: test_support::test_run_provenance(), - configured_providers: Vec::new(), + configured_providers: test_provider_ids(), web_url: None, }, storage_root, @@ -1633,7 +1796,7 @@ mod tests { fork_source_ref: None, parent_id: None, provenance: test_support::test_run_provenance(), - configured_providers: Vec::new(), + configured_providers: test_provider_ids(), web_url: None, }, storage_dir.clone(), @@ -1701,7 +1864,7 @@ mod tests { fabro_types::AuthMethod::Github, ), }, - configured_providers: Vec::new(), + configured_providers: test_provider_ids(), web_url: None, }, storage_dir, diff --git a/lib/crates/fabro-workflow/src/operations/start.rs b/lib/crates/fabro-workflow/src/operations/start.rs index 7fa47e938..5972d6cc4 100644 --- a/lib/crates/fabro-workflow/src/operations/start.rs +++ b/lib/crates/fabro-workflow/src/operations/start.rs @@ -1,4 +1,4 @@ -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::path::Path; use std::sync::{Arc, Mutex}; use std::time::{Duration, Instant}; @@ -7,7 +7,7 @@ use fabro_auth::{CredentialSource, EnvCredentialSource, VaultCredentialSource}; use fabro_interview::{AutoApproveInterviewer, Interviewer}; use fabro_llm::client::Client as LlmClient; use fabro_mcp::config::McpServerSettings; -use fabro_model::{Catalog, FallbackTarget, ProviderId}; +use fabro_model::{Catalog, FallbackTarget, ModelSelectionError, ProviderId, catalog}; use fabro_sandbox::daytona::DaytonaConfig; use fabro_sandbox::from_environment::{ daytona_config_from_environment, docker_config_from_environment_with_secrets, @@ -43,12 +43,15 @@ use crate::pipeline::{ }; use crate::records::Checkpoint; use crate::run_control::RunControlState; +use crate::run_materialization::resolve_run_model; use crate::run_metadata::metadata_branch_name; use crate::run_options::{GitCheckpointOptions, LifecycleOptions, RunOptions, SetupCommand}; use crate::run_status::{FailureReason, RunStatus}; use crate::runtime_store::RunStoreHandle; use crate::services::FabroRunToolServices; use crate::steering_hub::SteeringHub; +#[cfg(feature = "test-support")] +use crate::test_support as workflow_test_support; use crate::workflow_bundle::{RunDefinition, WorkflowBundle}; struct RunSession { @@ -372,6 +375,13 @@ impl RunSession { let catalog = Arc::clone(&services.catalog); let configured = configured_providers_for_start(services.vault.as_ref(), Arc::clone(&catalog)).await; + #[cfg(feature = "test-support")] + let configured = workflow_test_support::test_configured_provider_ids( + catalog.as_ref(), + configured, + process_env_var("FABRO_TEST_ASSUME_LLM_READY") + .is_some_and(|value| !matches!(value.as_str(), "" | "0" | "false" | "no")), + ); let llm = resolve_start_llm(catalog.as_ref(), &configured, resolved)?; let vault_guard = match services.vault.as_ref() { Some(vault) => Some(vault.read().await), @@ -607,25 +617,25 @@ fn resolve_start_llm( configured: &[ProviderId], settings: &ResolvedRunSettings, ) -> Result { - let model = settings - .model - .name - .clone() - .unwrap_or_else(|| catalog.default_for_configured_ids(configured).id.clone()); - let provider = settings - .model - .provider - .as_deref() - .filter(|value| !value.is_empty()); - - let default_provider_id = catalog - .default_for_configured_ids(configured) - .provider - .clone(); - let provider_context = - routing::resolve_provider_context(catalog, &default_provider_id, &model, provider)?; + let eligible = configured.iter().cloned().collect::>(); + let (model, provider) = resolve_run_model( + catalog, + &eligible, + settings.model.name.as_deref(), + settings.model.provider.as_deref(), + )?; + let provider_id = ProviderId::new( + provider.expect("catalog-backed run model resolution always selects a provider"), + ); + let provider_context = routing::resolve_provider_context( + catalog, + &provider_id, + &model, + Some(provider_id.as_str()), + )?; let provider_id = provider_context.provider_id; - let fallback_chain = resolve_fallback_chain(catalog, &provider_id, &model, &settings.model); + let fallback_chain = + resolve_fallback_chain(catalog, &provider_id, &model, &settings.model, &eligible)?; Ok(ResolvedStartLlm { model, @@ -636,44 +646,87 @@ fn resolve_start_llm( fn resolve_fallback_chain( catalog: &Catalog, - _provider: &ProviderId, + provider: &ProviderId, model: &str, settings: &ResolvedRunModelSettings, -) -> Vec { + eligible: &HashSet, +) -> Result, Error> { if settings.fallbacks.is_empty() { - return Vec::new(); + return Ok(Vec::new()); } let registry = CatalogModelRegistry { catalog }; - let primary = catalog.get(model); + let primary = catalog.get_on_provider(provider, model); + let mut chain = Vec::new(); - settings - .fallbacks - .iter() - .filter_map(|model_ref| match model_ref.resolve(®istry).ok()? { + for model_ref in &settings.fallbacks { + let identifier = model_ref.to_string(); + if let Some((provider, model)) = catalog::retired_model_replacement(&identifier) { + return Err(ModelSelectionError::RetiredModelIdentifier { + identifier, + provider, + model, + } + .into()); + } + match model_ref.resolve(®istry)? { ResolvedModelRef::Provider(provider_name) => { let provider_id = canonical_provider_id(catalog, &provider_name); - let reference = primary?; - catalog - .closest(&provider_id, reference) - .map(|model| FallbackTarget { + if !eligible.contains(&provider_id) { + return Err(ModelSelectionError::ProviderUnavailable { + provider: provider_id, + } + .into()); + } + if let Some(model) = + primary.and_then(|reference| catalog.closest(&provider_id, reference)) + { + chain.push(FallbackTarget { provider: provider_id.to_string(), - model: model.id.clone(), - }) - } - ResolvedModelRef::Model { provider, model } => { - let provider = - provider.map(|provider| canonical_provider_id(catalog, &provider).to_string()); - if let Some(info) = catalog.get(&model) { - let provider = provider.unwrap_or_else(|| info.provider.to_string()); - return Some(FallbackTarget { - provider, - model: info.id.clone(), + model: model.id.to_string(), }); } - provider.map(|provider| FallbackTarget { provider, model }) } - }) - .collect() + ResolvedModelRef::Model { + provider: fallback_provider, + model, + } => { + if let Some(provider) = fallback_provider { + let provider = canonical_provider_id(catalog, &provider); + if !eligible.contains(&provider) { + return Err(ModelSelectionError::ProviderUnavailable { provider }.into()); + } + match catalog.resolve_on_provider(&provider, &model) { + Ok(info) => chain.push(FallbackTarget { + provider: info.provider.to_string(), + model: info.id.to_string(), + }), + Err(ModelSelectionError::UnknownSelectorOnProvider { .. }) => { + chain.push(FallbackTarget { + provider: provider.to_string(), + model, + }); + } + Err(error) => return Err(error.into()), + } + } else { + match catalog.select(&model, None, eligible) { + Ok(info) => chain.push(FallbackTarget { + provider: info.provider.to_string(), + model: info.id.to_string(), + }), + Err(ModelSelectionError::UnknownSelector { .. }) => { + chain.push(FallbackTarget { + provider: provider.to_string(), + model, + }); + } + Err(error) => return Err(error.into()), + } + } + } + } + } + Ok(chain) } fn canonical_provider_id(catalog: &Catalog, provider_name: &str) -> ProviderId { @@ -693,13 +746,7 @@ impl ModelRegistry for CatalogModelRegistry<'_> { } fn is_model(&self, token: &str) -> bool { - self.catalog.get(token).is_some() - } - - fn provider_of(&self, token: &str) -> Option { - self.catalog - .get(token) - .map(|model| model.provider.to_string()) + self.catalog.is_model_selector(token) } } @@ -1229,6 +1276,59 @@ mod tests { Arc::new(Catalog::from_builtin().expect("default catalog should build")) } + fn test_provider_ids() -> Vec { + Catalog::builtin().all_provider_ids().into_iter().collect() + } + + fn portable_model_catalog() -> Catalog { + let settings: fabro_model::catalog::LlmCatalogSettings = toml::from_str( + r#" +[providers.openai] +display_name = "OpenAI" +adapter = "openai" +agent_profile = "openai" +priority = 90 + +[providers.openai.models."gpt-5.6-sol"] +display_name = "GPT-5.6 Sol" +family = "gpt-5" +aliases = ["gpt-56-sol"] +default = true + +[providers.openai.models."gpt-5.6-sol".limits] +context_window = 1000 + +[providers.openai.models."gpt-5.6-sol".features] +tools = true +vision = false +reasoning = false + +[providers.openrouter] +display_name = "OpenRouter" +adapter = "openai_compatible" +agent_profile = "openai" +priority = 25 + +[providers.openrouter.models."gpt-5.6-sol"] +api_id = "openai/gpt-5.6-sol" +display_name = "GPT-5.6 Sol (via OpenRouter)" +family = "gpt-5" +aliases = ["gpt-56-sol"] +default = true + +[providers.openrouter.models."gpt-5.6-sol".limits] +context_window = 1000 + +[providers.openrouter.models."gpt-5.6-sol".features] +tools = true +vision = false +reasoning = false +"#, + ) + .unwrap(); + Catalog::from_settings(&settings).unwrap() + } + #[test] fn resolve_fallback_chain_resolves_provider_fallbacks() { let catalog = test_catalog(); @@ -1242,7 +1342,9 @@ mod tests { &ProviderId::anthropic(), "claude-opus-4-6", &settings, - ); + &catalog.all_provider_ids(), + ) + .unwrap(); assert_eq!(chain, vec![FallbackTarget { provider: "openai".to_string(), @@ -1263,7 +1365,9 @@ mod tests { &ProviderId::anthropic(), "claude-opus-4-6", &settings, - ); + &catalog.all_provider_ids(), + ) + .unwrap(); assert_eq!(chain, vec![FallbackTarget { provider: "openai".to_string(), @@ -1271,6 +1375,162 @@ mod tests { }]); } + #[test] + fn resolve_fallback_chain_selects_shared_bare_alias_from_ready_providers() { + let catalog = portable_model_catalog(); + let settings = ResolvedRunModelSettings { + fallbacks: vec!["gpt-56-sol".parse::().unwrap()], + ..ResolvedRunModelSettings::default() + }; + + let chain = resolve_fallback_chain( + &catalog, + &ProviderId::openai(), + "gpt-5.6-sol", + &settings, + &HashSet::from([ProviderId::new("openrouter")]), + ) + .unwrap(); + + assert_eq!(chain, vec![FallbackTarget { + provider: "openrouter".to_string(), + model: "gpt-5.6-sol".to_string(), + }]); + } + + #[test] + fn resolve_fallback_chain_resolves_provider_qualified_shared_alias() { + let catalog = portable_model_catalog(); + let settings = ResolvedRunModelSettings { + fallbacks: vec!["openrouter/gpt-56-sol".parse::().unwrap()], + ..ResolvedRunModelSettings::default() + }; + + let chain = resolve_fallback_chain( + &catalog, + &ProviderId::openai(), + "gpt-5.6-sol", + &settings, + &catalog.all_provider_ids(), + ) + .unwrap(); + + assert_eq!(chain, vec![FallbackTarget { + provider: "openrouter".to_string(), + model: "gpt-5.6-sol".to_string(), + }]); + } + + #[test] + fn resolve_fallback_chain_rejects_retired_wire_identifier_before_qualification() { + let catalog = test_catalog(); + let settings = ResolvedRunModelSettings { + fallbacks: vec!["openai/gpt-5.6-sol".parse::().unwrap()], + ..ResolvedRunModelSettings::default() + }; + + let error = resolve_fallback_chain( + catalog.as_ref(), + &ProviderId::anthropic(), + "claude-opus-4-6", + &settings, + &catalog.all_provider_ids(), + ) + .unwrap_err(); + + assert!(matches!( + error, + Error::ModelSelection(ModelSelectionError::RetiredModelIdentifier { + identifier, + provider, + model, + }) if identifier == "openai/gpt-5.6-sol" + && provider == ProviderId::new("openrouter") + && model == fabro_model::ModelId::new("gpt-5.6-sol") + )); + } + + #[test] + fn resolve_fallback_chain_propagates_provider_model_ambiguity() { + let settings: fabro_model::catalog::LlmCatalogSettings = toml::from_str( + r#" +[providers.shared] +display_name = "Shared Provider" +adapter = "openai" +agent_profile = "openai" + +[providers.shared.models.default] +display_name = "Default" +family = "test" +default = true + +[providers.shared.models.default.limits] +context_window = 1000 + +[providers.shared.models.default.features] +tools = false +vision = false +reasoning = false + +[providers.other] +display_name = "Other" +adapter = "openai" +agent_profile = "openai" + +[providers.other.models.model] +display_name = "Shared Alias" +family = "test" +aliases = ["shared"] +default = true + +[providers.other.models.model.limits] +context_window = 1000 + +[providers.other.models.model.features] +tools = false +vision = false +reasoning = false +"#, + ) + .unwrap(); + let catalog = Catalog::from_settings(&settings).unwrap(); + let run_model = ResolvedRunModelSettings { + fallbacks: vec!["shared".parse::().unwrap()], + ..ResolvedRunModelSettings::default() + }; + + let error = resolve_fallback_chain( + &catalog, + &ProviderId::new("other"), + "model", + &run_model, + &catalog.all_provider_ids(), + ) + .unwrap_err(); + + assert!(matches!(error, Error::ModelReference(_))); + } + + #[test] + fn materialized_provider_pin_is_not_reselected_when_readiness_changes() { + let catalog = portable_model_catalog(); + let mut settings = ResolvedRunSettings::default(); + settings.model.name = Some("gpt-5.6-sol".to_string()); + settings.model.provider = Some("openai".to_string()); + + let Err(error) = resolve_start_llm(&catalog, &[ProviderId::new("openrouter")], &settings) + else { + panic!("materialized provider pin should remain fixed"); + }; + + assert!(matches!( + error, + Error::ModelSelection(fabro_model::ModelSelectionError::ProviderUnavailable { + provider + }) if provider == ProviderId::openai() + )); + } + #[test] fn resolve_start_llm_infers_provider_from_model_alias() { let overrides: fabro_model::catalog::LlmCatalogSettings = toml::from_str( @@ -1302,9 +1562,9 @@ reasoning = false let mut settings = ResolvedRunSettings::default(); settings.model.name = Some("ac".to_string()); - let resolved = resolve_start_llm(&catalog, &[], &settings).unwrap(); + let resolved = resolve_start_llm(&catalog, &[ProviderId::new("acme")], &settings).unwrap(); - assert_eq!(resolved.model, "ac"); + assert_eq!(resolved.model, "acme-claude"); assert_eq!(resolved.provider_id, ProviderId::new("acme")); } @@ -1529,7 +1789,11 @@ reasoning = false persisted_workflow_with_settings(MINIMAL_DOT, &storage_root, settings).await; let emitter = Arc::new(Emitter::new(fixtures::RUN_1)); let registry = Arc::new(test_registry()); - let vault = Arc::new(AsyncRwLock::new(token_vault("DEPLOY_TOKEN", "vault-token"))); + let vault = Arc::new(AsyncRwLock::new(start_vault(&[( + "DEPLOY_TOKEN", + "vault-token", + SecretType::Token, + )]))); let session = RunSession::new(&persisted, StartServices { vault: Some(vault), @@ -1587,7 +1851,7 @@ reasoning = false persisted_workflow_with_settings(MINIMAL_DOT, &storage_root, settings).await; let emitter = Arc::new(Emitter::new(fixtures::RUN_1)); let registry = Arc::new(test_registry()); - let vault = Arc::new(AsyncRwLock::new(temp_vault(&[]))); + let vault = Arc::new(AsyncRwLock::new(start_vault(&[]))); let Err(err) = RunSession::new(&persisted, StartServices { vault: Some(vault), @@ -1703,7 +1967,7 @@ reasoning = false fork_source_ref: None, parent_id: None, provenance: test_support::test_run_provenance(), - configured_providers: Vec::new(), + configured_providers: test_provider_ids(), web_url: None, }, storage_root.to_path_buf(), @@ -1756,7 +2020,7 @@ reasoning = false run_control: None, github_app: None, github_permissions: HashMap::new(), - vault: None, + vault: Some(Arc::new(AsyncRwLock::new(start_vault(&[])))), catalog: test_catalog(), on_node: None, registry_override: Some(registry), @@ -1777,6 +2041,12 @@ reasoning = false temp_vault(&[(name, value, SecretType::Token)]) } + fn start_vault(entries: &[(&str, &str, SecretType)]) -> Vault { + let mut all_entries = vec![("ANTHROPIC_API_KEY", "test-key", SecretType::Token)]; + all_entries.extend_from_slice(entries); + temp_vault(&all_entries) + } + fn vault_secret_lookup(vault: &Vault) -> impl FnMut(&str) -> Option + '_ { move |name| vault_token_lookup(Some(vault), name) } @@ -2183,7 +2453,7 @@ reasoning = false fork_source_ref: None, parent_id: None, provenance: test_support::test_run_provenance(), - configured_providers: Vec::new(), + configured_providers: test_provider_ids(), web_url: None, }, storage_root, diff --git a/lib/crates/fabro-workflow/src/operations/validate.rs b/lib/crates/fabro-workflow/src/operations/validate.rs index c11315d3c..a95b13920 100644 --- a/lib/crates/fabro-workflow/src/operations/validate.rs +++ b/lib/crates/fabro-workflow/src/operations/validate.rs @@ -34,6 +34,11 @@ pub fn validate(input: ValidateInput) -> Result { cwd: input.cwd, }) .map_err(|err| Error::Parse(err.to_string()))?; + let eligible_providers = input + .catalog + .all_provider_ids() + .into_iter() + .collect::>(); preprocess_and_validate( &resolved.raw_source, @@ -47,6 +52,15 @@ pub fn validate(input: ValidateInput) -> Result { template_context(Some(&resolved.settings), input.vars), resolved.goal_override.as_deref(), RenderMode::Structural, + resolved + .settings + .run + .model + .provider + .as_deref() + .filter(|provider| !provider.is_empty()) + .map(fabro_model::ProviderId::new), + &eligible_providers, &input.catalog, ) } diff --git a/lib/crates/fabro-workflow/src/outcome.rs b/lib/crates/fabro-workflow/src/outcome.rs index 532d26860..7d3c7096c 100644 --- a/lib/crates/fabro-workflow/src/outcome.rs +++ b/lib/crates/fabro-workflow/src/outcome.rs @@ -156,7 +156,7 @@ mod tests { fn model_ref(provider: ProviderId, model_id: &str, speed: Option) -> ModelRef { ModelRef { provider, - model_id: model_id.to_string(), + model_id: model_id.into(), speed, } } diff --git a/lib/crates/fabro-workflow/src/pipeline/pull_request.rs b/lib/crates/fabro-workflow/src/pipeline/pull_request.rs index 1015b6972..0dcc7745c 100644 --- a/lib/crates/fabro-workflow/src/pipeline/pull_request.rs +++ b/lib/crates/fabro-workflow/src/pipeline/pull_request.rs @@ -1,3 +1,4 @@ +use std::collections::HashSet; use std::sync::{Arc, LazyLock}; use fabro_auth::CredentialSource; @@ -5,7 +6,7 @@ use fabro_github::{self as github_app, ssh_url_to_https}; use fabro_graphviz::parser; use fabro_llm::client::Client; use fabro_llm::generate::{GenerateParams, generate_object}; -use fabro_model::Catalog; +use fabro_model::{Catalog, ProviderId}; use fabro_store::RunProjection; use fabro_types::PullRequestLink; use fabro_types::settings::run::MergeStrategy; @@ -66,9 +67,14 @@ const UNKNOWN_MODEL_CTX: usize = 200_000; /// Resolve truncation caps based on the model's context window. Unknown /// models use the baseline 200k context-window assumption. -fn truncation_caps(model: &str, catalog: &Catalog) -> TruncationCaps { +fn truncation_caps( + model: &str, + eligible: &HashSet, + catalog: &Catalog, +) -> TruncationCaps { let ctx = catalog - .get(model) + .select(model, None, eligible) + .ok() .and_then(|m| usize::try_from(m.context_window()).ok()) .unwrap_or(UNKNOWN_MODEL_CTX); @@ -395,7 +401,8 @@ async fn build_pr_content_with_client( let run_spec = run_state.map(|state| state.spec.clone()); let dot_source = run_state.and_then(|state| state.spec.graph_source.clone()); - let caps = truncation_caps(model, catalog); + let eligible = client.provider_ids(); + let caps = truncation_caps(model, &eligible, catalog); let truncated_diff = truncate_chars(diff, caps.diff); let prompt = if let Some(ref plan) = plan_text { @@ -1523,7 +1530,11 @@ mod tests { } ); assert_eq!( - truncation_caps("unknown-model", Catalog::builtin()), + truncation_caps( + "unknown-model", + &Catalog::builtin().all_provider_ids(), + Catalog::builtin(), + ), TruncationCaps { diff: 80_000, plan: 20_000, diff --git a/lib/crates/fabro-workflow/src/pipeline/transform.rs b/lib/crates/fabro-workflow/src/pipeline/transform.rs index 4f14ceeb7..6b76380d0 100644 --- a/lib/crates/fabro-workflow/src/pipeline/transform.rs +++ b/lib/crates/fabro-workflow/src/pipeline/transform.rs @@ -63,7 +63,12 @@ pub fn transform(parsed: Parsed, options: &TransformOptions) -> Result TransformOptions { TransformOptions { - current_dir: None, - file_resolver: None, - template_context: fabro_template::TemplateContext::new(), - source_name: None, - render_mode: crate::operations::RenderMode::Strict, - custom_transforms: vec![], - catalog: test_catalog(), + current_dir: None, + file_resolver: None, + template_context: fabro_template::TemplateContext::new(), + source_name: None, + render_mode: crate::operations::RenderMode::Strict, + custom_transforms: vec![], + catalog: test_catalog(), + default_provider: None, + eligible_providers: Catalog::builtin().all_provider_ids(), } } @@ -168,13 +175,15 @@ mod tests { ) .unwrap(); let transformed = transform(parsed, &TransformOptions { - current_dir: Some(dir.path().to_path_buf()), - file_resolver: Some(Arc::new(FilesystemFileResolver::new(None))), - template_context: fabro_template::TemplateContext::new(), - source_name: None, - render_mode: crate::operations::RenderMode::Strict, - custom_transforms: vec![], - catalog: test_catalog(), + current_dir: Some(dir.path().to_path_buf()), + file_resolver: Some(Arc::new(FilesystemFileResolver::new(None))), + template_context: fabro_template::TemplateContext::new(), + source_name: None, + render_mode: crate::operations::RenderMode::Strict, + custom_transforms: vec![], + catalog: test_catalog(), + default_provider: None, + eligible_providers: Catalog::builtin().all_provider_ids(), }) .unwrap(); @@ -215,18 +224,20 @@ mod tests { ) .unwrap(); let transformed = transform(parsed, &TransformOptions { - current_dir: Some(dir.path().to_path_buf()), - file_resolver: Some(Arc::new(FilesystemFileResolver::new(None))), - template_context: fabro_template::TemplateContext::new().with_inputs(HashMap::from([ - ( + current_dir: Some(dir.path().to_path_buf()), + file_resolver: Some(Arc::new(FilesystemFileResolver::new(None))), + template_context: fabro_template::TemplateContext::new().with_inputs(HashMap::from( + [( "task".to_string(), toml::Value::String("Launch".to_string()), - ), - ])), - source_name: None, - render_mode: crate::operations::RenderMode::Strict, - custom_transforms: vec![], - catalog: test_catalog(), + )], + )), + source_name: None, + render_mode: crate::operations::RenderMode::Strict, + custom_transforms: vec![], + catalog: test_catalog(), + default_provider: None, + eligible_providers: Catalog::builtin().all_provider_ids(), }) .unwrap(); @@ -350,13 +361,15 @@ mod tests { ) .unwrap(); let transformed = transform(parsed, &TransformOptions { - current_dir: Some(dir.path().to_path_buf()), - file_resolver: Some(Arc::new(FilesystemFileResolver::new(None))), - template_context: fabro_template::TemplateContext::new(), - source_name: None, - render_mode: crate::operations::RenderMode::Structural, - custom_transforms: vec![], - catalog: test_catalog(), + current_dir: Some(dir.path().to_path_buf()), + file_resolver: Some(Arc::new(FilesystemFileResolver::new(None))), + template_context: fabro_template::TemplateContext::new(), + source_name: None, + render_mode: crate::operations::RenderMode::Structural, + custom_transforms: vec![], + catalog: test_catalog(), + default_provider: None, + eligible_providers: Catalog::builtin().all_provider_ids(), }) .unwrap(); diff --git a/lib/crates/fabro-workflow/src/pipeline/types.rs b/lib/crates/fabro-workflow/src/pipeline/types.rs index 8449453a8..dc5b25c9e 100644 --- a/lib/crates/fabro-workflow/src/pipeline/types.rs +++ b/lib/crates/fabro-workflow/src/pipeline/types.rs @@ -1,4 +1,4 @@ -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::path::{Path, PathBuf}; use std::sync::Arc; @@ -324,13 +324,15 @@ pub struct Finalized { /// Options for the TRANSFORM phase. pub struct TransformOptions { - pub current_dir: Option, - pub file_resolver: Option>, - pub template_context: TemplateContext, - pub source_name: Option, - pub render_mode: RenderMode, - pub custom_transforms: Vec>, - pub catalog: Arc, + pub current_dir: Option, + pub file_resolver: Option>, + pub template_context: TemplateContext, + pub source_name: Option, + pub render_mode: RenderMode, + pub custom_transforms: Vec>, + pub catalog: Arc, + pub default_provider: Option, + pub eligible_providers: HashSet, } /// Options for the FINALIZE phase. diff --git a/lib/crates/fabro-workflow/src/pipeline/validate.rs b/lib/crates/fabro-workflow/src/pipeline/validate.rs index 14999afae..06120e8ce 100644 --- a/lib/crates/fabro-workflow/src/pipeline/validate.rs +++ b/lib/crates/fabro-workflow/src/pipeline/validate.rs @@ -42,13 +42,15 @@ mod tests { let catalog = test_catalog(); let parsed = parse(dot).unwrap(); let transformed = transform::transform(parsed, &TransformOptions { - current_dir: None, - file_resolver: None, - template_context: fabro_template::TemplateContext::new(), - source_name: None, - render_mode: crate::operations::RenderMode::Strict, - custom_transforms: vec![], - catalog: std::sync::Arc::clone(&catalog), + current_dir: None, + file_resolver: None, + template_context: fabro_template::TemplateContext::new(), + source_name: None, + render_mode: crate::operations::RenderMode::Strict, + custom_transforms: vec![], + catalog: std::sync::Arc::clone(&catalog), + default_provider: None, + eligible_providers: catalog.all_provider_ids(), }) .unwrap(); validate(transformed, catalog.as_ref(), &[]) diff --git a/lib/crates/fabro-workflow/src/run_materialization.rs b/lib/crates/fabro-workflow/src/run_materialization.rs index b6f9aa873..14ffa735a 100644 --- a/lib/crates/fabro-workflow/src/run_materialization.rs +++ b/lib/crates/fabro-workflow/src/run_materialization.rs @@ -1,15 +1,19 @@ +use std::collections::HashSet; + use fabro_graphviz::graph::Graph; -use fabro_model::{Catalog, ProviderId}; +use fabro_model::{Catalog, ModelSelectionError, ProviderId}; use fabro_types::WorkflowSettings; use fabro_types::settings::InterpString; use fabro_types::settings::run::RunGoal; +use crate::error::Error; + pub fn materialize_run( mut settings: WorkflowSettings, graph: &Graph, catalog: &Catalog, configured_providers: &[ProviderId], -) -> WorkflowSettings { +) -> Result { let configured_model = settings.run.model.name.take(); let configured_provider = settings.run.model.provider.take(); let graph_provider = graph @@ -24,23 +28,10 @@ pub fn materialize_run( .map(str::to_string); let provider = configured_provider.or(graph_provider); - let model = configured_model.or(graph_model).unwrap_or_else(|| { - provider - .as_deref() - .map(ProviderId::from) - .and_then(|provider| catalog.default_for_provider(&provider)) - .unwrap_or_else(|| catalog.default_for_configured_ids(configured_providers)) - .id - .clone() - }); - - let (resolved_model, resolved_provider) = match catalog.get(&model) { - Some(info) => ( - info.id.clone(), - provider.or(Some(info.provider.to_string())), - ), - None => (model, provider), - }; + let model = configured_model.or(graph_model); + let eligible = configured_providers.iter().cloned().collect::>(); + let (resolved_model, resolved_provider) = + resolve_run_model(catalog, &eligible, model.as_deref(), provider.as_deref())?; settings.run.model.name = Some(resolved_model); settings.run.model.provider = resolved_provider; @@ -61,5 +52,63 @@ pub fn materialize_run( settings.run.pull_request = None; } - settings + Ok(settings) +} + +pub(crate) fn resolve_run_model( + catalog: &Catalog, + eligible: &HashSet, + model: Option<&str>, + provider: Option<&str>, +) -> Result<(String, Option), ModelSelectionError> { + if let Some(provider) = provider.filter(|provider| !provider.is_empty()) { + let requested = ProviderId::new(provider); + let provider = + catalog + .provider(&requested) + .ok_or_else(|| ModelSelectionError::UnknownProvider { + provider: requested.clone(), + })?; + let canonical_provider = provider.id.clone(); + let canonical_eligible = eligible.iter().any(|eligible| { + catalog + .provider(eligible) + .is_some_and(|provider| provider.id == canonical_provider) + }); + if !canonical_eligible { + return Err(ModelSelectionError::ProviderUnavailable { + provider: canonical_provider, + }); + } + if let Some(model) = model { + return match catalog.resolve_on_provider(&provider.id, model) { + Ok(offering) => Ok((offering.id.to_string(), Some(offering.provider.to_string()))), + Err(ModelSelectionError::UnknownSelectorOnProvider { .. }) => { + Ok((model.to_string(), Some(provider.id.to_string()))) + } + Err(error) => Err(error), + }; + } + let offering = catalog.default_for_provider(&provider.id).ok_or_else(|| { + ModelSelectionError::UnknownSelectorOnProvider { + selector: "".to_string(), + provider: provider.id.clone(), + } + })?; + return Ok((offering.id.to_string(), Some(offering.provider.to_string()))); + } + + if let Some(model) = model { + return match catalog.select(model, None, eligible) { + Ok(offering) => Ok((offering.id.to_string(), Some(offering.provider.to_string()))), + Err(ModelSelectionError::UnknownSelector { .. }) => { + let default = catalog.select_default(eligible)?; + Ok((model.to_string(), Some(default.provider.to_string()))) + } + Err(error) => Err(error), + }; + } + + let default = catalog.select_default(eligible)?; + Ok((default.id.to_string(), Some(default.provider.to_string()))) } diff --git a/lib/crates/fabro-workflow/src/test_support.rs b/lib/crates/fabro-workflow/src/test_support.rs index 1acf410a4..4094b539d 100644 --- a/lib/crates/fabro-workflow/src/test_support.rs +++ b/lib/crates/fabro-workflow/src/test_support.rs @@ -9,6 +9,8 @@ use fabro_auth::{CredentialSource, EnvCredentialSource}; use fabro_graphviz::graph::Graph as GvGraph; use fabro_interview::AutoApproveInterviewer; use fabro_model::Catalog; +#[cfg(feature = "test-support")] +use fabro_model::ProviderId; use fabro_store::{ArtifactStore, Database, RunProjection}; use object_store::local::LocalFileSystem; @@ -26,6 +28,19 @@ use crate::run_options::RunOptions; use crate::sandbox_git_runtime::SandboxGitRuntime; use crate::services::{EngineServices, RunLocations, RunServices}; +#[cfg(feature = "test-support")] +pub(crate) fn test_configured_provider_ids( + catalog: &Catalog, + configured_provider_ids: Vec, + assume_ready: bool, +) -> Vec { + if assume_ready { + catalog.all_provider_ids().into_iter().collect() + } else { + configured_provider_ids + } +} + /// These helpers stop at EXECUTE, so they emit the terminal event here to /// keep test consumers seeing the same end-of-run signal as production /// (FINALIZE). diff --git a/lib/crates/fabro-workflow/src/transforms/model_resolution.rs b/lib/crates/fabro-workflow/src/transforms/model_resolution.rs index b58f9f44e..fc130bfb1 100644 --- a/lib/crates/fabro-workflow/src/transforms/model_resolution.rs +++ b/lib/crates/fabro-workflow/src/transforms/model_resolution.rs @@ -1,7 +1,8 @@ +use std::collections::HashSet; use std::sync::Arc; use fabro_graphviz::graph::{AttrValue, Graph}; -use fabro_model::Catalog; +use fabro_model::{Catalog, ModelSelectionError, ProviderId}; use super::Transform; use crate::error::Error; @@ -9,19 +10,100 @@ use crate::error::Error; /// Resolves model aliases to canonical IDs and infers the provider from the /// model catalog. pub struct ModelResolutionTransform { - catalog: Arc, + catalog: Arc, + default_provider: Option, + eligible_providers: HashSet, } impl ModelResolutionTransform { #[must_use] pub fn new(catalog: Arc) -> Self { - Self { catalog } + let eligible_providers = catalog.all_provider_ids(); + Self { + catalog, + default_provider: None, + eligible_providers, + } + } + + #[must_use] + pub fn for_eligible(catalog: Arc, eligible_providers: HashSet) -> Self { + Self { + catalog, + default_provider: None, + eligible_providers, + } + } + + #[must_use] + pub fn with_default_provider(mut self, provider: Option) -> Self { + self.default_provider = provider; + self + } + + fn resolve_model( + &self, + model: &str, + explicit_provider: Option<&ProviderId>, + ) -> Result<(String, ProviderId), Error> { + match self + .catalog + .select(model, explicit_provider, &self.eligible_providers) + { + Ok(info) => Ok((info.id.to_string(), info.provider.clone())), + Err(ModelSelectionError::UnknownSelectorOnProvider { provider, .. }) => { + Ok((model.to_string(), provider)) + } + Err(ModelSelectionError::UnknownSelector { .. }) => { + let provider = self + .catalog + .select_default(&self.eligible_providers)? + .provider + .clone(); + Ok((model.to_string(), provider)) + } + Err(error) => Err(error.into()), + } } } impl Transform for ModelResolutionTransform { fn apply(&self, graph: Graph) -> Result { let mut graph = graph; + let graph_default_provider = graph + .attrs + .get("default_provider") + .and_then(AttrValue::as_str) + .filter(|provider| !provider.is_empty()) + .map(ProviderId::new); + let requested_default_provider = self + .default_provider + .as_ref() + .or(graph_default_provider.as_ref()); + if let Some(default_model) = graph + .attrs + .get("default_model") + .and_then(AttrValue::as_str) + .map(str::to_string) + { + let (model, provider) = + self.resolve_model(&default_model, requested_default_provider)?; + graph + .attrs + .insert("default_model".to_string(), AttrValue::String(model)); + graph.attrs.insert( + "default_provider".to_string(), + AttrValue::String(provider.to_string()), + ); + } + let default_provider = self.default_provider.clone().or_else(|| { + graph + .attrs + .get("default_provider") + .and_then(AttrValue::as_str) + .filter(|provider| !provider.is_empty()) + .map(ProviderId::new) + }); for node in graph.nodes.values_mut() { let model = node .attrs @@ -29,19 +111,20 @@ impl Transform for ModelResolutionTransform { .and_then(AttrValue::as_str) .map(String::from); if let Some(model) = model { - if let Some(info) = self.catalog.get(&model) { - let canonical_id = info.id.clone(); - let provider = info.provider.to_string(); - // Resolve alias to canonical model ID - if model != canonical_id { - node.attrs - .insert("model".to_string(), AttrValue::String(canonical_id)); - } - if !node.attrs.contains_key("provider") { - node.attrs - .insert("provider".to_string(), AttrValue::String(provider)); - } - } + let explicit_provider = node + .attrs + .get("provider") + .and_then(AttrValue::as_str) + .filter(|provider| !provider.is_empty()) + .map(ProviderId::new) + .or_else(|| default_provider.clone()); + let (model, provider) = self.resolve_model(&model, explicit_provider.as_ref())?; + node.attrs + .insert("model".to_string(), AttrValue::String(model)); + node.attrs.insert( + "provider".to_string(), + AttrValue::String(provider.to_string()), + ); } } @@ -117,7 +200,7 @@ reasoning = false } #[test] - fn provider_inference_does_not_override_explicit_provider() { + fn explicit_provider_allows_unknown_model_passthrough() { let mut graph = Graph::new("test"); let mut node = Node::new("a"); node.attrs.insert( @@ -126,7 +209,7 @@ reasoning = false ); node.attrs.insert( "provider".to_string(), - AttrValue::String("custom".to_string()), + AttrValue::String("openai".to_string()), ); graph.nodes.insert("a".to_string(), node); @@ -137,12 +220,19 @@ reasoning = false .attrs .get("provider") .and_then(AttrValue::as_str), - Some("custom") + Some("openai") + ); + assert_eq!( + graph.nodes["a"] + .attrs + .get("model") + .and_then(AttrValue::as_str), + Some("claude-sonnet-4-5") ); } #[test] - fn provider_inference_unknown_model_leaves_no_provider() { + fn provider_inference_unknown_model_pins_default_eligible_provider() { let mut graph = Graph::new("test"); let mut node = Node::new("a"); node.attrs.insert( @@ -153,7 +243,13 @@ reasoning = false let graph = builtin_transform().apply(graph).unwrap(); - assert_eq!(graph.nodes["a"].attrs.get("provider"), None); + assert_eq!( + graph.nodes["a"] + .attrs + .get("provider") + .and_then(AttrValue::as_str), + Some("anthropic") + ); } #[test] @@ -241,4 +337,29 @@ reasoning = false Some("venice") ); } + + #[test] + fn graph_default_alias_materializes_to_canonical_offering() { + let mut graph = Graph::new("test"); + graph.attrs.insert( + "default_model".to_string(), + AttrValue::String("vl".to_string()), + ); + + let graph = ModelResolutionTransform::new(custom_catalog()) + .apply(graph) + .unwrap(); + + assert_eq!( + graph.attrs.get("default_model").and_then(AttrValue::as_str), + Some("venice-large") + ); + assert_eq!( + graph + .attrs + .get("default_provider") + .and_then(AttrValue::as_str), + Some("venice") + ); + } } diff --git a/lib/crates/fabro-workflow/tests/it/integration.rs b/lib/crates/fabro-workflow/tests/it/integration.rs index 3ceeced5d..086b443af 100644 --- a/lib/crates/fabro-workflow/tests/it/integration.rs +++ b/lib/crates/fabro-workflow/tests/it/integration.rs @@ -4680,15 +4680,17 @@ async fn import_e2e_through_engine() { ) .expect("parse should succeed"); let transformed = transform(parsed, &TransformOptions { - current_dir: Some(dir.path().to_path_buf()), - file_resolver: Some(std::sync::Arc::new( + current_dir: Some(dir.path().to_path_buf()), + file_resolver: Some(std::sync::Arc::new( fabro_workflow::file_resolver::FilesystemFileResolver::new(None), )), - template_context: fabro_template::TemplateContext::new(), - source_name: None, - render_mode: fabro_workflow::operations::RenderMode::Strict, - custom_transforms: vec![], - catalog: std::sync::Arc::clone(&catalog), + template_context: fabro_template::TemplateContext::new(), + source_name: None, + render_mode: fabro_workflow::operations::RenderMode::Strict, + custom_transforms: vec![], + catalog: std::sync::Arc::clone(&catalog), + default_provider: None, + eligible_providers: catalog.all_provider_ids(), }) .unwrap(); let validated = validate(transformed, catalog.as_ref(), &[]); diff --git a/lib/crates/fabro-workflow/tests/materialize_run.rs b/lib/crates/fabro-workflow/tests/materialize_run.rs index 77fdf6dd0..4fec97585 100644 --- a/lib/crates/fabro-workflow/tests/materialize_run.rs +++ b/lib/crates/fabro-workflow/tests/materialize_run.rs @@ -34,7 +34,10 @@ fn materialize_run_applies_graph_and_catalog_defaults() { ..WorkflowSettings::default() }; - let materialized = materialize_run(settings, &graph(source), Catalog::builtin(), &[]); + let materialized = materialize_run(settings, &graph(source), Catalog::builtin(), &[ + ProviderId::anthropic(), + ]) + .unwrap(); let resolved = &materialized.run; assert_eq!(resolved.model.name.as_deref(), Some("claude-sonnet-4-6")); @@ -60,7 +63,8 @@ fn materialize_run_uses_configured_provider_defaults() { &graph(source), Catalog::builtin(), &[ProviderId::openai()], - ); + ) + .unwrap(); let resolved = &materialized.run; assert_eq!(resolved.model.provider.as_deref(), Some("openai")); diff --git a/lib/packages/fabro-api-client/src/.openapi-generator/FILES b/lib/packages/fabro-api-client/src/.openapi-generator/FILES index 6c4bc5e4e..57f760daa 100644 --- a/lib/packages/fabro-api-client/src/.openapi-generator/FILES +++ b/lib/packages/fabro-api-client/src/.openapi-generator/FILES @@ -293,6 +293,8 @@ models/principal-webhook.ts models/principal-worker.ts models/principal.ts models/project-namespace.ts +models/provider-credential-test-request.ts +models/provider-credential-test-response.ts models/provider-list.ts models/provider-test-list.ts models/provider-test-result.ts diff --git a/lib/packages/fabro-api-client/src/api/models-api.ts b/lib/packages/fabro-api-client/src/api/models-api.ts index 6b19c88c3..03a90fac4 100644 --- a/lib/packages/fabro-api-client/src/api/models-api.ts +++ b/lib/packages/fabro-api-client/src/api/models-api.ts @@ -30,6 +30,10 @@ import type { ModelTestResult } from '../models'; // @ts-ignore import type { PaginatedModelList } from '../models'; // @ts-ignore +import type { ProviderCredentialTestRequest } from '../models'; +// @ts-ignore +import type { ProviderCredentialTestResponse } from '../models'; +// @ts-ignore import type { ProviderList } from '../models'; // @ts-ignore import type { ProviderTestList } from '../models'; @@ -39,7 +43,7 @@ import type { ProviderTestList } from '../models'; export const ModelsApiAxiosParamCreator = function (configuration?: Configuration) { return { /** - * Returns a paginated list of available LLM models from the built-in catalog. + * Returns one row per provider/model offering from the catalog. Model IDs are unique within a provider; `(provider, id)` is the resource identity. * @summary List Models * @param {string} [provider] Filter models by provider ID. Unknown provider IDs return an empty result set. * @param {string} [query] Case-insensitive substring search across `id`, `display_name`, and `aliases`. @@ -133,12 +137,13 @@ export const ModelsApiAxiosParamCreator = function (configuration?: Configuratio /** * Tests a model by sending a simple prompt and reporting pass/fail. * @summary Test Model - * @param {string} id The model identifier. + * @param {string} id The canonical model ID or an alias. + * @param {string} [provider] Pin the test to this provider\'s offering. When omitted, the server selects among ready providers by catalog priority. * @param {ModelTestMode} [mode] Test mode for the single-model test endpoint. Defaults to `basic`. * @param {*} [options] Override http request option. * @throws {RequiredError} */ - testModel: async (id: string, mode?: ModelTestMode, options: RawAxiosRequestConfig = {}): Promise => { + testModel: async (id: string, provider?: string, mode?: ModelTestMode, options: RawAxiosRequestConfig = {}): Promise => { // verify required parameter 'id' is not null or undefined assertParamExists('testModel', 'id', id) const localVarPath = `/api/v1/models/{id}/test` @@ -160,6 +165,10 @@ export const ModelsApiAxiosParamCreator = function (configuration?: Configuratio // http bearer authentication required await setBearerAuthToObject(localVarHeaderParameter, configuration) + if (provider !== undefined) { + localVarQueryParameter['provider'] = provider; + } + if (mode !== undefined) { localVarQueryParameter['mode'] = mode; } @@ -175,6 +184,51 @@ export const ModelsApiAxiosParamCreator = function (configuration?: Configuratio options: localVarRequestOptions, }; }, + /** + * Validates an LLM provider API key against the server\'s effective catalog without persisting it. + * @summary Test Provider Credentials + * @param {string} provider The provider identifier. + * @param {ProviderCredentialTestRequest} providerCredentialTestRequest + * @param {*} [options] Override http request option. + * @throws {RequiredError} + */ + testProviderCredentials: async (provider: string, providerCredentialTestRequest: ProviderCredentialTestRequest, options: RawAxiosRequestConfig = {}): Promise => { + // verify required parameter 'provider' is not null or undefined + assertParamExists('testProviderCredentials', 'provider', provider) + // verify required parameter 'providerCredentialTestRequest' is not null or undefined + assertParamExists('testProviderCredentials', 'providerCredentialTestRequest', providerCredentialTestRequest) + const localVarPath = `/api/v1/providers/{provider}/credentials/test` + .replace(`{${"provider"}}`, encodeURIComponent(String(provider))); + // use dummy base URL string because the URL constructor only accepts absolute URLs. + const localVarUrlObj = new URL(localVarPath, DUMMY_BASE_URL); + let baseOptions; + if (configuration) { + baseOptions = configuration.baseOptions; + } + + const localVarRequestOptions = { method: 'POST', ...baseOptions, ...options}; + const localVarHeaderParameter = {} as any; + const localVarQueryParameter = {} as any; + + // authentication SessionCookie required + + // authentication BearerAuth required + // http bearer authentication required + await setBearerAuthToObject(localVarHeaderParameter, configuration) + + localVarHeaderParameter['Content-Type'] = 'application/json'; + localVarHeaderParameter['Accept'] = 'application/json'; + + setSearchParams(localVarUrlObj, localVarQueryParameter); + let headersFromBaseOptions = baseOptions && baseOptions.headers ? baseOptions.headers : {}; + localVarRequestOptions.headers = {...localVarHeaderParameter, ...headersFromBaseOptions, ...options.headers}; + localVarRequestOptions.data = serializeDataIfNeeded(providerCredentialTestRequest, localVarRequestOptions, configuration) + + return { + url: toPathString(localVarUrlObj), + options: localVarRequestOptions, + }; + }, /** * Tests every configured LLM provider once using the catalog probe model. Provider-level failures are returned in the response body with HTTP 200. * @summary Test Providers @@ -221,7 +275,7 @@ export const ModelsApiFp = function(configuration?: Configuration) { const localVarAxiosParamCreator = ModelsApiAxiosParamCreator(configuration) return { /** - * Returns a paginated list of available LLM models from the built-in catalog. + * Returns one row per provider/model offering from the catalog. Model IDs are unique within a provider; `(provider, id)` is the resource identity. * @summary List Models * @param {string} [provider] Filter models by provider ID. Unknown provider IDs return an empty result set. * @param {string} [query] Case-insensitive substring search across `id`, `display_name`, and `aliases`. @@ -251,17 +305,32 @@ export const ModelsApiFp = function(configuration?: Configuration) { /** * Tests a model by sending a simple prompt and reporting pass/fail. * @summary Test Model - * @param {string} id The model identifier. + * @param {string} id The canonical model ID or an alias. + * @param {string} [provider] Pin the test to this provider\'s offering. When omitted, the server selects among ready providers by catalog priority. * @param {ModelTestMode} [mode] Test mode for the single-model test endpoint. Defaults to `basic`. * @param {*} [options] Override http request option. * @throws {RequiredError} */ - async testModel(id: string, mode?: ModelTestMode, options?: RawAxiosRequestConfig): Promise<(axios?: AxiosInstance, basePath?: string) => AxiosPromise> { - const localVarAxiosArgs = await localVarAxiosParamCreator.testModel(id, mode, options); + async testModel(id: string, provider?: string, mode?: ModelTestMode, options?: RawAxiosRequestConfig): Promise<(axios?: AxiosInstance, basePath?: string) => AxiosPromise> { + const localVarAxiosArgs = await localVarAxiosParamCreator.testModel(id, provider, mode, options); const localVarOperationServerIndex = configuration?.serverIndex ?? 0; const localVarOperationServerBasePath = operationServerMap['ModelsApi.testModel']?.[localVarOperationServerIndex]?.url; return (axios, basePath) => createRequestFunction(localVarAxiosArgs, globalAxios, BASE_PATH, configuration)(axios, localVarOperationServerBasePath || basePath); }, + /** + * Validates an LLM provider API key against the server\'s effective catalog without persisting it. + * @summary Test Provider Credentials + * @param {string} provider The provider identifier. + * @param {ProviderCredentialTestRequest} providerCredentialTestRequest + * @param {*} [options] Override http request option. + * @throws {RequiredError} + */ + async testProviderCredentials(provider: string, providerCredentialTestRequest: ProviderCredentialTestRequest, options?: RawAxiosRequestConfig): Promise<(axios?: AxiosInstance, basePath?: string) => AxiosPromise> { + const localVarAxiosArgs = await localVarAxiosParamCreator.testProviderCredentials(provider, providerCredentialTestRequest, options); + const localVarOperationServerIndex = configuration?.serverIndex ?? 0; + const localVarOperationServerBasePath = operationServerMap['ModelsApi.testProviderCredentials']?.[localVarOperationServerIndex]?.url; + return (axios, basePath) => createRequestFunction(localVarAxiosArgs, globalAxios, BASE_PATH, configuration)(axios, localVarOperationServerBasePath || basePath); + }, /** * Tests every configured LLM provider once using the catalog probe model. Provider-level failures are returned in the response body with HTTP 200. * @summary Test Providers @@ -284,7 +353,7 @@ export const ModelsApiFactory = function (configuration?: Configuration, basePat const localVarFp = ModelsApiFp(configuration) return { /** - * Returns a paginated list of available LLM models from the built-in catalog. + * Returns one row per provider/model offering from the catalog. Model IDs are unique within a provider; `(provider, id)` is the resource identity. * @summary List Models * @param {string} [provider] Filter models by provider ID. Unknown provider IDs return an empty result set. * @param {string} [query] Case-insensitive substring search across `id`, `display_name`, and `aliases`. @@ -308,13 +377,25 @@ export const ModelsApiFactory = function (configuration?: Configuration, basePat /** * Tests a model by sending a simple prompt and reporting pass/fail. * @summary Test Model - * @param {string} id The model identifier. + * @param {string} id The canonical model ID or an alias. + * @param {string} [provider] Pin the test to this provider\'s offering. When omitted, the server selects among ready providers by catalog priority. * @param {ModelTestMode} [mode] Test mode for the single-model test endpoint. Defaults to `basic`. * @param {*} [options] Override http request option. * @throws {RequiredError} */ - testModel(id: string, mode?: ModelTestMode, options?: RawAxiosRequestConfig): AxiosPromise { - return localVarFp.testModel(id, mode, options).then((request) => request(axios, basePath)); + testModel(id: string, provider?: string, mode?: ModelTestMode, options?: RawAxiosRequestConfig): AxiosPromise { + return localVarFp.testModel(id, provider, mode, options).then((request) => request(axios, basePath)); + }, + /** + * Validates an LLM provider API key against the server\'s effective catalog without persisting it. + * @summary Test Provider Credentials + * @param {string} provider The provider identifier. + * @param {ProviderCredentialTestRequest} providerCredentialTestRequest + * @param {*} [options] Override http request option. + * @throws {RequiredError} + */ + testProviderCredentials(provider: string, providerCredentialTestRequest: ProviderCredentialTestRequest, options?: RawAxiosRequestConfig): AxiosPromise { + return localVarFp.testProviderCredentials(provider, providerCredentialTestRequest, options).then((request) => request(axios, basePath)); }, /** * Tests every configured LLM provider once using the catalog probe model. Provider-level failures are returned in the response body with HTTP 200. @@ -333,7 +414,7 @@ export const ModelsApiFactory = function (configuration?: Configuration, basePat */ export class ModelsApi extends BaseAPI { /** - * Returns a paginated list of available LLM models from the built-in catalog. + * Returns one row per provider/model offering from the catalog. Model IDs are unique within a provider; `(provider, id)` is the resource identity. * @summary List Models * @param {string} [provider] Filter models by provider ID. Unknown provider IDs return an empty result set. * @param {string} [query] Case-insensitive substring search across `id`, `display_name`, and `aliases`. @@ -359,13 +440,26 @@ export class ModelsApi extends BaseAPI { /** * Tests a model by sending a simple prompt and reporting pass/fail. * @summary Test Model - * @param {string} id The model identifier. + * @param {string} id The canonical model ID or an alias. + * @param {string} [provider] Pin the test to this provider\'s offering. When omitted, the server selects among ready providers by catalog priority. * @param {ModelTestMode} [mode] Test mode for the single-model test endpoint. Defaults to `basic`. * @param {*} [options] Override http request option. * @throws {RequiredError} */ - public testModel(id: string, mode?: ModelTestMode, options?: RawAxiosRequestConfig) { - return ModelsApiFp(this.configuration).testModel(id, mode, options).then((request) => request(this.axios, this.basePath)); + public testModel(id: string, provider?: string, mode?: ModelTestMode, options?: RawAxiosRequestConfig) { + return ModelsApiFp(this.configuration).testModel(id, provider, mode, options).then((request) => request(this.axios, this.basePath)); + } + + /** + * Validates an LLM provider API key against the server\'s effective catalog without persisting it. + * @summary Test Provider Credentials + * @param {string} provider The provider identifier. + * @param {ProviderCredentialTestRequest} providerCredentialTestRequest + * @param {*} [options] Override http request option. + * @throws {RequiredError} + */ + public testProviderCredentials(provider: string, providerCredentialTestRequest: ProviderCredentialTestRequest, options?: RawAxiosRequestConfig) { + return ModelsApiFp(this.configuration).testProviderCredentials(provider, providerCredentialTestRequest, options).then((request) => request(this.axios, this.basePath)); } /** diff --git a/lib/packages/fabro-api-client/src/models/completion-response.ts b/lib/packages/fabro-api-client/src/models/completion-response.ts index 02c8f4b6d..5b3d34e4b 100644 --- a/lib/packages/fabro-api-client/src/models/completion-response.ts +++ b/lib/packages/fabro-api-client/src/models/completion-response.ts @@ -25,7 +25,14 @@ import type { CostSource } from './cost-source'; export interface CompletionResponse { 'id': string; + /** + * Canonical model ID selected for the request. + */ 'model': string; + /** + * LLM provider identifier. + */ + 'provider': string; 'message': CompletionMessage; /** * Why generation stopped (end_turn, max_tokens, tool_calls). diff --git a/lib/packages/fabro-api-client/src/models/create-completion-request.ts b/lib/packages/fabro-api-client/src/models/create-completion-request.ts index 2722bc9b9..923431e9b 100644 --- a/lib/packages/fabro-api-client/src/models/create-completion-request.ts +++ b/lib/packages/fabro-api-client/src/models/create-completion-request.ts @@ -29,7 +29,7 @@ export interface CreateCompletionRequest { */ 'messages': Array; /** - * Model ID or alias. Server picks default if omitted. + * Model ID or alias. Server picks a ready-provider default if omitted. */ 'model'?: string; /** @@ -58,7 +58,7 @@ export interface CreateCompletionRequest { */ 'reasoning_effort'?: string; /** - * Provider to route to. + * Optional provider pin. */ 'provider'?: string; 'provider_options'?: any; diff --git a/lib/packages/fabro-api-client/src/models/create-playground-chat-request.ts b/lib/packages/fabro-api-client/src/models/create-playground-chat-request.ts index d0a12f33c..1d1ddc216 100644 --- a/lib/packages/fabro-api-client/src/models/create-playground-chat-request.ts +++ b/lib/packages/fabro-api-client/src/models/create-playground-chat-request.ts @@ -30,7 +30,11 @@ export interface CreatePlaygroundChatRequest { */ 'workflow_fabro': string; /** - * Model id or alias. Server picks the default if omitted. + * Model ID or alias. Server picks a ready-provider default if omitted. */ 'model'?: string; + /** + * Optional provider pin. + */ + 'provider'?: string; } diff --git a/lib/packages/fabro-api-client/src/models/create-run-session-request.ts b/lib/packages/fabro-api-client/src/models/create-run-session-request.ts index 6b11c4b39..bea185e6d 100644 --- a/lib/packages/fabro-api-client/src/models/create-run-session-request.ts +++ b/lib/packages/fabro-api-client/src/models/create-run-session-request.ts @@ -17,7 +17,11 @@ export interface CreateRunSessionRequest { 'title'?: string; /** - * Catalog model ID or alias, or provider-qualified provider/model reference. Stored as the canonical catalog model ID. + * Catalog model ID or alias. The server selects among ready providers and stores the canonical model ID. */ 'model'?: string; + /** + * Optional provider pin. Provider-qualified model references remain accepted for compatibility. + */ + 'provider'?: string; } diff --git a/lib/packages/fabro-api-client/src/models/hook-definition.ts b/lib/packages/fabro-api-client/src/models/hook-definition.ts index a7d2f7cf1..85092d14f 100644 --- a/lib/packages/fabro-api-client/src/models/hook-definition.ts +++ b/lib/packages/fabro-api-client/src/models/hook-definition.ts @@ -27,6 +27,9 @@ export interface HookDefinition { 'type'?: HookDefinitionTypeEnum | null; 'url'?: string | null; 'headers'?: { [key: string]: string; } | null; + /** + * Allowlist of environment variable names that an http hook header may read via `{{ env.NAME }}`. An empty list (the default) permits no env vars in headers. + */ 'allowed_env_vars'?: Array; 'tls'?: TlsMode; 'prompt'?: string | null; diff --git a/lib/packages/fabro-api-client/src/models/index.ts b/lib/packages/fabro-api-client/src/models/index.ts index 7b51ee78c..c8c1e5538 100644 --- a/lib/packages/fabro-api-client/src/models/index.ts +++ b/lib/packages/fabro-api-client/src/models/index.ts @@ -264,6 +264,8 @@ export * from './principal-webhook'; export * from './principal-worker'; export * from './project-namespace'; export * from './provider'; +export * from './provider-credential-test-request'; +export * from './provider-credential-test-response'; export * from './provider-list'; export * from './provider-test-list'; export * from './provider-test-result'; diff --git a/lib/packages/fabro-api-client/src/models/model-test-result.ts b/lib/packages/fabro-api-client/src/models/model-test-result.ts index 0155da8e7..cba04c056 100644 --- a/lib/packages/fabro-api-client/src/models/model-test-result.ts +++ b/lib/packages/fabro-api-client/src/models/model-test-result.ts @@ -19,9 +19,13 @@ */ export interface ModelTestResult { /** - * The model identifier that was tested. + * The canonical model ID that was tested. */ 'model_id': string; + /** + * LLM provider identifier. + */ + 'provider': string; /** * Whether the model responded successfully, failed, or was skipped because its provider is not configured. */ diff --git a/lib/packages/fabro-api-client/src/models/model.ts b/lib/packages/fabro-api-client/src/models/model.ts index f02cb0e51..d3326e2db 100644 --- a/lib/packages/fabro-api-client/src/models/model.ts +++ b/lib/packages/fabro-api-client/src/models/model.ts @@ -24,11 +24,11 @@ import type { ModelFeatures } from './model-features'; import type { ModelLimits } from './model-limits'; /** - * An available LLM model from the built-in catalog. + * One provider\'s offering of an LLM model. The `id` is unique within `provider`; `(provider, id)` is the stable resource identity. */ export interface Model { /** - * Unique model identifier. + * Canonical human-facing model ID, unique within the provider. */ 'id': string; /** diff --git a/lib/packages/fabro-api-client/src/models/provider-credential-test-request.ts b/lib/packages/fabro-api-client/src/models/provider-credential-test-request.ts new file mode 100644 index 000000000..390b00848 --- /dev/null +++ b/lib/packages/fabro-api-client/src/models/provider-credential-test-request.ts @@ -0,0 +1,22 @@ +/* tslint:disable */ +/* eslint-disable */ +/** + * Fabro Run API + * HTTP API for managing Fabro workflow run executions. + * + * The version of the OpenAPI document: 0.1.0 + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + + + +/** + * API key to validate against an LLM provider without persisting it. + */ +export interface ProviderCredentialTestRequest { + 'api_key': string; +} diff --git a/lib/packages/fabro-api-client/src/models/provider-credential-test-response.ts b/lib/packages/fabro-api-client/src/models/provider-credential-test-response.ts new file mode 100644 index 000000000..b74a688eb --- /dev/null +++ b/lib/packages/fabro-api-client/src/models/provider-credential-test-response.ts @@ -0,0 +1,22 @@ +/* tslint:disable */ +/* eslint-disable */ +/** + * Fabro Run API + * HTTP API for managing Fabro workflow run executions. + * + * The version of the OpenAPI document: 0.1.0 + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + + + +/** + * Successful response from provider credential validation. + */ +export interface ProviderCredentialTestResponse { + 'ok': boolean; +} diff --git a/lib/packages/fabro-api-client/src/models/session-detail.ts b/lib/packages/fabro-api-client/src/models/session-detail.ts index d2f072227..e1dce8d26 100644 --- a/lib/packages/fabro-api-client/src/models/session-detail.ts +++ b/lib/packages/fabro-api-client/src/models/session-detail.ts @@ -34,7 +34,14 @@ export interface SessionDetail { 'run_id': string; 'title'?: string | null; 'status': SessionStatus; + /** + * Canonical model ID selected when the session was created. + */ 'model'?: string | null; + /** + * LLM provider identifier. + */ + 'provider'?: string | null; 'active_turn': SessionTurn | null; 'created_at': string; 'updated_at': string; diff --git a/lib/packages/fabro-api-client/src/models/session-record.ts b/lib/packages/fabro-api-client/src/models/session-record.ts index 94aa77aa7..5bb2ee06d 100644 --- a/lib/packages/fabro-api-client/src/models/session-record.ts +++ b/lib/packages/fabro-api-client/src/models/session-record.ts @@ -31,7 +31,14 @@ export interface SessionRecord { 'run_id': string; 'title'?: string | null; 'status': SessionStatus; + /** + * Canonical model ID selected when the session was created. + */ 'model'?: string | null; + /** + * LLM provider identifier. + */ + 'provider'?: string | null; 'active_turn': SessionTurn | null; 'created_at': string; 'updated_at': string; diff --git a/lib/packages/fabro-api-client/src/models/session-summary.ts b/lib/packages/fabro-api-client/src/models/session-summary.ts index 4c5e3c75e..6520d77c7 100644 --- a/lib/packages/fabro-api-client/src/models/session-summary.ts +++ b/lib/packages/fabro-api-client/src/models/session-summary.ts @@ -31,7 +31,14 @@ export interface SessionSummary { 'run_id': string; 'title'?: string | null; 'status': SessionStatus; + /** + * Canonical model ID selected when the session was created. + */ 'model'?: string | null; + /** + * LLM provider identifier. + */ + 'provider'?: string | null; 'active_turn': SessionTurn | null; 'created_at': string; 'updated_at': string; From 05e65070954d2dcbb7588eeb9a35fac455c7019a Mon Sep 17 00:00:00 2001 From: Bryan Helmkamp Date: Thu, 23 Jul 2026 11:07:47 -0400 Subject: [PATCH 02/16] refactor: unify model selection and legacy-model normalization Consolidate duplicated resolution logic introduced by the provider-aware model alias work: - Add Catalog::resolve_selection (with SelectedModel and ready_provider) as the single home for the select -> passthrough -> default-fallback policy, replacing six hand-rolled copies across fabro-server, fabro-workflow, and fabro-llm. - Move legacy [models] row resolution into a shared LegacyModelIndex and LegacyModelError in fabro-model; delete fabro-config's parallel implementation and its LlmNormalizationError enum, plus the now-unused builtin_* catalog helpers. - Drop redundant client.resolve_request calls (and their full-request clones) from the completions and playground handlers. - Remove the redundant resolve_provider_context round-trip in resolve_start_llm and make resolve_run_model return a ProviderId instead of a never-None Option. - Replace the "" sentinel selector with a dedicated ModelSelectionError::NoDefaultModel variant. - Add a CatalogRoute trait so provider adapters call self.api_model_id(...) instead of threading catalog/provider args. - Delete the unused FromStr impl for ModelId; dedupe the CLI's id-or-alias predicate. Co-Authored-By: Claude Fable 5 --- lib/crates/fabro-cli/src/commands/model.rs | 13 +- lib/crates/fabro-config/src/layers/llm.rs | 284 ++------- lib/crates/fabro-config/src/layers/mod.rs | 6 +- lib/crates/fabro-config/src/lib.rs | 25 +- lib/crates/fabro-config/src/parse.rs | 13 +- lib/crates/fabro-llm/src/client.rs | 17 +- .../fabro-llm/src/providers/anthropic.rs | 39 +- .../fabro-llm/src/providers/bedrock/mod.rs | 30 +- lib/crates/fabro-llm/src/providers/common.rs | 17 + lib/crates/fabro-llm/src/providers/gemini.rs | 36 +- lib/crates/fabro-llm/src/providers/openai.rs | 36 +- .../src/providers/openai_compatible.rs | 20 +- lib/crates/fabro-model/src/catalog.rs | 568 +++++++++++------- lib/crates/fabro-model/src/ids.rs | 8 - lib/crates/fabro-model/src/lib.rs | 2 +- .../src/server/handler/completions.rs | 59 +- .../fabro-server/src/server/handler/models.rs | 3 +- .../src/server/handler/playground.rs | 8 +- .../src/server/handler/sessions.rs | 83 +-- lib/crates/fabro-server/src/server/tests.rs | 2 +- .../fabro-workflow/src/operations/start.rs | 13 +- .../fabro-workflow/src/run_materialization.rs | 59 +- .../src/transforms/model_resolution.rs | 26 +- 23 files changed, 560 insertions(+), 807 deletions(-) diff --git a/lib/crates/fabro-cli/src/commands/model.rs b/lib/crates/fabro-cli/src/commands/model.rs index d5d41bb20..4e5bd6f39 100644 --- a/lib/crates/fabro-cli/src/commands/model.rs +++ b/lib/crates/fabro-cli/src/commands/model.rs @@ -45,6 +45,10 @@ struct CompletedModelTest { status: String, } +fn model_matches_selector(model: &Model, selector: &str) -> bool { + model.id == selector || model.aliases.iter().any(|alias| alias == selector) +} + fn find_model_by_id_or_alias( models: &[Model], id: &str, @@ -54,7 +58,7 @@ fn find_model_by_id_or_alias( .iter() .find(|model| { provider.is_none_or(|provider| &model.provider == provider) - && (model.id == id || model.aliases.iter().any(|alias| alias == id)) + && model_matches_selector(model, id) }) .cloned() } @@ -275,10 +279,9 @@ async fn test_models_via_server( if !json_output { eprint!("Testing {model_id}..."); } - let has_configured_match = listed_models.iter().any(|model| { - model.configured - && (model.id == model_id || model.aliases.iter().any(|alias| alias == model_id)) - }); + let has_configured_match = listed_models + .iter() + .any(|model| model.configured && model_matches_selector(model, model_id)); let result = if requested_provider.is_none() && listed_info.is_some() && !has_configured_match { None diff --git a/lib/crates/fabro-config/src/layers/llm.rs b/lib/crates/fabro-config/src/layers/llm.rs index 32169b980..ca29d2bbd 100644 --- a/lib/crates/fabro-config/src/layers/llm.rs +++ b/lib/crates/fabro-config/src/layers/llm.rs @@ -25,7 +25,7 @@ //! Resolution against the static adapter registry happens in `fabro-model` //! when the resolved [`Catalog`](fabro_model::Catalog) is built. -use std::collections::{BTreeMap, BTreeSet, HashMap}; +use std::collections::{BTreeMap, HashMap}; use fabro_model::catalog::deserialize_knowledge_cutoff; use fabro_model::{ @@ -160,90 +160,16 @@ pub struct ModelSettings { pub costs: Option, } -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum LlmNormalizationError { - BuiltinCatalog { - message: String, - }, - RetiredModel { - identifier: String, - provider: ProviderId, - model: ModelId, - }, - UnknownLegacyModel { - model: String, - }, - AmbiguousLegacyModel { - model: String, - candidates: Vec<(ProviderId, ModelId)>, - }, - AmbiguousProviderModel { - provider: ProviderId, - selector: String, - models: Vec, - }, - DuplicateModelDefinition { - provider: ProviderId, - model: ModelId, - }, - ScopedModelDeclaresProvider { - provider: ProviderId, - model: ModelId, - }, -} - -impl std::fmt::Display for LlmNormalizationError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - Self::BuiltinCatalog { message } => { - write!(f, "failed to inspect the built-in model catalog: {message}") - } - Self::RetiredModel { - identifier, - provider, - model, - } => write!( - f, - "model identifier '{identifier}' has been retired; use '{provider}/{model}' instead" - ), - Self::UnknownLegacyModel { model } => write!( - f, - "legacy model row '{model}' omits provider and does not match a unique known offering" - ), - Self::AmbiguousLegacyModel { model, candidates } => write!( - f, - "legacy model row '{model}' omits provider and matches multiple offerings: {candidates:?}" - ), - Self::AmbiguousProviderModel { - provider, - selector, - models, - } => write!( - f, - "legacy model selector '{selector}' is ambiguous on provider '{provider}': {models:?}" - ), - Self::DuplicateModelDefinition { provider, model } => write!( - f, - "provider '{provider}' model '{model}' is defined through both provider-scoped and legacy top-level syntax" - ), - Self::ScopedModelDeclaresProvider { provider, model } => write!( - f, - "provider-scoped model '{provider}/{model}' must not declare a provider field" - ), - } - } -} - -impl std::error::Error for LlmNormalizationError {} - impl LlmLayer { /// Normalize the temporary legacy model table before this source is - /// combined with any other settings source. - pub(crate) fn normalize_legacy_models(&mut self) -> Result<(), LlmNormalizationError> { + /// combined with any other settings source. Resolution runs against this + /// layer's own providers plus the built-in catalog, because a single + /// source may reference built-in offerings that merge in later. + pub(crate) fn normalize_legacy_models(&mut self) -> Result<(), catalog::LegacyModelError> { for (provider, settings) in self.providers.iter() { for (model, settings) in settings.models.iter() { if settings.provider.is_some() { - return Err(LlmNormalizationError::ScopedModelDeclaresProvider { + return Err(catalog::LegacyModelError::ScopedModelDeclaresProvider { provider: ProviderId::new(provider.clone()), model: ModelId::new(model.clone()), }); @@ -251,46 +177,45 @@ impl LlmLayer { } } - let mut legacy_models = std::mem::take(&mut self.models.0) - .into_iter() - .collect::>(); + let legacy_models = std::mem::take(&mut self.models.0); + if legacy_models.is_empty() { + return Ok(()); + } + let mut legacy_models = legacy_models.into_iter().collect::>(); legacy_models.sort_by(|(left, _), (right, _)| left.cmp(right)); - for (legacy_id, mut settings) in legacy_models { - if let Some((provider, model)) = catalog::retired_model_replacement(&legacy_id) { - return Err(LlmNormalizationError::RetiredModel { - identifier: legacy_id, - provider, - model, - }); - } + let mut index = catalog::LegacyModelIndex::default(); + let mut provider_ids = self.providers.keys().cloned().collect::>(); + provider_ids.sort_unstable(); + for provider_id in &provider_ids { + let settings = self + .providers + .get(provider_id) + .expect("provider ID came from provider map keys"); + let mut model_ids = settings.models.keys().cloned().collect::>(); + model_ids.sort_unstable(); + index.add_provider( + ProviderId::new(provider_id.clone()), + settings.aliases.clone().unwrap_or_default(), + model_ids.into_iter().map(|model_id| { + let model = settings + .models + .get(&model_id) + .expect("model ID came from model map keys"); + let aliases = model.aliases.clone().unwrap_or_default(); + (ModelId::new(model_id), aliases) + }), + ); + } + let index = index.with_builtin()?; + for (legacy_id, mut settings) in legacy_models { let explicit_provider = settings.provider.take(); - let (provider, model) = if let Some(provider) = explicit_provider { - let provider = self.canonical_provider(&provider)?; - let model = self - .canonical_model_on_provider(&provider, &legacy_id)? - .unwrap_or_else(|| ModelId::new(legacy_id.clone())); - (provider, model) - } else { - let candidates = self.model_candidates(&legacy_id)?; - match candidates.as_slice() { - [(provider, model)] => (provider.clone(), model.clone()), - [] => { - return Err(LlmNormalizationError::UnknownLegacyModel { model: legacy_id }); - } - _ => { - return Err(LlmNormalizationError::AmbiguousLegacyModel { - model: legacy_id, - candidates, - }); - } - } - }; + let (provider, model) = index.resolve(&legacy_id, explicit_provider.as_deref())?; let provider_settings = self.providers.entry(provider.to_string()).or_default(); if provider_settings.models.contains_key(model.as_str()) { - return Err(LlmNormalizationError::DuplicateModelDefinition { provider, model }); + return Err(catalog::LegacyModelError::DuplicateModel { provider, model }); } provider_settings .models @@ -298,139 +223,6 @@ impl LlmLayer { } Ok(()) } - - fn canonical_provider(&self, selector: &str) -> Result { - if self.providers.contains_key(selector) { - return Ok(ProviderId::new(selector)); - } - let mut aliases = self - .providers - .iter() - .filter(|(_, settings)| { - settings - .aliases - .as_ref() - .is_some_and(|aliases| aliases.iter().any(|alias| alias == selector)) - }) - .map(|(provider, _)| ProviderId::new(provider.clone())) - .collect::>(); - aliases.sort(); - if let Some(provider) = aliases.into_iter().next() { - return Ok(provider); - } - catalog::builtin_provider_id(selector) - .map_err(|error| normalization_catalog_error(&error)) - .map(|provider| provider.unwrap_or_else(|| ProviderId::new(selector))) - } - - fn canonical_model_on_provider( - &self, - provider: &ProviderId, - selector: &str, - ) -> Result, LlmNormalizationError> { - let current = self - .providers - .get(provider.as_str()) - .map(|settings| &settings.models); - if current.is_some_and(|models| models.contains_key(selector)) { - return Ok(Some(ModelId::new(selector))); - } - - let builtin_canonical = catalog::builtin_canonical_model_offerings(selector) - .map_err(|error| normalization_catalog_error(&error))? - .into_iter() - .find_map(|(candidate_provider, model)| { - (candidate_provider == *provider).then_some(model) - }); - if builtin_canonical.is_some() { - return Ok(builtin_canonical); - } - - let mut aliases = current - .into_iter() - .flat_map(|models| models.iter()) - .filter(|(_, settings)| { - settings - .aliases - .as_ref() - .is_some_and(|aliases| aliases.iter().any(|alias| alias == selector)) - }) - .map(|(model, _)| ModelId::new(model.clone())) - .collect::>(); - aliases.extend( - catalog::builtin_alias_model_offerings(selector) - .map_err(|error| normalization_catalog_error(&error))? - .into_iter() - .filter_map(|(candidate_provider, model)| { - (candidate_provider == *provider).then_some(model) - }), - ); - match aliases.len() { - 0 => Ok(None), - 1 => Ok(aliases.into_iter().next()), - _ => Err(LlmNormalizationError::AmbiguousProviderModel { - provider: provider.clone(), - selector: selector.to_string(), - models: aliases.into_iter().collect(), - }), - } - } - - fn model_candidates( - &self, - selector: &str, - ) -> Result, LlmNormalizationError> { - let mut canonical = self - .providers - .iter() - .filter(|(_, settings)| settings.models.contains_key(selector)) - .map(|(provider, _)| { - ( - ProviderId::new(provider.clone()), - ModelId::new(selector.to_string()), - ) - }) - .collect::>(); - canonical.extend( - catalog::builtin_canonical_model_offerings(selector) - .map_err(|error| normalization_catalog_error(&error))?, - ); - if !canonical.is_empty() { - return Ok(canonical.into_iter().collect()); - } - - let mut aliases = - self.providers - .iter() - .flat_map(|(provider, settings)| { - settings - .models - .iter() - .filter(move |(_, settings)| { - settings.aliases.as_ref().is_some_and(|aliases| { - aliases.iter().any(|alias| alias == selector) - }) - }) - .map(move |(model, _)| { - ( - ProviderId::new(provider.clone()), - ModelId::new(model.clone()), - ) - }) - }) - .collect::>(); - aliases.extend( - catalog::builtin_alias_model_offerings(selector) - .map_err(|error| normalization_catalog_error(&error))?, - ); - Ok(aliases.into_iter().collect()) - } -} - -fn normalization_catalog_error(error: &catalog::CatalogBuildError) -> LlmNormalizationError { - LlmNormalizationError::BuiltinCatalog { - message: error.to_string(), - } } #[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize, fabro_macros::Combine)] diff --git a/lib/crates/fabro-config/src/layers/mod.rs b/lib/crates/fabro-config/src/layers/mod.rs index c25838781..c3fa1632c 100644 --- a/lib/crates/fabro-config/src/layers/mod.rs +++ b/lib/crates/fabro-config/src/layers/mod.rs @@ -21,9 +21,9 @@ pub use environment::{ EnvironmentNetworkLayer, EnvironmentResourcesLayer, RunEnvironmentLayer, }; pub use llm::{ - CostRates, CredentialRef, CredentialRefParseError, LlmLayer, LlmNormalizationError, - ModelControls, ModelCostTable, ModelFeatures as LlmModelFeatures, - ModelLimits as LlmModelLimits, ModelSettings, ProviderSettings, ReasoningEffortFeature, + CostRates, CredentialRef, CredentialRefParseError, LlmLayer, ModelControls, ModelCostTable, + ModelFeatures as LlmModelFeatures, ModelLimits as LlmModelLimits, ModelSettings, + ProviderSettings, ReasoningEffortFeature, }; pub use log_filter::LogFilter; pub use maps::{MergeMap, ReplaceMap, StickyMap}; diff --git a/lib/crates/fabro-config/src/lib.rs b/lib/crates/fabro-config/src/lib.rs index 694ab1773..f6097b9be 100644 --- a/lib/crates/fabro-config/src/lib.rs +++ b/lib/crates/fabro-config/src/lib.rs @@ -46,19 +46,18 @@ pub use layers::{ CredentialRefParseError, EnvironmentDockerfileLayer, EnvironmentImageLayer, EnvironmentLayer, EnvironmentLifecycleLayer, EnvironmentNetworkLayer, EnvironmentResourcesLayer, GitAuthorLayer, GithubIntegrationLayer, HookAgentMarker, HookEntry, HookTlsMode, IntegrationWebhooksLayer, - InterviewProviderLayer, InterviewsLayer, LlmLayer, LlmModelFeatures, LlmModelLimits, - LlmNormalizationError, LogFilter, McpEntryLayer, MergeMap, ModelControls, ModelCostTable, - ModelRefOrSplice, ModelSettings, NotificationProviderLayer, NotificationRouteLayer, - ObjectStoreLocalLayer, ObjectStoreS3Layer, PrepareStep, ProjectLayer, ProviderSettings, - ReasoningEffortFeature, ReplaceMap, RunAgentLayer, RunArtifactsLayer, RunCheckpointLayer, - RunCloneLayer, RunEnvironmentLayer, RunExecutionLayer, RunGitLayer, RunGoalLayer, - RunIntegrationsGithubLayer, RunIntegrationsLayer, RunLayer, RunMetaBranchLayer, - RunModelControlsLayer, RunModelLayer, RunPrepareLayer, RunPullRequestLayer, RunRunBranchLayer, - RunScmLayer, ScmGitHubLayer, ServerApiLayer, ServerArtifactsLayer, ServerAuthGithubLayer, - ServerAuthLayer, ServerIntegrationsLayer, ServerLayer, ServerListenLayer, ServerLoggingLayer, - ServerSandboxLayer, ServerSandboxProviderLayer, ServerSandboxProvidersLayer, - ServerSchedulerLayer, ServerSlateDbLayer, ServerStorageLayer, ServerWebLayer, SettingsLayer, - SlackIntegrationLayer, StickyMap, StringOrSplice, WorkflowLayer, + InterviewProviderLayer, InterviewsLayer, LlmLayer, LlmModelFeatures, LlmModelLimits, LogFilter, + McpEntryLayer, MergeMap, ModelControls, ModelCostTable, ModelRefOrSplice, ModelSettings, + NotificationProviderLayer, NotificationRouteLayer, ObjectStoreLocalLayer, ObjectStoreS3Layer, + PrepareStep, ProjectLayer, ProviderSettings, ReasoningEffortFeature, ReplaceMap, RunAgentLayer, + RunArtifactsLayer, RunCheckpointLayer, RunCloneLayer, RunEnvironmentLayer, RunExecutionLayer, + RunGitLayer, RunGoalLayer, RunIntegrationsGithubLayer, RunIntegrationsLayer, RunLayer, + RunMetaBranchLayer, RunModelControlsLayer, RunModelLayer, RunPrepareLayer, RunPullRequestLayer, + RunRunBranchLayer, RunScmLayer, ScmGitHubLayer, ServerApiLayer, ServerArtifactsLayer, + ServerAuthGithubLayer, ServerAuthLayer, ServerIntegrationsLayer, ServerLayer, + ServerListenLayer, ServerLoggingLayer, ServerSandboxLayer, ServerSandboxProviderLayer, + ServerSandboxProvidersLayer, ServerSchedulerLayer, ServerSlateDbLayer, ServerStorageLayer, + ServerWebLayer, SettingsLayer, SlackIntegrationLayer, StickyMap, StringOrSplice, WorkflowLayer, }; pub use logging::{resolve_log_destination, resolve_log_destination_with_env}; pub use parse::ParseError; diff --git a/lib/crates/fabro-config/src/parse.rs b/lib/crates/fabro-config/src/parse.rs index 025e0b130..fb72c35af 100644 --- a/lib/crates/fabro-config/src/parse.rs +++ b/lib/crates/fabro-config/src/parse.rs @@ -1,7 +1,8 @@ use std::fmt; +use fabro_model::catalog::LegacyModelError; + use crate::SettingsLayer; -use crate::layers::LlmNormalizationError; const CURRENT_VERSION: u32 = 1; @@ -31,7 +32,7 @@ const LEGACY_LLM_KEYS: &[&str] = &[ #[derive(Debug, Clone, PartialEq, Eq)] pub enum ParseError { Toml(String), - LlmCatalog(LlmNormalizationError), + LlmCatalog(LegacyModelError), Version(VersionError), UnknownTopLevelKey { key: String, @@ -325,7 +326,7 @@ provider = "openai" assert!(matches!( error, - ParseError::LlmCatalog(LlmNormalizationError::ScopedModelDeclaresProvider { + ParseError::LlmCatalog(LegacyModelError::ScopedModelDeclaresProvider { provider, model, }) if provider.as_str() == "openai" && model.as_str() == "gpt-5.4" @@ -398,7 +399,7 @@ display_name = "Ambiguous" assert!(matches!( error, - ParseError::LlmCatalog(LlmNormalizationError::AmbiguousLegacyModel { + ParseError::LlmCatalog(LegacyModelError::AmbiguousModel { model, candidates, }) if model == "gpt-5.6-sol" && candidates.len() >= 2 @@ -420,7 +421,7 @@ display_name = "Legacy" assert!(matches!( error, - ParseError::LlmCatalog(LlmNormalizationError::DuplicateModelDefinition { + ParseError::LlmCatalog(LegacyModelError::DuplicateModel { provider, model, }) if provider.as_str() == "openai" && model.as_str() == "gpt-5.4" @@ -438,7 +439,7 @@ provider = "openrouter" assert!(matches!( error, - ParseError::LlmCatalog(LlmNormalizationError::RetiredModel { + ParseError::LlmCatalog(LegacyModelError::Retired { identifier, provider, model, diff --git a/lib/crates/fabro-llm/src/client.rs b/lib/crates/fabro-llm/src/client.rs index e40a8ede2..446b0b77a 100644 --- a/lib/crates/fabro-llm/src/client.rs +++ b/lib/crates/fabro-llm/src/client.rs @@ -309,20 +309,19 @@ impl Client { let eligible = self.eligible_provider_ids(); if let Some(explicit) = request.provider.as_deref() { let explicit = ProviderId::new(explicit); - if let Some(catalog_provider) = catalog.provider(&explicit) { + if catalog.provider(&explicit).is_some() { + let selected = catalog + .resolve_selection(Some(&request.model), Some(&explicit), &eligible) + .map_err(selection_error)?; let provider = self - .provider_adapter(catalog_provider.id.as_str()) + .provider_adapter(selected.provider.as_str()) .ok_or_else(|| { selection_error(ModelSelectionError::ProviderUnavailable { - provider: catalog_provider.id.clone(), + provider: selected.provider.clone(), }) })?; - match catalog.resolve_on_provider(&catalog_provider.id, &request.model) { - Ok(model) => resolved.model = model.id.to_string(), - Err(ModelSelectionError::UnknownSelectorOnProvider { .. }) => {} - Err(error) => return Err(selection_error(error)), - } - resolved.provider = Some(catalog_provider.id.to_string()); + resolved.model = selected.model; + resolved.provider = Some(selected.provider.into_inner()); return Ok(ResolvedRequest { provider, request: resolved, diff --git a/lib/crates/fabro-llm/src/providers/anthropic.rs b/lib/crates/fabro-llm/src/providers/anthropic.rs index 539b2b0fc..83894e9a9 100644 --- a/lib/crates/fabro-llm/src/providers/anthropic.rs +++ b/lib/crates/fabro-llm/src/providers/anthropic.rs @@ -7,7 +7,7 @@ use crate::codec::anthropic_messages::{AnthropicMessages, anthropic_option}; use crate::codec::{AnthropicVersion, Codec, CodecCtx, CodecParams, EncodedRequest}; use crate::error::Error; use crate::provider::{self, ProviderAdapter, StreamEventStream}; -use crate::providers::common::{self as common}; +use crate::providers::common::{self as common, CatalogRoute}; use crate::token_count::{InputTokenCount, InputTokenCountMethod}; use crate::transport::{self, HttpTransport, SseFraming}; use crate::types::{AdapterTimeout, Request, Response, StreamEvent}; @@ -118,11 +118,7 @@ impl Adapter { request, provider_name: &self.provider_name, deployment_id, - model: common::catalog_model( - self.catalog.as_deref(), - &self.provider_name, - &request.model, - ), + model: self.catalog_model(&request.model), params, } } @@ -214,6 +210,16 @@ fn anthropic_thinking_type(provider_options: Option<&serde_json::Value>) -> Opti .and_then(serde_json::Value::as_str) } +impl common::CatalogRoute for Adapter { + fn catalog(&self) -> Option<&Catalog> { + self.catalog.as_deref() + } + + fn provider_name(&self) -> &str { + &self.provider_name + } +} + #[async_trait::async_trait] impl ProviderAdapter for Adapter { fn name(&self) -> &str { @@ -232,11 +238,7 @@ impl ProviderAdapter for Adapter { self.validate_request(request)?; let resolved = self.resolve_request(request).await; let codec = AnthropicMessages; - let deployment_id = common::api_model_id( - self.catalog.as_deref(), - &self.provider_name, - &resolved.model, - ); + let deployment_id = self.api_model_id(&resolved.model); let ctx = self.codec_ctx(&resolved, &deployment_id, &route.codec_params); let Some(encoded) = codec.encode_count_tokens(&ctx).transpose()? else { @@ -272,11 +274,7 @@ impl ProviderAdapter for Adapter { let resolved = self.resolve_request(request).await; let codec = AnthropicMessages; - let deployment_id = common::api_model_id( - self.catalog.as_deref(), - &self.provider_name, - &resolved.model, - ); + let deployment_id = self.api_model_id(&resolved.model); let ctx = self.codec_ctx(&resolved, &deployment_id, &route.codec_params); let encoded = codec.encode(&ctx, false)?; @@ -293,11 +291,7 @@ impl ProviderAdapter for Adapter { let route = self.route_config(); let resolved = self.resolve_request(request).await; let codec = AnthropicMessages; - let deployment_id = common::api_model_id( - self.catalog.as_deref(), - &self.provider_name, - &resolved.model, - ); + let deployment_id = self.api_model_id(&resolved.model); let ctx = self.codec_ctx(&resolved, &deployment_id, &route.codec_params); let encoded = codec.encode(&ctx, true)?; @@ -323,8 +317,7 @@ impl ProviderAdapter for Adapter { // Always-adaptive models reject manual enabled/disabled thinking // configs at the API, so fail them locally with a clear message // instead. - let model_info = - common::catalog_model(self.catalog.as_deref(), &self.provider_name, &request.model); + let model_info = self.catalog_model(&request.model); if let Some(model) = model_info .filter(|m| m.features.reasoning_effort == ReasoningEffortFeature::AlwaysAdaptive) { diff --git a/lib/crates/fabro-llm/src/providers/bedrock/mod.rs b/lib/crates/fabro-llm/src/providers/bedrock/mod.rs index 97c15dc02..deca69bd0 100644 --- a/lib/crates/fabro-llm/src/providers/bedrock/mod.rs +++ b/lib/crates/fabro-llm/src/providers/bedrock/mod.rs @@ -29,7 +29,7 @@ use crate::codec::bedrock_converse::BedrockConverse; use crate::codec::{Codec, CodecCtx, CodecParams, EncodedRequest, RawEvent, StreamDecoder}; use crate::error::Error; use crate::provider::{self, ProviderAdapter, StreamEventStream}; -use crate::providers::common::{self as common}; +use crate::providers::common::{self as common, CatalogRoute}; use crate::transport::{self, HttpTransport}; use crate::types::{AdapterTimeout, Request, Response, StreamEvent}; @@ -169,11 +169,7 @@ impl Adapter { request, provider_name: &self.provider_name, deployment_id, - model: common::catalog_model( - self.catalog.as_deref(), - &self.provider_name, - &request.model, - ), + model: self.catalog_model(&request.model), params, } } @@ -232,6 +228,16 @@ impl Adapter { } } +impl common::CatalogRoute for Adapter { + fn catalog(&self) -> Option<&Catalog> { + self.catalog.as_deref() + } + + fn provider_name(&self) -> &str { + &self.provider_name + } +} + #[async_trait::async_trait] impl ProviderAdapter for Adapter { fn name(&self) -> &str { @@ -243,11 +249,7 @@ impl ProviderAdapter for Adapter { let resolved = self.resolve_request(request).await; let codec = BedrockConverse; - let deployment_id = common::api_model_id( - self.catalog.as_deref(), - &self.provider_name, - &resolved.model, - ); + let deployment_id = self.api_model_id(&resolved.model); let params = CodecParams::default(); let ctx = self.codec_ctx(&resolved, &deployment_id, ¶ms); @@ -261,11 +263,7 @@ impl ProviderAdapter for Adapter { let resolved = self.resolve_request(request).await; let codec = BedrockConverse; - let deployment_id = common::api_model_id( - self.catalog.as_deref(), - &self.provider_name, - &resolved.model, - ); + let deployment_id = self.api_model_id(&resolved.model); let params = CodecParams::default(); let ctx = self.codec_ctx(&resolved, &deployment_id, ¶ms); diff --git a/lib/crates/fabro-llm/src/providers/common.rs b/lib/crates/fabro-llm/src/providers/common.rs index f85397935..6eff3b727 100644 --- a/lib/crates/fabro-llm/src/providers/common.rs +++ b/lib/crates/fabro-llm/src/providers/common.rs @@ -20,6 +20,23 @@ pub fn api_model_id(catalog: Option<&Catalog>, provider: &str, model: &str) -> S .map_or_else(|| model.to_string(), |settings| settings.api_id.clone()) } +/// Adapters that route models through an optional catalog scoped to one +/// provider name. +pub trait CatalogRoute { + fn catalog(&self) -> Option<&Catalog>; + fn provider_name(&self) -> &str; + + /// Catalog offering for a canonical ID or alias on this provider. + fn catalog_model(&self, model: &str) -> Option<&Model> { + catalog_model(self.catalog(), self.provider_name(), model) + } + + /// Identifier sent to the provider API for a model. + fn api_model_id(&self, model: &str) -> String { + api_model_id(self.catalog(), self.provider_name(), model) + } +} + /// Check if a URL string looks like a local file path. #[must_use] pub fn is_file_path(url: &str) -> bool { diff --git a/lib/crates/fabro-llm/src/providers/gemini.rs b/lib/crates/fabro-llm/src/providers/gemini.rs index a9e2fab0f..e2a33337b 100644 --- a/lib/crates/fabro-llm/src/providers/gemini.rs +++ b/lib/crates/fabro-llm/src/providers/gemini.rs @@ -9,7 +9,7 @@ use crate::error::Error; use crate::provider::{ ProviderAdapter, StreamEventStream, validate_standard_speed, validate_tool_choice, }; -use crate::providers::common::{self as common}; +use crate::providers::common::{self as common, CatalogRoute}; use crate::token_count::{InputTokenCount, InputTokenCountMethod}; use crate::transport::{self, HttpTransport, SseFraming}; use crate::types::{AdapterTimeout, Request, Response}; @@ -102,11 +102,7 @@ impl Adapter { request, provider_name: &self.provider_name, deployment_id, - model: common::catalog_model( - self.catalog.as_deref(), - &self.provider_name, - &request.model, - ), + model: self.catalog_model(&request.model), params, } } @@ -129,6 +125,16 @@ impl Adapter { } } +impl common::CatalogRoute for Adapter { + fn catalog(&self) -> Option<&Catalog> { + self.catalog.as_deref() + } + + fn provider_name(&self) -> &str { + &self.provider_name + } +} + #[async_trait::async_trait] impl ProviderAdapter for Adapter { fn name(&self) -> &str { @@ -151,11 +157,7 @@ impl ProviderAdapter for Adapter { let resolved = self.resolve_request(request).await; let codec = GeminiGenerate; - let deployment_id = common::api_model_id( - self.catalog.as_deref(), - &self.provider_name, - &resolved.model, - ); + let deployment_id = self.api_model_id(&resolved.model); let params = CodecParams::default(); let ctx = self.codec_ctx(&resolved, &deployment_id, ¶ms); @@ -185,11 +187,7 @@ impl ProviderAdapter for Adapter { let resolved = self.resolve_request(request).await; let codec = GeminiGenerate; - let deployment_id = common::api_model_id( - self.catalog.as_deref(), - &self.provider_name, - &resolved.model, - ); + let deployment_id = self.api_model_id(&resolved.model); let params = CodecParams::default(); let ctx = self.codec_ctx(&resolved, &deployment_id, ¶ms); @@ -206,11 +204,7 @@ impl ProviderAdapter for Adapter { let resolved = self.resolve_request(request).await; let codec = GeminiGenerate; - let deployment_id = common::api_model_id( - self.catalog.as_deref(), - &self.provider_name, - &resolved.model, - ); + let deployment_id = self.api_model_id(&resolved.model); let params = CodecParams::default(); let ctx = self.codec_ctx(&resolved, &deployment_id, ¶ms); diff --git a/lib/crates/fabro-llm/src/providers/openai.rs b/lib/crates/fabro-llm/src/providers/openai.rs index 185af1714..56eb76c3d 100644 --- a/lib/crates/fabro-llm/src/providers/openai.rs +++ b/lib/crates/fabro-llm/src/providers/openai.rs @@ -9,7 +9,7 @@ use crate::error::Error; use crate::provider::{ ProviderAdapter, StreamEventStream, validate_standard_speed, validate_tool_choice, }; -use crate::providers::common::{self as common}; +use crate::providers::common::{self as common, CatalogRoute}; use crate::token_count::{InputTokenCount, InputTokenCountMethod}; use crate::transport::{self, HttpTransport, SseFraming}; use crate::types::{AdapterTimeout, Request, Response, StreamEvent}; @@ -131,11 +131,7 @@ impl Adapter { request, provider_name: &self.provider_name, deployment_id, - model: common::catalog_model( - self.catalog.as_deref(), - &self.provider_name, - &request.model, - ), + model: self.catalog_model(&request.model), params, } } @@ -196,6 +192,16 @@ impl Adapter { } } +impl common::CatalogRoute for Adapter { + fn catalog(&self) -> Option<&Catalog> { + self.catalog.as_deref() + } + + fn provider_name(&self) -> &str { + &self.provider_name + } +} + #[async_trait::async_trait] impl ProviderAdapter for Adapter { fn name(&self) -> &str { @@ -218,11 +224,7 @@ impl ProviderAdapter for Adapter { let resolved = self.resolve_request(request).await; let codec = OpenAiResponses; - let deployment_id = common::api_model_id( - self.catalog.as_deref(), - &self.provider_name, - &resolved.model, - ); + let deployment_id = self.api_model_id(&resolved.model); let params = self.codec_params(); let ctx = self.codec_ctx(&resolved, &deployment_id, ¶ms); @@ -258,11 +260,7 @@ impl ProviderAdapter for Adapter { let resolved = self.resolve_request(request).await; let codec = OpenAiResponses; - let deployment_id = common::api_model_id( - self.catalog.as_deref(), - &self.provider_name, - &resolved.model, - ); + let deployment_id = self.api_model_id(&resolved.model); let params = self.codec_params(); let ctx = self.codec_ctx(&resolved, &deployment_id, ¶ms); @@ -279,11 +277,7 @@ impl ProviderAdapter for Adapter { let resolved = self.resolve_request(request).await; let codec = OpenAiResponses; - let deployment_id = common::api_model_id( - self.catalog.as_deref(), - &self.provider_name, - &resolved.model, - ); + let deployment_id = self.api_model_id(&resolved.model); let params = self.codec_params(); let ctx = self.codec_ctx(&resolved, &deployment_id, ¶ms); diff --git a/lib/crates/fabro-llm/src/providers/openai_compatible.rs b/lib/crates/fabro-llm/src/providers/openai_compatible.rs index 3feb0b577..b46444636 100644 --- a/lib/crates/fabro-llm/src/providers/openai_compatible.rs +++ b/lib/crates/fabro-llm/src/providers/openai_compatible.rs @@ -8,7 +8,7 @@ use crate::error::Error; use crate::provider::{ ProviderAdapter, StreamEventStream, validate_standard_speed, validate_tool_choice, }; -use crate::providers::common::{self as common}; +use crate::providers::common::{self as common, CatalogRoute}; use crate::transport::{self, HttpTransport, SseFraming}; use crate::types::{AdapterTimeout, Request, Response}; @@ -88,7 +88,7 @@ impl Adapter { /// Resolve the wire model id (catalog `api_id`, falling back to the /// requested model). fn deployment_id(&self, request: &Request) -> String { - common::api_model_id(self.catalog.as_deref(), &self.provider_name, &request.model) + self.api_model_id(&request.model) } /// Build the borrowed codec context. `deployment_id` and `params` are @@ -103,11 +103,7 @@ impl Adapter { request, provider_name: &self.provider_name, deployment_id, - model: common::catalog_model( - self.catalog.as_deref(), - &self.provider_name, - &request.model, - ), + model: self.catalog_model(&request.model), params, } } @@ -131,6 +127,16 @@ impl Adapter { } } +impl common::CatalogRoute for Adapter { + fn catalog(&self) -> Option<&Catalog> { + self.catalog.as_deref() + } + + fn provider_name(&self) -> &str { + &self.provider_name + } +} + #[async_trait::async_trait] impl ProviderAdapter for Adapter { fn name(&self) -> &str { diff --git a/lib/crates/fabro-model/src/catalog.rs b/lib/crates/fabro-model/src/catalog.rs index ee362d414..ea089cfba 100644 --- a/lib/crates/fabro-model/src/catalog.rs +++ b/lib/crates/fabro-model/src/catalog.rs @@ -544,33 +544,8 @@ pub enum CatalogBuildError { first: ModelId, second: ModelId, }, - #[error( - "provider '{provider}' model '{model}' is defined through both provider-scoped and legacy top-level syntax" - )] - DuplicateProviderModelDefinition { - provider: ProviderId, - model: ModelId, - }, - #[error("provider-scoped model '{provider}/{model}' must not declare a provider field")] - ScopedModelDeclaresProvider { - provider: ProviderId, - model: ModelId, - }, - #[error("legacy model row '{model}' omits provider and does not match a unique known offering")] - LegacyModelProviderUnknown { model: String }, - #[error( - "legacy model row '{model}' omits provider and matches multiple providers: {providers:?}" - )] - LegacyModelProviderAmbiguous { - model: String, - providers: Vec, - }, - #[error("model identifier '{identifier}' has been retired; use '{provider}/{model}' instead")] - RetiredModelIdentifier { - identifier: String, - provider: ProviderId, - model: ModelId, - }, + #[error(transparent)] + LegacyModel(#[from] LegacyModelError), #[error("provider '{provider}' model '{model}' has an empty api_id")] EmptyModelApiId { provider: ProviderId, @@ -637,6 +612,10 @@ pub enum ModelSelectionError { selector: String, providers: Vec, }, + #[error( + "no default model is available on an eligible provider; providers with defaults: {providers:?}" + )] + NoDefaultModel { providers: Vec }, #[error("model identifier '{identifier}' has been retired; use '{provider}/{model}' instead")] RetiredModelIdentifier { identifier: String, @@ -645,6 +624,15 @@ pub enum ModelSelectionError { }, } +/// One provider/model pair chosen by [`Catalog::resolve_selection`]. The +/// model is the canonical catalog ID when the selector matched an offering, +/// or the caller's selector passed through verbatim when it did not. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SelectedModel { + pub provider: ProviderId, + pub model: String, +} + /// Typed model catalog backed by a `Vec`. /// /// Use [`Catalog::builtin()`] for the embedded settings-backed catalog. @@ -724,13 +712,14 @@ impl Catalog { } if let Some((replacement_provider, replacement_model)) = - retired_model_address(&model_id) + retired_model_replacement(&model_id) { - return Err(CatalogBuildError::RetiredModelIdentifier { + return Err(LegacyModelError::Retired { identifier: model_id, provider: replacement_provider, model: replacement_model, - }); + } + .into()); } let (model, resolved_settings) = build_model(&model_id, model_settings, provider)?; @@ -905,7 +894,8 @@ impl Catalog { provider: &ProviderId, selector: &str, ) -> Result<&Model, ModelSelectionError> { - if let Some((replacement_provider, replacement_model)) = retired_model_address(selector) { + if let Some((replacement_provider, replacement_model)) = retired_model_replacement(selector) + { return Err(ModelSelectionError::RetiredModelIdentifier { identifier: selector.to_string(), provider: replacement_provider, @@ -937,7 +927,8 @@ impl Catalog { explicit_provider: Option<&ProviderId>, eligible_providers: &HashSet, ) -> Result<&'a Model, ModelSelectionError> { - if let Some((replacement_provider, replacement_model)) = retired_model_address(selector) { + if let Some((replacement_provider, replacement_model)) = retired_model_replacement(selector) + { return Err(ModelSelectionError::RetiredModelIdentifier { identifier: selector.to_string(), provider: replacement_provider, @@ -1042,10 +1033,82 @@ impl Catalog { .collect::>(); providers.sort(); providers.dedup(); - Err(ModelSelectionError::NoEligibleOffering { - selector: "".to_string(), - providers, - }) + Err(ModelSelectionError::NoDefaultModel { providers }) + } + + /// Canonicalize a provider ID or alias and require it to be in the + /// eligible snapshot. + pub fn ready_provider( + &self, + provider: &ProviderId, + eligible_providers: &HashSet, + ) -> Result { + let provider = + self.provider(provider) + .ok_or_else(|| ModelSelectionError::UnknownProvider { + provider: provider.clone(), + })?; + let ready = eligible_providers.iter().any(|eligible| { + self.provider(eligible) + .is_some_and(|eligible| eligible.id == provider.id) + }); + if !ready { + return Err(ModelSelectionError::ProviderUnavailable { + provider: provider.id.clone(), + }); + } + Ok(provider.id.clone()) + } + + /// Resolve an optional selector to one provider/model pair, applying the + /// passthrough policy shared by every dispatch boundary: + /// + /// - A selector known to the catalog resolves to its canonical offering. + /// - An unknown selector pinned to a provider passes through verbatim on + /// that provider. + /// - An unqualified unknown selector passes through on the default + /// provider. + /// - No selector picks the default offering (of the pinned provider, when + /// one is given). + pub fn resolve_selection( + &self, + selector: Option<&str>, + explicit_provider: Option<&ProviderId>, + eligible_providers: &HashSet, + ) -> Result { + let Some(selector) = selector else { + let eligible = match explicit_provider { + Some(provider) => { + HashSet::from([self.ready_provider(provider, eligible_providers)?]) + } + None => eligible_providers.clone(), + }; + let offering = self.select_default(&eligible)?; + return Ok(SelectedModel { + provider: offering.provider.clone(), + model: offering.id.to_string(), + }); + }; + match self.select(selector, explicit_provider, eligible_providers) { + Ok(offering) => Ok(SelectedModel { + provider: offering.provider.clone(), + model: offering.id.to_string(), + }), + Err(ModelSelectionError::UnknownSelectorOnProvider { provider, .. }) => { + Ok(SelectedModel { + provider, + model: selector.to_string(), + }) + } + Err(ModelSelectionError::UnknownSelector { .. }) => { + let default = self.select_default(eligible_providers)?; + Ok(SelectedModel { + provider: default.provider.clone(), + model: selector.to_string(), + }) + } + Err(error) => Err(error), + } } #[must_use] @@ -1411,52 +1474,24 @@ fn normalize_catalog_settings( mut settings: LlmCatalogSettings, known: Option<&LlmCatalogSettings>, ) -> Result { - for (provider, settings) in &settings.providers { - for (model, settings) in &settings.models { - if settings.provider.is_some() { - return Err(CatalogBuildError::ScopedModelDeclaresProvider { - provider: ProviderId::new(provider.clone()), - model: ModelId::new(model.clone()), - }); - } - } - } + reject_scoped_provider_fields(&settings)?; let legacy_models = std::mem::take(&mut settings.models); - for (legacy_id, mut model_settings) in legacy_models { - if let Some((provider, model)) = retired_model_address(&legacy_id) { - return Err(CatalogBuildError::RetiredModelIdentifier { - identifier: legacy_id, - provider, - model, - }); - } + if legacy_models.is_empty() { + return Ok(settings); + } + let mut legacy_models = legacy_models.into_iter().collect::>(); + legacy_models.sort_by(|(left, _), (right, _)| left.cmp(right)); + let mut index = LegacyModelIndex::default(); + index.add_settings(&settings); + if let Some(known) = known { + index.add_settings(known); + } + + for (legacy_id, mut model_settings) in legacy_models { let explicit_provider = model_settings.provider.take(); - let (provider, model_id) = if let Some(provider) = explicit_provider { - let provider = canonical_settings_provider(&provider, &settings, known) - .unwrap_or_else(|| ProviderId::new(provider)); - let model_id = canonical_settings_model(&provider, &legacy_id, &settings, known) - .unwrap_or_else(|| ModelId::new(legacy_id.clone())); - (provider, model_id) - } else { - let candidates = settings_model_candidates(&legacy_id, &settings, known); - match candidates.as_slice() { - [(provider, model)] => (provider.clone(), model.clone()), - [] => { - return Err(CatalogBuildError::LegacyModelProviderUnknown { model: legacy_id }); - } - _ => { - return Err(CatalogBuildError::LegacyModelProviderAmbiguous { - model: legacy_id, - providers: candidates - .into_iter() - .map(|(provider, _)| provider) - .collect(), - }); - } - } - }; + let (provider, model_id) = index.resolve(&legacy_id, explicit_provider.as_deref())?; if !settings.providers.contains_key(provider.as_str()) && !known.is_some_and(|known| known.providers.contains_key(provider.as_str())) @@ -1469,10 +1504,11 @@ fn normalize_catalog_settings( let provider_settings = settings.providers.entry(provider.to_string()).or_default(); if provider_settings.models.contains_key(model_id.as_str()) { - return Err(CatalogBuildError::DuplicateProviderModelDefinition { + return Err(LegacyModelError::DuplicateModel { provider, model: model_id, - }); + } + .into()); } provider_settings .models @@ -1481,154 +1517,242 @@ fn normalize_catalog_settings( Ok(settings) } -fn canonical_settings_provider( - selector: &str, - settings: &LlmCatalogSettings, - known: Option<&LlmCatalogSettings>, -) -> Option { - let providers = || { - settings - .providers - .iter() - .chain(known.into_iter().flat_map(|known| known.providers.iter())) - }; - providers() - .find(|(id, _)| id.as_str() == selector) - .or_else(|| { - providers().find(|(_, provider)| { - provider - .aliases - .as_ref() - .is_some_and(|aliases| aliases.iter().any(|alias| alias == selector)) - }) - }) - .map(|(id, _)| ProviderId::new(id.clone())) -} - -fn canonical_settings_model( - provider: &ProviderId, - selector: &str, - settings: &LlmCatalogSettings, - known: Option<&LlmCatalogSettings>, -) -> Option { - let models = || { - settings - .providers - .get(provider.as_str()) - .into_iter() - .chain(known.and_then(|known| known.providers.get(provider.as_str()))) - .flat_map(|provider| provider.models.iter()) - }; - models() - .find(|(id, _)| id.as_str() == selector) - .or_else(|| { - models().find(|(_, model)| { - model - .aliases - .as_ref() - .is_some_and(|aliases| aliases.iter().any(|alias| alias == selector)) - }) - }) - .map(|(id, _)| ModelId::new(id.clone())) -} - -fn settings_model_candidates( - selector: &str, - settings: &LlmCatalogSettings, - known: Option<&LlmCatalogSettings>, -) -> Vec<(ProviderId, ModelId)> { - let collect = |matches: &dyn Fn(&str, &ModelCatalogSettings) -> bool| { - let mut candidates = BTreeSet::<(ProviderId, ModelId)>::new(); - for (provider_id, provider) in settings - .providers - .iter() - .chain(known.into_iter().flat_map(|known| known.providers.iter())) - { - for (model_id, model) in &provider.models { - if matches(model_id, model) { - candidates.insert(( - ProviderId::new(provider_id.clone()), - ModelId::new(model_id.clone()), - )); - } +fn reject_scoped_provider_fields(settings: &LlmCatalogSettings) -> Result<(), LegacyModelError> { + for (provider, settings) in &settings.providers { + for (model, settings) in &settings.models { + if settings.provider.is_some() { + return Err(LegacyModelError::ScopedModelDeclaresProvider { + provider: ProviderId::new(provider.clone()), + model: ModelId::new(model.clone()), + }); } } - candidates.into_iter().collect::>() - }; - - let canonical = collect(&|model_id, _| model_id == selector); - if canonical.is_empty() { - collect(&|_, model| { - model - .aliases - .as_ref() - .is_some_and(|aliases| aliases.iter().any(|alias| alias == selector)) - }) - } else { - canonical } + Ok(()) } -/// Return every built-in offering whose canonical model ID is `selector`. -/// -/// This includes disabled providers because config compatibility -/// normalization happens before runtime availability is known. -pub fn builtin_canonical_model_offerings( - selector: &str, -) -> Result, CatalogBuildError> { - let settings = Catalog::builtin_settings()?; - Ok(settings - .providers - .iter() - .filter_map(|(provider, settings)| { - settings.models.get_key_value(selector).map(|(model, _)| { - ( - ProviderId::new(provider.clone()), - ModelId::new(model.clone()), - ) - }) - }) - .collect()) +/// Failure to resolve a legacy top-level `[models.]` row onto its +/// provider. +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +pub enum LegacyModelError { + #[error("failed to inspect the built-in model catalog: {message}")] + BuiltinCatalog { message: String }, + #[error("model identifier '{identifier}' has been retired; use '{provider}/{model}' instead")] + Retired { + identifier: String, + provider: ProviderId, + model: ModelId, + }, + #[error("legacy model row '{model}' omits provider and does not match a unique known offering")] + UnknownModel { model: String }, + #[error( + "legacy model row '{model}' omits provider and matches multiple offerings: {candidates:?}" + )] + AmbiguousModel { + model: String, + candidates: Vec<(ProviderId, ModelId)>, + }, + #[error("legacy model selector '{selector}' is ambiguous on provider '{provider}': {models:?}")] + AmbiguousAlias { + provider: ProviderId, + selector: String, + models: Vec, + }, + #[error("provider-scoped model '{provider}/{model}' must not declare a provider field")] + ScopedModelDeclaresProvider { + provider: ProviderId, + model: ModelId, + }, + #[error( + "provider '{provider}' model '{model}' is defined through both provider-scoped and legacy top-level syntax" + )] + DuplicateModel { + provider: ProviderId, + model: ModelId, + }, } -/// Return every built-in offering that declares `selector` as an alias. +/// Identifier/alias view used to resolve legacy top-level `[models.]` +/// rows onto their provider before provider-scoped settings merge. /// -/// This includes disabled providers because config compatibility -/// normalization happens before runtime availability is known. -pub fn builtin_alias_model_offerings( - selector: &str, -) -> Result, CatalogBuildError> { - let settings = Catalog::builtin_settings()?; - let mut offerings = settings - .providers - .iter() - .flat_map(|(provider, settings)| { - settings - .models - .iter() - .filter(move |(_, settings)| { - settings - .aliases - .as_ref() - .is_some_and(|aliases| aliases.iter().any(|alias| alias == selector)) - }) - .map(move |(model, _)| { +/// Both the settings-layer normalization in `fabro-config` and catalog-build +/// normalization here feed this index: local entries first, lower-precedence +/// known entries (e.g. the built-in catalog) after. Canonical IDs always win +/// over aliases; alias ties resolve to the first entry added. +#[derive(Debug, Default)] +pub struct LegacyModelIndex { + providers: Vec, +} + +#[derive(Debug)] +struct LegacyProviderEntry { + id: ProviderId, + aliases: Vec, + models: Vec, +} + +#[derive(Debug)] +struct LegacyModelEntry { + id: ModelId, + aliases: Vec, +} + +impl LegacyModelIndex { + pub fn add_provider( + &mut self, + id: ProviderId, + aliases: Vec, + models: impl IntoIterator)>, + ) { + self.providers.push(LegacyProviderEntry { + id, + aliases, + models: models + .into_iter() + .map(|(id, aliases)| LegacyModelEntry { id, aliases }) + .collect(), + }); + } + + fn add_settings(&mut self, settings: &LlmCatalogSettings) { + let mut provider_ids = settings.providers.keys().collect::>(); + provider_ids.sort_unstable(); + for provider_id in provider_ids { + let provider = &settings.providers[provider_id]; + let mut model_ids = provider.models.keys().collect::>(); + model_ids.sort_unstable(); + self.add_provider( + ProviderId::new(provider_id.clone()), + provider.aliases.clone().unwrap_or_default(), + model_ids.into_iter().map(|model_id| { + let model = &provider.models[model_id]; ( - ProviderId::new(provider.clone()), - ModelId::new(model.clone()), + ModelId::new(model_id.clone()), + model.aliases.clone().unwrap_or_default(), ) - }) - }) - .collect::>(); - offerings.sort(); - offerings.dedup(); - Ok(offerings) -} + }), + ); + } + } -/// Resolve a built-in provider ID or alias without filtering disabled rows. -pub fn builtin_provider_id(selector: &str) -> Result, CatalogBuildError> { - let settings = Catalog::builtin_settings()?; - Ok(canonical_settings_provider(selector, &settings, None)) + /// Append the built-in catalog as the lowest-precedence tier. Includes + /// disabled providers because config compatibility normalization happens + /// before runtime availability is known. + pub fn with_builtin(mut self) -> Result { + let builtin = + Catalog::builtin_settings().map_err(|error| LegacyModelError::BuiltinCatalog { + message: error.to_string(), + })?; + self.add_settings(&builtin); + Ok(self) + } + + /// Resolve one legacy row to its provider-scoped address. An unknown + /// explicit provider or model selector passes through verbatim; rows + /// without an explicit provider must match exactly one known offering. + pub fn resolve( + &self, + legacy_id: &str, + explicit_provider: Option<&str>, + ) -> Result<(ProviderId, ModelId), LegacyModelError> { + if let Some((provider, model)) = retired_model_replacement(legacy_id) { + return Err(LegacyModelError::Retired { + identifier: legacy_id.to_string(), + provider, + model, + }); + } + if let Some(explicit) = explicit_provider { + let provider = self + .canonical_provider(explicit) + .unwrap_or_else(|| ProviderId::new(explicit)); + let model = self + .canonical_model_on(&provider, legacy_id)? + .unwrap_or_else(|| ModelId::new(legacy_id)); + return Ok((provider, model)); + } + let candidates = self.candidates(legacy_id); + match candidates.as_slice() { + [(provider, model)] => Ok((provider.clone(), model.clone())), + [] => Err(LegacyModelError::UnknownModel { + model: legacy_id.to_string(), + }), + _ => Err(LegacyModelError::AmbiguousModel { + model: legacy_id.to_string(), + candidates, + }), + } + } + + fn canonical_provider(&self, selector: &str) -> Option { + self.providers + .iter() + .find(|provider| provider.id.as_str() == selector) + .or_else(|| { + self.providers + .iter() + .find(|provider| provider.aliases.iter().any(|alias| alias == selector)) + }) + .map(|provider| provider.id.clone()) + } + + fn canonical_model_on( + &self, + provider: &ProviderId, + selector: &str, + ) -> Result, LegacyModelError> { + let models = || { + self.providers + .iter() + .filter(|entry| entry.id == *provider) + .flat_map(|entry| entry.models.iter()) + }; + if models().any(|model| model.id.as_str() == selector) { + return Ok(Some(ModelId::new(selector))); + } + let matches = models() + .filter(|model| model.aliases.iter().any(|alias| alias == selector)) + .map(|model| model.id.clone()) + .collect::>(); + match matches.len() { + 0 => Ok(None), + 1 => Ok(matches.into_iter().next()), + _ => Err(LegacyModelError::AmbiguousAlias { + provider: provider.clone(), + selector: selector.to_string(), + models: matches.into_iter().collect(), + }), + } + } + + fn candidates(&self, selector: &str) -> Vec<(ProviderId, ModelId)> { + let canonical = self + .providers + .iter() + .filter(|entry| { + entry + .models + .iter() + .any(|model| model.id.as_str() == selector) + }) + .map(|entry| (entry.id.clone(), ModelId::new(selector))) + .collect::>(); + if !canonical.is_empty() { + return canonical.into_iter().collect(); + } + self.providers + .iter() + .flat_map(|entry| { + entry + .models + .iter() + .filter(|model| model.aliases.iter().any(|alias| alias == selector)) + .map(|model| (entry.id.clone(), model.id.clone())) + }) + .collect::>() + .into_iter() + .collect() + } } /// Built-in catalog keys retired when provider API identifiers stopped being @@ -1718,10 +1842,6 @@ pub fn retired_model_replacement(identifier: &str) -> Option<(ProviderId, ModelI .map(|(_, provider, model)| (ProviderId::new(*provider), ModelId::new(*model))) } -fn retired_model_address(identifier: &str) -> Option<(ProviderId, ModelId)> { - retired_model_replacement(identifier) -} - fn merge_catalog_settings( higher: LlmCatalogSettings, mut fallback: LlmCatalogSettings, @@ -3676,8 +3796,10 @@ provider = "test" assert!(matches!( error, - CatalogBuildError::ScopedModelDeclaresProvider { provider, model } - if provider == ProviderId::new("test") && model == "one" + CatalogBuildError::LegacyModel(LegacyModelError::ScopedModelDeclaresProvider { + provider, + model, + }) if provider == ProviderId::new("test") && model == "one" )); } diff --git a/lib/crates/fabro-model/src/ids.rs b/lib/crates/fabro-model/src/ids.rs index 3429ae094..02d6db0d7 100644 --- a/lib/crates/fabro-model/src/ids.rs +++ b/lib/crates/fabro-model/src/ids.rs @@ -150,14 +150,6 @@ impl From for ModelId { } } -impl FromStr for ModelId { - type Err = std::convert::Infallible; - - fn from_str(s: &str) -> Result { - Ok(Self::from(s)) - } -} - impl AsRef for ModelId { fn as_ref(&self) -> &str { &self.0 diff --git a/lib/crates/fabro-model/src/lib.rs b/lib/crates/fabro-model/src/lib.rs index 337c00fdd..5f3511096 100644 --- a/lib/crates/fabro-model/src/lib.rs +++ b/lib/crates/fabro-model/src/lib.rs @@ -19,7 +19,7 @@ pub use billing::{ }; pub use catalog::{ ApiKeyHeaderPolicy, BillingPolicy, Catalog, CredentialRef, CredentialRefParseError, - FallbackTarget, ModelSelectionError, ProviderAuthConfig, + FallbackTarget, ModelSelectionError, ProviderAuthConfig, SelectedModel, }; pub use codec::CodecKind; pub use ids::{ModelId, ProviderId}; diff --git a/lib/crates/fabro-server/src/server/handler/completions.rs b/lib/crates/fabro-server/src/server/handler/completions.rs index 3d36f9036..b53e56d31 100644 --- a/lib/crates/fabro-server/src/server/handler/completions.rs +++ b/lib/crates/fabro-server/src/server/handler/completions.rs @@ -53,7 +53,7 @@ async fn create_completion( let (model_id, selected_provider) = match resolve_request_model( catalog.as_ref(), &client.provider_ids(), - req.model, + req.model.as_deref(), req.provider, ) { Ok(selection) => selection, @@ -114,20 +114,8 @@ async fn create_completion( metadata: None, provider_options: req.provider_options, }; - let request = match client.resolve_request(&request) { - Ok(request) => request, - Err(error) => return ApiError::bad_request(error.to_string()).into_response(), - }; - let selected_provider = ProviderId::new( - request - .provider - .as_deref() - .expect("resolved LLM request always has a provider"), - ); - let selected_model = request.model.clone(); - info!( - model = %selected_model, + model = %model_id, provider = %selected_provider, "Completion request received" ); @@ -177,7 +165,7 @@ async fn create_completion( let stop_reason = finish_reason_to_api_stop_reason(&response.finish_reason); Json(CompletionResponse { id: msg_id, - model: selected_model, + model: model_id, provider: selected_provider, message: response.message, stop_reason, @@ -224,44 +212,11 @@ async fn create_completion( pub(super) fn resolve_request_model( catalog: &Catalog, eligible: &HashSet, - requested_model: Option, + requested_model: Option<&str>, explicit_provider: Option, ) -> Result<(String, ProviderId), ModelSelectionError> { let explicit_provider = explicit_provider.map(ProviderId::new); - if let Some(model) = requested_model { - return match catalog.select(&model, explicit_provider.as_ref(), eligible) { - Ok(offering) => Ok((offering.id.to_string(), offering.provider.clone())), - Err(ModelSelectionError::UnknownSelectorOnProvider { provider, .. }) => { - Ok((model, provider)) - } - Err(ModelSelectionError::UnknownSelector { .. }) => { - let default = catalog.select_default(eligible)?; - Ok((model, default.provider.clone())) - } - Err(error) => Err(error), - }; - } - - let eligible = if let Some(requested_provider) = explicit_provider { - let provider = catalog.provider(&requested_provider).ok_or_else(|| { - ModelSelectionError::UnknownProvider { - provider: requested_provider.clone(), - } - })?; - let provider_is_ready = eligible.iter().any(|eligible_provider| { - catalog - .provider(eligible_provider) - .is_some_and(|eligible_provider| eligible_provider.id == provider.id) - }); - if !provider_is_ready { - return Err(ModelSelectionError::ProviderUnavailable { - provider: provider.id.clone(), - }); - } - HashSet::from([provider.id.clone()]) - } else { - eligible.clone() - }; - let default = catalog.select_default(&eligible)?; - Ok((default.id.to_string(), default.provider.clone())) + let selected = + catalog.resolve_selection(requested_model, explicit_provider.as_ref(), eligible)?; + Ok((selected.model, selected.provider)) } diff --git a/lib/crates/fabro-server/src/server/handler/models.rs b/lib/crates/fabro-server/src/server/handler/models.rs index 9b639af99..59b6b8623 100644 --- a/lib/crates/fabro-server/src/server/handler/models.rs +++ b/lib/crates/fabro-server/src/server/handler/models.rs @@ -272,7 +272,8 @@ fn model_selection_response(error: &ModelSelectionError) -> Response { } ModelSelectionError::RetiredModelIdentifier { .. } | ModelSelectionError::ProviderUnavailable { .. } - | ModelSelectionError::NoEligibleOffering { .. } => { + | ModelSelectionError::NoEligibleOffering { .. } + | ModelSelectionError::NoDefaultModel { .. } => { ApiError::bad_request(error.to_string()).into_response() } } diff --git a/lib/crates/fabro-server/src/server/handler/playground.rs b/lib/crates/fabro-server/src/server/handler/playground.rs index 046cf2ef3..d404c2b58 100644 --- a/lib/crates/fabro-server/src/server/handler/playground.rs +++ b/lib/crates/fabro-server/src/server/handler/playground.rs @@ -130,7 +130,7 @@ async fn create_playground_chat( let (model_id, selected_provider) = match super::completions::resolve_request_model( catalog.as_ref(), &client.provider_ids(), - req.model, + req.model.as_deref(), req.provider.map(|provider| provider.to_string()), ) { Ok(selection) => selection, @@ -146,7 +146,7 @@ async fn create_playground_chat( let request = LlmRequest { model: model_id, messages, - provider: Some(selected_provider.to_string()), + provider: Some(selected_provider.into_inner()), tools: Some(playground_tools()), tool_choice: Some(ToolChoice::Auto), response_format: None, @@ -159,10 +159,6 @@ async fn create_playground_chat( metadata: None, provider_options: None, }; - let request = match client.resolve_request(&request) { - Ok(request) => request, - Err(error) => return ApiError::bad_request(error.to_string()).into_response(), - }; info!( model = %request.model, provider = request.provider.as_deref().unwrap_or(""), diff --git a/lib/crates/fabro-server/src/server/handler/sessions.rs b/lib/crates/fabro-server/src/server/handler/sessions.rs index eb72bd386..936b8e582 100644 --- a/lib/crates/fabro-server/src/server/handler/sessions.rs +++ b/lib/crates/fabro-server/src/server/handler/sessions.rs @@ -805,37 +805,22 @@ fn selected_session_model( .provider_ids() .into_iter() .collect::>(); - let (provider_id, model) = if let Some(model_id) = session.record.model.as_deref() { - match catalog.select(model_id, session.record.provider.as_ref(), &eligible) { - Ok(selected) => (selected.provider.clone(), selected.id.to_string()), - Err(ModelSelectionError::UnknownSelectorOnProvider { provider, .. }) => { - (provider, model_id.to_string()) - } - Err(ModelSelectionError::UnknownSelector { .. }) - if session.record.provider.is_none() => + let record = &session.record; + let selected = catalog + .resolve_selection(record.model.as_deref(), record.provider.as_ref(), &eligible) + .map_err(|error| { + // A missing default with no provider pin means no LLM is + // configured at all; every other failure is about the requested + // model/provider. + if record.provider.is_none() + && matches!(error, ModelSelectionError::NoDefaultModel { .. }) { - let selected = catalog - .select_default(&eligible) - .map_err(|error| AskFabroBuildError::LlmUnconfigured(error.to_string()))?; - (selected.provider.clone(), model_id.to_string()) + AskFabroBuildError::LlmUnconfigured(error.to_string()) + } else { + AskFabroBuildError::ModelUnavailable(error.to_string()) } - Err(error) => { - return Err(AskFabroBuildError::ModelUnavailable(error.to_string())); - } - } - } else { - let selected = if let Some(provider) = session.record.provider.as_ref() { - let provider_only = std::collections::HashSet::from([provider.clone()]); - catalog - .select_default(&provider_only) - .map_err(|error| AskFabroBuildError::ModelUnavailable(error.to_string()))? - } else { - catalog - .select_default(&eligible) - .map_err(|error| AskFabroBuildError::LlmUnconfigured(error.to_string()))? - }; - (selected.provider.clone(), selected.id.to_string()) - }; + })?; + let (provider_id, model) = (selected.provider, selected.model); let profile_kind = catalog .effective_agent_profile(&provider_id, Some(&model)) .ok_or_else(|| { @@ -865,27 +850,10 @@ fn canonical_session_model( }) .transpose()?; let Some(requested) = requested else { - let eligible = if let Some(provider) = explicit_provider.as_ref() { - let provider_is_ready = eligible.iter().any(|eligible_provider| { - catalog - .provider(eligible_provider) - .is_some_and(|eligible_provider| eligible_provider.id == *provider) - }); - if !provider_is_ready { - return Err(session_selection_error( - &ModelSelectionError::ProviderUnavailable { - provider: provider.clone(), - }, - )); - } - std::collections::HashSet::from([provider.clone()]) - } else { - eligible.clone() - }; - let model = catalog - .select_default(&eligible) + let selected = catalog + .resolve_selection(None, explicit_provider.as_ref(), eligible) .map_err(|error| session_selection_error(&error))?; - return Ok((model.provider.clone(), model.id.to_string())); + return Ok((selected.provider, selected.model)); }; let requested = requested.trim(); if requested.is_empty() { @@ -942,19 +910,10 @@ fn canonical_session_model( } }; let provider = qualified_provider.as_ref().or(explicit_provider.as_ref()); - match catalog.select(&model, provider, eligible) { - Ok(selected) => Ok((selected.provider.clone(), selected.id.to_string())), - Err(ModelSelectionError::UnknownSelectorOnProvider { provider, .. }) => { - Ok((provider, model)) - } - Err(ModelSelectionError::UnknownSelector { .. }) if provider.is_none() => { - let selected = catalog - .select_default(eligible) - .map_err(|error| session_selection_error(&error))?; - Ok((selected.provider.clone(), model)) - } - Err(error) => Err(session_selection_error(&error)), - } + let selected = catalog + .resolve_selection(Some(&model), provider, eligible) + .map_err(|error| session_selection_error(&error))?; + Ok((selected.provider, selected.model)) } fn session_selection_error(error: &ModelSelectionError) -> ApiError { diff --git a/lib/crates/fabro-server/src/server/tests.rs b/lib/crates/fabro-server/src/server/tests.rs index ca29053a9..74095bbf5 100644 --- a/lib/crates/fabro-server/src/server/tests.rs +++ b/lib/crates/fabro-server/src/server/tests.rs @@ -3414,7 +3414,7 @@ async fn create_run_without_ready_llm_provider_rejects_implicit_model_selection( assert!( body["errors"][0]["detail"] .as_str() - .is_some_and(|detail| detail.contains("no offering on an eligible provider")), + .is_some_and(|detail| detail.contains("no default model is available")), "unexpected response: {body}" ); assert!(state.runs.lock().expect("runs lock poisoned").is_empty()); diff --git a/lib/crates/fabro-workflow/src/operations/start.rs b/lib/crates/fabro-workflow/src/operations/start.rs index 5972d6cc4..b1e33f4c7 100644 --- a/lib/crates/fabro-workflow/src/operations/start.rs +++ b/lib/crates/fabro-workflow/src/operations/start.rs @@ -35,7 +35,6 @@ use crate::event::{ Emitter, Event, EventBody, RunEventLogger, RunEventSink, RunNoticeLevel, append_event_to_sink, }; use crate::handler::HandlerRegistry; -use crate::handler::llm::routing; use crate::outcome::{Outcome, StageOutcome}; use crate::pipeline::{ self, FinalizeOptions, Finalized, InitOptions, LlmSpec, Persisted, PullRequestOptions, @@ -618,22 +617,12 @@ fn resolve_start_llm( settings: &ResolvedRunSettings, ) -> Result { let eligible = configured.iter().cloned().collect::>(); - let (model, provider) = resolve_run_model( + let (model, provider_id) = resolve_run_model( catalog, &eligible, settings.model.name.as_deref(), settings.model.provider.as_deref(), )?; - let provider_id = ProviderId::new( - provider.expect("catalog-backed run model resolution always selects a provider"), - ); - let provider_context = routing::resolve_provider_context( - catalog, - &provider_id, - &model, - Some(provider_id.as_str()), - )?; - let provider_id = provider_context.provider_id; let fallback_chain = resolve_fallback_chain(catalog, &provider_id, &model, &settings.model, &eligible)?; diff --git a/lib/crates/fabro-workflow/src/run_materialization.rs b/lib/crates/fabro-workflow/src/run_materialization.rs index 14ffa735a..7e82964ac 100644 --- a/lib/crates/fabro-workflow/src/run_materialization.rs +++ b/lib/crates/fabro-workflow/src/run_materialization.rs @@ -34,7 +34,7 @@ pub fn materialize_run( resolve_run_model(catalog, &eligible, model.as_deref(), provider.as_deref())?; settings.run.model.name = Some(resolved_model); - settings.run.model.provider = resolved_provider; + settings.run.model.provider = Some(resolved_provider.into_inner()); let goal = graph.goal().to_string(); settings.run.goal = if goal.is_empty() { @@ -60,55 +60,10 @@ pub(crate) fn resolve_run_model( eligible: &HashSet, model: Option<&str>, provider: Option<&str>, -) -> Result<(String, Option), ModelSelectionError> { - if let Some(provider) = provider.filter(|provider| !provider.is_empty()) { - let requested = ProviderId::new(provider); - let provider = - catalog - .provider(&requested) - .ok_or_else(|| ModelSelectionError::UnknownProvider { - provider: requested.clone(), - })?; - let canonical_provider = provider.id.clone(); - let canonical_eligible = eligible.iter().any(|eligible| { - catalog - .provider(eligible) - .is_some_and(|provider| provider.id == canonical_provider) - }); - if !canonical_eligible { - return Err(ModelSelectionError::ProviderUnavailable { - provider: canonical_provider, - }); - } - if let Some(model) = model { - return match catalog.resolve_on_provider(&provider.id, model) { - Ok(offering) => Ok((offering.id.to_string(), Some(offering.provider.to_string()))), - Err(ModelSelectionError::UnknownSelectorOnProvider { .. }) => { - Ok((model.to_string(), Some(provider.id.to_string()))) - } - Err(error) => Err(error), - }; - } - let offering = catalog.default_for_provider(&provider.id).ok_or_else(|| { - ModelSelectionError::UnknownSelectorOnProvider { - selector: "".to_string(), - provider: provider.id.clone(), - } - })?; - return Ok((offering.id.to_string(), Some(offering.provider.to_string()))); - } - - if let Some(model) = model { - return match catalog.select(model, None, eligible) { - Ok(offering) => Ok((offering.id.to_string(), Some(offering.provider.to_string()))), - Err(ModelSelectionError::UnknownSelector { .. }) => { - let default = catalog.select_default(eligible)?; - Ok((model.to_string(), Some(default.provider.to_string()))) - } - Err(error) => Err(error), - }; - } - - let default = catalog.select_default(eligible)?; - Ok((default.id.to_string(), Some(default.provider.to_string()))) +) -> Result<(String, ProviderId), ModelSelectionError> { + let provider = provider + .filter(|provider| !provider.is_empty()) + .map(ProviderId::new); + let selected = catalog.resolve_selection(model, provider.as_ref(), eligible)?; + Ok((selected.model, selected.provider)) } diff --git a/lib/crates/fabro-workflow/src/transforms/model_resolution.rs b/lib/crates/fabro-workflow/src/transforms/model_resolution.rs index fc130bfb1..7eecbcfd1 100644 --- a/lib/crates/fabro-workflow/src/transforms/model_resolution.rs +++ b/lib/crates/fabro-workflow/src/transforms/model_resolution.rs @@ -2,7 +2,7 @@ use std::collections::HashSet; use std::sync::Arc; use fabro_graphviz::graph::{AttrValue, Graph}; -use fabro_model::{Catalog, ModelSelectionError, ProviderId}; +use fabro_model::{Catalog, ProviderId}; use super::Transform; use crate::error::Error; @@ -46,24 +46,12 @@ impl ModelResolutionTransform { model: &str, explicit_provider: Option<&ProviderId>, ) -> Result<(String, ProviderId), Error> { - match self - .catalog - .select(model, explicit_provider, &self.eligible_providers) - { - Ok(info) => Ok((info.id.to_string(), info.provider.clone())), - Err(ModelSelectionError::UnknownSelectorOnProvider { provider, .. }) => { - Ok((model.to_string(), provider)) - } - Err(ModelSelectionError::UnknownSelector { .. }) => { - let provider = self - .catalog - .select_default(&self.eligible_providers)? - .provider - .clone(); - Ok((model.to_string(), provider)) - } - Err(error) => Err(error.into()), - } + let selected = self.catalog.resolve_selection( + Some(model), + explicit_provider, + &self.eligible_providers, + )?; + Ok((selected.model, selected.provider)) } } From 14317ccaf79fd72d6cb6aa188e55daf3eb2ebcf6 Mon Sep 17 00:00:00 2001 From: Bryan Helmkamp Date: Thu, 23 Jul 2026 12:12:09 -0400 Subject: [PATCH 03/16] fix: preserve legacy model selector compatibility --- docs/public/core-concepts/models.mdx | 8 +- docs/public/execution/failures.mdx | 2 + docs/public/execution/run-configuration.mdx | 2 + lib/crates/fabro-config/src/parse.rs | 48 +++-- lib/crates/fabro-model/src/catalog.rs | 202 +++++++++++++----- .../fabro-server/src/server/handler/models.rs | 3 +- .../src/server/handler/sessions.rs | 73 +++++-- .../fabro-workflow/src/operations/start.rs | 42 ++-- 8 files changed, 271 insertions(+), 109 deletions(-) diff --git a/docs/public/core-concepts/models.mdx b/docs/public/core-concepts/models.mdx index efa60f8cf..ab88318dd 100644 --- a/docs/public/core-concepts/models.mdx +++ b/docs/public/core-concepts/models.mdx @@ -145,9 +145,9 @@ reasoning = false `api_id` is the opaque model name sent to that provider's API. It defaults to the exact model slug, so omit it when the two strings match. Fabro does not infer vendor prefixes or rewrite the value. - -Built-in catalog keys that formerly exposed an aggregator's wire API ID are retired selectors. Fabro rejects them with their replacement address instead of silently routing them. Some contain a slash and look provider-qualified: for example, the old OpenRouter key `openai/gpt-5.6-sol` is retired in favor of `openrouter/gpt-5.6-sol`. To pin the direct OpenAI offering, set `provider = "openai"` and `name = "gpt-5.6-sol"` as separate fields. - + +Historical built-in catalog keys that exposed provider API IDs remain accepted as compatibility selectors. Fabro normalizes a primary or node selector such as `openai/gpt-5.6-sol` to the canonical `gpt-5.6-sol` slug before normal provider-aware selection. With no provider pin, the highest-priority ready offering wins; a separate `provider = "openrouter"` pin selects the OpenRouter offering. Fabro also normalizes these keys in legacy top-level `[llm.models]` rows without rewriting the settings file. + Model roles are separate: `default = true` controls normal model selection for workflow execution, while `small_default = true` marks the provider's small/cheap utility model for metadata tasks such as generated run titles. If a provider has no small default, Fabro falls back to that provider's normal default. @@ -271,7 +271,7 @@ Then launch with: fabro run run.toml ``` -The `fallbacks` array is optional. Each entry may be a bare provider token (like `"gemini"`), a bare model alias (like `"gpt-5.4"`), or a qualified `"provider/model"` reference. Fabro tries them in order when the primary provider is unavailable. +The `fallbacks` array is optional. Each entry may be a bare provider token (like `"gemini"`), a bare model alias (like `"gpt-5.4"`), or a qualified `"provider/model"` reference. Fabro tries them in order when the primary provider is unavailable. In this field, qualified references keep their established provider-pin meaning: `"openai/gpt-5.6-sol"` selects the direct OpenAI offering. The precedence order is: node-level stylesheet > run config TOML > CLI flags > server defaults. More specific settings always win. diff --git a/docs/public/execution/failures.mdx b/docs/public/execution/failures.mdx index ef91658be..1caa84184 100644 --- a/docs/public/execution/failures.mdx +++ b/docs/public/execution/failures.mdx @@ -127,6 +127,8 @@ When Anthropic fails, Fabro tries Gemini first, then OpenAI. Fallback resolution - A qualified selector such as `"openrouter/gpt-56-sol"` resolves only within that provider. - A bare model slug or alias considers ready providers and uses provider priority. +Qualified fallback references always remain provider pins, including strings that were historical built-in API IDs. For example, `"openai/gpt-5.6-sol"` pins the direct OpenAI offering. + The primary provider and model were already resolved and persisted when the run was created; resuming does not re-run primary selection. Fallbacks are only considered after an eligible runtime failure. ### What triggers failover diff --git a/docs/public/execution/run-configuration.mdx b/docs/public/execution/run-configuration.mdx index 43bb9a3d1..3cb544906 100644 --- a/docs/public/execution/run-configuration.mdx +++ b/docs/public/execution/run-configuration.mdx @@ -146,6 +146,8 @@ Provider values are catalog provider ID strings. Built-in IDs like `anthropic` a At run creation, Fabro resolves the primary selector and every node selector against the ready-provider snapshot. It persists the selected canonical model slug and provider, so resuming the run does not choose a different provider just because credentials or priorities changed. The configured fallback chain remains available for failures that occur while the materialized run is executing. +Historical built-in provider API IDs are accepted for compatibility and normalize before this selection. For example, `name = "openai/gpt-5.6-sol"` is treated as the canonical `gpt-5.6-sol` selector; omit `provider` to use readiness and priority, or set `provider` separately to pin an offering. + #### `[run.model.controls]` Set default model controls for all nodes that do not override them in the workflow stylesheet: diff --git a/lib/crates/fabro-config/src/parse.rs b/lib/crates/fabro-config/src/parse.rs index fb72c35af..47b88b728 100644 --- a/lib/crates/fabro-config/src/parse.rs +++ b/lib/crates/fabro-config/src/parse.rs @@ -429,24 +429,46 @@ display_name = "Legacy" } #[test] - fn retired_legacy_model_id_names_replacement() { - let error = r#" + fn legacy_builtin_model_id_normalizes_with_explicit_provider() { + let parsed = r#" [llm.models."openai/gpt-5.6-sol"] provider = "openrouter" +display_name = "Configured Sol" "# .parse::() - .unwrap_err(); + .unwrap(); + let llm = parsed.llm.unwrap(); - assert!(matches!( - error, - ParseError::LlmCatalog(LegacyModelError::Retired { - identifier, - provider, - model, - }) if identifier == "openai/gpt-5.6-sol" - && provider.as_str() == "openrouter" - && model.as_str() == "gpt-5.6-sol" - )); + assert!(llm.models.is_empty()); + let model = llm + .providers + .get("openrouter") + .unwrap() + .models + .get("gpt-5.6-sol") + .unwrap(); + assert_eq!(model.display_name.as_deref(), Some("Configured Sol")); + assert!(model.provider.is_none()); + } + + #[test] + fn legacy_builtin_model_id_without_provider_uses_historical_catalog_provider() { + let parsed = r#" +[llm.models."anthropic/claude-fable-5"] +display_name = "Configured Fable" +"# + .parse::() + .unwrap(); + let llm = parsed.llm.unwrap(); + + assert!(llm.models.is_empty()); + assert!( + llm.providers + .get("openrouter") + .unwrap() + .models + .contains_key("claude-fable-5") + ); } #[test] diff --git a/lib/crates/fabro-model/src/catalog.rs b/lib/crates/fabro-model/src/catalog.rs index ea089cfba..5877bd5f2 100644 --- a/lib/crates/fabro-model/src/catalog.rs +++ b/lib/crates/fabro-model/src/catalog.rs @@ -616,12 +616,6 @@ pub enum ModelSelectionError { "no default model is available on an eligible provider; providers with defaults: {providers:?}" )] NoDefaultModel { providers: Vec }, - #[error("model identifier '{identifier}' has been retired; use '{provider}/{model}' instead")] - RetiredModelIdentifier { - identifier: String, - provider: ProviderId, - model: ModelId, - }, } /// One provider/model pair chosen by [`Catalog::resolve_selection`]. The @@ -711,13 +705,11 @@ impl Catalog { continue; } - if let Some((replacement_provider, replacement_model)) = - retired_model_replacement(&model_id) - { - return Err(LegacyModelError::Retired { + if let Some((_, canonical_model)) = legacy_builtin_model(&model_id) { + return Err(LegacyModelError::LegacyIdentifierAsModelId { identifier: model_id, - provider: replacement_provider, - model: replacement_model, + provider: provider.id.clone(), + model: canonical_model, } .into()); } @@ -870,12 +862,14 @@ impl Catalog { } /// Look up a selector on exactly one provider, without considering - /// provider availability. + /// provider availability. Historical built-in API identifiers normalize + /// to their canonical model slug before lookup. #[must_use] pub fn get_on_provider(&self, provider: &ProviderId, selector: &str) -> Option<&Model> { let provider = self.provider(provider)?; + let selector = normalize_legacy_builtin_selector(selector); self.provider_selector_index - .get(&(provider.id.clone(), selector.to_string())) + .get(&(provider.id.clone(), selector.into_owned())) .and_then(|idx| self.models.get(*idx)) } @@ -894,14 +888,6 @@ impl Catalog { provider: &ProviderId, selector: &str, ) -> Result<&Model, ModelSelectionError> { - if let Some((replacement_provider, replacement_model)) = retired_model_replacement(selector) - { - return Err(ModelSelectionError::RetiredModelIdentifier { - identifier: selector.to_string(), - provider: replacement_provider, - model: replacement_model, - }); - } let provider = self.provider(provider) .ok_or_else(|| ModelSelectionError::UnknownProvider { @@ -919,6 +905,9 @@ impl Catalog { /// Select one concrete offering for a selector and ready-provider /// snapshot. /// + /// Historical built-in API identifiers normalize to their canonical model + /// slug before selection. + /// /// An explicit provider is a pin. Unqualified selection checks canonical /// IDs before aliases and uses the catalog's provider priority ordering. pub fn select<'a>( @@ -927,14 +916,6 @@ impl Catalog { explicit_provider: Option<&ProviderId>, eligible_providers: &HashSet, ) -> Result<&'a Model, ModelSelectionError> { - if let Some((replacement_provider, replacement_model)) = retired_model_replacement(selector) - { - return Err(ModelSelectionError::RetiredModelIdentifier { - identifier: selector.to_string(), - provider: replacement_provider, - model: replacement_model, - }); - } let eligible = eligible_providers .iter() .filter_map(|provider| self.provider(provider).map(|provider| provider.id.clone())) @@ -954,7 +935,10 @@ impl Catalog { return self.resolve_on_provider(&provider.id, selector); } - let canonical = self.canonical_candidates.get(&ModelId::new(selector)); + let normalized_selector = normalize_legacy_builtin_selector(selector); + let canonical = self + .canonical_candidates + .get(&ModelId::new(normalized_selector.as_ref())); if let Some(indices) = canonical { if let Some(model) = indices .iter() @@ -965,7 +949,7 @@ impl Catalog { } } - let aliases = self.alias_candidates.get(selector); + let aliases = self.alias_candidates.get(normalized_selector.as_ref()); if let Some(indices) = aliases { if let Some(model) = indices .iter() @@ -1117,9 +1101,10 @@ impl Catalog { } fn candidate_indices(&self, selector: &str) -> Option<&Vec> { + let selector = normalize_legacy_builtin_selector(selector); self.canonical_candidates - .get(&ModelId::new(selector)) - .or_else(|| self.alias_candidates.get(selector)) + .get(&ModelId::new(selector.as_ref())) + .or_else(|| self.alias_candidates.get(selector.as_ref())) } #[must_use] @@ -1537,8 +1522,10 @@ fn reject_scoped_provider_fields(settings: &LlmCatalogSettings) -> Result<(), Le pub enum LegacyModelError { #[error("failed to inspect the built-in model catalog: {message}")] BuiltinCatalog { message: String }, - #[error("model identifier '{identifier}' has been retired; use '{provider}/{model}' instead")] - Retired { + #[error( + "legacy built-in model identifier '{identifier}' cannot be used as a canonical model ID under provider '{provider}'; use '{model}'" + )] + LegacyIdentifierAsModelId { identifier: String, provider: ProviderId, model: ModelId, @@ -1647,20 +1634,23 @@ impl LegacyModelIndex { Ok(self) } - /// Resolve one legacy row to its provider-scoped address. An unknown - /// explicit provider or model selector passes through verbatim; rows - /// without an explicit provider must match exactly one known offering. + /// Resolve one legacy row to its provider-scoped address. Historical + /// built-in identifiers normalize to their canonical slug and use their + /// historical provider when no explicit provider is present. Other + /// unknown explicit providers or model selectors pass through verbatim; + /// rows without an explicit provider must match exactly one known + /// offering. pub fn resolve( &self, legacy_id: &str, explicit_provider: Option<&str>, ) -> Result<(ProviderId, ModelId), LegacyModelError> { - if let Some((provider, model)) = retired_model_replacement(legacy_id) { - return Err(LegacyModelError::Retired { - identifier: legacy_id.to_string(), - provider, - model, + if let Some((historical_provider, model)) = legacy_builtin_model(legacy_id) { + let provider = explicit_provider.map_or(historical_provider, |explicit| { + self.canonical_provider(explicit) + .unwrap_or_else(|| ProviderId::new(explicit)) }); + return Ok((provider, model)); } if let Some(explicit) = explicit_provider { let provider = self @@ -1755,11 +1745,11 @@ impl LegacyModelIndex { } } -/// Built-in catalog keys retired when provider API identifiers stopped being -/// Fabro model IDs. Keep this list explicit so old workflow and persisted-run -/// references fail with an actionable replacement instead of silently -/// selecting another route. -const RETIRED_MODEL_IDENTIFIERS: &[(&str, &str, &str)] = &[ +/// Historical built-in catalog keys from before Fabro separated canonical +/// model slugs from provider API identifiers. The provider records the key's +/// original offering for legacy catalog-row normalization; runtime selectors +/// normalize to the model slug and use normal provider-aware selection. +const LEGACY_BUILTIN_MODEL_IDENTIFIERS: &[(&str, &str, &str)] = &[ ("openai.gpt-5.5", "bedrock-openai", "gpt-5.5"), ("openai.gpt-5.4", "bedrock-openai", "gpt-5.4"), ( @@ -1833,15 +1823,23 @@ const RETIRED_MODEL_IDENTIFIERS: &[(&str, &str, &str)] = &[ ("mistralai/devstral-2512", "openrouter", "devstral-2512"), ]; -/// Return the replacement address for a retired built-in catalog key. +/// Return the historical provider and canonical model slug for a legacy +/// built-in catalog key. #[must_use] -pub fn retired_model_replacement(identifier: &str) -> Option<(ProviderId, ModelId)> { - RETIRED_MODEL_IDENTIFIERS +pub fn legacy_builtin_model(identifier: &str) -> Option<(ProviderId, ModelId)> { + LEGACY_BUILTIN_MODEL_IDENTIFIERS .iter() - .find(|(retired, _, _)| *retired == identifier) + .find(|(legacy, _, _)| *legacy == identifier) .map(|(_, provider, model)| (ProviderId::new(*provider), ModelId::new(*model))) } +fn normalize_legacy_builtin_selector(selector: &str) -> Cow<'_, str> { + legacy_builtin_model(selector).map_or_else( + || Cow::Borrowed(selector), + |(_, model)| Cow::Owned(model.into_inner()), + ) +} + fn merge_catalog_settings( higher: LlmCatalogSettings, mut fallback: LlmCatalogSettings, @@ -3105,6 +3103,47 @@ enabled = true } } + #[test] + fn builtin_legacy_vendor_ids_normalize_for_pinned_and_unpinned_selection() { + let catalog = Catalog::from_builtin_with_overrides(&minimal_settings( + r" +[providers.openrouter] +enabled = true +", + )) + .expect("enabled OpenRouter override should build from the built-in provider settings"); + let openrouter = ProviderId::new("openrouter"); + + for (selector, canonical_id) in [ + ("anthropic/claude-fable-5", "claude-fable-5"), + ("openai/gpt-5.6-sol", "gpt-5.6-sol"), + ] { + let model = catalog + .resolve_on_provider(&openrouter, selector) + .unwrap_or_else(|error| panic!("{selector} should resolve on OpenRouter: {error}")); + assert_eq!(model.provider, openrouter, "{selector}"); + assert_eq!(model.id, canonical_id, "{selector}"); + } + + let anthropic = ProviderId::anthropic(); + let selector = "anthropic/claude-fable-5"; + let selected = catalog + .resolve_selection( + Some(selector), + None, + &HashSet::from([anthropic.clone(), openrouter.clone()]), + ) + .unwrap(); + assert_eq!(selected.provider, anthropic); + assert_eq!(selected.model, "claude-fable-5"); + + let selected = catalog + .resolve_selection(Some(selector), None, &HashSet::from([openrouter.clone()])) + .unwrap(); + assert_eq!(selected.provider, openrouter); + assert_eq!(selected.model, "claude-fable-5"); + } + #[test] fn builtin_openrouter_includes_glm_5_2_when_enabled() { let catalog = Catalog::from_builtin_with_overrides(&minimal_settings( @@ -3803,6 +3842,33 @@ provider = "test" )); } + #[test] + fn provider_scoped_model_rejects_legacy_builtin_id_as_canonical_id() { + let error = Catalog::from_settings(&minimal_settings( + r#" +[providers.openrouter] +display_name = "OpenRouter" +adapter = "openai_compatible" + +[providers.openrouter.models."openai/gpt-5.6-sol"] +"#, + )) + .unwrap_err(); + + assert!(matches!( + error, + CatalogBuildError::LegacyModel( + LegacyModelError::LegacyIdentifierAsModelId { + identifier, + provider, + model, + } + ) if identifier == "openai/gpt-5.6-sol" + && provider == ProviderId::new("openrouter") + && model == "gpt-5.6-sol" + )); + } + #[test] fn provider_aware_selection_uses_readiness_priority_and_api_ids() { let catalog = portable_model_catalog(); @@ -3856,6 +3922,36 @@ provider = "test" )); } + #[test] + fn legacy_builtin_selector_uses_readiness_priority_and_explicit_pins() { + let catalog = portable_model_catalog(); + let openai = ProviderId::openai(); + let openrouter = ProviderId::new("openrouter"); + let selector = "openai/gpt-5.6-sol"; + + for (eligible, expected_provider) in [ + (HashSet::from([openai.clone()]), openai.clone()), + (HashSet::from([openrouter.clone()]), openrouter.clone()), + ( + HashSet::from([openai.clone(), openrouter.clone()]), + openai.clone(), + ), + ] { + let selected = catalog + .resolve_selection(Some(selector), None, &eligible) + .unwrap(); + assert_eq!(selected.provider, expected_provider); + assert_eq!(selected.model, "gpt-5.6-sol"); + } + + let both = HashSet::from([openai, openrouter.clone()]); + let selected = catalog + .resolve_selection(Some(selector), Some(&openrouter), &both) + .unwrap(); + assert_eq!(selected.provider, openrouter); + assert_eq!(selected.model, "gpt-5.6-sol"); + } + #[test] fn equal_provider_priorities_use_canonical_provider_id_as_tie_breaker() { let catalog = Catalog::from_settings(&minimal_settings( diff --git a/lib/crates/fabro-server/src/server/handler/models.rs b/lib/crates/fabro-server/src/server/handler/models.rs index 59b6b8623..06af1d3de 100644 --- a/lib/crates/fabro-server/src/server/handler/models.rs +++ b/lib/crates/fabro-server/src/server/handler/models.rs @@ -270,8 +270,7 @@ fn model_selection_response(error: &ModelSelectionError) -> Response { | ModelSelectionError::UnknownSelectorOnProvider { .. } => { ApiError::not_found(error.to_string()).into_response() } - ModelSelectionError::RetiredModelIdentifier { .. } - | ModelSelectionError::ProviderUnavailable { .. } + ModelSelectionError::ProviderUnavailable { .. } | ModelSelectionError::NoEligibleOffering { .. } | ModelSelectionError::NoDefaultModel { .. } => { ApiError::bad_request(error.to_string()).into_response() diff --git a/lib/crates/fabro-server/src/server/handler/sessions.rs b/lib/crates/fabro-server/src/server/handler/sessions.rs index 936b8e582..f053207f6 100644 --- a/lib/crates/fabro-server/src/server/handler/sessions.rs +++ b/lib/crates/fabro-server/src/server/handler/sessions.rs @@ -859,14 +859,11 @@ fn canonical_session_model( if requested.is_empty() { return Err(ApiError::bad_request("Session model must not be empty.")); } - if let Some((provider, model)) = catalog::retired_model_replacement(requested) { - return Err(session_selection_error( - &ModelSelectionError::RetiredModelIdentifier { - identifier: requested.to_string(), - provider, - model, - }, - )); + if catalog::legacy_builtin_model(requested).is_some() { + let selected = catalog + .resolve_selection(Some(requested), explicit_provider.as_ref(), eligible) + .map_err(|error| session_selection_error(&error))?; + return Ok((selected.provider, selected.model)); } let model_ref = requested .parse::() @@ -1715,13 +1712,63 @@ reasoning = false } #[test] - fn canonical_session_model_rejects_retired_wire_identifier_before_qualification() { - let catalog = Catalog::from_builtin().unwrap(); + fn canonical_session_model_normalizes_legacy_builtin_selector_before_qualification() { + let catalog = portable_session_catalog(); + let openai = ProviderId::openai(); + let openrouter = ProviderId::new("openrouter"); + let both = std::collections::HashSet::from([openai.clone(), openrouter.clone()]); + + assert_eq!( + canonical_session_model(&catalog, &both, Some("openai/gpt-5.6-sol"), None,).unwrap(), + (openai, "gpt-5.6-sol".to_string()) + ); + assert_eq!( + canonical_session_model( + &catalog, + &both, + Some("openai/gpt-5.6-sol"), + Some(&openrouter), + ) + .unwrap(), + (openrouter.clone(), "gpt-5.6-sol".to_string()) + ); + assert_eq!( + canonical_session_model( + &catalog, + &std::collections::HashSet::from([openrouter.clone()]), + Some("openai/gpt-5.6-sol"), + None, + ) + .unwrap(), + (openrouter, "gpt-5.6-sol".to_string()) + ); + } + + #[test] + fn canonical_session_model_still_treats_non_legacy_qualified_model_as_a_pin() { + let catalog = portable_session_catalog(); + let openrouter = ProviderId::new("openrouter"); + + assert_eq!( + canonical_session_model( + &catalog, + &catalog.all_provider_ids(), + Some("openrouter/gpt-56-sol"), + None, + ) + .unwrap(), + (openrouter, "gpt-5.6-sol".to_string()) + ); + } + + #[test] + fn canonical_session_model_rejects_conflicting_non_legacy_provider_pins() { + let catalog = portable_session_catalog(); let error = canonical_session_model( &catalog, &catalog.all_provider_ids(), - Some("openai/gpt-5.6-sol"), - None, + Some("openrouter/gpt-56-sol"), + Some(&ProviderId::openai()), ) .unwrap_err(); @@ -1730,7 +1777,7 @@ reasoning = false error .into_response_entry() .detail - .contains("openrouter/gpt-5.6-sol") + .contains("conflicts with model reference provider") ); } diff --git a/lib/crates/fabro-workflow/src/operations/start.rs b/lib/crates/fabro-workflow/src/operations/start.rs index b1e33f4c7..25f7a8014 100644 --- a/lib/crates/fabro-workflow/src/operations/start.rs +++ b/lib/crates/fabro-workflow/src/operations/start.rs @@ -7,7 +7,7 @@ use fabro_auth::{CredentialSource, EnvCredentialSource, VaultCredentialSource}; use fabro_interview::{AutoApproveInterviewer, Interviewer}; use fabro_llm::client::Client as LlmClient; use fabro_mcp::config::McpServerSettings; -use fabro_model::{Catalog, FallbackTarget, ModelSelectionError, ProviderId, catalog}; +use fabro_model::{Catalog, FallbackTarget, ModelSelectionError, ProviderId}; use fabro_sandbox::daytona::DaytonaConfig; use fabro_sandbox::from_environment::{ daytona_config_from_environment, docker_config_from_environment_with_secrets, @@ -648,15 +648,6 @@ fn resolve_fallback_chain( let mut chain = Vec::new(); for model_ref in &settings.fallbacks { - let identifier = model_ref.to_string(); - if let Some((provider, model)) = catalog::retired_model_replacement(&identifier) { - return Err(ModelSelectionError::RetiredModelIdentifier { - identifier, - provider, - model, - } - .into()); - } match model_ref.resolve(®istry)? { ResolvedModelRef::Provider(provider_name) => { let provider_id = canonical_provider_id(catalog, &provider_name); @@ -1411,32 +1402,35 @@ reasoning = false } #[test] - fn resolve_fallback_chain_rejects_retired_wire_identifier_before_qualification() { + fn resolve_fallback_chain_keeps_qualified_legacy_references_as_provider_pins() { let catalog = test_catalog(); let settings = ResolvedRunModelSettings { - fallbacks: vec!["openai/gpt-5.6-sol".parse::().unwrap()], + fallbacks: vec![ + "openai/gpt-5.6-sol".parse::().unwrap(), + "anthropic/claude-fable-5".parse::().unwrap(), + ], ..ResolvedRunModelSettings::default() }; - let error = resolve_fallback_chain( + let chain = resolve_fallback_chain( catalog.as_ref(), &ProviderId::anthropic(), "claude-opus-4-6", &settings, &catalog.all_provider_ids(), ) - .unwrap_err(); + .unwrap(); - assert!(matches!( - error, - Error::ModelSelection(ModelSelectionError::RetiredModelIdentifier { - identifier, - provider, - model, - }) if identifier == "openai/gpt-5.6-sol" - && provider == ProviderId::new("openrouter") - && model == fabro_model::ModelId::new("gpt-5.6-sol") - )); + assert_eq!(chain, vec![ + FallbackTarget { + provider: "openai".to_string(), + model: "gpt-5.6-sol".to_string(), + }, + FallbackTarget { + provider: "anthropic".to_string(), + model: "claude-fable-5".to_string(), + }, + ]); } #[test] From 4f697c527c50f888c6a50c93a55a9dec8bb3c53f Mon Sep 17 00:00:00 2001 From: Bryan Helmkamp Date: Thu, 23 Jul 2026 13:11:49 -0400 Subject: [PATCH 04/16] fix: preserve provider pins during model routing --- lib/crates/fabro-model/src/catalog.rs | 91 +++++++++++ lib/crates/fabro-server/src/server/tests.rs | 2 +- .../fabro-types/src/run_event/session.rs | 19 +++ lib/crates/fabro-types/src/session.rs | 22 ++- .../fabro-workflow/src/event/convert.rs | 2 +- .../fabro-workflow/src/handler/llm/api.rs | 27 ++++ .../fabro-workflow/src/handler/llm/routing.rs | 5 + .../fabro-workflow/src/operations/create.rs | 149 +++++++++++------- 8 files changed, 254 insertions(+), 63 deletions(-) diff --git a/lib/crates/fabro-model/src/catalog.rs b/lib/crates/fabro-model/src/catalog.rs index 5877bd5f2..69790f96d 100644 --- a/lib/crates/fabro-model/src/catalog.rs +++ b/lib/crates/fabro-model/src/catalog.rs @@ -3144,6 +3144,42 @@ enabled = true assert_eq!(selected.model, "claude-fable-5"); } + #[test] + fn every_legacy_builtin_identifier_targets_an_existing_offering() { + let catalog = Catalog::from_builtin_with_overrides(&minimal_settings( + r" +[providers.bedrock] +enabled = true + +[providers.bedrock-openai] +enabled = true + +[providers.openrouter] +enabled = true +", + )) + .expect("all providers referenced by the legacy table should build"); + + for (legacy_id, provider_id, canonical_id) in LEGACY_BUILTIN_MODEL_IDENTIFIERS { + let provider = ProviderId::new(*provider_id); + let model = catalog + .resolve_on_provider(&provider, legacy_id) + .unwrap_or_else(|error| { + panic!( + "legacy identifier '{legacy_id}' should resolve on '{provider}': {error}" + ) + }); + + assert_eq!(model.provider, provider, "{legacy_id}"); + assert_eq!(model.id, *canonical_id, "{legacy_id}"); + assert_eq!( + legacy_builtin_model(legacy_id), + Some((provider, ModelId::new(*canonical_id))), + "{legacy_id}" + ); + } + } + #[test] fn builtin_openrouter_includes_glm_5_2_when_enabled() { let catalog = Catalog::from_builtin_with_overrides(&minimal_settings( @@ -5118,6 +5154,61 @@ reasoning = false ); } + #[test] + fn effective_agent_profile_is_scoped_by_provider_for_shared_model_id() { + let layer = minimal_settings( + r#" +[providers.one] +display_name = "One" +adapter = "openai" +agent_profile = "openai" + +[providers.one.models.shared] +display_name = "Shared on One" +family = "test" +default = true + +[providers.one.models.shared.limits] +context_window = 1000 + +[providers.one.models.shared.features] +tools = false +vision = false +reasoning = false + +[providers.two] +display_name = "Two" +adapter = "openai" +agent_profile = "anthropic" + +[providers.two.models.shared] +display_name = "Shared on Two" +family = "test" +default = true +agent_profile = "gemini" + +[providers.two.models.shared.limits] +context_window = 1000 + +[providers.two.models.shared.features] +tools = false +vision = false +reasoning = false +"#, + ); + + let catalog = Catalog::from_settings(&layer).unwrap(); + + assert_eq!( + catalog.effective_agent_profile(&ProviderId::new("one"), Some("shared")), + Some(AgentProfileKind::OpenAi) + ); + assert_eq!( + catalog.effective_agent_profile(&ProviderId::new("two"), Some("shared")), + Some(AgentProfileKind::Gemini) + ); + } + #[test] fn omitted_agent_profile_uses_adapter_default() { let layer = minimal_settings( diff --git a/lib/crates/fabro-server/src/server/tests.rs b/lib/crates/fabro-server/src/server/tests.rs index 9ecdabcae..7aafcec03 100644 --- a/lib/crates/fabro-server/src/server/tests.rs +++ b/lib/crates/fabro-server/src/server/tests.rs @@ -13785,7 +13785,7 @@ async fn get_aggregate_billing_saturates_total_cost_across_models() { agg.by_model.insert( ModelRef { provider: ProviderId::openai(), - model_id: model_id.to_string(), + model_id: model_id.into(), speed: None, }, ModelBillingTotals { diff --git a/lib/crates/fabro-types/src/run_event/session.rs b/lib/crates/fabro-types/src/run_event/session.rs index 504d4f653..78321afa4 100644 --- a/lib/crates/fabro-types/src/run_event/session.rs +++ b/lib/crates/fabro-types/src/run_event/session.rs @@ -110,3 +110,22 @@ pub struct RunSessionTurnInterruptedProps { #[serde(default, skip_serializing_if = "Option::is_none")] pub error: Option, } + +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::RunSessionCreatedProps; + + #[test] + fn session_created_deserializes_legacy_payload_without_provider() { + let props: RunSessionCreatedProps = serde_json::from_value(json!({ + "title": "Legacy session", + "model": "gpt-5.4" + })) + .unwrap(); + + assert_eq!(props.model.as_deref(), Some("gpt-5.4")); + assert_eq!(props.provider, None); + } +} diff --git a/lib/crates/fabro-types/src/session.rs b/lib/crates/fabro-types/src/session.rs index eaad05181..b825338d9 100644 --- a/lib/crates/fabro-types/src/session.rs +++ b/lib/crates/fabro-types/src/session.rs @@ -188,13 +188,33 @@ impl SessionMessage { #[cfg(test)] mod tests { + use chrono::Utc; use serde_json::json; - use super::SessionStatus; + use super::{SessionId, SessionRecord, SessionStatus}; + use crate::fixtures; #[test] fn session_status_rejects_removed_terminal_states() { assert!(serde_json::from_value::(json!("closed")).is_err()); assert!(serde_json::from_value::(json!("deleted")).is_err()); } + + #[test] + fn session_record_deserializes_legacy_json_without_provider() { + let mut value = serde_json::to_value(SessionRecord::new( + SessionId::new(), + fixtures::RUN_1, + Utc::now(), + )) + .unwrap(); + value + .as_object_mut() + .expect("session record should serialize as an object") + .remove("provider"); + + let record: SessionRecord = serde_json::from_value(value).unwrap(); + + assert_eq!(record.provider, None); + } } diff --git a/lib/crates/fabro-workflow/src/event/convert.rs b/lib/crates/fabro-workflow/src/event/convert.rs index b2ba1d715..528371880 100644 --- a/lib/crates/fabro-workflow/src/event/convert.rs +++ b/lib/crates/fabro-workflow/src/event/convert.rs @@ -2184,7 +2184,7 @@ mod tests { text: "ok".to_string(), model: ModelRef { provider: ProviderId::new("openrouter"), - model_id: "openai/gpt-5.4".to_string(), + model_id: "openai/gpt-5.4".into(), speed: None, }, usage: LlmTokenCounts { diff --git a/lib/crates/fabro-workflow/src/handler/llm/api.rs b/lib/crates/fabro-workflow/src/handler/llm/api.rs index 573542d9f..4e60dfbff 100644 --- a/lib/crates/fabro-workflow/src/handler/llm/api.rs +++ b/lib/crates/fabro-workflow/src/handler/llm/api.rs @@ -2624,6 +2624,33 @@ reasoning = false assert_eq!(provider.profile_kind, AgentProfileKind::Anthropic); } + #[test] + fn api_backend_preserves_default_provider_for_legacy_model_identifier() { + let settings: LlmCatalogSettings = toml::from_str( + r" +[providers.openrouter] +enabled = true +", + ) + .unwrap(); + let catalog = Arc::new(Catalog::from_builtin_with_overrides(&settings).unwrap()); + let backend = AgentApiBackend::new_with_catalog( + "openai/gpt-5.4".to_string(), + ProviderId::from("openrouter"), + Vec::new(), + Arc::new(EnvCredentialSource::new()), + SteeringHub::for_tests(), + catalog, + ); + + let provider = backend + .resolve_provider_context("openai/gpt-5.4", None) + .unwrap(); + + assert_eq!(provider.provider_id, ProviderId::from("openrouter")); + assert_eq!(provider.profile_kind, AgentProfileKind::OpenAi); + } + #[test] fn run_model_controls_apply_when_node_omits_controls() { let backend = AgentApiBackend::new_from_env( diff --git a/lib/crates/fabro-workflow/src/handler/llm/routing.rs b/lib/crates/fabro-workflow/src/handler/llm/routing.rs index 0bb05b216..b7d499b8c 100644 --- a/lib/crates/fabro-workflow/src/handler/llm/routing.rs +++ b/lib/crates/fabro-workflow/src/handler/llm/routing.rs @@ -58,6 +58,11 @@ pub(crate) fn resolve_provider_context( })? .id .clone() + } else if catalog + .get_on_provider(default_provider_id, model) + .is_some() + { + default_provider_id.clone() } else { match catalog.select(model, None, &catalog.all_provider_ids()) { Ok(model) => model.provider.clone(), diff --git a/lib/crates/fabro-workflow/src/operations/create.rs b/lib/crates/fabro-workflow/src/operations/create.rs index 488bb5c74..144cf5976 100644 --- a/lib/crates/fabro-workflow/src/operations/create.rs +++ b/lib/crates/fabro-workflow/src/operations/create.rs @@ -1462,11 +1462,11 @@ reasoning = false } #[tokio::test] - async fn create_materializes_shared_alias_for_ready_provider_snapshot_and_pin() { - const ALIAS_DOT: &str = r#"digraph Test { + async fn create_materializes_portable_selectors_for_ready_provider_snapshot_and_pin() { + const MODEL_DOT: &str = r#"digraph Test { graph [goal="Test"] start [shape=Mdiamond] - work [prompt="Do work", model="gpt-56-sol"] + work [prompt="Do work", model="MODEL_SELECTOR"] exit [shape=Msquare] start -> work -> exit }"#; @@ -1490,65 +1490,94 @@ reasoning = false ), ]; - for (ready, explicit_provider, expected_provider) in cases { - let dir = tempfile::tempdir().unwrap(); - let mut settings = test_default_settings(); - settings.run.model.name = Some("gpt-56-sol".to_string()); - settings.run.model.provider = explicit_provider.map(str::to_string); - let store = memory_store(); - let created = create( - store.as_ref(), - CreateRunInput { - workflow: WorkflowInput::DotSource { - source: ALIAS_DOT.to_string(), - base_dir: None, + for selector in ["gpt-56-sol", "openai/gpt-5.6-sol"] { + for (ready, explicit_provider, expected_provider) in &cases { + let dir = tempfile::tempdir().unwrap(); + let mut settings = test_default_settings(); + settings.run.model.name = Some(selector.to_string()); + settings.run.model.provider = explicit_provider.map(str::to_string); + let store = memory_store(); + let created = create( + store.as_ref(), + CreateRunInput { + workflow: WorkflowInput::DotSource { + source: MODEL_DOT.replace("MODEL_SELECTOR", selector), + base_dir: None, + }, + settings, + vars: HashMap::new(), + cwd: dir.path().to_path_buf(), + workflow_slug: None, + workflow_path: None, + workflow_bundle: None, + submitted_manifest_bytes: None, + run_id: None, + title: None, + automation: None, + git: None, + fork_source_ref: None, + parent_id: None, + provenance: test_support::test_run_provenance(), + configured_providers: ready.clone(), + web_url: None, }, - settings, - vars: HashMap::new(), - cwd: dir.path().to_path_buf(), - workflow_slug: None, - workflow_path: None, - workflow_bundle: None, - submitted_manifest_bytes: None, - run_id: None, - title: None, - automation: None, - git: None, - fork_source_ref: None, - parent_id: None, - provenance: test_support::test_run_provenance(), - configured_providers: ready, - web_url: None, - }, - dir.path().join("storage"), - Arc::clone(&catalog), - ) - .await - .unwrap(); - let run_spec = created.persisted.run_spec(); + dir.path().join("storage"), + Arc::clone(&catalog), + ) + .await + .unwrap(); + let run_spec = created.persisted.run_spec(); - assert_eq!( - run_spec.settings.run.model.name.as_deref(), - Some("gpt-5.6-sol") - ); - assert_eq!( - run_spec.settings.run.model.provider.as_deref(), - Some(expected_provider.as_str()) - ); - assert_eq!( - run_spec.graph.nodes["work"] - .attrs - .get("model") - .and_then(AttrValue::as_str), - Some("gpt-5.6-sol") - ); - assert_eq!( - run_spec.graph.nodes["work"] - .attrs - .get("provider") - .and_then(AttrValue::as_str), - Some(expected_provider.as_str()) - ); + assert_eq!( + run_spec.settings.run.model.name.as_deref(), + Some("gpt-5.6-sol"), + "{selector}" + ); + assert_eq!( + run_spec.settings.run.model.provider.as_deref(), + Some(expected_provider.as_str()), + "{selector}" + ); + assert_eq!( + run_spec.graph.nodes["work"] + .attrs + .get("model") + .and_then(AttrValue::as_str), + Some("gpt-5.6-sol"), + "{selector}" + ); + assert_eq!( + run_spec.graph.nodes["work"] + .attrs + .get("provider") + .and_then(AttrValue::as_str), + Some(expected_provider.as_str()), + "{selector}" + ); + + let run_store = store.open_run(&created.run_id).await.unwrap(); + let run_store = run_store.into(); + let reloaded = Persisted::load_from_store(&run_store, &created.run_dir) + .await + .unwrap(); + assert_eq!( + reloaded.run_spec().settings.run.model.provider.as_deref(), + Some(expected_provider.as_str()), + "{selector}" + ); + assert_eq!( + reloaded.run_spec().graph.nodes["work"] + .attrs + .get("provider") + .and_then(AttrValue::as_str), + Some(expected_provider.as_str()), + "{selector}" + ); + assert!( + reloaded.source().contains(selector), + "persisted source should preserve the user's selector '{selector}'" + ); + } } } From 831b7506645ae2f4c44c44ab988e86c6cc7a8d22 Mon Sep 17 00:00:00 2001 From: Bryan Helmkamp Date: Thu, 23 Jul 2026 13:13:45 -0400 Subject: [PATCH 05/16] fix: honor the run's provider pin during request-time routing Merging main brought in billing tests that construct ModelRef with String model ids and an integration test that pins an OpenRouter run via the backend's provider id. The ModelRef sites now use ModelId conversions. The integration test also exposed a real regression: resolve_provider_context ignored the persisted run provider whenever the model selector resolved globally, re-routing pinned OpenRouter runs to a higher-priority provider for nodes without explicit model/provider attrs. Request-time routing now treats the run's selected provider as a pin with custom-model passthrough, matching transform-time selection semantics. Co-Authored-By: Claude Fable 5 --- lib/crates/fabro-server/src/server/tests.rs | 2 +- .../fabro-workflow/src/event/convert.rs | 2 +- .../fabro-workflow/src/handler/llm/api.rs | 41 +++++++++++++++++++ .../fabro-workflow/src/handler/llm/routing.rs | 22 ++++++++-- 4 files changed, 61 insertions(+), 6 deletions(-) diff --git a/lib/crates/fabro-server/src/server/tests.rs b/lib/crates/fabro-server/src/server/tests.rs index 9ecdabcae..7aafcec03 100644 --- a/lib/crates/fabro-server/src/server/tests.rs +++ b/lib/crates/fabro-server/src/server/tests.rs @@ -13785,7 +13785,7 @@ async fn get_aggregate_billing_saturates_total_cost_across_models() { agg.by_model.insert( ModelRef { provider: ProviderId::openai(), - model_id: model_id.to_string(), + model_id: model_id.into(), speed: None, }, ModelBillingTotals { diff --git a/lib/crates/fabro-workflow/src/event/convert.rs b/lib/crates/fabro-workflow/src/event/convert.rs index b2ba1d715..528371880 100644 --- a/lib/crates/fabro-workflow/src/event/convert.rs +++ b/lib/crates/fabro-workflow/src/event/convert.rs @@ -2184,7 +2184,7 @@ mod tests { text: "ok".to_string(), model: ModelRef { provider: ProviderId::new("openrouter"), - model_id: "openai/gpt-5.4".to_string(), + model_id: "openai/gpt-5.4".into(), speed: None, }, usage: LlmTokenCounts { diff --git a/lib/crates/fabro-workflow/src/handler/llm/api.rs b/lib/crates/fabro-workflow/src/handler/llm/api.rs index 573542d9f..349960574 100644 --- a/lib/crates/fabro-workflow/src/handler/llm/api.rs +++ b/lib/crates/fabro-workflow/src/handler/llm/api.rs @@ -2531,6 +2531,47 @@ reasoning = false assert!(names.contains(&"close_agent".to_string())); } + #[test] + fn api_backend_provider_pin_wins_over_priority_selection() { + let settings: LlmCatalogSettings = toml::from_str( + r" +[providers.openrouter] +enabled = true +", + ) + .unwrap(); + let backend = AgentApiBackend::new_with_catalog( + "gpt-5.4".to_string(), + ProviderId::from("openrouter"), + Vec::new(), + Arc::new(EnvCredentialSource::new()), + SteeringHub::for_tests(), + Arc::new(Catalog::from_builtin_with_overrides(&settings).unwrap()), + ); + + let provider = backend.resolve_provider_context("gpt-5.4", None).unwrap(); + + assert_eq!(provider.provider_id, ProviderId::from("openrouter")); + } + + #[test] + fn api_backend_node_provider_attr_overrides_backend_pin() { + let backend = AgentApiBackend::new_with_catalog( + "gpt-5.4".to_string(), + ProviderId::from("openrouter"), + Vec::new(), + Arc::new(EnvCredentialSource::new()), + SteeringHub::for_tests(), + Arc::new(Catalog::from_builtin().unwrap()), + ); + + let provider = backend + .resolve_provider_context("gpt-5.4", Some("openai")) + .unwrap(); + + assert_eq!(provider.provider_id, ProviderId::openai()); + } + #[test] fn api_backend_resolves_custom_catalog_provider_profile() { let settings: LlmCatalogSettings = toml::from_str( diff --git a/lib/crates/fabro-workflow/src/handler/llm/routing.rs b/lib/crates/fabro-workflow/src/handler/llm/routing.rs index 0bb05b216..cfe7a85a0 100644 --- a/lib/crates/fabro-workflow/src/handler/llm/routing.rs +++ b/lib/crates/fabro-workflow/src/handler/llm/routing.rs @@ -59,10 +59,24 @@ pub(crate) fn resolve_provider_context( .id .clone() } else { - match catalog.select(model, None, &catalog.all_provider_ids()) { - Ok(model) => model.provider.clone(), - Err(fabro_model::ModelSelectionError::UnknownSelector { .. }) => { - default_provider_id.clone() + // The run's selected provider is a pin, matching transform-time + // semantics: a selector the pinned provider does not offer passes + // through to it as a custom model rather than re-selecting a + // higher-priority provider. + match catalog.resolve_selection( + Some(model), + Some(default_provider_id), + &catalog.all_provider_ids(), + ) { + Ok(selected) => selected.provider, + Err(fabro_model::ModelSelectionError::UnknownProvider { .. }) => { + match catalog.select(model, None, &catalog.all_provider_ids()) { + Ok(model) => model.provider.clone(), + Err(fabro_model::ModelSelectionError::UnknownSelector { .. }) => { + default_provider_id.clone() + } + Err(error) => return Err(error.into()), + } } Err(error) => return Err(error.into()), } From 96102854d8b00239bc0ea95bee7a9366121aaa7f Mon Sep 17 00:00:00 2001 From: Bryan Helmkamp Date: Thu, 23 Jul 2026 13:13:45 -0400 Subject: [PATCH 06/16] chore: address review feedback - rename resolve_route catalog-instance test to describe its actual id-based resolution assertion - use EnvVars::OPENAI_API_KEY instead of a raw string in the automation scheduler test fixture Co-Authored-By: Claude Fable 5 --- lib/crates/fabro-llm/src/adapter_registry.rs | 2 +- lib/crates/fabro-server/src/server/automation_scheduler.rs | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/crates/fabro-llm/src/adapter_registry.rs b/lib/crates/fabro-llm/src/adapter_registry.rs index 810f6d6f4..dd880d200 100644 --- a/lib/crates/fabro-llm/src/adapter_registry.rs +++ b/lib/crates/fabro-llm/src/adapter_registry.rs @@ -367,7 +367,7 @@ mod tests { } #[test] - fn resolve_route_returns_none_for_model_from_another_catalog() { + fn resolve_route_resolves_by_id_for_model_from_another_catalog_instance() { let other = Catalog::from_builtin().unwrap(); let model = select_from_all(&other, "gpt-5.4"); assert!(resolve_route(Catalog::builtin(), model).is_some()); diff --git a/lib/crates/fabro-server/src/server/automation_scheduler.rs b/lib/crates/fabro-server/src/server/automation_scheduler.rs index 9499d4170..0abd3e988 100644 --- a/lib/crates/fabro-server/src/server/automation_scheduler.rs +++ b/lib/crates/fabro-server/src/server/automation_scheduler.rs @@ -336,6 +336,7 @@ fn run_due_schedules_once<'a>( mod tests { use fabro_api::types::RunManifest; use fabro_automation::{AutomationDraft, AutomationTarget, AutomationTrigger, ScheduleTrigger}; + use fabro_static::EnvVars; use fabro_store::ListRunsQuery; use fabro_types::RunStatus; use serde_json::json; @@ -428,7 +429,7 @@ mod tests { fn test_state_with_materializer(materializer: TestAutomationRunMaterializer) -> Arc { TestAppStateBuilder::new() .env_lookup(|_| None) - .vault_entries([("OPENAI_API_KEY", "test-openai-api-key")]) + .vault_entries([(EnvVars::OPENAI_API_KEY, "test-openai-api-key")]) .automation_materializer(materializer) .build() } From 712b7702a283b325baa77f42c2a44e4eb524b1f0 Mon Sep 17 00:00:00 2001 From: Bryan Helmkamp Date: Thu, 23 Jul 2026 13:16:09 -0400 Subject: [PATCH 07/16] fix: simplify provider-pin routing after concurrent fixes merged Both this branch and the remote qa branch fixed the same provider-pin regression; the merge stacked the two implementations. Keep the remote's semantics: pin the run's provider whenever it offers the model, otherwise fall back to priority selection. Co-Authored-By: Claude Fable 5 --- .../fabro-workflow/src/handler/llm/routing.rs | 23 ++++--------------- 1 file changed, 5 insertions(+), 18 deletions(-) diff --git a/lib/crates/fabro-workflow/src/handler/llm/routing.rs b/lib/crates/fabro-workflow/src/handler/llm/routing.rs index a838ac30a..3ce9618db 100644 --- a/lib/crates/fabro-workflow/src/handler/llm/routing.rs +++ b/lib/crates/fabro-workflow/src/handler/llm/routing.rs @@ -62,26 +62,13 @@ pub(crate) fn resolve_provider_context( .get_on_provider(default_provider_id, model) .is_some() { + // The run's selected provider is a pin whenever it offers the model. default_provider_id.clone() } else { - // The run's selected provider is a pin, matching transform-time - // semantics: a selector the pinned provider does not offer passes - // through to it as a custom model rather than re-selecting a - // higher-priority provider. - match catalog.resolve_selection( - Some(model), - Some(default_provider_id), - &catalog.all_provider_ids(), - ) { - Ok(selected) => selected.provider, - Err(fabro_model::ModelSelectionError::UnknownProvider { .. }) => { - match catalog.select(model, None, &catalog.all_provider_ids()) { - Ok(model) => model.provider.clone(), - Err(fabro_model::ModelSelectionError::UnknownSelector { .. }) => { - default_provider_id.clone() - } - Err(error) => return Err(error.into()), - } + match catalog.select(model, None, &catalog.all_provider_ids()) { + Ok(model) => model.provider.clone(), + Err(fabro_model::ModelSelectionError::UnknownSelector { .. }) => { + default_provider_id.clone() } Err(error) => return Err(error.into()), } From 1448d996e2bf74268cc969d88580768a149ca036 Mon Sep 17 00:00:00 2001 From: Release Repro Date: Thu, 23 Jul 2026 13:26:20 -0400 Subject: [PATCH 08/16] feat(llm): send x-session-id trace header with the run ID Tag every LLM request in a run with an x-session-id header carrying the run ID, so gateways that understand session tracing (e.g. OpenRouter broadcast) can group a run's requests into one session. Adds ExtraHeadersCredentialSource to fabro-auth: a CredentialSource decorator that appends fixed headers to every resolved credential, leaving operator-configured extra_headers untouched. The run pipeline wraps its vault/env source with it, so agent stages, prompt stages, hooks, and PR-content generation all pick up the header through the existing extra_headers plumbing with no fabro-llm changes. Co-Authored-By: Claude Fable 5 --- .../fabro-auth/src/extra_headers_source.rs | 164 ++++++++++++++++++ lib/crates/fabro-auth/src/lib.rs | 2 + .../fabro-workflow/src/pipeline/initialize.rs | 49 +++++- 3 files changed, 210 insertions(+), 5 deletions(-) create mode 100644 lib/crates/fabro-auth/src/extra_headers_source.rs diff --git a/lib/crates/fabro-auth/src/extra_headers_source.rs b/lib/crates/fabro-auth/src/extra_headers_source.rs new file mode 100644 index 000000000..2e57ee6ea --- /dev/null +++ b/lib/crates/fabro-auth/src/extra_headers_source.rs @@ -0,0 +1,164 @@ +use std::collections::HashMap; +use std::sync::Arc; + +use async_trait::async_trait; +use fabro_model::{Catalog, ProviderId}; + +use crate::credential_source::{CredentialSource, ResolvedCredentials}; + +/// Decorates another [`CredentialSource`] by appending fixed extra headers to +/// every credential it resolves. +/// +/// Headers already present on a credential (for example from explicit +/// provider configuration) are left untouched. +pub struct ExtraHeadersCredentialSource { + inner: Arc, + headers: HashMap, +} + +impl ExtraHeadersCredentialSource { + #[must_use] + pub fn new(inner: Arc, headers: HashMap) -> Self { + Self { inner, headers } + } +} + +#[async_trait] +impl CredentialSource for ExtraHeadersCredentialSource { + async fn resolve(&self, catalog: &Catalog) -> anyhow::Result { + let mut resolved = self.inner.resolve(catalog).await?; + for credential in &mut resolved.credentials { + for (name, value) in &self.headers { + credential + .extra_headers + .entry(name.clone()) + .or_insert_with(|| value.clone()); + } + } + Ok(resolved) + } + + async fn configured_providers(&self, catalog: &Catalog) -> Vec { + self.inner.configured_providers(catalog).await + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{ApiCredential, ResolveError}; + + struct StubSource { + credentials: Vec, + auth_issues: Vec<(ProviderId, ResolveError)>, + } + + #[async_trait] + impl CredentialSource for StubSource { + async fn resolve(&self, _catalog: &Catalog) -> anyhow::Result { + Ok(ResolvedCredentials { + credentials: self.credentials.clone(), + auth_issues: self + .auth_issues + .iter() + .map(|(provider, _)| { + ( + provider.clone(), + ResolveError::NotConfigured(provider.clone()), + ) + }) + .collect(), + }) + } + + async fn configured_providers(&self, _catalog: &Catalog) -> Vec { + self.credentials + .iter() + .map(|c| c.provider.clone()) + .collect() + } + } + + fn credential(provider: &str, extra_headers: HashMap) -> ApiCredential { + ApiCredential { + provider: ProviderId::new(provider), + auth_header: None, + extra_headers, + base_url: None, + codex_mode: false, + org_id: None, + project_id: None, + } + } + + fn catalog() -> Catalog { + Catalog::from_builtin().unwrap() + } + + #[tokio::test] + async fn appends_headers_to_every_resolved_credential() { + let source = ExtraHeadersCredentialSource::new( + Arc::new(StubSource { + credentials: vec![ + credential("anthropic", HashMap::new()), + credential("openai", HashMap::new()), + ], + auth_issues: Vec::new(), + }), + HashMap::from([("x-session-id".to_string(), "run-123".to_string())]), + ); + + let resolved = source.resolve(&catalog()).await.unwrap(); + + assert_eq!(resolved.credentials.len(), 2); + for credential in &resolved.credentials { + assert_eq!( + credential.extra_headers.get("x-session-id"), + Some(&"run-123".to_string()) + ); + } + } + + #[tokio::test] + async fn preserves_headers_already_set_on_a_credential() { + let source = ExtraHeadersCredentialSource::new( + Arc::new(StubSource { + credentials: vec![credential( + "openrouter", + HashMap::from([("x-session-id".to_string(), "configured".to_string())]), + )], + auth_issues: Vec::new(), + }), + HashMap::from([("x-session-id".to_string(), "run-123".to_string())]), + ); + + let resolved = source.resolve(&catalog()).await.unwrap(); + + assert_eq!( + resolved.credentials[0].extra_headers.get("x-session-id"), + Some(&"configured".to_string()) + ); + } + + #[tokio::test] + async fn passes_through_auth_issues_and_configured_providers() { + let provider = ProviderId::new("anthropic"); + let source = ExtraHeadersCredentialSource::new( + Arc::new(StubSource { + credentials: vec![credential("openai", HashMap::new())], + auth_issues: vec![( + provider.clone(), + ResolveError::NotConfigured(provider.clone()), + )], + }), + HashMap::from([("x-session-id".to_string(), "run-123".to_string())]), + ); + + let resolved = source.resolve(&catalog()).await.unwrap(); + assert_eq!(resolved.auth_issues.len(), 1); + assert_eq!(resolved.auth_issues[0].0, provider); + + let providers = source.configured_providers(&catalog()).await; + assert_eq!(providers, vec![ProviderId::new("openai")]); + } +} diff --git a/lib/crates/fabro-auth/src/lib.rs b/lib/crates/fabro-auth/src/lib.rs index 1bc135149..77c217317 100644 --- a/lib/crates/fabro-auth/src/lib.rs +++ b/lib/crates/fabro-auth/src/lib.rs @@ -2,6 +2,7 @@ mod context; mod credential; mod credential_source; mod env_source; +mod extra_headers_source; mod refresh; mod resolve; mod sql_vault_source; @@ -15,6 +16,7 @@ pub use context::{AuthContextRequest, AuthContextResponse}; pub use credential::{ApiKeyHeader, OAuthConfig, OAuthCredential, OAuthTokens}; pub use credential_source::{CredentialSource, ResolvedCredentials}; pub use env_source::EnvCredentialSource; +pub use extra_headers_source::ExtraHeadersCredentialSource; pub use refresh::refresh_oauth_credential; pub use resolve::{ ApiCredential, CredentialResolver, CredentialUsage, EnvLookup, ResolveError, diff --git a/lib/crates/fabro-workflow/src/pipeline/initialize.rs b/lib/crates/fabro-workflow/src/pipeline/initialize.rs index 54a6f1f45..ffe53e180 100644 --- a/lib/crates/fabro-workflow/src/pipeline/initialize.rs +++ b/lib/crates/fabro-workflow/src/pipeline/initialize.rs @@ -5,7 +5,8 @@ use std::time::Instant; use fabro_agent::{Sandbox, ToolSecrets}; use fabro_auth::{ - CredentialSource, EnvCredentialSource, VaultCredentialSource, auth_issue_message, + CredentialSource, EnvCredentialSource, ExtraHeadersCredentialSource, VaultCredentialSource, + auth_issue_message, }; use fabro_graphviz::graph; use fabro_hooks::{HookContext, HookDecision, HookEvent, HookExecutionContext, HookRunner}; @@ -260,11 +261,23 @@ fn graph_needs_api_backend(graph: &graph::Graph) -> bool { graph.nodes.values().any(routing::node_needs_api_backend) } -fn build_llm_source(vault: Option>>) -> Arc { - match vault { +/// Trace header attached to every LLM request in a run so gateways that +/// understand it (e.g. OpenRouter broadcast) can group the run's requests +/// into one session. Explicit `extra_headers` provider configuration wins. +const SESSION_ID_HEADER: &str = "x-session-id"; + +fn build_llm_source( + vault: Option>>, + run_id: fabro_types::RunId, +) -> Arc { + let inner: Arc = match vault { Some(vault) => Arc::new(VaultCredentialSource::new(vault)), None => Arc::new(EnvCredentialSource::new()), - } + }; + Arc::new(ExtraHeadersCredentialSource::new( + inner, + HashMap::from([(SESSION_ID_HEADER.to_string(), run_id.to_string())]), + )) } /// INITIALIZE phase: prepare the sandbox, env, and handlers for execution. @@ -277,7 +290,7 @@ pub async fn initialize( options.run_options.run_dir = run_dir.clone(); options.run_options.git = options.git.clone(); - let llm_source = build_llm_source(options.vault.clone()); + let llm_source = build_llm_source(options.vault.clone(), options.run_options.run_id); let tool_secrets = tool_secrets_from_configured_sources(options.vault.as_ref()).await; let catalog = Arc::clone(&options.catalog); let sandbox_git = Arc::new(SandboxGitRuntime::new()); @@ -1028,6 +1041,32 @@ mod tests { assert!(!effective_dry_run); } + #[tokio::test] + async fn build_llm_source_appends_run_session_trace_header() { + let dir = tempfile::tempdir().unwrap(); + let mut vault = Vault::load(dir.path().join("secrets.json")).unwrap(); + vault + .set( + "ANTHROPIC_API_KEY", + "anthropic-key", + SecretType::Token, + None, + ) + .unwrap(); + let vault = Arc::new(AsyncRwLock::new(vault)); + + let source = build_llm_source(Some(vault), test_run_id()); + let resolved = source.resolve(test_catalog().as_ref()).await.unwrap(); + + assert!(!resolved.credentials.is_empty()); + for credential in &resolved.credentials { + assert_eq!( + credential.extra_headers.get(SESSION_ID_HEADER), + Some(&test_run_id().to_string()) + ); + } + } + #[tokio::test] async fn initialize_executes_acp_backend_node_from_registry() { let temp = tempfile::tempdir().unwrap(); From e8db4e1a9b4b328e871b37edeb0ea117b185c9fa Mon Sep 17 00:00:00 2001 From: Release Repro Date: Thu, 23 Jul 2026 13:38:18 -0400 Subject: [PATCH 09/16] refactor(llm): simplify session trace header plumbing --- .../fabro-auth/src/extra_headers_source.rs | 99 ++++++++++--------- .../fabro-workflow/src/operations/start.rs | 1 - .../src/pipeline/execute/tests.rs | 3 - .../fabro-workflow/src/pipeline/initialize.rs | 29 +++--- .../fabro-workflow/src/pipeline/types.rs | 1 - 5 files changed, 66 insertions(+), 67 deletions(-) diff --git a/lib/crates/fabro-auth/src/extra_headers_source.rs b/lib/crates/fabro-auth/src/extra_headers_source.rs index 2e57ee6ea..29ad806cd 100644 --- a/lib/crates/fabro-auth/src/extra_headers_source.rs +++ b/lib/crates/fabro-auth/src/extra_headers_source.rs @@ -29,10 +29,14 @@ impl CredentialSource for ExtraHeadersCredentialSource { let mut resolved = self.inner.resolve(catalog).await?; for credential in &mut resolved.credentials { for (name, value) in &self.headers { - credential + if credential .extra_headers - .entry(name.clone()) - .or_insert_with(|| value.clone()); + .keys() + .any(|existing| existing.eq_ignore_ascii_case(name)) + { + continue; + } + credential.extra_headers.insert(name.clone(), value.clone()); } } Ok(resolved) @@ -49,8 +53,9 @@ mod tests { use crate::{ApiCredential, ResolveError}; struct StubSource { - credentials: Vec, - auth_issues: Vec<(ProviderId, ResolveError)>, + credentials: Vec, + auth_issue_provider: Option, + configured_providers: Vec, } #[async_trait] @@ -59,12 +64,12 @@ mod tests { Ok(ResolvedCredentials { credentials: self.credentials.clone(), auth_issues: self - .auth_issues + .auth_issue_provider .iter() - .map(|(provider, _)| { + .map(|provider| { ( provider.clone(), - ResolveError::NotConfigured(provider.clone()), + ResolveError::RefreshTokenMissing(provider.clone()), ) }) .collect(), @@ -72,16 +77,13 @@ mod tests { } async fn configured_providers(&self, _catalog: &Catalog) -> Vec { - self.credentials - .iter() - .map(|c| c.provider.clone()) - .collect() + self.configured_providers.clone() } } - fn credential(provider: &str, extra_headers: HashMap) -> ApiCredential { + fn credential(provider: ProviderId, extra_headers: HashMap) -> ApiCredential { ApiCredential { - provider: ProviderId::new(provider), + provider, auth_header: None, extra_headers, base_url: None, @@ -91,74 +93,83 @@ mod tests { } } - fn catalog() -> Catalog { - Catalog::from_builtin().unwrap() - } - #[tokio::test] async fn appends_headers_to_every_resolved_credential() { let source = ExtraHeadersCredentialSource::new( Arc::new(StubSource { - credentials: vec![ - credential("anthropic", HashMap::new()), - credential("openai", HashMap::new()), + credentials: vec![ + credential(ProviderId::anthropic(), HashMap::new()), + credential(ProviderId::openai(), HashMap::new()), ], - auth_issues: Vec::new(), + auth_issue_provider: None, + configured_providers: Vec::new(), }), HashMap::from([("x-session-id".to_string(), "run-123".to_string())]), ); - let resolved = source.resolve(&catalog()).await.unwrap(); + let resolved = source.resolve(Catalog::builtin()).await.unwrap(); assert_eq!(resolved.credentials.len(), 2); for credential in &resolved.credentials { assert_eq!( - credential.extra_headers.get("x-session-id"), - Some(&"run-123".to_string()) + credential + .extra_headers + .get("x-session-id") + .map(String::as_str), + Some("run-123") ); } } #[tokio::test] - async fn preserves_headers_already_set_on_a_credential() { + async fn preserves_case_insensitive_headers_already_set_on_a_credential() { let source = ExtraHeadersCredentialSource::new( Arc::new(StubSource { - credentials: vec![credential( - "openrouter", - HashMap::from([("x-session-id".to_string(), "configured".to_string())]), + credentials: vec![credential( + ProviderId::new("openrouter"), + HashMap::from([("X-Session-Id".to_string(), "configured".to_string())]), )], - auth_issues: Vec::new(), + auth_issue_provider: None, + configured_providers: Vec::new(), }), HashMap::from([("x-session-id".to_string(), "run-123".to_string())]), ); - let resolved = source.resolve(&catalog()).await.unwrap(); + let resolved = source.resolve(Catalog::builtin()).await.unwrap(); assert_eq!( - resolved.credentials[0].extra_headers.get("x-session-id"), - Some(&"configured".to_string()) + resolved.credentials[0] + .extra_headers + .get("X-Session-Id") + .map(String::as_str), + Some("configured") ); + assert_eq!(resolved.credentials[0].extra_headers.len(), 1); } #[tokio::test] async fn passes_through_auth_issues_and_configured_providers() { - let provider = ProviderId::new("anthropic"); + let auth_issue_provider = ProviderId::anthropic(); + let configured_provider = ProviderId::gemini(); let source = ExtraHeadersCredentialSource::new( Arc::new(StubSource { - credentials: vec![credential("openai", HashMap::new())], - auth_issues: vec![( - provider.clone(), - ResolveError::NotConfigured(provider.clone()), - )], + credentials: vec![credential(ProviderId::openai(), HashMap::new())], + auth_issue_provider: Some(auth_issue_provider.clone()), + configured_providers: vec![configured_provider.clone()], }), HashMap::from([("x-session-id".to_string(), "run-123".to_string())]), ); - let resolved = source.resolve(&catalog()).await.unwrap(); - assert_eq!(resolved.auth_issues.len(), 1); - assert_eq!(resolved.auth_issues[0].0, provider); + let resolved = source.resolve(Catalog::builtin()).await.unwrap(); + let [(reported_provider, ResolveError::RefreshTokenMissing(error_provider))] = + resolved.auth_issues.as_slice() + else { + panic!("expected the inner source's refresh-token issue"); + }; + assert_eq!(reported_provider, &auth_issue_provider); + assert_eq!(error_provider, &auth_issue_provider); - let providers = source.configured_providers(&catalog()).await; - assert_eq!(providers, vec![ProviderId::new("openai")]); + let providers = source.configured_providers(Catalog::builtin()).await; + assert_eq!(providers, vec![configured_provider]); } } diff --git a/lib/crates/fabro-workflow/src/operations/start.rs b/lib/crates/fabro-workflow/src/operations/start.rs index 7fa47e938..8d0be3d5f 100644 --- a/lib/crates/fabro-workflow/src/operations/start.rs +++ b/lib/crates/fabro-workflow/src/operations/start.rs @@ -831,7 +831,6 @@ impl RunSession { store_progress_logger.register(self.emitter.as_ref()); let init_options = InitOptions { - run_id: record.run_id, run_store: self.run_store.clone(), dry_run: run_options.dry_run_enabled(), emitter: self.emitter, diff --git a/lib/crates/fabro-workflow/src/pipeline/execute/tests.rs b/lib/crates/fabro-workflow/src/pipeline/execute/tests.rs index e2b96ed37..400050896 100644 --- a/lib/crates/fabro-workflow/src/pipeline/execute/tests.rs +++ b/lib/crates/fabro-workflow/src/pipeline/execute/tests.rs @@ -256,7 +256,6 @@ async fn execute_test_run_with_options( let initialized = initialize( persisted_workflow(graph, String::new(), &run_options.run_dir, run_id_value), InitOptions { - run_id: run_id_value, run_store: run_store.into(), dry_run: false, emitter: emitter.clone(), @@ -317,7 +316,6 @@ async fn execute_runs_start_to_exit_and_returns_final_context() { let initialized = initialize( persisted_workflow(graph, source, &run_dir, test_run_id("run-test")), InitOptions { - run_id: test_run_id("run-test"), run_store: run_store.into(), dry_run: false, emitter: test_emitter_arc("run-test"), @@ -393,7 +391,6 @@ async fn run_with_lifecycle( let initialized = initialize( persisted_workflow(graph.clone(), String::new(), &run_dir, run_id), InitOptions { - run_id, run_store: run_store.into(), dry_run: false, emitter: emitter.clone(), diff --git a/lib/crates/fabro-workflow/src/pipeline/initialize.rs b/lib/crates/fabro-workflow/src/pipeline/initialize.rs index ffe53e180..e20b2fb61 100644 --- a/lib/crates/fabro-workflow/src/pipeline/initialize.rs +++ b/lib/crates/fabro-workflow/src/pipeline/initialize.rs @@ -360,7 +360,7 @@ pub async fn initialize( let sandbox = reconnect_for_run_with_callback( instance, daytona_api_key, - Some(options.run_id), + Some(options.run_options.run_id), Some(Arc::clone(&sandbox_event_callback)), ) .await @@ -825,7 +825,6 @@ mod tests { }); let result = initialize(persisted, InitOptions { - run_id: test_run_id(), run_store: { let store = memory_store(); let inner = store.create_run(&test_run_id()).await.unwrap(); @@ -907,7 +906,6 @@ mod tests { let emitter = Arc::new(crate::event::Emitter::new(test_run_id())); let initialized = initialize(persisted, InitOptions { - run_id: test_run_id(), run_store: { let store = memory_store(); let inner = store.create_run(&test_run_id()).await.unwrap(); @@ -1043,26 +1041,24 @@ mod tests { #[tokio::test] async fn build_llm_source_appends_run_session_trace_header() { - let dir = tempfile::tempdir().unwrap(); - let mut vault = Vault::load(dir.path().join("secrets.json")).unwrap(); - vault - .set( - "ANTHROPIC_API_KEY", - "anthropic-key", - SecretType::Token, - None, - ) + let mut vault = Vault::from_entries(HashMap::new()); + fabro_auth::vault_set_token(&mut vault, EnvVars::ANTHROPIC_API_KEY, "anthropic-key") .unwrap(); let vault = Arc::new(AsyncRwLock::new(vault)); + let run_id = test_run_id(); + let expected_session_id = run_id.to_string(); - let source = build_llm_source(Some(vault), test_run_id()); + let source = build_llm_source(Some(vault), run_id); let resolved = source.resolve(test_catalog().as_ref()).await.unwrap(); assert!(!resolved.credentials.is_empty()); for credential in &resolved.credentials { assert_eq!( - credential.extra_headers.get(SESSION_ID_HEADER), - Some(&test_run_id().to_string()) + credential + .extra_headers + .get(SESSION_ID_HEADER) + .map(String::as_str), + Some(expected_session_id.as_str()) ); } } @@ -1135,7 +1131,6 @@ mod tests { let store = memory_store(); let run_store = store.create_run(&test_run_id()).await.unwrap(); let initialized = initialize(test_persisted(graph, source, &run_dir), InitOptions { - run_id: test_run_id(), run_store: run_store.into(), dry_run: false, emitter: emitter.clone(), @@ -1231,7 +1226,6 @@ mod tests { store_logger.register(&emitter); let initialized = initialize(persisted, InitOptions { - run_id: test_run_id(), run_store: run_store.into(), dry_run: false, emitter: emitter.clone(), @@ -1370,7 +1364,6 @@ mod tests { let emitter = Arc::new(crate::event::Emitter::new(test_run_id())); let result = initialize(persisted, InitOptions { - run_id: test_run_id(), run_store: { let store = memory_store(); let inner = store.create_run(&test_run_id()).await.unwrap(); diff --git a/lib/crates/fabro-workflow/src/pipeline/types.rs b/lib/crates/fabro-workflow/src/pipeline/types.rs index 8449453a8..69a03c9fe 100644 --- a/lib/crates/fabro-workflow/src/pipeline/types.rs +++ b/lib/crates/fabro-workflow/src/pipeline/types.rs @@ -249,7 +249,6 @@ pub struct SandboxEnvSpec { } pub struct InitOptions { - pub run_id: RunId, pub run_store: RunStoreHandle, pub dry_run: bool, pub emitter: Arc, From 0432c019ee3d3004866d9254d77ff2ef8e900e6b Mon Sep 17 00:00:00 2001 From: Bryan Helmkamp Date: Thu, 23 Jul 2026 13:38:55 -0400 Subject: [PATCH 10/16] chore: address review feedback on error mapping and test-support gating - Return 400 (not 500) for WorkflowError::ModelReference from run creation, matching ModelSelection: an ambiguous model/provider token is user input, not a server fault. - Gate fabro-workflow's test_support module behind cfg(any(test, feature = "test-support")) so the feature actually controls exposure, per the repo's test-support boundary guidance. Add the self dev-dependency so tests/it keeps compiling, and gate the pipeline helpers that only test_support consumed. Co-Authored-By: Claude Fable 5 --- Cargo.lock | 1 + lib/crates/fabro-server/src/server/handler/runs.rs | 2 +- lib/crates/fabro-workflow/Cargo.toml | 1 + lib/crates/fabro-workflow/src/lib.rs | 2 +- lib/crates/fabro-workflow/src/pipeline/finalize.rs | 1 + lib/crates/fabro-workflow/src/pipeline/mod.rs | 6 +++--- 6 files changed, 8 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 098d71cbf..c4bffa44c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3368,6 +3368,7 @@ dependencies = [ "fabro-util", "fabro-validate", "fabro-vault", + "fabro-workflow", "futures", "git2", "hex", diff --git a/lib/crates/fabro-server/src/server/handler/runs.rs b/lib/crates/fabro-server/src/server/handler/runs.rs index e3e7f3dd2..57562fc4c 100644 --- a/lib/crates/fabro-server/src/server/handler/runs.rs +++ b/lib/crates/fabro-server/src/server/handler/runs.rs @@ -640,7 +640,7 @@ pub(crate) async fn create_run_from_manifest( Err(WorkflowError::ValidationFailed { .. } | WorkflowError::Parse(_)) => { return ApiError::bad_request("Validation failed").into_response(); } - Err(err @ WorkflowError::ModelSelection(_)) => { + Err(err @ (WorkflowError::ModelSelection(_) | WorkflowError::ModelReference(_))) => { return ApiError::bad_request(err.to_string()).into_response(); } Err(err) => { diff --git a/lib/crates/fabro-workflow/Cargo.toml b/lib/crates/fabro-workflow/Cargo.toml index fc7166d7a..e719de2fd 100644 --- a/lib/crates/fabro-workflow/Cargo.toml +++ b/lib/crates/fabro-workflow/Cargo.toml @@ -76,6 +76,7 @@ fabro-vault = { path = "../fabro-vault" } [dev-dependencies] base64.workspace = true fabro-acp = { path = "../fabro-acp", features = ["test-support"] } +fabro-workflow = { path = ".", features = ["test-support"] } fabro-api = { path = "../fabro-api" } fabro-environment = { path = "../fabro-environment" } fabro-sandbox = { path = "../fabro-sandbox", features = ["daytona", "docker", "test-support"] } diff --git a/lib/crates/fabro-workflow/src/lib.rs b/lib/crates/fabro-workflow/src/lib.rs index d5efaaebb..b4d3cc727 100644 --- a/lib/crates/fabro-workflow/src/lib.rs +++ b/lib/crates/fabro-workflow/src/lib.rs @@ -332,7 +332,7 @@ pub mod services; mod stage_scope; pub mod static_reference; pub mod steering_hub; -#[doc(hidden)] +#[cfg(any(test, feature = "test-support"))] pub mod test_support; #[doc(hidden)] pub mod transforms; diff --git a/lib/crates/fabro-workflow/src/pipeline/finalize.rs b/lib/crates/fabro-workflow/src/pipeline/finalize.rs index 6e90991dc..9fdcad717 100644 --- a/lib/crates/fabro-workflow/src/pipeline/finalize.rs +++ b/lib/crates/fabro-workflow/src/pipeline/finalize.rs @@ -438,6 +438,7 @@ async fn compute_final_patch( (final_patch, diff_summary) } +#[cfg(any(test, feature = "test-support"))] pub(crate) fn billing_from_projection(projection: &RunProjection) -> Option { billing_rollup_from_projection(projection, None).billing_if_present() } diff --git a/lib/crates/fabro-workflow/src/pipeline/mod.rs b/lib/crates/fabro-workflow/src/pipeline/mod.rs index ef517e9cc..ac77cc992 100644 --- a/lib/crates/fabro-workflow/src/pipeline/mod.rs +++ b/lib/crates/fabro-workflow/src/pipeline/mod.rs @@ -9,9 +9,9 @@ pub(crate) mod types; mod validate; pub use execute::execute; -pub(crate) use finalize::{ - billing_from_projection, build_conclusion_from_store, build_terminal_event, -}; +pub(crate) use finalize::build_conclusion_from_store; +#[cfg(any(test, feature = "test-support"))] +pub(crate) use finalize::{billing_from_projection, build_terminal_event}; pub use finalize::{classify_engine_result, finalize, write_finalize_commit}; pub use initialize::initialize; pub use parse::parse; From 187e10879a47d650f5748d1225fa6484674401ca Mon Sep 17 00:00:00 2001 From: Bryan Helmkamp Date: Thu, 23 Jul 2026 14:18:04 -0400 Subject: [PATCH 11/16] fix(server): make generated title updates atomic --- .../fabro-server/src/server/handler/runs.rs | 21 ++---------- lib/crates/fabro-server/src/server/tests.rs | 2 +- lib/crates/fabro-store/src/slate/mod.rs | 34 +++++++++++++++++++ lib/crates/fabro-store/src/slate/run_store.rs | 27 +++++++++++++++ lib/crates/fabro-workflow/src/event.rs | 3 +- lib/crates/fabro-workflow/src/event/sink.rs | 17 +++++++++- 6 files changed, 83 insertions(+), 21 deletions(-) diff --git a/lib/crates/fabro-server/src/server/handler/runs.rs b/lib/crates/fabro-server/src/server/handler/runs.rs index 57562fc4c..614f2d719 100644 --- a/lib/crates/fabro-server/src/server/handler/runs.rs +++ b/lib/crates/fabro-server/src/server/handler/runs.rs @@ -743,23 +743,6 @@ fn spawn_generated_title_task(task: GeneratedTitleTask) { return; } - let current = match task - .state - .stores - .runs - .get_cached_summary(&task.run_id, Utc::now()) - .await - { - Ok(Some(summary)) => summary, - Ok(None) => return, - Err(err) => { - tracing::debug!(run_id = %task.run_id, error = %err, "Failed to re-read run summary for title update"); - return; - } - }; - if current.title != task.deterministic_title { - return; - } let run_store = match task.state.stores.runs.open_run(&task.run_id).await { Ok(store) => store, Err(err) => { @@ -767,7 +750,8 @@ fn spawn_generated_title_task(task: GeneratedTitleTask) { return; } }; - if let Err(err) = workflow_event::append_event( + let expected_title = task.deterministic_title; + if let Err(err) = workflow_event::append_event_if( &run_store, &task.run_id, &workflow_event::Event::RunTitleUpdated { @@ -776,6 +760,7 @@ fn spawn_generated_title_task(task: GeneratedTitleTask) { system_kind: SystemActorKind::Engine, }), }, + move |projection| projection.title().as_ref() == expected_title, ) .await { diff --git a/lib/crates/fabro-server/src/server/tests.rs b/lib/crates/fabro-server/src/server/tests.rs index 7aafcec03..c90e1d10f 100644 --- a/lib/crates/fabro-server/src/server/tests.rs +++ b/lib/crates/fabro-server/src/server/tests.rs @@ -3495,7 +3495,7 @@ async fn generated_title_does_not_overwrite_user_title_edit() { response_json!(response, StatusCode::OK).await; wait_for_mock_hits(&title_mock, 1).await; - tokio::time::sleep(std::time::Duration::from_millis(50)).await; + tokio::time::sleep(std::time::Duration::from_millis(250)).await; assert_eq!( state diff --git a/lib/crates/fabro-store/src/slate/mod.rs b/lib/crates/fabro-store/src/slate/mod.rs index 60c40d1c4..7546a741f 100644 --- a/lib/crates/fabro-store/src/slate/mod.rs +++ b/lib/crates/fabro-store/src/slate/mod.rs @@ -803,6 +803,40 @@ mod tests { assert!(matches!(err, Error::ReadOnly)); } + #[tokio::test] + async fn append_event_if_evaluates_latest_projection_before_appending() { + let (_object_store, store) = make_store(); + let run = store.create_run(&test_run_id("run-1")).await.unwrap(); + append_created(&run, "run-1", dt("2026-03-27T12:00:00Z")).await; + let initial_title = run.state().await.unwrap().title().into_owned(); + + run.append_event(&event_payload( + "run-1", + "2026-03-27T12:00:01Z", + "run.title.updated", + &serde_json::json!({ "title": "User title" }), + )) + .await + .unwrap(); + + let generated_update = event_payload( + "run-1", + "2026-03-27T12:00:02Z", + "run.title.updated", + &serde_json::json!({ "title": "Generated title" }), + ); + let appended = run + .append_event_if(&generated_update, |projection| { + projection.title() == initial_title + }) + .await + .unwrap(); + + assert_eq!(appended, None); + assert_eq!(run.state().await.unwrap().title(), "User title"); + assert_eq!(run.list_events().await.unwrap().len(), 2); + } + #[tokio::test] async fn control_request_events_set_pending_control_without_overwriting_status() { let (_object_store, store) = make_store(); diff --git a/lib/crates/fabro-store/src/slate/run_store.rs b/lib/crates/fabro-store/src/slate/run_store.rs index 2b67a066a..a1d7f8441 100644 --- a/lib/crates/fabro-store/src/slate/run_store.rs +++ b/lib/crates/fabro-store/src/slate/run_store.rs @@ -169,6 +169,10 @@ impl RunDatabase { async fn projected_state(&self) -> Result { let _state_guard = self.inner.state_lock.lock().await; + self.projected_state_locked().await + } + + async fn projected_state_locked(&self) -> Result { let next_seq = { let cache = self.inner.projection_cache.lock().await; cache.last_seq.saturating_add(1) @@ -254,12 +258,35 @@ impl RunDatabase { Ok(self.append_event_envelope(payload).await?.seq) } + /// Atomically appends `payload` when `predicate` matches the latest run + /// projection. + pub async fn append_event_if( + &self, + payload: &EventPayload, + predicate: impl FnOnce(&RunProjection) -> bool, + ) -> Result> { + if self.read_only { + return Err(Error::ReadOnly); + } + payload.validate(&self.inner.run_id)?; + let _state_guard = self.inner.state_lock.lock().await; + let projection = self.projected_state_locked().await?; + if !predicate(&projection) { + return Ok(None); + } + Ok(Some(self.append_event_envelope_locked(payload).await?.seq)) + } + pub async fn append_event_envelope(&self, payload: &EventPayload) -> Result { if self.read_only { return Err(Error::ReadOnly); } payload.validate(&self.inner.run_id)?; let _state_guard = self.inner.state_lock.lock().await; + self.append_event_envelope_locked(payload).await + } + + async fn append_event_envelope_locked(&self, payload: &EventPayload) -> Result { let seq = self.inner.event_seq.fetch_add(1, Ordering::SeqCst); let event = EventEnvelope { seq, diff --git a/lib/crates/fabro-workflow/src/event.rs b/lib/crates/fabro-workflow/src/event.rs index e64b2ae30..a5c1f583e 100644 --- a/lib/crates/fabro-workflow/src/event.rs +++ b/lib/crates/fabro-workflow/src/event.rs @@ -18,6 +18,7 @@ pub use self::redaction::{ build_redacted_event_payload, event_payload_from_redacted_json, redacted_event_json, }; pub use self::sink::{ - RunEventLogger, RunEventSink, StoreProgressLogger, append_event, append_event_to_sink, + RunEventLogger, RunEventSink, StoreProgressLogger, append_event, append_event_if, + append_event_to_sink, }; pub use crate::stage_scope::StageScope; diff --git a/lib/crates/fabro-workflow/src/event/sink.rs b/lib/crates/fabro-workflow/src/event/sink.rs index 1017b2b29..7d62b7151 100644 --- a/lib/crates/fabro-workflow/src/event/sink.rs +++ b/lib/crates/fabro-workflow/src/event/sink.rs @@ -2,7 +2,7 @@ use std::future::Future; use std::pin::Pin; use std::sync::Arc; -use ::fabro_types::{RunEvent, RunId}; +use ::fabro_types::{RunEvent, RunId, RunProjection}; use anyhow::Result; use fabro_store::RunDatabase; use tokio::io::{AsyncWrite, AsyncWriteExt}; @@ -23,6 +23,21 @@ pub async fn append_event(run_store: &RunDatabase, run_id: &RunId, event: &Event .map_err(anyhow::Error::from) } +pub async fn append_event_if( + run_store: &RunDatabase, + run_id: &RunId, + event: &Event, + predicate: impl FnOnce(&RunProjection) -> bool, +) -> Result { + let stored = to_run_event(run_id, event); + let payload = build_redacted_event_payload(&stored, run_id)?; + run_store + .append_event_if(&payload, predicate) + .await + .map(|seq| seq.is_some()) + .map_err(anyhow::Error::from) +} + pub async fn append_event_to_sink( sink: &RunEventSink, run_id: &RunId, From 5d7ecd7348ca8fe798f90d3231f9d4910f381c49 Mon Sep 17 00:00:00 2001 From: "fabro-releases[bot]" Date: Thu, 23 Jul 2026 18:35:29 +0000 Subject: [PATCH 12/16] Bump version to 0.303.0-nightly.3 --- Cargo.lock | 102 ++++++++++++++++++++++++++--------------------------- Cargo.toml | 2 +- 2 files changed, 52 insertions(+), 52 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c4bffa44c..d02be647e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2239,7 +2239,7 @@ dependencies = [ [[package]] name = "fabro-acp" -version = "0.303.0-nightly.2" +version = "0.303.0-nightly.3" dependencies = [ "agent-client-protocol", "agent-client-protocol-tokio", @@ -2258,7 +2258,7 @@ dependencies = [ [[package]] name = "fabro-agent" -version = "0.303.0-nightly.2" +version = "0.303.0-nightly.3" dependencies = [ "anyhow", "async-trait", @@ -2300,7 +2300,7 @@ dependencies = [ [[package]] name = "fabro-api" -version = "0.303.0-nightly.2" +version = "0.303.0-nightly.3" dependencies = [ "chrono", "fabro-automation", @@ -2323,7 +2323,7 @@ dependencies = [ [[package]] name = "fabro-auth" -version = "0.303.0-nightly.2" +version = "0.303.0-nightly.3" dependencies = [ "anyhow", "async-trait", @@ -2348,7 +2348,7 @@ dependencies = [ [[package]] name = "fabro-automation" -version = "0.303.0-nightly.2" +version = "0.303.0-nightly.3" dependencies = [ "anyhow", "chrono", @@ -2367,11 +2367,11 @@ dependencies = [ [[package]] name = "fabro-build-support" -version = "0.303.0-nightly.2" +version = "0.303.0-nightly.3" [[package]] name = "fabro-checkpoint" -version = "0.303.0-nightly.2" +version = "0.303.0-nightly.3" dependencies = [ "chrono", "fabro-config", @@ -2387,7 +2387,7 @@ dependencies = [ [[package]] name = "fabro-cli" -version = "0.303.0-nightly.2" +version = "0.303.0-nightly.3" dependencies = [ "anyhow", "assert_cmd", @@ -2489,7 +2489,7 @@ dependencies = [ [[package]] name = "fabro-client" -version = "0.303.0-nightly.2" +version = "0.303.0-nightly.3" dependencies = [ "anyhow", "bytes", @@ -2518,7 +2518,7 @@ dependencies = [ [[package]] name = "fabro-config" -version = "0.303.0-nightly.2" +version = "0.303.0-nightly.3" dependencies = [ "anyhow", "chrono", @@ -2547,7 +2547,7 @@ dependencies = [ [[package]] name = "fabro-core" -version = "0.303.0-nightly.2" +version = "0.303.0-nightly.3" dependencies = [ "async-trait", "fabro-types", @@ -2562,7 +2562,7 @@ dependencies = [ [[package]] name = "fabro-db" -version = "0.303.0-nightly.2" +version = "0.303.0-nightly.3" dependencies = [ "anyhow", "chrono", @@ -2574,7 +2574,7 @@ dependencies = [ [[package]] name = "fabro-dev" -version = "0.303.0-nightly.2" +version = "0.303.0-nightly.3" dependencies = [ "anyhow", "assert_cmd", @@ -2593,7 +2593,7 @@ dependencies = [ [[package]] name = "fabro-dump" -version = "0.303.0-nightly.2" +version = "0.303.0-nightly.3" dependencies = [ "anyhow", "bytes", @@ -2607,7 +2607,7 @@ dependencies = [ [[package]] name = "fabro-environment" -version = "0.303.0-nightly.2" +version = "0.303.0-nightly.3" dependencies = [ "anyhow", "chrono", @@ -2629,7 +2629,7 @@ dependencies = [ [[package]] name = "fabro-github" -version = "0.303.0-nightly.2" +version = "0.303.0-nightly.3" dependencies = [ "anyhow", "base64", @@ -2651,7 +2651,7 @@ dependencies = [ [[package]] name = "fabro-graphviz" -version = "0.303.0-nightly.2" +version = "0.303.0-nightly.3" dependencies = [ "anyhow", "fabro-types", @@ -2665,7 +2665,7 @@ dependencies = [ [[package]] name = "fabro-hooks" -version = "0.303.0-nightly.2" +version = "0.303.0-nightly.3" dependencies = [ "async-trait", "fabro-agent", @@ -2688,7 +2688,7 @@ dependencies = [ [[package]] name = "fabro-http" -version = "0.303.0-nightly.2" +version = "0.303.0-nightly.3" dependencies = [ "fabro-static", "http 1.4.0", @@ -2698,7 +2698,7 @@ dependencies = [ [[package]] name = "fabro-install" -version = "0.303.0-nightly.2" +version = "0.303.0-nightly.3" dependencies = [ "anyhow", "base64", @@ -2717,7 +2717,7 @@ dependencies = [ [[package]] name = "fabro-interview" -version = "0.303.0-nightly.2" +version = "0.303.0-nightly.3" dependencies = [ "async-trait", "dialoguer", @@ -2732,7 +2732,7 @@ dependencies = [ [[package]] name = "fabro-llm" -version = "0.303.0-nightly.2" +version = "0.303.0-nightly.3" dependencies = [ "anyhow", "async-trait", @@ -2773,7 +2773,7 @@ dependencies = [ [[package]] name = "fabro-macros" -version = "0.303.0-nightly.2" +version = "0.303.0-nightly.3" dependencies = [ "clap", "fabro-options-metadata", @@ -2784,7 +2784,7 @@ dependencies = [ [[package]] name = "fabro-manifest" -version = "0.303.0-nightly.2" +version = "0.303.0-nightly.3" dependencies = [ "anyhow", "fabro-api", @@ -2802,7 +2802,7 @@ dependencies = [ [[package]] name = "fabro-mcp" -version = "0.303.0-nightly.2" +version = "0.303.0-nightly.3" dependencies = [ "anyhow", "axum", @@ -2822,7 +2822,7 @@ dependencies = [ [[package]] name = "fabro-mcp-server" -version = "0.303.0-nightly.2" +version = "0.303.0-nightly.3" dependencies = [ "anyhow", "chrono", @@ -2849,7 +2849,7 @@ dependencies = [ [[package]] name = "fabro-mcp-store" -version = "0.303.0-nightly.2" +version = "0.303.0-nightly.3" dependencies = [ "chrono", "fabro-db", @@ -2867,7 +2867,7 @@ dependencies = [ [[package]] name = "fabro-model" -version = "0.303.0-nightly.2" +version = "0.303.0-nightly.3" dependencies = [ "fabro-static", "http 1.4.0", @@ -2883,7 +2883,7 @@ dependencies = [ [[package]] name = "fabro-oauth" -version = "0.303.0-nightly.2" +version = "0.303.0-nightly.3" dependencies = [ "anyhow", "axum", @@ -2905,7 +2905,7 @@ dependencies = [ [[package]] name = "fabro-options-metadata" -version = "0.303.0-nightly.2" +version = "0.303.0-nightly.3" dependencies = [ "serde", "serde_json", @@ -2913,7 +2913,7 @@ dependencies = [ [[package]] name = "fabro-proc" -version = "0.303.0-nightly.2" +version = "0.303.0-nightly.3" dependencies = [ "cc", "libc", @@ -2922,7 +2922,7 @@ dependencies = [ [[package]] name = "fabro-redact" -version = "0.303.0-nightly.2" +version = "0.303.0-nightly.3" dependencies = [ "aho-corasick", "ref-cast", @@ -2938,7 +2938,7 @@ dependencies = [ [[package]] name = "fabro-sandbox" -version = "0.303.0-nightly.2" +version = "0.303.0-nightly.3" dependencies = [ "anyhow", "async-trait", @@ -2983,7 +2983,7 @@ dependencies = [ [[package]] name = "fabro-server" -version = "0.303.0-nightly.2" +version = "0.303.0-nightly.3" dependencies = [ "anyhow", "async-trait", @@ -3075,7 +3075,7 @@ dependencies = [ [[package]] name = "fabro-slack" -version = "0.303.0-nightly.2" +version = "0.303.0-nightly.3" dependencies = [ "fabro-http", "fabro-interview", @@ -3097,18 +3097,18 @@ dependencies = [ [[package]] name = "fabro-spa" -version = "0.303.0-nightly.2" +version = "0.303.0-nightly.3" dependencies = [ "rust-embed", ] [[package]] name = "fabro-static" -version = "0.303.0-nightly.2" +version = "0.303.0-nightly.3" [[package]] name = "fabro-store" -version = "0.303.0-nightly.2" +version = "0.303.0-nightly.3" dependencies = [ "async-trait", "bytes", @@ -3138,7 +3138,7 @@ dependencies = [ [[package]] name = "fabro-telemetry" -version = "0.303.0-nightly.2" +version = "0.303.0-nightly.3" dependencies = [ "anyhow", "base64", @@ -3164,7 +3164,7 @@ dependencies = [ [[package]] name = "fabro-template" -version = "0.303.0-nightly.2" +version = "0.303.0-nightly.3" dependencies = [ "anyhow", "fabro-types", @@ -3178,7 +3178,7 @@ dependencies = [ [[package]] name = "fabro-test" -version = "0.303.0-nightly.2" +version = "0.303.0-nightly.3" dependencies = [ "anyhow", "assert_cmd", @@ -3203,7 +3203,7 @@ dependencies = [ [[package]] name = "fabro-tool" -version = "0.303.0-nightly.2" +version = "0.303.0-nightly.3" dependencies = [ "anyhow", "async-trait", @@ -3224,7 +3224,7 @@ dependencies = [ [[package]] name = "fabro-tracker" -version = "0.303.0-nightly.2" +version = "0.303.0-nightly.3" dependencies = [ "anyhow", "async-trait", @@ -3238,7 +3238,7 @@ dependencies = [ [[package]] name = "fabro-types" -version = "0.303.0-nightly.2" +version = "0.303.0-nightly.3" dependencies = [ "chrono", "clap", @@ -3260,7 +3260,7 @@ dependencies = [ [[package]] name = "fabro-util" -version = "0.303.0-nightly.2" +version = "0.303.0-nightly.3" dependencies = [ "anyhow", "console 0.15.11", @@ -3281,7 +3281,7 @@ dependencies = [ [[package]] name = "fabro-validate" -version = "0.303.0-nightly.2" +version = "0.303.0-nightly.3" dependencies = [ "fabro-acp", "fabro-graphviz", @@ -3294,7 +3294,7 @@ dependencies = [ [[package]] name = "fabro-variable" -version = "0.303.0-nightly.2" +version = "0.303.0-nightly.3" dependencies = [ "anyhow", "chrono", @@ -3311,7 +3311,7 @@ dependencies = [ [[package]] name = "fabro-vault" -version = "0.303.0-nightly.2" +version = "0.303.0-nightly.3" dependencies = [ "anyhow", "chrono", @@ -3330,7 +3330,7 @@ dependencies = [ [[package]] name = "fabro-workflow" -version = "0.303.0-nightly.2" +version = "0.303.0-nightly.3" dependencies = [ "anyhow", "assert_cmd", @@ -8494,7 +8494,7 @@ dependencies = [ [[package]] name = "twin-github" -version = "0.303.0-nightly.2" +version = "0.303.0-nightly.3" dependencies = [ "axum", "base64", @@ -8513,7 +8513,7 @@ dependencies = [ [[package]] name = "twin-openai" -version = "0.303.0-nightly.2" +version = "0.303.0-nightly.3" dependencies = [ "anyhow", "async-stream", diff --git a/Cargo.toml b/Cargo.toml index 5bd216ed8..32978ca47 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,7 +5,7 @@ resolver = "2" [workspace.package] edition = "2021" -version = "0.303.0-nightly.2" +version = "0.303.0-nightly.3" license = "MIT" [workspace.dependencies] From e75c3dcd37c7a2aac7709b9146f9911c1663d5c5 Mon Sep 17 00:00:00 2001 From: Bryan Helmkamp Date: Thu, 23 Jul 2026 14:58:01 -0400 Subject: [PATCH 13/16] fix(test): replay SSE events before asserting stage types --- .../fabro-server/tests/it/scenario/sse.rs | 25 ++++++++----------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/lib/crates/fabro-server/tests/it/scenario/sse.rs b/lib/crates/fabro-server/tests/it/scenario/sse.rs index cef47a545..d95f1b174 100644 --- a/lib/crates/fabro-server/tests/it/scenario/sse.rs +++ b/lib/crates/fabro-server/tests/it/scenario/sse.rs @@ -60,16 +60,17 @@ async fn sse_stream_contains_expected_event_types() { wait_for_run_status_not_in(&app, &run_id, &["runnable", "starting"]).await; - // Get SSE stream + // Replay from the beginning so the assertion does not depend on whether + // the run advances before the attach request is handled. let req = Request::builder() .method("GET") - .uri(api(&format!("/runs/{run_id}/attach"))) + .uri(api(&format!("/runs/{run_id}/attach?since_seq=1"))) .body(Body::empty()) .unwrap(); let response = checked_response( app.clone().oneshot(req).await.unwrap(), StatusCode::OK, - format!("GET /api/v1/runs/{run_id}/attach"), + format!("GET /api/v1/runs/{run_id}/attach?since_seq=1"), ) .await; @@ -103,18 +104,12 @@ async fn sse_stream_contains_expected_event_types() { } } - // Because we subscribe while the run is only guaranteed to be past - // "runnable", a live stream should include at least one stage event. - // If the run completes before we attach with no unread events, an empty - // stream is still a valid 200 response. - if !event_types.is_empty() { - assert!( - event_types - .iter() - .any(|t| t == "stage.started" || t == "stage.completed"), - "should contain stage events, got: {event_types:?}" - ); - } + assert!( + event_types + .iter() + .any(|t| t == "stage.started" || t == "stage.completed"), + "should contain stage events, got: {event_types:?}" + ); // Pipeline is complete (SSE stream ended), verify checkpoint let cp_body = wait_for_checkpoint(&app, &run_id).await; From 7ff153d2229d37c004b513397d656cc3f617c59d Mon Sep 17 00:00:00 2001 From: Bryan Helmkamp Date: Thu, 23 Jul 2026 15:21:56 -0400 Subject: [PATCH 14/16] chore: bump verify timeout to 20m --- .fabro/workflows/implement-plan/workflow.fabro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.fabro/workflows/implement-plan/workflow.fabro b/.fabro/workflows/implement-plan/workflow.fabro index 8c7ad4256..e2a59dcae 100644 --- a/.fabro/workflows/implement-plan/workflow.fabro +++ b/.fabro/workflows/implement-plan/workflow.fabro @@ -12,7 +12,7 @@ digraph ImplementPlan { implement [label="Implement", prompt="Read the plan file referenced in the goal and implement every step. Make all the code changes described in the plan. Use red/green TDD.", model="openai/gpt-5.6-sol", provider="openrouter", reasoning_effort="xhigh"] simplify_fable [label="Simplify (Claude Fable 5)", prompt="@prompts/simplify.md", model="anthropic/claude-fable-5", provider="openrouter", reasoning_effort="xhigh"] simplify_sol [label="Simplify (GPT-5.6 Sol)", prompt="@prompts/simplify.md", model="openai/gpt-5.6-sol", provider="openrouter", reasoning_effort="max"] - verify [label="Verify", shape=parallelogram, script="git fetch origin main 2>&1 && git merge --no-edit --no-stat origin/main 2>&1 && cargo +nightly-2026-04-14 fmt --all 2>&1 && cargo dev docs refresh 2>&1 && cargo +nightly-2026-04-14 fmt --check --all 2>&1 && { command -v rg >/dev/null 2>&1 || { echo 'rg is required for verify'; exit 127; }; } && ! rg -n 'AuthMode::Disabled|RunAuthMethod|RunSubjectProvenance|\bActorRef\b|\bActorKind\b|AuthenticatedSubject|AuthenticatedService|AuthorizeRunScoped|AuthorizeRunBlob|AuthorizeStageArtifact|AuthorizeCommandLog|auth_method\s*==\s*\"disabled\"' lib/crates apps lib/packages docs/public/api-reference/fabro-api.yaml 2>&1 && cargo +nightly-2026-04-14 clippy --workspace --all-targets -- -D warnings 2>&1 && cargo nextest run --workspace --status-level slow --profile ci 2>&1 && cargo dev docs check 2>&1 && bun install --frozen-lockfile 2>&1 && (cd apps/fabro-web && bun run typecheck) 2>&1 && (cd apps/fabro-web && bun run test) 2>&1 && (cd lib/packages/fabro-api-client && bun run typecheck) 2>&1 && cargo dev build -- -p fabro-cli --release 2>&1", goal_gate=true, retry_target="fixup"] + verify [label="Verify", shape=parallelogram, script="git fetch origin main 2>&1 && git merge --no-edit --no-stat origin/main 2>&1 && cargo +nightly-2026-04-14 fmt --all 2>&1 && cargo dev docs refresh 2>&1 && cargo +nightly-2026-04-14 fmt --check --all 2>&1 && { command -v rg >/dev/null 2>&1 || { echo 'rg is required for verify'; exit 127; }; } && ! rg -n 'AuthMode::Disabled|RunAuthMethod|RunSubjectProvenance|\bActorRef\b|\bActorKind\b|AuthenticatedSubject|AuthenticatedService|AuthorizeRunScoped|AuthorizeRunBlob|AuthorizeStageArtifact|AuthorizeCommandLog|auth_method\s*==\s*\"disabled\"' lib/crates apps lib/packages docs/public/api-reference/fabro-api.yaml 2>&1 && cargo +nightly-2026-04-14 clippy --workspace --all-targets -- -D warnings 2>&1 && cargo nextest run --workspace --status-level slow --profile ci 2>&1 && cargo dev docs check 2>&1 && bun install --frozen-lockfile 2>&1 && (cd apps/fabro-web && bun run typecheck) 2>&1 && (cd apps/fabro-web && bun run test) 2>&1 && (cd lib/packages/fabro-api-client && bun run typecheck) 2>&1 && cargo dev build -- -p fabro-cli --release 2>&1", timeout="20m", goal_gate=true, retry_target="fixup"] fixup [label="Fixup", prompt="The verify step failed. Read the build output from context and fix all format, clippy, Rust test, docs, TypeScript typecheck/test, and build failures.", model="anthropic/claude-fable-5", provider="openrouter", reasoning_effort="xhigh", max_visits=3] start -> toolchain From c9b53031282fa360d27e5de2ac147bb955a64fca Mon Sep 17 00:00:00 2001 From: Bryan Helmkamp Date: Thu, 23 Jul 2026 15:52:25 -0400 Subject: [PATCH 15/16] fix(workflow): flush agent events before stage completion --- .../fabro-workflow/src/handler/llm/api.rs | 58 +++++++++++++++++-- .../fabro-workflow/tests/it/integration.rs | 30 +++++++++- 2 files changed, 81 insertions(+), 7 deletions(-) diff --git a/lib/crates/fabro-workflow/src/handler/llm/api.rs b/lib/crates/fabro-workflow/src/handler/llm/api.rs index 2b0ffc1df..93ee9fed8 100644 --- a/lib/crates/fabro-workflow/src/handler/llm/api.rs +++ b/lib/crates/fabro-workflow/src/handler/llm/api.rs @@ -24,7 +24,7 @@ use fabro_model::{AgentProfileKind, Catalog, FallbackTarget, ModelRef, ProviderI use fabro_types::settings::run::RunModelControls; use fabro_types::{PermissionLevel, RunId, SessionCapability, StageId, StageTiming}; use serde::de::DeserializeOwned; -use tokio::sync::Mutex as TokioMutex; +use tokio::sync::{Mutex as TokioMutex, mpsc}; use tokio::task::JoinHandle; use tokio_util::sync::CancellationToken; @@ -532,16 +532,50 @@ fn emit_agent_tools_available( /// Spawn a task that subscribes to session events and: /// 1. Tracks file changes (write_file/edit_file tool calls) into shared state. /// 2. Forwards non-streaming agent events to the pipeline emitter. +/// +/// The returned handle exposes a per-input barrier. A successful +/// `process_input_with_runtime` emits `ProcessingEnd` after all events for +/// that input, so waiting for the barrier keeps terminal stage events from +/// overtaking queued agent events. +struct EventForwarder { + processing_end_rx: mpsc::UnboundedReceiver<()>, + task: JoinHandle<()>, +} + +impl EventForwarder { + async fn wait_for_processing_end(&mut self) { + if self.processing_end_rx.recv().await.is_none() { + tracing::warn!("Agent event forwarder stopped before processing input events"); + } + } + + fn abort(&self) { + self.task.abort(); + } +} + +impl Drop for EventForwarder { + fn drop(&mut self) { + self.task.abort(); + } +} + fn spawn_event_forwarder( session: &Session, node_id: String, scope: StageScope, emitter: Arc, file_tracking: Arc>, -) { +) -> EventForwarder { let mut rx = session.subscribe(); - tokio::spawn(async move { + let root_session_id = session.id().to_string(); + let (processing_end_tx, processing_end_rx) = mpsc::unbounded_channel(); + let task = tokio::spawn(async move { while let Ok(event) = rx.recv().await { + let is_root_processing_end = event.session_id == root_session_id + && event.parent_session_id.is_none() + && matches!(&event.event, AgentEvent::ProcessingEnd); + // Reset watchdog on every event, including streaming deltas emitter.touch(); @@ -573,8 +607,17 @@ fn spawn_event_forwarder( &scope, ); } + + if is_root_processing_end { + let _ = processing_end_tx.send(()); + } } }); + + EventForwarder { + processing_end_rx, + task, + } } /// LLM backend that delegates to an `agent` Session per invocation. @@ -1213,7 +1256,7 @@ impl CodergenBackend for AgentApiBackend { let stage_scope = StageScope::for_handler(context, &node.id); // Subscribe to session events: forward to pipeline emitter + track files. - spawn_event_forwarder( + let mut event_forwarder = spawn_event_forwarder( &session, node.id.clone(), stage_scope.clone(), @@ -1284,6 +1327,7 @@ impl CodergenBackend for AgentApiBackend { inference_duration = inference_duration.saturating_add(timing.inference); tool_duration = tool_duration.saturating_add(timing.tool); if process_result.is_ok() { + event_forwarder.wait_for_processing_end().await; total_usage += session.last_input_usage(); UsdMicros::accumulate(&mut total_cost, session.last_input_cost()); } @@ -1315,6 +1359,7 @@ impl CodergenBackend for AgentApiBackend { let mut succeeded = false; bridge.abort(); + event_forwarder.abort(); discard_session(&mut session, &mut lease, emitter); for (index, target) in self.fallback_chain.iter().enumerate() { @@ -1368,7 +1413,7 @@ impl CodergenBackend for AgentApiBackend { bridge.replace(cancel_token.clone(), &session); // Re-subscribe to forward events + track files from the new session - spawn_event_forwarder( + event_forwarder = spawn_event_forwarder( &session, node.id.clone(), stage_scope.clone(), @@ -1420,6 +1465,7 @@ impl CodergenBackend for AgentApiBackend { tool_duration = tool_duration.saturating_add(timing.tool); match process_result { Ok(()) => { + event_forwarder.wait_for_processing_end().await; total_usage += session.last_input_usage(); UsdMicros::accumulate(&mut total_cost, session.last_input_cost()); succeeded = true; @@ -1492,6 +1538,7 @@ impl CodergenBackend for AgentApiBackend { tool_duration = tool_duration.saturating_add(timing.tool); match repair_result { Ok(()) => { + event_forwarder.wait_for_processing_end().await; total_usage += session.last_input_usage(); UsdMicros::accumulate(&mut total_cost, session.last_input_cost()); repair_attempts += 1; @@ -1534,6 +1581,7 @@ impl CodergenBackend for AgentApiBackend { // Collect files_touched from the shared tracking state. let (files_touched, last_file_touched) = file_tracking_snapshot(&file_tracking); + drop(event_forwarder); if let Some(lease) = lease.take() { lease.release(); diff --git a/lib/crates/fabro-workflow/tests/it/integration.rs b/lib/crates/fabro-workflow/tests/it/integration.rs index 2b817434a..71a1e1103 100644 --- a/lib/crates/fabro-workflow/tests/it/integration.rs +++ b/lib/crates/fabro-workflow/tests/it/integration.rs @@ -2319,7 +2319,7 @@ reasoning = false ); } -#[tokio::test] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn workflow_persists_authoritative_openrouter_cost_for_agent_stage() { use fabro_auth::EnvCredentialSource; use fabro_workflow::steering_hub::SteeringHub; @@ -2398,8 +2398,18 @@ base_url = "{}" registry.register("start", Box::new(StartHandler)); registry.register("exit", Box::new(ExitHandler)); + let events = Arc::new(std::sync::Mutex::new(Vec::new())); + let events_for_listener = Arc::clone(&events); + let emitter = Arc::new(Emitter::default()); + emitter.on_event(move |event| { + if event.event_name() == "agent.message" { + std::thread::sleep(Duration::from_millis(50)); + } + events_for_listener.lock().unwrap().push(event.clone()); + }); + let dir = tempfile::tempdir().unwrap(); - let engine = WorkflowRunner::new(registry, Arc::new(Emitter::default()), local_env()); + let engine = WorkflowRunner::new(registry, emitter, local_env()); let run_options = RunOptions { settings: WorkflowSettings::default(), run_dir: dir.path().to_path_buf(), @@ -2432,6 +2442,22 @@ base_url = "{}" Some(AUTHORITATIVE_COST_USD_MICROS), "provider-reported usage.cost should override the catalog estimate" ); + + let events = events.lock().unwrap(); + let agent_message = events + .iter() + .position(|event| event.event_name() == "agent.message") + .expect("agent message should be emitted"); + let stage_completed = events + .iter() + .position(|event| { + event.event_name() == "stage.completed" && event.node_id.as_deref() == Some("work") + }) + .expect("work stage completion should be emitted"); + assert!( + agent_message < stage_completed, + "agent messages must be forwarded before terminal stage events" + ); } // --------------------------------------------------------------------------- From 8c37f5fae55624edb1f922f509a478f454edc7d5 Mon Sep 17 00:00:00 2001 From: "fabro-releases[bot]" Date: Thu, 23 Jul 2026 20:10:14 +0000 Subject: [PATCH 16/16] Bump version to 0.303.0-nightly.4 --- Cargo.lock | 102 ++++++++++++++++++++++++++--------------------------- Cargo.toml | 2 +- 2 files changed, 52 insertions(+), 52 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d02be647e..014915d5c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2239,7 +2239,7 @@ dependencies = [ [[package]] name = "fabro-acp" -version = "0.303.0-nightly.3" +version = "0.303.0-nightly.4" dependencies = [ "agent-client-protocol", "agent-client-protocol-tokio", @@ -2258,7 +2258,7 @@ dependencies = [ [[package]] name = "fabro-agent" -version = "0.303.0-nightly.3" +version = "0.303.0-nightly.4" dependencies = [ "anyhow", "async-trait", @@ -2300,7 +2300,7 @@ dependencies = [ [[package]] name = "fabro-api" -version = "0.303.0-nightly.3" +version = "0.303.0-nightly.4" dependencies = [ "chrono", "fabro-automation", @@ -2323,7 +2323,7 @@ dependencies = [ [[package]] name = "fabro-auth" -version = "0.303.0-nightly.3" +version = "0.303.0-nightly.4" dependencies = [ "anyhow", "async-trait", @@ -2348,7 +2348,7 @@ dependencies = [ [[package]] name = "fabro-automation" -version = "0.303.0-nightly.3" +version = "0.303.0-nightly.4" dependencies = [ "anyhow", "chrono", @@ -2367,11 +2367,11 @@ dependencies = [ [[package]] name = "fabro-build-support" -version = "0.303.0-nightly.3" +version = "0.303.0-nightly.4" [[package]] name = "fabro-checkpoint" -version = "0.303.0-nightly.3" +version = "0.303.0-nightly.4" dependencies = [ "chrono", "fabro-config", @@ -2387,7 +2387,7 @@ dependencies = [ [[package]] name = "fabro-cli" -version = "0.303.0-nightly.3" +version = "0.303.0-nightly.4" dependencies = [ "anyhow", "assert_cmd", @@ -2489,7 +2489,7 @@ dependencies = [ [[package]] name = "fabro-client" -version = "0.303.0-nightly.3" +version = "0.303.0-nightly.4" dependencies = [ "anyhow", "bytes", @@ -2518,7 +2518,7 @@ dependencies = [ [[package]] name = "fabro-config" -version = "0.303.0-nightly.3" +version = "0.303.0-nightly.4" dependencies = [ "anyhow", "chrono", @@ -2547,7 +2547,7 @@ dependencies = [ [[package]] name = "fabro-core" -version = "0.303.0-nightly.3" +version = "0.303.0-nightly.4" dependencies = [ "async-trait", "fabro-types", @@ -2562,7 +2562,7 @@ dependencies = [ [[package]] name = "fabro-db" -version = "0.303.0-nightly.3" +version = "0.303.0-nightly.4" dependencies = [ "anyhow", "chrono", @@ -2574,7 +2574,7 @@ dependencies = [ [[package]] name = "fabro-dev" -version = "0.303.0-nightly.3" +version = "0.303.0-nightly.4" dependencies = [ "anyhow", "assert_cmd", @@ -2593,7 +2593,7 @@ dependencies = [ [[package]] name = "fabro-dump" -version = "0.303.0-nightly.3" +version = "0.303.0-nightly.4" dependencies = [ "anyhow", "bytes", @@ -2607,7 +2607,7 @@ dependencies = [ [[package]] name = "fabro-environment" -version = "0.303.0-nightly.3" +version = "0.303.0-nightly.4" dependencies = [ "anyhow", "chrono", @@ -2629,7 +2629,7 @@ dependencies = [ [[package]] name = "fabro-github" -version = "0.303.0-nightly.3" +version = "0.303.0-nightly.4" dependencies = [ "anyhow", "base64", @@ -2651,7 +2651,7 @@ dependencies = [ [[package]] name = "fabro-graphviz" -version = "0.303.0-nightly.3" +version = "0.303.0-nightly.4" dependencies = [ "anyhow", "fabro-types", @@ -2665,7 +2665,7 @@ dependencies = [ [[package]] name = "fabro-hooks" -version = "0.303.0-nightly.3" +version = "0.303.0-nightly.4" dependencies = [ "async-trait", "fabro-agent", @@ -2688,7 +2688,7 @@ dependencies = [ [[package]] name = "fabro-http" -version = "0.303.0-nightly.3" +version = "0.303.0-nightly.4" dependencies = [ "fabro-static", "http 1.4.0", @@ -2698,7 +2698,7 @@ dependencies = [ [[package]] name = "fabro-install" -version = "0.303.0-nightly.3" +version = "0.303.0-nightly.4" dependencies = [ "anyhow", "base64", @@ -2717,7 +2717,7 @@ dependencies = [ [[package]] name = "fabro-interview" -version = "0.303.0-nightly.3" +version = "0.303.0-nightly.4" dependencies = [ "async-trait", "dialoguer", @@ -2732,7 +2732,7 @@ dependencies = [ [[package]] name = "fabro-llm" -version = "0.303.0-nightly.3" +version = "0.303.0-nightly.4" dependencies = [ "anyhow", "async-trait", @@ -2773,7 +2773,7 @@ dependencies = [ [[package]] name = "fabro-macros" -version = "0.303.0-nightly.3" +version = "0.303.0-nightly.4" dependencies = [ "clap", "fabro-options-metadata", @@ -2784,7 +2784,7 @@ dependencies = [ [[package]] name = "fabro-manifest" -version = "0.303.0-nightly.3" +version = "0.303.0-nightly.4" dependencies = [ "anyhow", "fabro-api", @@ -2802,7 +2802,7 @@ dependencies = [ [[package]] name = "fabro-mcp" -version = "0.303.0-nightly.3" +version = "0.303.0-nightly.4" dependencies = [ "anyhow", "axum", @@ -2822,7 +2822,7 @@ dependencies = [ [[package]] name = "fabro-mcp-server" -version = "0.303.0-nightly.3" +version = "0.303.0-nightly.4" dependencies = [ "anyhow", "chrono", @@ -2849,7 +2849,7 @@ dependencies = [ [[package]] name = "fabro-mcp-store" -version = "0.303.0-nightly.3" +version = "0.303.0-nightly.4" dependencies = [ "chrono", "fabro-db", @@ -2867,7 +2867,7 @@ dependencies = [ [[package]] name = "fabro-model" -version = "0.303.0-nightly.3" +version = "0.303.0-nightly.4" dependencies = [ "fabro-static", "http 1.4.0", @@ -2883,7 +2883,7 @@ dependencies = [ [[package]] name = "fabro-oauth" -version = "0.303.0-nightly.3" +version = "0.303.0-nightly.4" dependencies = [ "anyhow", "axum", @@ -2905,7 +2905,7 @@ dependencies = [ [[package]] name = "fabro-options-metadata" -version = "0.303.0-nightly.3" +version = "0.303.0-nightly.4" dependencies = [ "serde", "serde_json", @@ -2913,7 +2913,7 @@ dependencies = [ [[package]] name = "fabro-proc" -version = "0.303.0-nightly.3" +version = "0.303.0-nightly.4" dependencies = [ "cc", "libc", @@ -2922,7 +2922,7 @@ dependencies = [ [[package]] name = "fabro-redact" -version = "0.303.0-nightly.3" +version = "0.303.0-nightly.4" dependencies = [ "aho-corasick", "ref-cast", @@ -2938,7 +2938,7 @@ dependencies = [ [[package]] name = "fabro-sandbox" -version = "0.303.0-nightly.3" +version = "0.303.0-nightly.4" dependencies = [ "anyhow", "async-trait", @@ -2983,7 +2983,7 @@ dependencies = [ [[package]] name = "fabro-server" -version = "0.303.0-nightly.3" +version = "0.303.0-nightly.4" dependencies = [ "anyhow", "async-trait", @@ -3075,7 +3075,7 @@ dependencies = [ [[package]] name = "fabro-slack" -version = "0.303.0-nightly.3" +version = "0.303.0-nightly.4" dependencies = [ "fabro-http", "fabro-interview", @@ -3097,18 +3097,18 @@ dependencies = [ [[package]] name = "fabro-spa" -version = "0.303.0-nightly.3" +version = "0.303.0-nightly.4" dependencies = [ "rust-embed", ] [[package]] name = "fabro-static" -version = "0.303.0-nightly.3" +version = "0.303.0-nightly.4" [[package]] name = "fabro-store" -version = "0.303.0-nightly.3" +version = "0.303.0-nightly.4" dependencies = [ "async-trait", "bytes", @@ -3138,7 +3138,7 @@ dependencies = [ [[package]] name = "fabro-telemetry" -version = "0.303.0-nightly.3" +version = "0.303.0-nightly.4" dependencies = [ "anyhow", "base64", @@ -3164,7 +3164,7 @@ dependencies = [ [[package]] name = "fabro-template" -version = "0.303.0-nightly.3" +version = "0.303.0-nightly.4" dependencies = [ "anyhow", "fabro-types", @@ -3178,7 +3178,7 @@ dependencies = [ [[package]] name = "fabro-test" -version = "0.303.0-nightly.3" +version = "0.303.0-nightly.4" dependencies = [ "anyhow", "assert_cmd", @@ -3203,7 +3203,7 @@ dependencies = [ [[package]] name = "fabro-tool" -version = "0.303.0-nightly.3" +version = "0.303.0-nightly.4" dependencies = [ "anyhow", "async-trait", @@ -3224,7 +3224,7 @@ dependencies = [ [[package]] name = "fabro-tracker" -version = "0.303.0-nightly.3" +version = "0.303.0-nightly.4" dependencies = [ "anyhow", "async-trait", @@ -3238,7 +3238,7 @@ dependencies = [ [[package]] name = "fabro-types" -version = "0.303.0-nightly.3" +version = "0.303.0-nightly.4" dependencies = [ "chrono", "clap", @@ -3260,7 +3260,7 @@ dependencies = [ [[package]] name = "fabro-util" -version = "0.303.0-nightly.3" +version = "0.303.0-nightly.4" dependencies = [ "anyhow", "console 0.15.11", @@ -3281,7 +3281,7 @@ dependencies = [ [[package]] name = "fabro-validate" -version = "0.303.0-nightly.3" +version = "0.303.0-nightly.4" dependencies = [ "fabro-acp", "fabro-graphviz", @@ -3294,7 +3294,7 @@ dependencies = [ [[package]] name = "fabro-variable" -version = "0.303.0-nightly.3" +version = "0.303.0-nightly.4" dependencies = [ "anyhow", "chrono", @@ -3311,7 +3311,7 @@ dependencies = [ [[package]] name = "fabro-vault" -version = "0.303.0-nightly.3" +version = "0.303.0-nightly.4" dependencies = [ "anyhow", "chrono", @@ -3330,7 +3330,7 @@ dependencies = [ [[package]] name = "fabro-workflow" -version = "0.303.0-nightly.3" +version = "0.303.0-nightly.4" dependencies = [ "anyhow", "assert_cmd", @@ -8494,7 +8494,7 @@ dependencies = [ [[package]] name = "twin-github" -version = "0.303.0-nightly.3" +version = "0.303.0-nightly.4" dependencies = [ "axum", "base64", @@ -8513,7 +8513,7 @@ dependencies = [ [[package]] name = "twin-openai" -version = "0.303.0-nightly.3" +version = "0.303.0-nightly.4" dependencies = [ "anyhow", "async-stream", diff --git a/Cargo.toml b/Cargo.toml index 32978ca47..6cdec975b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,7 +5,7 @@ resolver = "2" [workspace.package] edition = "2021" -version = "0.303.0-nightly.3" +version = "0.303.0-nightly.4" license = "MIT" [workspace.dependencies]