From 1b26702f95032d094f5de509f1e767e895b36f4e Mon Sep 17 00:00:00 2001 From: Bryan Helmkamp Date: Sat, 28 Feb 2026 17:51:31 -0500 Subject: [PATCH] Add guardrail tests and fix config hygiene across provider system - Add Provider::ALL constant for iterating all variants in tests - Add catalog guardrail tests: every provider has models, provider strings round-trip, as_str round-trips through from_str - Add arc-agent guardrail tests: every default_model exists in catalog, profile context_window matches catalog for default models - Fix context window drift: profiles now look up catalog instead of hardcoding sizes, with conservative fallbacks for unknown models - Add #[serde(deny_unknown_fields)] to config structs so typos like [lmm] instead of [llm] produce parse errors - Extract DEFAULT_BASE_URL constant in OpenAI adapter Co-Authored-By: Claude Opus 4.6 --- crates/arc-agent/src/cli.rs | 2 +- crates/arc-agent/src/profiles/anthropic.rs | 14 +++--- crates/arc-agent/src/profiles/gemini.rs | 5 ++- crates/arc-agent/src/profiles/openai.rs | 5 ++- crates/arc-agent/tests/guardrails.rs | 49 +++++++++++++++++++++ crates/arc-attractor/src/cli/task_config.rs | 4 ++ crates/arc-attractor/src/daytona_env.rs | 3 ++ crates/arc-llm/src/catalog.rs | 39 ++++++++++++++++ crates/arc-llm/src/provider.rs | 11 +++++ crates/arc-llm/src/providers/openai.rs | 4 +- 10 files changed, 127 insertions(+), 9 deletions(-) create mode 100644 crates/arc-agent/tests/guardrails.rs diff --git a/crates/arc-agent/src/cli.rs b/crates/arc-agent/src/cli.rs index 29e22020b..bd5e5d0b0 100644 --- a/crates/arc-agent/src/cli.rs +++ b/crates/arc-agent/src/cli.rs @@ -70,7 +70,7 @@ pub enum PermissionLevel { Full, } -fn default_model(provider: Provider) -> &'static str { +pub fn default_model(provider: Provider) -> &'static str { match provider { Provider::OpenAi => "gpt-5.2-codex", Provider::Gemini => "gemini-3.1-pro-preview", diff --git a/crates/arc-agent/src/profiles/anthropic.rs b/crates/arc-agent/src/profiles/anthropic.rs index 13614fde2..d9c117496 100644 --- a/crates/arc-agent/src/profiles/anthropic.rs +++ b/crates/arc-agent/src/profiles/anthropic.rs @@ -154,11 +154,15 @@ in the project. Keep changes minimal and focused on the task."; } fn capabilities(&self) -> ProfileCapabilities { - let context_window_size = if self.model().contains("opus-4-6") { - 1_000_000 - } else { - 200_000 - }; + let context_window_size = arc_llm::catalog::get_model_info(self.model()) + .map(|info| info.context_window as usize) + .unwrap_or_else(|| { + if self.model().contains("opus-4-6") { + 1_000_000 + } else { + 200_000 + } + }); ProfileCapabilities { supports_reasoning: true, supports_streaming: true, diff --git a/crates/arc-agent/src/profiles/gemini.rs b/crates/arc-agent/src/profiles/gemini.rs index 428905bcd..fbeb4eafc 100644 --- a/crates/arc-agent/src/profiles/gemini.rs +++ b/crates/arc-agent/src/profiles/gemini.rs @@ -189,11 +189,14 @@ in the project."; } fn capabilities(&self) -> ProfileCapabilities { + let context_window_size = arc_llm::catalog::get_model_info(self.model()) + .map(|info| info.context_window as usize) + .unwrap_or(1_000_000); ProfileCapabilities { supports_reasoning: true, supports_streaming: true, supports_parallel_tool_calls: true, - context_window_size: 1_000_000, + context_window_size, } } diff --git a/crates/arc-agent/src/profiles/openai.rs b/crates/arc-agent/src/profiles/openai.rs index 7f455cb1c..451f652f3 100644 --- a/crates/arc-agent/src/profiles/openai.rs +++ b/crates/arc-agent/src/profiles/openai.rs @@ -169,11 +169,14 @@ in the project."; } fn capabilities(&self) -> ProfileCapabilities { + let context_window_size = arc_llm::catalog::get_model_info(self.model()) + .map(|info| info.context_window as usize) + .unwrap_or(128_000); ProfileCapabilities { supports_reasoning: true, supports_streaming: true, supports_parallel_tool_calls: true, - context_window_size: 128_000, + context_window_size, } } diff --git a/crates/arc-agent/tests/guardrails.rs b/crates/arc-agent/tests/guardrails.rs new file mode 100644 index 000000000..c011aca33 --- /dev/null +++ b/crates/arc-agent/tests/guardrails.rs @@ -0,0 +1,49 @@ +use arc_agent::cli::default_model; +use arc_agent::{AnthropicProfile, GeminiProfile, OpenAiProfile, ProviderProfile}; +use arc_llm::catalog; +use arc_llm::provider::Provider; + +#[test] +fn every_default_model_exists_in_catalog() { + for &provider in Provider::ALL { + let model = default_model(provider); + assert!( + catalog::get_model_info(model).is_some(), + "default_model for {:?} is '{}' but it is not in the catalog", + provider, + model + ); + } +} + +#[test] +fn profile_context_window_matches_catalog_for_default_models() { + for &provider in Provider::ALL { + let model = default_model(provider); + let catalog_info = catalog::get_model_info(model).unwrap_or_else(|| { + panic!( + "default_model '{}' for {:?} not in catalog", + model, provider + ) + }); + + let profile: Box = match provider { + Provider::OpenAi => Box::new(OpenAiProfile::new(model)), + Provider::Kimi | Provider::Zai | Provider::Minimax | Provider::Inception => { + Box::new(OpenAiProfile::new(model).with_provider(provider)) + } + Provider::Gemini => Box::new(GeminiProfile::new(model)), + Provider::Anthropic => Box::new(AnthropicProfile::new(model)), + }; + + assert_eq!( + profile.context_window_size(), + catalog_info.context_window as usize, + "context_window_size mismatch for {:?} model '{}': profile={} catalog={}", + provider, + model, + profile.context_window_size(), + catalog_info.context_window as usize + ); + } +} diff --git a/crates/arc-attractor/src/cli/task_config.rs b/crates/arc-attractor/src/cli/task_config.rs index 02ec27b29..65260be7c 100644 --- a/crates/arc-attractor/src/cli/task_config.rs +++ b/crates/arc-attractor/src/cli/task_config.rs @@ -9,6 +9,7 @@ use crate::daytona_env::DaytonaConfig; const SUPPORTED_VERSION: u32 = 1; #[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] pub struct TaskConfig { pub version: u32, pub task: String, @@ -21,18 +22,21 @@ pub struct TaskConfig { } #[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] pub struct LlmConfig { pub model: Option, pub provider: Option, } #[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] pub struct SetupConfig { pub commands: Vec, pub timeout_ms: Option, } #[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] pub struct ExecutionConfig { pub environment: Option, pub daytona: Option, diff --git a/crates/arc-attractor/src/daytona_env.rs b/crates/arc-attractor/src/daytona_env.rs index b6133ea69..c363fe56e 100644 --- a/crates/arc-attractor/src/daytona_env.rs +++ b/crates/arc-attractor/src/daytona_env.rs @@ -13,6 +13,7 @@ const DEFAULT_IMAGE: &str = "ubuntu:22.04"; /// /// Doubles as the TOML deserialization target for `[execution.daytona]`. #[derive(Clone, Debug, Default, Deserialize)] +#[serde(deny_unknown_fields)] pub struct DaytonaConfig { #[serde(default)] pub sandbox: DaytonaSandboxConfig, @@ -21,6 +22,7 @@ pub struct DaytonaConfig { /// Sandbox-level settings (labels, auto-stop). #[derive(Clone, Debug, Default, Deserialize)] +#[serde(deny_unknown_fields)] pub struct DaytonaSandboxConfig { pub auto_stop_interval: Option, pub labels: Option>, @@ -29,6 +31,7 @@ pub struct DaytonaSandboxConfig { /// Snapshot configuration: when present, the sandbox is created from a snapshot /// instead of a bare Docker image. #[derive(Clone, Debug, Deserialize)] +#[serde(deny_unknown_fields)] pub struct DaytonaSnapshotConfig { pub name: String, pub cpu: Option, diff --git a/crates/arc-llm/src/catalog.rs b/crates/arc-llm/src/catalog.rs index 89514d382..4aedec1a4 100644 --- a/crates/arc-llm/src/catalog.rs +++ b/crates/arc-llm/src/catalog.rs @@ -29,6 +29,45 @@ pub fn list_models(provider: Option<&str>) -> Vec { #[cfg(test)] mod tests { use super::*; + use crate::provider::Provider; + use std::str::FromStr; + + #[test] + fn every_provider_has_catalog_models() { + for &provider in Provider::ALL { + let models = list_models(Some(provider.as_str())); + assert!( + !models.is_empty(), + "Provider {:?} has no models in catalog", + provider + ); + } + } + + #[test] + fn catalog_provider_strings_roundtrip_through_provider() { + for model in list_models(None) { + let parsed = Provider::from_str(&model.provider); + assert!( + parsed.is_ok(), + "catalog model '{}' has provider '{}' which does not parse as Provider", + model.id, model.provider + ); + } + } + + #[test] + fn provider_as_str_roundtrips_through_from_str() { + for &provider in Provider::ALL { + let roundtripped = Provider::from_str(provider.as_str()); + assert_eq!( + roundtripped, + Ok(provider), + "Provider::{:?}.as_str() does not round-trip through from_str", + provider + ); + } + } #[test] fn get_model_info_by_id() { diff --git a/crates/arc-llm/src/provider.rs b/crates/arc-llm/src/provider.rs index 9bf84d711..5f43ae386 100644 --- a/crates/arc-llm/src/provider.rs +++ b/crates/arc-llm/src/provider.rs @@ -24,6 +24,17 @@ pub enum Provider { } impl Provider { + /// All known provider variants, for use in guardrail tests and iteration. + pub const ALL: &[Provider] = &[ + Provider::Anthropic, + Provider::OpenAi, + Provider::Gemini, + Provider::Kimi, + Provider::Zai, + Provider::Minimax, + Provider::Inception, + ]; + /// Stable lowercase string representation used in `Request.provider`, /// adapter names, and other serialization boundaries. #[must_use] diff --git a/crates/arc-llm/src/providers/openai.rs b/crates/arc-llm/src/providers/openai.rs index ebe003e77..a524675d2 100644 --- a/crates/arc-llm/src/providers/openai.rs +++ b/crates/arc-llm/src/providers/openai.rs @@ -11,6 +11,8 @@ use crate::types::{ Role, StreamEvent, ToolCall, ToolChoice, ToolDefinition, Usage, }; +const DEFAULT_BASE_URL: &str = "https://api.openai.com/v1"; + /// Provider adapter for the `OpenAI` Responses API (`/v1/responses`). /// /// Per spec Section 2.7, this adapter uses the Responses API (not Chat Completions) @@ -25,7 +27,7 @@ impl Adapter { #[must_use] pub fn new(api_key: impl Into) -> Self { Self { - http: super::http_api::HttpApi::new(api_key, "https://api.openai.com/v1"), + http: super::http_api::HttpApi::new(api_key, DEFAULT_BASE_URL), org_id: None, project_id: None, }