Add guardrail tests and fix config hygiene across provider system

- Add Provider::ALL constant for iterating all variants in tests
- Add catalog guardrail tests: every provider has models, provider strings
  round-trip, as_str round-trips through from_str
- Add arc-agent guardrail tests: every default_model exists in catalog,
  profile context_window matches catalog for default models
- Fix context window drift: profiles now look up catalog instead of
  hardcoding sizes, with conservative fallbacks for unknown models
- Add #[serde(deny_unknown_fields)] to config structs so typos like
  [lmm] instead of [llm] produce parse errors
- Extract DEFAULT_BASE_URL constant in OpenAI adapter

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Bryan Helmkamp 2026-02-28 17:51:31 -05:00
parent 8adbf79b12
commit 1b26702f95
10 changed files with 127 additions and 9 deletions

View file

@ -70,7 +70,7 @@ pub enum PermissionLevel {
Full,
}
fn default_model(provider: Provider) -> &'static str {
pub fn default_model(provider: Provider) -> &'static str {
match provider {
Provider::OpenAi => "gpt-5.2-codex",
Provider::Gemini => "gemini-3.1-pro-preview",

View file

@ -154,11 +154,15 @@ in the project. Keep changes minimal and focused on the task.";
}
fn capabilities(&self) -> ProfileCapabilities {
let context_window_size = if self.model().contains("opus-4-6") {
1_000_000
} else {
200_000
};
let context_window_size = arc_llm::catalog::get_model_info(self.model())
.map(|info| info.context_window as usize)
.unwrap_or_else(|| {
if self.model().contains("opus-4-6") {
1_000_000
} else {
200_000
}
});
ProfileCapabilities {
supports_reasoning: true,
supports_streaming: true,

View file

@ -189,11 +189,14 @@ in the project.";
}
fn capabilities(&self) -> ProfileCapabilities {
let context_window_size = arc_llm::catalog::get_model_info(self.model())
.map(|info| info.context_window as usize)
.unwrap_or(1_000_000);
ProfileCapabilities {
supports_reasoning: true,
supports_streaming: true,
supports_parallel_tool_calls: true,
context_window_size: 1_000_000,
context_window_size,
}
}

View file

@ -169,11 +169,14 @@ in the project.";
}
fn capabilities(&self) -> ProfileCapabilities {
let context_window_size = arc_llm::catalog::get_model_info(self.model())
.map(|info| info.context_window as usize)
.unwrap_or(128_000);
ProfileCapabilities {
supports_reasoning: true,
supports_streaming: true,
supports_parallel_tool_calls: true,
context_window_size: 128_000,
context_window_size,
}
}

View file

@ -0,0 +1,49 @@
use arc_agent::cli::default_model;
use arc_agent::{AnthropicProfile, GeminiProfile, OpenAiProfile, ProviderProfile};
use arc_llm::catalog;
use arc_llm::provider::Provider;
#[test]
fn every_default_model_exists_in_catalog() {
for &provider in Provider::ALL {
let model = default_model(provider);
assert!(
catalog::get_model_info(model).is_some(),
"default_model for {:?} is '{}' but it is not in the catalog",
provider,
model
);
}
}
#[test]
fn profile_context_window_matches_catalog_for_default_models() {
for &provider in Provider::ALL {
let model = default_model(provider);
let catalog_info = catalog::get_model_info(model).unwrap_or_else(|| {
panic!(
"default_model '{}' for {:?} not in catalog",
model, provider
)
});
let profile: Box<dyn ProviderProfile> = match provider {
Provider::OpenAi => Box::new(OpenAiProfile::new(model)),
Provider::Kimi | Provider::Zai | Provider::Minimax | Provider::Inception => {
Box::new(OpenAiProfile::new(model).with_provider(provider))
}
Provider::Gemini => Box::new(GeminiProfile::new(model)),
Provider::Anthropic => Box::new(AnthropicProfile::new(model)),
};
assert_eq!(
profile.context_window_size(),
catalog_info.context_window as usize,
"context_window_size mismatch for {:?} model '{}': profile={} catalog={}",
provider,
model,
profile.context_window_size(),
catalog_info.context_window as usize
);
}
}

View file

@ -9,6 +9,7 @@ use crate::daytona_env::DaytonaConfig;
const SUPPORTED_VERSION: u32 = 1;
#[derive(Debug, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct TaskConfig {
pub version: u32,
pub task: String,
@ -21,18 +22,21 @@ pub struct TaskConfig {
}
#[derive(Debug, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct LlmConfig {
pub model: Option<String>,
pub provider: Option<String>,
}
#[derive(Debug, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct SetupConfig {
pub commands: Vec<String>,
pub timeout_ms: Option<u64>,
}
#[derive(Debug, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct ExecutionConfig {
pub environment: Option<String>,
pub daytona: Option<DaytonaConfig>,

View file

@ -13,6 +13,7 @@ const DEFAULT_IMAGE: &str = "ubuntu:22.04";
///
/// Doubles as the TOML deserialization target for `[execution.daytona]`.
#[derive(Clone, Debug, Default, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct DaytonaConfig {
#[serde(default)]
pub sandbox: DaytonaSandboxConfig,
@ -21,6 +22,7 @@ pub struct DaytonaConfig {
/// Sandbox-level settings (labels, auto-stop).
#[derive(Clone, Debug, Default, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct DaytonaSandboxConfig {
pub auto_stop_interval: Option<i32>,
pub labels: Option<HashMap<String, String>>,
@ -29,6 +31,7 @@ pub struct DaytonaSandboxConfig {
/// Snapshot configuration: when present, the sandbox is created from a snapshot
/// instead of a bare Docker image.
#[derive(Clone, Debug, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct DaytonaSnapshotConfig {
pub name: String,
pub cpu: Option<i32>,

View file

@ -29,6 +29,45 @@ pub fn list_models(provider: Option<&str>) -> Vec<ModelInfo> {
#[cfg(test)]
mod tests {
use super::*;
use crate::provider::Provider;
use std::str::FromStr;
#[test]
fn every_provider_has_catalog_models() {
for &provider in Provider::ALL {
let models = list_models(Some(provider.as_str()));
assert!(
!models.is_empty(),
"Provider {:?} has no models in catalog",
provider
);
}
}
#[test]
fn catalog_provider_strings_roundtrip_through_provider() {
for model in list_models(None) {
let parsed = Provider::from_str(&model.provider);
assert!(
parsed.is_ok(),
"catalog model '{}' has provider '{}' which does not parse as Provider",
model.id, model.provider
);
}
}
#[test]
fn provider_as_str_roundtrips_through_from_str() {
for &provider in Provider::ALL {
let roundtripped = Provider::from_str(provider.as_str());
assert_eq!(
roundtripped,
Ok(provider),
"Provider::{:?}.as_str() does not round-trip through from_str",
provider
);
}
}
#[test]
fn get_model_info_by_id() {

View file

@ -24,6 +24,17 @@ pub enum Provider {
}
impl Provider {
/// All known provider variants, for use in guardrail tests and iteration.
pub const ALL: &[Provider] = &[
Provider::Anthropic,
Provider::OpenAi,
Provider::Gemini,
Provider::Kimi,
Provider::Zai,
Provider::Minimax,
Provider::Inception,
];
/// Stable lowercase string representation used in `Request.provider`,
/// adapter names, and other serialization boundaries.
#[must_use]

View file

@ -11,6 +11,8 @@ use crate::types::{
Role, StreamEvent, ToolCall, ToolChoice, ToolDefinition, Usage,
};
const DEFAULT_BASE_URL: &str = "https://api.openai.com/v1";
/// Provider adapter for the `OpenAI` Responses API (`/v1/responses`).
///
/// Per spec Section 2.7, this adapter uses the Responses API (not Chat Completions)
@ -25,7 +27,7 @@ impl Adapter {
#[must_use]
pub fn new(api_key: impl Into<String>) -> Self {
Self {
http: super::http_api::HttpApi::new(api_key, "https://api.openai.com/v1"),
http: super::http_api::HttpApi::new(api_key, DEFAULT_BASE_URL),
org_id: None,
project_id: None,
}