mirror of
https://github.com/fabro-sh/fabro.git
synced 2026-09-06 08:18:58 +00:00
Add guardrail tests and fix config hygiene across provider system
- Add Provider::ALL constant for iterating all variants in tests - Add catalog guardrail tests: every provider has models, provider strings round-trip, as_str round-trips through from_str - Add arc-agent guardrail tests: every default_model exists in catalog, profile context_window matches catalog for default models - Fix context window drift: profiles now look up catalog instead of hardcoding sizes, with conservative fallbacks for unknown models - Add #[serde(deny_unknown_fields)] to config structs so typos like [lmm] instead of [llm] produce parse errors - Extract DEFAULT_BASE_URL constant in OpenAI adapter Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
parent
8adbf79b12
commit
1b26702f95
10 changed files with 127 additions and 9 deletions
|
|
@ -70,7 +70,7 @@ pub enum PermissionLevel {
|
|||
Full,
|
||||
}
|
||||
|
||||
fn default_model(provider: Provider) -> &'static str {
|
||||
pub fn default_model(provider: Provider) -> &'static str {
|
||||
match provider {
|
||||
Provider::OpenAi => "gpt-5.2-codex",
|
||||
Provider::Gemini => "gemini-3.1-pro-preview",
|
||||
|
|
|
|||
|
|
@ -154,11 +154,15 @@ in the project. Keep changes minimal and focused on the task.";
|
|||
}
|
||||
|
||||
fn capabilities(&self) -> ProfileCapabilities {
|
||||
let context_window_size = if self.model().contains("opus-4-6") {
|
||||
1_000_000
|
||||
} else {
|
||||
200_000
|
||||
};
|
||||
let context_window_size = arc_llm::catalog::get_model_info(self.model())
|
||||
.map(|info| info.context_window as usize)
|
||||
.unwrap_or_else(|| {
|
||||
if self.model().contains("opus-4-6") {
|
||||
1_000_000
|
||||
} else {
|
||||
200_000
|
||||
}
|
||||
});
|
||||
ProfileCapabilities {
|
||||
supports_reasoning: true,
|
||||
supports_streaming: true,
|
||||
|
|
|
|||
|
|
@ -189,11 +189,14 @@ in the project.";
|
|||
}
|
||||
|
||||
fn capabilities(&self) -> ProfileCapabilities {
|
||||
let context_window_size = arc_llm::catalog::get_model_info(self.model())
|
||||
.map(|info| info.context_window as usize)
|
||||
.unwrap_or(1_000_000);
|
||||
ProfileCapabilities {
|
||||
supports_reasoning: true,
|
||||
supports_streaming: true,
|
||||
supports_parallel_tool_calls: true,
|
||||
context_window_size: 1_000_000,
|
||||
context_window_size,
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -169,11 +169,14 @@ in the project.";
|
|||
}
|
||||
|
||||
fn capabilities(&self) -> ProfileCapabilities {
|
||||
let context_window_size = arc_llm::catalog::get_model_info(self.model())
|
||||
.map(|info| info.context_window as usize)
|
||||
.unwrap_or(128_000);
|
||||
ProfileCapabilities {
|
||||
supports_reasoning: true,
|
||||
supports_streaming: true,
|
||||
supports_parallel_tool_calls: true,
|
||||
context_window_size: 128_000,
|
||||
context_window_size,
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
49
crates/arc-agent/tests/guardrails.rs
Normal file
49
crates/arc-agent/tests/guardrails.rs
Normal file
|
|
@ -0,0 +1,49 @@
|
|||
use arc_agent::cli::default_model;
|
||||
use arc_agent::{AnthropicProfile, GeminiProfile, OpenAiProfile, ProviderProfile};
|
||||
use arc_llm::catalog;
|
||||
use arc_llm::provider::Provider;
|
||||
|
||||
#[test]
|
||||
fn every_default_model_exists_in_catalog() {
|
||||
for &provider in Provider::ALL {
|
||||
let model = default_model(provider);
|
||||
assert!(
|
||||
catalog::get_model_info(model).is_some(),
|
||||
"default_model for {:?} is '{}' but it is not in the catalog",
|
||||
provider,
|
||||
model
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn profile_context_window_matches_catalog_for_default_models() {
|
||||
for &provider in Provider::ALL {
|
||||
let model = default_model(provider);
|
||||
let catalog_info = catalog::get_model_info(model).unwrap_or_else(|| {
|
||||
panic!(
|
||||
"default_model '{}' for {:?} not in catalog",
|
||||
model, provider
|
||||
)
|
||||
});
|
||||
|
||||
let profile: Box<dyn ProviderProfile> = match provider {
|
||||
Provider::OpenAi => Box::new(OpenAiProfile::new(model)),
|
||||
Provider::Kimi | Provider::Zai | Provider::Minimax | Provider::Inception => {
|
||||
Box::new(OpenAiProfile::new(model).with_provider(provider))
|
||||
}
|
||||
Provider::Gemini => Box::new(GeminiProfile::new(model)),
|
||||
Provider::Anthropic => Box::new(AnthropicProfile::new(model)),
|
||||
};
|
||||
|
||||
assert_eq!(
|
||||
profile.context_window_size(),
|
||||
catalog_info.context_window as usize,
|
||||
"context_window_size mismatch for {:?} model '{}': profile={} catalog={}",
|
||||
provider,
|
||||
model,
|
||||
profile.context_window_size(),
|
||||
catalog_info.context_window as usize
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
@ -9,6 +9,7 @@ use crate::daytona_env::DaytonaConfig;
|
|||
const SUPPORTED_VERSION: u32 = 1;
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct TaskConfig {
|
||||
pub version: u32,
|
||||
pub task: String,
|
||||
|
|
@ -21,18 +22,21 @@ pub struct TaskConfig {
|
|||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct LlmConfig {
|
||||
pub model: Option<String>,
|
||||
pub provider: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct SetupConfig {
|
||||
pub commands: Vec<String>,
|
||||
pub timeout_ms: Option<u64>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct ExecutionConfig {
|
||||
pub environment: Option<String>,
|
||||
pub daytona: Option<DaytonaConfig>,
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ const DEFAULT_IMAGE: &str = "ubuntu:22.04";
|
|||
///
|
||||
/// Doubles as the TOML deserialization target for `[execution.daytona]`.
|
||||
#[derive(Clone, Debug, Default, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct DaytonaConfig {
|
||||
#[serde(default)]
|
||||
pub sandbox: DaytonaSandboxConfig,
|
||||
|
|
@ -21,6 +22,7 @@ pub struct DaytonaConfig {
|
|||
|
||||
/// Sandbox-level settings (labels, auto-stop).
|
||||
#[derive(Clone, Debug, Default, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct DaytonaSandboxConfig {
|
||||
pub auto_stop_interval: Option<i32>,
|
||||
pub labels: Option<HashMap<String, String>>,
|
||||
|
|
@ -29,6 +31,7 @@ pub struct DaytonaSandboxConfig {
|
|||
/// Snapshot configuration: when present, the sandbox is created from a snapshot
|
||||
/// instead of a bare Docker image.
|
||||
#[derive(Clone, Debug, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct DaytonaSnapshotConfig {
|
||||
pub name: String,
|
||||
pub cpu: Option<i32>,
|
||||
|
|
|
|||
|
|
@ -29,6 +29,45 @@ pub fn list_models(provider: Option<&str>) -> Vec<ModelInfo> {
|
|||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::provider::Provider;
|
||||
use std::str::FromStr;
|
||||
|
||||
#[test]
|
||||
fn every_provider_has_catalog_models() {
|
||||
for &provider in Provider::ALL {
|
||||
let models = list_models(Some(provider.as_str()));
|
||||
assert!(
|
||||
!models.is_empty(),
|
||||
"Provider {:?} has no models in catalog",
|
||||
provider
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn catalog_provider_strings_roundtrip_through_provider() {
|
||||
for model in list_models(None) {
|
||||
let parsed = Provider::from_str(&model.provider);
|
||||
assert!(
|
||||
parsed.is_ok(),
|
||||
"catalog model '{}' has provider '{}' which does not parse as Provider",
|
||||
model.id, model.provider
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn provider_as_str_roundtrips_through_from_str() {
|
||||
for &provider in Provider::ALL {
|
||||
let roundtripped = Provider::from_str(provider.as_str());
|
||||
assert_eq!(
|
||||
roundtripped,
|
||||
Ok(provider),
|
||||
"Provider::{:?}.as_str() does not round-trip through from_str",
|
||||
provider
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn get_model_info_by_id() {
|
||||
|
|
|
|||
|
|
@ -24,6 +24,17 @@ pub enum Provider {
|
|||
}
|
||||
|
||||
impl Provider {
|
||||
/// All known provider variants, for use in guardrail tests and iteration.
|
||||
pub const ALL: &[Provider] = &[
|
||||
Provider::Anthropic,
|
||||
Provider::OpenAi,
|
||||
Provider::Gemini,
|
||||
Provider::Kimi,
|
||||
Provider::Zai,
|
||||
Provider::Minimax,
|
||||
Provider::Inception,
|
||||
];
|
||||
|
||||
/// Stable lowercase string representation used in `Request.provider`,
|
||||
/// adapter names, and other serialization boundaries.
|
||||
#[must_use]
|
||||
|
|
|
|||
|
|
@ -11,6 +11,8 @@ use crate::types::{
|
|||
Role, StreamEvent, ToolCall, ToolChoice, ToolDefinition, Usage,
|
||||
};
|
||||
|
||||
const DEFAULT_BASE_URL: &str = "https://api.openai.com/v1";
|
||||
|
||||
/// Provider adapter for the `OpenAI` Responses API (`/v1/responses`).
|
||||
///
|
||||
/// Per spec Section 2.7, this adapter uses the Responses API (not Chat Completions)
|
||||
|
|
@ -25,7 +27,7 @@ impl Adapter {
|
|||
#[must_use]
|
||||
pub fn new(api_key: impl Into<String>) -> Self {
|
||||
Self {
|
||||
http: super::http_api::HttpApi::new(api_key, "https://api.openai.com/v1"),
|
||||
http: super::http_api::HttpApi::new(api_key, DEFAULT_BASE_URL),
|
||||
org_id: None,
|
||||
project_id: None,
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue