Add node-level model validation + missing catalog aliases (#110)

This PR addresses two related issues that caused `fabro run` to fail at
runtime with an unhelpful LLM error when using model names like
`gpt-54`: the model catalog was missing hyphenated aliases, and the
validator had no rule to catch unknown model names on workflow nodes.

The catalog changes add `gpt-54`, `gpt-54-pro`, and `gpt-54-mini` as
recognized aliases alongside the existing `gpt54`-style variants for the
`gpt-5.4` model family. This makes the hyphenated form a valid input
that resolves correctly at both validation and runtime, and the updated
`fabro model list` output and insta snapshots reflect the expanded alias
columns. Three new alias-resolution unit tests confirm each mapping.

On the validation side, a new `NodeModelKnownRule` mirrors the existing
`StylesheetModelKnownRule` but iterates over graph nodes instead of
stylesheet declarations, emitting `Severity::Warning` diagnostics with
the relevant `node_id` when a node's `model` or `provider` attribute
doesn't match anything in the catalog. To avoid duplication, the inline
validation logic in `StylesheetModelKnownRule` was refactored into
shared `check_model_known` and `check_provider_known` helpers that both
rules now call. Five unit tests cover the valid-model, unknown-model,
alias, unknown-provider, and no-attribute cases. An unrelated dead-code
helper in `git.rs` was also removed as part of the cleanup.

### Fabro Details

<details>
<summary>Ran 11 stages in 12m 36s for $2.85</summary>

| Stage | Duration | Cost | Retries |
|---|---|---|---|
| start | 0s | – | 0 |
| toolchain | 0s | – | 0 |
| preflight_compile | 58s | – | 0 |
| preflight_lint | 10s | – | 0 |
| implement | 2m 56s | $1.13 | 0 |
| simplify_opus | 4m 12s | $1.13 | 0 |
| simplify_gpt | 0s | – | 0 |
| verify | 10s | – | 0 |
| fixup | 2m 27s | $0.59 | 0 |
| verify | 10s | – | 0 |
| fmt | 0s | – | 0 |
| **Total** | **12m 36s** | **$2.85** | **0** |

</details>

<details>
<summary>Ran <code>ImplementAndSimplify.fabro</code> (12 nodes and 15
edges)</summary>

```dot
digraph ImplementAndSimplify {
    graph [
        goal="Implement and simplify",
        model_stylesheet="
            * { backend: api; model: claude-opus-4-6;}
        "
    ]
    rankdir=LR

    start [shape=Mdiamond, label="Start"]
    exit  [shape=Msquare, label="Exit"]

    toolchain         [label="Toolchain", shape=parallelogram, script="command -v cargo >/dev/null || { curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y && sudo ln -sf $HOME/.cargo/bin/* /usr/local/bin/; }; cargo --version 2>&1", max_retries=0]
    preflight_compile [label="Preflight Compile", shape=parallelogram, script="cargo check -q --workspace 2>&1", max_retries=0]
    preflight_lint    [label="Preflight Lint", shape=parallelogram, script="cargo clippy -q --workspace -- -D warnings 2>&1", max_retries=0]
    fix_lints         [label="Fix Lints", prompt="The preflight lint step failed. Read the build output from context and fix all clippy lint warnings.", max_visits=3]
    implement         [label="Implement", prompt="Read the plan file referenced in the goal and implement every step. Make all the code changes described in the plan. Use red/green TDD."]
    simplify_opus     [label="Simplify (Opus)", prompt="@prompts/simplify.md"]
    simplify_gpt      [label="Simplify (GPT-54)", prompt="@prompts/simplify.md", model="gpt-54"]
    verify            [label="Verify", shape=parallelogram, script="cargo clippy -q --workspace -- -D warnings 2>&1 && cargo nextest run --cargo-quiet --workspace --status-level fail 2>&1", goal_gate=true, retry_target="fixup"]
    fixup             [label="Fixup", prompt="The verify step failed. Read the build output from context and fix all clippy lint warnings and test failures.", max_visits=3]
    fmt               [label="Format", shape=parallelogram, script="cargo fmt --all 2>&1", max_retries=0]

    start -> toolchain
    toolchain -> preflight_compile [condition="outcome=success"]
    toolchain -> exit
    preflight_compile -> preflight_lint [condition="outcome=success"]
    preflight_compile -> exit
    preflight_lint -> implement [condition="outcome=success"]
    preflight_lint -> fix_lints
    fix_lints -> preflight_lint
    implement -> simplify_opus -> simplify_gpt -> verify
    verify -> fmt   [condition="outcome=success"]
    verify -> fixup
    fixup -> verify
    fmt -> exit
}

```

</details>

⚒️ Generated with [Fabro](https://fabro.sh)

---------

Co-authored-by: Fabro <noreply@fabro.sh>
Co-authored-by: Bryan Helmkamp <bryan@brynary.com>
Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
brynary-fabro[bot] 2026-03-19 22:48:08 -04:00 committed by GitHub
parent 4a5f9f698f
commit 69ef57cd4a
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
6 changed files with 220 additions and 120 deletions

View file

@ -1,25 +1,25 @@
```console
$ fabro model
 MODEL   PROVIDER   ALIASES   CONTEXT   COST   SPEED 
 claude-opus-4-6   anthropic  opus, claude-opus    1m   $15.0 / $75.0   25 tok/s 
 claude-sonnet-4-5   anthropic      200k   $3.0 / $15.0   50 tok/s 
 claude-sonnet-4-6   anthropic  sonnet, claude-sonnet   200k   $3.0 / $15.0   50 tok/s 
 claude-haiku-4-5   anthropic  haiku, claude-haiku    200k   $0.8 / $4.0   100 tok/s 
 gpt-5.2   openai   gpt5    1m   $1.8 / $14.0   65 tok/s 
 gpt-5-mini   openai   gpt5-mini    1m   $0.2 / $2.0   70 tok/s 
 gpt-5.2-codex   openai       1m   $1.8 / $14.0   100 tok/s 
 gpt-5.3-codex   openai   codex    1m   $1.8 / $14.0   100 tok/s 
 gpt-5.3-codex-spark   openai   codex-spark    131k   - / -  1000 tok/s 
 gpt-5.4   openai   gpt54    1m   $2.5 / $15.0   70 tok/s 
 gpt-5.4-pro   openai   gpt54-pro    1m  $30.0 / $180.0   20 tok/s 
 gpt-5.4-mini   openai   gpt54-mini    400k   $0.8 / $4.5   140 tok/s 
 gemini-3.1-pro-preview   gemini   gemini-pro    1m   $2.0 / $12.0   85 tok/s 
 gemini-3.1-pro-preview-customtools  gemini   gemini-customtools    1m   $2.0 / $12.0   85 tok/s 
 gemini-3-flash-preview   gemini   gemini-flash    1m   $0.5 / $3.0   150 tok/s 
 gemini-3.1-flash-lite-preview   gemini   gemini-flash-lite    1m   $0.2 / $1.5   200 tok/s 
 kimi-k2.5   kimi   kimi    262k   $0.6 / $3.0   50 tok/s 
 glm-4.7   zai   glm, glm4    203k   $0.6 / $2.2   100 tok/s 
 minimax-m2.5   minimax   minimax    197k   $0.3 / $1.2   45 tok/s 
 mercury-2   inception  mercury    131k   $0.2 / $0.8  1000 tok/s 
 MODEL   PROVIDER   ALIASES   CONTEXT   COST   SPEED 
 claude-opus-4-6   anthropic  opus, claude-opus    1m   $15.0 / $75.0   25 tok/s 
 claude-sonnet-4-5   anthropic      200k   $3.0 / $15.0   50 tok/s 
 claude-sonnet-4-6   anthropic  sonnet, claude-sonnet    200k   $3.0 / $15.0   50 tok/s 
 claude-haiku-4-5   anthropic  haiku, claude-haiku    200k   $0.8 / $4.0   100 tok/s 
 gpt-5.2   openai   gpt5    1m   $1.8 / $14.0   65 tok/s 
 gpt-5-mini   openai   gpt5-mini    1m   $0.2 / $2.0   70 tok/s 
 gpt-5.2-codex   openai       1m   $1.8 / $14.0   100 tok/s 
 gpt-5.3-codex   openai   codex    1m   $1.8 / $14.0   100 tok/s 
 gpt-5.3-codex-spark   openai   codex-spark    131k   - / -  1000 tok/s 
 gpt-5.4   openai   gpt54, gpt-54    1m   $2.5 / $15.0   70 tok/s 
 gpt-5.4-pro   openai   gpt54-pro, gpt-54-pro    1m  $30.0 / $180.0   20 tok/s 
 gpt-5.4-mini   openai   gpt54-mini, gpt-54-mini   400k   $0.8 / $4.5   140 tok/s 
 gemini-3.1-pro-preview   gemini   gemini-pro    1m   $2.0 / $12.0   85 tok/s 
 gemini-3.1-pro-preview-customtools  gemini   gemini-customtools    1m   $2.0 / $12.0   85 tok/s 
 gemini-3-flash-preview   gemini   gemini-flash    1m   $0.5 / $3.0   150 tok/s 
 gemini-3.1-flash-lite-preview   gemini   gemini-flash-lite    1m   $0.2 / $1.5   200 tok/s 
 kimi-k2.5   kimi   kimi    262k   $0.6 / $3.0   50 tok/s 
 glm-4.7   zai   glm, glm4    203k   $0.6 / $2.2   100 tok/s 
 minimax-m2.5   minimax   minimax    197k   $0.3 / $1.2   45 tok/s 
 mercury-2   inception  mercury    131k   $0.2 / $0.8  1000 tok/s 

```

View file

@ -1,25 +1,25 @@
```console
$ fabro model list
 MODEL   PROVIDER   ALIASES   CONTEXT   COST   SPEED 
 claude-opus-4-6   anthropic  opus, claude-opus    1m   $15.0 / $75.0   25 tok/s 
 claude-sonnet-4-5   anthropic      200k   $3.0 / $15.0   50 tok/s 
 claude-sonnet-4-6   anthropic  sonnet, claude-sonnet   200k   $3.0 / $15.0   50 tok/s 
 claude-haiku-4-5   anthropic  haiku, claude-haiku    200k   $0.8 / $4.0   100 tok/s 
 gpt-5.2   openai   gpt5    1m   $1.8 / $14.0   65 tok/s 
 gpt-5-mini   openai   gpt5-mini    1m   $0.2 / $2.0   70 tok/s 
 gpt-5.2-codex   openai       1m   $1.8 / $14.0   100 tok/s 
 gpt-5.3-codex   openai   codex    1m   $1.8 / $14.0   100 tok/s 
 gpt-5.3-codex-spark   openai   codex-spark    131k   - / -  1000 tok/s 
 gpt-5.4   openai   gpt54    1m   $2.5 / $15.0   70 tok/s 
 gpt-5.4-pro   openai   gpt54-pro    1m  $30.0 / $180.0   20 tok/s 
 gpt-5.4-mini   openai   gpt54-mini    400k   $0.8 / $4.5   140 tok/s 
 gemini-3.1-pro-preview   gemini   gemini-pro    1m   $2.0 / $12.0   85 tok/s 
 gemini-3.1-pro-preview-customtools  gemini   gemini-customtools    1m   $2.0 / $12.0   85 tok/s 
 gemini-3-flash-preview   gemini   gemini-flash    1m   $0.5 / $3.0   150 tok/s 
 gemini-3.1-flash-lite-preview   gemini   gemini-flash-lite    1m   $0.2 / $1.5   200 tok/s 
 kimi-k2.5   kimi   kimi    262k   $0.6 / $3.0   50 tok/s 
 glm-4.7   zai   glm, glm4    203k   $0.6 / $2.2   100 tok/s 
 minimax-m2.5   minimax   minimax    197k   $0.3 / $1.2   45 tok/s 
 mercury-2   inception  mercury    131k   $0.2 / $0.8  1000 tok/s 
 MODEL   PROVIDER   ALIASES   CONTEXT   COST   SPEED 
 claude-opus-4-6   anthropic  opus, claude-opus    1m   $15.0 / $75.0   25 tok/s 
 claude-sonnet-4-5   anthropic      200k   $3.0 / $15.0   50 tok/s 
 claude-sonnet-4-6   anthropic  sonnet, claude-sonnet    200k   $3.0 / $15.0   50 tok/s 
 claude-haiku-4-5   anthropic  haiku, claude-haiku    200k   $0.8 / $4.0   100 tok/s 
 gpt-5.2   openai   gpt5    1m   $1.8 / $14.0   65 tok/s 
 gpt-5-mini   openai   gpt5-mini    1m   $0.2 / $2.0   70 tok/s 
 gpt-5.2-codex   openai       1m   $1.8 / $14.0   100 tok/s 
 gpt-5.3-codex   openai   codex    1m   $1.8 / $14.0   100 tok/s 
 gpt-5.3-codex-spark   openai   codex-spark    131k   - / -  1000 tok/s 
 gpt-5.4   openai   gpt54, gpt-54    1m   $2.5 / $15.0   70 tok/s 
 gpt-5.4-pro   openai   gpt54-pro, gpt-54-pro    1m  $30.0 / $180.0   20 tok/s 
 gpt-5.4-mini   openai   gpt54-mini, gpt-54-mini   400k   $0.8 / $4.5   140 tok/s 
 gemini-3.1-pro-preview   gemini   gemini-pro    1m   $2.0 / $12.0   85 tok/s 
 gemini-3.1-pro-preview-customtools  gemini   gemini-customtools    1m   $2.0 / $12.0   85 tok/s 
 gemini-3-flash-preview   gemini   gemini-flash    1m   $0.5 / $3.0   150 tok/s 
 gemini-3.1-flash-lite-preview   gemini   gemini-flash-lite    1m   $0.2 / $1.5   200 tok/s 
 kimi-k2.5   kimi   kimi    262k   $0.6 / $3.0   50 tok/s 
 glm-4.7   zai   glm, glm4    203k   $0.6 / $2.2   100 tok/s 
 minimax-m2.5   minimax   minimax    197k   $0.3 / $1.2   45 tok/s 
 mercury-2   inception  mercury    131k   $0.2 / $0.8  1000 tok/s 

```

View file

@ -158,7 +158,7 @@
"cache_input_cost_per_mtok": 0.25
},
"estimated_output_tps": 70,
"aliases": ["gpt54"],
"aliases": ["gpt54", "gpt-54"],
"default": true
},
{
@ -175,7 +175,7 @@
"cache_input_cost_per_mtok": 3.0
},
"estimated_output_tps": 20,
"aliases": ["gpt54-pro"]
"aliases": ["gpt54-pro", "gpt-54-pro"]
},
{
"id": "gpt-5.4-mini",
@ -191,7 +191,7 @@
"cache_input_cost_per_mtok": 0.075
},
"estimated_output_tps": 140,
"aliases": ["gpt54-mini"]
"aliases": ["gpt54-mini", "gpt-54-mini"]
},
{
"id": "gemini-3.1-pro-preview",

View file

@ -525,6 +525,7 @@ mod tests {
),
aliases: [
"gpt54",
"gpt-54",
],
default: true,
}
@ -570,6 +571,7 @@ mod tests {
),
aliases: [
"gpt54-pro",
"gpt-54-pro",
],
default: false,
}
@ -581,6 +583,21 @@ mod tests {
assert_eq!(get_model_info("gpt54").unwrap().id, "gpt-5.4");
}
#[test]
fn gpt_54_hyphenated_alias() {
assert_eq!(get_model_info("gpt-54").unwrap().id, "gpt-5.4");
}
#[test]
fn gpt_54_pro_hyphenated_alias() {
assert_eq!(get_model_info("gpt-54-pro").unwrap().id, "gpt-5.4-pro");
}
#[test]
fn gpt_54_mini_hyphenated_alias() {
assert_eq!(get_model_info("gpt-54-mini").unwrap().id, "gpt-5.4-mini");
}
#[test]
fn gpt_5_3_codex_spark_in_catalog() {
let m = get_model_info("gpt-5.3-codex-spark").unwrap();

View file

@ -31,6 +31,7 @@ pub fn built_in_rules() -> Vec<Box<dyn LintRule>> {
Box::new(OrphanCustomOutcomeRule),
Box::new(ScriptAbsoluteCdRule),
Box::new(StylesheetModelKnownRule),
Box::new(NodeModelKnownRule),
Box::new(UnresolvedFileRefRule),
Box::new(ThreadIdRequiresFidelityFullRule),
Box::new(SelectionValidRule),
@ -898,6 +899,53 @@ impl LintRule for ScriptAbsoluteCdRule {
}
}
// --- Shared helpers for model/provider validation ---
fn check_model_known(
rule_name: &str,
model: &str,
context: &str,
node_id: Option<String>,
) -> Option<Diagnostic> {
if fabro_model::get_model_info(model).is_some() {
return None;
}
Some(Diagnostic {
rule: rule_name.to_string(),
severity: Severity::Warning,
message: format!(
"Unknown model '{model}' {context}. Run `fabro model list` to see available models"
),
node_id,
edge: None,
fix: Some("Use a model ID from `fabro model list`".to_string()),
})
}
fn check_provider_known(
rule_name: &str,
provider: &str,
context: &str,
node_id: Option<String>,
) -> Option<Diagnostic> {
if fabro_model::Provider::from_str(provider).is_ok() {
return None;
}
let valid: Vec<&str> = fabro_model::Provider::ALL
.iter()
.map(|p| p.as_str())
.collect();
let valid_str = valid.join(", ");
Some(Diagnostic {
rule: rule_name.to_string(),
severity: Severity::Warning,
message: format!("Unknown provider '{provider}' {context}. Valid providers: {valid_str}"),
node_id,
edge: None,
fix: Some(format!("Use one of: {valid_str}")),
})
}
// --- Rule 20: stylesheet_model_known (WARNING) ---
struct StylesheetModelKnownRule;
@ -932,40 +980,19 @@ impl LintRule for StylesheetModelKnownRule {
for rule in &stylesheet.rules {
let label = Self::selector_label(&rule.selector);
for decl in &rule.declarations {
let context = format!("in stylesheet rule '{label}'");
match decl.property.as_str() {
"model" => {
if fabro_model::get_model_info(&decl.value).is_none() {
diagnostics.push(Diagnostic {
rule: self.name().to_string(),
severity: Severity::Warning,
message: format!(
"Unknown model '{}' in stylesheet rule '{label}'. Run `fabro model list` to see available models",
decl.value
),
node_id: None,
edge: None,
fix: Some("Use a model ID from `fabro model list`".to_string()),
});
if let Some(d) = check_model_known(self.name(), &decl.value, &context, None)
{
diagnostics.push(d);
}
}
"provider" => {
if fabro_model::Provider::from_str(&decl.value).is_err() {
let valid: Vec<&str> = fabro_model::Provider::ALL
.iter()
.map(|p| p.as_str())
.collect();
diagnostics.push(Diagnostic {
rule: self.name().to_string(),
severity: Severity::Warning,
message: format!(
"Unknown provider '{}' in stylesheet rule '{label}'. Valid providers: {}",
decl.value,
valid.join(", ")
),
node_id: None,
edge: None,
fix: Some(format!("Use one of: {}", valid.join(", "))),
});
if let Some(d) =
check_provider_known(self.name(), &decl.value, &context, None)
{
diagnostics.push(d);
}
}
_ => {}
@ -976,7 +1003,38 @@ impl LintRule for StylesheetModelKnownRule {
}
}
// --- Rule 21: unresolved_file_ref (ERROR) ---
// --- Rule 21: node_model_known (WARNING) ---
struct NodeModelKnownRule;
impl LintRule for NodeModelKnownRule {
fn name(&self) -> &'static str {
"node_model_known"
}
fn apply(&self, graph: &Graph) -> Vec<Diagnostic> {
let mut diagnostics = Vec::new();
for node in graph.nodes.values() {
let context = format!("on node '{}'", node.id);
let node_id = Some(node.id.clone());
if let Some(model) = node.model() {
if let Some(d) = check_model_known(self.name(), model, &context, node_id.clone()) {
diagnostics.push(d);
}
}
if let Some(provider) = node.provider() {
if let Some(d) =
check_provider_known(self.name(), provider, &context, node_id.clone())
{
diagnostics.push(d);
}
}
}
diagnostics
}
}
// --- Rule 22: unresolved_file_ref (ERROR) ---
struct UnresolvedFileRefRule;
@ -3035,6 +3093,76 @@ mod tests {
assert!(d.is_empty());
}
// node_model_known rule tests
#[test]
fn node_model_known_rule_valid_model() {
let mut g = minimal_graph();
let mut node = Node::new("work");
node.attrs.insert(
"model".to_string(),
AttrValue::String("claude-sonnet-4-5".to_string()),
);
g.nodes.insert("work".to_string(), node);
let rule = NodeModelKnownRule;
let d = rule.apply(&g);
assert!(d.is_empty());
}
#[test]
fn node_model_known_rule_unknown_model() {
let mut g = minimal_graph();
let mut node = Node::new("work");
node.attrs.insert(
"model".to_string(),
AttrValue::String("nonexistent-model-xyz".to_string()),
);
g.nodes.insert("work".to_string(), node);
let rule = NodeModelKnownRule;
let d = rule.apply(&g);
assert_eq!(d.len(), 1);
assert_eq!(d[0].severity, Severity::Warning);
assert!(d[0].message.contains("nonexistent-model-xyz"));
assert_eq!(d[0].node_id.as_deref(), Some("work"));
}
#[test]
fn node_model_known_rule_alias() {
let mut g = minimal_graph();
let mut node = Node::new("work");
node.attrs
.insert("model".to_string(), AttrValue::String("opus".to_string()));
g.nodes.insert("work".to_string(), node);
let rule = NodeModelKnownRule;
let d = rule.apply(&g);
assert!(d.is_empty());
}
#[test]
fn node_model_known_rule_unknown_provider() {
let mut g = minimal_graph();
let mut node = Node::new("work");
node.attrs.insert(
"provider".to_string(),
AttrValue::String("google".to_string()),
);
g.nodes.insert("work".to_string(), node);
let rule = NodeModelKnownRule;
let d = rule.apply(&g);
assert_eq!(d.len(), 1);
assert_eq!(d[0].severity, Severity::Warning);
assert!(d[0].message.contains("google"));
assert_eq!(d[0].node_id.as_deref(), Some("work"));
}
#[test]
fn node_model_known_rule_no_model_no_provider() {
let g = minimal_graph();
let rule = NodeModelKnownRule;
let d = rule.apply(&g);
assert!(d.is_empty());
}
// unresolved_file_ref rule tests
#[test]

View file

@ -1149,51 +1149,6 @@ mod tests {
assert!(branch_needs_push(repo_dir, "origin", "main"));
}
/// Helper: create a local repo with a bare remote and push main.
fn init_repo_with_remote(dir: &Path) -> (std::path::PathBuf, std::path::PathBuf) {
let repo_dir = dir.join("repo");
let remote_dir = dir.join("remote.git");
Command::new("git")
.args(["init", "--bare"])
.arg(&remote_dir)
.output()
.unwrap();
Command::new("git")
.args(["init"])
.arg(&repo_dir)
.output()
.unwrap();
Command::new("git")
.args(["remote", "add", "origin"])
.arg(&remote_dir)
.current_dir(&repo_dir)
.output()
.unwrap();
Command::new("git")
.args([
"-c",
"user.name=test",
"-c",
"user.email=test@test",
"commit",
"--allow-empty",
"-m",
"init",
])
.current_dir(&repo_dir)
.output()
.unwrap();
Command::new("git")
.args(["branch", "-M", "main"])
.current_dir(&repo_dir)
.output()
.unwrap();
push_branch(&repo_dir, "origin", "main").unwrap();
(repo_dir, remote_dir)
}
#[test]
fn metadata_branch_name_uses_meta_prefix() {
assert_eq!(MetadataStore::branch_name("abc-123"), "fabro/meta/abc-123");