Introduce insta snapshot testing for arc-devcontainer and arc-cli

Replace assert_eq!/contains() chains with insta::assert_snapshot! in
dockerfile unit tests (11 inline snapshots) and CLI model/help tests
(7 file-based snapshots), making test output easier to read and update.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Bryan Helmkamp 2026-03-10 21:09:15 -04:00
parent 490dc2e5eb
commit bd96b94acf
13 changed files with 258 additions and 131 deletions

48
Cargo.lock generated
View file

@ -98,7 +98,7 @@ checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c"
[[package]]
name = "arc-agent"
version = "0.1.0"
version = "0.2.0"
dependencies = [
"anyhow",
"arc-llm",
@ -131,7 +131,7 @@ dependencies = [
[[package]]
name = "arc-api"
version = "0.1.0"
version = "0.2.0"
dependencies = [
"anyhow",
"arc-agent",
@ -181,7 +181,7 @@ dependencies = [
[[package]]
name = "arc-cli"
version = "0.1.0"
version = "0.2.0"
dependencies = [
"anyhow",
"arc-agent",
@ -205,6 +205,7 @@ dependencies = [
"dotenvy",
"futures",
"httpmock",
"insta",
"jsonwebtoken",
"open",
"predicates",
@ -227,7 +228,7 @@ dependencies = [
[[package]]
name = "arc-config"
version = "0.1.0"
version = "0.2.0"
dependencies = [
"anyhow",
"arc-agent",
@ -242,7 +243,7 @@ dependencies = [
[[package]]
name = "arc-db"
version = "0.1.0"
version = "0.2.0"
dependencies = [
"chrono",
"sqlx",
@ -253,8 +254,9 @@ dependencies = [
[[package]]
name = "arc-devcontainer"
version = "0.1.0"
version = "0.2.0"
dependencies = [
"insta",
"reqwest 0.12.28",
"serde",
"serde_json",
@ -267,7 +269,7 @@ dependencies = [
[[package]]
name = "arc-exe"
version = "0.1.0"
version = "0.2.0"
dependencies = [
"arc-agent",
"arc-github",
@ -286,7 +288,7 @@ dependencies = [
[[package]]
name = "arc-git-storage"
version = "0.1.0"
version = "0.2.0"
dependencies = [
"git2",
"tempfile",
@ -297,7 +299,7 @@ dependencies = [
[[package]]
name = "arc-github"
version = "0.1.0"
version = "0.2.0"
dependencies = [
"base64",
"chrono",
@ -312,7 +314,7 @@ dependencies = [
[[package]]
name = "arc-llm"
version = "0.1.0"
version = "0.2.0"
dependencies = [
"anyhow",
"arc-util",
@ -339,7 +341,7 @@ dependencies = [
[[package]]
name = "arc-mcp"
version = "0.1.0"
version = "0.2.0"
dependencies = [
"anyhow",
"futures",
@ -353,7 +355,7 @@ dependencies = [
[[package]]
name = "arc-openai-oauth"
version = "0.1.0"
version = "0.2.0"
dependencies = [
"axum",
"base64",
@ -371,7 +373,7 @@ dependencies = [
[[package]]
name = "arc-slack"
version = "0.1.0"
version = "0.2.0"
dependencies = [
"arc-workflows",
"futures-util",
@ -389,7 +391,7 @@ dependencies = [
[[package]]
name = "arc-sprites"
version = "0.1.0"
version = "0.2.0"
dependencies = [
"arc-agent",
"async-trait",
@ -406,7 +408,7 @@ dependencies = [
[[package]]
name = "arc-types"
version = "0.1.0"
version = "0.2.0"
dependencies = [
"chrono",
"prettyplease",
@ -421,7 +423,7 @@ dependencies = [
[[package]]
name = "arc-util"
version = "0.1.0"
version = "0.2.0"
dependencies = [
"aho-corasick",
"anyhow",
@ -446,7 +448,7 @@ dependencies = [
[[package]]
name = "arc-workflows"
version = "0.1.0"
version = "0.2.0"
dependencies = [
"anyhow",
"arc-agent",
@ -2325,6 +2327,18 @@ dependencies = [
"web-time",
]
[[package]]
name = "insta"
version = "1.46.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e82db8c87c7f1ccecb34ce0c24399b8a73081427f3c7c50a5d597925356115e4"
dependencies = [
"console 0.15.11",
"once_cell",
"similar",
"tempfile",
]
[[package]]
name = "ipnet"
version = "2.11.0"

View file

@ -52,6 +52,7 @@ jsonwebtoken = { version = "10", features = ["aws_lc_rs"] }
hmac = "0.12"
sha2 = "0.10"
hex = "0.4"
insta = "1"
tokio-tungstenite = { version = "0.26", features = ["rustls-tls-webpki-roots"] }
futures-util = "0.3"
openssh = "0.11"

View file

@ -54,6 +54,7 @@ serde_json.workspace = true
[dev-dependencies]
assert_cmd = "2"
insta = { workspace = true }
predicates = "3"
tempfile = "3"
serde_json.workspace = true

View file

@ -10,72 +10,62 @@ fn arc() -> Command {
#[test]
fn model_list_prints_all_models() {
arc()
.args(["model", "list"])
.assert()
.success()
.stdout(predicate::str::contains("claude-opus-4-6"))
.stdout(predicate::str::contains("claude-sonnet-4-5"))
.stdout(predicate::str::contains("gpt-5.2"))
.stdout(predicate::str::contains("gemini-3.1-pro-preview"))
.stdout(predicate::str::contains("anthropic"))
.stdout(predicate::str::contains("openai"))
.stdout(predicate::str::contains("gemini"))
.stdout(predicate::str::contains("gpt-5.4"));
let output = arc().args(["model", "list"]).output().expect("runs");
assert!(output.status.success());
let stdout = String::from_utf8(output.stdout).unwrap();
insta::assert_snapshot!(stdout);
}
#[test]
fn model_list_filters_by_provider() {
let assert = arc()
let output = arc()
.args(["model", "list", "--provider", "anthropic"])
.assert()
.success()
.stdout(predicate::str::contains("claude-opus-4-6"))
.stdout(predicate::str::contains("claude-sonnet-4-5"));
// Should NOT contain other providers
assert
.stdout(predicate::str::contains("gpt-5.2").not())
.stdout(predicate::str::contains("gemini-3.1-pro-preview").not());
.output()
.expect("runs");
assert!(output.status.success());
let stdout = String::from_utf8(output.stdout).unwrap();
insta::assert_snapshot!(stdout);
}
#[test]
fn model_list_filters_by_query() {
arc()
let output = arc()
.args(["model", "list", "--query", "opus"])
.assert()
.success()
.stdout(predicate::str::contains("claude-opus-4-6"))
.stdout(predicate::str::contains("claude-sonnet-4-5").not());
.output()
.expect("runs");
assert!(output.status.success());
let stdout = String::from_utf8(output.stdout).unwrap();
insta::assert_snapshot!(stdout);
}
#[test]
fn model_list_query_is_case_insensitive() {
arc()
let output = arc()
.args(["model", "list", "--query", "OPUS"])
.assert()
.success()
.stdout(predicate::str::contains("claude-opus-4-6"));
.output()
.expect("runs");
assert!(output.status.success());
let stdout = String::from_utf8(output.stdout).unwrap();
insta::assert_snapshot!(stdout);
}
#[test]
fn model_list_query_matches_aliases() {
arc()
let output = arc()
.args(["model", "list", "--query", "codex"])
.assert()
.success()
.stdout(predicate::str::contains("gpt-5.2-codex"));
.output()
.expect("runs");
assert!(output.status.success());
let stdout = String::from_utf8(output.stdout).unwrap();
insta::assert_snapshot!(stdout);
}
#[test]
fn model_bare_defaults_to_list() {
arc()
.args(["model"])
.assert()
.success()
.stdout(predicate::str::contains("claude-opus-4-6"))
.stdout(predicate::str::contains("gpt-5.2"))
.stdout(predicate::str::contains("gemini-3.1-pro-preview"));
let output = arc().args(["model"]).output().expect("runs");
assert!(output.status.success());
let stdout = String::from_utf8(output.stdout).unwrap();
insta::assert_snapshot!(stdout);
}
// == LLM: prompt ==============================================================
@ -296,11 +286,10 @@ fn exec_no_prompt_prints_usage() {
#[test]
fn exec_help_flag_prints_help() {
arc()
.args(["exec", "--help"])
.assert()
.success()
.stdout(predicate::str::contains("Task prompt"));
let output = arc().args(["exec", "--help"]).output().expect("runs");
assert!(output.status.success());
let stdout = String::from_utf8(output.stdout).unwrap();
insta::assert_snapshot!(stdout);
}
#[test]
@ -543,15 +532,10 @@ fn validate_invalid() {
#[test]
#[cfg(feature = "server")]
fn serve_help() {
arc()
.args(["serve", "--help"])
.assert()
.success()
.stdout(predicate::str::contains("--port"))
.stdout(predicate::str::contains("--host"))
.stdout(predicate::str::contains("--dry-run"))
.stdout(predicate::str::contains("--model"))
.stdout(predicate::str::contains("--provider"));
let output = arc().args(["serve", "--help"]).output().expect("runs");
assert!(output.status.success());
let stdout = String::from_utf8(output.stdout).unwrap();
insta::assert_snapshot!(stdout);
}
// == Arc: run --dry-run =================================================

View file

@ -0,0 +1,21 @@
---
source: lib/crates/arc-cli/tests/cli.rs
expression: stdout
---
Run an agentic coding session
Usage: arc exec [OPTIONS] <PROMPT>
Arguments:
<PROMPT> Task prompt
Options:
--provider <PROVIDER> LLM provider (anthropic, openai, gemini, kimi, zai, minimax, inception)
--model <MODEL> Model name (defaults per provider)
--permissions <PERMISSIONS> Permission level for tool execution [possible values: read-only, read-write, full]
--auto-approve Skip interactive prompts; deny tools outside permission level
--debug Print LLM request/response debug info to stderr
--verbose Print full LLM request/response JSON to stderr
--skills-dir <SKILLS_DIR> Directory containing skill files (overrides default discovery)
--output-format <OUTPUT_FORMAT> Output format (text for human-readable, json for NDJSON event stream) [possible values: text, json]
-h, --help Print help

View file

@ -0,0 +1,23 @@
---
source: lib/crates/arc-cli/tests/cli.rs
expression: stdout
---
MODEL PROVIDER ALIASES CONTEXT COST SPEED
claude-opus-4-6 anthropic opus, claude-opus 1m $15.0 / $75.0 25 tok/s
claude-sonnet-4-5 anthropic 200k $3.0 / $15.0 50 tok/s
claude-sonnet-4-6 anthropic sonnet, claude-sonnet 200k $3.0 / $15.0 50 tok/s
claude-haiku-4-5 anthropic haiku, claude-haiku 200k $0.8 / $4.0 100 tok/s
gpt-5.2 openai gpt5 1m $1.8 / $14.0 65 tok/s
gpt-5-mini openai gpt5-mini 1m $0.2 / $2.0 70 tok/s
gpt-5.2-codex openai 1m $1.8 / $14.0 100 tok/s
gpt-5.3-codex openai codex 1m $1.8 / $14.0 100 tok/s
gpt-5.3-codex-spark openai codex-spark 131k - / - 1000 tok/s
gpt-5.4 openai gpt54 1m $2.5 / $15.0 70 tok/s
gpt-5.4-pro openai gpt54-pro 1m $30.0 / $180.0 20 tok/s
gemini-3.1-pro-preview gemini gemini-pro 1m $2.0 / $12.0 85 tok/s
gemini-3-flash-preview gemini gemini-flash 1m $0.5 / $3.0 150 tok/s
gemini-3.1-flash-lite-preview gemini gemini-flash-lite 1m $0.2 / $1.5 200 tok/s
kimi-k2.5 kimi kimi 262k $0.6 / $3.0 50 tok/s
glm-4.7 zai glm, glm4 203k $0.6 / $2.2 100 tok/s
minimax-m2.5 minimax minimax 197k $0.3 / $1.2 45 tok/s
mercury-2 inception mercury 131k $0.2 / $0.8 1000 tok/s

View file

@ -0,0 +1,9 @@
---
source: lib/crates/arc-cli/tests/cli.rs
expression: stdout
---
MODEL PROVIDER ALIASES CONTEXT COST SPEED
claude-opus-4-6 anthropic opus, claude-opus 1m $15.0 / $75.0 25 tok/s
claude-sonnet-4-5 anthropic 200k $3.0 / $15.0 50 tok/s
claude-sonnet-4-6 anthropic sonnet, claude-sonnet 200k $3.0 / $15.0 50 tok/s
claude-haiku-4-5 anthropic haiku, claude-haiku 200k $0.8 / $4.0 100 tok/s

View file

@ -0,0 +1,6 @@
---
source: lib/crates/arc-cli/tests/cli.rs
expression: stdout
---
MODEL PROVIDER ALIASES CONTEXT COST SPEED
claude-opus-4-6 anthropic opus, claude-opus 1m $15.0 / $75.0 25 tok/s

View file

@ -0,0 +1,23 @@
---
source: lib/crates/arc-cli/tests/cli.rs
expression: stdout
---
MODEL PROVIDER ALIASES CONTEXT COST SPEED
claude-opus-4-6 anthropic opus, claude-opus 1m $15.0 / $75.0 25 tok/s
claude-sonnet-4-5 anthropic 200k $3.0 / $15.0 50 tok/s
claude-sonnet-4-6 anthropic sonnet, claude-sonnet 200k $3.0 / $15.0 50 tok/s
claude-haiku-4-5 anthropic haiku, claude-haiku 200k $0.8 / $4.0 100 tok/s
gpt-5.2 openai gpt5 1m $1.8 / $14.0 65 tok/s
gpt-5-mini openai gpt5-mini 1m $0.2 / $2.0 70 tok/s
gpt-5.2-codex openai 1m $1.8 / $14.0 100 tok/s
gpt-5.3-codex openai codex 1m $1.8 / $14.0 100 tok/s
gpt-5.3-codex-spark openai codex-spark 131k - / - 1000 tok/s
gpt-5.4 openai gpt54 1m $2.5 / $15.0 70 tok/s
gpt-5.4-pro openai gpt54-pro 1m $30.0 / $180.0 20 tok/s
gemini-3.1-pro-preview gemini gemini-pro 1m $2.0 / $12.0 85 tok/s
gemini-3-flash-preview gemini gemini-flash 1m $0.5 / $3.0 150 tok/s
gemini-3.1-flash-lite-preview gemini gemini-flash-lite 1m $0.2 / $1.5 200 tok/s
kimi-k2.5 kimi kimi 262k $0.6 / $3.0 50 tok/s
glm-4.7 zai glm, glm4 203k $0.6 / $2.2 100 tok/s
minimax-m2.5 minimax minimax 197k $0.3 / $1.2 45 tok/s
mercury-2 inception mercury 131k $0.2 / $0.8 1000 tok/s

View file

@ -0,0 +1,6 @@
---
source: lib/crates/arc-cli/tests/cli.rs
expression: stdout
---
MODEL PROVIDER ALIASES CONTEXT COST SPEED
claude-opus-4-6 anthropic opus, claude-opus 1m $15.0 / $75.0 25 tok/s

View file

@ -0,0 +1,8 @@
---
source: lib/crates/arc-cli/tests/cli.rs
expression: stdout
---
MODEL PROVIDER ALIASES CONTEXT COST SPEED
gpt-5.2-codex openai 1m $1.8 / $14.0 100 tok/s
gpt-5.3-codex openai codex 1m $1.8 / $14.0 100 tok/s
gpt-5.3-codex-spark openai codex-spark 131k - / - 1000 tok/s

View file

@ -18,5 +18,6 @@ tokio = { workspace = true }
reqwest = { workspace = true }
[dev-dependencies]
insta = { workspace = true }
tokio = { workspace = true, features = ["test-util", "macros"] }
tempfile = "3"

View file

@ -52,20 +52,24 @@ mod tests {
#[test]
fn base_image_only() {
let result = generate("FROM ubuntu:22.04", &[], &HashMap::new(), None);
assert_eq!(
result,
"# Generated by arc-devcontainer\n\nFROM ubuntu:22.04\n"
);
insta::assert_snapshot!(result, @r"
# Generated by arc-devcontainer
FROM ubuntu:22.04
");
}
#[test]
fn base_dockerfile_preserved_as_is() {
let base = "FROM ubuntu:22.04\nRUN apt-get update\nRUN apt-get install -y curl";
let result = generate(base, &[], &HashMap::new(), None);
assert_eq!(
result,
"# Generated by arc-devcontainer\n\nFROM ubuntu:22.04\nRUN apt-get update\nRUN apt-get install -y curl\n"
);
insta::assert_snapshot!(result, @r"
# Generated by arc-devcontainer
FROM ubuntu:22.04
RUN apt-get update
RUN apt-get install -y curl
");
}
#[test]
@ -75,13 +79,15 @@ mod tests {
make_layer("python", "python-1", "RUN install-python.sh"),
];
let result = generate("FROM ubuntu:22.04", &layers, &HashMap::new(), None);
assert_eq!(
result,
"# Generated by arc-devcontainer\n\n\
FROM ubuntu:22.04\n\n\
RUN install-node.sh\n\n\
RUN install-python.sh\n"
);
insta::assert_snapshot!(result, @r"
# Generated by arc-devcontainer
FROM ubuntu:22.04
RUN install-node.sh
RUN install-python.sh
");
}
#[test]
@ -91,25 +97,27 @@ mod tests {
env.insert("APPLE".to_string(), "red".to_string());
env.insert("MANGO".to_string(), "yellow".to_string());
let result = generate("FROM alpine", &[], &env, None);
assert_eq!(
result,
"# Generated by arc-devcontainer\n\n\
FROM alpine\n\n\
ENV APPLE=red\n\
ENV MANGO=yellow\n\
ENV ZEBRA=stripes\n"
);
insta::assert_snapshot!(result, @r"
# Generated by arc-devcontainer
FROM alpine
ENV APPLE=red
ENV MANGO=yellow
ENV ZEBRA=stripes
");
}
#[test]
fn with_remote_user() {
let result = generate("FROM alpine", &[], &HashMap::new(), Some("vscode"));
assert_eq!(
result,
"# Generated by arc-devcontainer\n\n\
FROM alpine\n\n\
USER vscode\n"
);
insta::assert_snapshot!(result, @r"
# Generated by arc-devcontainer
FROM alpine
USER vscode
");
}
#[test]
@ -119,33 +127,41 @@ mod tests {
env.insert("PATH".to_string(), "/usr/local/bin".to_string());
env.insert("HOME".to_string(), "/home/vscode".to_string());
let result = generate("FROM ubuntu:22.04", &layers, &env, Some("vscode"));
assert_eq!(
result,
"# Generated by arc-devcontainer\n\n\
FROM ubuntu:22.04\n\n\
RUN install-node.sh\n\n\
ENV HOME=/home/vscode\n\
ENV PATH=/usr/local/bin\n\n\
USER vscode\n"
);
insta::assert_snapshot!(result, @r"
# Generated by arc-devcontainer
FROM ubuntu:22.04
RUN install-node.sh
ENV HOME=/home/vscode
ENV PATH=/usr/local/bin
USER vscode
");
}
#[test]
fn empty_feature_layers_no_extra_blank_lines() {
let result = generate("FROM alpine", &[], &HashMap::new(), Some("dev"));
assert_eq!(
result,
"# Generated by arc-devcontainer\n\n\
FROM alpine\n\n\
USER dev\n"
);
insta::assert_snapshot!(result, @r"
# Generated by arc-devcontainer
FROM alpine
USER dev
");
}
#[test]
fn empty_env_map_treated_as_none() {
let env = HashMap::new();
let result = generate("FROM alpine", &[], &env, None);
assert_eq!(result, "# Generated by arc-devcontainer\n\nFROM alpine\n");
insta::assert_snapshot!(result, @r"
# Generated by arc-devcontainer
FROM alpine
");
}
#[test]
@ -158,10 +174,17 @@ mod tests {
FROM ubuntu:22.04\n\
COPY --from=builder /app/bin /usr/local/bin";
let result = generate(base, &[], &HashMap::new(), None);
assert!(result.starts_with("# Generated by arc-devcontainer\n\n"));
assert!(result.contains("FROM ubuntu:22.04 AS builder"));
assert!(result.contains("COPY --from=builder /app/bin /usr/local/bin"));
assert!(result.ends_with('\n'));
insta::assert_snapshot!(result, @r"
# Generated by arc-devcontainer
FROM ubuntu:22.04 AS builder
RUN apt-get update && apt-get install -y build-essential
COPY . /app
RUN make
FROM ubuntu:22.04
COPY --from=builder /app/bin /usr/local/bin
");
}
#[test]
@ -169,12 +192,13 @@ mod tests {
let mut cenv = HashMap::new();
cenv.insert("DEBIAN_FRONTEND".to_string(), "noninteractive".to_string());
let result = generate("FROM alpine", &[], &cenv, None);
assert_eq!(
result,
"# Generated by arc-devcontainer\n\n\
FROM alpine\n\n\
ENV DEBIAN_FRONTEND=noninteractive\n"
);
insta::assert_snapshot!(result, @r"
# Generated by arc-devcontainer
FROM alpine
ENV DEBIAN_FRONTEND=noninteractive
");
}
#[test]
@ -184,8 +208,14 @@ mod tests {
cenv.insert("BETA".to_string(), "second".to_string());
cenv.insert("GAMMA".to_string(), "third".to_string());
let result = generate("FROM alpine", &[], &cenv, None);
assert!(result.contains("ENV ALPHA=first"));
assert!(result.contains("ENV BETA=second"));
assert!(result.contains("ENV GAMMA=third"));
insta::assert_snapshot!(result, @r"
# Generated by arc-devcontainer
FROM alpine
ENV ALPHA=first
ENV BETA=second
ENV GAMMA=third
");
}
}