diff --git a/Cargo.lock b/Cargo.lock index 395b5c24a..c5544df46 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1578,6 +1578,7 @@ version = "0.219.0-nightly.0" dependencies = [ "chrono", "fabro-config", + "fabro-model", "fabro-types", "openapiv3", "prettyplease", diff --git a/docs/public/api-reference/fabro-api.yaml b/docs/public/api-reference/fabro-api.yaml index d6a39bb3b..dfde7debb 100644 --- a/docs/public/api-reference/fabro-api.yaml +++ b/docs/public/api-reference/fabro-api.yaml @@ -3009,7 +3009,7 @@ components: required: false description: Filter models by provider name. Invalid values return `400`. schema: - type: string + $ref: "#/components/schemas/Provider" example: anthropic ModelQueryFilter: @@ -3431,11 +3431,25 @@ components: meta: $ref: "#/components/schemas/PaginationMeta" + Provider: + description: LLM provider identifier. + type: string + enum: + - anthropic + - openai + - gemini + - kimi + - zai + - minimax + - inception + - openai_compatible + ModelLimits: description: Token limits for a model. type: object required: - context_window + - max_output properties: context_window: type: integer @@ -3455,6 +3469,7 @@ components: - tools - vision - reasoning + - effort properties: tools: type: boolean @@ -3465,10 +3480,17 @@ components: reasoning: type: boolean description: Whether the model supports extended reasoning. + effort: + type: boolean + description: Whether the model supports direct reasoning effort controls. ModelCosts: description: Pricing per million tokens in USD. type: object + required: + - input_cost_per_mtok + - output_cost_per_mtok + - cache_input_cost_per_mtok properties: input_cost_per_mtok: type: ["number", "null"] @@ -3495,8 +3517,11 @@ components: - family - display_name - limits + - training + - knowledge_cutoff - features - costs + - estimated_output_tps - aliases - default properties: @@ -3505,9 +3530,7 @@ components: description: Unique model identifier. example: "claude-opus-4-6" provider: - type: string - description: Provider that serves this model. - example: "anthropic" + $ref: "#/components/schemas/Provider" family: type: string description: Model family grouping. @@ -3522,6 +3545,10 @@ components: type: ["string", "null"] description: Training data cutoff date (YYYY-MM-DD). example: "2025-08-01" + knowledge_cutoff: + type: ["string", "null"] + description: Public knowledge cutoff label, if known. + example: "May 2025" features: $ref: "#/components/schemas/ModelFeatures" costs: @@ -5375,29 +5402,38 @@ components: - input_tokens - output_tokens - total_tokens + - reasoning_tokens + - cache_read_tokens + - cache_write_tokens properties: input_tokens: type: integer + format: int64 description: Number of input tokens consumed. example: 28640 output_tokens: type: integer + format: int64 description: Number of output tokens generated. example: 8750 total_tokens: type: integer + format: int64 description: Total billable tokens aggregated across categories. example: 37390 reasoning_tokens: type: integer + format: int64 description: Number of reasoning tokens. example: 1200 cache_read_tokens: type: integer + format: int64 description: Number of cache read tokens. example: 4800 cache_write_tokens: type: integer + format: int64 description: Number of cache write tokens. example: 1500 total_usd_micros: @@ -5722,6 +5758,9 @@ components: - input_tokens - output_tokens - total_tokens + - reasoning_tokens + - cache_read_tokens + - cache_write_tokens - runtime_secs properties: runs: @@ -6235,6 +6274,9 @@ components: - input_tokens - output_tokens - total_tokens + - reasoning_tokens + - cache_read_tokens + - cache_write_tokens properties: runtime_secs: type: number diff --git a/lib/crates/fabro-api/Cargo.toml b/lib/crates/fabro-api/Cargo.toml index fe476f2c2..8b347f032 100644 --- a/lib/crates/fabro-api/Cargo.toml +++ b/lib/crates/fabro-api/Cargo.toml @@ -16,6 +16,7 @@ wildcard_imports = "warn" [dependencies] chrono = { workspace = true, features = ["serde"] } fabro-config = { path = "../fabro-config" } +fabro-model = { path = "../fabro-model" } fabro-types = { path = "../fabro-types" } progenitor-client = "0.13" regress = "0.10" diff --git a/lib/crates/fabro-api/build.rs b/lib/crates/fabro-api/build.rs index e6422d61e..6f4564917 100644 --- a/lib/crates/fabro-api/build.rs +++ b/lib/crates/fabro-api/build.rs @@ -327,6 +327,13 @@ fn main() { "fabro_types::PendingInterviewRecord", &[], ), + ("BilledTokenCounts", "fabro_types::BilledTokenCounts", &[]), + ("Provider", "fabro_model::Provider", &[]), + ("Model", "fabro_model::Model", &[]), + ("ModelLimits", "fabro_model::ModelLimits", &[]), + ("ModelFeatures", "fabro_model::ModelFeatures", &[]), + ("ModelCosts", "fabro_model::ModelCosts", &[]), + ("ModelTestMode", "fabro_model::ModelTestMode", &[]), ("RunProjection", "fabro_types::RunProjection", &[]), ("RunEvent", "fabro_types::RunEvent", &[]), ("EventEnvelope", "fabro_types::EventEnvelope", &[]), diff --git a/lib/crates/fabro-api/src/lib.rs b/lib/crates/fabro-api/src/lib.rs index 4c6709763..97c05591d 100644 --- a/lib/crates/fabro-api/src/lib.rs +++ b/lib/crates/fabro-api/src/lib.rs @@ -14,6 +14,7 @@ mod generated { include!(concat!(env!("OUT_DIR"), "/codegen.rs")); } pub mod types { + pub use fabro_model::{Model, ModelCosts, ModelFeatures, ModelLimits, ModelTestMode, Provider}; pub use fabro_types::settings::server::{ DiscordIntegrationSettings, GithubIntegrationSettings, GithubIntegrationStrategy, IntegrationWebhooksSettings, IpAllowEntry, LogDestination, ObjectStoreSettings, @@ -28,11 +29,11 @@ pub mod types { BlockedReason, FailureReason, RunControlAction, RunStatus, SuccessReason, TerminalStatus, }; pub use fabro_types::{ - ActorKind, ActorRef, DiffStats, DirtyStatus, EventEnvelope, GitContext, InterviewOption, - InterviewQuestionRecord, NodeState, NodeStatusRecord, PendingInterviewRecord, - PreRunPushOutcome, QuestionType, RepositoryReference, RunEvent, RunProjection, RunSummary, - SecretMetadata, SecretType, ServerSettings, StageStatus as InternalStageStatus, - WorkflowSettings, + ActorKind, ActorRef, BilledTokenCounts, DiffStats, DirtyStatus, EventEnvelope, GitContext, + InterviewOption, InterviewQuestionRecord, NodeState, NodeStatusRecord, + PendingInterviewRecord, PreRunPushOutcome, QuestionType, RepositoryReference, RunEvent, + RunProjection, RunSummary, SecretMetadata, SecretType, ServerSettings, + StageStatus as InternalStageStatus, WorkflowSettings, }; pub use crate::generated::types::*; diff --git a/lib/crates/fabro-cli/src/commands/model.rs b/lib/crates/fabro-cli/src/commands/model.rs index 2c33be0b8..79aed76d5 100644 --- a/lib/crates/fabro-cli/src/commands/model.rs +++ b/lib/crates/fabro-cli/src/commands/model.rs @@ -2,7 +2,7 @@ use anyhow::{Context, Result, bail}; use cli_table::format::{Border, Justify, Separator}; use cli_table::{Cell, CellStruct, Color, Style, Table}; use fabro_api::types as api_types; -use fabro_model::{Catalog, Model, Provider}; +use fabro_model::{Catalog, Model, ModelTestMode, Provider}; use fabro_util::terminal::Styles; use serde::Serialize; @@ -200,7 +200,7 @@ async fn test_models_via_server( styles: &Styles, json_output: bool, ) -> Result<()> { - let request_mode = deep.then_some(api_types::ModelTestMode::Deep); + let request_mode = deep.then_some(ModelTestMode::Deep); let use_color = styles.use_color; let mut title = models_title(use_color); @@ -559,7 +559,7 @@ mod tests { let client = test_client(&server.url("")); let response = client - .test_model("test-model", Some(api_types::ModelTestMode::Deep)) + .test_model("test-model", Some(ModelTestMode::Deep)) .await .unwrap(); diff --git a/lib/crates/fabro-cli/src/commands/run/attach.rs b/lib/crates/fabro-cli/src/commands/run/attach.rs index ecca558cc..5cfb23bda 100644 --- a/lib/crates/fabro-cli/src/commands/run/attach.rs +++ b/lib/crates/fabro-cli/src/commands/run/attach.rs @@ -24,7 +24,7 @@ use fabro_types::{EventBody, InterviewOption, RunId}; use fabro_util::json::normalize_json_value; use fabro_util::printer::Printer; use fabro_util::terminal::Styles; -use fabro_workflow::outcome::StageStatus; +use fabro_workflow::outcome::StageOutcome; use fabro_workflow::run_status::RunStatus; use tokio::signal::ctrl_c; use tokio::time::sleep; @@ -436,7 +436,7 @@ fn state_exit_code(state: &server_client::RunProjection) -> Option { if let Some(conclusion) = &state.conclusion { let success = matches!( conclusion.status, - StageStatus::Success | StageStatus::PartialSuccess + StageOutcome::Succeeded | StageOutcome::PartiallySucceeded ); return Some(if success { ExitCode::from(0) diff --git a/lib/crates/fabro-cli/src/commands/run/output.rs b/lib/crates/fabro-cli/src/commands/run/output.rs index 4b4cec3d4..696833845 100644 --- a/lib/crates/fabro-cli/src/commands/run/output.rs +++ b/lib/crates/fabro-cli/src/commands/run/output.rs @@ -12,7 +12,7 @@ use fabro_util::check_report::{CheckDetail, CheckReport, CheckResult, CheckSecti use fabro_util::printer::Printer; use fabro_util::terminal::Styles; use fabro_util::text::strip_goal_decoration; -use fabro_workflow::outcome::StageStatus; +use fabro_workflow::outcome::StageOutcome; use fabro_workflow::records::Conclusion; use indicatif::HumanDuration; @@ -178,7 +178,7 @@ pub(crate) fn print_run_conclusion( let status_str = conclusion.status.to_string().to_uppercase(); let status_color = match conclusion.status { - StageStatus::Success | StageStatus::PartialSuccess => &styles.bold_green, + StageOutcome::Succeeded | StageOutcome::PartiallySucceeded => &styles.bold_green, _ => &styles.bold_red, }; fabro_util::printerr!(printer, "Status: {}", status_color.apply_to(&status_str)); diff --git a/lib/crates/fabro-cli/src/commands/run/run_progress/stage_display.rs b/lib/crates/fabro-cli/src/commands/run/run_progress/stage_display.rs index e02124787..a87861284 100644 --- a/lib/crates/fabro-cli/src/commands/run/run_progress/stage_display.rs +++ b/lib/crates/fabro-cli/src/commands/run/run_progress/stage_display.rs @@ -3,7 +3,7 @@ use std::convert::TryFrom; use std::time::Duration; use chrono::{DateTime, Utc}; -use fabro_workflow::outcome::{StageStatus, format_cost}; +use fabro_workflow::outcome::{StageOutcome, format_cost}; use indicatif::ProgressBar; use super::event::ProgressUsage; @@ -135,9 +135,14 @@ impl StageDisplay { status: &str, usage: Option<&ProgressUsage>, ) { - let succeeded = status.parse::().map_or_else( - |_| matches!(status, "success" | "partial_success"), - |status| matches!(status, StageStatus::Success | StageStatus::PartialSuccess), + let succeeded = status.parse::().map_or_else( + |_| matches!(status, "succeeded" | "partially_succeeded"), + |status| { + matches!( + status, + StageOutcome::Succeeded | StageOutcome::PartiallySucceeded + ) + }, ); let cost_str = usage .and_then(ProgressUsage::display_cost) diff --git a/lib/crates/fabro-cli/src/commands/run/wait.rs b/lib/crates/fabro-cli/src/commands/run/wait.rs index fbd2a1e96..f82c8ec56 100644 --- a/lib/crates/fabro-cli/src/commands/run/wait.rs +++ b/lib/crates/fabro-cli/src/commands/run/wait.rs @@ -135,8 +135,7 @@ fn print_human_output( #[cfg(test)] mod tests { - use fabro_types::{BilledTokenCounts, RunStatus, SuccessReason, fixtures}; - use fabro_workflow::outcome::StageStatus; + use fabro_types::{BilledTokenCounts, RunStatus, StageOutcome, SuccessReason, fixtures}; use fabro_workflow::records::Conclusion; use super::*; @@ -150,7 +149,7 @@ mod tests { let run_id = fixtures::RUN_1; let conclusion = Conclusion { timestamp: chrono::Utc::now(), - status: StageStatus::Success, + status: StageOutcome::Succeeded, duration_ms: 12345, failure_reason: None, final_git_commit_sha: None, @@ -206,7 +205,9 @@ mod tests { let run_id = fixtures::RUN_4; let conclusion = Conclusion { timestamp: chrono::Utc::now(), - status: StageStatus::Fail, + status: StageOutcome::Failed { + retry_requested: false, + }, duration_ms: 500, failure_reason: Some("error".into()), final_git_commit_sha: None, @@ -231,7 +232,7 @@ mod tests { let run_id = fixtures::RUN_5; let conclusion = Conclusion { timestamp: chrono::Utc::now(), - status: StageStatus::Success, + status: StageOutcome::Succeeded, duration_ms: 8000, failure_reason: None, final_git_commit_sha: None, diff --git a/lib/crates/fabro-client/src/client.rs b/lib/crates/fabro-client/src/client.rs index 3d76f9c42..b22e38f4a 100644 --- a/lib/crates/fabro-client/src/client.rs +++ b/lib/crates/fabro-client/src/client.rs @@ -9,7 +9,7 @@ use bytes::Bytes; use fabro_api::types; use fabro_http::header::{AUTHORIZATION, CONTENT_LENGTH, CONTENT_TYPE}; use fabro_http::multipart::{Form, Part}; -use fabro_model::Model; +use fabro_model::{Model, ModelTestMode, Provider}; use fabro_types::settings::run::MergeStrategy; use fabro_types::{ ArtifactUpload, EventEnvelope, RunBlobId, RunEvent, RunId, RunProjection, RunSummary, StageId, @@ -569,6 +569,13 @@ impl Client { provider: Option<&str>, query: Option<&str>, ) -> Result> { + let provider = provider + .map(|provider| { + provider + .parse::() + .map_err(|_| anyhow!("unknown provider: {provider}")) + }) + .transpose()?; let mut offset = 0u64; let mut models = Vec::new(); @@ -577,7 +584,7 @@ impl Client { .send_api(|client| async move { let mut request = client.list_models().page_limit(100u64).page_offset(offset); if let Some(provider) = provider { - request = request.provider(provider.to_string()); + request = request.provider(provider); } if let Some(query) = query { request = request.query(query.to_string()); @@ -600,7 +607,7 @@ impl Client { pub async fn test_model( &self, id: &str, - mode: Option, + mode: Option, ) -> Result { let response = self .send_api(|client| async move { diff --git a/lib/crates/fabro-llm/src/model_test.rs b/lib/crates/fabro-llm/src/model_test.rs index fa72ed460..bf9c2f55e 100644 --- a/lib/crates/fabro-llm/src/model_test.rs +++ b/lib/crates/fabro-llm/src/model_test.rs @@ -2,7 +2,8 @@ use std::sync::Arc; use std::time::Duration; use fabro_model::Model; -use strum::{EnumString, IntoStaticStr}; +pub use fabro_model::ModelTestMode; +use strum::IntoStaticStr; use tokio::time; use crate::client::Client; @@ -10,24 +11,6 @@ use crate::generate::{self, GenerateParams}; use crate::tools::Tool; use crate::types::{GenerateResult, ReasoningEffort}; -#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, EnumString, IntoStaticStr)] -#[strum(serialize_all = "lowercase")] -pub enum ModelTestMode { - #[default] - Basic, - Deep, -} - -impl ModelTestMode { - #[must_use] - pub const fn timeout_secs(self) -> u64 { - match self { - Self::Basic => 30, - Self::Deep => 90, - } - } -} - #[derive(Debug, Clone, Copy, PartialEq, Eq, IntoStaticStr)] #[strum(serialize_all = "lowercase")] pub enum ModelTestStatus { diff --git a/lib/crates/fabro-model/src/lib.rs b/lib/crates/fabro-model/src/lib.rs index 80fb26087..9a95e5815 100644 --- a/lib/crates/fabro-model/src/lib.rs +++ b/lib/crates/fabro-model/src/lib.rs @@ -1,6 +1,7 @@ pub mod billing; pub mod catalog; pub mod model_ref; +pub mod model_test; pub mod provider; pub mod types; @@ -12,5 +13,6 @@ pub use billing::{ }; pub use catalog::{Catalog, FallbackTarget}; pub use model_ref::ModelHandle; +pub use model_test::ModelTestMode; pub use provider::Provider; pub use types::{Model, ModelCosts, ModelFeatures, ModelLimits}; diff --git a/lib/crates/fabro-model/src/model_test.rs b/lib/crates/fabro-model/src/model_test.rs new file mode 100644 index 000000000..f099bf4c2 --- /dev/null +++ b/lib/crates/fabro-model/src/model_test.rs @@ -0,0 +1,33 @@ +use serde::{Deserialize, Serialize}; +use strum::{Display, EnumString, IntoStaticStr}; + +#[derive( + Debug, + Clone, + Copy, + PartialEq, + Eq, + Default, + Serialize, + Deserialize, + Display, + EnumString, + IntoStaticStr, +)] +#[serde(rename_all = "lowercase")] +#[strum(serialize_all = "lowercase")] +pub enum ModelTestMode { + #[default] + Basic, + Deep, +} + +impl ModelTestMode { + #[must_use] + pub const fn timeout_secs(self) -> u64 { + match self { + Self::Basic => 30, + Self::Deep => 90, + } + } +} diff --git a/lib/crates/fabro-server/src/demo/mod.rs b/lib/crates/fabro-server/src/demo/mod.rs index 8920c94c6..90239e08d 100644 --- a/lib/crates/fabro-server/src/demo/mod.rs +++ b/lib/crates/fabro-server/src/demo/mod.rs @@ -1213,11 +1213,11 @@ mod runs { id: "Opus 4.6".into(), }, billing: BilledTokenCounts { - cache_read_tokens: None, - cache_write_tokens: None, + cache_read_tokens: 0, + cache_write_tokens: 0, input_tokens: 12480, output_tokens: 3210, - reasoning_tokens: None, + reasoning_tokens: 0, total_tokens: 15690, total_usd_micros: Some(480_000), }, @@ -1232,11 +1232,11 @@ mod runs { id: "Gemini 3.1".into(), }, billing: BilledTokenCounts { - cache_read_tokens: None, - cache_write_tokens: None, + cache_read_tokens: 0, + cache_write_tokens: 0, input_tokens: 28640, output_tokens: 8750, - reasoning_tokens: None, + reasoning_tokens: 0, total_tokens: 37390, total_usd_micros: Some(720_000), }, @@ -1251,11 +1251,11 @@ mod runs { id: "Codex 5.3".into(), }, billing: BilledTokenCounts { - cache_read_tokens: None, - cache_write_tokens: None, + cache_read_tokens: 0, + cache_write_tokens: 0, input_tokens: 9120, output_tokens: 2640, - reasoning_tokens: None, + reasoning_tokens: 0, total_tokens: 11760, total_usd_micros: Some(190_000), }, @@ -1270,11 +1270,11 @@ mod runs { id: "Opus 4.6".into(), }, billing: BilledTokenCounts { - cache_read_tokens: None, - cache_write_tokens: None, + cache_read_tokens: 0, + cache_write_tokens: 0, input_tokens: 21300, output_tokens: 6480, - reasoning_tokens: None, + reasoning_tokens: 0, total_tokens: 27780, total_usd_micros: Some(870_000), }, @@ -1282,23 +1282,23 @@ mod runs { }, ], totals: RunBillingTotals { - cache_read_tokens: None, - cache_write_tokens: None, + cache_read_tokens: 0, + cache_write_tokens: 0, runtime_secs: 389.0, input_tokens: 71540, output_tokens: 21080, - reasoning_tokens: None, + reasoning_tokens: 0, total_tokens: 92620, total_usd_micros: Some(2_260_000), }, by_model: vec![ BillingByModel { billing: BilledTokenCounts { - cache_read_tokens: None, - cache_write_tokens: None, + cache_read_tokens: 0, + cache_write_tokens: 0, input_tokens: 33780, output_tokens: 9690, - reasoning_tokens: None, + reasoning_tokens: 0, total_tokens: 43470, total_usd_micros: Some(1_350_000), }, @@ -1309,11 +1309,11 @@ mod runs { }, BillingByModel { billing: BilledTokenCounts { - cache_read_tokens: None, - cache_write_tokens: None, + cache_read_tokens: 0, + cache_write_tokens: 0, input_tokens: 28640, output_tokens: 8750, - reasoning_tokens: None, + reasoning_tokens: 0, total_tokens: 37390, total_usd_micros: Some(720_000), }, @@ -1324,11 +1324,11 @@ mod runs { }, BillingByModel { billing: BilledTokenCounts { - cache_read_tokens: None, - cache_write_tokens: None, + cache_read_tokens: 0, + cache_write_tokens: 0, input_tokens: 9120, output_tokens: 2640, - reasoning_tokens: None, + reasoning_tokens: 0, total_tokens: 11760, total_usd_micros: Some(190_000), }, @@ -1516,12 +1516,12 @@ mod billing { pub(super) fn aggregate() -> AggregateBilling { AggregateBilling { totals: AggregateBillingTotals { - cache_read_tokens: None, - cache_write_tokens: None, + cache_read_tokens: 0, + cache_write_tokens: 0, runs: 9, input_tokens: 643_860, output_tokens: 189_720, - reasoning_tokens: None, + reasoning_tokens: 0, runtime_secs: 3_501.0, total_tokens: 833_580, total_usd_micros: Some(20_340_000), @@ -1529,11 +1529,11 @@ mod billing { by_model: vec![ BillingByModel { billing: BilledTokenCounts { - cache_read_tokens: None, - cache_write_tokens: None, + cache_read_tokens: 0, + cache_write_tokens: 0, input_tokens: 304_020, output_tokens: 87_210, - reasoning_tokens: None, + reasoning_tokens: 0, total_tokens: 391_230, total_usd_micros: Some(12_150_000), }, @@ -1544,11 +1544,11 @@ mod billing { }, BillingByModel { billing: BilledTokenCounts { - cache_read_tokens: None, - cache_write_tokens: None, + cache_read_tokens: 0, + cache_write_tokens: 0, input_tokens: 257_760, output_tokens: 78_750, - reasoning_tokens: None, + reasoning_tokens: 0, total_tokens: 336_510, total_usd_micros: Some(6_480_000), }, @@ -1559,11 +1559,11 @@ mod billing { }, BillingByModel { billing: BilledTokenCounts { - cache_read_tokens: None, - cache_write_tokens: None, + cache_read_tokens: 0, + cache_write_tokens: 0, input_tokens: 82_080, output_tokens: 23_760, - reasoning_tokens: None, + reasoning_tokens: 0, total_tokens: 105_840, total_usd_micros: Some(1_710_000), }, diff --git a/lib/crates/fabro-server/src/server.rs b/lib/crates/fabro-server/src/server.rs index 4d5726d10..6f926dc3c 100644 --- a/lib/crates/fabro-server/src/server.rs +++ b/lib/crates/fabro-server/src/server.rs @@ -24,19 +24,19 @@ use base64::engine::general_purpose::STANDARD as BASE64_STANDARD; use bytes::Bytes; pub use fabro_api::types::{ AggregateBilling, AggregateBillingTotals, ApiQuestion, ApiQuestionOption, AppendEventResponse, - ArtifactEntry, ArtifactListResponse, BilledTokenCounts as ApiBilledTokenCounts, BillingByModel, - BillingStageRef, CloseRunPullRequestResponse, CompletionContentPart, CompletionMessage, - CompletionMessageRole, CompletionResponse, CompletionToolChoiceMode, CompletionUsage, - CreateCompletionRequest, CreateRunPullRequestRequest, CreateSecretRequest, DeleteSecretRequest, - DiskUsageResponse, DiskUsageRunRow, DiskUsageSummaryRow, ForkRequest, ForkResponse, - MergeRunPullRequestRequest, MergeRunPullRequestResponse, ModelReference, PaginatedEventList, - PaginatedRunList, PaginationMeta, PreflightResponse, PreviewUrlRequest, PreviewUrlResponse, - PruneRunEntry, PruneRunsRequest, PruneRunsResponse, RenderWorkflowGraphDirection, - RenderWorkflowGraphRequest, RewindRequest, RewindResponse, RunArtifactEntry, - RunArtifactListResponse, RunBilling, RunBillingStage, RunBillingTotals, RunError, RunManifest, - RunStage, RunStatusResponse, SandboxFileEntry, SandboxFileListResponse, SshAccessRequest, - SshAccessResponse, StageStatus as ApiStageStatus, StartRunRequest, SubmitAnswerRequest, - SystemFeatures, SystemInfoResponse, SystemRunCounts, TimelineEntryResponse, WriteBlobResponse, + ArtifactEntry, ArtifactListResponse, BillingByModel, BillingStageRef, + CloseRunPullRequestResponse, CompletionContentPart, CompletionMessage, CompletionMessageRole, + CompletionResponse, CompletionToolChoiceMode, CompletionUsage, CreateCompletionRequest, + CreateRunPullRequestRequest, CreateSecretRequest, DeleteSecretRequest, DiskUsageResponse, + DiskUsageRunRow, DiskUsageSummaryRow, ForkRequest, ForkResponse, MergeRunPullRequestRequest, + MergeRunPullRequestResponse, ModelReference, PaginatedEventList, PaginatedRunList, + PaginationMeta, PreflightResponse, PreviewUrlRequest, PreviewUrlResponse, PruneRunEntry, + PruneRunsRequest, PruneRunsResponse, RenderWorkflowGraphDirection, RenderWorkflowGraphRequest, + RewindRequest, RewindResponse, RunArtifactEntry, RunArtifactListResponse, RunBilling, + RunBillingStage, RunBillingTotals, RunError, RunManifest, RunStage, RunStatusResponse, + SandboxFileEntry, SandboxFileListResponse, SshAccessRequest, SshAccessResponse, + StageStatus as ApiStageStatus, StartRunRequest, SubmitAnswerRequest, SystemFeatures, + SystemInfoResponse, SystemRunCounts, TimelineEntryResponse, WriteBlobResponse, }; use fabro_auth::{ CredentialSource, VaultCredentialSource, auth_issue_message, parse_credential_secret, @@ -46,12 +46,12 @@ use fabro_config::{RunLayer, RunSettingsBuilder, ServerSettingsBuilder, Storage, use fabro_interview::{Answer, ControlInterviewer, Interviewer, Question, WorkerControlEnvelope}; use fabro_llm::client::Client as LlmClient; use fabro_llm::generate::{GenerateParams, generate_object}; -use fabro_llm::model_test::{ModelTestMode, run_model_test}; +use fabro_llm::model_test::run_model_test; use fabro_llm::types::{ ContentPart, FinishReason, Message as LlmMessage, Request as LlmRequest, Role, ToolChoice, ToolDefinition, }; -use fabro_model::{BilledModelUsage, BilledTokenCounts, Catalog}; +use fabro_model::{BilledModelUsage, BilledTokenCounts, Catalog, ModelTestMode, Provider}; use fabro_redact::redact_jsonl_line; use fabro_sandbox::daytona::DaytonaSandbox; use fabro_sandbox::reconnect::reconnect; @@ -637,35 +637,6 @@ pub(crate) struct ResolvedAppStateSettings { pub(crate) manifest_run_settings: std::result::Result, } -fn nonzero_i64(value: i64) -> Option { - (value != 0).then_some(value) -} - -fn api_billed_token_counts_from_domain(billing: &BilledTokenCounts) -> ApiBilledTokenCounts { - ApiBilledTokenCounts { - cache_read_tokens: nonzero_i64(billing.cache_read_tokens), - cache_write_tokens: nonzero_i64(billing.cache_write_tokens), - input_tokens: billing.input_tokens, - output_tokens: billing.output_tokens, - reasoning_tokens: nonzero_i64(billing.reasoning_tokens), - total_tokens: billing.total_tokens, - total_usd_micros: billing.total_usd_micros, - } -} - -fn api_billed_token_counts_from_usage(usage: &BilledModelUsage) -> ApiBilledTokenCounts { - let tokens = usage.tokens(); - ApiBilledTokenCounts { - cache_read_tokens: nonzero_i64(tokens.cache_read_tokens), - cache_write_tokens: nonzero_i64(tokens.cache_write_tokens), - input_tokens: tokens.input_tokens, - output_tokens: tokens.output_tokens, - reasoning_tokens: nonzero_i64(tokens.reasoning_tokens), - total_tokens: tokens.total_tokens(), - total_usd_micros: usage.total_usd_micros, - } -} - fn accumulate_model_billing(entry: &mut ModelBillingTotals, usage: &BilledModelUsage) { let tokens = usage.tokens(); entry.stages += 1; @@ -2190,32 +2161,34 @@ async fn get_aggregate_billing( .by_model .iter() .map(|(model, totals)| BillingByModel { - billing: api_billed_token_counts_from_domain(&totals.billing), + billing: totals.billing.clone(), model: ModelReference { id: model.clone() }, stages: totals.stages, }) .collect(); - let total_billing = by_model - .iter() - .fold(BilledTokenCounts::default(), |mut acc, model| { - acc.input_tokens += model.billing.input_tokens; - acc.output_tokens += model.billing.output_tokens; - acc.reasoning_tokens += model.billing.reasoning_tokens.unwrap_or(0); - acc.cache_read_tokens += model.billing.cache_read_tokens.unwrap_or(0); - acc.cache_write_tokens += model.billing.cache_write_tokens.unwrap_or(0); - acc.total_tokens += model.billing.total_tokens; - if let Some(value) = model.billing.total_usd_micros { - *acc.total_usd_micros.get_or_insert(0) += value; - } - acc - }); + let total_billing = + agg.by_model + .values() + .fold(BilledTokenCounts::default(), |mut acc, totals| { + let billing = &totals.billing; + acc.input_tokens += billing.input_tokens; + acc.output_tokens += billing.output_tokens; + acc.reasoning_tokens += billing.reasoning_tokens; + acc.cache_read_tokens += billing.cache_read_tokens; + acc.cache_write_tokens += billing.cache_write_tokens; + acc.total_tokens += billing.total_tokens; + if let Some(value) = billing.total_usd_micros { + *acc.total_usd_micros.get_or_insert(0) += value; + } + acc + }); let response = AggregateBilling { totals: AggregateBillingTotals { - cache_read_tokens: nonzero_i64(total_billing.cache_read_tokens), - cache_write_tokens: nonzero_i64(total_billing.cache_write_tokens), + cache_read_tokens: total_billing.cache_read_tokens, + cache_write_tokens: total_billing.cache_write_tokens, input_tokens: total_billing.input_tokens, output_tokens: total_billing.output_tokens, - reasoning_tokens: nonzero_i64(total_billing.reasoning_tokens), + reasoning_tokens: total_billing.reasoning_tokens, runs: agg.total_runs, runtime_secs: agg.total_runtime_secs, total_tokens: total_billing.total_tokens, @@ -2352,11 +2325,11 @@ async fn get_run_billing( by_model: Vec::new(), stages: Vec::new(), totals: RunBillingTotals { - cache_read_tokens: None, - cache_write_tokens: None, + cache_read_tokens: 0, + cache_write_tokens: 0, input_tokens: 0, output_tokens: 0, - reasoning_tokens: None, + reasoning_tokens: 0, runtime_secs: 0.0, total_tokens: 0, total_usd_micros: None, @@ -2391,7 +2364,16 @@ async fn get_run_billing( }; billed_usages.push(usage.clone()); - let billing = api_billed_token_counts_from_usage(usage); + let tokens = usage.tokens(); + let billing = BilledTokenCounts { + cache_read_tokens: tokens.cache_read_tokens, + cache_write_tokens: tokens.cache_write_tokens, + input_tokens: tokens.input_tokens, + output_tokens: tokens.output_tokens, + reasoning_tokens: tokens.reasoning_tokens, + total_tokens: tokens.total_tokens(), + total_usd_micros: usage.total_usd_micros, + }; let model_id = usage.model_id().to_string(); accumulate_model_billing(by_model_totals.entry(model_id.clone()).or_default(), usage); stages.push(RunBillingStage { @@ -2409,7 +2391,7 @@ async fn get_run_billing( let by_model = by_model_totals .into_iter() .map(|(model, totals)| BillingByModel { - billing: api_billed_token_counts_from_domain(&totals.billing), + billing: totals.billing, model: ModelReference { id: model }, stages: totals.stages, }) @@ -2419,11 +2401,11 @@ async fn get_run_billing( by_model, stages, totals: RunBillingTotals { - cache_read_tokens: nonzero_i64(totals.cache_read_tokens), - cache_write_tokens: nonzero_i64(totals.cache_write_tokens), + cache_read_tokens: totals.cache_read_tokens, + cache_write_tokens: totals.cache_write_tokens, input_tokens: totals.input_tokens, output_tokens: totals.output_tokens, - reasoning_tokens: nonzero_i64(totals.reasoning_tokens), + reasoning_tokens: totals.reasoning_tokens, runtime_secs, total_tokens: totals.total_tokens, total_usd_micros: totals.total_usd_micros, @@ -7439,7 +7421,7 @@ async fn list_models( Query(params): Query, ) -> Response { let provider = match params.provider.as_deref() { - Some(value) => match fabro_model::Provider::from_str(value) { + Some(value) => match Provider::from_str(value) { Ok(provider) => Some(provider), Err(_) => { return ApiError::new(