mirror of
https://github.com/fabro-sh/fabro.git
synced 2026-10-09 03:20:56 +00:00
Simplify workflow bundling extraction
This commit is contained in:
parent
17bc48acf1
commit
227e520400
3 changed files with 717 additions and 1768 deletions
|
|
@ -3,10 +3,10 @@
|
|||
reason = "CLI manifest builder: sync file I/O building install manifests"
|
||||
)]
|
||||
|
||||
mod workflow_bundle;
|
||||
mod workflow_bundler;
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::path::{Component, Path, PathBuf};
|
||||
|
||||
use anyhow::{Context, Result, anyhow};
|
||||
use fabro_api::types;
|
||||
|
|
@ -21,17 +21,13 @@ use fabro_graphviz::graph::AttrValue;
|
|||
use fabro_graphviz::parser;
|
||||
use fabro_types::settings::interp::InterpString;
|
||||
use fabro_types::settings::run::{ApprovalMode, ResolvedGoalSource, ResolvedRunGoal, RunMode};
|
||||
use fabro_types::{DirtyStatus, GitContext, WorkflowSettings};
|
||||
use fabro_types::{DirtyStatus, GitContext, ManifestPath, WorkflowSettings};
|
||||
use fabro_workflow::git::{
|
||||
GitSyncStatus, branch_needs_push, head_sha, push_branch_noninteractive, sync_status,
|
||||
};
|
||||
use fabro_workflow::static_reference::ReferenceKind;
|
||||
|
||||
use crate::workflow_bundle::{
|
||||
CollectWorkflowBundleInput, CollectedDocument, CollectedFileReferenceType,
|
||||
CollectedSourceInput, CollectedWorkflowBundle, manifest_path_from_absolute,
|
||||
normalize_absolute_path,
|
||||
};
|
||||
use crate::workflow_bundler::WorkflowBundler;
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
pub struct ManifestBuildInput {
|
||||
|
|
@ -138,13 +134,14 @@ pub fn build_run_manifest(input: ManifestBuildInput) -> Result<BuiltManifest> {
|
|||
}
|
||||
let project_config = discover_project_config(&root_location.dir)?;
|
||||
let project_config_source = project_config
|
||||
.as_deref()
|
||||
.map(read_source_input)
|
||||
.as_ref()
|
||||
.map(|path| {
|
||||
let source = std::fs::read_to_string(path)
|
||||
.with_context(|| format!("Failed to read {}", path.display()))?;
|
||||
let manifest_path = manifest_path_from_absolute(path, &input.cwd)?;
|
||||
Ok::<_, anyhow::Error>((path.clone(), manifest_path, source))
|
||||
})
|
||||
.transpose()?;
|
||||
let user_settings_path = input
|
||||
.user_settings_path
|
||||
.as_deref()
|
||||
.filter(|path| path.is_file());
|
||||
|
||||
let mut workflow_settings_builder = WorkflowSettingsBuilder::new()
|
||||
.server_manifest_defaults(RunLayer::default(), input.environment_defaults.clone());
|
||||
|
|
@ -160,7 +157,11 @@ pub fn build_run_manifest(input: ManifestBuildInput) -> Result<BuiltManifest> {
|
|||
if let Some(path) = project_config.as_ref() {
|
||||
workflow_settings_builder = workflow_settings_builder.project_file(path)?;
|
||||
}
|
||||
if let Some(path) = user_settings_path {
|
||||
if let Some(path) = input
|
||||
.user_settings_path
|
||||
.as_ref()
|
||||
.filter(|path| path.is_file())
|
||||
{
|
||||
workflow_settings_builder = workflow_settings_builder.user_file(path)?;
|
||||
}
|
||||
let mut workflow_settings = workflow_settings_builder
|
||||
|
|
@ -168,15 +169,35 @@ pub fn build_run_manifest(input: ManifestBuildInput) -> Result<BuiltManifest> {
|
|||
.context("failed to resolve manifest settings")?;
|
||||
workflow_settings.run.inputs.extend(input.input_overrides);
|
||||
let target_path = root_location.graph.clone();
|
||||
let user_config_source = user_settings_path.map(read_source_input).transpose()?;
|
||||
let collected = workflow_bundle::collect_workflow_bundle(CollectWorkflowBundleInput {
|
||||
cwd: &input.cwd,
|
||||
root_location,
|
||||
inputs: &workflow_settings.run.inputs,
|
||||
project_config: project_config_source,
|
||||
user_config: user_config_source,
|
||||
})?;
|
||||
let assembled = assemble_current_manifest(collected, &input.cwd)?;
|
||||
let target_manifest_path = manifest_path_from_absolute(&target_path, &input.cwd)?;
|
||||
let target_key = target_manifest_path.to_string();
|
||||
let project_config_input = project_config_source
|
||||
.as_ref()
|
||||
.map(|(_, path, source)| (path, source.as_str()));
|
||||
let workflows = WorkflowBundler::new(&input.cwd, &workflow_settings.run.inputs)
|
||||
.bundle(&root_location, project_config_input)?;
|
||||
let root_source = workflows
|
||||
.get(&target_key)
|
||||
.map(|workflow| workflow.source.clone())
|
||||
.ok_or_else(|| anyhow!("root workflow missing from manifest bundle"))?;
|
||||
|
||||
let mut configs = Vec::new();
|
||||
if let Some((path, _, source)) = project_config_source {
|
||||
configs.push(types::ManifestConfig {
|
||||
path: Some(path.display().to_string()),
|
||||
source: Some(source),
|
||||
type_: types::ManifestConfigType::Project,
|
||||
});
|
||||
}
|
||||
if let Some(path) = input.user_settings_path.filter(|path| path.is_file()) {
|
||||
let source = std::fs::read_to_string(&path)
|
||||
.with_context(|| format!("Failed to read {}", path.display()))?;
|
||||
configs.push(types::ManifestConfig {
|
||||
path: Some(path.display().to_string()),
|
||||
source: Some(source),
|
||||
type_: types::ManifestConfigType::User,
|
||||
});
|
||||
}
|
||||
|
||||
let working_directory =
|
||||
project::resolve_working_directory_from_run(&workflow_settings.run, &input.cwd);
|
||||
|
|
@ -184,7 +205,7 @@ pub fn build_run_manifest(input: ManifestBuildInput) -> Result<BuiltManifest> {
|
|||
let goal = resolve_manifest_goal(
|
||||
input.run_overrides.as_ref(),
|
||||
&workflow_settings,
|
||||
&assembled.root_source,
|
||||
&root_source,
|
||||
&target_path,
|
||||
&working_directory,
|
||||
)?;
|
||||
|
|
@ -196,118 +217,20 @@ pub fn build_run_manifest(input: ManifestBuildInput) -> Result<BuiltManifest> {
|
|||
Ok(BuiltManifest {
|
||||
manifest: types::RunManifest {
|
||||
args,
|
||||
configs,
|
||||
cwd: input.cwd.display().to_string(),
|
||||
git,
|
||||
goal,
|
||||
parent_id: None,
|
||||
title: None,
|
||||
target: types::ManifestTarget {
|
||||
path: assembled.target_key,
|
||||
},
|
||||
target: types::ManifestTarget { path: target_key },
|
||||
version: 1,
|
||||
workflows: assembled.workflows,
|
||||
configs: assembled.configs,
|
||||
workflows,
|
||||
},
|
||||
target_path,
|
||||
})
|
||||
}
|
||||
|
||||
struct AssembledCurrentManifest {
|
||||
target_key: String,
|
||||
root_source: String,
|
||||
workflows: HashMap<String, types::ManifestWorkflow>,
|
||||
configs: Vec<types::ManifestConfig>,
|
||||
}
|
||||
|
||||
fn assemble_current_manifest(
|
||||
collected: CollectedWorkflowBundle,
|
||||
cwd: &Path,
|
||||
) -> Result<AssembledCurrentManifest> {
|
||||
let root = collected
|
||||
.workflows
|
||||
.get(&collected.entrypoint)
|
||||
.ok_or_else(|| anyhow!("root workflow missing from collected workflow bundle"))?;
|
||||
let target_key = manifest_path_from_absolute(&root.graph.access_path, cwd)?.to_string();
|
||||
let root_source = root.graph.source.clone();
|
||||
|
||||
let mut workflows = HashMap::new();
|
||||
for workflow in collected.workflows.into_values() {
|
||||
let graph_key = manifest_path_from_absolute(&workflow.graph.access_path, cwd)?.to_string();
|
||||
let config = workflow
|
||||
.config
|
||||
.map(|config| {
|
||||
Ok::<_, anyhow::Error>(types::ManifestWorkflowConfig {
|
||||
path: manifest_path_from_absolute(&config.access_path, cwd)?.to_string(),
|
||||
source: config.source,
|
||||
})
|
||||
})
|
||||
.transpose()?;
|
||||
let mut files = HashMap::new();
|
||||
for file in workflow.files.into_values() {
|
||||
let key = manifest_path_from_absolute(&file.document.access_path, cwd)?.to_string();
|
||||
let from = file
|
||||
.reference
|
||||
.from_access_path
|
||||
.as_deref()
|
||||
.map(|path| manifest_path_from_absolute(path, cwd).map(|path| path.to_string()))
|
||||
.transpose()?;
|
||||
let type_ = match file.reference.type_ {
|
||||
CollectedFileReferenceType::FileInline => types::ManifestFileRefType::FileInline,
|
||||
CollectedFileReferenceType::Import => types::ManifestFileRefType::Import,
|
||||
CollectedFileReferenceType::Dockerfile => types::ManifestFileRefType::Dockerfile,
|
||||
};
|
||||
files.insert(key, types::ManifestFileEntry {
|
||||
content: file.document.source,
|
||||
ref_: types::ManifestFileRef {
|
||||
from,
|
||||
original: file.reference.original,
|
||||
type_,
|
||||
},
|
||||
});
|
||||
}
|
||||
workflows.insert(graph_key, types::ManifestWorkflow {
|
||||
source: workflow.graph.source,
|
||||
config,
|
||||
files,
|
||||
});
|
||||
}
|
||||
|
||||
let mut configs = Vec::new();
|
||||
if let Some(config) = collected.project_config {
|
||||
configs.push(manifest_config(config, types::ManifestConfigType::Project));
|
||||
}
|
||||
if let Some(config) = collected.user_config {
|
||||
configs.push(manifest_config(config, types::ManifestConfigType::User));
|
||||
}
|
||||
|
||||
Ok(AssembledCurrentManifest {
|
||||
target_key,
|
||||
root_source,
|
||||
workflows,
|
||||
configs,
|
||||
})
|
||||
}
|
||||
|
||||
fn manifest_config(
|
||||
document: CollectedDocument,
|
||||
type_: types::ManifestConfigType,
|
||||
) -> types::ManifestConfig {
|
||||
types::ManifestConfig {
|
||||
path: Some(document.access_path.display().to_string()),
|
||||
source: Some(document.source),
|
||||
type_,
|
||||
}
|
||||
}
|
||||
|
||||
fn read_source_input(path: &Path) -> Result<CollectedSourceInput> {
|
||||
let source = std::fs::read_to_string(path)
|
||||
.with_context(|| format!("Failed to read {}", path.display()))?;
|
||||
Ok(CollectedSourceInput {
|
||||
access_path: path.to_path_buf(),
|
||||
source,
|
||||
})
|
||||
}
|
||||
|
||||
fn resolve_manifest_goal(
|
||||
run_overrides: Option<&RunLayer>,
|
||||
settings: &WorkflowSettings,
|
||||
|
|
@ -473,6 +396,32 @@ fn push_manifest_branch_best_effort(
|
|||
let _ = push_branch_noninteractive(repo_path, "origin", branch);
|
||||
}
|
||||
|
||||
fn normalize_absolute_path(base_dir: &Path, reference: &str) -> Option<PathBuf> {
|
||||
let path = Path::new(reference);
|
||||
if path.is_absolute() || reference.starts_with('~') {
|
||||
return None;
|
||||
}
|
||||
|
||||
let mut normalized = PathBuf::new();
|
||||
for component in base_dir.join(path).components() {
|
||||
match component {
|
||||
Component::CurDir => {}
|
||||
Component::Normal(part) => normalized.push(part),
|
||||
Component::ParentDir => {
|
||||
normalized.pop();
|
||||
}
|
||||
Component::RootDir => normalized.push(Path::new("/")),
|
||||
Component::Prefix(prefix) => normalized.push(prefix.as_os_str()),
|
||||
}
|
||||
}
|
||||
Some(normalized)
|
||||
}
|
||||
|
||||
fn manifest_path_from_absolute(path: &Path, cwd: &Path) -> Result<ManifestPath> {
|
||||
ManifestPath::from_absolute(path, cwd)
|
||||
.ok_or_else(|| anyhow!("Failed to compute manifest path for {}", path.display()))
|
||||
}
|
||||
|
||||
pub fn manifest_args_is_empty(args: &types::ManifestArgs) -> bool {
|
||||
args.auto_approve.is_none()
|
||||
&& args.dry_run.is_none()
|
||||
|
|
@ -699,19 +648,19 @@ graph = "workflow.fabro"
|
|||
),
|
||||
".fabro/workflows/root/prompts/deep.md": file(
|
||||
"deep\n",
|
||||
".fabro/workflows/root/workflow.fabro",
|
||||
".fabro/workflows/root/prompts/plan.md",
|
||||
".fabro/workflows/root/prompts/deep.md",
|
||||
"file_inline",
|
||||
),
|
||||
".fabro/workflows/root/prompts/helpers.md": file(
|
||||
helpers,
|
||||
".fabro/workflows/root/workflow.fabro",
|
||||
".fabro/workflows/root/prompts/plan.md",
|
||||
".fabro/workflows/root/prompts/helpers.md",
|
||||
"file_inline",
|
||||
),
|
||||
".fabro/workflows/root/prompts/partial.md": file(
|
||||
"partial\n",
|
||||
".fabro/workflows/root/workflow.fabro",
|
||||
".fabro/workflows/root/prompts/plan.md",
|
||||
".fabro/workflows/root/prompts/partial.md",
|
||||
"file_inline",
|
||||
),
|
||||
|
|
@ -1057,17 +1006,26 @@ graph = "workflow.fabro"
|
|||
.unwrap();
|
||||
|
||||
let root = &built.manifest.workflows[".fabro/workflows/demo/workflow.fabro"];
|
||||
assert!(
|
||||
root.files
|
||||
.contains_key(".fabro/workflows/demo/prompts/goal.tpl.md")
|
||||
assert_eq!(
|
||||
root.files[".fabro/workflows/demo/prompts/goal.tpl.md"]
|
||||
.ref_
|
||||
.from
|
||||
.as_deref(),
|
||||
Some(".fabro/workflows/demo/prompts/goal.md")
|
||||
);
|
||||
assert!(
|
||||
root.files
|
||||
.contains_key(".fabro/workflows/demo/prompts/plan.tpl.md")
|
||||
assert_eq!(
|
||||
root.files[".fabro/workflows/demo/prompts/plan.tpl.md"]
|
||||
.ref_
|
||||
.from
|
||||
.as_deref(),
|
||||
Some(".fabro/workflows/demo/prompts/plan.md")
|
||||
);
|
||||
assert!(
|
||||
root.files
|
||||
.contains_key(".fabro/workflows/demo/inline.tpl.md")
|
||||
assert_eq!(
|
||||
root.files[".fabro/workflows/demo/inline.tpl.md"]
|
||||
.ref_
|
||||
.from
|
||||
.as_deref(),
|
||||
Some(".fabro/workflows/demo/workflow.fabro")
|
||||
);
|
||||
}
|
||||
|
||||
|
|
@ -1166,8 +1124,9 @@ graph = "workflow.fabro"
|
|||
.unwrap_err();
|
||||
|
||||
assert!(
|
||||
err.chain()
|
||||
.any(|cause| cause.to_string().contains("dynamic template dependency")),
|
||||
err.chain().any(|cause| cause
|
||||
.downcast_ref::<fabro_template::TemplateDiscoveryError>()
|
||||
.is_some()),
|
||||
"unexpected error: {err:#}"
|
||||
);
|
||||
}
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
618
lib/components/fabro-manifest/src/workflow_bundler.rs
Normal file
618
lib/components/fabro-manifest/src/workflow_bundler.rs
Normal file
|
|
@ -0,0 +1,618 @@
|
|||
use std::collections::{HashMap, HashSet};
|
||||
use std::path::{Component, Path, PathBuf};
|
||||
use std::sync::Arc;
|
||||
|
||||
use anyhow::{Context as _, Result, anyhow};
|
||||
use fabro_api::types;
|
||||
use fabro_config::project::WorkflowLocation;
|
||||
use fabro_config::{EnvironmentDockerfileLayer, EnvironmentImageLayer, SettingsLayer};
|
||||
use fabro_graphviz::graph::AttrValue;
|
||||
use fabro_graphviz::parser;
|
||||
use fabro_template::{
|
||||
BundleTemplateStore, FilesystemTemplateStore, RecordingTemplateStore, TemplateContext,
|
||||
TemplateDependencyClosure, TemplateRenderMode, TemplateSource,
|
||||
};
|
||||
use fabro_types::ManifestPath;
|
||||
use fabro_workflow::static_reference::{self, AttributeScope, ReferenceKind};
|
||||
|
||||
use crate::{manifest_path_from_absolute, normalize_absolute_path};
|
||||
|
||||
pub(super) struct WorkflowBundler<'a> {
|
||||
cwd: &'a Path,
|
||||
inputs: &'a HashMap<String, toml::Value>,
|
||||
workflows: HashMap<String, types::ManifestWorkflow>,
|
||||
visited_workflows: HashSet<String>,
|
||||
}
|
||||
|
||||
impl<'a> WorkflowBundler<'a> {
|
||||
pub(super) fn new(cwd: &'a Path, inputs: &'a HashMap<String, toml::Value>) -> Self {
|
||||
Self {
|
||||
cwd,
|
||||
inputs,
|
||||
workflows: HashMap::new(),
|
||||
visited_workflows: HashSet::new(),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn bundle(
|
||||
mut self,
|
||||
root_location: &WorkflowLocation,
|
||||
project_config: Option<(&ManifestPath, &str)>,
|
||||
) -> Result<HashMap<String, types::ManifestWorkflow>> {
|
||||
self.collect_workflow_location(root_location)?;
|
||||
|
||||
if let Some((config_path, source)) = project_config {
|
||||
let root_key = manifest_path_from_absolute(&root_location.graph, self.cwd)?.to_string();
|
||||
let mut root = self
|
||||
.workflows
|
||||
.remove(&root_key)
|
||||
.ok_or_else(|| anyhow!("root workflow missing from manifest bundle"))?;
|
||||
self.collect_config_dockerfile(config_path, source, &mut root.files)?;
|
||||
self.workflows.insert(root_key, root);
|
||||
}
|
||||
|
||||
Ok(self.workflows)
|
||||
}
|
||||
|
||||
fn collect_workflow_location(&mut self, location: &WorkflowLocation) -> Result<()> {
|
||||
let dot_path = manifest_path_from_absolute(&location.graph, self.cwd)?;
|
||||
let dot_key = dot_path.to_string();
|
||||
if !self.visited_workflows.insert(dot_key.clone()) {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let source = std::fs::read_to_string(&location.graph)
|
||||
.with_context(|| format!("Failed to read {}", location.graph.display()))?;
|
||||
let config = if let Some(workflow_toml_path) = location.toml.as_ref() {
|
||||
Some(types::ManifestWorkflowConfig {
|
||||
path: manifest_path_from_absolute(workflow_toml_path, self.cwd)?.to_string(),
|
||||
source: std::fs::read_to_string(workflow_toml_path)
|
||||
.with_context(|| format!("Failed to read {}", workflow_toml_path.display()))?,
|
||||
})
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
let scan = WorkflowScanInput {
|
||||
absolute_dot_path: location.graph.clone(),
|
||||
dot_path,
|
||||
source: source.clone(),
|
||||
};
|
||||
let mut files = HashMap::new();
|
||||
let mut visited_imports = HashSet::new();
|
||||
if let Some(config) = config.as_ref() {
|
||||
let config_path = ManifestPath::from_wire(&config.path)
|
||||
.ok_or_else(|| anyhow!("invalid manifest workflow config path: {}", config.path))?;
|
||||
self.collect_config_dockerfile(&config_path, &config.source, &mut files)?;
|
||||
}
|
||||
self.collect_workflow_files(&scan, &mut files, &mut visited_imports)?;
|
||||
|
||||
self.workflows.insert(dot_key, types::ManifestWorkflow {
|
||||
config,
|
||||
files,
|
||||
source,
|
||||
});
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn collect_workflow_entry(&mut self, workflow: &Path, resolve_from: &Path) -> Result<()> {
|
||||
let normalized_workflow = if workflow.extension().is_some() && workflow.is_relative() {
|
||||
normalize_absolute_path(resolve_from, &workflow.to_string_lossy()).ok_or_else(|| {
|
||||
anyhow!(
|
||||
"unsupported manifest workflow reference: {}",
|
||||
workflow.display()
|
||||
)
|
||||
})?
|
||||
} else {
|
||||
workflow.to_path_buf()
|
||||
};
|
||||
let location = WorkflowLocation::resolve(&normalized_workflow, resolve_from)?;
|
||||
self.collect_workflow_location(&location)
|
||||
}
|
||||
|
||||
fn collect_workflow_files(
|
||||
&mut self,
|
||||
workflow: &WorkflowScanInput,
|
||||
files: &mut HashMap<String, types::ManifestFileEntry>,
|
||||
visited_imports: &mut HashSet<String>,
|
||||
) -> Result<()> {
|
||||
let graph = parser::parse(&workflow.source)
|
||||
.with_context(|| format!("Failed to parse {}", workflow.absolute_dot_path.display()))?;
|
||||
let workflow_base_dir = workflow
|
||||
.absolute_dot_path
|
||||
.parent()
|
||||
.unwrap_or_else(|| Path::new("."));
|
||||
let workflow_template_root = manifest_parent_or_dot(&workflow.dot_path)?;
|
||||
|
||||
if let Some(goal_ref) = graph.attrs.get("goal").and_then(AttrValue::as_str) {
|
||||
if goal_ref.starts_with('@') {
|
||||
let bundled = self.collect_bundled_file(
|
||||
files,
|
||||
workflow_base_dir,
|
||||
goal_ref.trim_start_matches('@'),
|
||||
types::ManifestFileRefType::FileInline,
|
||||
manifest_attr_reference_kind(AttributeScope::Graph, "goal", goal_ref)?,
|
||||
Some(workflow.dot_path.clone()),
|
||||
)?;
|
||||
self.collect_bundled_template_includes(files, &bundled, &workflow_template_root)?;
|
||||
} else {
|
||||
self.collect_template_include_files(
|
||||
files,
|
||||
TemplateSource::new(
|
||||
workflow.dot_path.clone(),
|
||||
workflow_template_root.clone(),
|
||||
goal_ref.to_owned(),
|
||||
),
|
||||
Some(&workflow.dot_path),
|
||||
)?;
|
||||
}
|
||||
}
|
||||
|
||||
for node in graph.nodes.values() {
|
||||
if let Some(prompt_ref) = node.attrs.get("prompt").and_then(AttrValue::as_str) {
|
||||
if !prompt_ref.starts_with('@') {
|
||||
self.collect_template_include_files(
|
||||
files,
|
||||
TemplateSource::new(
|
||||
workflow.dot_path.clone(),
|
||||
workflow_template_root.clone(),
|
||||
prompt_ref.to_owned(),
|
||||
),
|
||||
Some(&workflow.dot_path),
|
||||
)?;
|
||||
}
|
||||
}
|
||||
|
||||
for (name, value) in &node.attrs {
|
||||
let Some(value) = value.as_str() else {
|
||||
continue;
|
||||
};
|
||||
let Some(ReferenceKind::FileInline) =
|
||||
static_reference::reference_kind_for_attribute(
|
||||
AttributeScope::Node,
|
||||
name,
|
||||
value,
|
||||
)
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
let reference = value.strip_prefix('@').ok_or_else(|| {
|
||||
anyhow!("file inline reference must start with '@': {name}={value}")
|
||||
})?;
|
||||
let bundled = self.collect_bundled_file(
|
||||
files,
|
||||
workflow_base_dir,
|
||||
reference,
|
||||
types::ManifestFileRefType::FileInline,
|
||||
ReferenceKind::FileInline,
|
||||
Some(workflow.dot_path.clone()),
|
||||
)?;
|
||||
|
||||
if name == "prompt" {
|
||||
self.collect_bundled_template_includes(
|
||||
files,
|
||||
&bundled,
|
||||
&workflow_template_root,
|
||||
)?;
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(import_ref) = node.attrs.get("import").and_then(AttrValue::as_str) {
|
||||
let imported = self.collect_bundled_file(
|
||||
files,
|
||||
workflow_base_dir,
|
||||
import_ref,
|
||||
types::ManifestFileRefType::Import,
|
||||
manifest_attr_reference_kind(AttributeScope::Node, "import", import_ref)?,
|
||||
Some(workflow.dot_path.clone()),
|
||||
)?;
|
||||
let import_key = imported.path.to_string();
|
||||
if visited_imports.insert(import_key) {
|
||||
let imported_source = std::fs::read_to_string(&imported.absolute_path)
|
||||
.with_context(|| {
|
||||
format!("Failed to read {}", imported.absolute_path.display())
|
||||
})?;
|
||||
let imported_scan = WorkflowScanInput {
|
||||
absolute_dot_path: imported.absolute_path,
|
||||
dot_path: imported.path,
|
||||
source: imported_source,
|
||||
};
|
||||
self.collect_workflow_files(&imported_scan, files, visited_imports)?;
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(child_ref) = node
|
||||
.attrs
|
||||
.get("stack.child_workflow")
|
||||
.and_then(AttrValue::as_str)
|
||||
{
|
||||
manifest_attr_reference_kind(
|
||||
AttributeScope::Node,
|
||||
"stack.child_workflow",
|
||||
child_ref,
|
||||
)?
|
||||
.validate(child_ref)
|
||||
.map_err(anyhow::Error::new)?;
|
||||
self.collect_workflow_entry(Path::new(child_ref), workflow_base_dir)?;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn collect_bundled_template_includes(
|
||||
&self,
|
||||
files: &mut HashMap<String, types::ManifestFileEntry>,
|
||||
bundled: &BundledFile,
|
||||
workflow_template_root: &ManifestPath,
|
||||
) -> Result<()> {
|
||||
let source = std::fs::read_to_string(&bundled.absolute_path)
|
||||
.with_context(|| format!("Failed to read {}", bundled.absolute_path.display()))?;
|
||||
let template_root = template_root_for_bundled_file(&bundled.path, workflow_template_root)?;
|
||||
self.collect_template_include_files(
|
||||
files,
|
||||
TemplateSource::new(bundled.path.clone(), template_root, source),
|
||||
Some(&bundled.path),
|
||||
)
|
||||
}
|
||||
|
||||
fn collect_template_include_files(
|
||||
&self,
|
||||
files: &mut HashMap<String, types::ManifestFileEntry>,
|
||||
source: TemplateSource,
|
||||
from: Option<&ManifestPath>,
|
||||
) -> Result<()> {
|
||||
let source_path = source.path.clone();
|
||||
let store = FilesystemTemplateStore::new(self.cwd.to_path_buf());
|
||||
let closure = fabro_template::discover_static_dependency_closure([source], &store)
|
||||
.context("failed to discover template dependencies")?;
|
||||
self.verify_recorded_template_dependencies(&source_path, &closure, files, from)?;
|
||||
|
||||
for (path, source) in closure.sources {
|
||||
if path == source_path {
|
||||
continue;
|
||||
}
|
||||
let key = path.to_string();
|
||||
files
|
||||
.entry(key)
|
||||
.or_insert_with(|| types::ManifestFileEntry {
|
||||
content: source.content,
|
||||
ref_: types::ManifestFileRef {
|
||||
from: from.map(std::string::ToString::to_string),
|
||||
original: path.to_string(),
|
||||
type_: types::ManifestFileRefType::FileInline,
|
||||
},
|
||||
});
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn verify_recorded_template_dependencies(
|
||||
&self,
|
||||
source_path: &ManifestPath,
|
||||
closure: &TemplateDependencyClosure,
|
||||
files: &HashMap<String, types::ManifestFileEntry>,
|
||||
from: Option<&ManifestPath>,
|
||||
) -> Result<()> {
|
||||
let Some(source) = closure.sources.get(source_path) else {
|
||||
return Ok(());
|
||||
};
|
||||
let mut bundled_files = closure
|
||||
.sources
|
||||
.iter()
|
||||
.map(|(path, source)| (path.clone(), source.content.clone()))
|
||||
.collect::<HashMap<_, _>>();
|
||||
for (path, entry) in files {
|
||||
if let Some(path) = ManifestPath::from_wire(path) {
|
||||
bundled_files.insert(path, entry.content.clone());
|
||||
}
|
||||
}
|
||||
let allowed = bundled_files.keys().cloned().collect();
|
||||
let store =
|
||||
RecordingTemplateStore::with_allowed(BundleTemplateStore::new(bundled_files), allowed);
|
||||
let context = TemplateContext::for_input_scan(self.inputs.clone());
|
||||
fabro_template::render_source(
|
||||
source,
|
||||
&context,
|
||||
Arc::new(store),
|
||||
TemplateRenderMode::Lenient,
|
||||
)
|
||||
.with_context(|| {
|
||||
let from =
|
||||
from.map_or_else(|| source_path.to_string(), std::string::ToString::to_string);
|
||||
format!("failed to verify template dependencies for {from}")
|
||||
})?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn collect_config_dockerfile(
|
||||
&self,
|
||||
config_path: &ManifestPath,
|
||||
source: &str,
|
||||
files: &mut HashMap<String, types::ManifestFileEntry>,
|
||||
) -> Result<()> {
|
||||
let layer = source
|
||||
.parse::<SettingsLayer>()
|
||||
.context("Failed to parse run config TOML")?;
|
||||
let absolute_config_path = self.cwd.join(config_path.as_path());
|
||||
let base_dir = absolute_config_path
|
||||
.parent()
|
||||
.unwrap_or_else(|| Path::new("."));
|
||||
|
||||
for environment in layer.environments.values() {
|
||||
self.collect_environment_dockerfile(
|
||||
files,
|
||||
base_dir,
|
||||
config_path,
|
||||
environment.image.as_ref(),
|
||||
)?;
|
||||
}
|
||||
if let Some(run_environment) = layer.run.as_ref().and_then(|run| run.environment.as_ref()) {
|
||||
self.collect_environment_dockerfile(
|
||||
files,
|
||||
base_dir,
|
||||
config_path,
|
||||
run_environment.image.as_ref(),
|
||||
)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn collect_environment_dockerfile(
|
||||
&self,
|
||||
files: &mut HashMap<String, types::ManifestFileEntry>,
|
||||
base_dir: &Path,
|
||||
config_path: &ManifestPath,
|
||||
image: Option<&EnvironmentImageLayer>,
|
||||
) -> Result<()> {
|
||||
let dockerfile = image.and_then(|image| image.dockerfile.as_ref());
|
||||
let Some(EnvironmentDockerfileLayer::Path { path }) = dockerfile else {
|
||||
return Ok(());
|
||||
};
|
||||
self.collect_bundled_file(
|
||||
files,
|
||||
base_dir,
|
||||
path,
|
||||
types::ManifestFileRefType::Dockerfile,
|
||||
ReferenceKind::Dockerfile,
|
||||
Some(config_path.clone()),
|
||||
)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn collect_bundled_file(
|
||||
&self,
|
||||
files: &mut HashMap<String, types::ManifestFileEntry>,
|
||||
base_dir: &Path,
|
||||
reference: &str,
|
||||
ref_type: types::ManifestFileRefType,
|
||||
reference_kind: ReferenceKind,
|
||||
from: Option<ManifestPath>,
|
||||
) -> Result<BundledFile> {
|
||||
reference_kind
|
||||
.validate(reference)
|
||||
.map_err(anyhow::Error::new)?;
|
||||
|
||||
let absolute_path = normalize_absolute_path(base_dir, reference)
|
||||
.ok_or_else(|| anyhow!("unsupported manifest reference: {reference}"))?;
|
||||
let path = manifest_path_from_absolute(&absolute_path, self.cwd)?;
|
||||
let key = path.to_string();
|
||||
if !files.contains_key(&key) {
|
||||
let content = std::fs::read_to_string(&absolute_path)
|
||||
.with_context(|| format!("Failed to read {}", absolute_path.display()))?;
|
||||
files.insert(key.clone(), types::ManifestFileEntry {
|
||||
content,
|
||||
ref_: types::ManifestFileRef {
|
||||
from: from.map(|value| value.to_string()),
|
||||
original: reference.to_owned(),
|
||||
type_: ref_type,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
Ok(BundledFile {
|
||||
absolute_path,
|
||||
path,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct WorkflowScanInput {
|
||||
absolute_dot_path: PathBuf,
|
||||
dot_path: ManifestPath,
|
||||
source: String,
|
||||
}
|
||||
|
||||
struct BundledFile {
|
||||
absolute_path: PathBuf,
|
||||
path: ManifestPath,
|
||||
}
|
||||
|
||||
fn manifest_parent_or_dot(path: &ManifestPath) -> Result<ManifestPath> {
|
||||
let parent = path.parent_or_dot().to_string_lossy();
|
||||
ManifestPath::from_wire(&parent)
|
||||
.ok_or_else(|| anyhow!("invalid manifest parent path for {path}: {parent}"))
|
||||
}
|
||||
|
||||
fn template_root_for_bundled_file(
|
||||
path: &ManifestPath,
|
||||
workflow_template_root: &ManifestPath,
|
||||
) -> Result<ManifestPath> {
|
||||
if manifest_path_is_within_root(path, workflow_template_root) {
|
||||
Ok(workflow_template_root.clone())
|
||||
} else {
|
||||
manifest_parent_or_dot(path)
|
||||
}
|
||||
}
|
||||
|
||||
fn manifest_path_is_within_root(path: &ManifestPath, root: &ManifestPath) -> bool {
|
||||
if root.as_path().as_os_str().is_empty() {
|
||||
return !matches!(
|
||||
path.as_path().components().next(),
|
||||
Some(Component::ParentDir)
|
||||
);
|
||||
}
|
||||
path.starts_with(root)
|
||||
}
|
||||
|
||||
fn manifest_attr_reference_kind(
|
||||
scope: AttributeScope,
|
||||
key: &str,
|
||||
value: &str,
|
||||
) -> Result<ReferenceKind> {
|
||||
static_reference::reference_kind_for_attribute(scope, key, value)
|
||||
.ok_or_else(|| anyhow!("unsupported manifest reference attribute: {key}={value}"))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn write_file(path: &Path, source: &str) {
|
||||
if let Some(parent) = path.parent() {
|
||||
std::fs::create_dir_all(parent).expect("fixture directory should be created");
|
||||
}
|
||||
std::fs::write(path, source).expect("fixture file should be written");
|
||||
}
|
||||
|
||||
fn bundle_graph(cwd: &Path, graph: &Path) -> Result<HashMap<String, types::ManifestWorkflow>> {
|
||||
let inputs = HashMap::new();
|
||||
let root_location = WorkflowLocation::resolve(graph, cwd)?;
|
||||
WorkflowBundler::new(cwd, &inputs).bundle(&root_location, None)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn repeated_references_collect_one_file() {
|
||||
let temp = tempfile::tempdir().expect("temp directory should be created");
|
||||
let graph = temp.path().join("workflow.fabro");
|
||||
write_file(
|
||||
&graph,
|
||||
r#"digraph Root {
|
||||
start [shape=Mdiamond]
|
||||
first [prompt="@prompt.md"]
|
||||
second [prompt="@prompt.md"]
|
||||
exit [shape=Msquare]
|
||||
start -> first -> second -> exit
|
||||
}"#,
|
||||
);
|
||||
write_file(&temp.path().join("prompt.md"), "prompt\n");
|
||||
|
||||
let workflows = bundle_graph(temp.path(), &graph).expect("workflow should bundle");
|
||||
|
||||
assert_eq!(workflows["workflow.fabro"].files.len(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_errors_keep_the_graphviz_error_in_the_source_chain() {
|
||||
let temp = tempfile::tempdir().expect("temp directory should be created");
|
||||
let graph = temp.path().join("workflow.fabro");
|
||||
write_file(&graph, "not a graph");
|
||||
|
||||
let error = bundle_graph(temp.path(), &graph).expect_err("invalid graph should fail");
|
||||
|
||||
assert!(
|
||||
error
|
||||
.chain()
|
||||
.any(|cause| cause.downcast_ref::<fabro_graphviz::Error>().is_some()),
|
||||
"unexpected error chain: {error:#}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn read_errors_keep_the_io_error_in_the_source_chain() {
|
||||
let temp = tempfile::tempdir().expect("temp directory should be created");
|
||||
let graph = temp.path().join("workflow.fabro");
|
||||
write_file(
|
||||
&graph,
|
||||
r#"digraph Root {
|
||||
start [shape=Mdiamond]
|
||||
work [prompt="@missing.md"]
|
||||
exit [shape=Msquare]
|
||||
start -> work -> exit
|
||||
}"#,
|
||||
);
|
||||
|
||||
let error = bundle_graph(temp.path(), &graph).expect_err("missing file should fail");
|
||||
|
||||
assert!(
|
||||
error
|
||||
.chain()
|
||||
.any(|cause| cause.downcast_ref::<std::io::Error>().is_some()),
|
||||
"unexpected error chain: {error:#}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bundler_does_not_push_an_ahead_branch() {
|
||||
fn commit_all(repository: &git2::Repository, message: &str) -> git2::Oid {
|
||||
let mut index = repository.index().expect("index should open");
|
||||
index
|
||||
.add_all(["*"], git2::IndexAddOption::DEFAULT, None)
|
||||
.expect("fixture files should be staged");
|
||||
index.write().expect("index should be written");
|
||||
let tree_id = index.write_tree().expect("tree should be written");
|
||||
let tree = repository.find_tree(tree_id).expect("tree should exist");
|
||||
let signature = git2::Signature::now("Fabro Test", "fabro@example.com")
|
||||
.expect("signature should be valid");
|
||||
let parents = repository
|
||||
.head()
|
||||
.ok()
|
||||
.and_then(|head| head.target())
|
||||
.map(|oid| {
|
||||
repository
|
||||
.find_commit(oid)
|
||||
.expect("parent commit should exist")
|
||||
});
|
||||
let parent_refs = parents.iter().collect::<Vec<_>>();
|
||||
repository
|
||||
.commit(
|
||||
Some("refs/heads/main"),
|
||||
&signature,
|
||||
&signature,
|
||||
message,
|
||||
&tree,
|
||||
&parent_refs,
|
||||
)
|
||||
.expect("commit should be created")
|
||||
}
|
||||
|
||||
let temp = tempfile::tempdir().expect("temp directory should be created");
|
||||
let origin_path = temp.path().join("origin.git");
|
||||
let checkout = temp.path().join("checkout");
|
||||
let origin =
|
||||
git2::Repository::init_bare(&origin_path).expect("bare origin should be initialized");
|
||||
let repository = git2::Repository::init(&checkout).expect("checkout should be initialized");
|
||||
repository
|
||||
.set_head("refs/heads/main")
|
||||
.expect("main should be selected");
|
||||
write_file(
|
||||
&checkout.join("workflow.fabro"),
|
||||
"digraph Root { start [shape=Mdiamond] exit [shape=Msquare] start -> exit }",
|
||||
);
|
||||
let first_commit = commit_all(&repository, "initial");
|
||||
let mut remote = repository
|
||||
.remote(
|
||||
"origin",
|
||||
origin_path.to_str().expect("origin path should be UTF-8"),
|
||||
)
|
||||
.expect("origin should be configured");
|
||||
remote
|
||||
.push(&["refs/heads/main:refs/heads/main"], None)
|
||||
.expect("initial commit should be pushed");
|
||||
drop(remote);
|
||||
write_file(&checkout.join("README.md"), "ahead\n");
|
||||
let ahead_commit = commit_all(&repository, "ahead");
|
||||
assert_ne!(first_commit, ahead_commit);
|
||||
|
||||
bundle_graph(&checkout, &checkout.join("workflow.fabro")).expect("workflow should bundle");
|
||||
|
||||
let origin_commit = origin
|
||||
.find_reference("refs/heads/main")
|
||||
.expect("origin main should exist")
|
||||
.target()
|
||||
.expect("origin main should point to a commit");
|
||||
assert_eq!(origin_commit, first_commit);
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Reference in a new issue