diff --git a/Cargo.lock b/Cargo.lock index 3f0bfb5..3cc03c4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -781,6 +781,7 @@ dependencies = [ name = "flowproof-agent" version = "0.9.0" dependencies = [ + "base64", "chrono", "flowproof-driver", "flowproof-trace", diff --git a/Cargo.toml b/Cargo.toml index 4b865be..5506eaf 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,6 +27,7 @@ flowproof-adapters = { path = "crates/flowproof-adapters" } flowproof-cli = { path = "crates/flowproof-cli" } anyhow = "1" +base64 = "0.22" chrono = { version = "0.4", default-features = false, features = ["clock"] } clap = { version = "4", features = ["derive"] } serde = { version = "1", features = ["derive"] } diff --git a/crates/flowproof-agent/Cargo.toml b/crates/flowproof-agent/Cargo.toml index 105590e..58c9434 100644 --- a/crates/flowproof-agent/Cargo.toml +++ b/crates/flowproof-agent/Cargo.toml @@ -8,6 +8,7 @@ repository.workspace = true authors.workspace = true [dependencies] +base64 = { workspace = true } chrono = { workspace = true } flowproof-driver = { workspace = true, features = ["oob"] } flowproof-trace = { workspace = true } diff --git a/crates/flowproof-agent/src/lib.rs b/crates/flowproof-agent/src/lib.rs index 87b1158..febfd06 100644 --- a/crates/flowproof-agent/src/lib.rs +++ b/crates/flowproof-agent/src/lib.rs @@ -12,6 +12,7 @@ pub mod llm; pub mod recorder; pub mod rules; pub mod spec; +pub mod video_author; pub use clarify::{Clarification, ClarifyStage}; pub use heal::{heal, heal_with_author, HealError, HealReport}; diff --git a/crates/flowproof-agent/src/video_author.rs b/crates/flowproof-agent/src/video_author.rs new file mode 100644 index 0000000..f61b462 --- /dev/null +++ b/crates/flowproof-agent/src/video_author.rs @@ -0,0 +1,373 @@ +//! SPIKE: draft `.flow.yaml` authoring from a screen-recording video. +//! +//! docs/recording.md and issue #227 both say video should never be a +//! machine-parsed input - video is the human surface, not the machine +//! surface. This exists anyway, as a scoped experiment: extract key +//! frames, ask a vision model what single action bridges each pair (in +//! flowproof's existing step vocabulary), and write a DRAFT. It never +//! infers `assert:` steps - a recording shows behaviour, not a belief +//! about correctness - so a human adds at least one before the draft +//! can survive the live `flowproof record` pass every spec still needs. + +use std::path::{Path, PathBuf}; +use std::process::Command; + +use base64::Engine; +use serde_json::json; + +use crate::{AgentError, BackendConfig, BackendKind, FlowSpec}; + +#[derive(Debug, thiserror::Error)] +pub enum VideoAuthorError { + #[error("ffmpeg not found on PATH: install ffmpeg to extract frames from a video")] + FfmpegMissing, + #[error("ffmpeg failed extracting frames: {0}")] + FfmpegFailed(String), + #[error("no steps were inferred from the video's frames — nothing to draft")] + NoStepsInferred, + #[error("model produced a draft that does not parse as a flow spec: {0}")] + DraftInvalid(#[from] crate::spec::SpecError), + #[error(transparent)] + Agent(#[from] AgentError), + #[error("io error at '{path}': {source}")] + Io { + path: String, + source: std::io::Error, + }, +} + +fn io_err(path: &Path, source: std::io::Error) -> VideoAuthorError { + VideoAuthorError::Io { + path: path.display().to_string(), + source, + } +} + +/// One frame every `interval_ms` from `video` into `out_dir`, via a +/// system `ffmpeg` (never bundled, never run in CI — local only). +pub fn extract_keyframes( + video: &Path, + interval_ms: u64, + out_dir: &Path, +) -> Result, VideoAuthorError> { + check_ffmpeg_available()?; + std::fs::create_dir_all(out_dir).map_err(|e| io_err(out_dir, e))?; + + let fps = 1000.0 / interval_ms.max(1) as f64; + let pattern = out_dir.join("frame_%05d.png"); + let output = Command::new("ffmpeg") + .arg("-y") + .arg("-i") + .arg(video) + .args(["-vf", &format!("fps={fps}")]) + .arg(&pattern) + .output() + .map_err(|e| VideoAuthorError::FfmpegFailed(e.to_string()))?; + if !output.status.success() { + return Err(VideoAuthorError::FfmpegFailed( + String::from_utf8_lossy(&output.stderr).trim().to_string(), + )); + } + + // Empty (too-short video) is not an error here; assemble_draft_spec + // refuses an empty result downstream, with one message for that. + let mut frames: Vec = std::fs::read_dir(out_dir) + .map_err(|e| io_err(out_dir, e))? + .filter_map(|entry| entry.ok()) + .map(|entry| entry.path()) + .filter(|path| path.extension().and_then(|ext| ext.to_str()) == Some("png")) + .collect(); + frames.sort(); + Ok(frames) +} + +fn check_ffmpeg_available() -> Result<(), VideoAuthorError> { + Command::new("ffmpeg") + .arg("-version") + .output() + .map(|_| ()) + .map_err(|_| VideoAuthorError::FfmpegMissing) +} + +const TRANSITION_SYSTEM_PROMPT: &str = "\ +Two frames from a screen recording: FIRST is before an action, SECOND is \ +after. Describe the SINGLE action a flowproof step would perform to go \ +from first to second, using ONLY these forms: +- \"Go to /nVA01\" (a transaction code in the command field) +- \"Type into the