diff --git a/Cargo.lock b/Cargo.lock index 9e4577b254..035912cfde 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2129,6 +2129,7 @@ dependencies = [ "derive_more", "dunce", "home", + "indexmap", "insta", "miette 7.6.0", "pacquet-config", @@ -2146,10 +2147,13 @@ dependencies = [ "pacquet-store-dir", "pacquet-tarball", "pacquet-testing-utils", + "pacquet-workspace", + "pacquet-workspace-projects-graph", "pipe-trait", "pnpr", "pretty_assertions", "rayon", + "serde", "serde-saphyr", "serde_json", "tempfile", diff --git a/pacquet/crates/cli/Cargo.toml b/pacquet/crates/cli/Cargo.toml index 2a032c84fb..ed5c81b9dd 100644 --- a/pacquet/crates/cli/Cargo.toml +++ b/pacquet/crates/cli/Cargo.toml @@ -15,27 +15,31 @@ name = "pacquet" path = "src/bin/main.rs" [dependencies] -pacquet-pnpr-client = { workspace = true } -pacquet-executor = { workspace = true } -pacquet-fs = { workspace = true } -pacquet-lockfile = { workspace = true } -pacquet-network = { workspace = true } -pacquet-config = { workspace = true } -pacquet-package-manifest = { workspace = true } -pacquet-package-manager = { workspace = true } -pacquet-package-is-installable = { workspace = true } -pacquet-registry = { workspace = true } -pacquet-reporter = { workspace = true } -pacquet-tarball = { workspace = true } -pacquet-diagnostics = { workspace = true } +pacquet-pnpr-client = { workspace = true } +pacquet-executor = { workspace = true } +pacquet-fs = { workspace = true } +pacquet-lockfile = { workspace = true } +pacquet-network = { workspace = true } +pacquet-config = { workspace = true } +pacquet-package-manifest = { workspace = true } +pacquet-package-manager = { workspace = true } +pacquet-package-is-installable = { workspace = true } +pacquet-registry = { workspace = true } +pacquet-reporter = { workspace = true } +pacquet-tarball = { workspace = true } +pacquet-diagnostics = { workspace = true } +pacquet-workspace = { workspace = true } +pacquet-workspace-projects-graph = { workspace = true } clap = { workspace = true } derive_more = { workspace = true } dunce = { workspace = true } home = { workspace = true } +indexmap = { workspace = true } miette = { workspace = true } pipe-trait = { workspace = true } rayon = { workspace = true } +serde = { workspace = true } serde_json = { workspace = true } tokio = { workspace = true } diff --git a/pacquet/crates/cli/src/cli_args.rs b/pacquet/crates/cli/src/cli_args.rs index f2cc9cdf85..e85a7edd6c 100644 --- a/pacquet/crates/cli/src/cli_args.rs +++ b/pacquet/crates/cli/src/cli_args.rs @@ -233,7 +233,13 @@ impl CliArgs { .wrap_err(format!("executing command: \"{0}\"", script))?; } } - CliCommand::Run(args) => args.run(manifest_path())?, + CliCommand::Run(args) => { + if recursive { + args.run_recursive(config()?, &dir)?; + } else { + args.run(manifest_path())?; + } + } CliCommand::Start => { // Runs an arbitrary command specified in the package's start property of its scripts // object. If no start property is specified on the scripts object, it will attempt to diff --git a/pacquet/crates/cli/src/cli_args/run.rs b/pacquet/crates/cli/src/cli_args/run.rs index 0ab4cefa26..dc1af6bd5d 100644 --- a/pacquet/crates/cli/src/cli_args/run.rs +++ b/pacquet/crates/cli/src/cli_args/run.rs @@ -1,8 +1,11 @@ use clap::Args; use miette::Context; +use pacquet_config::Config; use pacquet_executor::execute_shell; use pacquet_package_manifest::PackageManifest; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; + +mod recursive; #[derive(Debug, Args)] pub struct RunArgs { @@ -17,12 +20,32 @@ pub struct RunArgs { /// execution chain. #[clap(long)] pub if_present: bool, + + /// Run the script starting from the given package, skipping every + /// package that sorts before it. Only meaningful together with the + /// global `-r` / `--recursive` flag. Mirrors pnpm's `--resume-from`. + #[clap(long = "resume-from")] + pub resume_from: Option, + + /// Save the execution result of every package to + /// `pnpm-exec-summary.json`. Only meaningful together with the + /// global `-r` / `--recursive` flag. Mirrors pnpm's + /// `--report-summary`. + #[clap(long = "report-summary")] + pub report_summary: bool, + + /// Keep running the remaining packages after a script fails instead + /// of aborting on the first failure. Only meaningful together with + /// the global `-r` / `--recursive` flag. Mirrors pnpm's `--no-bail` + /// (recursive runs bail by default). + #[clap(long = "no-bail")] + pub no_bail: bool, } impl RunArgs { - /// Execute the subcommand. + /// Execute the subcommand for a single project. pub fn run(self, manifest_path: PathBuf) -> miette::Result<()> { - let RunArgs { command, args, if_present } = self; + let RunArgs { command, args, if_present, .. } = self; let manifest = PackageManifest::from_path(manifest_path) .wrap_err("getting the package.json in current directory")?; @@ -38,4 +61,11 @@ impl RunArgs { Ok(()) } + + /// Execute the subcommand for every project in the workspace, in + /// topological order. The recursive counterpart of [`Self::run`], + /// selected when the global `-r` / `--recursive` flag is set. + pub fn run_recursive(&self, config: &Config, dir: &Path) -> miette::Result<()> { + recursive::run_recursive(self, config, dir) + } } diff --git a/pacquet/crates/cli/src/cli_args/run/recursive.rs b/pacquet/crates/cli/src/cli_args/run/recursive.rs new file mode 100644 index 0000000000..70e30d38f3 --- /dev/null +++ b/pacquet/crates/cli/src/cli_args/run/recursive.rs @@ -0,0 +1,325 @@ +//! Recursive `pacquet run` — run a package script in every project of +//! the workspace, in topological order. +//! +//! Port of pnpm's +//! [`runRecursive`](https://github.com/pnpm/pnpm/blob/8eb1be4988/exec/commands/src/runRecursive.ts) +//! together with the `getResumedPackageChunks` / `writeRecursiveSummary` +//! helpers from +//! [`exec.ts`](https://github.com/pnpm/pnpm/blob/8eb1be4988/exec/commands/src/exec.ts) +//! and the `throwOnCommandFail` failure check from +//! [`@pnpm/cli.utils`](https://github.com/pnpm/pnpm/blob/8eb1be4988/cli/utils/src/recursiveSummary.ts). +//! +//! Scope versus upstream: projects are sorted topologically (upstream's +//! default) and run sequentially. `--no-sort`, `--reverse`, +//! `--workspace-concurrency` parallelism, `--filter` narrowing of the +//! selected set, and the RegExp script selector are not ported yet — the +//! selected set is every workspace project, matching pacquet's +//! currently-unfiltered `install`. + +use super::RunArgs; +use derive_more::{Display, Error}; +use indexmap::IndexMap; +use miette::{Context, Diagnostic, IntoDiagnostic}; +use pacquet_config::Config; +use pacquet_executor::execute_shell_with_status; +use pacquet_package_manager::graph_sequencer; +use pacquet_package_manifest::DependencyGroup; +use pacquet_workspace::{ + FindWorkspaceProjectsOpts, Project, find_workspace_projects, read_workspace_manifest, +}; +use pacquet_workspace_projects_graph::{ + BaseProject, CreateProjectsGraphOptions, GraphProject, ProjectGraph, create_projects_graph, +}; +use serde::Serialize; +use std::{ + collections::{HashMap, HashSet}, + path::{Path, PathBuf}, + time::Instant, +}; + +/// Errors surfaced by a recursive run. Codes mirror pnpm's so log +/// consumers and `pnpm.io/errors` references stay valid across the two +/// implementations. +#[derive(Debug, Display, Error, Diagnostic)] +#[non_exhaustive] +pub enum RecursiveRunError { + #[display("Cannot find package {resume_from}. Could not determine where to resume from.")] + #[diagnostic(code(ERR_PNPM_RESUME_FROM_NOT_FOUND))] + ResumeFromNotFound { + #[error(not(source))] + resume_from: String, + }, + + #[display("None of the packages has a \"{script_name}\" script")] + #[diagnostic(code(ERR_PNPM_RECURSIVE_RUN_NO_SCRIPT))] + NoScript { + #[error(not(source))] + script_name: String, + }, + + #[display("\"pnpm recursive run\" failed in {count} packages")] + #[diagnostic(code(ERR_PNPM_RECURSIVE_FAIL))] + RecursiveFail { + #[error(not(source))] + count: usize, + }, + + #[display("\"pnpm recursive run\" failed in {prefix}")] + #[diagnostic(code(ERR_PNPM_RECURSIVE_RUN_FIRST_FAIL))] + RecursiveRunFirstFail { + #[error(not(source))] + prefix: String, + }, +} + +/// Run `args.command` across every workspace project, sorted +/// topologically. `dir` is the canonicalized working directory; the +/// workspace root (and the directory the summary is written to) is +/// `config.workspace_dir`, falling back to `dir` when no +/// `pnpm-workspace.yaml` exists. +pub fn run_recursive(args: &RunArgs, config: &Config, dir: &Path) -> miette::Result<()> { + let script_name = args.command.as_str(); + let workspace_root = config.workspace_dir.as_deref().unwrap_or(dir); + + let patterns = read_workspace_manifest(workspace_root) + .into_diagnostic() + .wrap_err("reading pnpm-workspace.yaml")? + .and_then(|manifest| manifest.packages); + let projects = find_workspace_projects(workspace_root, &FindWorkspaceProjectsOpts { patterns }) + .wrap_err("finding workspace projects")?; + + let adapters = projects.iter().map(|project| GraphPkg { project }).collect(); + let graph = create_projects_graph(adapters, &CreateProjectsGraphOptions::default()).graph; + + let mut chunks = sort_projects(&graph); + if let Some(resume_from) = &args.resume_from { + chunks = get_resumed_package_chunks(resume_from, chunks, &graph)?; + } + + let bail = !args.no_bail; + let passed_through_args = args.args.join(" "); + let mut result: IndexMap = + chunks.iter().flatten().map(|root| (root.clone(), ExecutionStatus::queued())).collect(); + let mut has_command = 0_usize; + + for chunk in &chunks { + for root in chunk { + let manifest = &graph[root].package.project.manifest; + let Some(script) = manifest.script(script_name, true)? else { + result[root].status = Status::Skipped; + continue; + }; + + result[root].status = Status::Running; + has_command += 1; + let start = Instant::now(); + let command = format!("{script} {passed_through_args}"); + let status = execute_shell_with_status(command.trim(), root).into_diagnostic()?; + let duration = start.elapsed().as_secs_f64() * 1e3; + + if status.success() { + let entry = &mut result[root]; + entry.status = Status::Passed; + entry.duration = Some(duration); + } else { + let prefix = root.to_string_lossy().into_owned(); + let entry = &mut result[root]; + entry.status = Status::Failure; + entry.duration = Some(duration); + entry.message = + Some(format!("command failed with exit code {}", status.code().unwrap_or(1))); + entry.prefix = Some(prefix.clone()); + + if bail { + if args.report_summary { + write_recursive_summary(workspace_root, &result)?; + } + return Err(RecursiveRunError::RecursiveRunFirstFail { prefix }.into()); + } + } + } + } + + // `test` is exempt because `pnpm test` falls back to a default and + // should not error on a workspace with no `test` script; otherwise a + // recursive run that matched nothing is a user error, unless + // `--if-present` opted out of it. + if script_name != "test" && has_command == 0 && !args.if_present { + return Err(RecursiveRunError::NoScript { script_name: script_name.to_string() }.into()); + } + + if args.report_summary { + write_recursive_summary(workspace_root, &result)?; + } + + throw_on_command_fail(&result)?; + Ok(()) +} + +/// Sort the workspace graph into topologically ordered chunks: every +/// project in chunk `i` depends only on projects in earlier chunks, so +/// chunk `i` may run after chunks `0..i`. +/// +/// Port of pnpm's +/// [`sortProjects`](https://github.com/pnpm/pnpm/blob/8eb1be4988/workspace/projects-sorter/src/index.ts): +/// build a node → in-set-dependencies map (dropping self-edges and edges +/// leaving the selected set) and run it through [`graph_sequencer`]. +fn sort_projects(graph: &ProjectGraph>) -> Vec> { + let keys: Vec = graph.keys().cloned().collect(); + let key_set: HashSet<&PathBuf> = keys.iter().collect(); + let dependency_graph: HashMap> = graph + .iter() + .map(|(root, node)| { + let dependencies = node + .dependencies + .iter() + .filter(|dependency| *dependency != root && key_set.contains(dependency)) + .cloned() + .collect(); + (root.clone(), dependencies) + }) + .collect(); + graph_sequencer(&dependency_graph, &keys).chunks +} + +/// Drop every chunk before the one containing the `resume_from` package, +/// so execution resumes from that package. +/// +/// Port of pnpm's +/// [`getResumedPackageChunks`](https://github.com/pnpm/pnpm/blob/8eb1be4988/exec/commands/src/exec.ts#L100-L118): +/// the package is located by manifest name; an unknown name is a +/// `RESUME_FROM_NOT_FOUND` error. +fn get_resumed_package_chunks( + resume_from: &str, + chunks: Vec>, + graph: &ProjectGraph>, +) -> Result>, RecursiveRunError> { + let resume_root = graph + .iter() + .find(|(_, node)| node.package.manifest_name() == Some(resume_from)) + .map(|(root, _)| root.clone()) + .ok_or_else(|| RecursiveRunError::ResumeFromNotFound { + resume_from: resume_from.to_string(), + })?; + let position = chunks + .iter() + .position(|chunk| chunk.contains(&resume_root)) + .expect("the resume-from package is present in the sorted chunks"); + Ok(chunks.into_iter().skip(position).collect()) +} + +/// Write the recursive summary to `pnpm-exec-summary.json` under `dir`. +/// +/// Port of pnpm's +/// [`writeRecursiveSummary`](https://github.com/pnpm/pnpm/blob/8eb1be4988/exec/commands/src/exec.ts#L120-L124): +/// the per-package map is nested under an `executionStatus` key. +fn write_recursive_summary( + dir: &Path, + summary: &IndexMap, +) -> miette::Result<()> { + let execution_status = summary + .iter() + .map(|(root, status)| (root.to_string_lossy().into_owned(), status.clone())) + .collect(); + let path = dir.join("pnpm-exec-summary.json"); + let mut contents = + serde_json::to_string_pretty(&ExecSummaryFile { execution_status }).into_diagnostic()?; + contents.push('\n'); + std::fs::write(&path, contents) + .into_diagnostic() + .wrap_err_with(|| format!("writing {}", path.display())) +} + +/// Fail when any package's script failed. +/// +/// Port of pnpm's +/// [`throwOnCommandFail`](https://github.com/pnpm/pnpm/blob/8eb1be4988/cli/utils/src/recursiveSummary.ts#L28-L33). +fn throw_on_command_fail( + summary: &IndexMap, +) -> Result<(), RecursiveRunError> { + let count = summary.values().filter(|status| status.status == Status::Failure).count(); + if count > 0 { + return Err(RecursiveRunError::RecursiveFail { count }); + } + Ok(()) +} + +/// Adapter that lets a [`Project`] feed [`create_projects_graph`]. Owns +/// nothing beyond a borrow of the project; the graph reads the manifest +/// name, version, and dependency groups through it. +struct GraphPkg<'a> { + project: &'a Project, +} + +impl BaseProject for GraphPkg<'_> { + fn root_dir(&self) -> &Path { + &self.project.root_dir + } + + fn manifest_name(&self) -> Option<&str> { + self.project.manifest.value().get("name").and_then(|name| name.as_str()) + } +} + +impl GraphProject for GraphPkg<'_> { + fn manifest_version(&self) -> Option<&str> { + self.project.manifest.value().get("version").and_then(|version| version.as_str()) + } + + fn merged_dependencies(&self, ignore_dev_deps: bool) -> Vec<(String, String)> { + // Precedence mirrors upstream's `createNode` spread: peer, then + // dev (unless excluded), then optional, then prod, with a later + // group overwriting an earlier duplicate's specifier while + // keeping the first-seen position. + let mut merged: IndexMap = IndexMap::new(); + let mut absorb = |group: DependencyGroup| { + for (name, spec) in self.project.manifest.dependencies([group]) { + merged.insert(name.to_string(), spec.to_string()); + } + }; + absorb(DependencyGroup::Peer); + if !ignore_dev_deps { + absorb(DependencyGroup::Dev); + } + absorb(DependencyGroup::Optional); + absorb(DependencyGroup::Prod); + merged.into_iter().collect() + } +} + +/// `pnpm-exec-summary.json` top-level shape: `{ "executionStatus": { ... } }`. +#[derive(Serialize)] +struct ExecSummaryFile { + #[serde(rename = "executionStatus")] + execution_status: IndexMap, +} + +/// One package's entry in the recursive summary. `duration` is in +/// milliseconds and present only once the script has run; `prefix` and +/// `message` are filled in for failures. +#[derive(Debug, Clone, Serialize)] +struct ExecutionStatus { + status: Status, + #[serde(skip_serializing_if = "Option::is_none")] + duration: Option, + #[serde(skip_serializing_if = "Option::is_none")] + prefix: Option, + #[serde(skip_serializing_if = "Option::is_none")] + message: Option, +} + +impl ExecutionStatus { + fn queued() -> Self { + ExecutionStatus { status: Status::Queued, duration: None, prefix: None, message: None } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "lowercase")] +enum Status { + Queued, + Running, + Passed, + Skipped, + Failure, +} diff --git a/pacquet/crates/cli/tests/run_recursive.rs b/pacquet/crates/cli/tests/run_recursive.rs new file mode 100644 index 0000000000..3f33a3ea57 --- /dev/null +++ b/pacquet/crates/cli/tests/run_recursive.rs @@ -0,0 +1,327 @@ +//! Recursive-run integration tests. The build scripts run through +//! pacquet's `sh -c` executor, so the whole file is gated to Unix — +//! same as the single-package `run` tests. +#![cfg(unix)] + +use assert_cmd::prelude::*; +use command_extra::CommandExtra; +use pacquet_testing_utils::bin::CommandTempCwd; +use serde_json::{Value, json}; +use std::{collections::HashMap, fs, path::Path}; + +/// Write a `pnpm-workspace.yaml` listing `names` as packages, plus a +/// `package.json` per name under its own subdirectory of `workspace`. +fn write_workspace(workspace: &Path, manifests: &[(&str, Value)]) { + let packages = manifests.iter().map(|(name, _)| format!(" - {name}")).collect::>(); + let workspace_yaml = format!("packages:\n{}\n", packages.join("\n")); + fs::write(workspace.join("pnpm-workspace.yaml"), workspace_yaml) + .expect("write pnpm-workspace.yaml"); + for (name, manifest) in manifests { + let dir = workspace.join(name); + fs::create_dir_all(&dir).expect("create project dir"); + fs::write(dir.join("package.json"), manifest.to_string()).expect("write package.json"); + } +} + +/// Map each summary entry to `(basename, status)` so assertions don't +/// depend on the absolute tempdir path used as the key. +fn summary_statuses(workspace: &Path) -> HashMap { + let contents = + fs::read_to_string(workspace.join("pnpm-exec-summary.json")).expect("read summary file"); + let value: Value = serde_json::from_str(&contents).expect("parse summary file"); + value["executionStatus"] + .as_object() + .expect("executionStatus is an object") + .iter() + .map(|(prefix, entry)| { + let basename = Path::new(prefix) + .file_name() + .expect("prefix has a basename") + .to_string_lossy() + .into_owned(); + let status = entry["status"].as_str().expect("status is a string").to_string(); + (basename, status) + }) + .collect() +} + +/// A package whose `build` script writes a marker via a *relative* path +/// (`touch ran.txt`), so it lands in the script's working directory. +/// Tests assert the marker appears under the package's own root, which +/// only holds if each script runs with cwd == its package root rather +/// than the workspace root. +fn build_writes_marker(name: &str) -> Value { + json!({ + "name": name, + "version": "1.0.0", + "scripts": { "build": "touch ran.txt" }, + }) +} + +/// `pacquet -r run