diff --git a/.agents/skills/spacedrive-cli/SKILL.md b/.agents/skills/spacedrive-cli/SKILL.md index 23325f619..8075c7f3d 100644 --- a/.agents/skills/spacedrive-cli/SKILL.md +++ b/.agents/skills/spacedrive-cli/SKILL.md @@ -246,6 +246,117 @@ leaves small contents alone. compares integrity hashes; a pair whose bytes differ stays and is reported. What the reads learn is written to the stores. +## Rename files + +`file rename --to ` renames one path; `file rename ...` +with rules renames many, applying the rules to each name in a fixed order: +`--replace FIND WITH` (`--regex` for captures, `--whole-name` to include the +extension), `--case lower|upper|title` and `--lower-extension`, `--prefix` +and `--suffix`, `--sequence "IMG_{n:04}"` (`--start`, `--step`) for the stem +from a counter, and `--template "{date:%Y-%m-%d} {name}{ext}"` for the whole +name. Tokens: `{name}`, `{ext}` (with its dot), `{n}` or `{n:04}`, `{parent}`, +`{date:FORMAT}` from the modification time, `{captured:FORMAT}` from the +capture time the store holds, or the modification time where it holds none. + +- It validates and previews first: each old name beside its new one, then a + y/N prompt; `--dry-run` stops after the plan. An error finding refuses: + a name the target filesystem does not write (`rename.illegal_name`, NTFS + and exFAT rules on those volumes and on SMB shares), a file already at the + new name (`rename.exists`), a name the directory cannot tell from an + existing one on a case-insensitive volume (`rename.case_collision`), and + two files wanting one name (`rename.collision`). A change of case alone is + `rename.case_only`, an info. +- The job renames in an order that never overwrites: a chain like + `1 -> 2 -> 3` waits for each name to free, and a swap parks one file under + a temporary name. Each rename is one `rename` call, so records keep their + identity, and each is journaled for undo. + +## Undo, the journal, and the trash + +Every job that changes the filesystem records what it did as effects with +enough to reverse them: created, moved from and to, trashed with where it +went, replaced with where the previous bytes went, removed for good, and +attributes before and after. `job journal ` prints them in order. + +`file undo ` reverses a job from its journal, newest effect first, +on preflight: validation refuses a job still running or with no journal +(`undo.no_journal`), warns of each file that changed since the job ran +(`undo.changed`) or whose place is now taken (`undo.occupied`), both left as +they are, and counts what cannot be reversed (`undo.irreversible`: permanent +removals, and replacements or trashings whose previous bytes were not kept). +The plan is the reverse: a delete for each creation, a move back for each +move and trashing, a replace for each replacement. `--effects 3,4` reverses +only those sequences. An undo writes its own journal, so undoing an undo is +the same command over it. + +- A deletion to the trash records where the item went: `NSFileManager` on + macOS reports the location, and on Windows and Linux the item is found in + the trash by its original path. A volume with no trash of its own, a + network mount among them, gets a Spacedrive trash directory at its root, + `.spacedrive/trash//`, reached by a rename. +- An overwrite, by copy, merge or extract, moves the previous file to the + trash the same way, so a replacement is undoable until the trash is + emptied and costs a rename rather than a copy. + +`file trash list` prints what Spacedrive trashed with a known location, +newest first, with the job and sequence to restore by; `file trash restore + ` is `file undo` over that one effect; `file trash empty` +removes those items for good along with the Spacedrive trash directories, +and `--os` empties the platform's own trash as well. + +## Mirror a folder + +`file merge --into --remove-extras` makes the destination +match the source: the merge as above, then every file the destination holds +that no source does goes to the trash, and the folders left empty are +pruned. The plan lists those files as deletes flagged where they are the +last copy of their content anywhere in the library, and validation counts +them (`merge.last_copies`). With `--on-conflict keep-newer` it is the one +way sync of a working folder to a backup, previewed. Junk is left alone. + +## Organize and flatten + +`file organize --by date|kind|extension` moves the folder's files +into subfolders: by date (`--field modified|created|captured`, +`--granularity year|year-month|year-month-day`), by the content kind the +indexer assigned (Images, Videos, Documents), or by extension. `--recursive` +takes the files beneath the folder at any depth. Every move is a rename +inside the folder, so records keep their identity; two files wanting one +place are a conflict left alone (`organize.conflicts`). + +`file flatten ` moves every file beneath a folder up to the folder +itself and prunes the emptied folders. `--on-conflict keep-both` numbers a +file whose name is taken at the root; `skip` leaves it where it is. + +## Archive and extract + +`file archive ... --to ` writes a zip or a tar.zst, by the +name's extension or `--format`, to a temporary name beside the destination +and renames it into place when complete. Validation refuses a name already +taken (`archive.exists`) and warns of free space and, with +`--remove-sources`, of the last copies among the sources; the sources go to +the trash once the archive is complete. A zip does not carry symlinks; they +are left out with a warning. + +`file extract --to ` plans from the archive's own directory: +a create, replace or skip per entry against the folder, `--on-conflict` +taking the merge policies, `--strip-components N` dropping leading folders. +An entry that would land outside the folder refuses the extract +(`extract.escape`). The job checkpoints the entry it reached and resumes +there; a replaced file's previous bytes go to the trash. + +## Attributes and links + +`file attributes ... --mode 644 --modified --hidden true` +sets what the filesystem lets a file carry; each flag left out stays as it +is. Validation refuses a mode on FAT32 and exFAT and hidden where it is a +leading dot rather than a flag (`attributes.unsupported`). The job journals +the attributes before and after, so undo sets them back. + +`file link --target ` makes a symlink, or a hard link with +`--hard`, which validation refuses across volumes (`link.cross_volume`) and +to a directory (`link.directory`). + ## Merge folders `file merge ... --into ` merges folders into an existing diff --git a/Cargo.lock b/Cargo.lock index 910e12efd..001086da6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3735,6 +3735,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4a3d7db9596fecd151c5f638c0ee5d5bd487b6e0ea232e5dc96d5250f6f94b1d" dependencies = [ "crc32fast", + "libz-rs-sys", "miniz_oxide", ] @@ -7018,6 +7019,15 @@ dependencies = [ "glob", ] +[[package]] +name = "libz-rs-sys" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c10501e7805cee23da17c7790e59df2870c0d4043ec6d03f67d31e2b53e77415" +dependencies = [ + "zlib-rs", +] + [[package]] name = "linebender_resource_handle" version = "0.1.1" @@ -11565,6 +11575,7 @@ dependencies = [ "strum 0.26.3", "supports-color", "sysinfo 0.31.4", + "tar", "tempfile", "thiserror 1.0.69", "tokio", @@ -11584,6 +11595,7 @@ dependencies = [ "whoami", "windows-sys 0.52.0", "x25519-dalek", + "zip", "zstd", ] @@ -17723,10 +17735,19 @@ checksum = "caa8cd6af31c3b31c6631b8f483848b91589021b28fffe50adada48d4f4d2ed1" dependencies = [ "arbitrary", "crc32fast", + "flate2", "indexmap 2.11.4", "memchr", + "time", + "zopfli", ] +[[package]] +name = "zlib-rs" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40990edd51aae2c2b6907af74ffb635029d5788228222c4bb811e9351c0caad3" + [[package]] name = "zlog" version = "0.1.0" @@ -17744,6 +17765,18 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" +[[package]] +name = "zopfli" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f05cd8797d63865425ff89b5c4a48804f35ba0ce8d125800027ad6017d2b5249" +dependencies = [ + "bumpalo", + "crc32fast", + "log 0.4.33", + "simd-adler32", +] + [[package]] name = "zstd" version = "0.13.3" diff --git a/PROJECT_STATUS.md b/PROJECT_STATUS.md index 2faded01c..9a8253f06 100644 --- a/PROJECT_STATUS.md +++ b/PROJECT_STATUS.md @@ -205,7 +205,7 @@ Recent landed. | Permission-aware automatic startup | first slice committed (`4b84e2c05`) | unowned | Library loading no longer dispatches a new filesystem discovery pass. The desktop requests one idempotent pass after the connected shell renders; a bare daemon does not initiate discovery, and `sd index start --defaults` gives CLI and server operators the same explicit path. Restored volume snapshots now prevent repeat full walks. Next: model denied coverage and add the macOS permission explanation and Full Disk Access affordance. | | Host media tools | committed (`bbc53866e`); live proof pending | unowned | The machine-scoped registry discovers host FFmpeg outside GUI PATHs, reports its version and media capabilities, and exposes explicit Homebrew/WinGet installation. Services settings now shows the resolved version, path, capability coverage, missing FFprobe state, refresh, and a confirmed package-manager install action. Default builds use QuickLook or host FFmpeg for hot video thumbnails and generate immutable 5 by 5 thumbstrips on hover. Core, linked-feature, server, Tauri shell, generated-client, and desktop production builds pass; real FFmpeg output is 384×216 and 800×450 in the focused fixtures. Restart the daemon, inspect Services settings, then open a video with no old sidecars and confirm its poster and hover scrub sheet. | | Opening remote files | proposal | unowned | Plan: `docs/plans/2026-09-21-opening-remote-files.md`. The byte plane already streams titan's files to the Mac over the loopback share with ranges and a block cache, but Open and QuickPreview pass a remote file's path to the local opener and asset protocol, so they fail or open a local file with the same path. O1 fixes that; then in-app streaming, open by copy, and open by stream through an attached mount, behind two on/off settings: download before opening (off) and cache streamed files on disk (on). Four decisions are James's. | -| File operations on preflight | plan captured 2026-09-23; F1 next | unowned | Plan: `docs/plans/2026-09-23-file-operations.md`. The operations a file manager is expected to have that are missing or off preflight, in seven phases: delete and duplicate in the explorer on preflight (the explorer's delete still uses a browser `confirm()`), rename on preflight and batch rename, an operation journal with undo and restore from the trash, mirror as a merge option that removes extras, organize and flatten, archive and extract, attributes and links with a trash view. Seven decisions are James's, the trash mechanism first. | +| File operations on preflight | F1 through F7 landed 2026-09-23; live acceptance next | unowned | Plan: `docs/plans/2026-09-23-file-operations.md`. Delete and duplicate in the explorer on preflight; rename and batch rename on preflight with filesystem name rules; the operation journal every mutating job writes, the trash with recorded locations and a Spacedrive trash directory where a volume has none, stashing on replace, and `files.undo` on preflight reached from the job list, ⌘Z and the trash view; mirror as `remove_extras` on merge; organize and flatten; zip and tar.zst archive and extract; attributes and links. CLI commands and dialogs for each, 30 tests over the fixture. Next: the acceptance case on the Expansion drive and a run of trash restore on Windows and Linux. | | Action validation, preview, and folder merge | V1 through V9 and dedupe landed; live acceptance on titan next | Opus | Plan: `docs/plans/2026-09-22-action-previews.md`. Preflight (`validate:` and `preview:`), `FsPlan`, copy, move, merge, and delete on both methods, the merge job, dedupe as a delete target (first copy in walk order or chosen copies, read in full before removal) with `sd file dedupe` and a Duplicates screen, `sd file copy`, `merge`, and `delete` rendering preflight, the merge dialog, the modifier drop, the preflight-driven operation modal, plan handles, and overlay listings with the explorer's preview mode; the old validation hook is gone. Remote execution rides the existing `--device` forwarding. Two opt-in preflight methods on actions, both taking the action's exact input: `validate:` answers whether and how an operation runs (findings with stable codes plus execution facts, re-run server-side at dispatch, errors refuse) and `preview:` answers what will exist afterward (`FsPlan` from the index, advisory, browsable through plan handles as an overlay on `files.directory_listing`). The plan catalogs both methods across copy, move, merge, delete, batch rename, and dedupe. Folder merge is the first action on them: recursive merge with integrity-confirmed duplicate skipping, per-leaf conflict policy, optional source consumption, remote execution via `--device`. Also fixes copy's silent blind overwrite inside directory collisions, deletes the CLI's local conflict logic per the operations-are-the-unit principle, and removes the dead `ValidationResult`/`ConfirmationRequest` hook. Six decisions are James's. Landed ahead of preflight: `files.delete` takes a comparison as its targets (`sd file delete A --against B --show both`); the job derives the set from the index as it runs, reads both copies in full before removing one, checkpoints the compare cursor, and reports skips. | | Retire Spacebot, Console, and storefront surfaces | blocked on exact private backup | unowned | The company tree contains product and research documents, but no exact copy of the current Spacebot UI, configuration, and design files was found. Back those up, then remove product-specific routes, configuration, UI, build aliases, generated types, and docs. Preserve generic filesystem operations, permissions, remote access, jobs, and logs. | diff --git a/core/Cargo.toml b/core/Cargo.toml index 6a87a3db6..dd10781e8 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -80,6 +80,9 @@ regex = "1.11" # File operations blake3 = "1.5" # Content addressing +# Archives: zip with deflate, and tar wrapped in the zstd already used for snapshots +zip = { version = "4", default-features = false, features = ["deflate", "time"] } +tar = "0.4" # Loopback SMB2 mount frontend. The `ixr-` fork is upstream plus the # re-exports a backend needs to be implementable from outside the crate. smb-server = { package = "ixr-smb-server", version = "0.5", default-features = false } @@ -249,6 +252,15 @@ windows-sys = { version = "0.52", features = ["Win32_Storage_FileSystem", "Win32 # System codecs via ImageIO and QuickLook, so the default macOS build needs no # bundled FFmpeg, libheif or pdfium. sd-imageio = { path = "../crates/imageio" } +# NSFileManager's trash call, which reports where the item went; the trash +# crate's does not. +objc2 = "0.6" +objc2-foundation = { version = "0.3", features = [ + "NSFileManager", + "NSURL", + "NSString", + "NSError", +] } [target.'cfg(any(target_os = "macos", target_os = "ios"))'.dependencies] whisper-rs = { version = "0.15.1", features = ["metal"], optional = true } diff --git a/core/src/infra/job/context.rs b/core/src/infra/job/context.rs index c64241494..be12a1b57 100644 --- a/core/src/infra/job/context.rs +++ b/core/src/infra/job/context.rs @@ -193,6 +193,27 @@ impl<'a> JobContext<'a> { Ok(()) } + /// Record what the job did to the filesystem, in order. The journal is + /// what undo reads, so a failure to write it is logged and never fails + /// the operation that already happened. + pub async fn record(&self, effects: Vec) { + if effects.is_empty() { + return; + } + if let Err(error) = self + .library + .jobs() + .database() + .append_journal(self.id, &effects) + .await + { + warn!(job_id = %self.id, "Could not record {} effects: {error}", effects.len()); + if let Some(logger) = &self.file_logger { + let _ = logger.log("WARN", &format!("Could not record effects: {error}")); + } + } + } + /// Log a message pub fn log(&self, message: impl Into) { let msg = message.into(); diff --git a/core/src/infra/job/database.rs b/core/src/infra/job/database.rs index dfb02b860..f0fc48357 100644 --- a/core/src/infra/job/database.rs +++ b/core/src/infra/job/database.rs @@ -15,8 +15,8 @@ use sea_orm::{ sea_query::{Expr, Query}, ActiveModelTrait, ActiveValue::Set, - ConnectionTrait, DatabaseConnection, DbBackend, DbErr, EntityTrait, QueryFilter, Schema, - TransactionTrait, + ColumnTrait, ConnectionTrait, DatabaseConnection, DbBackend, DbErr, EntityTrait, QueryFilter, + QueryOrder, QuerySelect, Schema, TransactionTrait, }; use serde::{Deserialize, Serialize}; use serde_json::Value as JsonValue; @@ -114,6 +114,28 @@ pub mod checkpoint { impl ActiveModelBehavior for ActiveModel {} } +pub mod journal { + use super::*; + + /// One effect a job had on the filesystem, in the order it happened. + #[derive(Clone, Debug, PartialEq, DeriveEntityModel, Serialize, Deserialize)] + #[sea_orm(table_name = "job_journal")] + pub struct Model { + #[sea_orm(primary_key, auto_increment = false)] + pub job_id: String, + #[sea_orm(primary_key, auto_increment = false)] + pub sequence: i64, + /// The effect, as MessagePack. + pub effect: Vec, + pub recorded_at: DateTime, + } + + #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] + pub enum Relation {} + + impl ActiveModelBehavior for ActiveModel {} +} + /// Initialize job database pub async fn init_database(db_file_path: &Path) -> JobResult { // Ensure the parent directory exists @@ -153,6 +175,11 @@ async fn create_tables(db: &DatabaseConnection) -> JobResult<()> { db.execute(db.get_database_backend().build(&checkpoint_statement)) .await?; + let mut journal_statement = schema.create_table_from_entity(journal::Entity); + journal_statement.if_not_exists(); + db.execute(db.get_database_backend().build(&journal_statement)) + .await?; + Ok(()) } @@ -291,6 +318,129 @@ impl JobDb { Ok(()) } + /// Append effects to a job's journal, in order after what it holds. + pub async fn append_journal( + &self, + job_id: JobId, + effects: &[super::journal::Effect], + ) -> JobResult<()> { + if effects.is_empty() { + return Ok(()); + } + let transaction = self.conn.begin().await?; + let last: Option = journal::Entity::find() + .filter(journal::Column::JobId.eq(job_id.to_string())) + .select_only() + .column_as(journal::Column::Sequence.max(), "last") + .into_tuple() + .one(&transaction) + .await? + .flatten(); + let mut sequence = last.unwrap_or(0); + let now = Utc::now(); + let rows: Vec = effects + .iter() + .map(|effect| { + sequence += 1; + Ok(journal::ActiveModel { + job_id: Set(job_id.to_string()), + sequence: Set(sequence), + effect: Set( + rmp_serde::to_vec_named(effect).map_err(|e| JobError::serialization(e))? + ), + recorded_at: Set(now), + }) + }) + .collect::>()?; + journal::Entity::insert_many(rows) + .exec(&transaction) + .await?; + transaction.commit().await?; + Ok(()) + } + + /// A job's journal, in the order it was written. + pub async fn journal(&self, job_id: JobId) -> JobResult> { + let rows = journal::Entity::find() + .filter(journal::Column::JobId.eq(job_id.to_string())) + .order_by_asc(journal::Column::Sequence) + .all(&self.conn) + .await?; + rows.into_iter() + .map(|row| { + Ok(super::journal::Recorded { + sequence: row.sequence, + effect: rmp_serde::from_slice(&row.effect) + .map_err(|e| JobError::serialization(e))?, + recorded_at: row.recorded_at, + }) + }) + .collect() + } + + /// What each job's journal amounts to, for the jobs that have one. + pub async fn journal_summaries( + &self, + ) -> JobResult> { + let rows = journal::Entity::find().all(&self.conn).await?; + let mut summaries: std::collections::HashMap = + std::collections::HashMap::new(); + for row in rows { + let effect: super::journal::Effect = match rmp_serde::from_slice(&row.effect) { + Ok(effect) => effect, + Err(_) => continue, + }; + let summary = summaries.entry(row.job_id).or_default(); + summary.effects += 1; + if effect.reversible() { + summary.reversible += 1; + } + } + Ok(summaries) + } + + /// Every trashed effect with a known location, newest first, with the + /// job it belongs to. + pub async fn trashed(&self) -> JobResult> { + let rows = journal::Entity::find() + .order_by_desc(journal::Column::RecordedAt) + .order_by_desc(journal::Column::Sequence) + .all(&self.conn) + .await?; + let mut trashed = Vec::new(); + for row in rows { + let effect: super::journal::Effect = match rmp_serde::from_slice(&row.effect) { + Ok(effect) => effect, + Err(_) => continue, + }; + if matches!(effect, super::journal::Effect::Trashed { to: Some(_), .. }) { + trashed.push(( + row.job_id, + super::journal::Recorded { + sequence: row.sequence, + effect, + recorded_at: row.recorded_at, + }, + )); + } + } + Ok(trashed) + } + + /// Drop the trashed effects whose items are gone, after the trash was + /// emptied. + pub async fn forget_trashed(&self, job_id: JobId, sequences: &[i64]) -> JobResult<()> { + if sequences.is_empty() { + return Ok(()); + } + journal::Entity::delete_many() + .filter(journal::Column::JobId.eq(job_id.to_string())) + .filter(journal::Column::Sequence.is_in(sequences.iter().copied())) + .exec(&self.conn) + .await?; + Ok(()) + } + /// Clean up old job history pub async fn cleanup_history(&self, older_than: DateTime) -> JobResult { let result = history::Entity::delete_many() diff --git a/core/src/infra/job/journal.rs b/core/src/infra/job/journal.rs new file mode 100644 index 000000000..c1f4a21a0 --- /dev/null +++ b/core/src/infra/job/journal.rs @@ -0,0 +1,169 @@ +//! # The operation journal +//! +//! What a job did to the filesystem, as effects with enough to reverse +//! them: a file created, moved from one place to another, trashed with +//! where it went, replaced with where the previous bytes went, or removed +//! for good. A job records effects as it goes, in order, against its own +//! id, so an interrupted job's journal is exact up to the interruption. +//! Undo is an action over a journal, and each effect carries the state of +//! its result, so undo can refuse an effect whose subject changed since. + +use std::path::{Path, PathBuf}; + +use serde::{Deserialize, Serialize}; +use specta::Type; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Type)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum Effect { + Created { + path: PathBuf, + subject: Option, + }, + Moved { + from: PathBuf, + to: PathBuf, + subject: Option, + }, + /// `to` is where the item went, where the platform reports it. + Trashed { + from: PathBuf, + to: Option, + subject: Option, + }, + /// `previous` is where the bytes that were at `path` went, where they + /// were kept. + Replaced { + path: PathBuf, + previous: Option, + subject: Option, + }, + /// Gone for good. + Removed { path: PathBuf }, + /// The filesystem attributes of a path, before and after. + Attributes { + path: PathBuf, + from: Attributes, + to: Attributes, + }, +} + +/// The state of an effect's result when it was recorded: what undo checks +/// against before touching it. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Type)] +pub struct Subject { + pub size: u64, + pub mtime_ms: i64, + pub is_dir: bool, +} + +impl Subject { + pub fn of(meta: &std::fs::Metadata) -> Self { + Self { + size: if meta.is_dir() { 0 } else { meta.len() }, + mtime_ms: meta + .modified() + .ok() + .and_then(|time| time.duration_since(std::time::UNIX_EPOCH).ok()) + .map(|elapsed| elapsed.as_millis() as i64) + .unwrap_or(0), + is_dir: meta.is_dir(), + } + } + + /// Whether the path holds what the subject recorded. + pub async fn still_holds(&self, path: &Path) -> bool { + match tokio::fs::symlink_metadata(path).await { + Ok(meta) => Self::of(&meta) == *self, + Err(_) => false, + } + } +} + +/// What an attribute change sets. +#[derive(Debug, Clone, PartialEq, Eq, Default, Serialize, Deserialize, Type)] +pub struct Attributes { + /// Unix permission bits. + pub mode: Option, + /// Modification time, unix milliseconds. + pub modified_ms: Option, + pub hidden: Option, +} + +impl Effect { + pub fn created(path: PathBuf, meta: Option<&std::fs::Metadata>) -> Self { + Self::Created { + path, + subject: meta.map(Subject::of), + } + } + + pub fn moved(from: PathBuf, to: PathBuf, meta: Option<&std::fs::Metadata>) -> Self { + Self::Moved { + from, + to, + subject: meta.map(Subject::of), + } + } + + pub fn trashed(from: PathBuf, to: Option, meta: Option<&std::fs::Metadata>) -> Self { + Self::Trashed { + from, + to, + subject: meta.map(Subject::of), + } + } + + pub fn replaced( + path: PathBuf, + previous: Option, + meta: Option<&std::fs::Metadata>, + ) -> Self { + Self::Replaced { + path, + previous, + subject: meta.map(Subject::of), + } + } + + pub fn removed(path: PathBuf) -> Self { + Self::Removed { path } + } + + /// Whether the effect can be reversed from what it recorded. + pub fn reversible(&self) -> bool { + match self { + Self::Created { .. } | Self::Moved { .. } | Self::Attributes { .. } => true, + Self::Trashed { to, .. } => to.is_some(), + Self::Replaced { previous, .. } => previous.is_some(), + Self::Removed { .. } => false, + } + } + + /// The path the effect left its result at, where it left one. + pub fn result(&self) -> Option<&Path> { + match self { + Self::Created { path, .. } + | Self::Replaced { path, .. } + | Self::Attributes { path, .. } => Some(path), + Self::Moved { to, .. } => Some(to), + Self::Trashed { to, .. } => to.as_deref(), + Self::Removed { .. } => None, + } + } +} + +/// One effect as the journal holds it. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Type)] +pub struct Recorded { + pub sequence: i64, + pub effect: Effect, + pub recorded_at: chrono::DateTime, +} + +/// What a journal amounts to, for a list of jobs. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize, Type)] +pub struct JournalSummary { + pub effects: u64, + pub reversible: u64, +} diff --git a/core/src/infra/job/mod.rs b/core/src/infra/job/mod.rs index 2b44e660d..cd26a5b16 100644 --- a/core/src/infra/job/mod.rs +++ b/core/src/infra/job/mod.rs @@ -8,6 +8,7 @@ pub mod error; pub mod executor; pub mod generic_progress; pub mod handle; +pub mod journal; pub mod logger; pub mod manager; pub mod output; diff --git a/core/src/ops/files/archive/action.rs b/core/src/ops/files/archive/action.rs new file mode 100644 index 000000000..d83f45af7 --- /dev/null +++ b/core/src/ops/files/archive/action.rs @@ -0,0 +1,80 @@ +//! Archive and extract action handlers + +use std::sync::Arc; + +use super::{ + input::{FileArchiveInput, FileExtractInput}, + job::{ArchiveJob, ExtractJob}, +}; +use crate::{ + context::CoreContext, + infra::{ + action::{error::ActionError, LibraryAction}, + job::handle::JobReceipt, + }, +}; + +pub struct FileArchiveAction { + input: FileArchiveInput, +} + +impl LibraryAction for FileArchiveAction { + type Input = FileArchiveInput; + type Output = JobReceipt; + + fn from_input(input: Self::Input) -> Result { + input.validate().map_err(|errors| errors.join("; "))?; + Ok(Self { input }) + } + + async fn execute( + self, + library: Arc, + _context: Arc, + ) -> Result { + let job_handle = library + .jobs() + .dispatch(ArchiveJob::new(self.input)) + .await + .map_err(ActionError::Job)?; + Ok(job_handle.into()) + } + + fn action_kind(&self) -> &'static str { + "files.archive" + } +} + +crate::register_library_action!(FileArchiveAction, "files.archive"); + +pub struct FileExtractAction { + input: FileExtractInput, +} + +impl LibraryAction for FileExtractAction { + type Input = FileExtractInput; + type Output = JobReceipt; + + fn from_input(input: Self::Input) -> Result { + Ok(Self { input }) + } + + async fn execute( + self, + library: Arc, + _context: Arc, + ) -> Result { + let job_handle = library + .jobs() + .dispatch(ExtractJob::new(self.input)) + .await + .map_err(ActionError::Job)?; + Ok(job_handle.into()) + } + + fn action_kind(&self) -> &'static str { + "files.extract" + } +} + +crate::register_library_action!(FileExtractAction, "files.extract"); diff --git a/core/src/ops/files/archive/directory.rs b/core/src/ops/files/archive/directory.rs new file mode 100644 index 000000000..7037bb4bb --- /dev/null +++ b/core/src/ops/files/archive/directory.rs @@ -0,0 +1,137 @@ +//! An archive's own directory: what it holds, before anything is written. +//! +//! A zip carries a central directory, read without decompressing a byte. A +//! tar has no directory, so its headers are read in one pass through the +//! zstd stream, which costs the decompression and no writes. + +use std::{ + io, + path::{Path, PathBuf}, +}; + +use super::input::ArchiveFormat; + +/// One entry of an archive. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ArchiveEntry { + /// Its position, which an extract resumes from. + pub index: usize, + /// The path as the archive spells it. + pub name: String, + pub size: u64, + pub is_dir: bool, + pub modified_ms: Option, +} + +impl ArchiveEntry { + /// The path the entry lands at beneath a destination, with leading + /// components stripped; `None` where the entry would escape, or where + /// stripping leaves nothing. + pub fn landing(&self, strip: u32) -> Option { + let normalized = self.name.replace('\\', "/"); + let mut components = Vec::new(); + for part in normalized.split('/') { + match part { + "" | "." => continue, + ".." => return None, + part if part.contains('\0') => return None, + part => components.push(part), + } + } + if normalized.starts_with('/') && !components.is_empty() { + // An absolute name is read relative to the destination. + } + let kept: Vec<&str> = components.into_iter().skip(strip as usize).collect(); + if kept.is_empty() { + return None; + } + Some(kept.iter().collect()) + } +} + +/// Read the archive's directory, in entry order. +pub fn read(path: &Path, format: ArchiveFormat) -> io::Result> { + match format { + ArchiveFormat::Zip => read_zip(path), + ArchiveFormat::TarZstd => read_tar_zstd(path), + } +} + +fn read_zip(path: &Path) -> io::Result> { + let file = std::fs::File::open(path)?; + let mut archive = zip::ZipArchive::new(file).map_err(io::Error::other)?; + let mut entries = Vec::with_capacity(archive.len()); + for index in 0..archive.len() { + let entry = archive.by_index_raw(index).map_err(io::Error::other)?; + entries.push(ArchiveEntry { + index, + name: entry.name().to_string(), + size: entry.size(), + is_dir: entry.is_dir(), + modified_ms: entry.last_modified().and_then(zip_time_ms), + }); + } + Ok(entries) +} + +fn zip_time_ms(time: zip::DateTime) -> Option { + let date = chrono::NaiveDate::from_ymd_opt( + i32::from(time.year()), + u32::from(time.month()), + u32::from(time.day()), + )?; + let when = date.and_hms_opt( + u32::from(time.hour()), + u32::from(time.minute()), + u32::from(time.second()), + )?; + Some(when.and_utc().timestamp_millis()) +} + +fn read_tar_zstd(path: &Path) -> io::Result> { + let file = std::fs::File::open(path)?; + let decoder = zstd::stream::read::Decoder::new(file)?; + let mut archive = tar::Archive::new(decoder); + let mut entries = Vec::new(); + for (index, entry) in archive.entries()?.enumerate() { + let entry = entry?; + let header = entry.header(); + entries.push(ArchiveEntry { + index, + name: entry.path()?.to_string_lossy().into_owned(), + size: header.size()?, + is_dir: header.entry_type().is_dir(), + modified_ms: header.mtime().ok().map(|seconds| seconds as i64 * 1000), + }); + } + Ok(entries) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn entry(name: &str) -> ArchiveEntry { + ArchiveEntry { + index: 0, + name: name.to_string(), + size: 0, + is_dir: false, + modified_ms: None, + } + } + + #[test] + fn an_entry_lands_beneath_the_destination_or_not_at_all() { + assert_eq!(entry("a/b.txt").landing(0), Some(PathBuf::from("a/b.txt"))); + assert_eq!(entry("a/b.txt").landing(1), Some(PathBuf::from("b.txt"))); + assert_eq!(entry("a/b.txt").landing(2), None); + assert_eq!(entry("../x").landing(0), None); + assert_eq!(entry("a/../../x").landing(0), None); + assert_eq!( + entry("/etc/passwd").landing(0), + Some(PathBuf::from("etc/passwd")) + ); + assert_eq!(entry("./a//b").landing(0), Some(PathBuf::from("a/b"))); + } +} diff --git a/core/src/ops/files/archive/input.rs b/core/src/ops/files/archive/input.rs new file mode 100644 index 000000000..9503ea6a7 --- /dev/null +++ b/core/src/ops/files/archive/input.rs @@ -0,0 +1,68 @@ +//! Inputs for writing an archive and extracting one. + +use serde::{Deserialize, Serialize}; +use specta::Type; + +use crate::{domain::SdPath, ops::files::merge::MergeConflictPolicy}; + +/// Write one archive holding the sources. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Type)] +pub struct FileArchiveInput { + pub sources: Vec, + /// The archive file to write. It must not exist. + pub destination: SdPath, + pub format: ArchiveFormat, + /// Trash the sources once the archive is complete. + #[serde(default)] + pub remove_sources: bool, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Type)] +#[serde(rename_all = "snake_case")] +#[cfg_attr(feature = "cli", derive(clap::ValueEnum))] +pub enum ArchiveFormat { + Zip, + TarZstd, +} + +impl ArchiveFormat { + pub fn extension(self) -> &'static str { + match self { + Self::Zip => "zip", + Self::TarZstd => "tar.zst", + } + } + + /// The format an archive's name says it is in. + pub fn of_name(name: &str) -> Option { + let lower = name.to_lowercase(); + if lower.ends_with(".zip") { + Some(Self::Zip) + } else if lower.ends_with(".tar.zst") || lower.ends_with(".tzst") { + Some(Self::TarZstd) + } else { + None + } + } +} + +/// Extract an archive into a folder. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Type)] +pub struct FileExtractInput { + pub archive: SdPath, + /// An existing folder the entries are written into. + pub destination: SdPath, + pub on_conflict: MergeConflictPolicy, + /// Leading path components to drop from every entry. + #[serde(default)] + pub strip_components: u32, +} + +impl FileArchiveInput { + pub fn validate(&self) -> Result<(), Vec> { + if self.sources.is_empty() { + return Err(vec!["name at least one source".to_string()]); + } + Ok(()) + } +} diff --git a/core/src/ops/files/archive/job.rs b/core/src/ops/files/archive/job.rs new file mode 100644 index 000000000..1354950bf --- /dev/null +++ b/core/src/ops/files/archive/job.rs @@ -0,0 +1,868 @@ +//! Writing an archive, and extracting one, entry by entry. +//! +//! Both do their reading and writing on a blocking thread that reports +//! each entry over a channel, so the job's side keeps progress, the +//! journal, checkpoints and interruption. An archive is written to a +//! temporary name beside its destination and renamed into place when +//! complete, so an interrupted job leaves nothing half-written. An extract +//! checkpoints the entry it reached and resumes there; a file it replaces +//! goes to the trash first, so the replacement can be undone. + +use std::{ + collections::HashSet, + io, + path::{Path, PathBuf}, + sync::{ + atomic::{AtomicBool, Ordering}, + Arc, + }, + time::Instant, +}; + +use serde::{Deserialize, Serialize}; +use specta::Type; +use tokio::sync::mpsc; + +use super::{ + input::{ArchiveFormat, FileArchiveInput, FileExtractInput}, + plan::decide, + preflight::read_directory, +}; +use crate::{ + infra::job::{generic_progress::GenericProgress, journal::Effect, prelude::*}, + ops::files::{merge::MergeConflictPolicy, plan::ChangeKind, trash}, +}; + +const CHECKPOINT_EVERY: usize = 64; + +/// The suffix an archive carries until it is complete. +const PARTIAL: &str = "sd-partial"; + +/// What the blocking side reports. +enum Report { + /// One entry written, with its bytes. + Entry { + index: usize, + bytes: u64, + }, + Failed { + index: usize, + error: String, + }, + Done(Result<(), String>), +} + +#[derive(Debug, Serialize, Deserialize, Job)] +pub struct ArchiveJob { + pub input: FileArchiveInput, + #[serde(skip, default = "Instant::now")] + started_at: Instant, +} + +impl ArchiveJob { + pub fn new(input: FileArchiveInput) -> Self { + Self { + input, + started_at: Instant::now(), + } + } +} + +impl Job for ArchiveJob { + const NAME: &'static str = "archive_files"; + const RESUMABLE: bool = false; + const DESCRIPTION: Option<&'static str> = Some("Write an archive"); +} + +impl crate::infra::job::traits::DynJob for ArchiveJob { + fn job_name(&self) -> &'static str { + Self::NAME + } + + fn dedup_key(&self) -> Option { + Some(self.input.destination.to_string()) + } +} + +#[derive(Debug, Clone, Default, Serialize, Deserialize, Type)] +pub struct ArchiveOutput { + pub archive: PathBuf, + pub files: u64, + pub bytes: u64, + pub left: Vec, + pub sources_removed: u64, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Type)] +pub struct ArchiveProblem { + pub path: PathBuf, + pub reason: String, +} + +impl From for JobOutput { + fn from(output: ArchiveOutput) -> Self { + JobOutput::custom(output) + } +} + +/// One file or folder to put in the archive, under its name there. +struct Member { + path: PathBuf, + name: String, + is_dir: bool, + size: u64, +} + +#[async_trait::async_trait] +impl JobHandler for ArchiveJob { + type Output = ArchiveOutput; + + async fn run(&mut self, ctx: JobContext<'_>) -> JobResult { + let destination = self + .input + .destination + .as_local_path() + .ok_or_else(|| JobError::execution("the archive would be written on another device"))? + .to_path_buf(); + if tokio::fs::symlink_metadata(&destination).await.is_ok() { + return Err(JobError::execution(format!( + "{} is already there", + destination.display() + ))); + } + let partial = partial_name(&destination); + let _ = tokio::fs::remove_file(&partial).await; + + ctx.progress(Progress::Indeterminate("Listing the sources".to_string())); + let mut members = Vec::new(); + let mut left = Vec::new(); + for source in &self.input.sources { + let Some(path) = source.as_local_path() else { + left.push(ArchiveProblem { + path: PathBuf::new(), + reason: format!("{source} is on another device"), + }); + continue; + }; + gather(path, &mut members, &mut left, self.input.format).await; + } + let total_bytes: u64 = members.iter().map(|member| member.size).sum(); + let total_files = members.iter().filter(|member| !member.is_dir).count() as u64; + ctx.log(format!( + "{total_files} files ({total_bytes} bytes) into {}", + destination.display() + )); + + let (report_tx, mut report_rx) = mpsc::unbounded_channel(); + let cancel = Arc::new(AtomicBool::new(false)); + let worker = { + let partial = partial.clone(); + let format = self.input.format; + let cancel = cancel.clone(); + let members: Vec<(PathBuf, String, bool)> = members + .iter() + .map(|member| (member.path.clone(), member.name.clone(), member.is_dir)) + .collect(); + tokio::task::spawn_blocking(move || { + let result = write_archive(&partial, format, &members, &cancel, &report_tx); + let _ = report_tx.send(Report::Done(result.map_err(|error| error.to_string()))); + }) + }; + + let mut bytes = 0u64; + let mut files = 0u64; + let mut outcome: Option> = None; + while let Some(report) = report_rx.recv().await { + if ctx.check_interrupt().await.is_err() { + cancel.store(true, Ordering::Relaxed); + } + match report { + Report::Entry { + index, + bytes: written, + } => { + bytes += written; + if !members[index].is_dir { + files += 1; + } + ctx.progress(Progress::generic( + GenericProgress::new( + (bytes as f32 / total_bytes.max(1) as f32).min(1.0), + "Archiving", + format!("{files} of {total_files} files"), + ) + .with_completion(files, total_files) + .with_bytes(bytes, total_bytes), + )); + } + Report::Failed { index, error } => left.push(ArchiveProblem { + path: members[index].path.clone(), + reason: error, + }), + Report::Done(result) => { + outcome = Some(result); + break; + } + } + } + let _ = worker.await; + if cancel.load(Ordering::Relaxed) { + let _ = tokio::fs::remove_file(&partial).await; + return Err(JobError::Interrupted); + } + if let Some(Err(error)) = outcome { + let _ = tokio::fs::remove_file(&partial).await; + return Err(JobError::execution(format!( + "could not write the archive: {error}" + ))); + } + + tokio::fs::rename(&partial, &destination) + .await + .map_err(|error| { + JobError::execution(format!("could not finish the archive: {error}")) + })?; + let subject = tokio::fs::symlink_metadata(&destination).await.ok(); + ctx.record(vec![Effect::created(destination.clone(), subject.as_ref())]) + .await; + + let mut sources_removed = 0; + if self.input.remove_sources && left.is_empty() { + for source in &self.input.sources { + let Some(path) = source.as_local_path() else { + continue; + }; + match trash::trash(path, ctx.volume_manager().as_deref(), ctx.id()).await { + Ok(location) => { + let subject = match &location { + Some(location) => tokio::fs::symlink_metadata(location).await.ok(), + None => None, + }; + ctx.record(vec![Effect::trashed( + path.to_path_buf(), + location, + subject.as_ref(), + )]) + .await; + sources_removed += 1; + } + Err(error) => ctx.log(format!("Could not remove {}: {error}", path.display())), + } + } + } else if self.input.remove_sources { + ctx.log("Sources kept: not every file made it into the archive"); + } + + ctx.progress(Progress::generic( + GenericProgress::new(1.0, "Complete", format!("{files} files archived")) + .with_completion(total_files, total_files) + .with_bytes(bytes, bytes) + .with_performance(0.0, None, Some(self.started_at.elapsed())) + .with_errors(left.len() as u64, 0), + )); + ctx.log(format!( + "Archive completed: {files} files, {bytes} bytes, {} left out, {sources_removed} sources removed", + left.len() + )); + Ok(ArchiveOutput { + archive: destination, + files, + bytes, + left, + sources_removed, + }) + } +} + +fn partial_name(destination: &Path) -> PathBuf { + let name = destination + .file_name() + .map(|name| name.to_string_lossy().into_owned()) + .unwrap_or_default(); + destination.with_file_name(format!(".{name}.{PARTIAL}")) +} + +/// Every file and folder beneath a source, named from the source's own +/// name down, in sorted order. +async fn gather( + source: &Path, + members: &mut Vec, + left: &mut Vec, + format: ArchiveFormat, +) { + let base = source + .file_name() + .map(|name| name.to_string_lossy().into_owned()) + .unwrap_or_default(); + let mut stack = vec![(source.to_path_buf(), base)]; + while let Some((path, name)) = stack.pop() { + let Ok(meta) = tokio::fs::symlink_metadata(&path).await else { + left.push(ArchiveProblem { + path, + reason: "not there".to_string(), + }); + continue; + }; + if meta.file_type().is_symlink() { + if format == ArchiveFormat::Zip { + left.push(ArchiveProblem { + path, + reason: "a zip does not carry a symlink".to_string(), + }); + continue; + } + members.push(Member { + path, + name, + is_dir: false, + size: 0, + }); + continue; + } + if meta.is_dir() { + members.push(Member { + path: path.clone(), + name: name.clone(), + is_dir: true, + size: 0, + }); + let mut children = Vec::new(); + if let Ok(mut entries) = tokio::fs::read_dir(&path).await { + while let Ok(Some(entry)) = entries.next_entry().await { + children.push(entry.file_name()); + } + } + children.sort(); + for child in children.into_iter().rev() { + let child_name = child.to_string_lossy().into_owned(); + stack.push((path.join(&child), format!("{name}/{child_name}"))); + } + continue; + } + members.push(Member { + path, + name, + is_dir: false, + size: meta.len(), + }); + } +} + +/// Write every member, reporting each. +fn write_archive( + partial: &Path, + format: ArchiveFormat, + members: &[(PathBuf, String, bool)], + cancel: &AtomicBool, + report: &mpsc::UnboundedSender, +) -> io::Result<()> { + let file = std::fs::File::create(partial)?; + match format { + ArchiveFormat::Zip => { + let mut writer = zip::ZipWriter::new(file); + for (index, (path, name, is_dir)) in members.iter().enumerate() { + if cancel.load(Ordering::Relaxed) { + return Err(io::Error::new(io::ErrorKind::Interrupted, "cancelled")); + } + let written = if *is_dir { + writer + .add_directory(name, zip_options(path)) + .map(|()| 0) + .map_err(io::Error::other) + } else { + writer + .start_file(name, zip_options(path)) + .map_err(io::Error::other) + .and_then(|()| { + let mut source = std::fs::File::open(path)?; + io::copy(&mut source, &mut writer) + }) + }; + match written { + Ok(bytes) => { + let _ = report.send(Report::Entry { index, bytes }); + } + Err(error) => { + let _ = report.send(Report::Failed { + index, + error: error.to_string(), + }); + } + } + } + writer.finish().map_err(io::Error::other)?; + } + ArchiveFormat::TarZstd => { + let encoder = zstd::stream::write::Encoder::new(file, 3)?; + let mut builder = tar::Builder::new(encoder); + builder.follow_symlinks(false); + for (index, (path, name, is_dir)) in members.iter().enumerate() { + if cancel.load(Ordering::Relaxed) { + return Err(io::Error::new(io::ErrorKind::Interrupted, "cancelled")); + } + let written = if *is_dir { + builder.append_dir(name, path).map(|()| 0) + } else { + builder.append_path_with_name(path, name).and_then(|()| { + std::fs::symlink_metadata(path).map(|meta| { + if meta.file_type().is_symlink() { + 0 + } else { + meta.len() + } + }) + }) + }; + match written { + Ok(bytes) => { + let _ = report.send(Report::Entry { index, bytes }); + } + Err(error) => { + let _ = report.send(Report::Failed { + index, + error: error.to_string(), + }); + } + } + } + let encoder = builder.into_inner()?; + encoder.finish()?; + } + } + Ok(()) +} + +/// Deflate, with the file's modification time and unix mode where the +/// platform has them. +fn zip_options(path: &Path) -> zip::write::SimpleFileOptions { + let mut options = zip::write::SimpleFileOptions::default() + .compression_method(zip::CompressionMethod::Deflated) + .large_file(true); + if let Ok(meta) = std::fs::symlink_metadata(path) { + if let Ok(modified) = meta.modified() { + let when: chrono::DateTime = modified.into(); + use chrono::{Datelike, Timelike}; + if let Ok(time) = zip::DateTime::from_date_and_time( + when.year() as u16, + when.month() as u8, + when.day() as u8, + when.hour() as u8, + when.minute() as u8, + when.second() as u8, + ) { + options = options.last_modified_time(time); + } + } + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + options = options.unix_permissions(meta.permissions().mode()); + } + } + options +} + +#[derive(Debug, Serialize, Deserialize, Job)] +pub struct ExtractJob { + pub input: FileExtractInput, + /// The entry index the job reached, which a resumed job continues from. + pub next: usize, + #[serde(skip, default = "Instant::now")] + started_at: Instant, +} + +impl ExtractJob { + pub fn new(input: FileExtractInput) -> Self { + Self { + input, + next: 0, + started_at: Instant::now(), + } + } +} + +impl Job for ExtractJob { + const NAME: &'static str = "extract_archive"; + const RESUMABLE: bool = true; + const DESCRIPTION: Option<&'static str> = Some("Extract an archive"); +} + +impl crate::infra::job::traits::DynJob for ExtractJob { + fn job_name(&self) -> &'static str { + Self::NAME + } + + fn dedup_key(&self) -> Option { + Some(format!( + "{} -> {}", + self.input.archive, self.input.destination + )) + } +} + +#[derive(Debug, Clone, Default, Serialize, Deserialize, Type)] +pub struct ExtractOutput { + pub extracted: u64, + pub replaced: u64, + pub skipped: u64, + pub conflicts: u64, + pub bytes: u64, + pub failed: Vec, +} + +impl From for JobOutput { + fn from(output: ExtractOutput) -> Self { + JobOutput::custom(output) + } +} + +#[derive(Debug, Serialize, Deserialize)] +struct Resume { + next: usize, +} + +/// One entry the worker writes, at its final path. +struct Write { + index: usize, + path: PathBuf, + is_dir: bool, +} + +#[async_trait::async_trait] +impl JobHandler for ExtractJob { + type Output = ExtractOutput; + + async fn run(&mut self, ctx: JobContext<'_>) -> JobResult { + if let Some(resume) = ctx.load_state::().await? { + self.next = resume.next; + ctx.log(format!("Resuming at entry {}", self.next)); + } + let (Some(archive), Some(destination)) = ( + self.input.archive.as_local_path().map(Path::to_path_buf), + self.input + .destination + .as_local_path() + .map(Path::to_path_buf), + ) else { + return Err(JobError::execution( + "the archive and the destination must be on this device", + )); + }; + ctx.progress(Progress::Indeterminate("Reading the archive".to_string())); + let (format, entries) = read_directory(&archive) + .await + .map_err(JobError::execution)?; + let (decisions, escapes) = decide(&self.input, &destination, entries).await; + if escapes > 0 { + return Err(JobError::execution(format!( + "{escapes} entries would land outside the destination" + ))); + } + let total_bytes: u64 = decisions + .iter() + .filter_map(|decision| match decision.change { + ChangeKind::Create { size } => Some(size), + ChangeKind::Replace { incoming_size, .. } => Some(incoming_size), + _ => None, + }) + .sum(); + let mut output = ExtractOutput::default(); + + // The places, settled before anything is written: numbered names + // for what is kept beside an existing file, and the previous bytes + // of what is replaced set aside in the trash. + let mut writes: Vec = Vec::new(); + let mut previous: std::collections::HashMap> = + std::collections::HashMap::new(); + let mut claimed: HashSet = HashSet::new(); + for decision in &decisions { + let Some(path) = &decision.path else { + continue; + }; + match &decision.change { + ChangeKind::Skip { .. } => output.skipped += 1, + ChangeKind::Conflict { .. } => output.conflicts += 1, + ChangeKind::MergeInto => {} + ChangeKind::CreateDirectory => writes.push(Write { + index: decision.entry.index, + path: path.clone(), + is_dir: true, + }), + ChangeKind::Create { .. } => { + let final_path = if tokio::fs::symlink_metadata(path).await.is_ok() + && self.input.on_conflict == MergeConflictPolicy::KeepBoth + { + let numbered = crate::ops::files::merge::job::unique_name(path).await; + claimed.insert(numbered.clone()); + numbered + } else { + path.clone() + }; + writes.push(Write { + index: decision.entry.index, + path: final_path, + is_dir: false, + }); + } + ChangeKind::Replace { .. } => { + if decision.entry.index >= self.next { + match trash::trash(path, ctx.volume_manager().as_deref(), ctx.id()).await { + Ok(location) => { + previous.insert(decision.entry.index, location); + } + Err(error) => { + output.failed.push(ArchiveProblem { + path: path.clone(), + reason: format!( + "could not set the previous file aside: {error}" + ), + }); + continue; + } + } + } + writes.push(Write { + index: decision.entry.index, + path: path.clone(), + is_dir: false, + }); + } + _ => {} + } + } + let pending: Vec = writes + .into_iter() + .filter(|write| write.index >= self.next) + .collect(); + let total_files = pending.iter().filter(|write| !write.is_dir).count() as u64; + ctx.log(format!( + "{total_files} entries to write from entry {}", + self.next + )); + let by_index: std::collections::HashMap = pending + .iter() + .map(|write| (write.index, (write.path.clone(), write.is_dir))) + .collect(); + + let (report_tx, mut report_rx) = mpsc::unbounded_channel(); + let cancel = Arc::new(AtomicBool::new(false)); + let worker = { + let archive = archive.clone(); + let cancel = cancel.clone(); + let plan: Vec<(usize, PathBuf, bool)> = pending + .iter() + .map(|write| (write.index, write.path.clone(), write.is_dir)) + .collect(); + tokio::task::spawn_blocking(move || { + let result = extract_entries(&archive, format, &plan, &cancel, &report_tx); + let _ = report_tx.send(Report::Done(result.map_err(|error| error.to_string()))); + }) + }; + + let mut since_checkpoint = 0; + let mut outcome = None; + while let Some(report) = report_rx.recv().await { + if ctx.check_interrupt().await.is_err() { + cancel.store(true, Ordering::Relaxed); + } + match report { + Report::Entry { index, bytes } => { + let Some((path, is_dir)) = by_index.get(&index) else { + continue; + }; + let subject = tokio::fs::symlink_metadata(path).await.ok(); + let effect = match previous.remove(&index) { + Some(location) => { + output.replaced += 1; + Effect::replaced(path.clone(), location, subject.as_ref()) + } + None => { + if !is_dir { + output.extracted += 1; + } + Effect::created(path.clone(), subject.as_ref()) + } + }; + ctx.record(vec![effect]).await; + output.bytes += bytes; + self.next = index + 1; + since_checkpoint += 1; + if since_checkpoint >= CHECKPOINT_EVERY { + since_checkpoint = 0; + ctx.checkpoint_with_state(&Resume { next: self.next }) + .await?; + } + ctx.progress(Progress::generic( + GenericProgress::new( + (output.bytes as f32 / total_bytes.max(1) as f32).min(1.0), + "Extracting", + format!( + "{} of {total_files} files", + output.extracted + output.replaced + ), + ) + .with_completion(output.extracted + output.replaced, total_files) + .with_bytes(output.bytes, total_bytes), + )); + } + Report::Failed { index, error } => { + let path = by_index + .get(&index) + .map(|(path, _)| path.clone()) + .unwrap_or_default(); + if let Some(Some(location)) = previous.remove(&index) { + if let Err(restore_error) = trash::restore(&location, &path).await { + ctx.log(format!( + "Could not put {} back: {restore_error}", + path.display() + )); + } + } + output.failed.push(ArchiveProblem { + path, + reason: error, + }); + } + Report::Done(result) => { + outcome = Some(result); + break; + } + } + } + let _ = worker.await; + if cancel.load(Ordering::Relaxed) { + ctx.checkpoint_with_state(&Resume { next: self.next }) + .await?; + return Err(JobError::Interrupted); + } + if let Some(Err(error)) = outcome { + return Err(JobError::execution(format!( + "could not read the archive: {error}" + ))); + } + + ctx.progress(Progress::generic( + GenericProgress::new( + 1.0, + "Complete", + format!( + "{} extracted, {} replaced", + output.extracted, output.replaced + ), + ) + .with_completion(total_files, total_files) + .with_bytes(output.bytes, output.bytes) + .with_performance(0.0, None, Some(self.started_at.elapsed())) + .with_errors(output.failed.len() as u64, output.conflicts), + )); + ctx.log(format!( + "Extract completed: {} extracted, {} replaced, {} skipped, {} conflicts, {} failed", + output.extracted, + output.replaced, + output.skipped, + output.conflicts, + output.failed.len() + )); + Ok(output) + } +} + +/// Write the planned entries, reporting each. +fn extract_entries( + archive: &Path, + format: ArchiveFormat, + plan: &[(usize, PathBuf, bool)], + cancel: &AtomicBool, + report: &mpsc::UnboundedSender, +) -> io::Result<()> { + let wanted: std::collections::HashMap = plan + .iter() + .map(|(index, path, is_dir)| (*index, (path, *is_dir))) + .collect(); + match format { + ArchiveFormat::Zip => { + let file = std::fs::File::open(archive)?; + let mut zip = zip::ZipArchive::new(file).map_err(io::Error::other)?; + for (index, path, is_dir) in plan { + if cancel.load(Ordering::Relaxed) { + return Err(io::Error::new(io::ErrorKind::Interrupted, "cancelled")); + } + let written = if *is_dir { + std::fs::create_dir_all(path).map(|()| 0) + } else { + zip.by_index(*index) + .map_err(io::Error::other) + .and_then(|mut entry| { + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent)?; + } + let mut out = std::fs::File::create(path)?; + let bytes = io::copy(&mut entry, &mut out)?; + if let Some(time) = entry.last_modified().and_then(zip_system_time) { + let _ = out.set_modified(time); + } + Ok(bytes) + }) + }; + let _ = report.send(match written { + Ok(bytes) => Report::Entry { + index: *index, + bytes, + }, + Err(error) => Report::Failed { + index: *index, + error: error.to_string(), + }, + }); + } + } + ArchiveFormat::TarZstd => { + let file = std::fs::File::open(archive)?; + let decoder = zstd::stream::read::Decoder::new(file)?; + let mut tar = tar::Archive::new(decoder); + for (index, entry) in tar.entries()?.enumerate() { + if cancel.load(Ordering::Relaxed) { + return Err(io::Error::new(io::ErrorKind::Interrupted, "cancelled")); + } + let mut entry = entry?; + let Some((path, is_dir)) = wanted.get(&index) else { + continue; + }; + let written = if *is_dir { + std::fs::create_dir_all(path).map(|()| 0) + } else { + (|| { + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent)?; + } + entry.unpack(path)?; + Ok::(entry.header().size().unwrap_or(0)) + })() + }; + let _ = report.send(match written { + Ok(bytes) => Report::Entry { index, bytes }, + Err(error) => Report::Failed { + index, + error: error.to_string(), + }, + }); + } + } + } + Ok(()) +} + +fn zip_system_time(time: zip::DateTime) -> Option { + let date = chrono::NaiveDate::from_ymd_opt( + i32::from(time.year()), + u32::from(time.month()), + u32::from(time.day()), + )?; + let when = date.and_hms_opt( + u32::from(time.hour()), + u32::from(time.minute()), + u32::from(time.second()), + )?; + let local = when.and_local_timezone(chrono::Local).single()?; + Some(local.into()) +} diff --git a/core/src/ops/files/archive/mod.rs b/core/src/ops/files/archive/mod.rs new file mode 100644 index 000000000..bd7e9f414 --- /dev/null +++ b/core/src/ops/files/archive/mod.rs @@ -0,0 +1,21 @@ +//! # Archive and extract +//! +//! Write one archive, zip or tar with zstd, holding the sources, or write +//! an archive's entries into a folder. Both on preflight: an archive +//! previews as one create with the sources' bytes and validation checks +//! the space; an extract is planned from the archive's own directory +//! against the live destination and refuses an entry that would land +//! outside it. + +pub mod action; +pub mod directory; +pub mod input; +pub mod job; +pub mod plan; +mod preflight; +#[cfg(test)] +mod tests; + +pub use action::{FileArchiveAction, FileExtractAction}; +pub use input::{ArchiveFormat, FileArchiveInput, FileExtractInput}; +pub use job::{ArchiveJob, ArchiveOutput, ExtractJob, ExtractOutput}; diff --git a/core/src/ops/files/archive/plan.rs b/core/src/ops/files/archive/plan.rs new file mode 100644 index 000000000..d62e6b5e7 --- /dev/null +++ b/core/src/ops/files/archive/plan.rs @@ -0,0 +1,113 @@ +//! What an extract would leave, entry by entry, and what an archive holds. + +use std::{ + collections::HashSet, + path::{Path, PathBuf}, +}; + +use super::{directory::ArchiveEntry, input::FileExtractInput}; +use crate::ops::files::{ + merge::MergeConflictPolicy, + plan::{ChangeKind, ConflictKind, ReplaceReason, SkipReason}, + planner::is_junk, +}; + +/// Where one entry goes and what happens there. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Decision { + pub entry: ArchiveEntry, + /// Beneath the destination; `None` for an entry that escapes or strips + /// to nothing. + pub path: Option, + pub change: ChangeKind, +} + +/// Decide each entry against the live destination. +pub async fn decide( + input: &FileExtractInput, + destination: &Path, + entries: Vec, +) -> (Vec, u64) { + let mut decisions = Vec::with_capacity(entries.len()); + let mut claimed: HashSet = HashSet::new(); + let mut escapes = 0; + for entry in entries { + let Some(relative) = entry.landing(input.strip_components) else { + escapes += 1; + decisions.push(Decision { + entry, + path: None, + change: ChangeKind::Conflict { + kind: ConflictKind::Sources, + }, + }); + continue; + }; + let path = destination.join(&relative); + let name = relative + .file_name() + .map(|name| name.to_string_lossy().into_owned()) + .unwrap_or_default(); + let existing = tokio::fs::symlink_metadata(&path).await.ok(); + let change = if entry.is_dir { + match &existing { + Some(meta) if meta.is_dir() => ChangeKind::MergeInto, + Some(_) => ChangeKind::Conflict { + kind: ConflictKind::FileVsDirectory, + }, + None => ChangeKind::CreateDirectory, + } + } else if is_junk(&name) { + ChangeKind::Skip { + reason: SkipReason::Junk, + } + } else if !claimed.insert(path.clone()) { + ChangeKind::Conflict { + kind: ConflictKind::Sources, + } + } else { + match &existing { + None => ChangeKind::Create { size: entry.size }, + Some(meta) if meta.is_dir() => ChangeKind::Conflict { + kind: ConflictKind::FileVsDirectory, + }, + Some(meta) => match input.on_conflict { + MergeConflictPolicy::Skip => ChangeKind::Skip { + reason: SkipReason::Policy, + }, + MergeConflictPolicy::Overwrite => ChangeKind::Replace { + existing_size: meta.len(), + incoming_size: entry.size, + reason: ReplaceReason::Overwrite, + }, + MergeConflictPolicy::KeepBoth => ChangeKind::Create { size: entry.size }, + MergeConflictPolicy::KeepNewer => { + let existing_ms = meta + .modified() + .ok() + .and_then(|time| time.duration_since(std::time::UNIX_EPOCH).ok()) + .map(|elapsed| elapsed.as_millis() as i64) + .unwrap_or(0); + if entry.modified_ms.unwrap_or(0) > existing_ms { + ChangeKind::Replace { + existing_size: meta.len(), + incoming_size: entry.size, + reason: ReplaceReason::Newer, + } + } else { + ChangeKind::Skip { + reason: SkipReason::Policy, + } + } + } + }, + } + }; + decisions.push(Decision { + entry, + path: Some(path), + change, + }); + } + (decisions, escapes) +} diff --git a/core/src/ops/files/archive/preflight.rs b/core/src/ops/files/archive/preflight.rs new file mode 100644 index 000000000..c9372bda0 --- /dev/null +++ b/core/src/ops/files/archive/preflight.rs @@ -0,0 +1,482 @@ +//! Whether an archive can be written or extracted, and what would exist +//! after. +//! +//! An archive's plan is one `Create` with the sources' byte estimate, and +//! the sources as `Delete` rows when they are to be removed. An extract's +//! plan is read from the archive's own directory and decided against the +//! live destination: a `Create`, `Replace` or `Skip` per entry, a conflict +//! where an entry meets a folder or escapes the destination, which +//! validation refuses. + +use std::path::Path; + +use super::{ + action::{FileArchiveAction, FileExtractAction}, + directory::{self, ArchiveEntry}, + input::{ArchiveFormat, FileArchiveInput, FileExtractInput}, + plan::decide, +}; +use crate::{ + domain::SdPath, + infra::action::{ + error::ActionError, + preflight::{ + ExecutionFacts, Finding, PreviewContext, PreviewableAction, ValidatedAction, Validation, + }, + }, + ops::files::{ + copy::database::CopyDatabaseQuery, + delete::last_copies, + plan::{ + ChangeKind, ConflictKind, FsPlan, FsPlanSummary, PlanBasis, PlanChanges, PlannedChange, + }, + }, +}; + +pub const REMOTE_ROOT: &str = "archive.remote_root"; +pub const SOURCE_MISSING: &str = "archive.source_missing"; +pub const EXISTS: &str = "archive.exists"; +pub const DESTINATION_MISSING: &str = "archive.destination_missing"; +pub const UNSUPPORTED: &str = "archive.unsupported"; +pub const SPACE: &str = "archive.space"; +pub const LAST_COPY: &str = "archive.last_copy"; +pub const UNREADABLE: &str = "extract.unreadable"; +pub const ESCAPE: &str = "extract.escape"; +pub const EXTRACT_DESTINATION: &str = "extract.destination"; +pub const EXTRACT_SPACE: &str = "extract.space"; +pub const CONFLICTS: &str = "extract.conflicts"; + +impl ValidatedAction for FileArchiveAction { + async fn validate( + input: &FileArchiveInput, + ctx: &PreviewContext, + ) -> Result { + let mut findings = Vec::new(); + let mut facts = ExecutionFacts { + executes_on: ctx.executes_on(), + strategy: Some(input.format.extension().to_string()), + ..Default::default() + }; + let Some(destination) = input.destination.as_local_path() else { + findings.push( + Finding::error( + REMOTE_ROOT, + "the archive would be written on another device", + ) + .at(input.destination.clone()), + ); + return Ok(Validation { findings, facts }); + }; + if tokio::fs::symlink_metadata(destination).await.is_ok() { + findings.push( + Finding::error(EXISTS, "a file is already at the archive's name") + .at(input.destination.clone()), + ); + } + if !destination.parent().is_some_and(|parent| parent.is_dir()) { + findings.push( + Finding::error( + DESTINATION_MISSING, + "the folder the archive would be written in does not exist", + ) + .at(input.destination.clone()), + ); + } + let mut local = Vec::new(); + for source in &input.sources { + let Some(path) = source.as_local_path() else { + findings.push( + Finding::error(REMOTE_ROOT, "a source is on another device").at(source.clone()), + ); + continue; + }; + match tokio::fs::symlink_metadata(path).await { + Ok(meta) => { + if meta.file_type().is_symlink() && input.format == ArchiveFormat::Zip { + findings.push( + Finding::warning( + UNSUPPORTED, + "a zip does not carry a symlink; it is left out", + ) + .at(source.clone()), + ); + } + local.push(source.clone()); + } + Err(_) => findings.push( + Finding::error(SOURCE_MISSING, "a source is not there").at(source.clone()), + ), + } + } + + let estimate = estimate(ctx, &local).await; + facts.estimated_files = Some(estimate.0); + facts.estimated_bytes = Some(estimate.1); + if let Some(volume) = ctx.volumes().volume_for_path(destination).await { + facts.free_space_after = Some(volume.available_space as i64 - estimate.1 as i64); + if estimate.1 > volume.available_space { + findings.push(Finding::warning( + SPACE, + format!( + "{} bytes to archive against {} free on the destination volume; compression may bring that down", + estimate.1, volume.available_space + ), + )); + } + } + if input.remove_sources { + let stranded = last_copies_among(ctx, &local).await; + if stranded > 0 { + findings.push(Finding::warning( + LAST_COPY, + format!( + "{stranded} of the sources are the last copy of their content anywhere in your library; the archive becomes the only copy" + ), + )); + } + } + Ok(Validation { findings, facts }) + } +} + +impl PreviewableAction for FileArchiveAction { + type Plan = FsPlan; + + async fn preview(input: FileArchiveInput, ctx: &PreviewContext) -> Result { + let device = crate::device::get_current_device_slug(); + let mut summary = FsPlanSummary::default(); + let mut changes = PlanChanges::default(); + let (_, bytes) = estimate(ctx, &input.sources).await; + let create = ChangeKind::Create { size: bytes }; + summary.count(&create, bytes); + changes.push(PlannedChange { + path: input.destination.clone(), + change: create, + }); + if input.remove_sources { + for source in &input.sources { + let Some(path) = source.as_local_path() else { + continue; + }; + let size = tokio::fs::symlink_metadata(path) + .await + .map(|meta| if meta.is_dir() { 0 } else { meta.len() }) + .unwrap_or(0); + let change = ChangeKind::Delete { last_copy: false }; + summary.count(&change, size); + changes.push(PlannedChange { + path: SdPath::Physical { + device_slug: device.clone(), + path: path.to_path_buf(), + }, + change, + }); + } + } + let (changes, truncated) = changes.finish(); + Ok(ctx.plans().retain(FsPlan { + handle: None, + basis: PlanBasis::Index { + revisions: Vec::new(), + }, + roots: Vec::new(), + summary, + changes, + truncated, + })) + } +} + +impl ValidatedAction for FileExtractAction { + async fn validate( + input: &FileExtractInput, + ctx: &PreviewContext, + ) -> Result { + let mut findings = Vec::new(); + let mut facts = ExecutionFacts { + executes_on: ctx.executes_on(), + ..Default::default() + }; + let (Some(archive), Some(destination)) = ( + input.archive.as_local_path(), + input.destination.as_local_path(), + ) else { + findings.push(Finding::error( + REMOTE_ROOT, + "the archive or the destination is on another device", + )); + return Ok(Validation { findings, facts }); + }; + if !destination.is_dir() { + findings.push( + Finding::error( + EXTRACT_DESTINATION, + "the destination does not exist; an extract writes into an existing folder", + ) + .at(input.destination.clone()), + ); + } + let entries = match read_directory(archive).await { + Ok((format, entries)) => { + facts.strategy = Some(format.extension().to_string()); + entries + } + Err(error) => { + findings.push( + Finding::error(UNREADABLE, format!("the archive cannot be read: {error}")) + .at(input.archive.clone()), + ); + return Ok(Validation { findings, facts }); + } + }; + let (decisions, escapes) = decide(input, destination, entries).await; + if escapes > 0 { + findings.push( + Finding::error( + ESCAPE, + format!( + "{escapes} entries would land outside the destination, or strip to nothing" + ), + ) + .at(input.archive.clone()), + ); + } + let conflicts = decisions + .iter() + .filter(|decision| { + matches!( + decision.change, + ChangeKind::Conflict { + kind: ConflictKind::FileVsDirectory + } + ) + }) + .count(); + if conflicts > 0 { + findings.push(Finding::warning( + CONFLICTS, + format!("{conflicts} entries meet a folder where a file goes, or a file where a folder goes; they are left"), + )); + } + let bytes: u64 = decisions + .iter() + .filter_map(|decision| match decision.change { + ChangeKind::Create { size } => Some(size), + ChangeKind::Replace { incoming_size, .. } => Some(incoming_size), + _ => None, + }) + .sum(); + facts.estimated_files = Some( + decisions + .iter() + .filter(|decision| { + matches!( + decision.change, + ChangeKind::Create { .. } | ChangeKind::Replace { .. } + ) + }) + .count() as u64, + ); + facts.estimated_bytes = Some(bytes); + if let Some(volume) = ctx.volumes().volume_for_path(destination).await { + facts.free_space_after = Some(volume.available_space as i64 - bytes as i64); + if bytes > volume.available_space { + findings.push(Finding::warning( + EXTRACT_SPACE, + format!( + "{bytes} bytes to extract against {} free on the destination volume", + volume.available_space + ), + )); + } + } + Ok(Validation { findings, facts }) + } +} + +impl PreviewableAction for FileExtractAction { + type Plan = FsPlan; + + async fn preview(input: FileExtractInput, ctx: &PreviewContext) -> Result { + let (Some(archive), Some(destination)) = ( + input.archive.as_local_path(), + input.destination.as_local_path(), + ) else { + return Err(ActionError::InvalidInput( + "the archive and the destination must be on this device".into(), + )); + }; + let (_, entries) = read_directory(archive).await.map_err(|error| { + ActionError::InvalidInput(format!("the archive cannot be read: {error}")) + })?; + let count = entries.len() as u64; + let (decisions, _) = decide(&input, destination, entries).await; + let device = crate::device::get_current_device_slug(); + let mut summary = FsPlanSummary::default(); + let mut changes = PlanChanges::default(); + for decision in decisions { + let path = decision + .path + .unwrap_or_else(|| destination.join(&decision.entry.name)); + let bytes = match decision.change { + ChangeKind::Create { size } => size, + ChangeKind::Replace { incoming_size, .. } => incoming_size, + ChangeKind::Skip { .. } => decision.entry.size, + _ => 0, + }; + summary.count(&decision.change, bytes); + changes.push(PlannedChange { + path: SdPath::Physical { + device_slug: device.clone(), + path, + }, + change: decision.change, + }); + } + let (changes, truncated) = changes.finish(); + Ok(ctx.plans().retain(FsPlan { + handle: None, + basis: PlanBasis::Archive { + path: input.archive.clone(), + entries: count, + }, + roots: Vec::new(), + summary, + changes, + truncated, + })) + } +} + +crate::register_validate!(FileArchiveAction, "files.archive"); +crate::register_preview!(FileArchiveAction, "files.archive"); +crate::register_validate!(FileExtractAction, "files.extract"); +crate::register_preview!(FileExtractAction, "files.extract"); + +/// The archive's format from its name, and its directory. +pub(super) async fn read_directory( + archive: &Path, +) -> Result<(ArchiveFormat, Vec), String> { + let name = archive + .file_name() + .map(|name| name.to_string_lossy().into_owned()) + .unwrap_or_default(); + let format = ArchiveFormat::of_name(&name) + .ok_or_else(|| "only .zip and .tar.zst are read".to_string())?; + let path = archive.to_path_buf(); + let entries = tokio::task::spawn_blocking(move || directory::read(&path, format)) + .await + .map_err(|error| error.to_string())? + .map_err(|error| error.to_string())?; + Ok((format, entries)) +} + +/// Files and bytes beneath the sources, from the index where it has them, +/// else from the disk. +async fn estimate(ctx: &PreviewContext, sources: &[SdPath]) -> (u64, u64) { + let mut spelled = Vec::with_capacity(sources.len()); + for source in sources { + let Some(path) = source.as_local_path() else { + continue; + }; + spelled.push(match ctx.volumes().locate_path(path).await { + Some((_, path)) => SdPath::local(path), + None => source.clone(), + }); + } + if let Ok(estimates) = CopyDatabaseQuery::new(ctx.index()) + .get_estimates_for_paths(&spelled) + .await + { + if estimates.is_complete() { + return (estimates.file_count, estimates.total_size); + } + } + let mut files = 0; + let mut bytes = 0; + for source in sources { + let Some(path) = source.as_local_path() else { + continue; + }; + walk(path, &mut files, &mut bytes).await; + } + (files, bytes) +} + +async fn walk(path: &Path, files: &mut u64, bytes: &mut u64) { + let mut stack = vec![path.to_path_buf()]; + while let Some(current) = stack.pop() { + let Ok(meta) = tokio::fs::symlink_metadata(¤t).await else { + continue; + }; + if meta.is_dir() { + if let Ok(mut entries) = tokio::fs::read_dir(¤t).await { + while let Ok(Some(entry)) = entries.next_entry().await { + stack.push(entry.path()); + } + } + } else if meta.is_file() { + *files += 1; + *bytes += meta.len(); + } + } +} + +/// How many of the sources' files are the last copy of their content. +async fn last_copies_among(ctx: &PreviewContext, sources: &[SdPath]) -> u64 { + let mut hashes = Vec::new(); + for source in sources { + for reach in ctx.reach(source).await { + let Some(db) = ctx.index().read_store(reach.source.id).await else { + continue; + }; + if reach.prefix.is_empty() { + if let Ok(Some(entry)) = + sd_store::read::entry_by_path(db.pool(), &reach.scope).await + { + if entry.kind == sd_store::FileKind::File { + hashes.extend(entry.sampled_hash); + continue; + } + } + } + let mut cursor: Option<(String, String)> = None; + loop { + let batch = { + let start = match &cursor { + Some((directory, name)) => sd_store::read::Start::After { directory, name }, + None => sd_store::read::Start::First, + }; + match sd_store::read::files_beneath( + db.pool(), + &reach.scope, + start, + None, + true, + 1000, + ) + .await + { + Ok(batch) => batch, + Err(_) => break, + } + }; + let full = batch.len() == 1000; + for entry in &batch { + hashes.extend(entry.sampled_hash.clone()); + } + cursor = batch + .last() + .map(|entry| (entry.directory().to_string(), entry.name.clone())); + if !full { + break; + } + } + } + } + let stranded = last_copies(ctx, hashes.clone()).await; + hashes + .iter() + .filter(|hash| stranded.contains(*hash)) + .count() as u64 +} diff --git a/core/src/ops/files/archive/tests.rs b/core/src/ops/files/archive/tests.rs new file mode 100644 index 000000000..e72d59356 --- /dev/null +++ b/core/src/ops/files/archive/tests.rs @@ -0,0 +1,264 @@ +//! An archive round trip in both formats, with a folder collision, a path +//! escape refused, and an extract resumed at its entry. + +use std::time::Duration; + +use sd_store::file::FileKind; + +use super::{ + input::{ArchiveFormat, FileArchiveInput, FileExtractInput}, + job::{ArchiveJob, ExtractJob}, + preflight::{ESCAPE, EXISTS}, + FileArchiveAction, FileExtractAction, +}; +use crate::{ + domain::SdPath, + infra::{ + action::preflight::{PreviewableAction, ValidatedAction}, + job::{journal::Effect, output::JobOutput}, + }, + ops::files::{ + fixture::{Fixture, T}, + merge::MergeConflictPolicy, + plan::{ChangeKind, ConflictKind, PlanBasis}, + }, +}; + +async fn wait(handle: crate::infra::job::handle::JobHandle) -> serde_json::Value { + let output = tokio::time::timeout(Duration::from_secs(60), handle.wait()) + .await + .expect("in time") + .expect("completed"); + let JobOutput::Custom(value) = output else { + panic!("not a custom output: {output:?}"); + }; + value +} + +async fn round_trip(format: ArchiveFormat) { + let fixture = Fixture::new().await; + let file = FileKind::File; + let tree = [ + ("a.txt", file, 4, T, Some("a"), None, None), + ("sub/b.txt", file, 3, T, Some("b"), None, None), + ("sub/deep/c.txt", file, 2, T, Some("c"), None, None), + ]; + fixture.materialize(&fixture.source, &tree); + fixture.index(&fixture.source, &tree).await; + let archive = fixture.other.join(format!("src.{}", format.extension())); + + let input = FileArchiveInput { + sources: vec![SdPath::local(&fixture.source)], + destination: SdPath::local(&archive), + format, + remove_sources: false, + }; + let validation = FileArchiveAction::validate(&input, &fixture.preview()) + .await + .expect("validated"); + assert!(!validation.refuses(), "{:?}", validation.findings); + assert_eq!(validation.facts.estimated_files, Some(3)); + let plan = FileArchiveAction::preview(input.clone(), &fixture.preview()) + .await + .expect("planned"); + assert_eq!(plan.summary.creates.files, 1); + assert_eq!(plan.summary.creates.bytes, 9); + + let handle = fixture + .library + .jobs() + .dispatch(ArchiveJob::new(input.clone())) + .await + .expect("dispatched"); + let job_id = handle.id(); + let output = wait(handle).await; + assert_eq!(output["files"], 3); + assert!(archive.exists()); + let partial: Vec<_> = std::fs::read_dir(&fixture.other) + .expect("dir") + .filter_map(|entry| entry.ok()) + .filter(|entry| entry.file_name().to_string_lossy().contains("sd-partial")) + .collect(); + assert!(partial.is_empty(), "no temporary file is left behind"); + let journal = fixture + .library + .jobs() + .database() + .journal(job_id) + .await + .expect("journal"); + assert!(matches!(journal[0].effect, Effect::Created { .. })); + + // Writing again at the same name is refused. + let validation = FileArchiveAction::validate(&input, &fixture.preview()) + .await + .expect("validated"); + assert!(validation.errors().any(|finding| finding.code == EXISTS)); + + // Extract into the destination, where a folder already holds one name + // as a file and one file differs. + let src_name = fixture + .source + .file_name() + .expect("name") + .to_string_lossy() + .into_owned(); + std::fs::create_dir_all(fixture.destination.join(&src_name)).expect("dir"); + std::fs::write(fixture.destination.join(&src_name).join("a.txt"), b"old").expect("file"); + std::fs::create_dir_all(fixture.destination.join(&src_name).join("sub/b.txt")) + .expect("dir in the way"); + + let extract = FileExtractInput { + archive: SdPath::local(&archive), + destination: SdPath::local(&fixture.destination), + on_conflict: MergeConflictPolicy::Overwrite, + strip_components: 0, + }; + let validation = FileExtractAction::validate(&extract, &fixture.preview()) + .await + .expect("validated"); + assert!(!validation.refuses(), "{:?}", validation.findings); + let plan = FileExtractAction::preview(extract.clone(), &fixture.preview()) + .await + .expect("planned"); + assert!(matches!(plan.basis, PlanBasis::Archive { entries, .. } if entries >= 3)); + let at = |name: &str| { + plan.changes + .iter() + .find(|change| { + change.path.path().map(std::path::PathBuf::as_path) + == Some(fixture.destination.join(&src_name).join(name).as_path()) + }) + .map(|change| change.change.clone()) + }; + assert!(matches!(at("a.txt"), Some(ChangeKind::Replace { .. }))); + assert_eq!( + at("sub/b.txt"), + Some(ChangeKind::Conflict { + kind: ConflictKind::FileVsDirectory + }) + ); + assert_eq!(at("sub/deep/c.txt"), Some(ChangeKind::Create { size: 2 })); + + let handle = fixture + .library + .jobs() + .dispatch(ExtractJob::new(extract)) + .await + .expect("dispatched"); + let output = wait(handle).await; + assert_eq!(output["replaced"], 1, "{output}"); + assert_eq!(output["conflicts"], 1); + assert_eq!( + std::fs::read_to_string(fixture.destination.join(&src_name).join("a.txt")) + .expect("replaced"), + "xxxx" + ); + assert_eq!( + std::fs::read_to_string(fixture.destination.join(&src_name).join("sub/deep/c.txt")) + .expect("created"), + "xx" + ); +} + +#[tokio::test] +async fn a_zip_round_trips_with_a_collision_and_a_replacement() { + round_trip(ArchiveFormat::Zip).await; +} + +#[tokio::test] +async fn a_tar_zstd_round_trips_with_a_collision_and_a_replacement() { + round_trip(ArchiveFormat::TarZstd).await; +} + +/// An entry that would land outside the destination refuses the extract, +/// and stripping a component lands the rest one level up. +#[tokio::test] +async fn an_escaping_entry_is_refused_and_components_strip() { + let fixture = Fixture::new().await; + let archive = fixture.other.join("bad.zip"); + { + let file = std::fs::File::create(&archive).expect("file"); + let mut writer = zip::ZipWriter::new(file); + let options = zip::write::SimpleFileOptions::default(); + writer.start_file("top/inner.txt", options).expect("entry"); + std::io::Write::write_all(&mut writer, b"in").expect("bytes"); + writer.start_file("../escape.txt", options).expect("entry"); + std::io::Write::write_all(&mut writer, b"out").expect("bytes"); + writer.finish().expect("finished"); + } + let extract = FileExtractInput { + archive: SdPath::local(&archive), + destination: SdPath::local(&fixture.destination), + on_conflict: MergeConflictPolicy::Skip, + strip_components: 0, + }; + let validation = FileExtractAction::validate(&extract, &fixture.preview()) + .await + .expect("validated"); + assert!(validation.errors().any(|finding| finding.code == ESCAPE)); + + let stripped = FileExtractInput { + strip_components: 1, + ..extract + }; + let plan = FileExtractAction::preview(stripped, &fixture.preview()) + .await + .expect("planned"); + assert!(plan.changes.iter().any(|change| { + change.path.path().map(std::path::PathBuf::as_path) + == Some(fixture.destination.join("inner.txt").as_path()) + && matches!(change.change, ChangeKind::Create { .. }) + })); +} + +/// A resumed extract starts at the entry it reached and writes the rest. +#[tokio::test] +async fn a_resumed_extract_continues_at_its_entry() { + let fixture = Fixture::new().await; + let archive = fixture.other.join("many.zip"); + { + let file = std::fs::File::create(&archive).expect("file"); + let mut writer = zip::ZipWriter::new(file); + let options = zip::write::SimpleFileOptions::default(); + for index in 0..6 { + writer + .start_file(format!("f{index}.txt"), options) + .expect("entry"); + std::io::Write::write_all(&mut writer, format!("{index}").as_bytes()).expect("bytes"); + } + writer.finish().expect("finished"); + } + let mut job = ExtractJob::new(FileExtractInput { + archive: SdPath::local(&archive), + destination: SdPath::local(&fixture.destination), + on_conflict: MergeConflictPolicy::Skip, + strip_components: 0, + }); + // The first three entries were written before the interruption. + for index in 0..3 { + std::fs::write(fixture.destination.join(format!("f{index}.txt")), b"kept").expect("file"); + } + job.next = 3; + let handle = fixture + .library + .jobs() + .dispatch(job) + .await + .expect("dispatched"); + let output = wait(handle).await; + assert_eq!(output["extracted"], 3, "{output}"); + for index in 0..3 { + assert_eq!( + std::fs::read_to_string(fixture.destination.join(format!("f{index}.txt"))).unwrap(), + "kept", + "entries before the cursor are left as they were" + ); + } + for index in 3..6 { + assert_eq!( + std::fs::read_to_string(fixture.destination.join(format!("f{index}.txt"))).unwrap(), + index.to_string() + ); + } +} diff --git a/core/src/ops/files/attributes_action/action.rs b/core/src/ops/files/attributes_action/action.rs new file mode 100644 index 000000000..353c3a623 --- /dev/null +++ b/core/src/ops/files/attributes_action/action.rs @@ -0,0 +1,77 @@ +use std::sync::Arc; + +use serde::{Deserialize, Serialize}; +use specta::Type; + +use super::job::AttributesJob; +use crate::{ + context::CoreContext, + domain::SdPath, + infra::{ + action::{error::ActionError, LibraryAction}, + job::{handle::JobReceipt, journal::Attributes}, + }, +}; + +/// Set attributes on files; each absent attribute stays as it is. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Type)] +pub struct FileSetAttributesInput { + pub paths: Vec, + pub attributes: Attributes, +} + +impl FileSetAttributesInput { + pub fn validate(&self) -> Result<(), Vec> { + let mut errors = Vec::new(); + if self.paths.is_empty() { + errors.push("name at least one file".to_string()); + } + if self.attributes.mode.is_none() + && self.attributes.modified_ms.is_none() + && self.attributes.hidden.is_none() + { + errors.push("name at least one attribute to set".to_string()); + } + if self.attributes.mode.is_some_and(|mode| mode > 0o7777) { + errors.push("a mode is at most 0o7777".to_string()); + } + if errors.is_empty() { + Ok(()) + } else { + Err(errors) + } + } +} + +pub struct FileSetAttributesAction { + input: FileSetAttributesInput, +} + +impl LibraryAction for FileSetAttributesAction { + type Input = FileSetAttributesInput; + type Output = JobReceipt; + + fn from_input(input: Self::Input) -> Result { + input.validate().map_err(|errors| errors.join("; "))?; + Ok(Self { input }) + } + + async fn execute( + self, + library: Arc, + _context: Arc, + ) -> Result { + let job_handle = library + .jobs() + .dispatch(AttributesJob::new(self.input)) + .await + .map_err(ActionError::Job)?; + Ok(job_handle.into()) + } + + fn action_kind(&self) -> &'static str { + "files.set_attributes" + } +} + +crate::register_library_action!(FileSetAttributesAction, "files.set_attributes"); diff --git a/core/src/ops/files/attributes_action/fs.rs b/core/src/ops/files/attributes_action/fs.rs new file mode 100644 index 000000000..4db42c6c4 --- /dev/null +++ b/core/src/ops/files/attributes_action/fs.rs @@ -0,0 +1,180 @@ +//! Filesystem attributes a path carries: the permission mode, the +//! modification time, and whether it is hidden. +//! +//! Hidden is a flag on macOS (`UF_HIDDEN`) and Windows +//! (`FILE_ATTRIBUTE_HIDDEN`); elsewhere it is a leading dot in the name, +//! which is a rename rather than an attribute, so setting it there is +//! refused. + +use std::{ + io, + path::Path, + time::{Duration, UNIX_EPOCH}, +}; + +use crate::infra::job::journal::Attributes; + +/// The attributes as the filesystem holds them now. +pub async fn read(path: &Path) -> io::Result { + let meta = tokio::fs::symlink_metadata(path).await?; + Ok(Attributes { + mode: mode_of(&meta), + modified_ms: meta + .modified() + .ok() + .and_then(|time| time.duration_since(UNIX_EPOCH).ok()) + .map(|elapsed| elapsed.as_millis() as i64), + hidden: Some(hidden_of(path, &meta)), + }) +} + +/// Set each attribute the change names. +pub async fn apply(path: &Path, attributes: &Attributes) -> io::Result<()> { + if let Some(mode) = attributes.mode { + set_mode(path, mode).await?; + } + if let Some(modified_ms) = attributes.modified_ms { + let time = UNIX_EPOCH + Duration::from_millis(modified_ms.max(0) as u64); + let file = std::fs::OpenOptions::new() + .write(true) + .open(path) + .or_else(|_| std::fs::File::open(path))?; + file.set_modified(time)?; + } + if let Some(hidden) = attributes.hidden { + set_hidden(path, hidden).await?; + } + Ok(()) +} + +/// Whether any named attribute differs from what the path carries. +pub fn differs(current: &Attributes, wanted: &Attributes) -> bool { + wanted.mode.is_some_and(|mode| current.mode != Some(mode)) + || wanted + .modified_ms + .is_some_and(|ms| current.modified_ms != Some(ms)) + || wanted + .hidden + .is_some_and(|hidden| current.hidden != Some(hidden)) +} + +/// The attributes that change: each named one that differs, the rest +/// absent. +pub fn changing(current: &Attributes, wanted: &Attributes) -> Attributes { + Attributes { + mode: wanted.mode.filter(|mode| current.mode != Some(*mode)), + modified_ms: wanted + .modified_ms + .filter(|ms| current.modified_ms != Some(*ms)), + hidden: wanted + .hidden + .filter(|hidden| current.hidden != Some(*hidden)), + } +} + +/// Whether the filesystem holding `path` can carry each attribute. +pub fn supported( + file_system: &crate::volume::types::FileSystem, + attributes: &Attributes, +) -> Result<(), String> { + use crate::volume::types::FileSystem; + if attributes.mode.is_some() && matches!(file_system, FileSystem::FAT32 | FileSystem::ExFAT) { + return Err(format!("{file_system} has no permission bits")); + } + if attributes.hidden.is_some() && !cfg!(any(target_os = "macos", target_os = "windows")) { + return Err("hidden is a leading dot on this platform, which is a rename".to_string()); + } + Ok(()) +} + +#[cfg(unix)] +fn mode_of(meta: &std::fs::Metadata) -> Option { + use std::os::unix::fs::PermissionsExt; + Some(meta.permissions().mode() & 0o7777) +} + +#[cfg(not(unix))] +fn mode_of(_meta: &std::fs::Metadata) -> Option { + None +} + +#[cfg(unix)] +async fn set_mode(path: &Path, mode: u32) -> io::Result<()> { + use std::os::unix::fs::PermissionsExt; + tokio::fs::set_permissions(path, std::fs::Permissions::from_mode(mode)).await +} + +#[cfg(not(unix))] +async fn set_mode(_path: &Path, _mode: u32) -> io::Result<()> { + Err(io::Error::new( + io::ErrorKind::Unsupported, + "this platform has no permission mode", + )) +} + +#[cfg(target_os = "macos")] +fn hidden_of(_path: &Path, meta: &std::fs::Metadata) -> bool { + use std::os::macos::fs::MetadataExt; + meta.st_flags() & libc::UF_HIDDEN != 0 +} + +#[cfg(target_os = "macos")] +async fn set_hidden(path: &Path, hidden: bool) -> io::Result<()> { + use std::os::macos::fs::MetadataExt; + let meta = tokio::fs::symlink_metadata(path).await?; + let flags = if hidden { + meta.st_flags() | libc::UF_HIDDEN + } else { + meta.st_flags() & !libc::UF_HIDDEN + }; + let owned = path.to_path_buf(); + tokio::task::spawn_blocking(move || { + let c_path = std::ffi::CString::new(owned.as_os_str().as_encoded_bytes()) + .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "a NUL in the path"))?; + if unsafe { libc::chflags(c_path.as_ptr(), flags) } == 0 { + Ok(()) + } else { + Err(io::Error::last_os_error()) + } + }) + .await + .map_err(|error| io::Error::other(error.to_string()))? +} + +#[cfg(windows)] +fn hidden_of(_path: &Path, meta: &std::fs::Metadata) -> bool { + use std::os::windows::fs::MetadataExt; + meta.file_attributes() & windows_sys::Win32::Storage::FileSystem::FILE_ATTRIBUTE_HIDDEN != 0 +} + +#[cfg(windows)] +async fn set_hidden(path: &Path, hidden: bool) -> io::Result<()> { + use std::os::windows::{ffi::OsStrExt, fs::MetadataExt}; + use windows_sys::Win32::Storage::FileSystem::{SetFileAttributesW, FILE_ATTRIBUTE_HIDDEN}; + let meta = tokio::fs::symlink_metadata(path).await?; + let attributes = if hidden { + meta.file_attributes() | FILE_ATTRIBUTE_HIDDEN + } else { + meta.file_attributes() & !FILE_ATTRIBUTE_HIDDEN + }; + let wide: Vec = path.as_os_str().encode_wide().chain(Some(0)).collect(); + if unsafe { SetFileAttributesW(wide.as_ptr(), attributes) } == 0 { + return Err(io::Error::last_os_error()); + } + Ok(()) +} + +#[cfg(not(any(target_os = "macos", windows)))] +fn hidden_of(path: &Path, _meta: &std::fs::Metadata) -> bool { + path.file_name() + .and_then(|name| name.to_str()) + .is_some_and(|name| name.starts_with('.')) +} + +#[cfg(not(any(target_os = "macos", windows)))] +async fn set_hidden(_path: &Path, _hidden: bool) -> io::Result<()> { + Err(io::Error::new( + io::ErrorKind::Unsupported, + "hidden is a leading dot on this platform", + )) +} diff --git a/core/src/ops/files/attributes_action/job.rs b/core/src/ops/files/attributes_action/job.rs new file mode 100644 index 000000000..a6458a7e6 --- /dev/null +++ b/core/src/ops/files/attributes_action/job.rs @@ -0,0 +1,120 @@ +use std::{path::PathBuf, time::Instant}; + +use serde::{Deserialize, Serialize}; +use specta::Type; + +use super::{action::FileSetAttributesInput, fs}; +use crate::infra::job::{generic_progress::GenericProgress, journal::Effect, prelude::*}; + +#[derive(Debug, Serialize, Deserialize, Job)] +pub struct AttributesJob { + pub input: FileSetAttributesInput, + #[serde(skip, default = "Instant::now")] + started_at: Instant, +} + +impl AttributesJob { + pub fn new(input: FileSetAttributesInput) -> Self { + Self { + input, + started_at: Instant::now(), + } + } +} + +impl Job for AttributesJob { + const NAME: &'static str = "set_attributes"; + const RESUMABLE: bool = false; + const DESCRIPTION: Option<&'static str> = Some("Set file attributes"); +} + +impl crate::infra::job::traits::DynJob for AttributesJob { + fn job_name(&self) -> &'static str { + Self::NAME + } +} + +#[derive(Debug, Clone, Default, Serialize, Deserialize, Type)] +pub struct AttributesOutput { + pub changed: u64, + pub unchanged: u64, + pub failed: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Type)] +pub struct AttributesProblem { + pub path: PathBuf, + pub reason: String, +} + +impl From for JobOutput { + fn from(output: AttributesOutput) -> Self { + JobOutput::custom(output) + } +} + +#[async_trait::async_trait] +impl JobHandler for AttributesJob { + type Output = AttributesOutput; + + async fn run(&mut self, ctx: JobContext<'_>) -> JobResult { + let mut output = AttributesOutput::default(); + let total = self.input.paths.len() as u64; + for (done, path) in self.input.paths.iter().enumerate() { + ctx.check_interrupt().await?; + let Some(local) = path.as_local_path() else { + continue; + }; + let before = match fs::read(local).await { + Ok(before) => before, + Err(error) => { + output.failed.push(AttributesProblem { + path: local.to_path_buf(), + reason: error.to_string(), + }); + continue; + } + }; + if !fs::differs(&before, &self.input.attributes) { + output.unchanged += 1; + continue; + } + let to = fs::changing(&before, &self.input.attributes); + match fs::apply(local, &to).await { + Ok(()) => { + let from = fs::changing(&to, &before); + ctx.record(vec![Effect::Attributes { + path: local.to_path_buf(), + from, + to, + }]) + .await; + output.changed += 1; + } + Err(error) => { + ctx.log(format!("{}: {error}", local.display())); + output.failed.push(AttributesProblem { + path: local.to_path_buf(), + reason: error.to_string(), + }); + } + } + ctx.progress(Progress::generic( + GenericProgress::new( + ((done + 1) as f32 / total.max(1) as f32).min(1.0), + "Setting attributes", + format!("{} of {total}", done + 1), + ) + .with_completion(done as u64 + 1, total) + .with_errors(output.failed.len() as u64, 0), + )); + } + ctx.progress(Progress::generic( + GenericProgress::new(1.0, "Complete", format!("{} changed", output.changed)) + .with_completion(total, total) + .with_performance(0.0, None, Some(self.started_at.elapsed())) + .with_errors(output.failed.len() as u64, 0), + )); + Ok(output) + } +} diff --git a/core/src/ops/files/attributes_action/mod.rs b/core/src/ops/files/attributes_action/mod.rs new file mode 100644 index 000000000..58d2be54a --- /dev/null +++ b/core/src/ops/files/attributes_action/mod.rs @@ -0,0 +1,17 @@ +//! # Attributes +//! +//! Set what the filesystem lets a path carry: the permission mode, the +//! modification time, and whether it is hidden. On preflight: validation +//! refuses what the target filesystem cannot express, and the plan is one +//! row per file naming what changes. The job journals each path's +//! attributes before and after, so an undo sets them back. + +mod action; +pub mod fs; +mod job; +mod preflight; +#[cfg(test)] +mod tests; + +pub use action::{FileSetAttributesAction, FileSetAttributesInput}; +pub use job::{AttributesJob, AttributesOutput}; diff --git a/core/src/ops/files/attributes_action/preflight.rs b/core/src/ops/files/attributes_action/preflight.rs new file mode 100644 index 000000000..6d808e87f --- /dev/null +++ b/core/src/ops/files/attributes_action/preflight.rs @@ -0,0 +1,113 @@ +use super::{ + action::{FileSetAttributesAction, FileSetAttributesInput}, + fs, +}; +use crate::{ + infra::action::{ + error::ActionError, + preflight::{ + ExecutionFacts, Finding, PreviewContext, PreviewableAction, ValidatedAction, Validation, + }, + }, + ops::files::plan::{ChangeKind, FsPlan, FsPlanSummary, PlanBasis, PlanChanges, PlannedChange}, +}; + +pub const REMOTE_ROOT: &str = "attributes.remote_root"; +pub const MISSING: &str = "attributes.missing"; +pub const UNSUPPORTED: &str = "attributes.unsupported"; +pub const UNCHANGED: &str = "attributes.unchanged"; + +impl ValidatedAction for FileSetAttributesAction { + async fn validate( + input: &FileSetAttributesInput, + ctx: &PreviewContext, + ) -> Result { + let mut findings = Vec::new(); + let mut changing = 0; + let mut unchanged = 0; + for path in &input.paths { + let Some(local) = path.as_local_path() else { + findings.push( + Finding::error(REMOTE_ROOT, "a file is on another device").at(path.clone()), + ); + continue; + }; + let Ok(current) = fs::read(local).await else { + findings.push(Finding::error(MISSING, "a file is not there").at(path.clone())); + continue; + }; + if let Some(volume) = ctx.volumes().volume_for_path(local).await { + if let Err(reason) = fs::supported(&volume.file_system, &input.attributes) { + findings.push(Finding::error(UNSUPPORTED, reason).at(path.clone())); + continue; + } + } + if fs::differs(¤t, &input.attributes) { + changing += 1; + } else { + unchanged += 1; + } + } + if unchanged > 0 { + findings.push(Finding::info( + UNCHANGED, + format!("{unchanged} files already carry these attributes"), + )); + } + Ok(Validation { + findings, + facts: ExecutionFacts { + executes_on: ctx.executes_on(), + strategy: Some("attributes".to_string()), + estimated_files: Some(changing), + estimated_bytes: Some(0), + free_space_after: None, + }, + }) + } +} + +impl PreviewableAction for FileSetAttributesAction { + type Plan = FsPlan; + + async fn preview( + input: FileSetAttributesInput, + ctx: &PreviewContext, + ) -> Result { + let mut summary = FsPlanSummary::default(); + let mut changes = PlanChanges::default(); + for path in &input.paths { + let Some(local) = path.as_local_path() else { + continue; + }; + let Ok(current) = fs::read(local).await else { + continue; + }; + if !fs::differs(¤t, &input.attributes) { + continue; + } + let change = ChangeKind::SetAttributes { + attributes: fs::changing(¤t, &input.attributes), + }; + summary.count(&change, 0); + changes.push(PlannedChange { + path: path.clone(), + change, + }); + } + let (changes, truncated) = changes.finish(); + Ok(ctx.plans().retain(FsPlan { + handle: None, + basis: PlanBasis::Index { + revisions: Vec::new(), + }, + roots: Vec::new(), + summary, + changes, + truncated, + })) + } +} + +crate::register_validate!(FileSetAttributesAction, "files.set_attributes"); +crate::register_preview!(FileSetAttributesAction, "files.set_attributes"); diff --git a/core/src/ops/files/attributes_action/tests.rs b/core/src/ops/files/attributes_action/tests.rs new file mode 100644 index 000000000..da2b8dfc1 --- /dev/null +++ b/core/src/ops/files/attributes_action/tests.rs @@ -0,0 +1,117 @@ +//! Attributes preview what changes, refuse what a filesystem cannot carry, +//! and undo sets them back. + +use std::time::Duration; + +use sd_store::file::FileKind; + +use super::{ + fs, preflight::UNSUPPORTED, AttributesJob, FileSetAttributesAction, FileSetAttributesInput, +}; +use crate::{ + domain::SdPath, + infra::{ + action::preflight::{PreviewableAction, ValidatedAction}, + job::{journal::Attributes, output::JobOutput}, + }, + ops::files::{ + fixture::{Fixture, T}, + plan::ChangeKind, + undo::UndoJob, + }, + volume::types::FileSystem, +}; + +#[test] +fn exfat_refuses_a_mode_and_a_dot_platform_refuses_hidden() { + let mode = Attributes { + mode: Some(0o600), + modified_ms: None, + hidden: None, + }; + assert!(fs::supported(&FileSystem::ExFAT, &mode).is_err()); + assert!(fs::supported(&FileSystem::FAT32, &mode).is_err()); + assert!(fs::supported(&FileSystem::APFS, &mode).is_ok()); + let hidden = Attributes { + mode: None, + modified_ms: None, + hidden: Some(true), + }; + assert_eq!( + fs::supported(&FileSystem::Ext4, &hidden).is_ok(), + cfg!(any(target_os = "macos", target_os = "windows")) + ); +} + +#[cfg(unix)] +#[tokio::test] +async fn attributes_preview_what_changes_and_undo_sets_them_back() { + let fixture = Fixture::new().await; + let tree = [("a.txt", FileKind::File, 4, T, Some("a"), None, None)]; + fixture.materialize(&fixture.source, &tree); + fixture.index(&fixture.source, &tree).await; + let a = fixture.source.join("a.txt"); + let before = fs::read(&a).await.expect("read"); + + let input = FileSetAttributesInput { + paths: vec![SdPath::local(&a)], + attributes: Attributes { + mode: Some(0o600), + modified_ms: Some(1_700_000_000_000), + hidden: None, + }, + }; + let validation = FileSetAttributesAction::validate(&input, &fixture.preview()) + .await + .expect("validated"); + assert!(!validation.refuses(), "{:?}", validation.findings); + assert!(!validation + .errors() + .any(|finding| finding.code == UNSUPPORTED)); + let plan = FileSetAttributesAction::preview(input.clone(), &fixture.preview()) + .await + .expect("planned"); + assert_eq!(plan.summary.attributes, 1); + assert!(matches!( + plan.changes[0].change, + ChangeKind::SetAttributes { + attributes: Attributes { + mode: Some(0o600), + .. + } + } + )); + + let handle = fixture + .library + .jobs() + .dispatch(AttributesJob::new(input)) + .await + .expect("dispatched"); + let job = handle.id(); + let output = tokio::time::timeout(Duration::from_secs(30), handle.wait()) + .await + .expect("in time") + .expect("completed"); + let JobOutput::Custom(value) = output else { + panic!("{output:?}"); + }; + assert_eq!(value["changed"], 1); + let after = fs::read(&a).await.expect("read"); + assert_eq!(after.mode, Some(0o600)); + assert_eq!(after.modified_ms, Some(1_700_000_000_000)); + + let handle = fixture + .library + .jobs() + .dispatch(UndoJob::new(job.0, None)) + .await + .expect("dispatched"); + tokio::time::timeout(Duration::from_secs(30), handle.wait()) + .await + .expect("in time") + .expect("completed"); + let restored = fs::read(&a).await.expect("read"); + assert_eq!(restored.mode, before.mode); + assert_eq!(restored.modified_ms, before.modified_ms); +} diff --git a/core/src/ops/files/copy/job.rs b/core/src/ops/files/copy/job.rs index f3dec5cea..698af0514 100644 --- a/core/src/ops/files/copy/job.rs +++ b/core/src/ops/files/copy/job.rs @@ -3,17 +3,25 @@ //! Implements file copy and move operations using the Strategy Pattern with real-time //! progress tracking and transfer speed calculation. Supports resume on interruption. -use super::{database::CopyDatabaseQuery, input::CopyMethod, routing::CopyStrategyRouter}; +use super::{ + action::FileConflictResolution, database::CopyDatabaseQuery, input::CopyMethod, + routing::CopyStrategyRouter, +}; use crate::{ domain::addressing::{SdPath, SdPathBatch}, infra::job::generic_progress::{GenericProgress, ToGenericProgress}, + infra::job::journal::Effect, infra::job::prelude::*, + ops::files::{ + merge::{job::Merge, MergeConflictPolicy}, + trash, + }, }; use serde::{Deserialize, Serialize}; use specta::Type; use std::{ collections::HashMap, - path::PathBuf, + path::{Path, PathBuf}, sync::{Arc, Mutex}, time::{Duration, Instant}, }; @@ -498,60 +506,149 @@ impl JobHandler for FileCopyJob { final_destination.display() ); - // Handle conflict resolution before copying - let final_destination = if let Some(resolution) = self.options.conflict_resolution { - match resolution { - super::action::FileConflictResolution::Skip => { - // Check if destination exists - if let Some(dest_path) = final_destination.as_local_path() { - if dest_path.exists() { - ctx.log(format!("Skipping existing file: {}", dest_path.display())); + // The policy for what is already at the destination: what the + // caller chose, else overwrite or skip as the option says. + let resolution = + self.options + .conflict_resolution + .unwrap_or(if self.options.overwrite { + FileConflictResolution::Overwrite + } else { + FileConflictResolution::Skip + }); + if resolution == FileConflictResolution::Abort { + return Err(JobError::execution("Operation aborted by user")); + } + let source_path = resolved_source.as_local_path().map(Path::to_path_buf); + let dest_path = final_destination.as_local_path().map(Path::to_path_buf); + let existing = match &dest_path { + Some(dest_path) => tokio::fs::symlink_metadata(dest_path).await.ok(), + None => None, + }; + let source_is_dir = match &source_path { + Some(source_path) => tokio::fs::symlink_metadata(source_path) + .await + .is_ok_and(|meta| meta.is_dir()), + None => false, + }; - // Mark as skipped in metadata - self.job_metadata.update_status( - &resolved_source, - super::metadata::CopyFileStatus::Skipped, - ); - - // Skip this file - progress_aggregator.complete_source(); - copied_count += files_in_source; - self.completed_indices.push(index); - continue; - } - } - final_destination + if let (Some(there), Some(source_path), Some(dest_path)) = + (existing.as_ref(), &source_path, &dest_path) + { + // A folder onto a folder already there merges leaf by leaf, + // as the plan said it would, unless both are to be kept. + if source_is_dir + && there.is_dir() + && resolution != FileConflictResolution::AutoModifyName + { + let policy = match resolution { + FileConflictResolution::Overwrite => MergeConflictPolicy::Overwrite, + _ => MergeConflictPolicy::Skip, + }; + let moving = is_move + && match volume_manager.as_deref() { + Some(vm) => vm.same_volume(source_path, dest_path).await, + None => false, + }; + let mut merge = Merge::new(&ctx, policy, is_move, moving, HashMap::new()); + merge + .directory(index, source_path, source_path, dest_path, "") + .await?; + let run = merge.finish().await?; + if is_move && tokio::fs::remove_dir(source_path).await.is_ok() { + ctx.record(vec![Effect::removed(source_path.clone())]).await; } - super::action::FileConflictResolution::AutoModifyName => { - // Generate unique name if destination exists - if let Some(dest_path) = final_destination.as_local_path() { - if dest_path.exists() { - let unique_dest = self.generate_unique_name(&dest_path).await?; - SdPath::Physical { - device_slug: final_destination - .device_slug() - .unwrap_or_default() - .to_string(), - path: unique_dest, - } - } else { - final_destination + for outcome in &run.outcomes { + match &outcome.result { + crate::ops::files::merge::MergeResult::Copied { .. } + | crate::ops::files::merge::MergeResult::Replaced { .. } => { + copied_count += 1; } - } else { - final_destination + crate::ops::files::merge::MergeResult::Failed { error } => { + failed_copies.push(CopyError { + source: source_path.join(&outcome.path), + destination: dest_path.join(&outcome.path), + error: error.clone(), + }); + } + _ => {} } } - super::action::FileConflictResolution::Overwrite => { - // Overwrite is already handled via options.overwrite - final_destination - } - super::action::FileConflictResolution::Abort => { - // Should have been caught earlier - return Err(JobError::execution("Operation aborted by user")); + total_bytes += run.bytes; + progress_aggregator.complete_source(); + self.completed_indices.push(index); + self.job_metadata.update_status( + &resolved_source, + super::metadata::CopyFileStatus::Completed, + ); + self.persist_job_state_to_db(&ctx).await?; + continue; + } + // A file where a folder goes, or a folder where a file goes, + // is a conflict nothing resolves. + if source_is_dir != there.is_dir() { + let error = if source_is_dir { + "a file is in the way of the folder" + } else { + "a folder is in the way of the file" + }; + failed_copies.push(CopyError { + source: source_path.clone(), + destination: dest_path.clone(), + error: error.to_string(), + }); + ctx.add_non_critical_error(format!("{}: {error}", dest_path.display())); + self.job_metadata + .set_error(&resolved_source, error.to_string()); + self.persist_job_state_to_db(&ctx).await?; + continue; + } + } + + // A file already at the destination: skipped, written beside + // under a numbered name, or replaced with its previous bytes + // kept in the trash so the replacement can be undone. + let mut previous: Option = None; + let final_destination = match (existing.as_ref(), &dest_path, resolution) { + (Some(_), Some(dest_path), FileConflictResolution::Skip) => { + ctx.log(format!("Skipping existing file: {}", dest_path.display())); + self.job_metadata + .update_status(&resolved_source, super::metadata::CopyFileStatus::Skipped); + progress_aggregator.complete_source(); + copied_count += files_in_source; + self.completed_indices.push(index); + self.persist_job_state_to_db(&ctx).await?; + continue; + } + (Some(_), Some(dest_path), FileConflictResolution::AutoModifyName) => { + let unique_dest = self.generate_unique_name(dest_path).await?; + SdPath::Physical { + device_slug: final_destination + .device_slug() + .unwrap_or_default() + .to_string(), + path: unique_dest, } } - } else { - final_destination + (Some(_), Some(dest_path), FileConflictResolution::Overwrite) => { + match trash::trash(dest_path, volume_manager.as_deref(), ctx.id()).await { + Ok(location) => previous = location, + Err(error) => { + let error = format!("could not set the previous file aside: {error}"); + failed_copies.push(CopyError { + source: source_path.clone().unwrap_or_default(), + destination: dest_path.clone(), + error: error.clone(), + }); + ctx.add_non_critical_error(format!("{}: {error}", dest_path.display())); + self.job_metadata.set_error(&resolved_source, error); + self.persist_job_state_to_db(&ctx).await?; + continue; + } + } + final_destination + } + _ => final_destination, }; // 2. Execute the strategy with progress callback @@ -619,6 +716,34 @@ impl JobHandler for FileCopyJob { } } } + + // The journal: what now sits at the destination, and + // where it came from. + if let (Some(source_path), Some(landed)) = ( + resolved_source.as_local_path(), + final_destination.as_local_path(), + ) { + let subject = tokio::fs::symlink_metadata(landed).await.ok(); + let source_gone = tokio::fs::symlink_metadata(source_path).await.is_err(); + let mut effects = Vec::new(); + if previous.is_some() { + effects.push(Effect::replaced( + landed.to_path_buf(), + previous.take(), + subject.as_ref(), + )); + } + if is_move && source_gone { + effects.push(Effect::moved( + source_path.to_path_buf(), + landed.to_path_buf(), + subject.as_ref(), + )); + } else if effects.is_empty() { + effects.push(Effect::created(landed.to_path_buf(), subject.as_ref())); + } + ctx.record(effects).await; + } } Err(e) => { failed_copies.push(CopyError { @@ -635,6 +760,16 @@ impl JobHandler for FileCopyJob { // Mark as failed in metadata self.job_metadata.set_error(&resolved_source, e.to_string()); + + // The previous file comes back where it was. + if let (Some(previous), Some(dest_path)) = (previous.take(), &dest_path) { + if let Err(error) = trash::restore(&previous, dest_path).await { + ctx.add_non_critical_error(format!( + "Could not put {} back after the copy failed: {error}", + dest_path.display() + )); + } + } } } diff --git a/core/src/ops/files/copy/preflight.rs b/core/src/ops/files/copy/preflight.rs index c51d7b13c..f3406572e 100644 --- a/core/src/ops/files/copy/preflight.rs +++ b/core/src/ops/files/copy/preflight.rs @@ -153,6 +153,7 @@ async fn validate(input: &FileCopyInput, ctx: &PreviewContext) -> Result Result Result Result ChangeKind::Create { size }, _ => ChangeKind::Skip { reason: SkipReason::Policy, }, @@ -510,6 +521,61 @@ mod tests { assert_eq!(plan.summary.creates.files, 1); } + /// A file or folder copied into its own directory with keep both is + /// written beside the original under a numbered name, which is what a + /// duplicate is. + #[tokio::test] + async fn a_duplicate_is_a_numbered_create_beside_the_original() { + let fixture = Fixture::new().await; + let file = FileKind::File; + let tree = [ + ("a.txt", file, 4, T, Some("a"), None, None), + ("sub/b.txt", file, 3, T, Some("b"), None, None), + ]; + fixture.materialize(&fixture.source, &tree); + fixture.index(&fixture.source, &tree).await; + + let mut duplicate = input(&[&fixture.source.join("a.txt")], &fixture.source); + duplicate.on_conflict = Some(FileConflictResolution::AutoModifyName); + let plan = preview(&duplicate, &fixture.preview()) + .await + .expect("planned"); + assert_eq!( + change_at(&plan, &fixture.source.join("a (1).txt")), + Some(&ChangeKind::Create { size: 4 }) + ); + assert_eq!(plan.summary.creates.files, 1); + assert_eq!(plan.summary.skips.policy.files, 0); + + let mut folder = input(&[&fixture.source.join("sub")], &fixture.source); + folder.on_conflict = Some(FileConflictResolution::AutoModifyName); + let validation = validate(&folder, &fixture.preview()) + .await + .expect("validated"); + let collision = validation + .findings + .iter() + .find(|finding| finding.code == FOLDER_COLLISION) + .expect("the collision is named"); + assert!( + collision.message.contains("numbered name"), + "{}", + collision.message + ); + let plan = preview(&folder, &fixture.preview()).await.expect("planned"); + let beside = fixture.source.join("sub (1)"); + assert_eq!( + change_at(&plan, &beside), + Some(&ChangeKind::CreateDirectory) + ); + assert_eq!( + change_at(&plan, &beside.join("b.txt")), + Some(&ChangeKind::Create { size: 3 }) + ); + assert_eq!(plan.summary.creates.files, 1); + assert_eq!(plan.summary.directories_created, 1); + } + /// A single file moved to a new name is one move in the plan, and a /// source that is not there is an error at its path. #[tokio::test] diff --git a/core/src/ops/files/create_folder/action.rs b/core/src/ops/files/create_folder/action.rs index e91662837..65a8858d9 100644 --- a/core/src/ops/files/create_folder/action.rs +++ b/core/src/ops/files/create_folder/action.rs @@ -8,7 +8,7 @@ use crate::{ infra::action::{error::ActionError, LibraryAction}, ops::files::{ copy::job::{FileCopyJob, MoveMode}, - rename::validation::validate_filename, + rename::naming::check_portable, }, volume::{LocalBackend, VolumeBackend}, }; @@ -52,7 +52,7 @@ impl LibraryAction for CreateFolderAction { type Output = CreateFolderOutput; fn from_input(input: Self::Input) -> Result { - validate_filename(&input.name).map_err(|e| e.to_string())?; + check_portable(&input.name).map_err(|e| e.to_string())?; match &input.parent { SdPath::Physical { .. } | SdPath::Cloud { .. } => {} SdPath::Content { .. } => { diff --git a/core/src/ops/files/delete/compared.rs b/core/src/ops/files/delete/compared.rs index 0781f318a..2b75f7a9f 100644 --- a/core/src/ops/files/delete/compared.rs +++ b/core/src/ops/files/delete/compared.rs @@ -117,7 +117,8 @@ pub(super) async fn delete( .execute(ctx, &paths, mode.clone()) .await .map_err(|e| JobError::execution(format!("Strategy execution failed: {e}")))?; - tally.record(results); + let effects = tally.record(results); + ctx.record(effects).await; } record_learned(ctx, &context, learned).await; diff --git a/core/src/ops/files/delete/duplicates.rs b/core/src/ops/files/delete/duplicates.rs index 9f902ebde..8f3562993 100644 --- a/core/src/ops/files/delete/duplicates.rs +++ b/core/src/ops/files/delete/duplicates.rs @@ -315,7 +315,8 @@ async fn settle_and_remove( .execute(ctx, &paths, mode.clone()) .await .map_err(|e| JobError::execution(format!("Strategy execution failed: {e}")))?; - tally.record(results); + let effects = tally.record(results); + ctx.record(effects).await; } record_learned(ctx, context, learned).await; Ok(()) diff --git a/core/src/ops/files/delete/job.rs b/core/src/ops/files/delete/job.rs index 4469f4b72..9ce1562aa 100644 --- a/core/src/ops/files/delete/job.rs +++ b/core/src/ops/files/delete/job.rs @@ -2,7 +2,7 @@ use crate::{ domain::addressing::SdPath, - infra::job::{generic_progress::GenericProgress, prelude::*}, + infra::job::{generic_progress::GenericProgress, journal::Effect, prelude::*}, }; use serde::{Deserialize, Serialize}; use std::{ @@ -164,7 +164,8 @@ async fn delete_paths( .map_err(|e| JobError::execution(format!("Strategy execution failed: {}", e)))?; let mut tally = Tally::default(); - tally.record(results); + let effects = tally.record(results); + ctx.record(effects).await; // Phase: Complete ctx.progress(Progress::Generic( @@ -213,11 +214,15 @@ pub(super) struct Tally { } impl Tally { - pub(super) fn record(&mut self, results: Vec) { + /// Count the results, and answer with what the successful ones did, for + /// the journal. + pub(super) fn record(&mut self, results: Vec) -> Vec { + let mut effects = Vec::new(); for result in results { if result.success { self.deleted += 1; self.bytes += result.bytes_freed; + effects.extend(result.effect); } else { self.failed.push(DeleteError { path: result @@ -229,6 +234,7 @@ impl Tally { }); } } + effects } pub(super) fn skip(&mut self, path: PathBuf, reason: SkipReason) { diff --git a/core/src/ops/files/delete/mod.rs b/core/src/ops/files/delete/mod.rs index 247dfd0b7..ec65e84a1 100644 --- a/core/src/ops/files/delete/mod.rs +++ b/core/src/ops/files/delete/mod.rs @@ -7,6 +7,7 @@ pub mod input; pub mod job; pub mod output; mod preflight; +pub(crate) use preflight::last_copies; pub mod routing; pub mod strategy; diff --git a/core/src/ops/files/delete/preflight.rs b/core/src/ops/files/delete/preflight.rs index 8327465f2..52ae8c161 100644 --- a/core/src/ops/files/delete/preflight.rs +++ b/core/src/ops/files/delete/preflight.rs @@ -509,7 +509,7 @@ async fn indexed(ctx: &PreviewContext, paths: &[SdPath]) -> Vec { /// Which of `hashes` one record in the whole library holds: the contents a /// deletion would remove the last copy of. -async fn last_copies(ctx: &PreviewContext, hashes: Vec) -> HashSet { +pub(crate) async fn last_copies(ctx: &PreviewContext, hashes: Vec) -> HashSet { if hashes.is_empty() { return HashSet::new(); } @@ -878,7 +878,9 @@ mod tests { plan.changes[0].change, ChangeKind::Delete { last_copy: false } ); - let PlanBasis::Index { revisions } = &plan.basis; + let PlanBasis::Index { revisions } = &plan.basis else { + panic!("not read from the index: {:?}", plan.basis); + }; assert_eq!(revisions.len(), 2, "both stores are named"); let untracked = delete(DeleteTargets::Comparison { diff --git a/core/src/ops/files/delete/strategy.rs b/core/src/ops/files/delete/strategy.rs index f8050fd08..34f5048b0 100644 --- a/core/src/ops/files/delete/strategy.rs +++ b/core/src/ops/files/delete/strategy.rs @@ -4,12 +4,16 @@ //! 1. **LocalDeleteStrategy** - Local file deletion (trash, permanent, secure) //! 2. **RemoteDeleteStrategy** - Cross-device deletion via network -use crate::{domain::addressing::SdPath, infra::job::prelude::*}; +use crate::{ + domain::addressing::SdPath, + infra::job::{journal::Effect, prelude::*}, + ops::files::trash, +}; use anyhow::Result; use async_trait::async_trait; use serde::{Deserialize, Serialize}; use std::collections::HashMap; -use std::path::Path; +use std::path::{Path, PathBuf}; use tokio::fs; use uuid::Uuid; @@ -22,6 +26,9 @@ pub struct DeleteResult { pub success: bool, pub bytes_freed: u64, pub error: Option, + /// What the deletion did, for the journal: trashed with where it went, + /// or removed for good. Absent where it happened on another device. + pub effect: Option, } /// Strategy for executing delete operations @@ -60,16 +67,49 @@ impl DeleteStrategy for LocalDeleteStrategy { let size = self.get_path_size(local_path).await.unwrap_or(0); let deletion_result = match mode { - DeleteMode::Trash => self.move_to_trash(local_path).await, - DeleteMode::Permanent => self.permanent_delete(local_path).await, - DeleteMode::Secure => self.secure_delete(local_path).await, + DeleteMode::Trash => { + self.move_to_trash(local_path, ctx).await.map(|location| { + Effect::Trashed { + from: local_path.to_path_buf(), + to: location, + subject: None, + } + }) + } + DeleteMode::Permanent => self + .permanent_delete(local_path) + .await + .map(|()| Effect::removed(local_path.to_path_buf())), + DeleteMode::Secure => self + .secure_delete(local_path) + .await + .map(|()| Effect::removed(local_path.to_path_buf())), + }; + let deletion_result = match deletion_result { + Ok(Effect::Trashed { + from, to: Some(to), .. + }) => { + let subject = fs::symlink_metadata(&to).await.ok(); + Ok(Effect::trashed(from, Some(to), subject.as_ref())) + } + other => other, }; - DeleteResult { - path: path.clone(), - success: deletion_result.is_ok(), - bytes_freed: if deletion_result.is_ok() { size } else { 0 }, - error: deletion_result.err().map(|e| e.to_string()), + match deletion_result { + Ok(effect) => DeleteResult { + path: path.clone(), + success: true, + bytes_freed: size, + error: None, + effect: Some(effect), + }, + Err(error) => DeleteResult { + path: path.clone(), + success: false, + bytes_freed: 0, + error: Some(error.to_string()), + effect: None, + }, } } @@ -82,6 +122,7 @@ impl DeleteStrategy for LocalDeleteStrategy { success: false, bytes_freed: 0, error: Some("Path is remote or unsupported".to_string()), + effect: None, }, }; @@ -110,6 +151,7 @@ impl LocalDeleteStrategy { "Delete mode {:?} not supported for cloud paths (only Permanent)", mode )), + effect: None, }; } @@ -122,6 +164,7 @@ impl LocalDeleteStrategy { success: false, bytes_freed: 0, error: Some("Volume manager not available".to_string()), + effect: None, } } }; @@ -135,6 +178,7 @@ impl LocalDeleteStrategy { success: false, bytes_freed: 0, error: Some("Path is not a cloud path".to_string()), + effect: None, } } }; @@ -152,6 +196,7 @@ impl LocalDeleteStrategy { service.scheme(), identifier )), + effect: None, } } }; @@ -165,6 +210,7 @@ impl LocalDeleteStrategy { success: false, bytes_freed: 0, error: Some("Volume backend not available".to_string()), + effect: None, } } }; @@ -182,12 +228,14 @@ impl LocalDeleteStrategy { success: true, bytes_freed: size, error: None, + effect: Some(Effect::removed(PathBuf::from(cloud_path))), }, Err(e) => DeleteResult { path: path.clone(), success: false, bytes_freed: 0, error: Some(format!("Cloud deletion failed: {}", e)), + effect: None, }, } } @@ -213,35 +261,15 @@ impl LocalDeleteStrategy { Ok(total) } - /// Move file to the system trash/recycle bin. - /// - /// Uses the `trash` crate for native platform support: - /// - Windows: SHFileOperation → Recycle Bin - /// - macOS: NSFileManager → Trash - /// - Linux: XDG trash spec - #[cfg(any(target_os = "windows", target_os = "macos", target_os = "linux"))] - pub async fn move_to_trash(&self, path: &Path) -> Result<(), std::io::Error> { - let path = path.to_path_buf(); - tokio::task::spawn_blocking(move || { - trash::delete(&path).map_err(|e| { - std::io::Error::new( - std::io::ErrorKind::Other, - format!("Failed to move to trash: {}", e), - ) - }) - }) - .await - .map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e.to_string()))??; - - Ok(()) - } - - #[cfg(not(any(target_os = "windows", target_os = "macos", target_os = "linux")))] - pub async fn move_to_trash(&self, _path: &Path) -> Result<(), std::io::Error> { - Err(std::io::Error::new( - std::io::ErrorKind::Unsupported, - "move to trash is not supported on this platform", - )) + /// Move a file to the trash, answering with where it went. A volume + /// with no trash of its own gets a Spacedrive trash directory at its + /// root. + pub async fn move_to_trash( + &self, + path: &Path, + ctx: &JobContext<'_>, + ) -> Result, std::io::Error> { + trash::trash(path, ctx.volume_manager().as_deref(), ctx.id()).await } /// Permanently delete file or directory @@ -414,6 +442,7 @@ impl RemoteDeleteStrategy { success: true, bytes_freed: 0, error: None, + effect: None, }) .collect(); diff --git a/core/src/ops/files/link/action.rs b/core/src/ops/files/link/action.rs new file mode 100644 index 000000000..a265d4dc1 --- /dev/null +++ b/core/src/ops/files/link/action.rs @@ -0,0 +1,62 @@ +use std::sync::Arc; + +use serde::{Deserialize, Serialize}; +use specta::Type; + +use super::job::LinkJob; +use crate::{ + context::CoreContext, + domain::SdPath, + infra::{ + action::{error::ActionError, LibraryAction}, + job::handle::JobReceipt, + }, +}; + +/// Make a link at `at` to `target`. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Type)] +pub struct FileLinkInput { + pub at: SdPath, + pub target: SdPath, + pub kind: LinkKind, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Type)] +#[serde(rename_all = "snake_case")] +#[cfg_attr(feature = "cli", derive(clap::ValueEnum))] +pub enum LinkKind { + Symlink, + Hardlink, +} + +pub struct FileLinkAction { + input: FileLinkInput, +} + +impl LibraryAction for FileLinkAction { + type Input = FileLinkInput; + type Output = JobReceipt; + + fn from_input(input: Self::Input) -> Result { + Ok(Self { input }) + } + + async fn execute( + self, + library: Arc, + _context: Arc, + ) -> Result { + let job_handle = library + .jobs() + .dispatch(LinkJob::new(self.input)) + .await + .map_err(ActionError::Job)?; + Ok(job_handle.into()) + } + + fn action_kind(&self) -> &'static str { + "files.link" + } +} + +crate::register_library_action!(FileLinkAction, "files.link"); diff --git a/core/src/ops/files/link/job.rs b/core/src/ops/files/link/job.rs new file mode 100644 index 000000000..8d34ac144 --- /dev/null +++ b/core/src/ops/files/link/job.rs @@ -0,0 +1,104 @@ +use std::{path::PathBuf, time::Instant}; + +use serde::{Deserialize, Serialize}; +use specta::Type; + +use super::action::{FileLinkInput, LinkKind}; +use crate::infra::job::{generic_progress::GenericProgress, journal::Effect, prelude::*}; + +#[derive(Debug, Serialize, Deserialize, Job)] +pub struct LinkJob { + pub input: FileLinkInput, + #[serde(skip, default = "Instant::now")] + started_at: Instant, +} + +impl LinkJob { + pub fn new(input: FileLinkInput) -> Self { + Self { + input, + started_at: Instant::now(), + } + } +} + +impl Job for LinkJob { + const NAME: &'static str = "link"; + const RESUMABLE: bool = false; + const DESCRIPTION: Option<&'static str> = Some("Make a link"); +} + +impl crate::infra::job::traits::DynJob for LinkJob { + fn job_name(&self) -> &'static str { + Self::NAME + } +} + +#[derive(Debug, Clone, Serialize, Deserialize, Type)] +pub struct LinkOutput { + pub at: PathBuf, + pub kind: LinkKind, +} + +impl From for JobOutput { + fn from(output: LinkOutput) -> Self { + JobOutput::custom(output) + } +} + +#[async_trait::async_trait] +impl JobHandler for LinkJob { + type Output = LinkOutput; + + async fn run(&mut self, ctx: JobContext<'_>) -> JobResult { + let (Some(at), Some(target)) = ( + self.input.at.as_local_path().map(PathBuf::from), + self.input.target.as_local_path().map(PathBuf::from), + ) else { + return Err(JobError::execution( + "the link and its target must be on this device", + )); + }; + if tokio::fs::symlink_metadata(&at).await.is_ok() { + return Err(JobError::execution(format!( + "{} is already there", + at.display() + ))); + } + let made = match self.input.kind { + LinkKind::Symlink => symlink(&target, &at).await, + LinkKind::Hardlink => tokio::fs::hard_link(&target, &at).await, + }; + made.map_err(|error| JobError::execution(format!("could not make the link: {error}")))?; + let subject = tokio::fs::symlink_metadata(&at).await.ok(); + ctx.record(vec![Effect::created(at.clone(), subject.as_ref())]) + .await; + ctx.progress(Progress::generic( + GenericProgress::new(1.0, "Complete", "linked") + .with_completion(1, 1) + .with_performance(0.0, None, Some(self.started_at.elapsed())), + )); + Ok(LinkOutput { + at, + kind: self.input.kind, + }) + } +} + +async fn symlink(target: &std::path::Path, at: &std::path::Path) -> std::io::Result<()> { + #[cfg(unix)] + { + tokio::fs::symlink(target, at).await + } + #[cfg(windows)] + { + if tokio::fs::metadata(target) + .await + .is_ok_and(|meta| meta.is_dir()) + { + tokio::fs::symlink_dir(target, at).await + } else { + tokio::fs::symlink_file(target, at).await + } + } +} diff --git a/core/src/ops/files/link/mod.rs b/core/src/ops/files/link/mod.rs new file mode 100644 index 000000000..17e1d7597 --- /dev/null +++ b/core/src/ops/files/link/mod.rs @@ -0,0 +1,15 @@ +//! # Links +//! +//! Make a symlink or a hard link at a path, on preflight: validation +//! refuses a place already taken, a target that is not there, a hard link +//! across volumes or to a directory, and the plan is one create. The job +//! journals the link, so an undo removes it. + +mod action; +mod job; +mod preflight; +#[cfg(test)] +mod tests; + +pub use action::{FileLinkAction, FileLinkInput, LinkKind}; +pub use job::{LinkJob, LinkOutput}; diff --git a/core/src/ops/files/link/preflight.rs b/core/src/ops/files/link/preflight.rs new file mode 100644 index 000000000..b14ab8bd1 --- /dev/null +++ b/core/src/ops/files/link/preflight.rs @@ -0,0 +1,133 @@ +use super::action::{FileLinkAction, FileLinkInput, LinkKind}; +use crate::{ + infra::action::{ + error::ActionError, + preflight::{ + ExecutionFacts, Finding, PreviewContext, PreviewableAction, ValidatedAction, Validation, + }, + }, + ops::files::plan::{ChangeKind, FsPlan, FsPlanSummary, PlanBasis, PlanChanges, PlannedChange}, +}; + +pub const REMOTE_ROOT: &str = "link.remote_root"; +pub const EXISTS: &str = "link.exists"; +pub const TARGET_MISSING: &str = "link.target_missing"; +pub const PARENT_MISSING: &str = "link.parent_missing"; +pub const CROSS_VOLUME: &str = "link.cross_volume"; +pub const DIRECTORY: &str = "link.directory"; + +/// Whether a hard link may be made, from what is known of the two sides. +pub fn hard_link_allowed(same_volume: bool, target_is_dir: bool) -> Result<(), &'static str> { + if target_is_dir { + return Err(DIRECTORY); + } + if !same_volume { + return Err(CROSS_VOLUME); + } + Ok(()) +} + +impl ValidatedAction for FileLinkAction { + async fn validate( + input: &FileLinkInput, + ctx: &PreviewContext, + ) -> Result { + let mut findings = Vec::new(); + let facts = ExecutionFacts { + executes_on: ctx.executes_on(), + strategy: Some( + match input.kind { + LinkKind::Symlink => "symlink", + LinkKind::Hardlink => "hardlink", + } + .to_string(), + ), + estimated_files: Some(1), + estimated_bytes: Some(0), + free_space_after: None, + }; + let (Some(at), Some(target)) = (input.at.as_local_path(), input.target.as_local_path()) + else { + findings.push(Finding::error( + REMOTE_ROOT, + "the link and its target must be on this device", + )); + return Ok(Validation { findings, facts }); + }; + if tokio::fs::symlink_metadata(at).await.is_ok() { + findings.push( + Finding::error(EXISTS, "something is already at the link's place") + .at(input.at.clone()), + ); + } + if !at.parent().is_some_and(|parent| parent.is_dir()) { + findings.push( + Finding::error( + PARENT_MISSING, + "the folder the link would sit in does not exist", + ) + .at(input.at.clone()), + ); + } + match tokio::fs::metadata(target).await { + Ok(meta) => { + if input.kind == LinkKind::Hardlink { + let theirs = ctx.volumes().volume_for_path(target).await; + let ours = match at.parent() { + Some(parent) => ctx.volumes().volume_for_path(parent).await, + None => None, + }; + let same_volume = match (theirs, ours) { + (Some(theirs), Some(ours)) => theirs.fingerprint == ours.fingerprint, + _ => true, + }; + if let Err(code) = hard_link_allowed(same_volume, meta.is_dir()) { + findings.push( + Finding::error( + code, + match code { + DIRECTORY => "a hard link cannot point at a directory", + _ => "a hard link cannot cross volumes; make a symlink", + }, + ) + .at(input.target.clone()), + ); + } + } + } + Err(_) => findings.push( + Finding::error(TARGET_MISSING, "the target is not there").at(input.target.clone()), + ), + } + Ok(Validation { findings, facts }) + } +} + +impl PreviewableAction for FileLinkAction { + type Plan = FsPlan; + + async fn preview(input: FileLinkInput, ctx: &PreviewContext) -> Result { + let mut summary = FsPlanSummary::default(); + let mut changes = PlanChanges::default(); + let change = ChangeKind::Create { size: 0 }; + summary.count(&change, 0); + changes.push(PlannedChange { + path: input.at.clone(), + change, + }); + let (changes, truncated) = changes.finish(); + Ok(ctx.plans().retain(FsPlan { + handle: None, + basis: PlanBasis::Index { + revisions: Vec::new(), + }, + roots: Vec::new(), + summary, + changes, + truncated, + })) + } +} + +crate::register_validate!(FileLinkAction, "files.link"); +crate::register_preview!(FileLinkAction, "files.link"); diff --git a/core/src/ops/files/link/tests.rs b/core/src/ops/files/link/tests.rs new file mode 100644 index 000000000..2704c4af5 --- /dev/null +++ b/core/src/ops/files/link/tests.rs @@ -0,0 +1,79 @@ +use std::time::Duration; + +use sd_store::file::FileKind; + +use super::{ + preflight::{hard_link_allowed, CROSS_VOLUME, DIRECTORY, EXISTS}, + FileLinkAction, FileLinkInput, LinkJob, LinkKind, +}; +use crate::{ + domain::SdPath, + infra::action::preflight::{PreviewableAction, ValidatedAction}, + ops::files::{ + fixture::{Fixture, T}, + undo::UndoJob, + }, +}; + +#[test] +fn a_hard_link_needs_one_volume_and_a_file() { + assert_eq!(hard_link_allowed(true, false), Ok(())); + assert_eq!(hard_link_allowed(false, false), Err(CROSS_VOLUME)); + assert_eq!(hard_link_allowed(true, true), Err(DIRECTORY)); +} + +#[cfg(unix)] +#[tokio::test] +async fn a_link_previews_one_create_and_undo_removes_it() { + let fixture = Fixture::new().await; + let tree = [("a.txt", FileKind::File, 4, T, Some("a"), None, None)]; + fixture.materialize(&fixture.source, &tree); + fixture.index(&fixture.source, &tree).await; + let at = fixture.source.join("a-link.txt"); + let input = FileLinkInput { + at: SdPath::local(&at), + target: SdPath::local(fixture.source.join("a.txt")), + kind: LinkKind::Symlink, + }; + let validation = FileLinkAction::validate(&input, &fixture.preview()) + .await + .expect("validated"); + assert!(!validation.refuses(), "{:?}", validation.findings); + let plan = FileLinkAction::preview(input.clone(), &fixture.preview()) + .await + .expect("planned"); + assert_eq!(plan.summary.creates.files, 1); + + let handle = fixture + .library + .jobs() + .dispatch(LinkJob::new(input.clone())) + .await + .expect("dispatched"); + let job = handle.id(); + tokio::time::timeout(Duration::from_secs(30), handle.wait()) + .await + .expect("in time") + .expect("completed"); + assert!(std::fs::symlink_metadata(&at) + .expect("linked") + .file_type() + .is_symlink()); + + let validation = FileLinkAction::validate(&input, &fixture.preview()) + .await + .expect("validated"); + assert!(validation.errors().any(|finding| finding.code == EXISTS)); + + let handle = fixture + .library + .jobs() + .dispatch(UndoJob::new(job.0, None)) + .await + .expect("dispatched"); + tokio::time::timeout(Duration::from_secs(30), handle.wait()) + .await + .expect("in time") + .expect("completed"); + assert!(std::fs::symlink_metadata(&at).is_err(), "the link is gone"); +} diff --git a/core/src/ops/files/merge/action.rs b/core/src/ops/files/merge/action.rs index 9e48da1e3..951b12c3e 100644 --- a/core/src/ops/files/merge/action.rs +++ b/core/src/ops/files/merge/action.rs @@ -59,11 +59,17 @@ impl PreviewableAction for FileMergeAction { async fn preview(input: FileMergeInput, ctx: &PreviewContext) -> Result { let mut planner = Planner::new(ctx, input.on_conflict, false); + if input.remove_extras { + planner = planner.removing_extras(); + } for source in &input.sources.paths { planner .pair(source, &input.destination, input.consume_sources) .await?; } + if let Some(destination) = input.destination.as_local_path() { + planner.remove_extras(destination).await; + } Ok(ctx.plans().retain(planner.finish())) } } diff --git a/core/src/ops/files/merge/input.rs b/core/src/ops/files/merge/input.rs index 146ab5002..09f6d88f5 100644 --- a/core/src/ops/files/merge/input.rs +++ b/core/src/ops/files/merge/input.rs @@ -19,6 +19,10 @@ pub struct FileMergeInput { /// confirmed identical to the destination's, and prune emptied source /// directories. What the merge did not settle stays where it was. pub consume_sources: bool, + /// Remove from the destination what no source holds, so it ends up + /// matching the sources: a mirror. The extras go to the trash. + #[serde(default)] + pub remove_extras: bool, } /// What to do with a file at the same path on both sides whose bytes differ. diff --git a/core/src/ops/files/merge/job.rs b/core/src/ops/files/merge/job.rs index 58c80835f..89d33202b 100644 --- a/core/src/ops/files/merge/job.rs +++ b/core/src/ops/files/merge/job.rs @@ -16,7 +16,10 @@ //! than repeating a half-merged tree. With `consume_sources` a settled leaf //! is removed from its source and emptied directories are pruned from the //! bottom up, so the source ends holding exactly what the merge left -//! unsettled. The job writes no records; the watcher observes the results. +//! unsettled; on one volume a consumed leaf is renamed into place rather +//! than copied and removed. A replaced file's previous bytes go to the +//! trash, and every effect is journaled. The job writes no records; the +//! watcher observes the results. use std::{ collections::HashMap, @@ -33,13 +36,14 @@ use crate::{ infra::{ action::preflight::PreviewContext, api::SessionContext, - job::{generic_progress::GenericProgress, prelude::*}, + job::{generic_progress::GenericProgress, journal::Effect, prelude::*}, }, ops::{ files::{ copy::{input::CopyMethod, routing::CopyStrategyRouter}, plan::{ChangeKind, ConflictKind, SkipReason}, planner::{is_junk, Planner}, + trash, }, paths::reach::stores_beneath, }, @@ -107,6 +111,8 @@ pub struct MergeOutput { /// Leaves removed from their source. pub consumed: u64, pub pruned_directories: u64, + /// Files the destination held that no source did, trashed by a mirror. + pub removed_extras: u64, /// Tag assertions still standing on records under the sources, which a /// consuming merge across volumes leaves behind. pub assertions_left: i64, @@ -193,20 +199,6 @@ impl JobHandler for FolderMergeJob { let total = plan.len() as u64; ctx.log(format!("{total} planned changes from the index")); - let mut merge = Merge { - ctx: &ctx, - policy: self.input.on_conflict, - consume: self.input.consume_sources, - plan, - total, - resuming: self.cursor.clone(), - cursor: self.cursor.take(), - outcomes: std::mem::take(&mut self.outcomes), - since_checkpoint: 0, - pruned: 0, - bytes: 0, - }; - let sources: Vec = self .input .sources @@ -214,12 +206,46 @@ impl JobHandler for FolderMergeJob { .iter() .filter_map(|source| source.as_local_path().map(Path::to_path_buf)) .collect(); + // A consuming merge on one volume renames each leaf into place. + let moving = self.input.consume_sources + && match ctx.volume_manager() { + Some(volumes) => { + let mut same = true; + for source in &sources { + same &= volumes.same_volume(source, &destination).await; + } + same + } + None => false, + }; + + let mut merge = Merge::new( + &ctx, + self.input.on_conflict, + self.input.consume_sources, + moving, + plan, + ); + merge.resuming = self.cursor.clone(); + merge.cursor = self.cursor.take(); + merge.outcomes = std::mem::take(&mut self.outcomes); + merge.total = total; + for (index, source) in sources.iter().enumerate() { merge .directory(index, source, source, &destination, "") .await?; } - merge.checkpoint().await?; + let run = merge.finish().await?; + + let removed_extras = if self.input.remove_extras { + ctx.progress(Progress::Indeterminate( + "Removing what no source holds".to_string(), + )); + remove_extras(&ctx, &sources, &destination).await? + } else { + 0 + }; let assertions_left = if self.input.consume_sources { assertions_left(&ctx, &self.input.sources.paths).await @@ -228,12 +254,13 @@ impl JobHandler for FolderMergeJob { }; let mut output = MergeOutput { - pruned_directories: merge.pruned, + pruned_directories: run.pruned, + removed_extras, assertions_left, - bytes: merge.bytes, + bytes: run.bytes, ..Default::default() }; - for outcome in &merge.outcomes { + for outcome in &run.outcomes { match &outcome.result { MergeResult::Copied { .. } => output.copied += 1, MergeResult::CreatedDirectory => output.created_directories += 1, @@ -255,9 +282,9 @@ impl JobHandler for FolderMergeJob { output.diverged += 1; } } - output.consumed = merge.consumed(); - self.cursor = merge.cursor; - self.outcomes = merge.outcomes; + output.consumed = run.consumed; + self.cursor = run.cursor; + self.outcomes = run.outcomes; output.outcomes = self.outcomes.clone(); ctx.progress(Progress::generic( @@ -310,11 +337,26 @@ impl FolderMergeJob { } } -/// One run of the merge over the live filesystem. -struct Merge<'a, 'c> { +/// What a run of the merge amounts to. +pub(crate) struct MergeRun { + pub(crate) outcomes: Vec, + pub(crate) cursor: Option, + pub(crate) pruned: u64, + pub(crate) bytes: u64, + /// Leaves removed from their source: everything settled while + /// consuming. + pub(crate) consumed: u64, +} + +/// One run of the merge over the live filesystem. The copy job runs one too, +/// for a folder copied onto a folder that is already there. +pub(crate) struct Merge<'a, 'c> { ctx: &'a JobContext<'c>, policy: MergeConflictPolicy, consume: bool, + /// Rename each leaf into place rather than copy and remove it; only on + /// one volume. + moving: bool, plan: HashMap, total: u64, /// The cursor a resumed job walks up to; leaves before it are done. @@ -324,13 +366,77 @@ struct Merge<'a, 'c> { since_checkpoint: usize, pruned: u64, bytes: u64, + consumed: u64, + /// Effects not yet written to the journal. + effects: Vec, } -impl Merge<'_, '_> { +/// What settling one leaf came to. +struct Settled { + result: MergeResult, + /// Where a leaf kept beside the existing file was written. + written: Option, + /// Where a replaced file's previous bytes went. + previous: Option, + /// Whether the leaf was renamed into place, so nothing is left to + /// remove from the source. + moved: bool, +} + +impl Settled { + fn of(result: MergeResult) -> Self { + Self { + result, + written: None, + previous: None, + moved: false, + } + } +} + +impl<'a, 'c> Merge<'a, 'c> { + pub(crate) fn new( + ctx: &'a JobContext<'c>, + policy: MergeConflictPolicy, + consume: bool, + moving: bool, + plan: HashMap, + ) -> Self { + Self { + ctx, + policy, + consume, + moving, + total: plan.len() as u64, + plan, + resuming: None, + cursor: None, + outcomes: Vec::new(), + since_checkpoint: 0, + pruned: 0, + bytes: 0, + consumed: 0, + effects: Vec::new(), + } + } + + /// Write what is pending to the journal and checkpoint, then hand back + /// the run. + pub(crate) async fn finish(mut self) -> JobResult { + self.checkpoint().await?; + Ok(MergeRun { + outcomes: self.outcomes, + cursor: self.cursor, + pruned: self.pruned, + bytes: self.bytes, + consumed: self.consumed, + }) + } + /// Settle every entry of one source directory, children of a directory /// before its next sibling, and prune the directory afterward when the /// merge consumes. - fn directory<'s>( + pub(crate) fn directory<'s>( &'s mut self, source: usize, root: &'s Path, @@ -374,6 +480,11 @@ impl Merge<'_, '_> { }; let descend = !matches!(result, Some(MergeResult::Conflict { .. })); if let Some(result) = result { + if result == MergeResult::CreatedDirectory { + let subject = tokio::fs::symlink_metadata(&to).await.ok(); + self.effects + .push(Effect::created(to.clone(), subject.as_ref())); + } self.record(source, relative.clone(), result).await?; } if descend { @@ -381,6 +492,7 @@ impl Merge<'_, '_> { .await?; if self.consume && tokio::fs::remove_dir(&from).await.is_ok() { self.pruned += 1; + self.effects.push(Effect::removed(from.clone())); } } continue; @@ -389,28 +501,39 @@ impl Merge<'_, '_> { if self.done(source, &relative) { continue; } - let (result, written) = if meta.file_type().is_symlink() { + let settled = if meta.file_type().is_symlink() { self.settle_link(&from, &to).await } else { self.settle_file(&from, &to, meta.len()).await }; - let settled = matches!( - result, - MergeResult::Copied { .. } - | MergeResult::Replaced { .. } - | MergeResult::Skipped { - reason: SkipReason::DuplicateConfirmed | SkipReason::Junk - } - ); - if self.consume && settled { - if let Err(error) = tokio::fs::remove_file(&from).await { - self.ctx - .log(format!("Could not remove {}: {error}", from.display())); + let landed = settled.written.clone().unwrap_or_else(|| to.clone()); + let consumed = self.consume + && matches!( + settled.result, + MergeResult::Copied { .. } + | MergeResult::Replaced { .. } + | MergeResult::Skipped { + reason: SkipReason::DuplicateConfirmed | SkipReason::Junk + } + ); + let mut removed = settled.moved; + if consumed && !settled.moved { + match tokio::fs::remove_file(&from).await { + Ok(()) => removed = true, + Err(error) => self + .ctx + .log(format!("Could not remove {}: {error}", from.display())), } } + if consumed && removed { + self.consumed += 1; + } + self.journal_leaf(&from, &landed, &settled, removed).await; + // A leaf kept beside the existing file is recorded where it // was written, which is where the plan put it too. - let written = written + let written = settled + .written .as_deref() .and_then(|path| path.strip_prefix(destination).ok()) .map(|path| { @@ -418,12 +541,49 @@ impl Merge<'_, '_> { .replace(std::path::MAIN_SEPARATOR, "/") }) .unwrap_or(relative); - self.record(source, written, result).await?; + self.record(source, written, settled.result).await?; } Ok(()) }) } + /// The effects one settled leaf had, for the journal. + async fn journal_leaf(&mut self, from: &Path, landed: &Path, settled: &Settled, removed: bool) { + let subject = tokio::fs::symlink_metadata(landed).await.ok(); + match &settled.result { + MergeResult::Copied { .. } => { + if removed { + self.effects.push(Effect::moved( + from.to_path_buf(), + landed.to_path_buf(), + subject.as_ref(), + )); + } else { + self.effects + .push(Effect::created(landed.to_path_buf(), subject.as_ref())); + } + } + MergeResult::Replaced { .. } => { + self.effects.push(Effect::replaced( + landed.to_path_buf(), + settled.previous.clone(), + subject.as_ref(), + )); + if removed { + self.effects.push(Effect::moved( + from.to_path_buf(), + landed.to_path_buf(), + subject.as_ref(), + )); + } + } + MergeResult::Skipped { .. } if removed => { + self.effects.push(Effect::removed(from.to_path_buf())); + } + _ => {} + } + } + /// Whether a resumed job has already settled this leaf. The cursor is /// the last leaf done; reaching it ends the skipping. fn done(&mut self, source: usize, relative: &str) -> bool { @@ -451,132 +611,166 @@ impl Merge<'_, '_> { } } - /// Settle a symlink, as the result and, for one kept beside the existing - /// link, where it was written. - async fn settle_link(&mut self, from: &Path, to: &Path) -> (MergeResult, Option) { + /// Settle a symlink. + async fn settle_link(&mut self, from: &Path, to: &Path) -> Settled { let target = match tokio::fs::read_link(from).await { Ok(target) => target, Err(error) => { - return ( - MergeResult::Failed { - error: error.to_string(), - }, - None, - ) + return Settled::of(MergeResult::Failed { + error: error.to_string(), + }) } }; - let result = match tokio::fs::symlink_metadata(to).await { - Err(_) => link(&target, to).await, + match tokio::fs::symlink_metadata(to).await { + Err(_) => self.place_link(from, &target, to).await, Ok(meta) if meta.file_type().is_symlink() => { if tokio::fs::read_link(to).await.ok().as_deref() == Some(target.as_path()) { - return ( - MergeResult::Skipped { - reason: SkipReason::DuplicateConfirmed, - }, - None, - ); + return Settled::of(MergeResult::Skipped { + reason: SkipReason::DuplicateConfirmed, + }); } match self.resolve(from, to, None).await { Resolution::Replace => { - if let Err(error) = tokio::fs::remove_file(to).await { - return ( - MergeResult::Failed { + let previous = match self.stash(to).await { + Ok(previous) => previous, + Err(error) => { + return Settled::of(MergeResult::Failed { error: error.to_string(), - }, - None, - ); - } - match link(&target, to).await { - MergeResult::Copied { bytes } => MergeResult::Replaced { bytes }, - other => other, + }) + } + }; + let mut settled = self.place_link(from, &target, to).await; + if let MergeResult::Copied { bytes } = settled.result { + settled.result = MergeResult::Replaced { bytes }; } + settled.previous = previous; + settled } Resolution::KeepBoth(renamed) => { - return (link(&target, &renamed).await, Some(renamed)); + let mut settled = self.place_link(from, &target, &renamed).await; + settled.written = Some(renamed); + settled } - Resolution::Skip => MergeResult::Skipped { + Resolution::Skip => Settled::of(MergeResult::Skipped { reason: SkipReason::Policy, - }, + }), } } - Ok(meta) if meta.is_dir() => MergeResult::Conflict { + Ok(meta) if meta.is_dir() => Settled::of(MergeResult::Conflict { kind: ConflictKind::FileVsDirectory, - }, - Ok(_) => MergeResult::Conflict { + }), + Ok(_) => Settled::of(MergeResult::Conflict { kind: ConflictKind::LinkVsFile, - }, - }; - (result, None) + }), + } } - /// Settle a file, as the result and, for one kept beside the existing - /// file, where it was written. - async fn settle_file( - &mut self, - from: &Path, - to: &Path, - size: u64, - ) -> (MergeResult, Option) { + /// A link at `at`: renamed from the source when moving, else made anew. + async fn place_link(&self, from: &Path, target: &Path, at: &Path) -> Settled { + if self.moving { + return match tokio::fs::rename(from, at).await { + Ok(()) => Settled { + result: MergeResult::Copied { bytes: 0 }, + written: None, + previous: None, + moved: true, + }, + Err(error) => Settled::of(MergeResult::Failed { + error: error.to_string(), + }), + }; + } + Settled::of(link(target, at).await) + } + + /// Settle a file. + async fn settle_file(&mut self, from: &Path, to: &Path, size: u64) -> Settled { let name = from .file_name() .map(|name| name.to_string_lossy().into_owned()) .unwrap_or_default(); if is_junk(&name) { - return ( - MergeResult::Skipped { - reason: SkipReason::Junk, - }, - None, - ); + return Settled::of(MergeResult::Skipped { + reason: SkipReason::Junk, + }); } - let result = match tokio::fs::symlink_metadata(to).await { - Err(_) => self.copy(from, to).await, - Ok(meta) if meta.is_dir() => MergeResult::Conflict { + match tokio::fs::symlink_metadata(to).await { + Err(_) => self.place(from, to).await, + Ok(meta) if meta.is_dir() => Settled::of(MergeResult::Conflict { kind: ConflictKind::FileVsDirectory, - }, - Ok(meta) if meta.file_type().is_symlink() => MergeResult::Conflict { + }), + Ok(meta) if meta.file_type().is_symlink() => Settled::of(MergeResult::Conflict { kind: ConflictKind::LinkVsFile, - }, + }), Ok(meta) => { if meta.len() == size { match identical(from, to).await { Ok(true) => { - return ( - MergeResult::Skipped { - reason: SkipReason::DuplicateConfirmed, - }, - None, - ) + return Settled::of(MergeResult::Skipped { + reason: SkipReason::DuplicateConfirmed, + }) } Ok(false) => {} - Err(error) => return (MergeResult::Failed { error }, None), + Err(error) => return Settled::of(MergeResult::Failed { error }), } } match self.resolve(from, to, Some(&meta)).await { Resolution::Replace => { - if let Err(error) = tokio::fs::remove_file(to).await { - return ( - MergeResult::Failed { + let previous = match self.stash(to).await { + Ok(previous) => previous, + Err(error) => { + return Settled::of(MergeResult::Failed { error: error.to_string(), - }, - None, - ); - } - match self.copy(from, to).await { - MergeResult::Copied { bytes } => MergeResult::Replaced { bytes }, - other => other, + }) + } + }; + let mut settled = self.place(from, to).await; + if let MergeResult::Copied { bytes } = settled.result { + settled.result = MergeResult::Replaced { bytes }; } + settled.previous = previous; + settled } Resolution::KeepBoth(renamed) => { - return (self.copy(from, &renamed).await, Some(renamed)); + let mut settled = self.place(from, &renamed).await; + settled.written = Some(renamed); + settled } - Resolution::Skip => MergeResult::Skipped { + Resolution::Skip => Settled::of(MergeResult::Skipped { reason: SkipReason::Policy, - }, + }), } } - }; - (result, None) + } + } + + /// The previous bytes at `to`, moved to the trash with their location + /// recorded, so the replacement can be undone. + async fn stash(&self, to: &Path) -> std::io::Result> { + trash::trash(to, self.ctx.volume_manager().as_deref(), self.ctx.id()).await + } + + /// A file at `to`: renamed from the source when moving, else copied + /// with the strategy the router picks. + async fn place(&self, from: &Path, to: &Path) -> Settled { + if self.moving { + let size = tokio::fs::symlink_metadata(from) + .await + .map(|meta| meta.len()) + .unwrap_or(0); + return match tokio::fs::rename(from, to).await { + Ok(()) => Settled { + result: MergeResult::Copied { bytes: size }, + written: None, + previous: None, + moved: true, + }, + Err(error) => Settled::of(MergeResult::Failed { + error: error.to_string(), + }), + }; + } + Settled::of(self.copy(from, to).await) } /// What the policy makes of a collision at execution. @@ -677,8 +871,11 @@ impl Merge<'_, '_> { Ok(()) } + /// The journal first, so a checkpoint never claims a leaf whose effects + /// are not written. async fn checkpoint(&mut self) -> JobResult<()> { self.since_checkpoint = 0; + self.ctx.record(std::mem::take(&mut self.effects)).await; self.ctx .checkpoint_with_state(&Resume { cursor: self.cursor.clone(), @@ -686,26 +883,6 @@ impl Merge<'_, '_> { }) .await } - - /// Leaves removed from their source: everything settled while consuming. - fn consumed(&self) -> u64 { - if !self.consume { - return 0; - } - self.outcomes - .iter() - .filter(|outcome| { - matches!( - outcome.result, - MergeResult::Copied { .. } - | MergeResult::Replaced { .. } - | MergeResult::Skipped { - reason: SkipReason::DuplicateConfirmed | SkipReason::Junk - } - ) - }) - .count() as u64 - } } enum Resolution { @@ -727,7 +904,7 @@ async fn identical(a: &Path, b: &Path) -> Result { /// A numbered name beside `path` that nothing holds, in the convention copy /// uses. -async fn unique_name(path: &Path) -> PathBuf { +pub(crate) async fn unique_name(path: &Path) -> PathBuf { let parent = path.parent().unwrap_or(Path::new("")); let name = path .file_name() @@ -777,6 +954,83 @@ fn agrees(planned: &ChangeKind, result: &MergeResult) -> bool { } } +/// Trash every file beneath the destination that no source holds at the +/// same relative path, then prune the directories no source has, bottom +/// up. Junk is left alone, as it is never copied. Answers with how many +/// files went. +async fn remove_extras( + ctx: &JobContext<'_>, + sources: &[PathBuf], + destination: &Path, +) -> JobResult { + let mut removed = 0; + let mut directories: Vec = Vec::new(); + let mut stack = vec![destination.to_path_buf()]; + while let Some(directory) = stack.pop() { + let mut entries = match tokio::fs::read_dir(&directory).await { + Ok(entries) => entries, + Err(error) => { + ctx.log(format!("Could not read {}: {error}", directory.display())); + continue; + } + }; + while let Ok(Some(entry)) = entries.next_entry().await { + ctx.check_interrupt().await?; + let path = entry.path(); + let Ok(relative) = path.strip_prefix(destination) else { + continue; + }; + let name = entry.file_name().to_string_lossy().into_owned(); + if relative.components().count() == 1 && name == ".spacedrive" { + continue; + } + let Ok(meta) = tokio::fs::symlink_metadata(&path).await else { + continue; + }; + let mut held = false; + for source in sources { + if tokio::fs::symlink_metadata(source.join(relative)) + .await + .is_ok() + { + held = true; + break; + } + } + if meta.is_dir() { + stack.push(path.clone()); + if !held { + directories.push(path); + } + continue; + } + if held || is_junk(&name) { + continue; + } + match trash::trash(&path, ctx.volume_manager().as_deref(), ctx.id()).await { + Ok(location) => { + let subject = match &location { + Some(location) => tokio::fs::symlink_metadata(location).await.ok(), + None => None, + }; + ctx.record(vec![Effect::trashed(path, location, subject.as_ref())]) + .await; + removed += 1; + } + Err(error) => ctx.log(format!("Could not remove {}: {error}", path.display())), + } + } + } + // Deepest first, so a directory is empty by the time its turn comes. + directories.sort_by(|a, b| b.components().count().cmp(&a.components().count())); + for directory in directories { + if tokio::fs::remove_dir(&directory).await.is_ok() { + ctx.record(vec![Effect::removed(directory)]).await; + } + } + Ok(removed) +} + /// Tag assertions still standing on records under the sources. async fn assertions_left(ctx: &JobContext<'_>, sources: &[SdPath]) -> i64 { let context = ctx.library().core_context(); diff --git a/core/src/ops/files/merge/tests.rs b/core/src/ops/files/merge/tests.rs index dddf92eb0..053ff4dd2 100644 --- a/core/src/ops/files/merge/tests.rs +++ b/core/src/ops/files/merge/tests.rs @@ -12,7 +12,7 @@ use super::{ use crate::{ domain::{SdPath, SdPathBatch}, infra::{ - action::preflight::{PreviewContext, Severity}, + action::preflight::{PreviewContext, PreviewableAction, Severity}, job::output::JobOutput, }, ops::files::{ @@ -43,6 +43,7 @@ impl Fixture { destination: SdPath::local(&self.destination), on_conflict: policy, consume_sources: false, + remove_extras: false, } } @@ -186,7 +187,9 @@ async fn a_plan_sorts_every_kind_of_leaf() { .iter() .any(|change| change.path.path() == Some(&dst.join("keep.txt")))); - let PlanBasis::Index { revisions } = &plan.basis; + let PlanBasis::Index { revisions } = &plan.basis else { + panic!("not read from the index: {:?}", plan.basis); + }; assert_eq!(revisions.len(), 2, "one revision per store read"); assert!(revisions.iter().all(|revision| revision.revision > 0)); } @@ -538,3 +541,105 @@ async fn an_outcome_reports_divergence_from_the_plan() { assert!(outcome.diverged); assert!(output.diverged >= 1); } + +/// A mirror plans a delete for each file only the destination holds, flags +/// the last copies, and the job removes those files and nothing else. +#[tokio::test] +async fn a_mirror_removes_what_no_source_holds() { + let fixture = Fixture::new().await; + let file = FileKind::File; + let source = [ + ("same.txt", file, 4, T, Some("s"), None, None), + ("new.txt", file, 3, T, Some("n"), None, None), + ]; + let destination = [ + ("same.txt", file, 4, T, Some("s"), None, None), + ("extra.txt", file, 5, T, Some("e"), None, None), + ("held.txt", file, 2, T, Some("h"), None, None), + ("old/gone.txt", file, 1, T, Some("g"), None, None), + ]; + // held.txt's bytes exist elsewhere in the library; extra.txt's do not. + let elsewhere = [("copy.txt", file, 2, T, Some("h"), None, None)]; + fixture.materialize(&fixture.source, &source); + fixture.index(&fixture.source, &source).await; + fixture.materialize(&fixture.destination, &destination); + fixture.index(&fixture.destination, &destination).await; + fixture.index(&fixture.other, &elsewhere).await; + + let mut input = fixture.input(MergeConflictPolicy::Overwrite); + input.remove_extras = true; + let validation = validate::validate(&input, &fixture.preview()) + .await + .expect("validated"); + let warning = validation + .findings + .iter() + .find(|finding| finding.code == validate::LAST_COPIES) + .expect("the last copies are counted"); + assert_eq!(warning.severity, Severity::Warning); + assert!( + warning.message.starts_with("2 of the 3 files"), + "{}", + warning.message + ); + + let plan = super::action::FileMergeAction::preview(input.clone(), &fixture.preview()) + .await + .expect("planned"); + let delete_at = |name: &str| { + plan.changes + .iter() + .find(|change| { + change.path.path().map(std::path::PathBuf::as_path) + == Some(fixture.destination.join(name).as_path()) + }) + .map(|change| change.change.clone()) + }; + assert_eq!( + delete_at("extra.txt"), + Some(ChangeKind::Delete { last_copy: true }) + ); + assert_eq!( + delete_at("held.txt"), + Some(ChangeKind::Delete { last_copy: false }) + ); + assert_eq!( + delete_at("old/gone.txt"), + Some(ChangeKind::Delete { last_copy: true }) + ); + assert_eq!( + delete_at("same.txt"), + Some(ChangeKind::Skip { + reason: SkipReason::DuplicateCandidate + }) + ); + assert_eq!(plan.summary.deletes.files, 3); + assert_eq!(plan.summary.creates.files, 1); + + let handle = fixture + .library + .jobs() + .dispatch(FolderMergeJob::new(input)) + .await + .expect("dispatched"); + let output = tokio::time::timeout(Duration::from_secs(30), handle.wait()) + .await + .expect("in time") + .expect("completed"); + let JobOutput::Custom(value) = output else { + panic!("not a merge output: {output:?}"); + }; + assert_eq!(value["removed_extras"], 3); + assert!(fixture.destination.join("same.txt").exists()); + assert!(fixture.destination.join("new.txt").exists()); + assert!(!fixture.destination.join("extra.txt").exists()); + assert!(!fixture.destination.join("held.txt").exists()); + assert!( + !fixture.destination.join("old").exists(), + "the emptied folder is pruned" + ); + assert!( + fixture.source.join("new.txt").exists(), + "the source is untouched" + ); +} diff --git a/core/src/ops/files/merge/validate.rs b/core/src/ops/files/merge/validate.rs index dc9bfeff1..4ef62ba50 100644 --- a/core/src/ops/files/merge/validate.rs +++ b/core/src/ops/files/merge/validate.rs @@ -13,8 +13,9 @@ use crate::{ error::ActionError, preflight::{ExecutionFacts, Finding, PreviewContext, Validation}, }, - ops::files::copy::{ - database::CopyDatabaseQuery, input::CopyMethod, routing::CopyStrategyRouter, + ops::files::{ + copy::{database::CopyDatabaseQuery, input::CopyMethod, routing::CopyStrategyRouter}, + planner::Planner, }, }; @@ -27,6 +28,7 @@ pub const SOURCE_NOT_DIRECTORY: &str = "merge.source_not_directory"; pub const UNTRACKED: &str = "merge.untracked"; pub const SPACE: &str = "merge.space"; pub const ASSERTIONS_STRANDED: &str = "merge.assertions_stranded"; +pub const LAST_COPIES: &str = "merge.last_copies"; pub(super) async fn validate( input: &FileMergeInput, @@ -158,6 +160,40 @@ pub(super) async fn validate( } } + // A mirror removes what no source holds; the last copies among those + // are the warning only an index can give. + if input.remove_extras && !findings.iter().any(|finding| finding.code == UNTRACKED) { + let mut planner = Planner::new(ctx, input.on_conflict, false).removing_extras(); + let mut planned = true; + for (source, _) in &sources { + if planner + .pair(source, &input.destination, input.consume_sources) + .await + .is_err() + { + planned = false; + break; + } + } + if planned { + let last_copies = planner.remove_extras(destination).await; + let extras = planner.summary().deletes.files; + if last_copies > 0 { + findings.push(Finding::warning( + LAST_COPIES, + format!( + "{last_copies} of the {extras} files the mirror removes are the last copy of their content anywhere in your library" + ), + )); + } else if extras > 0 { + findings.push(Finding::info( + LAST_COPIES, + format!("the mirror removes {extras} files no source holds; every one has another copy"), + )); + } + } + } + if let Some((source, _)) = sources.first() { let (_, metadata) = CopyStrategyRouter::select_strategy_with_metadata( source, diff --git a/core/src/ops/files/mod.rs b/core/src/ops/files/mod.rs index cc899f715..2d0a9370f 100644 --- a/core/src/ops/files/mod.rs +++ b/core/src/ops/files/mod.rs @@ -1,16 +1,23 @@ //! File operations - queries and actions for the File domain +pub mod archive; +pub mod attributes_action; pub mod copy; pub mod create_folder; pub mod delete; #[cfg(test)] pub(crate) mod fixture; +pub mod link; pub mod merge; +pub mod organize; pub mod plan; pub(crate) mod planner; pub mod query; pub mod rename; +pub mod trash; +pub mod trash_view; +pub mod undo; pub use create_folder::{CreateFolderAction, CreateFolderInput, CreateFolderOutput}; pub use query::*; -pub use rename::{FileRenameAction, FileRenameInput}; +pub use rename::{FileRenameAction, FileRenameBatchAction, FileRenameBatchInput, FileRenameInput}; diff --git a/core/src/ops/files/organize/action.rs b/core/src/ops/files/organize/action.rs new file mode 100644 index 000000000..4155dbc86 --- /dev/null +++ b/core/src/ops/files/organize/action.rs @@ -0,0 +1,79 @@ +//! Organize and flatten action handlers + +use std::sync::Arc; + +use super::{ + input::{FileFlattenInput, FileOrganizeInput}, + job::{Rearrange, RearrangeJob}, +}; +use crate::{ + context::CoreContext, + infra::{ + action::{error::ActionError, LibraryAction}, + job::handle::JobReceipt, + }, +}; + +pub struct FileOrganizeAction { + input: FileOrganizeInput, +} + +impl LibraryAction for FileOrganizeAction { + type Input = FileOrganizeInput; + type Output = JobReceipt; + + fn from_input(input: Self::Input) -> Result { + Ok(Self { input }) + } + + async fn execute( + self, + library: Arc, + _context: Arc, + ) -> Result { + let job_handle = library + .jobs() + .dispatch(RearrangeJob::new(Rearrange::Organize(self.input))) + .await + .map_err(ActionError::Job)?; + Ok(job_handle.into()) + } + + fn action_kind(&self) -> &'static str { + "files.organize" + } +} + +crate::register_library_action!(FileOrganizeAction, "files.organize"); + +pub struct FileFlattenAction { + input: FileFlattenInput, +} + +impl LibraryAction for FileFlattenAction { + type Input = FileFlattenInput; + type Output = JobReceipt; + + fn from_input(input: Self::Input) -> Result { + Ok(Self { input }) + } + + async fn execute( + self, + library: Arc, + _context: Arc, + ) -> Result { + let job_handle = library + .jobs() + .dispatch(RearrangeJob::new(Rearrange::Flatten(self.input))) + .await + .map_err(ActionError::Job)?; + Ok(job_handle.into()) + } + + fn action_kind(&self) -> &'static str { + "files.flatten" + } +} + +crate::register_library_action!(FileFlattenAction, "files.flatten"); diff --git a/core/src/ops/files/organize/input.rs b/core/src/ops/files/organize/input.rs new file mode 100644 index 000000000..74c5accf7 --- /dev/null +++ b/core/src/ops/files/organize/input.rs @@ -0,0 +1,72 @@ +//! Inputs for organizing a folder into subfolders and flattening one. + +use serde::{Deserialize, Serialize}; +use specta::Type; + +use crate::domain::SdPath; + +/// Move the files under a folder into subfolders named by a rule. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Type)] +pub struct FileOrganizeInput { + pub scope: SdPath, + pub rule: OrganizeRule, + /// Take the files beneath the folder at any depth, not only those + /// directly under it. Every file ends up directly under its new + /// subfolder. + #[serde(default)] + pub recursive: bool, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Type)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum OrganizeRule { + ByDate { + field: OrganizeDateField, + granularity: Granularity, + }, + /// The content kind the indexer assigned: Images, Videos, Documents. + ByKind, + /// The extension, lowercased; files without one go under "No extension". + ByExtension, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Type)] +#[serde(rename_all = "snake_case")] +#[cfg_attr(feature = "cli", derive(clap::ValueEnum))] +pub enum OrganizeDateField { + Modified, + Created, + /// The capture time the store holds for a photo or video, or the + /// modification time where it holds none. + Captured, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Type)] +#[serde(rename_all = "snake_case")] +#[cfg_attr(feature = "cli", derive(clap::ValueEnum))] +pub enum Granularity { + /// `2024` + Year, + /// `2024-05` + YearMonth, + /// `2024-05-06` + YearMonthDay, +} + +/// Move every file beneath a folder to the folder itself. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Type)] +pub struct FileFlattenInput { + pub scope: SdPath, + pub on_conflict: FlattenPolicy, +} + +/// What to do with a file whose name is already taken at the root. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Type)] +#[serde(rename_all = "snake_case")] +#[cfg_attr(feature = "cli", derive(clap::ValueEnum))] +pub enum FlattenPolicy { + /// The deeper file is written under a numbered name. + KeepBoth, + /// The deeper file stays where it is. + Skip, +} diff --git a/core/src/ops/files/organize/job.rs b/core/src/ops/files/organize/job.rs new file mode 100644 index 000000000..ce20eb62a --- /dev/null +++ b/core/src/ops/files/organize/job.rs @@ -0,0 +1,240 @@ +//! Organize and flatten: the moves, from the index, applied as renames. +//! +//! The places are read again from the index when the job starts, so the job +//! moves what the plan showed and nothing the folder gained since goes +//! anywhere unplanned. Every move is one `rename` inside the folder, so +//! records keep their identity; a place taken by the time its turn comes is +//! left with why. Flatten prunes the directories it emptied afterward, +//! deepest first. Every move, folder created and folder pruned is journaled. + +use std::{path::PathBuf, time::Instant}; + +use serde::{Deserialize, Serialize}; +use specta::Type; + +use super::{ + input::{FileFlattenInput, FileOrganizeInput}, + plan::{directories_beneath, flatten, organize, PlanError, Rearrangement}, +}; +use crate::infra::job::{generic_progress::GenericProgress, journal::Effect, prelude::*}; + +const CHECKPOINT_EVERY: usize = 50; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub enum Rearrange { + Organize(FileOrganizeInput), + Flatten(FileFlattenInput), +} + +#[derive(Debug, Serialize, Deserialize, Job)] +pub struct RearrangeJob { + pub what: Rearrange, + /// Origins already moved, for a resumed job. + pub done: Vec, + #[serde(skip, default = "Instant::now")] + started_at: Instant, +} + +impl RearrangeJob { + pub fn new(what: Rearrange) -> Self { + Self { + what, + done: Vec::new(), + started_at: Instant::now(), + } + } +} + +impl Job for RearrangeJob { + const NAME: &'static str = "rearrange_files"; + const RESUMABLE: bool = true; + const DESCRIPTION: Option<&'static str> = Some("Organize or flatten a folder"); +} + +impl crate::infra::job::traits::DynJob for RearrangeJob { + fn job_name(&self) -> &'static str { + Self::NAME + } + + fn dedup_key(&self) -> Option { + Some(match &self.what { + Rearrange::Organize(input) => format!("organize {}", input.scope), + Rearrange::Flatten(input) => format!("flatten {}", input.scope), + }) + } +} + +#[derive(Debug, Clone, Default, Serialize, Deserialize, Type)] +pub struct RearrangeOutput { + pub moved: u64, + pub directories_created: u64, + pub pruned_directories: u64, + /// Files left where they were, with why. + pub left: Vec, + pub failed: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Type)] +pub struct RearrangeProblem { + pub path: PathBuf, + pub reason: String, +} + +impl From for JobOutput { + fn from(output: RearrangeOutput) -> Self { + JobOutput::custom(output) + } +} + +#[derive(Debug, Serialize, Deserialize)] +struct Resume { + done: Vec, +} + +#[async_trait::async_trait] +impl JobHandler for RearrangeJob { + type Output = RearrangeOutput; + + async fn run(&mut self, ctx: JobContext<'_>) -> JobResult { + if let Some(resume) = ctx.load_state::().await? { + self.done = resume.done; + ctx.log(format!("Resuming past {} moves", self.done.len())); + } + let context = ctx.library().core_context(); + ctx.progress(Progress::Indeterminate("Planning".to_string())); + let (rearrangement, scope): (Rearrangement, PathBuf) = match &self.what { + Rearrange::Organize(input) => ( + organize(&context.volume_manager, context.volume_index(), input) + .await + .map_err(failed)?, + input + .scope + .as_local_path() + .map(PathBuf::from) + .unwrap_or_default(), + ), + Rearrange::Flatten(input) => ( + flatten(&context.volume_manager, context.volume_index(), input) + .await + .map_err(failed)?, + input + .scope + .as_local_path() + .map(PathBuf::from) + .unwrap_or_default(), + ), + }; + let mut output = RearrangeOutput::default(); + for (from, wanted, _) in &rearrangement.left { + output.left.push(RearrangeProblem { + path: from.clone(), + reason: format!("{} is taken", wanted.display()), + }); + } + + for directory in &rearrangement.directories { + match tokio::fs::create_dir_all(directory).await { + Ok(()) => { + let subject = tokio::fs::symlink_metadata(directory).await.ok(); + ctx.record(vec![Effect::created(directory.clone(), subject.as_ref())]) + .await; + output.directories_created += 1; + } + Err(error) => { + return Err(JobError::execution(format!( + "could not create {}: {error}", + directory.display() + ))) + } + } + } + + let total = rearrangement.moves.len() as u64; + ctx.log(format!("{total} files to move")); + let mut since_checkpoint = 0; + for planned in rearrangement.moves { + ctx.check_interrupt().await?; + if self.done.contains(&planned.from) { + output.moved += 1; + continue; + } + if tokio::fs::symlink_metadata(&planned.to).await.is_ok() { + output.left.push(RearrangeProblem { + path: planned.from, + reason: format!("{} is taken", planned.to.display()), + }); + continue; + } + match tokio::fs::rename(&planned.from, &planned.to).await { + Ok(()) => { + let subject = tokio::fs::symlink_metadata(&planned.to).await.ok(); + ctx.record(vec![Effect::moved( + planned.from.clone(), + planned.to.clone(), + subject.as_ref(), + )]) + .await; + output.moved += 1; + self.done.push(planned.from); + since_checkpoint += 1; + if since_checkpoint >= CHECKPOINT_EVERY { + since_checkpoint = 0; + ctx.checkpoint_with_state(&Resume { + done: self.done.clone(), + }) + .await?; + } + } + Err(error) => { + ctx.log(format!("{}: {error}", planned.from.display())); + output.failed.push(RearrangeProblem { + path: planned.from, + reason: error.to_string(), + }); + } + } + ctx.progress(Progress::generic( + GenericProgress::new( + (output.moved as f32 / total.max(1) as f32).min(1.0), + "Moving", + format!("{} of {total}", output.moved), + ) + .with_completion(output.moved, total) + .with_errors(output.failed.len() as u64, output.left.len() as u64), + )); + } + + if matches!(self.what, Rearrange::Flatten(_)) { + for directory in directories_beneath(&scope).await { + if tokio::fs::remove_dir(&directory).await.is_ok() { + ctx.record(vec![Effect::removed(directory)]).await; + output.pruned_directories += 1; + } + } + } + + ctx.progress(Progress::generic( + GenericProgress::new( + 1.0, + "Complete", + format!("{} moved, {} left", output.moved, output.left.len()), + ) + .with_completion(total, total) + .with_performance(0.0, None, Some(self.started_at.elapsed())) + .with_errors(output.failed.len() as u64, output.left.len() as u64), + )); + ctx.log(format!( + "Rearrange completed: {} moved, {} folders created, {} pruned, {} left, {} failed", + output.moved, + output.directories_created, + output.pruned_directories, + output.left.len(), + output.failed.len() + )); + Ok(output) + } +} + +fn failed(error: PlanError) -> JobError { + JobError::execution(error.to_string()) +} diff --git a/core/src/ops/files/organize/mod.rs b/core/src/ops/files/organize/mod.rs new file mode 100644 index 000000000..1b275e71d --- /dev/null +++ b/core/src/ops/files/organize/mod.rs @@ -0,0 +1,22 @@ +//! # Organize and flatten +//! +//! Move the files under a folder into subfolders named by a rule, or every +//! file beneath a folder to the folder itself. Both are renames inside one +//! folder, planned from the index and previewed as moves into new folders, +//! so records keep their identity and the listing shows the folder as it +//! would look before anything moves. + +pub mod action; +pub mod input; +pub mod job; +pub mod plan; +mod preflight; +#[cfg(test)] +mod tests; + +pub use action::{FileFlattenAction, FileOrganizeAction}; +pub use input::{ + FileFlattenInput, FileOrganizeInput, FlattenPolicy, Granularity, OrganizeDateField, + OrganizeRule, +}; +pub use job::{Rearrange, RearrangeJob, RearrangeOutput}; diff --git a/core/src/ops/files/organize/plan.rs b/core/src/ops/files/organize/plan.rs new file mode 100644 index 000000000..a2a964b15 --- /dev/null +++ b/core/src/ops/files/organize/plan.rs @@ -0,0 +1,343 @@ +//! Where each file goes, read from the index. +//! +//! Both operations are moves inside one folder, so every file keeps its +//! record and the plan is `Move` rows into `CreateDirectory` rows. Organize +//! names a subfolder per file from its date, its content kind, or its +//! extension; flatten names the folder itself. Two files wanting one place +//! are a conflict organize leaves alone and flatten resolves by its policy, +//! numbering the later one or leaving it where it is. Preflight and the job +//! read the same way, so the job moves exactly what the plan showed, and +//! checks each place is still free before it renames. + +use std::{ + collections::HashSet, + path::{Path, PathBuf}, +}; + +use chrono::{DateTime, Local, TimeZone}; +use sd_store::FsEntry; + +use super::input::{ + FileFlattenInput, FileOrganizeInput, FlattenPolicy, Granularity, OrganizeDateField, + OrganizeRule, +}; +use crate::{ + domain::{content_identity::ContentKind, SdPath}, + infra::query::QueryError, + ops::{ + files::{plan::StoreRevision, planner::is_junk, rename::resolve::captured_time}, + indexing::VolumeIndex, + paths::{ + compare::{Folder, Keyed}, + reach::stores_beneath_in, + }, + }, + volume::VolumeManager, +}; + +/// The name a file goes under when it has no extension. +pub const NO_EXTENSION: &str = "No extension"; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PlannedMove { + pub from: PathBuf, + pub to: PathBuf, + pub size: u64, +} + +/// Why a file stays where it is. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Stays { + /// Another file of the operation, or one already there, holds the place. + Conflict, + /// The policy leaves it. + Policy, +} + +#[derive(Debug, Default)] +pub struct Rearrangement { + pub moves: Vec, + /// Folders that have to exist first, in order. + pub directories: Vec, + /// Files left where they are, with the place they wanted. + pub left: Vec<(PathBuf, PathBuf, Stays)>, + /// Files already where the rule puts them. + pub in_place: u64, + pub revisions: Vec, +} + +#[derive(Debug, thiserror::Error)] +pub enum PlanError { + #[error("{0} is not on this device")] + Remote(SdPath), + #[error("{0} is not in a tracked source")] + Untracked(SdPath), + #[error("{0}")] + Read(#[from] QueryError), +} + +/// Where organize puts each file. +pub async fn organize( + volumes: &VolumeManager, + index: &VolumeIndex, + input: &FileOrganizeInput, +) -> Result { + let (root, files, revisions) = files_beneath(volumes, index, &input.scope).await?; + let mut out = Rearrangement { + revisions, + ..Default::default() + }; + let mut claimed: HashSet = HashSet::new(); + let mut directories: Vec = Vec::new(); + for file in files { + if !input.recursive && !file.key.0.is_empty() { + continue; + } + let folder = match &input.rule { + OrganizeRule::ByDate { field, granularity } => { + let when = date_of(volumes, index, &file, *field).await; + folder_for_date(when, *granularity) + } + OrganizeRule::ByKind => folder_for_kind(&file.entry), + OrganizeRule::ByExtension => file + .entry + .extension + .as_deref() + .filter(|extension| !extension.is_empty()) + .map(str::to_lowercase) + .unwrap_or_else(|| NO_EXTENSION.to_string()), + }; + let directory = root.join(&folder); + let to = directory.join(&file.entry.name); + if to == file.path { + out.in_place += 1; + continue; + } + if !directories.contains(&directory) { + directories.push(directory); + } + place(&mut out, &mut claimed, file, to, None).await; + } + for directory in directories { + if tokio::fs::symlink_metadata(&directory).await.is_err() { + out.directories.push(directory); + } + } + Ok(out) +} + +/// Where flatten puts each file. +pub async fn flatten( + volumes: &VolumeManager, + index: &VolumeIndex, + input: &FileFlattenInput, +) -> Result { + let (root, files, revisions) = files_beneath(volumes, index, &input.scope).await?; + let mut out = Rearrangement { + revisions, + ..Default::default() + }; + let mut claimed: HashSet = files + .iter() + .filter(|file| file.key.0.is_empty()) + .map(|file| file.path.clone()) + .collect(); + for file in files { + if file.key.0.is_empty() { + out.in_place += 1; + continue; + } + let to = root.join(&file.entry.name); + place(&mut out, &mut claimed, file, to, Some(input.on_conflict)).await; + } + Ok(out) +} + +/// Settle one file's place: free, taken by the policy's numbered name, or +/// left with why. +async fn place( + out: &mut Rearrangement, + claimed: &mut HashSet, + file: Keyed, + to: PathBuf, + policy: Option, +) { + let size = file.entry.size.unwrap_or(0).max(0) as u64; + let taken = claimed.contains(&to) || tokio::fs::symlink_metadata(&to).await.is_ok(); + if !taken { + claimed.insert(to.clone()); + out.moves.push(PlannedMove { + from: file.path, + to, + size, + }); + return; + } + match policy { + Some(FlattenPolicy::KeepBoth) => { + let numbered = numbered(&to, claimed).await; + claimed.insert(numbered.clone()); + out.moves.push(PlannedMove { + from: file.path, + to: numbered, + size, + }); + } + Some(FlattenPolicy::Skip) => out.left.push((file.path, to, Stays::Policy)), + None => out.left.push((file.path, to, Stays::Conflict)), + } +} + +/// A numbered name beside `path` that neither the disk nor the plan holds. +async fn numbered(path: &Path, claimed: &HashSet) -> PathBuf { + let parent = path.parent().unwrap_or(Path::new("")); + let name = path + .file_name() + .map(|name| name.to_string_lossy().into_owned()) + .unwrap_or_default(); + let (stem, extension) = match name.rsplit_once('.') { + Some((stem, extension)) if !stem.is_empty() => (stem.to_string(), format!(".{extension}")), + _ => (name.clone(), String::new()), + }; + for counter in 1.. { + let candidate = parent.join(format!("{stem} ({counter}){extension}")); + if !claimed.contains(&candidate) && tokio::fs::symlink_metadata(&candidate).await.is_err() { + return candidate; + } + } + unreachable!("the counter never runs out") +} + +/// Every file beneath the scope from the index, junk left out, with the +/// scope as the volume spells it and the revisions read. +async fn files_beneath( + volumes: &VolumeManager, + index: &VolumeIndex, + scope: &SdPath, +) -> Result<(PathBuf, Vec, Vec), PlanError> { + let local = scope + .as_local_path() + .ok_or_else(|| PlanError::Remote(scope.clone()))?; + let root = match volumes.locate_path(local).await { + Some((_, spelled)) => spelled, + None => local.to_path_buf(), + }; + let reaches = stores_beneath_in(volumes, index, scope).await; + if reaches.is_empty() { + return Err(PlanError::Untracked(scope.clone())); + } + let mut folder = Folder::open(index, reaches, false, None).await?; + let revisions = folder + .revisions() + .await? + .into_iter() + .map(|(source, revision)| StoreRevision { source, revision }) + .collect(); + let mut files = Vec::new(); + while let Some(file) = folder.next().await? { + if is_junk(&file.entry.name) { + continue; + } + files.push(file); + } + Ok((root, files, revisions)) +} + +async fn date_of( + volumes: &VolumeManager, + index: &VolumeIndex, + file: &Keyed, + field: OrganizeDateField, +) -> DateTime { + let modified = file + .entry + .mtime_ms + .and_then(|ms| Local.timestamp_millis_opt(ms).single()) + .unwrap_or_else(Local::now); + match field { + OrganizeDateField::Modified => modified, + OrganizeDateField::Created => file + .entry + .created_ms + .and_then(|ms| Local.timestamp_millis_opt(ms).single()) + .unwrap_or(modified), + OrganizeDateField::Captured => captured_time(volumes, index, &SdPath::local(&file.path)) + .await + .unwrap_or(modified), + } +} + +fn folder_for_date(when: DateTime, granularity: Granularity) -> String { + match granularity { + Granularity::Year => when.format("%Y").to_string(), + Granularity::YearMonth => when.format("%Y-%m").to_string(), + Granularity::YearMonthDay => when.format("%Y-%m-%d").to_string(), + } +} + +/// The folder a content kind's files go under. +pub fn folder_for_kind(entry: &FsEntry) -> String { + let kind = entry + .content_kind + .and_then(|kind| i32::try_from(kind).ok()) + .map(ContentKind::from_id) + .filter(|kind| *kind != ContentKind::Unknown) + .unwrap_or_else(|| { + crate::filetype::FileTypeRegistry::builtin() + .identify_by_extension(Path::new(&entry.name)) + }); + match kind { + ContentKind::Image => "Images", + ContentKind::Video => "Videos", + ContentKind::Audio => "Audio", + ContentKind::Document => "Documents", + ContentKind::Archive => "Archives", + ContentKind::Code => "Code", + ContentKind::Text => "Text", + ContentKind::Database => "Databases", + ContentKind::Book => "Books", + ContentKind::Font => "Fonts", + ContentKind::Mesh => "Meshes", + ContentKind::Config => "Config", + ContentKind::Encrypted => "Encrypted", + ContentKind::Key => "Keys", + ContentKind::Executable => "Executables", + ContentKind::Binary => "Binaries", + ContentKind::Spreadsheet => "Spreadsheets", + ContentKind::Presentation => "Presentations", + ContentKind::Email => "Email", + ContentKind::Calendar => "Calendars", + ContentKind::Contact => "Contacts", + ContentKind::Web => "Web", + ContentKind::Shortcut => "Shortcuts", + ContentKind::Package => "Packages", + ContentKind::ModelEntry => "Models", + ContentKind::Memory => "Memory", + ContentKind::Unknown => "Other", + } + .to_string() +} + +/// The directories beneath the scope, deepest first, for pruning after a +/// flatten. +pub async fn directories_beneath(root: &Path) -> Vec { + let mut found = Vec::new(); + let mut stack = vec![root.to_path_buf()]; + while let Some(directory) = stack.pop() { + let Ok(mut entries) = tokio::fs::read_dir(&directory).await else { + continue; + }; + while let Ok(Some(entry)) = entries.next_entry().await { + let path = entry.path(); + if tokio::fs::symlink_metadata(&path) + .await + .is_ok_and(|meta| meta.is_dir()) + { + found.push(path.clone()); + stack.push(path); + } + } + } + found.sort_by(|a, b| b.components().count().cmp(&a.components().count())); + found +} diff --git a/core/src/ops/files/organize/preflight.rs b/core/src/ops/files/organize/preflight.rs new file mode 100644 index 000000000..3d3d19822 --- /dev/null +++ b/core/src/ops/files/organize/preflight.rs @@ -0,0 +1,231 @@ +//! Whether organize or flatten would run, and what the folder would hold. + +use super::{ + action::{FileFlattenAction, FileOrganizeAction}, + input::{FileFlattenInput, FileOrganizeInput}, + plan::{flatten, organize, PlanError, Rearrangement, Stays}, +}; +use crate::{ + domain::SdPath, + infra::action::{ + error::ActionError, + preflight::{ + ExecutionFacts, Finding, PreviewContext, PreviewableAction, ValidatedAction, Validation, + }, + }, + ops::files::plan::{ + ChangeKind, ConflictKind, FsPlan, FsPlanSummary, PlanBasis, PlanChanges, PlannedChange, + SkipReason, + }, +}; + +pub const REMOTE_ROOT: &str = "organize.remote_root"; +pub const MISSING: &str = "organize.missing"; +pub const UNTRACKED: &str = "organize.untracked"; +pub const CONFLICTS: &str = "organize.conflicts"; +pub const NOTHING: &str = "organize.nothing"; + +impl ValidatedAction for FileOrganizeAction { + async fn validate( + input: &FileOrganizeInput, + ctx: &PreviewContext, + ) -> Result { + let planned = check(&input.scope, ctx).await?; + let plan = match planned { + Ok(()) => Some(organize(ctx.volumes(), ctx.index(), input).await), + Err(finding) => return Ok(refused(ctx, finding)), + }; + Ok(validation(ctx, plan)) + } +} + +impl PreviewableAction for FileOrganizeAction { + type Plan = FsPlan; + + async fn preview( + input: FileOrganizeInput, + ctx: &PreviewContext, + ) -> Result { + let rearrangement = organize(ctx.volumes(), ctx.index(), &input) + .await + .map_err(failed)?; + Ok(ctx.plans().retain(plan_of(rearrangement))) + } +} + +impl ValidatedAction for FileFlattenAction { + async fn validate( + input: &FileFlattenInput, + ctx: &PreviewContext, + ) -> Result { + let planned = check(&input.scope, ctx).await?; + let plan = match planned { + Ok(()) => Some(flatten(ctx.volumes(), ctx.index(), input).await), + Err(finding) => return Ok(refused(ctx, finding)), + }; + Ok(validation(ctx, plan)) + } +} + +impl PreviewableAction for FileFlattenAction { + type Plan = FsPlan; + + async fn preview(input: FileFlattenInput, ctx: &PreviewContext) -> Result { + let rearrangement = flatten(ctx.volumes(), ctx.index(), &input) + .await + .map_err(failed)?; + Ok(ctx.plans().retain(plan_of(rearrangement))) + } +} + +crate::register_validate!(FileOrganizeAction, "files.organize"); +crate::register_preview!(FileOrganizeAction, "files.organize"); +crate::register_validate!(FileFlattenAction, "files.flatten"); +crate::register_preview!(FileFlattenAction, "files.flatten"); + +/// The scope is on this device, there, a folder, and tracked. +async fn check(scope: &SdPath, ctx: &PreviewContext) -> Result, ActionError> { + let Some(local) = scope.as_local_path() else { + return Ok(Err(Finding::error( + REMOTE_ROOT, + "the folder is on another device; run this there with --device", + ) + .at(scope.clone()))); + }; + match tokio::fs::metadata(local).await { + Ok(meta) if meta.is_dir() => {} + _ => { + return Ok(Err( + Finding::error(MISSING, "the folder is not there").at(scope.clone()) + )) + } + } + if ctx.reach(scope).await.is_empty() { + return Ok(Err(Finding::error( + UNTRACKED, + "the folder is outside every tracked source; the plan needs its index", + ) + .at(scope.clone()))); + } + Ok(Ok(())) +} + +fn refused(ctx: &PreviewContext, finding: Finding) -> Validation { + Validation { + findings: vec![finding], + facts: ExecutionFacts { + executes_on: ctx.executes_on(), + strategy: Some("rename".to_string()), + ..Default::default() + }, + } +} + +fn validation( + ctx: &PreviewContext, + planned: Option>, +) -> Validation { + let mut findings = Vec::new(); + let mut facts = ExecutionFacts { + executes_on: ctx.executes_on(), + strategy: Some("rename".to_string()), + ..Default::default() + }; + match planned { + Some(Ok(rearrangement)) => { + let conflicts = rearrangement + .left + .iter() + .filter(|(_, _, stays)| *stays == Stays::Conflict) + .count(); + if conflicts > 0 { + findings.push(Finding::warning( + CONFLICTS, + format!("{conflicts} files want a place another file holds; they stay where they are"), + )); + } + if rearrangement.moves.is_empty() { + findings.push(Finding::info( + NOTHING, + format!( + "nothing moves: {} files are already where the rule puts them", + rearrangement.in_place + ), + )); + } + facts.estimated_files = Some(rearrangement.moves.len() as u64); + facts.estimated_bytes = Some(0); + } + Some(Err(error)) => findings.push(Finding::error(UNTRACKED, error.to_string())), + None => {} + } + Validation { findings, facts } +} + +fn plan_of(rearrangement: Rearrangement) -> FsPlan { + let device = crate::device::get_current_device_slug(); + let at = |path: std::path::PathBuf| SdPath::Physical { + device_slug: device.clone(), + path, + }; + let mut summary = FsPlanSummary::default(); + let mut changes = PlanChanges::default(); + for directory in rearrangement.directories { + summary.count(&ChangeKind::CreateDirectory, 0); + changes.push(PlannedChange { + path: at(directory), + change: ChangeKind::CreateDirectory, + }); + } + for planned in rearrangement.moves { + let change = ChangeKind::Move { + from: at(planned.from), + }; + summary.count(&change, planned.size); + changes.push(PlannedChange { + path: at(planned.to), + change, + }); + } + for (from, wanted, stays) in rearrangement.left { + let (path, change) = match stays { + Stays::Conflict => ( + wanted, + ChangeKind::Conflict { + kind: ConflictKind::Sources, + }, + ), + Stays::Policy => ( + from, + ChangeKind::Skip { + reason: SkipReason::Policy, + }, + ), + }; + summary.count(&change, 0); + changes.push(PlannedChange { + path: at(path), + change, + }); + } + let (changes, truncated) = changes.finish(); + FsPlan { + handle: None, + basis: PlanBasis::Index { + revisions: rearrangement.revisions, + }, + roots: Vec::new(), + summary, + changes, + truncated, + } +} + +fn failed(error: PlanError) -> ActionError { + match error { + PlanError::Remote(_) | PlanError::Untracked(_) => { + ActionError::InvalidInput(error.to_string()) + } + PlanError::Read(error) => ActionError::Internal(error.to_string()), + } +} diff --git a/core/src/ops/files/organize/tests.rs b/core/src/ops/files/organize/tests.rs new file mode 100644 index 000000000..930103be3 --- /dev/null +++ b/core/src/ops/files/organize/tests.rs @@ -0,0 +1,255 @@ +//! Organize previews the folders and moves and leaves records with their +//! identity; flatten numbers collisions and prunes the folders it empties. + +use std::{path::PathBuf, time::Duration}; + +use sd_store::file::FileKind; + +use super::{ + input::{ + FileFlattenInput, FileOrganizeInput, FlattenPolicy, Granularity, OrganizeDateField, + OrganizeRule, + }, + job::{Rearrange, RearrangeJob}, + preflight::CONFLICTS, + FileFlattenAction, FileOrganizeAction, +}; +use crate::{ + domain::SdPath, + infra::{ + action::preflight::{PreviewableAction, ValidatedAction}, + job::output::JobOutput, + }, + ops::files::{ + fixture::{Fixture, T}, + plan::{ChangeKind, ConflictKind, FsPlan}, + }, +}; + +fn change_at(plan: &FsPlan, path: &std::path::Path) -> Option { + plan.changes + .iter() + .find(|change| change.path.path().map(PathBuf::as_path) == Some(path)) + .map(|change| change.change.clone()) +} + +async fn run(fixture: &Fixture, what: Rearrange) -> serde_json::Value { + let handle = fixture + .library + .jobs() + .dispatch(RearrangeJob::new(what)) + .await + .expect("dispatched"); + let output = tokio::time::timeout(Duration::from_secs(30), handle.wait()) + .await + .expect("in time") + .expect("completed"); + let JobOutput::Custom(value) = output else { + panic!("not a rearrange output: {output:?}"); + }; + value +} + +/// Organizing by month plans a folder per month and a move per file, and +/// each move is a rename, so the inode the record is matched by survives. +#[tokio::test] +async fn organizing_by_month_previews_folders_and_moves_and_keeps_identity() { + let fixture = Fixture::new().await; + let file = FileKind::File; + // May and June 2024, in local time well inside each month. + let may = 1_715_500_000_000; + let june = 1_718_100_000_000; + let tree = [ + ("a.jpg", file, 4, may, Some("a"), None, None), + ("b.jpg", file, 3, may, Some("b"), None, None), + ("c.mov", file, 2, june, Some("c"), None, None), + ("2024-06/d.mov", file, 2, june, Some("d"), None, None), + ]; + fixture.materialize(&fixture.source, &tree); + fixture.index(&fixture.source, &tree).await; + + let input = FileOrganizeInput { + scope: SdPath::local(&fixture.source), + rule: OrganizeRule::ByDate { + field: OrganizeDateField::Modified, + granularity: Granularity::YearMonth, + }, + recursive: true, + }; + let validation = FileOrganizeAction::validate(&input, &fixture.preview()) + .await + .expect("validated"); + assert!(!validation.refuses(), "{:?}", validation.findings); + assert_eq!(validation.facts.estimated_files, Some(3)); + + let plan = FileOrganizeAction::preview(input.clone(), &fixture.preview()) + .await + .expect("planned"); + assert_eq!( + change_at(&plan, &fixture.source.join("2024-05")), + Some(ChangeKind::CreateDirectory) + ); + assert_eq!( + change_at(&plan, &fixture.source.join("2024-05/a.jpg")), + Some(ChangeKind::Move { + from: SdPath::local(fixture.source.join("a.jpg")) + }) + ); + assert_eq!( + change_at(&plan, &fixture.source.join("2024-06/d.mov")), + None, + "already in place" + ); + assert_eq!(plan.summary.moves.files, 3); + assert_eq!(plan.summary.directories_created, 1); + + let inode_before = inode(&fixture.source.join("a.jpg")); + let output = run(&fixture, Rearrange::Organize(input)).await; + assert_eq!(output["moved"], 3); + assert_eq!(output["directories_created"], 1); + assert!(fixture.source.join("2024-05/a.jpg").exists()); + assert!(fixture.source.join("2024-06/c.mov").exists()); + assert!(!fixture.source.join("a.jpg").exists()); + assert_eq!( + inode(&fixture.source.join("2024-05/a.jpg")), + inode_before, + "a rename keeps the inode the record is matched by" + ); +} + +#[cfg(unix)] +fn inode(path: &std::path::Path) -> u64 { + use std::os::unix::fs::MetadataExt; + std::fs::metadata(path).expect("metadata").ino() +} + +#[cfg(not(unix))] +fn inode(_path: &std::path::Path) -> u64 { + 0 +} + +/// Organizing by kind and by extension names the folders, and two files +/// wanting one place are a conflict left alone. +#[tokio::test] +async fn organizing_by_kind_names_folders_and_leaves_conflicts() { + let fixture = Fixture::new().await; + let file = FileKind::File; + let tree = [ + ("photo.jpg", file, 4, T, Some("p"), None, None), + ("notes.txt", file, 3, T, Some("n"), None, None), + ("README", file, 2, T, Some("r"), None, None), + ("old/photo.jpg", file, 5, T, Some("o"), None, None), + ]; + fixture.materialize(&fixture.source, &tree); + fixture.index(&fixture.source, &tree).await; + + let by_kind = FileOrganizeInput { + scope: SdPath::local(&fixture.source), + rule: OrganizeRule::ByKind, + recursive: true, + }; + let validation = FileOrganizeAction::validate(&by_kind, &fixture.preview()) + .await + .expect("validated"); + assert!(validation + .findings + .iter() + .any(|finding| finding.code == CONFLICTS)); + let plan = FileOrganizeAction::preview(by_kind, &fixture.preview()) + .await + .expect("planned"); + assert!(matches!( + change_at(&plan, &fixture.source.join("Images/photo.jpg")), + Some(ChangeKind::Move { .. }) + | Some(ChangeKind::Conflict { + kind: ConflictKind::Sources + }) + )); + assert_eq!(plan.summary.conflicts, 1); + assert!(matches!( + change_at(&plan, &fixture.source.join("Text/notes.txt")), + Some(ChangeKind::Move { .. }) + )); + + let by_extension = FileOrganizeInput { + scope: SdPath::local(&fixture.source), + rule: OrganizeRule::ByExtension, + recursive: false, + }; + let plan = FileOrganizeAction::preview(by_extension, &fixture.preview()) + .await + .expect("planned"); + assert!(matches!( + change_at(&plan, &fixture.source.join("jpg/photo.jpg")), + Some(ChangeKind::Move { .. }) + )); + assert!(matches!( + change_at(&plan, &fixture.source.join("No extension/README")), + Some(ChangeKind::Move { .. }) + )); + assert_eq!( + change_at(&plan, &fixture.source.join("jpg/photo.jpg")).is_some(), + true + ); + assert_eq!( + plan.summary.moves.files, 3, + "the nested file is left out without recursive" + ); +} + +/// Flattening numbers a collision under keep both, leaves it under skip, +/// and prunes the folders it emptied. +#[tokio::test] +async fn flattening_numbers_collisions_and_prunes_emptied_folders() { + let fixture = Fixture::new().await; + let file = FileKind::File; + let tree = [ + ("a.txt", file, 4, T, Some("a"), None, None), + ("x/a.txt", file, 3, T, Some("xa"), None, None), + ("x/deep/b.txt", file, 2, T, Some("b"), None, None), + ("y/c.txt", file, 1, T, Some("c"), None, None), + ]; + fixture.materialize(&fixture.source, &tree); + fixture.index(&fixture.source, &tree).await; + + let skip = FileFlattenInput { + scope: SdPath::local(&fixture.source), + on_conflict: FlattenPolicy::Skip, + }; + let plan = FileFlattenAction::preview(skip, &fixture.preview()) + .await + .expect("planned"); + assert_eq!(plan.summary.moves.files, 2); + assert_eq!(plan.summary.skips.policy.files, 1); + + let keep_both = FileFlattenInput { + scope: SdPath::local(&fixture.source), + on_conflict: FlattenPolicy::KeepBoth, + }; + let validation = FileFlattenAction::validate(&keep_both, &fixture.preview()) + .await + .expect("validated"); + assert!(!validation.refuses()); + let plan = FileFlattenAction::preview(keep_both.clone(), &fixture.preview()) + .await + .expect("planned"); + assert_eq!( + change_at(&plan, &fixture.source.join("a (1).txt")), + Some(ChangeKind::Move { + from: SdPath::local(fixture.source.join("x/a.txt")) + }) + ); + assert_eq!(plan.summary.moves.files, 3); + + let output = run(&fixture, Rearrange::Flatten(keep_both)).await; + assert_eq!(output["moved"], 3); + assert_eq!(output["pruned_directories"], 3); + assert_eq!( + std::fs::read_to_string(fixture.source.join("a (1).txt")).expect("numbered"), + "xxx" + ); + assert!(fixture.source.join("b.txt").exists()); + assert!(fixture.source.join("c.txt").exists()); + assert!(!fixture.source.join("x").exists()); + assert!(!fixture.source.join("y").exists()); +} diff --git a/core/src/ops/files/plan.rs b/core/src/ops/files/plan.rs index c289b8547..b91e44a63 100644 --- a/core/src/ops/files/plan.rs +++ b/core/src/ops/files/plan.rs @@ -23,7 +23,7 @@ use serde::{Deserialize, Serialize}; use specta::Type; use uuid::Uuid; -use crate::domain::SdPath; +use crate::{domain::SdPath, infra::job::journal::Attributes}; /// The most changes a plan lists. The summary counts past it. pub const CHANGE_CAP: usize = 5000; @@ -105,6 +105,10 @@ impl PlanHandles { pub enum PlanBasis { /// The stores the plan read, at the revisions it read them. Index { revisions: Vec }, + /// A job's journal, for an undo. + Journal { job: Uuid }, + /// An archive's own directory, for an extract. + Archive { path: SdPath, entries: u64 }, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Type)] @@ -147,6 +151,10 @@ pub enum ChangeKind { Conflict { kind: ConflictKind, }, + /// The filesystem attributes that change, each absent where it stays. + SetAttributes { + attributes: Attributes, + }, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Type)] @@ -200,6 +208,7 @@ pub struct FsPlanSummary { /// Files at the same path on both sides with different bytes, which the /// policy resolved one way or another. pub collisions: u64, + pub attributes: u64, } #[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize, Type)] @@ -240,6 +249,7 @@ impl FsPlanSummary { ChangeKind::Move { .. } => self.moves.add(bytes), ChangeKind::Delete { .. } => self.deletes.add(bytes), ChangeKind::Conflict { .. } => self.conflicts += 1, + ChangeKind::SetAttributes { .. } => self.attributes += 1, } } diff --git a/core/src/ops/files/planner.rs b/core/src/ops/files/planner.rs index 976e07288..4b13090ab 100644 --- a/core/src/ops/files/planner.rs +++ b/core/src/ops/files/planner.rs @@ -62,6 +62,13 @@ pub(crate) struct Planner<'a> { /// Every decision by destination-relative path, kept for a job that /// checks its own decisions against the plan's. decisions: Option>, + /// The files only the destination holds, by destination-relative path, + /// after every pair so far: what a mirror removes. `None` until asked + /// for. + extras: Option>, + /// Whether a pair has been planned, so the first pair's extras are the + /// starting set rather than an intersection with nothing. + paired: bool, } impl<'a> Planner<'a> { @@ -80,13 +87,54 @@ impl<'a> Planner<'a> { revisions: Vec::new(), roots: Vec::new(), decisions: keep_decisions.then(HashMap::new), + extras: None, + paired: false, } } + /// Plan the removal of what no source holds, after the pairs. + pub(crate) fn removing_extras(mut self) -> Self { + self.extras = Some(HashMap::new()); + self + } + pub(crate) fn summary(&self) -> &FsPlanSummary { &self.summary } + /// A `Delete` row for each file the destination holds that no source + /// does, flagged where it is the last copy of its bytes in the library. + /// Called once, after every pair. + pub(crate) async fn remove_extras(&mut self, dest_root: &Path) -> u64 { + let Some(extras) = self.extras.take() else { + return 0; + }; + let hashes: Vec = extras + .values() + .filter_map(|entry| entry.sampled_hash.clone()) + .collect(); + let last = crate::ops::files::delete::last_copies(self.ctx, hashes).await; + let mut last_copies = 0; + let mut paths: Vec<(String, FsEntry)> = extras.into_iter().collect(); + paths.sort_by(|a, b| a.0.cmp(&b.0)); + for (relative, entry) in paths { + let last_copy = entry + .sampled_hash + .as_ref() + .is_some_and(|hash| last.contains(hash)); + if last_copy { + last_copies += 1; + } + let size = bytes(&entry); + self.note( + dest_root.join(&relative), + ChangeKind::Delete { last_copy }, + size, + ); + } + last_copies + } + /// The plan so far, as a whole. pub(crate) fn finish(self) -> FsPlan { let (changes, truncated) = self.changes.finish(); @@ -175,10 +223,18 @@ impl<'a> Planner<'a> { } let mut matcher = Matcher::by_path(&mut a, &mut b); + let mut only_here: HashMap = HashMap::new(); while let Some(sorted) = matcher.next().await.map_err(read_failed)? { let Some(relative) = sorted.path() else { continue; }; + if self.extras.is_some() && sorted.a.is_none() { + if let Some(existing) = &sorted.b { + if existing.entry.kind == FileKind::File && !is_junk(&existing.entry.name) { + only_here.insert(relative.clone(), existing.entry.clone()); + } + } + } let Some((relative, change, bytes)) = self.decide(&dest_root, relative, sorted).await else { continue; @@ -195,6 +251,16 @@ impl<'a> Planner<'a> { change, }); } + // A file is an extra only if no source holds it: the first pair's + // set, narrowed by each pair after. + if let Some(extras) = &mut self.extras { + if self.paired { + extras.retain(|relative, _| only_here.contains_key(relative)); + } else { + *extras = only_here; + } + } + self.paired = true; Ok(()) } @@ -318,6 +384,17 @@ impl<'a> Planner<'a> { } } + /// A numbered name beside `path`, in the convention copy uses, that + /// neither the index nor this plan holds. + pub(crate) async fn unique_name(&self, path: &Path) -> PathBuf { + let parent = path.parent().unwrap_or(Path::new("")); + let name = path + .file_name() + .map(|name| name.to_string_lossy().into_owned()) + .unwrap_or_default(); + parent.join(self.unique(parent, &name).await) + } + /// A numbered name beside `relative`, in the convention copy uses, that /// neither the destination's index nor this plan holds. async fn unique(&self, dest_root: &Path, relative: &str) -> String { diff --git a/core/src/ops/files/query/directory_listing.rs b/core/src/ops/files/query/directory_listing.rs index 1e3877f3a..0c3c245b5 100644 --- a/core/src/ops/files/query/directory_listing.rs +++ b/core/src/ops/files/query/directory_listing.rs @@ -811,6 +811,7 @@ mod tests { destination: SdPath::local(&fixture.destination), on_conflict: MergeConflictPolicy::Skip, consume_sources: true, + remove_extras: false, }; let plan = FileMergeAction::preview(input, &fixture.preview()) .await diff --git a/core/src/ops/files/rename/action.rs b/core/src/ops/files/rename/action.rs index b4e423672..ce11801d1 100644 --- a/core/src/ops/files/rename/action.rs +++ b/core/src/ops/files/rename/action.rs @@ -1,7 +1,12 @@ -//! File rename action handler +//! Rename action handlers -use super::input::FileRenameInput; -use super::validation::validate_filename; +use std::sync::Arc; + +use super::{ + input::{FileRenameBatchInput, FileRenameInput}, + job::RenameJob, + naming::check_portable, +}; use crate::{ context::CoreContext, domain::addressing::SdPath, @@ -9,22 +14,17 @@ use crate::{ action::{error::ActionError, LibraryAction}, job::handle::JobReceipt, }, - ops::files::copy::job::FileCopyJob, }; use serde::{Deserialize, Serialize}; -use std::sync::Arc; -/// Action for renaming a file or directory +/// Rename one file or directory. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct FileRenameAction { - /// The file or directory to rename pub target: SdPath, - /// The new name (filename only, no path) pub new_name: String, } impl FileRenameAction { - /// Create a new rename action pub fn new(target: SdPath, new_name: impl Into) -> Self { Self { target, @@ -38,16 +38,8 @@ impl LibraryAction for FileRenameAction { type Output = JobReceipt; fn from_input(input: Self::Input) -> Result { - validate_filename(&input.new_name).map_err(|e| e.to_string())?; - match &input.target { - SdPath::Content { .. } => { - return Err("Cannot rename content-addressed files directly".to_string()); - } - SdPath::Sidecar { .. } => { - return Err("Cannot rename sidecar files directly".to_string()); - } - _ => {} - } + check_portable(&input.new_name).map_err(|e| e.to_string())?; + refuse_unnameable(&input.target)?; Ok(FileRenameAction { target: input.target, new_name: input.new_name, @@ -59,15 +51,11 @@ impl LibraryAction for FileRenameAction { library: Arc, _context: Arc, ) -> Result { - // Use FileCopyJob::new_rename which handles the rename as a move operation - let job = FileCopyJob::new_rename(self.target, self.new_name); - let job_handle = library .jobs() - .dispatch(job) + .dispatch(RenameJob::named(self.target, self.new_name)) .await .map_err(ActionError::Job)?; - Ok(job_handle.into()) } @@ -76,17 +64,62 @@ impl LibraryAction for FileRenameAction { } } -// Register with the action-centric registry crate::register_library_action!(FileRenameAction, "files.rename"); +/// Rename several files by rules. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct FileRenameBatchAction { + input: FileRenameBatchInput, +} + +impl LibraryAction for FileRenameBatchAction { + type Input = FileRenameBatchInput; + type Output = JobReceipt; + + fn from_input(input: Self::Input) -> Result { + input.validate().map_err(|errors| errors.join("; "))?; + for target in &input.targets { + refuse_unnameable(target)?; + } + Ok(Self { input }) + } + + async fn execute( + self, + library: Arc, + _context: Arc, + ) -> Result { + let job_handle = library + .jobs() + .dispatch(RenameJob::ruled(self.input.targets, self.input.rules)) + .await + .map_err(ActionError::Job)?; + Ok(job_handle.into()) + } + + fn action_kind(&self) -> &'static str { + "files.rename_batch" + } +} + +crate::register_library_action!(FileRenameBatchAction, "files.rename_batch"); + +fn refuse_unnameable(target: &SdPath) -> Result<(), String> { + match target { + SdPath::Content { .. } => Err("Cannot rename content-addressed files directly".to_string()), + SdPath::Sidecar { .. } => Err("Cannot rename sidecar files directly".to_string()), + _ => Ok(()), + } +} + #[cfg(test)] mod tests { use super::*; #[test] - fn test_action_creation() { + fn a_structurally_broken_name_never_reaches_a_job() { let target = SdPath::local(std::path::PathBuf::from("/test/file.txt")); - let action = FileRenameAction::new(target, "newname.txt"); - assert_eq!(action.new_name, "newname.txt"); + assert!(FileRenameAction::from_input(FileRenameInput::new(target.clone(), "a/b")).is_err()); + assert!(FileRenameAction::from_input(FileRenameInput::new(target, "b.txt")).is_ok()); } } diff --git a/core/src/ops/files/rename/input.rs b/core/src/ops/files/rename/input.rs index 545da241f..1a71c544e 100644 --- a/core/src/ops/files/rename/input.rs +++ b/core/src/ops/files/rename/input.rs @@ -1,11 +1,13 @@ -//! Input types for file rename operations +//! Input types for rename operations use crate::domain::addressing::SdPath; use serde::{Deserialize, Serialize}; use specta::Type; -/// Input for renaming a file or directory -#[derive(Debug, Clone, Serialize, Deserialize, Type)] +use super::rules::RenameRule; + +/// Rename one file or directory. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Type)] pub struct FileRenameInput { /// The file or directory to rename pub target: SdPath, @@ -14,7 +16,6 @@ pub struct FileRenameInput { } impl FileRenameInput { - /// Create a new file rename input pub fn new(target: SdPath, new_name: impl Into) -> Self { Self { target, @@ -22,3 +23,30 @@ impl FileRenameInput { } } } + +/// Rename several files by rules applied to each name in order. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Type)] +pub struct FileRenameBatchInput { + pub targets: Vec, + pub rules: Vec, +} + +impl FileRenameBatchInput { + pub fn validate(&self) -> Result<(), Vec> { + let mut errors = Vec::new(); + if self.targets.is_empty() { + errors.push("name at least one file to rename".to_string()); + } + if self.rules.is_empty() { + errors.push("give at least one rule".to_string()); + } + if let Err(error) = super::rules::check_rules(&self.rules) { + errors.push(error.to_string()); + } + if errors.is_empty() { + Ok(()) + } else { + Err(errors) + } + } +} diff --git a/core/src/ops/files/rename/job.rs b/core/src/ops/files/rename/job.rs new file mode 100644 index 000000000..b1b7e9965 --- /dev/null +++ b/core/src/ops/files/rename/job.rs @@ -0,0 +1,260 @@ +//! The rename itself. +//! +//! The set is resolved again against the live filesystem when the job +//! starts, so a name that would overwrite a file is refused here as it was +//! in validation. The renames then run in an order that never overwrites: +//! a rename whose new name another file of the set still holds waits for +//! that file to move, and a cycle, `a` to `b` while `b` goes to `a`, is +//! broken by parking one file under a temporary name in its directory. +//! Each rename is one `rename` call, so records keep their identity. + +use std::{ + path::{Path, PathBuf}, + time::Instant, +}; + +use serde::{Deserialize, Serialize}; +use specta::Type; + +use super::resolve::{resolve, Rename, RenameSet}; +use crate::{ + domain::SdPath, + infra::job::{generic_progress::GenericProgress, journal::Effect, prelude::*}, +}; + +/// Renames between checkpoints. +const CHECKPOINT_EVERY: usize = 50; + +#[derive(Debug, Serialize, Deserialize, Job)] +pub struct RenameJob { + pub set: RenameSet, + /// The original paths already renamed, for a resumed job. + pub done: Vec, + #[serde(skip, default = "Instant::now")] + started_at: Instant, +} + +impl RenameJob { + pub fn named(target: SdPath, new_name: String) -> Self { + Self::named_many(vec![(target, new_name)]) + } + + pub fn named_many(pairs: Vec<(SdPath, String)>) -> Self { + Self::new(RenameSet::Named(pairs)) + } + + pub fn ruled(targets: Vec, rules: Vec) -> Self { + Self::new(RenameSet::Ruled { targets, rules }) + } + + fn new(set: RenameSet) -> Self { + Self { + set, + done: Vec::new(), + started_at: Instant::now(), + } + } +} + +impl Job for RenameJob { + const NAME: &'static str = "rename_files"; + const RESUMABLE: bool = true; + const DESCRIPTION: Option<&'static str> = Some("Rename files"); +} + +impl crate::infra::job::traits::DynJob for RenameJob { + fn job_name(&self) -> &'static str { + Self::NAME + } +} + +#[derive(Debug, Clone, Default, Serialize, Deserialize, Type)] +pub struct RenameOutput { + pub renamed: u64, + pub unchanged: u64, + /// Targets a finding refused, with the finding's message. + pub refused: Vec, + pub failed: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Type)] +pub struct RenameProblem { + pub path: PathBuf, + pub reason: String, +} + +impl From for JobOutput { + fn from(output: RenameOutput) -> Self { + JobOutput::custom(output) + } +} + +#[derive(Debug, Serialize, Deserialize)] +struct Resume { + done: Vec, +} + +#[async_trait::async_trait] +impl JobHandler for RenameJob { + type Output = RenameOutput; + + async fn run(&mut self, ctx: JobContext<'_>) -> JobResult { + if let Some(resume) = ctx.load_state::().await? { + self.done = resume.done; + ctx.log(format!("Resuming past {} renames", self.done.len())); + } + let context = ctx.library().core_context(); + ctx.progress(Progress::Indeterminate("Resolving names".to_string())); + let resolved = resolve(&context.volume_manager, context.volume_index(), &self.set).await; + + let mut output = RenameOutput { + unchanged: resolved.unchanged, + ..Default::default() + }; + for finding in resolved + .findings + .iter() + .filter(|finding| finding.severity == crate::infra::action::preflight::Severity::Error) + { + output.refused.push(RenameProblem { + path: finding + .path + .as_ref() + .and_then(|path| path.as_local_path()) + .map(Path::to_path_buf) + .unwrap_or_default(), + reason: finding.message.clone(), + }); + ctx.log(format!("Refused: {}", finding.message)); + } + + // Each rename with where its file is right now, which differs from + // its original path once the file is parked mid-cycle. + let mut pending: Vec<(Rename, PathBuf)> = resolved + .renames + .into_iter() + .filter(|rename| !rename.refused) + .filter(|rename| !self.done.contains(&rename.from)) + .map(|rename| { + let at = rename.from.clone(); + (rename, at) + }) + .collect(); + let total = pending.len() as u64 + self.done.len() as u64; + output.renamed = self.done.len() as u64; + ctx.log(format!("{} names to change", pending.len())); + + let mut since_checkpoint = 0; + while !pending.is_empty() { + ctx.check_interrupt().await?; + let mut progressed = false; + let mut remaining = Vec::with_capacity(pending.len()); + for (rename, at) in pending { + let free = + rename.case_only || tokio::fs::symlink_metadata(&rename.to).await.is_err(); + if !free { + remaining.push((rename, at)); + continue; + } + progressed = true; + match tokio::fs::rename(&at, &rename.to).await { + Ok(()) => { + let subject = tokio::fs::symlink_metadata(&rename.to).await.ok(); + ctx.record(vec![Effect::moved( + rename.from.clone(), + rename.to.clone(), + subject.as_ref(), + )]) + .await; + output.renamed += 1; + self.done.push(rename.from.clone()); + since_checkpoint += 1; + } + Err(error) => { + ctx.log(format!("{}: {error}", rename.from.display())); + output.failed.push(RenameProblem { + path: rename.from.clone(), + reason: error.to_string(), + }); + } + } + ctx.progress(Progress::generic( + GenericProgress::new( + (output.renamed as f32 / total.max(1) as f32).min(1.0), + "Renaming", + format!("{} of {total}", output.renamed), + ) + .with_completion(output.renamed, total) + .with_errors(output.failed.len() as u64, output.refused.len() as u64), + )); + if since_checkpoint >= CHECKPOINT_EVERY { + since_checkpoint = 0; + ctx.checkpoint_with_state(&Resume { + done: self.done.clone(), + }) + .await?; + } + } + pending = remaining; + if !progressed && !pending.is_empty() { + // Every remaining name is held by another file of the set: + // a cycle. Park the first file under a temporary name so + // its own name frees up, and let it wait for its turn. The + // journal sees the whole move, from the original name to + // the final one, once it lands. + let (first, at) = &mut pending[0]; + let parked = temporary_name(&first.from); + match tokio::fs::rename(&*at, &parked).await { + Ok(()) => { + ctx.log(format!( + "Parked {} as {} to break a cycle", + at.display(), + parked.display() + )); + *at = parked; + } + Err(error) => { + let (rename, _) = pending.remove(0); + output.failed.push(RenameProblem { + path: rename.from, + reason: error.to_string(), + }); + } + } + } + } + + ctx.progress(Progress::generic( + GenericProgress::new( + 1.0, + "Complete", + format!( + "{} renamed, {} refused", + output.renamed, + output.refused.len() + ), + ) + .with_completion(total, total) + .with_performance(0.0, None, Some(self.started_at.elapsed())) + .with_errors(output.failed.len() as u64, output.refused.len() as u64), + )); + ctx.log(format!( + "Rename completed: {} renamed, {} unchanged, {} refused, {} failed", + output.renamed, + output.unchanged, + output.refused.len(), + output.failed.len() + )); + Ok(output) + } +} + +/// A name beside `path` that nothing holds, for a file parked mid-cycle. +fn temporary_name(path: &Path) -> PathBuf { + let name = path + .file_name() + .map(|name| name.to_string_lossy().into_owned()) + .unwrap_or_default(); + let short = uuid::Uuid::new_v4().simple().to_string(); + path.with_file_name(format!(".{name}.renaming-{}", &short[..8])) +} diff --git a/core/src/ops/files/rename/mod.rs b/core/src/ops/files/rename/mod.rs index 35ae88ae8..6f5667958 100644 --- a/core/src/ops/files/rename/mod.rs +++ b/core/src/ops/files/rename/mod.rs @@ -1,12 +1,21 @@ -//! File rename operations +//! # Rename //! -//! Provides a dedicated action API for renaming files and directories. -//! Wraps FileCopyJob::new_rename() for execution while providing input validation. +//! One file to a new name, or many files by rules applied to each name in +//! order, both on preflight: validation resolves every new name against the +//! live directory and the target volume's filesystem, the preview is one +//! move per changed name, and the job renames in an order that never +//! overwrites and journals each move. pub mod action; pub mod input; -pub mod validation; +pub mod job; +pub mod naming; +mod preflight; +pub mod resolve; +pub mod rules; -pub use action::FileRenameAction; -pub use input::FileRenameInput; -pub use validation::{validate_filename, FilenameValidationError}; +pub use action::{FileRenameAction, FileRenameBatchAction}; +pub use input::{FileRenameBatchInput, FileRenameInput}; +pub use job::{RenameJob, RenameOutput}; +pub use naming::{check_name, check_portable, NameProblem, NameRules}; +pub use rules::{CaseRule, ExtensionCase, RenameRule}; diff --git a/core/src/ops/files/rename/naming.rs b/core/src/ops/files/rename/naming.rs new file mode 100644 index 000000000..d96ae3ed5 --- /dev/null +++ b/core/src/ops/files/rename/naming.rs @@ -0,0 +1,207 @@ +//! What a filesystem accepts as a name. +//! +//! Two checks. The portable one refuses what no filesystem takes: an empty +//! name, a path separator, `.` and `..`, and NUL; it runs in `from_input`, +//! so a structurally broken name never reaches a job. The filesystem check +//! knows the target volume: Windows filesystems and SMB shares refuse the +//! characters and device names NTFS does and a trailing dot or space, Apple +//! filesystems refuse a colon, and every filesystem has a length past which +//! a name cannot be written. That check is a validation finding, since it +//! needs the volume registry. + +use std::path::Path; + +use thiserror::Error; + +use crate::volume::types::FileSystem; + +#[derive(Debug, Clone, PartialEq, Eq, Error)] +pub enum NameProblem { + #[error("the name is empty")] + Empty, + #[error("a name cannot contain a path separator")] + Separator, + #[error("a name cannot be '.' or '..'")] + Dot, + #[error("the name contains a character the filesystem refuses: {0:?}")] + Character(char), + #[error("{0} is a name Windows filesystems reserve for a device")] + Reserved(String), + #[error("a name cannot end with a space or a period on this filesystem")] + Ending, + #[error("the name is longer than the {0} {1} the filesystem allows")] + TooLong(usize, &'static str), +} + +/// Names Windows filesystems reserve for devices, matched on the stem +/// without regard to case. +const RESERVED: &[&str] = &[ + "CON", "PRN", "AUX", "NUL", "COM1", "COM2", "COM3", "COM4", "COM5", "COM6", "COM7", "COM8", + "COM9", "LPT1", "LPT2", "LPT3", "LPT4", "LPT5", "LPT6", "LPT7", "LPT8", "LPT9", +]; + +/// Which rules a filesystem applies to names. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum NameRules { + /// NTFS, ReFS, FAT32 and exFAT, and SMB, whose protocol carries the + /// same limits whatever sits behind the share. + Windows, + /// APFS and HFS+. + Apple, + /// Everything else: a separator and NUL are the only forbidden bytes. + Posix, +} + +impl NameRules { + pub fn of(file_system: &FileSystem) -> Self { + match file_system { + FileSystem::NTFS + | FileSystem::ReFS + | FileSystem::FAT32 + | FileSystem::ExFAT + | FileSystem::SMB => Self::Windows, + FileSystem::APFS | FileSystem::HFSPlus => Self::Apple, + FileSystem::Ext4 + | FileSystem::Btrfs + | FileSystem::ZFS + | FileSystem::NFS + | FileSystem::Other(_) => match file_system { + FileSystem::Other(name) if name.eq_ignore_ascii_case("cifs") => Self::Windows, + FileSystem::Other(name) if name.eq_ignore_ascii_case("smbfs") => Self::Windows, + FileSystem::Other(name) if name.eq_ignore_ascii_case("msdos") => Self::Windows, + _ => Self::Posix, + }, + } + } + + /// Whether the filesystem matches names without regard to case by + /// default. A volume can be formatted otherwise, so a rename probes the + /// directory before trusting this. + pub fn case_insensitive(file_system: &FileSystem) -> bool { + matches!(Self::of(file_system), Self::Windows | Self::Apple) + } +} + +/// What no filesystem accepts. +pub fn check_portable(name: &str) -> Result<(), NameProblem> { + if name.is_empty() { + return Err(NameProblem::Empty); + } + if name.contains('/') || name.contains('\\') { + return Err(NameProblem::Separator); + } + if name == "." || name == ".." { + return Err(NameProblem::Dot); + } + if name.contains('\0') { + return Err(NameProblem::Character('\0')); + } + Ok(()) +} + +/// Whether the filesystem writes the name, after [`check_portable`]. +pub fn check_name(name: &str, rules: NameRules) -> Result<(), NameProblem> { + check_portable(name)?; + match rules { + NameRules::Windows => { + if let Some(bad) = name.chars().find(|c| { + matches!(c, '<' | '>' | ':' | '"' | '|' | '?' | '*') || (*c as u32) < 0x20 + }) { + return Err(NameProblem::Character(bad)); + } + let stem = Path::new(name) + .file_stem() + .and_then(|stem| stem.to_str()) + .unwrap_or(name); + if RESERVED + .iter() + .any(|reserved| stem.eq_ignore_ascii_case(reserved)) + { + return Err(NameProblem::Reserved(stem.to_uppercase())); + } + if name.ends_with(' ') || name.ends_with('.') { + return Err(NameProblem::Ending); + } + if name.encode_utf16().count() > 255 { + return Err(NameProblem::TooLong(255, "characters")); + } + } + NameRules::Apple => { + if name.contains(':') { + return Err(NameProblem::Character(':')); + } + if name.len() > 255 { + return Err(NameProblem::TooLong(255, "bytes")); + } + } + NameRules::Posix => { + if name.len() > 255 { + return Err(NameProblem::TooLong(255, "bytes")); + } + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_portable_check_refuses_what_no_filesystem_takes() { + assert_eq!(check_portable(""), Err(NameProblem::Empty)); + assert_eq!(check_portable("a/b"), Err(NameProblem::Separator)); + assert_eq!(check_portable("a\\b"), Err(NameProblem::Separator)); + assert_eq!(check_portable(".."), Err(NameProblem::Dot)); + assert_eq!(check_portable("a\0b"), Err(NameProblem::Character('\0'))); + assert_eq!(check_portable("日本語ファイル.txt"), Ok(())); + } + + /// Windows rules refuse the NTFS character set, device names and a + /// trailing dot; Apple rules refuse a colon; POSIX takes both. + #[test] + fn each_filesystem_refuses_its_own_names() { + assert_eq!( + check_name("a:b", NameRules::Windows), + Err(NameProblem::Character(':')) + ); + assert_eq!( + check_name("con.txt", NameRules::Windows), + Err(NameProblem::Reserved("CON".into())) + ); + assert_eq!( + check_name("done.", NameRules::Windows), + Err(NameProblem::Ending) + ); + assert_eq!( + check_name("a:b", NameRules::Apple), + Err(NameProblem::Character(':')) + ); + assert_eq!(check_name("con.txt", NameRules::Apple), Ok(())); + assert_eq!(check_name("a:b", NameRules::Posix), Ok(())); + assert_eq!(check_name("a?b", NameRules::Posix), Ok(())); + assert_eq!( + check_name(&"x".repeat(256), NameRules::Posix), + Err(NameProblem::TooLong(255, "bytes")) + ); + assert_eq!(check_name(&"é".repeat(200), NameRules::Windows), Ok(())); + assert_eq!( + check_name(&"é".repeat(200), NameRules::Apple), + Err(NameProblem::TooLong(255, "bytes")) + ); + } + + #[test] + fn rules_follow_the_filesystem() { + assert_eq!(NameRules::of(&FileSystem::NTFS), NameRules::Windows); + assert_eq!(NameRules::of(&FileSystem::SMB), NameRules::Windows); + assert_eq!( + NameRules::of(&FileSystem::Other("cifs".into())), + NameRules::Windows + ); + assert_eq!(NameRules::of(&FileSystem::APFS), NameRules::Apple); + assert_eq!(NameRules::of(&FileSystem::Ext4), NameRules::Posix); + assert!(NameRules::case_insensitive(&FileSystem::APFS)); + assert!(!NameRules::case_insensitive(&FileSystem::Ext4)); + } +} diff --git a/core/src/ops/files/rename/preflight.rs b/core/src/ops/files/rename/preflight.rs new file mode 100644 index 000000000..18d13484e --- /dev/null +++ b/core/src/ops/files/rename/preflight.rs @@ -0,0 +1,435 @@ +//! Whether a rename would run, and what would exist after. +//! +//! Both actions resolve their set the way the job does, so the findings +//! name exactly what the job would refuse: a name the filesystem does not +//! write, a file already at the new name, a name the directory cannot tell +//! from an existing one, and two targets wanting one name. The plan is one +//! move per changed name, at the new name, so the listing shows every new +//! name before anything moves, and a conflict where two targets collide. + +use super::{ + action::{FileRenameAction, FileRenameBatchAction}, + input::{FileRenameBatchInput, FileRenameInput}, + resolve::{resolve, RenameSet, Resolved, UNCHANGED}, +}; +use crate::{ + domain::SdPath, + infra::action::{ + error::ActionError, + preflight::{ + ExecutionFacts, Finding, PreviewContext, PreviewableAction, ValidatedAction, Validation, + }, + }, + ops::files::plan::{ + ChangeKind, ConflictKind, FsPlan, FsPlanSummary, PlanBasis, PlanChanges, PlannedChange, + StoreRevision, + }, +}; + +impl ValidatedAction for FileRenameAction { + async fn validate( + input: &FileRenameInput, + ctx: &PreviewContext, + ) -> Result { + validate( + &RenameSet::Named(vec![(input.target.clone(), input.new_name.clone())]), + ctx, + ) + .await + } +} + +impl PreviewableAction for FileRenameAction { + type Plan = FsPlan; + + async fn preview(input: FileRenameInput, ctx: &PreviewContext) -> Result { + preview(&RenameSet::Named(vec![(input.target, input.new_name)]), ctx).await + } +} + +impl ValidatedAction for FileRenameBatchAction { + async fn validate( + input: &FileRenameBatchInput, + ctx: &PreviewContext, + ) -> Result { + validate(&set_of(input), ctx).await + } +} + +impl PreviewableAction for FileRenameBatchAction { + type Plan = FsPlan; + + async fn preview( + input: FileRenameBatchInput, + ctx: &PreviewContext, + ) -> Result { + preview(&set_of(&input), ctx).await + } +} + +crate::register_validate!(FileRenameAction, "files.rename"); +crate::register_preview!(FileRenameAction, "files.rename"); +crate::register_validate!(FileRenameBatchAction, "files.rename_batch"); +crate::register_preview!(FileRenameBatchAction, "files.rename_batch"); + +fn set_of(input: &FileRenameBatchInput) -> RenameSet { + RenameSet::Ruled { + targets: input.targets.clone(), + rules: input.rules.clone(), + } +} + +async fn validate(set: &RenameSet, ctx: &PreviewContext) -> Result { + let Resolved { + renames, + mut findings, + unchanged, + } = resolve(ctx.volumes(), ctx.index(), set).await; + if unchanged > 0 { + findings.push(Finding::info( + UNCHANGED, + format!( + "{unchanged} of {} names {} unchanged", + unchanged + renames.len() as u64, + if unchanged == 1 { "is" } else { "are" } + ), + )); + } + Ok(Validation { + findings, + facts: ExecutionFacts { + executes_on: ctx.executes_on(), + strategy: Some("rename".to_string()), + estimated_files: Some(renames.iter().filter(|rename| !rename.refused).count() as u64), + estimated_bytes: Some(0), + free_space_after: None, + }, + }) +} + +async fn preview(set: &RenameSet, ctx: &PreviewContext) -> Result { + let resolved = resolve(ctx.volumes(), ctx.index(), set).await; + let device = crate::device::get_current_device_slug(); + let mut summary = FsPlanSummary::default(); + let mut changes = PlanChanges::default(); + let mut revisions: Vec = Vec::new(); + + for target in set.targets() { + for reach in ctx.reach(target).await { + if revisions + .iter() + .any(|known| known.source == reach.source.id) + { + continue; + } + if let Some(db) = ctx.index().read_store(reach.source.id).await { + if let Ok(revision) = db.revision().await { + revisions.push(StoreRevision { + source: reach.source.id, + revision: revision.value, + }); + } + } + } + } + + for rename in resolved.renames { + let change = if rename.collides { + ChangeKind::Conflict { + kind: ConflictKind::Sources, + } + } else { + ChangeKind::Move { + from: SdPath::Physical { + device_slug: device.clone(), + path: rename.from, + }, + } + }; + summary.count(&change, rename.size); + changes.push(PlannedChange { + path: SdPath::Physical { + device_slug: device.clone(), + path: rename.to, + }, + change, + }); + } + + let (changes, truncated) = changes.finish(); + Ok(ctx.plans().retain(FsPlan { + handle: None, + basis: PlanBasis::Index { revisions }, + roots: Vec::new(), + summary, + changes, + truncated, + })) +} + +#[cfg(test)] +mod tests { + use std::path::PathBuf; + use std::time::Duration; + + use sd_store::file::FileKind; + + use super::*; + use crate::{ + infra::job::journal::Effect, + ops::files::{ + fixture::{Fixture, T}, + rename::{ + job::RenameJob, + resolve::{CASE_COLLISION, CASE_ONLY, COLLISION, EXISTS, ILLEGAL_NAME}, + rules::{CaseRule, ExtensionCase, RenameRule}, + }, + }, + }; + + fn change_at(plan: &FsPlan, path: &std::path::Path) -> Option { + plan.changes + .iter() + .find(|change| change.path.path().map(PathBuf::as_path) == Some(path)) + .map(|change| change.change.clone()) + } + + async fn run(fixture: &Fixture, job: RenameJob) -> crate::infra::job::output::JobOutput { + let handle = fixture + .library + .jobs() + .dispatch(job) + .await + .expect("dispatched"); + tokio::time::timeout(Duration::from_secs(30), handle.wait()) + .await + .expect("in time") + .expect("completed") + } + + /// A rename to a name already there is refused, to a name the directory + /// folds onto another file is refused, and to its own name in another + /// case is a case-only rename the plan shows as one move. + #[tokio::test] + async fn a_single_rename_answers_every_code() { + let fixture = Fixture::new().await; + let file = FileKind::File; + let tree = [ + ("a.txt", file, 4, T, Some("a"), None, None), + ("b.txt", file, 3, T, Some("b"), None, None), + ]; + fixture.materialize(&fixture.source, &tree); + fixture.index(&fixture.source, &tree).await; + let a = SdPath::local(fixture.source.join("a.txt")); + let insensitive = tokio::fs::symlink_metadata(fixture.source.join("A.TXT")) + .await + .is_ok(); + + let taken = FileRenameInput::new(a.clone(), "b.txt"); + let validation = FileRenameAction::validate(&taken, &fixture.preview()) + .await + .expect("validated"); + assert!(validation.errors().any(|finding| finding.code == EXISTS)); + + let illegal = FileRenameInput::new(a.clone(), "a/b.txt"); + let validation = FileRenameAction::validate(&illegal, &fixture.preview()) + .await + .expect("validated"); + assert!(validation + .errors() + .any(|finding| finding.code == ILLEGAL_NAME)); + + let cased = FileRenameInput::new(a.clone(), "A.txt"); + let validation = FileRenameAction::validate(&cased, &fixture.preview()) + .await + .expect("validated"); + assert!(!validation.refuses(), "{:?}", validation.findings); + assert_eq!( + validation + .findings + .iter() + .any(|finding| finding.code == CASE_ONLY), + insensitive + ); + let plan = FileRenameAction::preview(cased.clone(), &fixture.preview()) + .await + .expect("planned"); + assert_eq!( + change_at(&plan, &fixture.source.join("A.txt")), + Some(ChangeKind::Move { from: a.clone() }) + ); + assert_eq!(plan.summary.moves.files, 1); + + if insensitive { + let folded = FileRenameInput::new(a.clone(), "B.TXT"); + let validation = FileRenameAction::validate(&folded, &fixture.preview()) + .await + .expect("validated"); + assert!(validation + .errors() + .any(|finding| finding.code == CASE_COLLISION)); + } + + run(&fixture, RenameJob::named(a, "A.txt".to_string())).await; + assert!(fixture.source.join("A.txt").exists()); + let listed: Vec = std::fs::read_dir(&fixture.source) + .expect("dir") + .map(|entry| { + entry + .expect("entry") + .file_name() + .to_string_lossy() + .into_owned() + }) + .collect(); + assert!(listed.contains(&"A.txt".to_string()), "{listed:?}"); + } + + /// Rules preview each new name, two targets wanting one name are a + /// conflict, and the job renames a chain and a cycle without + /// overwriting, journaling each move. + #[tokio::test] + async fn a_batch_previews_collisions_and_renames_chains_and_cycles() { + let fixture = Fixture::new().await; + let file = FileKind::File; + let tree = [ + ("IMG_1.JPG", file, 4, T, Some("1"), None, None), + ("IMG_2.JPG", file, 3, T, Some("2"), None, None), + ("IMG_3.JPG", file, 2, T, Some("3"), None, None), + ]; + fixture.materialize(&fixture.source, &tree); + fixture.index(&fixture.source, &tree).await; + let targets: Vec = ["IMG_1.JPG", "IMG_2.JPG", "IMG_3.JPG"] + .iter() + .map(|name| SdPath::local(fixture.source.join(name))) + .collect(); + + // Every target wants the same name. + let collision = FileRenameBatchInput { + targets: targets.clone(), + rules: vec![RenameRule::Template { + pattern: "same{ext}".into(), + }], + }; + let validation = FileRenameBatchAction::validate(&collision, &fixture.preview()) + .await + .expect("validated"); + assert_eq!( + validation + .errors() + .filter(|finding| finding.code == COLLISION) + .count(), + 2 + ); + let plan = FileRenameBatchAction::preview(collision, &fixture.preview()) + .await + .expect("planned"); + assert_eq!(plan.summary.conflicts, 2); + assert_eq!(plan.summary.moves.files, 1); + + // A chain: 1 -> 2 -> 3 -> 4, each new name held by the next target + // until it moves, then lowercased extensions. + let chain = FileRenameBatchInput { + targets: targets.clone(), + rules: vec![ + RenameRule::Sequence { + pattern: "IMG_{n}".into(), + start: 2, + step: 1, + }, + RenameRule::Case { + stem: CaseRule::Upper, + extension: ExtensionCase::Lower, + }, + ], + }; + let validation = FileRenameBatchAction::validate(&chain, &fixture.preview()) + .await + .expect("validated"); + assert!(!validation.refuses(), "{:?}", validation.findings); + let plan = FileRenameBatchAction::preview(chain.clone(), &fixture.preview()) + .await + .expect("planned"); + assert_eq!( + change_at(&plan, &fixture.source.join("IMG_2.jpg")), + Some(ChangeKind::Move { + from: targets[0].clone() + }) + ); + assert_eq!(plan.summary.moves.files, 3); + + let output = run(&fixture, RenameJob::ruled(chain.targets, chain.rules)).await; + let crate::infra::job::output::JobOutput::Custom(value) = output else { + panic!("not a rename output: {output:?}"); + }; + assert_eq!(value["renamed"], 3); + assert_eq!(value["failed"].as_array().map(Vec::len), Some(0)); + assert_eq!( + std::fs::read_to_string(fixture.source.join("IMG_2.jpg")).expect("moved"), + "xxxx" + ); + assert_eq!( + std::fs::read_to_string(fixture.source.join("IMG_4.jpg")).expect("moved"), + "xx" + ); + assert!(!fixture.source.join("IMG_1.JPG").exists()); + + // A cycle: 2 <-> 3 swap names. + let swap = RenameJob::named_many(vec![ + ( + SdPath::local(fixture.source.join("IMG_2.jpg")), + "IMG_3.jpg".to_string(), + ), + ( + SdPath::local(fixture.source.join("IMG_3.jpg")), + "IMG_2.jpg".to_string(), + ), + ]); + let handle = fixture + .library + .jobs() + .dispatch(swap) + .await + .expect("dispatched"); + let job_id = handle.id(); + tokio::time::timeout(Duration::from_secs(30), handle.wait()) + .await + .expect("in time") + .expect("completed"); + assert_eq!( + std::fs::read_to_string(fixture.source.join("IMG_3.jpg")).expect("swapped"), + "xxxx" + ); + assert_eq!( + std::fs::read_to_string(fixture.source.join("IMG_2.jpg")).expect("swapped"), + "xxx" + ); + let parked: Vec<_> = std::fs::read_dir(&fixture.source) + .expect("dir") + .filter_map(|entry| entry.ok()) + .filter(|entry| entry.file_name().to_string_lossy().contains("renaming")) + .collect(); + assert!(parked.is_empty(), "no temporary name is left behind"); + + let journal = fixture + .library + .jobs() + .database() + .journal(job_id) + .await + .expect("journal"); + let moves: Vec<(PathBuf, PathBuf)> = journal + .iter() + .filter_map(|recorded| match &recorded.effect { + Effect::Moved { from, to, .. } => Some((from.clone(), to.clone())), + _ => None, + }) + .collect(); + assert_eq!(moves.len(), 2); + assert!(moves.iter().all(|(from, _)| !from + .file_name() + .unwrap() + .to_string_lossy() + .contains("renaming"))); + } +} diff --git a/core/src/ops/files/rename/resolve.rs b/core/src/ops/files/rename/resolve.rs new file mode 100644 index 000000000..8ced54b64 --- /dev/null +++ b/core/src/ops/files/rename/resolve.rs @@ -0,0 +1,393 @@ +//! Each target's new name, checked against the directory it sits in. +//! +//! Preflight and the job resolve the same way, so the job refuses exactly +//! what validation would have. Every check reads the live directory once +//! per parent: the sibling names, and whether the directory matches names +//! without regard to case, which is probed by asking for a sibling's name +//! with its case flipped rather than trusted from the filesystem's name, +//! since a volume can be formatted either way. + +use std::{ + collections::{HashMap, HashSet}, + path::{Path, PathBuf}, +}; + +use chrono::{DateTime, Local}; + +use super::{ + naming::{check_name, check_portable, NameRules}, + rules::{apply, RenameRule, Target}, +}; +use crate::{ + domain::SdPath, + infra::action::preflight::Finding, + ops::{indexing::VolumeIndex, paths::reach::stores_beneath_in}, + volume::VolumeManager, +}; + +pub const REMOTE_ROOT: &str = "rename.remote_root"; +pub const MISSING: &str = "rename.missing"; +pub const ILLEGAL_NAME: &str = "rename.illegal_name"; +pub const EXISTS: &str = "rename.exists"; +pub const CASE_COLLISION: &str = "rename.case_collision"; +pub const CASE_ONLY: &str = "rename.case_only"; +pub const COLLISION: &str = "rename.collision"; +pub const UNCHANGED: &str = "rename.unchanged"; +pub const RULE: &str = "rename.rule"; + +/// What is being renamed: names given one by one, or targets and the +/// rules that name them. +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +pub enum RenameSet { + Named(Vec<(SdPath, String)>), + Ruled { + targets: Vec, + rules: Vec, + }, +} + +impl RenameSet { + pub fn targets(&self) -> Vec<&SdPath> { + match self { + Self::Named(pairs) => pairs.iter().map(|(target, _)| target).collect(), + Self::Ruled { targets, .. } => targets.iter().collect(), + } + } +} + +/// One rename the set resolves to. +#[derive(Debug, Clone)] +pub struct Rename { + pub from: PathBuf, + pub to: PathBuf, + pub is_dir: bool, + pub size: u64, + /// The new name differs from the old in case alone, on a directory that + /// matches names without regard to it. + pub case_only: bool, + /// Whether a finding refuses this rename. + pub refused: bool, + /// Another target of the set wants the same name. + pub collides: bool, +} + +pub struct Resolved { + pub renames: Vec, + pub findings: Vec, + pub unchanged: u64, +} + +/// Resolve the set against the live filesystem. +pub async fn resolve(volumes: &VolumeManager, index: &VolumeIndex, set: &RenameSet) -> Resolved { + let mut findings = Vec::new(); + let mut unchanged = 0; + + // Each target's new name, before any check. + let mut wanted: Vec<(SdPath, PathBuf, std::fs::Metadata, String)> = Vec::new(); + let rules = match set { + RenameSet::Ruled { rules, .. } => Some(rules.as_slice()), + RenameSet::Named(_) => None, + }; + let wants_capture = rules.is_some_and(|rules| { + rules.iter().any(|rule| match rule { + RenameRule::Replace { with, .. } => with.contains("{captured"), + RenameRule::Affix { prefix, suffix } => { + prefix.contains("{captured") || suffix.contains("{captured") + } + RenameRule::Sequence { pattern, .. } | RenameRule::Template { pattern } => { + pattern.contains("{captured") + } + RenameRule::Case { .. } => false, + }) + }); + for (position, target) in set.targets().into_iter().enumerate() { + let Some(path) = target.as_local_path() else { + findings.push( + Finding::error( + REMOTE_ROOT, + "a file is on another device; rename it there with --device", + ) + .at(target.clone()), + ); + continue; + }; + let meta = match tokio::fs::symlink_metadata(path).await { + Ok(meta) => meta, + Err(_) => { + findings.push(Finding::error(MISSING, "a file is not there").at(target.clone())); + continue; + } + }; + let name = path + .file_name() + .map(|name| name.to_string_lossy().into_owned()) + .unwrap_or_default(); + let new_name = match set { + RenameSet::Named(pairs) => pairs[position].1.clone(), + RenameSet::Ruled { rules, .. } => { + let described = Target { + name: name.clone(), + parent: path + .parent() + .and_then(Path::file_name) + .map(|name| name.to_string_lossy().into_owned()) + .unwrap_or_default(), + index: position as u64, + modified: meta + .modified() + .map(DateTime::::from) + .unwrap_or_else(|_| Local::now()), + captured: if wants_capture { + captured_time(volumes, index, target).await + } else { + None + }, + }; + match apply(rules, &described) { + Ok(new_name) => new_name, + Err(error) => { + findings.push(Finding::error(RULE, error.to_string()).at(target.clone())); + continue; + } + } + } + }; + if new_name == name { + unchanged += 1; + continue; + } + wanted.push((target.clone(), path.to_path_buf(), meta, new_name)); + } + + // The directories the renames happen in, each read once. + let mut directories: HashMap = HashMap::new(); + for (_, path, _, _) in &wanted { + let Some(parent) = path.parent() else { + continue; + }; + if !directories.contains_key(parent) { + let directory = Directory::read(volumes, parent).await; + directories.insert(parent.to_path_buf(), directory); + } + } + // Names the set renames away from, per directory, by fold key. + let mut vacated: HashMap> = HashMap::new(); + for (_, path, _, _) in &wanted { + if let (Some(parent), Some(name)) = (path.parent(), path.file_name()) { + let directory = &directories[parent]; + vacated + .entry(parent.to_path_buf()) + .or_default() + .insert(directory.fold(&name.to_string_lossy())); + } + } + // Names the set claims, per directory, by fold key. + let mut claimed: HashMap> = HashMap::new(); + + let mut renames = Vec::with_capacity(wanted.len()); + for (target, path, meta, new_name) in wanted { + let Some(parent) = path.parent() else { + continue; + }; + let directory = &directories[parent]; + let name = path + .file_name() + .map(|name| name.to_string_lossy().into_owned()) + .unwrap_or_default(); + let mut refused = false; + let mut collides = false; + let mut case_only = false; + + if let Err(problem) = + check_portable(&new_name).and_then(|()| check_name(&new_name, directory.rules)) + { + findings.push(Finding::error(ILLEGAL_NAME, problem.to_string()).at(target.clone())); + refused = true; + } + + let key = directory.fold(&new_name); + if !claimed + .entry(parent.to_path_buf()) + .or_default() + .insert(key.clone()) + { + findings.push( + Finding::error(COLLISION, format!("two files want the name {new_name}")) + .at(target.clone()), + ); + refused = true; + collides = true; + } + + let vacated = vacated.get(parent); + let held = directory.siblings.get(&key); + match held { + Some(holder) if holder == &name => { + // Only a case-insensitive directory folds the new name onto + // the file itself. + case_only = true; + findings.push( + Finding::info(CASE_ONLY, format!("{name} changes only in case")) + .at(target.clone()), + ); + } + Some(holder) if vacated.is_some_and(|names| names.contains(&key)) => { + // The holder is renamed away by this set; the job orders it + // first. + let _ = holder; + } + Some(holder) if holder == &new_name => { + findings.push( + Finding::error(EXISTS, format!("a file named {new_name} is already there")) + .at(target.clone()), + ); + refused = true; + } + Some(holder) => { + findings.push( + Finding::error( + CASE_COLLISION, + format!( + "{holder} is already there, and this directory does not tell {new_name} from it" + ), + ) + .at(target.clone()), + ); + refused = true; + } + None => {} + } + + renames.push(Rename { + from: path.clone(), + to: parent.join(&new_name), + is_dir: meta.is_dir(), + size: if meta.is_dir() { 0 } else { meta.len() }, + case_only, + refused, + collides, + }); + } + + Resolved { + renames, + findings, + unchanged, + } +} + +/// One directory as the renames see it. +struct Directory { + /// Each sibling by its fold key. + siblings: HashMap, + case_insensitive: bool, + rules: NameRules, +} + +impl Directory { + async fn read(volumes: &VolumeManager, parent: &Path) -> Self { + let mut names = Vec::new(); + if let Ok(mut entries) = tokio::fs::read_dir(parent).await { + while let Ok(Some(entry)) = entries.next_entry().await { + names.push(entry.file_name().to_string_lossy().into_owned()); + } + } + let file_system = volumes + .volume_for_path(parent) + .await + .map(|volume| volume.file_system) + .unwrap_or_default(); + let case_insensitive = match probe_case(parent, &names).await { + Some(answer) => answer, + None => NameRules::case_insensitive(&file_system), + }; + let mut siblings = HashMap::new(); + for name in names { + let key = if case_insensitive { + name.to_lowercase() + } else { + name.clone() + }; + siblings.entry(key).or_insert(name); + } + Self { + siblings, + case_insensitive, + rules: NameRules::of(&file_system), + } + } + + /// The key two names collide on in this directory. + fn fold(&self, name: &str) -> String { + if self.case_insensitive { + name.to_lowercase() + } else { + name.to_string() + } + } +} + +/// Whether the directory matches names without regard to case: ask for a +/// sibling under its name with the case flipped, where that spelling is not +/// itself a sibling. `None` when no sibling has a letter to flip. +async fn probe_case(parent: &Path, names: &[String]) -> Option { + let present: HashSet<&String> = names.iter().collect(); + for name in names { + let flipped: String = name + .chars() + .map(|c| { + if c.is_lowercase() { + c.to_uppercase().next().unwrap_or(c) + } else if c.is_uppercase() { + c.to_lowercase().next().unwrap_or(c) + } else { + c + } + }) + .collect(); + if flipped == *name || present.contains(&flipped) { + continue; + } + return Some( + tokio::fs::symlink_metadata(parent.join(&flipped)) + .await + .is_ok(), + ); + } + None +} + +/// The capture time the store holds for a file: the image's date taken or +/// the video's date captured, where the media facet has been written. +pub(crate) async fn captured_time( + volumes: &VolumeManager, + index: &VolumeIndex, + target: &SdPath, +) -> Option> { + for reach in stores_beneath_in(volumes, index, target).await { + if !reach.prefix.is_empty() { + continue; + } + let db = index.read_store(reach.source.id).await?; + let entry = sd_store::read::entry_by_path(db.pool(), &reach.scope) + .await + .ok()??; + for (table, column) in [ + ("facet_image", "date_taken"), + ("facet_video", "date_captured"), + ] { + let sql = format!("SELECT {column} FROM {table} WHERE record_uuid = ?"); + let taken: Option> = sqlx::query_scalar(&sql) + .bind(entry.uuid) + .fetch_optional(db.pool()) + .await + .ok()?; + if let Some(Some(taken)) = taken { + if let Ok(taken) = DateTime::parse_from_rfc3339(&taken) { + return Some(taken.with_timezone(&Local)); + } + } + } + } + None +} diff --git a/core/src/ops/files/rename/rules.rs b/core/src/ops/files/rename/rules.rs new file mode 100644 index 000000000..03c2b038b --- /dev/null +++ b/core/src/ops/files/rename/rules.rs @@ -0,0 +1,386 @@ +//! Batch rename rules: an ordered list applied to each name. +//! +//! Each rule takes the name as the rules before it left it. Text a rule +//! introduces can hold tokens, expanded as the rule applies: `{name}` and +//! `{ext}` for the stem and the extension with its dot as they stand, +//! `{parent}` for the directory's name, `{n}` or `{n:04}` for the counter +//! of the target's position, `{date:%Y-%m-%d}` for the modification time, +//! and `{captured:%Y-%m-%d}` for the capture time the store holds, or the +//! modification time where it holds none. + +use std::path::Path; + +use chrono::{DateTime, Local}; +use regex::Regex; +use serde::{Deserialize, Serialize}; +use specta::Type; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Type)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum RenameRule { + /// Replace `find` with `with`, in the stem or the whole name. + Replace { + find: String, + with: String, + #[serde(default)] + regex: bool, + #[serde(default)] + whole_name: bool, + }, + Case { + stem: CaseRule, + #[serde(default)] + extension: ExtensionCase, + }, + Affix { + #[serde(default)] + prefix: String, + #[serde(default)] + suffix: String, + }, + /// The stem from a pattern holding `{n}`, counted from `start` in + /// steps of `step` over the targets in order. + Sequence { + pattern: String, + #[serde(default = "one")] + start: u64, + #[serde(default = "one")] + step: u64, + }, + /// The whole name from a pattern. + Template { pattern: String }, +} + +fn one() -> u64 { + 1 +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Type)] +#[serde(rename_all = "snake_case")] +pub enum CaseRule { + Lower, + Upper, + Title, + Keep, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize, Type)] +#[serde(rename_all = "snake_case")] +pub enum ExtensionCase { + Lower, + #[default] + Keep, +} + +/// What the tokens of one target expand to. +#[derive(Debug, Clone)] +pub struct Target { + pub name: String, + pub parent: String, + /// The target's position among the targets, from 0. + pub index: u64, + pub modified: DateTime, + pub captured: Option>, +} + +/// The name as the rules leave it. +#[derive(Debug, Clone, PartialEq, Eq)] +struct Name { + stem: String, + /// Without its dot. + extension: Option, +} + +impl Name { + fn parse(name: &str) -> Self { + let path = Path::new(name); + let stem = path + .file_stem() + .and_then(|stem| stem.to_str()) + .unwrap_or(name) + .to_string(); + let extension = path + .extension() + .and_then(|extension| extension.to_str()) + .map(str::to_string); + Self { stem, extension } + } + + fn whole(&self) -> String { + match &self.extension { + Some(extension) => format!("{}.{extension}", self.stem), + None => self.stem.clone(), + } + } + + fn dotted_extension(&self) -> String { + self.extension + .as_ref() + .map(|extension| format!(".{extension}")) + .unwrap_or_default() + } +} + +/// A rule that cannot apply: a regex that does not parse. +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +#[error("{0}")] +pub struct RuleError(pub String); + +/// Check every rule before any name is computed. +pub fn check_rules(rules: &[RenameRule]) -> Result<(), RuleError> { + for rule in rules { + if let RenameRule::Replace { + find, regex: true, .. + } = rule + { + Regex::new(find) + .map_err(|error| RuleError(format!("the pattern {find:?}: {error}")))?; + } + if let RenameRule::Sequence { step: 0, .. } = rule { + return Err(RuleError("a sequence cannot step by 0".to_string())); + } + } + Ok(()) +} + +/// The name the rules give one target. +pub fn apply(rules: &[RenameRule], target: &Target) -> Result { + let mut name = Name::parse(&target.name); + let mut counter = target.index + 1; + for rule in rules { + match rule { + RenameRule::Replace { + find, + with, + regex, + whole_name, + } => { + let with = expand(with, &name, target, counter); + let subject = if *whole_name { + name.whole() + } else { + name.stem.clone() + }; + let replaced = if *regex { + let pattern = Regex::new(find) + .map_err(|error| RuleError(format!("the pattern {find:?}: {error}")))?; + pattern.replace_all(&subject, with.as_str()).into_owned() + } else if find.is_empty() { + subject + } else { + subject.replace(find.as_str(), &with) + }; + if *whole_name { + name = Name::parse(&replaced); + } else { + name.stem = replaced; + } + } + RenameRule::Case { stem, extension } => { + name.stem = match stem { + CaseRule::Lower => name.stem.to_lowercase(), + CaseRule::Upper => name.stem.to_uppercase(), + CaseRule::Title => title_case(&name.stem), + CaseRule::Keep => name.stem, + }; + if *extension == ExtensionCase::Lower { + name.extension = name.extension.map(|extension| extension.to_lowercase()); + } + } + RenameRule::Affix { prefix, suffix } => { + let prefix = expand(prefix, &name, target, counter); + let suffix = expand(suffix, &name, target, counter); + name.stem = format!("{prefix}{}{suffix}", name.stem); + } + RenameRule::Sequence { + pattern, + start, + step, + } => { + counter = start + target.index * step; + name.stem = expand(pattern, &name, target, counter); + } + RenameRule::Template { pattern } => { + let whole = expand(pattern, &name, target, counter); + name = Name::parse(&whole); + } + } + } + Ok(name.whole()) +} + +/// Expand the tokens in `text` for the name as it stands. +fn expand(text: &str, name: &Name, target: &Target, counter: u64) -> String { + let mut out = String::with_capacity(text.len()); + let mut rest = text; + while let Some(open) = rest.find('{') { + out.push_str(&rest[..open]); + let Some(close) = rest[open..].find('}') else { + out.push_str(&rest[open..]); + return out; + }; + let token = &rest[open + 1..open + close]; + match token_value(token, name, target, counter) { + Some(value) => out.push_str(&value), + None => out.push_str(&rest[open..open + close + 1]), + } + rest = &rest[open + close + 1..]; + } + out.push_str(rest); + out +} + +fn token_value(token: &str, name: &Name, target: &Target, counter: u64) -> Option { + if token == "name" { + return Some(name.stem.clone()); + } + if token == "ext" { + return Some(name.dotted_extension()); + } + if token == "parent" { + return Some(target.parent.clone()); + } + if token == "n" { + return Some(counter.to_string()); + } + if let Some(pad) = token.strip_prefix("n:") { + let width: usize = pad.parse().ok()?; + return Some(format!("{counter:0width$}")); + } + if let Some(format) = token.strip_prefix("date:") { + return Some(target.modified.format(format).to_string()); + } + if let Some(format) = token.strip_prefix("captured:") { + return Some( + target + .captured + .unwrap_or(target.modified) + .format(format) + .to_string(), + ); + } + None +} + +/// The first letter of each word up, the rest down. +fn title_case(text: &str) -> String { + let mut out = String::with_capacity(text.len()); + let mut at_word_start = true; + for c in text.chars() { + if c.is_whitespace() || c == '_' || c == '-' { + out.push(c); + at_word_start = true; + } else if at_word_start { + out.extend(c.to_uppercase()); + at_word_start = false; + } else { + out.extend(c.to_lowercase()); + } + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + use chrono::TimeZone; + + fn target(name: &str, index: u64) -> Target { + Target { + name: name.to_string(), + parent: "Trip".to_string(), + index, + modified: Local.with_ymd_and_hms(2024, 5, 6, 7, 8, 9).unwrap(), + captured: None, + } + } + + #[test] + fn each_rule_shapes_the_name_in_order() { + let rules = [ + RenameRule::Replace { + find: "IMG_".into(), + with: String::new(), + regex: false, + whole_name: false, + }, + RenameRule::Case { + stem: CaseRule::Lower, + extension: ExtensionCase::Lower, + }, + RenameRule::Affix { + prefix: "{parent}-".into(), + suffix: "-{date:%Y}".into(), + }, + ]; + assert_eq!( + apply(&rules, &target("IMG_0041.JPG", 0)).unwrap(), + "Trip-0041-2024.jpg" + ); + } + + #[test] + fn a_sequence_counts_the_targets_and_a_template_names_the_whole() { + let sequence = [RenameRule::Sequence { + pattern: "IMG_{n:04}".into(), + start: 10, + step: 5, + }]; + assert_eq!( + apply(&sequence, &target("a.jpg", 0)).unwrap(), + "IMG_0010.jpg" + ); + assert_eq!( + apply(&sequence, &target("b.jpg", 2)).unwrap(), + "IMG_0020.jpg" + ); + + let template = [RenameRule::Template { + pattern: "{date:%Y-%m-%d} {name} ({n}){ext}".into(), + }]; + assert_eq!( + apply(&template, &target("clip.MOV", 3)).unwrap(), + "2024-05-06 clip (4).MOV" + ); + assert_eq!( + apply(&template, &target("README", 0)).unwrap(), + "2024-05-06 README (1)" + ); + } + + #[test] + fn a_regex_replace_captures_and_a_bad_one_is_refused() { + let rules = [RenameRule::Replace { + find: r"^(\d{4})-(\d{2})".into(), + with: "$2.$1".into(), + regex: true, + whole_name: false, + }]; + assert_eq!( + apply(&rules, &target("2024-05 trip.txt", 0)).unwrap(), + "05.2024 trip.txt" + ); + let broken = [RenameRule::Replace { + find: "(".into(), + with: String::new(), + regex: true, + whole_name: false, + }]; + assert!(check_rules(&broken).is_err()); + assert!(check_rules(&[RenameRule::Sequence { + pattern: "{n}".into(), + start: 1, + step: 0 + }]) + .is_err()); + } + + #[test] + fn title_case_and_an_unknown_token_are_left_readable() { + assert_eq!(title_case("my HOLIDAY_photos-2"), "My Holiday_Photos-2"); + let rules = [RenameRule::Template { + pattern: "{name}{what}".into(), + }]; + assert_eq!(apply(&rules, &target("x.txt", 0)).unwrap(), "x{what}"); + } +} diff --git a/core/src/ops/files/rename/validation.rs b/core/src/ops/files/rename/validation.rs deleted file mode 100644 index 1fa3f7045..000000000 --- a/core/src/ops/files/rename/validation.rs +++ /dev/null @@ -1,171 +0,0 @@ -//! Filename validation for rename operations - -use std::path::Path; -use thiserror::Error; - -/// Errors that can occur during filename validation -#[derive(Debug, Clone, Error)] -pub enum FilenameValidationError { - #[error("Filename cannot be empty")] - Empty, - - #[error("Filename cannot contain path separators (/ or \\)")] - ContainsPathSeparator, - - #[error("Filename cannot be '.' or '..'")] - InvalidDotName, - - #[error("Filename contains invalid character: {0}")] - InvalidCharacter(char), - - #[error("Filename is a Windows reserved name: {0}")] - WindowsReservedName(String), - - #[error("Filename cannot end with a space or period")] - InvalidEnding, - - #[error("Filename exceeds maximum length of {0} characters")] - TooLong(usize), -} - -/// Windows reserved device names (case-insensitive) -const WINDOWS_RESERVED_NAMES: &[&str] = &[ - "CON", "PRN", "AUX", "NUL", "COM1", "COM2", "COM3", "COM4", "COM5", "COM6", "COM7", "COM8", - "COM9", "LPT1", "LPT2", "LPT3", "LPT4", "LPT5", "LPT6", "LPT7", "LPT8", "LPT9", -]; - -/// Invalid characters for filenames across platforms -const INVALID_CHARS: &[char] = &['<', '>', ':', '"', '|', '?', '*', '\0']; - -/// Maximum filename length (conservative cross-platform limit) -const MAX_FILENAME_LENGTH: usize = 255; - -/// Validate a filename for use in rename operations -/// -/// Checks for: -/// - Empty names -/// - Path separators (/ or \) -/// - Invalid special names (. and ..) -/// - Invalid characters (platform-specific) -/// - Windows reserved names (on Windows) -/// - Invalid endings (space or period on Windows) -/// - Maximum length -pub fn validate_filename(name: &str) -> Result<(), FilenameValidationError> { - // Check for empty name - if name.is_empty() { - return Err(FilenameValidationError::Empty); - } - - // Check for path separators - if name.contains('/') || name.contains('\\') { - return Err(FilenameValidationError::ContainsPathSeparator); - } - - // Check for . and .. - if name == "." || name == ".." { - return Err(FilenameValidationError::InvalidDotName); - } - - // Check for invalid characters - for c in INVALID_CHARS { - if name.contains(*c) { - return Err(FilenameValidationError::InvalidCharacter(*c)); - } - } - - // Check maximum length - if name.len() > MAX_FILENAME_LENGTH { - return Err(FilenameValidationError::TooLong(MAX_FILENAME_LENGTH)); - } - - // Platform-specific validation - #[cfg(target_os = "windows")] - { - // Check for Windows reserved names - let name_upper = name.to_uppercase(); - let base_name = Path::new(&name_upper) - .file_stem() - .and_then(|s| s.to_str()) - .unwrap_or(&name_upper); - - if WINDOWS_RESERVED_NAMES.contains(&base_name) { - return Err(FilenameValidationError::WindowsReservedName( - base_name.to_string(), - )); - } - - // Check for invalid endings (Windows doesn't allow trailing space or period) - if name.ends_with(' ') || name.ends_with('.') { - return Err(FilenameValidationError::InvalidEnding); - } - } - - Ok(()) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_valid_filenames() { - assert!(validate_filename("file.txt").is_ok()); - assert!(validate_filename("my document.pdf").is_ok()); - assert!(validate_filename(".hidden").is_ok()); - assert!(validate_filename("file-name_123").is_ok()); - assert!(validate_filename("日本語ファイル.txt").is_ok()); - } - - #[test] - fn test_empty_filename() { - assert!(matches!( - validate_filename(""), - Err(FilenameValidationError::Empty) - )); - } - - #[test] - fn test_path_separators() { - assert!(matches!( - validate_filename("path/to/file"), - Err(FilenameValidationError::ContainsPathSeparator) - )); - assert!(matches!( - validate_filename("path\\to\\file"), - Err(FilenameValidationError::ContainsPathSeparator) - )); - } - - #[test] - fn test_dot_names() { - assert!(matches!( - validate_filename("."), - Err(FilenameValidationError::InvalidDotName) - )); - assert!(matches!( - validate_filename(".."), - Err(FilenameValidationError::InvalidDotName) - )); - } - - #[test] - fn test_invalid_characters() { - assert!(matches!( - validate_filename("file/`, reached by a rename on the same volume. +//! +//! Replacing a file goes through the same door: the previous bytes are +//! trashed and their location recorded, so an overwrite is undoable until +//! the trash is emptied and costs a rename rather than a copy. + +use std::{ + io, + path::{Path, PathBuf}, +}; + +use crate::{infra::job::types::JobId, volume::VolumeManager}; + +/// The directory beneath a volume's root that holds what Spacedrive trashed +/// there. +pub const SPACEDRIVE_TRASH: &str = ".spacedrive/trash"; + +/// Move `path` to the trash and answer with where it went, where the +/// platform says. +pub async fn trash( + path: &Path, + volumes: Option<&VolumeManager>, + job: JobId, +) -> io::Result> { + let owned = path.to_path_buf(); + let os = tokio::task::spawn_blocking(move || trash_os(&owned)) + .await + .map_err(|error| io::Error::other(error.to_string()))?; + match os { + Ok(location) => Ok(location), + Err(os_error) => match spacedrive_trash(path, volumes, job).await { + Ok(location) => Ok(Some(location)), + Err(_) => Err(os_error), + }, + } +} + +/// Put an item back where it was, from the location the trash reported. +pub async fn restore(location: &Path, original: &Path) -> io::Result<()> { + if tokio::fs::symlink_metadata(original).await.is_ok() { + return Err(io::Error::new( + io::ErrorKind::AlreadyExists, + format!("{} is already there", original.display()), + )); + } + if let Some(parent) = original.parent() { + tokio::fs::create_dir_all(parent).await?; + } + if tokio::fs::symlink_metadata(location).await.is_ok() { + return tokio::fs::rename(location, original).await; + } + let (location, original) = (location.to_path_buf(), original.to_path_buf()); + tokio::task::spawn_blocking(move || restore_os(&location, &original)) + .await + .map_err(|error| io::Error::other(error.to_string()))? +} + +/// Remove a trashed item for good. +pub async fn purge(location: &Path) -> io::Result<()> { + match tokio::fs::symlink_metadata(location).await { + Ok(meta) if meta.is_dir() => tokio::fs::remove_dir_all(location).await, + Ok(_) => tokio::fs::remove_file(location).await, + Err(_) => { + let location = location.to_path_buf(); + tokio::task::spawn_blocking(move || purge_os(&location)) + .await + .map_err(|error| io::Error::other(error.to_string()))? + } + } +} + +/// Whether a trash location is a Spacedrive trash directory rather than the +/// platform's. +pub fn is_spacedrive_trash(location: &Path) -> bool { + location + .components() + .collect::>() + .windows(2) + .any(|pair| pair[0].as_os_str() == ".spacedrive" && pair[1].as_os_str() == "trash") +} + +/// Remove every Spacedrive trash directory on the mounted volumes, and +/// answer with how many items went. +pub async fn empty_spacedrive_trash(volumes: &VolumeManager) -> io::Result { + let mut removed = 0; + for volume in volumes.get_all_volumes().await { + let trash = volume.mount_point.join(SPACEDRIVE_TRASH); + let Ok(mut jobs) = tokio::fs::read_dir(&trash).await else { + continue; + }; + while let Ok(Some(job)) = jobs.next_entry().await { + if let Ok(mut items) = tokio::fs::read_dir(job.path()).await { + while let Ok(Some(_)) = items.next_entry().await { + removed += 1; + } + } + } + tokio::fs::remove_dir_all(&trash).await?; + } + Ok(removed) +} + +/// Empty the platform's trash. +pub async fn empty_os_trash() -> io::Result<()> { + tokio::task::spawn_blocking(empty_os) + .await + .map_err(|error| io::Error::other(error.to_string()))? +} + +/// A rename into the Spacedrive trash directory at the root of the volume +/// holding `path`. +async fn spacedrive_trash( + path: &Path, + volumes: Option<&VolumeManager>, + job: JobId, +) -> io::Result { + let volume = match volumes { + Some(volumes) => volumes.volume_for_path(path).await, + None => None, + } + .ok_or_else(|| io::Error::other("no volume holds the path"))?; + if volume.is_read_only { + return Err(io::Error::other("the volume is read only")); + } + let directory = volume + .mount_point + .join(SPACEDRIVE_TRASH) + .join(job.to_string()); + tokio::fs::create_dir_all(&directory).await?; + let name = path + .file_name() + .map(|name| name.to_string_lossy().into_owned()) + .unwrap_or_else(|| "item".to_string()); + let mut location = directory.join(&name); + let mut counter = 1; + while tokio::fs::symlink_metadata(&location).await.is_ok() { + location = directory.join(format!("{name}.{counter}")); + counter += 1; + } + tokio::fs::rename(path, &location).await?; + Ok(location) +} + +#[cfg(target_os = "macos")] +fn trash_os(path: &Path) -> io::Result> { + use objc2::rc::Retained; + use objc2_foundation::{NSFileManager, NSString, NSURL}; + + let manager = NSFileManager::defaultManager(); + let url = NSURL::fileURLWithPath(&NSString::from_str(&path.to_string_lossy())); + let mut resulting: Option> = None; + manager + .trashItemAtURL_resultingItemURL_error(&url, Some(&mut resulting)) + .map_err(|error| io::Error::other(error.localizedDescription().to_string()))?; + Ok(resulting + .and_then(|url| url.path()) + .map(|path| PathBuf::from(path.to_string()))) +} + +#[cfg(target_os = "macos")] +fn restore_os(location: &Path, _original: &Path) -> io::Result<()> { + Err(io::Error::new( + io::ErrorKind::NotFound, + format!("{} is no longer in the trash", location.display()), + )) +} + +#[cfg(target_os = "macos")] +fn purge_os(location: &Path) -> io::Result<()> { + Err(io::Error::new( + io::ErrorKind::NotFound, + format!("{} is no longer in the trash", location.display()), + )) +} + +#[cfg(target_os = "macos")] +fn empty_os() -> io::Result<()> { + let status = std::process::Command::new("osascript") + .args(["-e", "tell application \"Finder\" to empty trash"]) + .status()?; + if status.success() { + Ok(()) + } else { + Err(io::Error::other("Finder did not empty the trash")) + } +} + +#[cfg(any(target_os = "windows", target_os = "linux"))] +fn trash_os(path: &Path) -> io::Result> { + trash::delete(path).map_err(|error| io::Error::other(error.to_string()))?; + let items = trash::os_limited::list().map_err(|error| io::Error::other(error.to_string()))?; + Ok(items + .into_iter() + .filter(|item| item.original_path() == path) + .max_by_key(|item| item.time_deleted) + .map(|item| PathBuf::from(item.id))) +} + +#[cfg(any(target_os = "windows", target_os = "linux"))] +fn trashed_item(location: &Path) -> io::Result { + trash::os_limited::list() + .map_err(|error| io::Error::other(error.to_string()))? + .into_iter() + .find(|item| Path::new(&item.id) == location) + .ok_or_else(|| { + io::Error::new( + io::ErrorKind::NotFound, + format!("{} is no longer in the trash", location.display()), + ) + }) +} + +#[cfg(any(target_os = "windows", target_os = "linux"))] +fn restore_os(location: &Path, _original: &Path) -> io::Result<()> { + let item = trashed_item(location)?; + trash::os_limited::restore_all([item]).map_err(|error| io::Error::other(error.to_string())) +} + +#[cfg(any(target_os = "windows", target_os = "linux"))] +fn purge_os(location: &Path) -> io::Result<()> { + let item = trashed_item(location)?; + trash::os_limited::purge_all([item]).map_err(|error| io::Error::other(error.to_string())) +} + +#[cfg(any(target_os = "windows", target_os = "linux"))] +fn empty_os() -> io::Result<()> { + let items = trash::os_limited::list().map_err(|error| io::Error::other(error.to_string()))?; + trash::os_limited::purge_all(items).map_err(|error| io::Error::other(error.to_string())) +} + +#[cfg(not(any(target_os = "windows", target_os = "linux", target_os = "macos")))] +fn trash_os(_path: &Path) -> io::Result> { + Err(io::Error::new( + io::ErrorKind::Unsupported, + "this platform has no trash", + )) +} + +#[cfg(not(any(target_os = "windows", target_os = "linux", target_os = "macos")))] +fn restore_os(_location: &Path, _original: &Path) -> io::Result<()> { + Err(io::Error::new( + io::ErrorKind::Unsupported, + "this platform has no trash", + )) +} + +#[cfg(not(any(target_os = "windows", target_os = "linux", target_os = "macos")))] +fn purge_os(_location: &Path) -> io::Result<()> { + Err(io::Error::new( + io::ErrorKind::Unsupported, + "this platform has no trash", + )) +} + +#[cfg(not(any(target_os = "windows", target_os = "linux", target_os = "macos")))] +fn empty_os() -> io::Result<()> { + Err(io::Error::new( + io::ErrorKind::Unsupported, + "this platform has no trash", + )) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_spacedrive_trash_location_is_told_apart() { + assert!(is_spacedrive_trash(Path::new( + "/Volumes/NAS/.spacedrive/trash/abc/file.txt" + ))); + assert!(!is_spacedrive_trash(Path::new("/Users/me/.Trash/file.txt"))); + } + + /// The macOS trash call reports where the item went, and the item comes + /// back from there. + #[cfg(target_os = "macos")] + #[tokio::test] + async fn the_trash_reports_where_an_item_went_and_gives_it_back() { + let dir = tempfile::tempdir().expect("tempdir"); + let file = dir.path().join("gone.txt"); + std::fs::write(&file, b"bytes").expect("file"); + let location = trash(&file, None, JobId::new()) + .await + .expect("trashed") + .expect("a location"); + assert!(!file.exists()); + assert!(location.exists(), "{}", location.display()); + restore(&location, &file).await.expect("restored"); + assert_eq!(std::fs::read(&file).expect("back"), b"bytes"); + } +} diff --git a/core/src/ops/files/trash_view/empty.rs b/core/src/ops/files/trash_view/empty.rs new file mode 100644 index 000000000..1c1f1b943 --- /dev/null +++ b/core/src/ops/files/trash_view/empty.rs @@ -0,0 +1,106 @@ +use std::sync::Arc; + +use serde::{Deserialize, Serialize}; +use specta::Type; +use uuid::Uuid; + +use crate::{ + context::CoreContext, + infra::{ + action::{error::ActionError, LibraryAction}, + job::{journal::Effect, types::JobId}, + }, + ops::files::trash, +}; + +/// Remove for good what the journals put in the trash, the Spacedrive +/// trash directories, and the platform's trash when asked. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Type)] +pub struct FileTrashEmptyInput { + /// Empty the platform's own trash as well, everything in it. + #[serde(default)] + pub os_trash: bool, +} + +#[derive(Debug, Clone, Default, Serialize, Deserialize, Type)] +pub struct TrashEmptyOutput { + /// Items the journals named that were removed. + pub purged: u64, + /// Items removed from Spacedrive trash directories beyond those. + pub spacedrive_trash: u64, + pub os_trash_emptied: bool, + pub failed: Vec, +} + +pub struct FileTrashEmptyAction { + input: FileTrashEmptyInput, +} + +impl LibraryAction for FileTrashEmptyAction { + type Input = FileTrashEmptyInput; + type Output = TrashEmptyOutput; + + fn from_input(input: Self::Input) -> Result { + Ok(Self { input }) + } + + async fn execute( + self, + library: Arc, + context: Arc, + ) -> Result { + let db = library.jobs().database(); + let trashed = db + .trashed() + .await + .map_err(|error| ActionError::Internal(error.to_string()))?; + let mut output = TrashEmptyOutput::default(); + let mut forget: std::collections::HashMap> = + std::collections::HashMap::new(); + for (job, recorded) in trashed { + let Effect::Trashed { + to: Some(location), .. + } = &recorded.effect + else { + continue; + }; + let Ok(job) = job.parse::() else { + continue; + }; + match trash::purge(location).await { + Ok(()) => { + output.purged += 1; + forget.entry(job).or_default().push(recorded.sequence); + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + forget.entry(job).or_default().push(recorded.sequence); + } + Err(error) => output + .failed + .push(format!("{}: {error}", location.display())), + } + } + for (job, sequences) in forget { + if let Err(error) = db.forget_trashed(JobId(job), &sequences).await { + output.failed.push(format!("journal of {job}: {error}")); + } + } + match trash::empty_spacedrive_trash(&context.volume_manager).await { + Ok(removed) => output.spacedrive_trash = removed, + Err(error) => output.failed.push(error.to_string()), + } + if self.input.os_trash { + match trash::empty_os_trash().await { + Ok(()) => output.os_trash_emptied = true, + Err(error) => output.failed.push(error.to_string()), + } + } + Ok(output) + } + + fn action_kind(&self) -> &'static str { + "files.trash_empty" + } +} + +crate::register_library_action!(FileTrashEmptyAction, "files.trash_empty"); diff --git a/core/src/ops/files/trash_view/list.rs b/core/src/ops/files/trash_view/list.rs new file mode 100644 index 000000000..b1422040d --- /dev/null +++ b/core/src/ops/files/trash_view/list.rs @@ -0,0 +1,116 @@ +use std::{path::PathBuf, sync::Arc}; + +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use specta::Type; +use uuid::Uuid; + +use crate::{ + context::CoreContext, + infra::{ + job::journal::Effect, + query::{LibraryQuery, QueryError, QueryResult}, + }, + ops::files::trash::is_spacedrive_trash, +}; + +#[derive(Debug, Clone, Serialize, Deserialize, Type)] +pub struct TrashListInput { + /// The most items to list; every item when absent. + pub limit: Option, +} + +/// One item a job put in the trash. +#[derive(Debug, Clone, Serialize, Deserialize, Type)] +pub struct TrashedItem { + pub job: Uuid, + pub sequence: i64, + /// Where it was. + pub from: PathBuf, + /// Where it is. + pub location: PathBuf, + pub trashed_at: DateTime, + pub size: u64, + pub is_dir: bool, + /// Whether the item is still at its location. + pub present: bool, + /// Whether it sits in a Spacedrive trash directory rather than the + /// platform's trash. + pub spacedrive_trash: bool, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Type)] +pub struct TrashListOutput { + pub items: Vec, +} + +pub struct TrashListQuery { + input: TrashListInput, +} + +impl LibraryQuery for TrashListQuery { + type Input = TrashListInput; + type Output = TrashListOutput; + + fn from_input(input: Self::Input) -> QueryResult { + Ok(Self { input }) + } + + async fn execute( + self, + context: Arc, + session: crate::infra::api::SessionContext, + ) -> QueryResult { + let library_id = session + .current_library_id + .ok_or_else(|| QueryError::Internal("No library selected".to_string()))?; + let library = context + .libraries() + .await + .get_library(library_id) + .await + .ok_or_else(|| QueryError::LibraryNotFound(library_id))?; + let trashed = library + .jobs() + .database() + .trashed() + .await + .map_err(|e| QueryError::Internal(e.to_string()))?; + let mut items = Vec::new(); + for (job, recorded) in trashed { + let Effect::Trashed { + from, + to: Some(location), + subject, + } = recorded.effect + else { + continue; + }; + let Ok(job) = job.parse::() else { + continue; + }; + let present = tokio::fs::symlink_metadata(&location).await.is_ok(); + items.push(TrashedItem { + job, + sequence: recorded.sequence, + spacedrive_trash: is_spacedrive_trash(&location), + from, + location, + trashed_at: recorded.recorded_at, + size: subject.map(|subject| subject.size).unwrap_or(0), + is_dir: subject.is_some_and(|subject| subject.is_dir), + present, + }); + if self + .input + .limit + .is_some_and(|limit| items.len() >= limit as usize) + { + break; + } + } + Ok(TrashListOutput { items }) + } +} + +crate::register_library_query!(TrashListQuery, "files.trash_list"); diff --git a/core/src/ops/files/trash_view/mod.rs b/core/src/ops/files/trash_view/mod.rs new file mode 100644 index 000000000..e2ab0dac2 --- /dev/null +++ b/core/src/ops/files/trash_view/mod.rs @@ -0,0 +1,13 @@ +//! # The trash +//! +//! What the journals put in the trash with a known location, newest first, +//! for restoring one item or all of a job's through `files.undo`, and +//! emptying: the items the journals name, the Spacedrive trash directories +//! on volumes without a trash of their own, and the platform's trash when +//! asked. + +mod empty; +mod list; + +pub use empty::{FileTrashEmptyAction, FileTrashEmptyInput, TrashEmptyOutput}; +pub use list::{TrashListInput, TrashListOutput, TrashListQuery, TrashedItem}; diff --git a/core/src/ops/files/undo/action.rs b/core/src/ops/files/undo/action.rs new file mode 100644 index 000000000..02151f910 --- /dev/null +++ b/core/src/ops/files/undo/action.rs @@ -0,0 +1,47 @@ +//! Undo action handler + +use std::sync::Arc; + +use super::{input::FileUndoInput, job::UndoJob}; +use crate::{ + context::CoreContext, + infra::{ + action::{error::ActionError, LibraryAction}, + job::handle::JobReceipt, + }, +}; + +pub struct FileUndoAction { + input: FileUndoInput, +} + +impl LibraryAction for FileUndoAction { + type Input = FileUndoInput; + type Output = JobReceipt; + + fn from_input(input: Self::Input) -> Result { + if input.effects.as_ref().is_some_and(Vec::is_empty) { + return Err("name at least one effect, or none for the whole job".to_string()); + } + Ok(Self { input }) + } + + async fn execute( + self, + library: Arc, + _context: Arc, + ) -> Result { + let job_handle = library + .jobs() + .dispatch(UndoJob::new(self.input.job, self.input.effects)) + .await + .map_err(ActionError::Job)?; + Ok(job_handle.into()) + } + + fn action_kind(&self) -> &'static str { + "files.undo" + } +} + +crate::register_library_action!(FileUndoAction, "files.undo"); diff --git a/core/src/ops/files/undo/input.rs b/core/src/ops/files/undo/input.rs new file mode 100644 index 000000000..a29385497 --- /dev/null +++ b/core/src/ops/files/undo/input.rs @@ -0,0 +1,15 @@ +//! Input for undoing a job + +use serde::{Deserialize, Serialize}; +use specta::Type; +use uuid::Uuid; + +/// Reverse what a job did, from its journal. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Type)] +pub struct FileUndoInput { + pub job: Uuid, + /// The effects to reverse, by sequence; every effect when absent. The + /// trash view restores one item this way. + #[serde(default)] + pub effects: Option>, +} diff --git a/core/src/ops/files/undo/job.rs b/core/src/ops/files/undo/job.rs new file mode 100644 index 000000000..6dd36db3e --- /dev/null +++ b/core/src/ops/files/undo/job.rs @@ -0,0 +1,234 @@ +//! Undoing a job, effect by effect, newest first. +//! +//! The journal is reversed as it stands when the undo runs: each effect's +//! subject is checked again, so a file that changed between the preview and +//! the run is left alone and reported. The undo writes its own journal, so +//! undoing an undo is the same action over it. + +use std::{path::PathBuf, time::Instant}; + +use serde::{Deserialize, Serialize}; +use specta::Type; +use uuid::Uuid; + +use super::reverse::{reversals, Left, Step}; +use crate::infra::job::{ + generic_progress::GenericProgress, + journal::{Attributes, Effect}, + prelude::*, + types::JobId, +}; +use crate::ops::files::trash; + +#[derive(Debug, Serialize, Deserialize, Job)] +pub struct UndoJob { + pub job: Uuid, + pub effects: Option>, + /// Sequences already reversed, for a resumed job. + pub done: Vec, + #[serde(skip, default = "Instant::now")] + started_at: Instant, +} + +impl UndoJob { + pub fn new(job: Uuid, effects: Option>) -> Self { + Self { + job, + effects, + done: Vec::new(), + started_at: Instant::now(), + } + } +} + +impl Job for UndoJob { + const NAME: &'static str = "undo"; + const RESUMABLE: bool = true; + const DESCRIPTION: Option<&'static str> = Some("Undo what a job did"); +} + +impl crate::infra::job::traits::DynJob for UndoJob { + fn job_name(&self) -> &'static str { + Self::NAME + } + + fn dedup_key(&self) -> Option { + Some(self.job.to_string()) + } +} + +#[derive(Debug, Clone, Default, Serialize, Deserialize, Type)] +pub struct UndoOutput { + pub reversed: u64, + pub left: Vec, + pub failed: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Type)] +pub struct UndoLeft { + pub sequence: i64, + pub reason: String, +} + +impl From for JobOutput { + fn from(output: UndoOutput) -> Self { + JobOutput::custom(output) + } +} + +#[derive(Debug, Serialize, Deserialize)] +struct Resume { + done: Vec, +} + +#[async_trait::async_trait] +impl JobHandler for UndoJob { + type Output = UndoOutput; + + async fn run(&mut self, ctx: JobContext<'_>) -> JobResult { + if let Some(resume) = ctx.load_state::().await? { + self.done = resume.done; + } + let journal = ctx + .library() + .jobs() + .database() + .journal(JobId(self.job)) + .await?; + ctx.log(format!( + "Undoing job {}: {} effects recorded", + self.job, + journal.len() + )); + let steps = reversals(&journal, self.effects.as_deref()).await; + let total = steps.len() as u64; + let mut output = UndoOutput::default(); + let mut handled = 0u64; + + for reversal in steps { + ctx.check_interrupt().await?; + handled += 1; + if self.done.contains(&reversal.sequence) { + output.reversed += 1; + continue; + } + let step = match reversal.outcome { + Ok(step) => step, + Err(left) => { + let reason = match left { + Left::Irreversible => "cannot be reversed", + Left::Changed => "changed since the job ran", + Left::Occupied => "something else is where it would go back to", + }; + output.left.push(UndoLeft { + sequence: reversal.sequence, + reason: reason.to_string(), + }); + continue; + } + }; + match apply(&ctx, step).await { + Ok(effects) => { + ctx.record(effects).await; + output.reversed += 1; + self.done.push(reversal.sequence); + ctx.checkpoint_with_state(&Resume { + done: self.done.clone(), + }) + .await?; + } + Err(error) => { + ctx.log(format!("Effect {}: {error}", reversal.sequence)); + output.failed.push(UndoLeft { + sequence: reversal.sequence, + reason: error.to_string(), + }); + } + } + ctx.progress(Progress::generic( + GenericProgress::new( + (handled as f32 / total.max(1) as f32).min(1.0), + "Undoing", + format!("{handled} of {total} effects"), + ) + .with_completion(handled, total) + .with_errors(output.failed.len() as u64, output.left.len() as u64), + )); + } + + ctx.progress(Progress::generic( + GenericProgress::new( + 1.0, + "Complete", + format!("{} reversed, {} left", output.reversed, output.left.len()), + ) + .with_completion(total, total) + .with_performance(0.0, None, Some(self.started_at.elapsed())) + .with_errors(output.failed.len() as u64, output.left.len() as u64), + )); + ctx.log(format!( + "Undo completed: {} reversed, {} left, {} failed", + output.reversed, + output.left.len(), + output.failed.len() + )); + Ok(output) + } +} + +/// Apply one step, answering with what it did for this job's journal. +async fn apply(ctx: &JobContext<'_>, step: Step) -> std::io::Result> { + match step { + Step::Trash { path, .. } => { + let location = trash::trash(&path, ctx.volume_manager().as_deref(), ctx.id()).await?; + let subject = match &location { + Some(location) => tokio::fs::symlink_metadata(location).await.ok(), + None => None, + }; + Ok(vec![Effect::trashed(path, location, subject.as_ref())]) + } + Step::MoveBack { from, to, .. } => { + if let Some(parent) = to.parent() { + tokio::fs::create_dir_all(parent).await?; + } + move_back(&from, &to).await?; + let subject = tokio::fs::symlink_metadata(&to).await.ok(); + Ok(vec![Effect::moved(from, to, subject.as_ref())]) + } + Step::Restore { path, previous, .. } => { + let location = trash::trash(&path, ctx.volume_manager().as_deref(), ctx.id()).await?; + let stashed = match &location { + Some(location) => tokio::fs::symlink_metadata(location).await.ok(), + None => None, + }; + move_back(&previous, &path).await?; + let subject = tokio::fs::symlink_metadata(&path).await.ok(); + Ok(vec![ + Effect::trashed(path.clone(), location, stashed.as_ref()), + Effect::moved(previous, path, subject.as_ref()), + ]) + } + Step::Attributes { path, to } => { + let from = crate::ops::files::attributes_action::fs::read(&path).await?; + crate::ops::files::attributes_action::fs::apply(&path, &to).await?; + Ok(vec![Effect::Attributes { + path, + from, + to: Attributes { + mode: to.mode, + modified_ms: to.modified_ms, + hidden: to.hidden, + }, + }]) + } + } +} + +/// A rename where both sit on one volume, and the trash's restore where +/// the item is in the platform's trash. +async fn move_back(from: &PathBuf, to: &PathBuf) -> std::io::Result<()> { + match tokio::fs::rename(from, to).await { + Ok(()) => Ok(()), + Err(_) => trash::restore(from, to).await, + } +} diff --git a/core/src/ops/files/undo/mod.rs b/core/src/ops/files/undo/mod.rs new file mode 100644 index 000000000..28a57c137 --- /dev/null +++ b/core/src/ops/files/undo/mod.rs @@ -0,0 +1,18 @@ +//! # Undo +//! +//! Reverse what a job did, from its journal, on preflight: validation says +//! what cannot be reversed and what changed since, the preview is the +//! reverse as an `FsPlan`, and the job applies it newest effect first, +//! writing its own journal so an undo can itself be undone. + +pub mod action; +pub mod input; +pub mod job; +mod preflight; +pub mod reverse; +#[cfg(test)] +mod tests; + +pub use action::FileUndoAction; +pub use input::FileUndoInput; +pub use job::{UndoJob, UndoOutput}; diff --git a/core/src/ops/files/undo/preflight.rs b/core/src/ops/files/undo/preflight.rs new file mode 100644 index 000000000..dca2bfec8 --- /dev/null +++ b/core/src/ops/files/undo/preflight.rs @@ -0,0 +1,201 @@ +//! Whether a job can be undone, and what undoing it would leave. +//! +//! Validation refuses a job with no journal or one still running, counts +//! what cannot be reversed, and names each effect whose subject changed +//! since. The plan is the reverse as an `FsPlan`: a delete for each +//! creation, a move back for each move and each trashing with a known +//! location, and a replacement's previous bytes moved back over it. + +use super::{ + action::FileUndoAction, + input::FileUndoInput, + reverse::{reversals, Left, Step}, +}; +use crate::{ + domain::SdPath, + infra::{ + action::{ + error::ActionError, + preflight::{ + ExecutionFacts, Finding, PreviewContext, PreviewableAction, ValidatedAction, + Validation, + }, + }, + job::{journal::Recorded, types::JobId}, + }, + ops::files::plan::{ + ChangeKind, FsPlan, FsPlanSummary, PlanBasis, PlanChanges, PlannedChange, ReplaceReason, + }, +}; + +pub const NO_JOURNAL: &str = "undo.no_journal"; +pub const RUNNING: &str = "undo.running"; +pub const CHANGED: &str = "undo.changed"; +pub const OCCUPIED: &str = "undo.occupied"; +pub const IRREVERSIBLE: &str = "undo.irreversible"; +pub const NOTHING: &str = "undo.nothing"; + +impl ValidatedAction for FileUndoAction { + async fn validate( + input: &FileUndoInput, + ctx: &PreviewContext, + ) -> Result { + let mut findings = Vec::new(); + let mut facts = ExecutionFacts { + executes_on: ctx.executes_on(), + strategy: Some("undo".to_string()), + ..Default::default() + }; + let jobs = ctx.library().jobs(); + if let Some(info) = jobs + .get_job_info(input.job) + .await + .map_err(|error| ActionError::Internal(error.to_string()))? + { + if !info.status.is_terminal() { + findings.push(Finding::error( + RUNNING, + "the job is still running; undo it once it has finished", + )); + } + } + let journal = journal_of(ctx, input.job).await?; + if journal.is_empty() { + findings.push(Finding::error( + NO_JOURNAL, + "the job recorded nothing that can be undone", + )); + return Ok(Validation { findings, facts }); + } + + let mut reversible = 0u64; + let mut irreversible = 0u64; + for reversal in reversals(&journal, input.effects.as_deref()).await { + match reversal.outcome { + Ok(_) => reversible += 1, + Err(Left::Irreversible) => irreversible += 1, + Err(Left::Changed) => { + let effect = journal + .iter() + .find(|recorded| recorded.sequence == reversal.sequence) + .map(|recorded| &recorded.effect); + let mut finding = Finding::warning( + CHANGED, + "a file changed since the job ran; it is left as it is", + ); + if let Some(path) = effect.and_then(|effect| effect.result()) { + finding = finding.at(SdPath::local(path)); + } + findings.push(finding); + } + Err(Left::Occupied) => { + let effect = journal + .iter() + .find(|recorded| recorded.sequence == reversal.sequence) + .map(|recorded| &recorded.effect); + let mut finding = Finding::warning( + OCCUPIED, + "something now sits where a file would go back to; it is left as it is", + ); + if let Some(path) = effect.and_then(|effect| match effect { + crate::infra::job::journal::Effect::Moved { from, .. } + | crate::infra::job::journal::Effect::Trashed { from, .. } => Some(from), + _ => None, + }) { + finding = finding.at(SdPath::local(path)); + } + findings.push(finding); + } + } + } + if irreversible > 0 { + findings.push(Finding::info( + IRREVERSIBLE, + format!( + "{irreversible} of the job's effects cannot be reversed: permanent removals, or replacements and trashings whose previous bytes were not kept" + ), + )); + } + if reversible == 0 { + findings.push(Finding::error( + NOTHING, + "nothing the job did can be reversed now", + )); + } + facts.estimated_files = Some(reversible); + Ok(Validation { findings, facts }) + } +} + +impl PreviewableAction for FileUndoAction { + type Plan = FsPlan; + + async fn preview(input: FileUndoInput, ctx: &PreviewContext) -> Result { + let journal = journal_of(ctx, input.job).await?; + let device = crate::device::get_current_device_slug(); + let at = |path: &std::path::Path| SdPath::Physical { + device_slug: device.clone(), + path: path.to_path_buf(), + }; + let mut summary = FsPlanSummary::default(); + let mut changes = PlanChanges::default(); + for reversal in reversals(&journal, input.effects.as_deref()).await { + let Ok(step) = reversal.outcome else { + continue; + }; + let (path, change, bytes) = match step { + Step::Trash { path, size, .. } => { + (at(&path), ChangeKind::Delete { last_copy: false }, size) + } + Step::MoveBack { from, to, size } => { + (at(&to), ChangeKind::Move { from: at(&from) }, size) + } + Step::Restore { + path, + previous, + size, + } => { + let incoming_size = tokio::fs::symlink_metadata(&previous) + .await + .map(|meta| meta.len()) + .unwrap_or(0); + ( + at(&path), + ChangeKind::Replace { + existing_size: size, + incoming_size, + reason: ReplaceReason::Overwrite, + }, + incoming_size, + ) + } + Step::Attributes { path, to } => { + (at(&path), ChangeKind::SetAttributes { attributes: to }, 0) + } + }; + summary.count(&change, bytes); + changes.push(PlannedChange { path, change }); + } + let (changes, truncated) = changes.finish(); + Ok(ctx.plans().retain(FsPlan { + handle: None, + basis: PlanBasis::Journal { job: input.job }, + roots: Vec::new(), + summary, + changes, + truncated, + })) + } +} + +crate::register_validate!(FileUndoAction, "files.undo"); +crate::register_preview!(FileUndoAction, "files.undo"); + +async fn journal_of(ctx: &PreviewContext, job: uuid::Uuid) -> Result, ActionError> { + ctx.library() + .jobs() + .database() + .journal(JobId(job)) + .await + .map_err(|error| ActionError::Internal(error.to_string())) +} diff --git a/core/src/ops/files/undo/reverse.rs b/core/src/ops/files/undo/reverse.rs new file mode 100644 index 000000000..e2688576b --- /dev/null +++ b/core/src/ops/files/undo/reverse.rs @@ -0,0 +1,157 @@ +//! What reversing a journal comes to, effect by effect. +//! +//! Effects reverse in the order opposite to the one they happened in. A +//! created path is trashed; a moved path goes back; a trashed item is put +//! back from where the trash reported it went; a replaced file gives way to +//! its previous bytes, kept in the trash; attributes go back to what they +//! were. A removal cannot be reversed, and neither can a trashing or a +//! replacement whose previous bytes were not kept. Before an effect is +//! reversed its subject is checked against what the journal recorded, so +//! undo never touches a file that changed since. + +use std::path::{Path, PathBuf}; + +use crate::infra::job::journal::{Effect, Recorded}; + +/// One step of an undo, as the plan and the job see it. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Step { + /// Trash what was created. + Trash { + path: PathBuf, + is_dir: bool, + size: u64, + }, + /// Put a moved or trashed item back. + MoveBack { + from: PathBuf, + to: PathBuf, + size: u64, + }, + /// Trash the replacement and put the previous bytes back. + Restore { + path: PathBuf, + previous: PathBuf, + size: u64, + }, + /// Set attributes back. + Attributes { + path: PathBuf, + to: crate::infra::job::journal::Attributes, + }, +} + +/// Why an effect stays as it is. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Left { + /// Nothing recorded can reverse it. + Irreversible, + /// The subject is not what the journal recorded, or is gone. + Changed, + /// Something else now sits where the item would go back to. + Occupied, +} + +/// What reversing one effect comes to. +pub struct Reversal { + pub sequence: i64, + pub outcome: Result, +} + +/// Each effect's reversal, newest first, checked against the filesystem. +pub async fn reversals(journal: &[Recorded], only: Option<&[i64]>) -> Vec { + let mut out = Vec::new(); + for recorded in journal.iter().rev() { + if only.is_some_and(|only| !only.contains(&recorded.sequence)) { + continue; + } + out.push(Reversal { + sequence: recorded.sequence, + outcome: reverse(&recorded.effect).await, + }); + } + out +} + +async fn reverse(effect: &Effect) -> Result { + match effect { + Effect::Created { path, subject } => { + let subject = subject.ok_or(Left::Irreversible)?; + if !subject.still_holds(path).await { + return Err(Left::Changed); + } + Ok(Step::Trash { + path: path.clone(), + is_dir: subject.is_dir, + size: subject.size, + }) + } + Effect::Moved { from, to, subject } => { + if let Some(subject) = subject { + if !subject.still_holds(to).await { + return Err(Left::Changed); + } + } else if tokio::fs::symlink_metadata(to).await.is_err() { + return Err(Left::Changed); + } + if occupied(from).await { + return Err(Left::Occupied); + } + Ok(Step::MoveBack { + from: to.clone(), + to: from.clone(), + size: subject.map(|subject| subject.size).unwrap_or(0), + }) + } + Effect::Trashed { from, to, subject } => { + let to = to.as_ref().ok_or(Left::Irreversible)?; + if let Some(subject) = subject { + if !subject.still_holds(to).await { + return Err(Left::Changed); + } + } + if occupied(from).await { + return Err(Left::Occupied); + } + Ok(Step::MoveBack { + from: to.clone(), + to: from.clone(), + size: subject.map(|subject| subject.size).unwrap_or(0), + }) + } + Effect::Replaced { + path, + previous, + subject, + } => { + let previous = previous.as_ref().ok_or(Left::Irreversible)?; + if let Some(subject) = subject { + if !subject.still_holds(path).await { + return Err(Left::Changed); + } + } + if tokio::fs::symlink_metadata(previous).await.is_err() { + return Err(Left::Changed); + } + Ok(Step::Restore { + path: path.clone(), + previous: previous.clone(), + size: subject.map(|subject| subject.size).unwrap_or(0), + }) + } + Effect::Removed { .. } => Err(Left::Irreversible), + Effect::Attributes { path, from, .. } => { + if tokio::fs::symlink_metadata(path).await.is_err() { + return Err(Left::Changed); + } + Ok(Step::Attributes { + path: path.clone(), + to: from.clone(), + }) + } + } +} + +async fn occupied(path: &Path) -> bool { + tokio::fs::symlink_metadata(path).await.is_ok() +} diff --git a/core/src/ops/files/undo/tests.rs b/core/src/ops/files/undo/tests.rs new file mode 100644 index 000000000..935f6ec71 --- /dev/null +++ b/core/src/ops/files/undo/tests.rs @@ -0,0 +1,384 @@ +//! Undo over the journals the other jobs write: a rename goes back, a copy +//! is trashed, a trashed file returns, a replacement gives way to the +//! previous bytes, and a file that changed since is left alone. + +use std::{path::Path, time::Duration}; + +use sd_store::file::FileKind; + +use super::{ + input::FileUndoInput, + job::UndoJob, + preflight::{CHANGED, NO_JOURNAL}, + FileUndoAction, +}; +use crate::{ + domain::{SdPath, SdPathBatch}, + infra::{ + action::preflight::{PreviewableAction, ValidatedAction}, + job::{ + journal::Effect, + output::JobOutput, + traits::{Job, JobHandler}, + types::JobId, + }, + }, + ops::files::{ + copy::{ + action::FileConflictResolution, + job::{CopyOptions, FileCopyJob}, + }, + delete::{DeleteJob, DeleteMode, DeleteTargets}, + fixture::{Fixture, T}, + merge::{FileMergeInput, FolderMergeJob, MergeConflictPolicy}, + plan::ChangeKind, + rename::RenameJob, + }, +}; + +async fn run(fixture: &Fixture, job: J) -> (JobId, JobOutput) +where + J: Job + JobHandler + crate::infra::job::types::ErasedJob + crate::infra::job::traits::DynJob, +{ + let handle = fixture + .library + .jobs() + .dispatch(job) + .await + .expect("dispatched"); + let id = handle.id(); + let output = tokio::time::timeout(Duration::from_secs(30), handle.wait()) + .await + .expect("in time") + .expect("completed"); + (id, output) +} + +async fn journal(fixture: &Fixture, job: JobId) -> Vec { + fixture + .library + .jobs() + .database() + .journal(job) + .await + .expect("journal") + .into_iter() + .map(|recorded| recorded.effect) + .collect() +} + +async fn undo(fixture: &Fixture, job: JobId) -> serde_json::Value { + let input = FileUndoInput { + job: job.0, + effects: None, + }; + let validation = FileUndoAction::validate(&input, &fixture.preview()) + .await + .expect("validated"); + assert!(!validation.refuses(), "{:?}", validation.findings); + let (_, output) = run(fixture, UndoJob::new(job.0, None)).await; + let JobOutput::Custom(value) = output else { + panic!("not an undo output: {output:?}"); + }; + value +} + +fn bytes(path: &Path) -> String { + std::fs::read_to_string(path).expect("readable") +} + +/// A rename journals a move, and undoing it moves the file back. +#[tokio::test] +async fn undoing_a_rename_moves_the_file_back() { + let fixture = Fixture::new().await; + let tree = [("a.txt", FileKind::File, 4, T, Some("a"), None, None)]; + fixture.materialize(&fixture.source, &tree); + fixture.index(&fixture.source, &tree).await; + let a = fixture.source.join("a.txt"); + + let (job, _) = run( + &fixture, + RenameJob::named(SdPath::local(&a), "b.txt".to_string()), + ) + .await; + assert!(matches!( + journal(&fixture, job).await.as_slice(), + [Effect::Moved { .. }] + )); + + let plan = FileUndoAction::preview( + FileUndoInput { + job: job.0, + effects: None, + }, + &fixture.preview(), + ) + .await + .expect("planned"); + assert_eq!(plan.summary.moves.files, 1); + assert!(matches!(plan.changes[0].change, ChangeKind::Move { .. })); + + let output = undo(&fixture, job).await; + assert_eq!(output["reversed"], 1); + assert!(a.exists()); + assert!(!fixture.source.join("b.txt").exists()); + + // Undoing the undo renames it again. + let undo_job = fixture + .library + .jobs() + .list_jobs(None) + .await + .expect("jobs") + .into_iter() + .find(|info| info.name == "undo") + .expect("the undo job"); + undo(&fixture, JobId(undo_job.id)).await; + assert!(fixture.source.join("b.txt").exists()); +} + +/// A copy journals a creation, which undo trashes, and a file that +/// changed since the copy is left alone with a warning. +#[tokio::test] +async fn undoing_a_copy_trashes_the_copy_unless_it_changed() { + let fixture = Fixture::new().await; + let tree = [ + ("a.txt", FileKind::File, 4, T, Some("a"), None, None), + ("c.txt", FileKind::File, 2, T, Some("c"), None, None), + ]; + fixture.materialize(&fixture.source, &tree); + fixture.index(&fixture.source, &tree).await; + fixture.index(&fixture.destination, &[]).await; + + let (job, _) = run( + &fixture, + FileCopyJob::new( + SdPathBatch::new(vec![ + SdPath::local(fixture.source.join("a.txt")), + SdPath::local(fixture.source.join("c.txt")), + ]), + SdPath::local(&fixture.destination), + ), + ) + .await; + let effects = journal(&fixture, job).await; + assert_eq!(effects.len(), 2, "{effects:?}"); + assert!(effects + .iter() + .all(|effect| matches!(effect, Effect::Created { .. }))); + + // One copy changes afterward. + std::fs::write(fixture.destination.join("c.txt"), b"changed bytes").expect("write"); + let input = FileUndoInput { + job: job.0, + effects: None, + }; + let validation = FileUndoAction::validate(&input, &fixture.preview()) + .await + .expect("validated"); + assert!(validation + .findings + .iter() + .any(|finding| finding.code == CHANGED)); + assert_eq!(validation.facts.estimated_files, Some(1)); + + let output = undo(&fixture, job).await; + assert_eq!(output["reversed"], 1); + assert_eq!(output["left"].as_array().map(Vec::len), Some(1)); + assert!(!fixture.destination.join("a.txt").exists()); + assert_eq!( + bytes(&fixture.destination.join("c.txt")), + "changed bytes", + "the changed copy stays" + ); + assert!(fixture.source.join("a.txt").exists(), "the original stays"); +} + +/// A trashed file journals where it went, and undo puts it back. +#[cfg(target_os = "macos")] +#[tokio::test] +async fn undoing_a_delete_restores_from_the_trash() { + let fixture = Fixture::new().await; + let tree = [("gone.txt", FileKind::File, 4, T, Some("g"), None, None)]; + fixture.materialize(&fixture.source, &tree); + fixture.index(&fixture.source, &tree).await; + let gone = fixture.source.join("gone.txt"); + + let (job, _) = run( + &fixture, + DeleteJob::new( + DeleteTargets::Paths { + paths: vec![SdPath::local(&gone)], + }, + DeleteMode::Trash, + ), + ) + .await; + let effects = journal(&fixture, job).await; + let [Effect::Trashed { + to: Some(location), .. + }] = effects.as_slice() + else { + panic!("not a trashing with a location: {effects:?}"); + }; + assert!(location.exists()); + assert!(!gone.exists()); + + let output = undo(&fixture, job).await; + assert_eq!(output["reversed"], 1); + assert_eq!(bytes(&gone), "xxxx"); +} + +/// A permanent delete cannot be undone, and validation says so. +#[tokio::test] +async fn a_permanent_delete_has_nothing_to_undo() { + let fixture = Fixture::new().await; + let tree = [("gone.txt", FileKind::File, 4, T, Some("g"), None, None)]; + fixture.materialize(&fixture.source, &tree); + fixture.index(&fixture.source, &tree).await; + + let (job, _) = run( + &fixture, + DeleteJob::new( + DeleteTargets::Paths { + paths: vec![SdPath::local(fixture.source.join("gone.txt"))], + }, + DeleteMode::Permanent, + ), + ) + .await; + assert!(matches!( + journal(&fixture, job).await.as_slice(), + [Effect::Removed { .. }] + )); + let validation = FileUndoAction::validate( + &FileUndoInput { + job: job.0, + effects: None, + }, + &fixture.preview(), + ) + .await + .expect("validated"); + assert!(validation.refuses()); + + let validation = FileUndoAction::validate( + &FileUndoInput { + job: uuid::Uuid::new_v4(), + effects: None, + }, + &fixture.preview(), + ) + .await + .expect("validated"); + assert!(validation + .errors() + .any(|finding| finding.code == NO_JOURNAL)); +} + +/// A merge that overwrites keeps the previous bytes in the trash, and undo +/// puts them back byte for byte; the copied files are trashed. +#[cfg(target_os = "macos")] +#[tokio::test] +async fn undoing_a_merge_restores_replaced_files() { + let fixture = Fixture::new().await; + let file = FileKind::File; + let incoming = [ + ("same.txt", file, 4, T, Some("s"), None, None), + ("new.txt", file, 3, T, Some("n"), None, None), + ]; + fixture.materialize(&fixture.source, &incoming); + fixture.index(&fixture.source, &incoming).await; + std::fs::write(fixture.destination.join("same.txt"), b"old bytes").expect("file"); + fixture + .index( + &fixture.destination, + &[("same.txt", file, 9, T - 1000, Some("o"), None, None)], + ) + .await; + + let (job, _) = run( + &fixture, + FolderMergeJob::new(FileMergeInput { + sources: SdPathBatch::new(vec![SdPath::local(&fixture.source)]), + destination: SdPath::local(&fixture.destination), + on_conflict: MergeConflictPolicy::Overwrite, + consume_sources: false, + remove_extras: false, + }), + ) + .await; + assert_eq!(bytes(&fixture.destination.join("same.txt")), "xxxx"); + let effects = journal(&fixture, job).await; + assert!(effects.iter().any(|effect| matches!( + effect, + Effect::Replaced { + previous: Some(_), + .. + } + ))); + assert!(effects + .iter() + .any(|effect| matches!(effect, Effect::Created { .. }))); + + let output = undo(&fixture, job).await; + assert_eq!(output["reversed"], 2, "{output}"); + assert_eq!(bytes(&fixture.destination.join("same.txt")), "old bytes"); + assert!(!fixture.destination.join("new.txt").exists()); +} + +/// A copy that overwrites honors the policy: skipped without overwrite, +/// replaced with the previous bytes kept when overwriting. +#[cfg(target_os = "macos")] +#[tokio::test] +async fn a_copy_skips_or_replaces_as_the_policy_says() { + let fixture = Fixture::new().await; + let tree = [("a.txt", FileKind::File, 4, T, Some("a"), None, None)]; + fixture.materialize(&fixture.source, &tree); + fixture.index(&fixture.source, &tree).await; + std::fs::write(fixture.destination.join("a.txt"), b"old").expect("file"); + fixture + .index( + &fixture.destination, + &[("a.txt", FileKind::File, 3, T, Some("o"), None, None)], + ) + .await; + + let sources = SdPathBatch::new(vec![SdPath::local(fixture.source.join("a.txt"))]); + let (job, _) = run( + &fixture, + FileCopyJob::new(sources.clone(), SdPath::local(&fixture.destination)), + ) + .await; + assert_eq!(bytes(&fixture.destination.join("a.txt")), "old", "skipped"); + assert!(journal(&fixture, job).await.is_empty()); + + let (job, _) = run( + &fixture, + FileCopyJob::new(sources, SdPath::local(&fixture.destination)).with_options(CopyOptions { + overwrite: true, + conflict_resolution: Some(FileConflictResolution::Overwrite), + ..Default::default() + }), + ) + .await; + assert_eq!( + bytes(&fixture.destination.join("a.txt")), + "xxxx", + "replaced" + ); + let effects = journal(&fixture, job).await; + assert!( + matches!( + effects.as_slice(), + [Effect::Replaced { + previous: Some(_), + .. + }] + ), + "{effects:?}" + ); + let output = undo(&fixture, job).await; + assert_eq!(output["reversed"], 1); + assert_eq!(bytes(&fixture.destination.join("a.txt")), "old"); +} diff --git a/core/src/ops/jobs/journal/mod.rs b/core/src/ops/jobs/journal/mod.rs new file mode 100644 index 000000000..e2fda8cec --- /dev/null +++ b/core/src/ops/jobs/journal/mod.rs @@ -0,0 +1,3 @@ +pub mod query; + +pub use query::*; diff --git a/core/src/ops/jobs/journal/query.rs b/core/src/ops/jobs/journal/query.rs new file mode 100644 index 000000000..3c07a0af7 --- /dev/null +++ b/core/src/ops/jobs/journal/query.rs @@ -0,0 +1,62 @@ +//! A job's journal: what it did to the filesystem, in order. + +use std::sync::Arc; + +use serde::{Deserialize, Serialize}; +use specta::Type; + +use crate::{ + context::CoreContext, + infra::{ + job::{journal::Recorded, types::JobId}, + query::{LibraryQuery, QueryError, QueryResult}, + }, +}; + +#[derive(Debug, Clone, Serialize, Deserialize, Type)] +pub struct JobJournalInput { + pub job_id: uuid::Uuid, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Type)] +pub struct JobJournalOutput { + pub effects: Vec, +} + +pub struct JobJournalQuery { + input: JobJournalInput, +} + +impl LibraryQuery for JobJournalQuery { + type Input = JobJournalInput; + type Output = JobJournalOutput; + + fn from_input(input: Self::Input) -> QueryResult { + Ok(Self { input }) + } + + async fn execute( + self, + context: Arc, + session: crate::infra::api::SessionContext, + ) -> QueryResult { + let library_id = session + .current_library_id + .ok_or_else(|| QueryError::Internal("No library selected".to_string()))?; + let library = context + .libraries() + .await + .get_library(library_id) + .await + .ok_or_else(|| QueryError::LibraryNotFound(library_id))?; + let effects = library + .jobs() + .database() + .journal(JobId(self.input.job_id)) + .await + .map_err(|e| QueryError::Internal(e.to_string()))?; + Ok(JobJournalOutput { effects }) + } +} + +crate::register_library_query!(JobJournalQuery, "jobs.journal"); diff --git a/core/src/ops/jobs/list/output.rs b/core/src/ops/jobs/list/output.rs index 8e1f25459..7a718a2e3 100644 --- a/core/src/ops/jobs/list/output.rs +++ b/core/src/ops/jobs/list/output.rs @@ -1,4 +1,4 @@ -use crate::infra::job::types::ActionContextInfo; +use crate::infra::job::{journal::JournalSummary, types::ActionContextInfo}; use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; use specta::Type; @@ -16,6 +16,9 @@ pub struct JobListItem { pub created_at: DateTime, pub started_at: Option>, pub completed_at: Option>, + /// What the job's journal amounts to, for jobs that changed the + /// filesystem. + pub journal: Option, } #[derive(Debug, Clone, Serialize, Deserialize, Type)] diff --git a/core/src/ops/jobs/list/query.rs b/core/src/ops/jobs/list/query.rs index e5ca48dbe..3378c0d95 100644 --- a/core/src/ops/jobs/list/query.rs +++ b/core/src/ops/jobs/list/query.rs @@ -46,9 +46,16 @@ impl LibraryQuery for JobListQuery { .list_jobs(self.input.status) .await .map_err(|e| QueryError::Internal(e.to_string()))?; + let journals = library + .jobs() + .database() + .journal_summaries() + .await + .map_err(|e| QueryError::Internal(e.to_string()))?; let items = jobs .into_iter() .map(|j| JobListItem { + journal: journals.get(&j.id.to_string()).copied(), id: j.id, name: j.name, device_id: j.device_id, diff --git a/core/src/ops/jobs/mod.rs b/core/src/ops/jobs/mod.rs index a74331f6c..0801cce86 100644 --- a/core/src/ops/jobs/mod.rs +++ b/core/src/ops/jobs/mod.rs @@ -2,6 +2,7 @@ pub mod active; pub mod control; pub mod copy_metadata; pub mod info; +pub mod journal; pub mod list; pub mod remote_list; @@ -9,5 +10,6 @@ pub use active::*; pub use control::*; pub use copy_metadata::*; pub use info::*; +pub use journal::*; pub use list::*; pub use remote_list::*; diff --git a/core/src/service/network/protocol/file_delete.rs b/core/src/service/network/protocol/file_delete.rs index aa06474e4..6c9c239de 100644 --- a/core/src/service/network/protocol/file_delete.rs +++ b/core/src/service/network/protocol/file_delete.rs @@ -72,7 +72,7 @@ impl FileDeleteProtocolHandler { // Note: We're creating a minimal job context for this operation // In a full implementation, this might integrate with the job system let results = match self - .execute_deletion_with_strategy(&strategy, &paths, mode.clone()) + .execute_deletion_with_strategy(&strategy, &paths, mode.clone(), context) .await { Ok(results) => results, @@ -86,6 +86,7 @@ impl FileDeleteProtocolHandler { success: false, bytes_freed: 0, error: Some(format!("Strategy execution failed: {}", e)), + effect: None, }) .collect(), }); @@ -109,7 +110,9 @@ impl FileDeleteProtocolHandler { strategy: &LocalDeleteStrategy, paths: &[crate::domain::addressing::SdPath], mode: DeleteMode, + context: &crate::context::CoreContext, ) -> anyhow::Result> { + let _ = strategy; let mut results = Vec::new(); for path in paths { @@ -128,6 +131,7 @@ impl FileDeleteProtocolHandler { success: false, bytes_freed: 0, error: Some("Path not within allowed roots".to_string()), + effect: None, }); continue; } @@ -144,18 +148,49 @@ impl FileDeleteProtocolHandler { }) .unwrap_or(0); - // Perform deletion based on mode + // Perform deletion based on mode. A remote request runs outside + // any job, so what it did is answered rather than journaled here. let result = match mode { - DeleteMode::Trash => LocalDeleteStrategy.move_to_trash(local_path).await, - DeleteMode::Permanent => LocalDeleteStrategy.permanent_delete(local_path).await, - DeleteMode::Secure => LocalDeleteStrategy.secure_delete(local_path).await, + DeleteMode::Trash => crate::ops::files::trash::trash( + local_path, + Some(&context.volume_manager), + crate::infra::job::types::JobId::new(), + ) + .await + .map(|to| { + crate::infra::job::journal::Effect::trashed(local_path.to_path_buf(), to, None) + }), + DeleteMode::Permanent => LocalDeleteStrategy + .permanent_delete(local_path) + .await + .map(|()| { + crate::infra::job::journal::Effect::removed(local_path.to_path_buf()) + }), + DeleteMode::Secure => { + LocalDeleteStrategy + .secure_delete(local_path) + .await + .map(|()| { + crate::infra::job::journal::Effect::removed(local_path.to_path_buf()) + }) + } }; - results.push(crate::ops::files::delete::strategy::DeleteResult { - path: path.clone(), - success: result.is_ok(), - bytes_freed: if result.is_ok() { size } else { 0 }, - error: result.err().map(|e| e.to_string()), + results.push(match result { + Ok(effect) => crate::ops::files::delete::strategy::DeleteResult { + path: path.clone(), + success: true, + bytes_freed: size, + error: None, + effect: Some(effect), + }, + Err(error) => crate::ops::files::delete::strategy::DeleteResult { + path: path.clone(), + success: false, + bytes_freed: 0, + error: Some(error.to_string()), + effect: None, + }, }); } diff --git a/docs/cli/overview.mdx b/docs/cli/overview.mdx index d2a33292e..f8c17386b 100644 --- a/docs/cli/overview.mdx +++ b/docs/cli/overview.mdx @@ -168,6 +168,48 @@ sd-cli file dedupe --keep ~/Photos/2024/IMG_0041.JPG sd-cli file dedupe ~/Downloads --keep-under /Volumes/NAS/Photos ~~~ +Rename one path, or many by rules applied to each name in order. Validation +resolves every new name against the directory and the volume's filesystem, so +a name NTFS refuses, a name already there, or two files wanting one name is a +finding before anything moves, and the plan lists each old name beside its +new one: + +~~~bash +sd-cli file rename ~/Photos/2024/IMG_0041.JPG --to "beach.jpg" +sd-cli file rename ~/Photos/2024/*.JPG --sequence "Trip_{n:03}" --lower-extension --dry-run +sd-cli file rename ~/Photos/2024/*.JPG --template "{captured:%Y-%m-%d} {name}{ext}" +~~~ + +Every job that changes the filesystem writes a journal of what it did, and +undo reverses it from the journal, newest effect first, after showing the +plan and refusing whatever changed since. A merge with `--remove-extras` is +a mirror: the destination ends up matching the sources, with the files it +alone held moved to the trash. The trash lists what Spacedrive put there +and puts it back through undo: + +~~~bash +sd-cli job journal 4f2c… +sd-cli file undo 4f2c… --dry-run +sd-cli file merge ~/Projects --into /Volumes/NAS/Projects --on-conflict keep-newer --remove-extras +sd-cli file trash list +sd-cli file trash restore 4f2c… 3 +sd-cli file trash empty +~~~ + +Organize a folder into subfolders by date, kind or extension, or flatten +one; write an archive or extract one; set attributes; make a link. Each +shows its plan first: + +~~~bash +sd-cli file organize ~/Downloads --by kind --dry-run +sd-cli file organize ~/Photos/2024 --by date --field captured --granularity year-month --recursive +sd-cli file flatten ~/Downloads/unpacked --on-conflict keep-both +sd-cli file archive ~/Projects/site --to ~/Archive/site.tar.zst --remove-sources +sd-cli file extract ~/Downloads/kit.zip --to ~/Downloads/kit --strip-components 1 +sd-cli file attributes ~/Scripts/*.sh --mode 755 +sd-cli file link ~/Desktop/Photos --target ~/Pictures/Photos +~~~ + ## Structured output Pass **--format json** before the command to receive structured output: diff --git a/docs/core/ops.mdx b/docs/core/ops.mdx index 330ed2c7f..f4fd43902 100644 --- a/docs/core/ops.mdx +++ b/docs/core/ops.mdx @@ -138,7 +138,10 @@ get `useLibraryValidate` and `useLibraryPreview`, `CoreClient::validate` and with a cheap structural check does it in `from_input`, and everything that needs the index or the filesystem is a finding. -`files.copy`, `files.merge`, and `files.delete` answer both methods, and their +Every filesystem-mutating action answers both methods: `files.copy`, +`files.merge`, `files.delete`, `files.rename`, `files.rename_batch`, +`files.undo`, `files.organize`, `files.flatten`, `files.archive`, +`files.extract`, `files.set_attributes`, and `files.link`. Their plans share one shape, `FsPlan` in `core/src/ops/files/plan.rs`: a summary of counts and bytes per kind of change, the changes themselves up to a cap that conflicts always survive, the roots the plan pairs, and the store @@ -153,6 +156,54 @@ keeping either the first copy in walk order beneath a folder or the copies the caller names. A copy and the copy that stands for it are read in full before it goes, and what the reads learn is written back to the stores. +`files.rename` and `files.rename_batch` resolve their new names the way their +job does (`core/src/ops/files/rename/resolve.rs`): against the live +directory, read once per parent, and the target volume's filesystem, whose +name rules live in `rename/naming.rs`. Whether a directory matches names +without regard to case is probed rather than assumed from the filesystem's +name. Batch rename applies an ordered list of rules to each name +(`rename/rules.rs`): replace, case, affix, sequence, and template, with +tokens for the stem, the extension, a counter, the parent, and dates. The +plan is one `Move` per changed name and a `Conflict` where two targets want +one, and the job renames in an order that never overwrites. + +## The journal and undo + +Every job that changes the filesystem records what it did as effects +(`core/src/infra/job/journal.rs`), in order, against its own id, in the +`job_journal` table of the library's job database: `Created`, `Moved`, +`Trashed` with where the item went, `Replaced` with where the previous +bytes went, `Removed`, and `Attributes` before and after. Each carries the +state of its result, size and modification time, so undo can tell whether +the subject changed since. A job records through `JobContext::record`, and +a failure to write the journal is logged rather than failing the operation +that already happened. + +The trash reports locations (`core/src/ops/files/trash.rs`): `NSFileManager` +on macOS, the `trash` crate's listing on Windows and Linux, and on a volume +with no trash of its own a Spacedrive trash directory at the volume's +root, `.spacedrive/trash//`, reached by a rename. An overwrite goes +through the same door, so a replacement is undoable until the trash is +emptied. `files.undo` (`core/src/ops/files/undo/`) previews the reverse as +an `FsPlan` whose basis is the journal, refuses a job still running or with +nothing to reverse, warns of each subject changed or place taken, and its +job applies the reverse newest effect first, writing its own journal. +`jobs.list` carries each job's journal summary and `jobs.journal` the +effects; `files.trash_list` lists the trashed effects with a location and +`files.trash_empty` purges them. + +The other operations follow the same two rules, one plan shape and one +journal. Mirror is `remove_extras` on `files.merge`: the planner keeps the +files only the destination holds across every pair and adds a flagged +`Delete` for each, and the job trashes them after the merge. +`files.organize` and `files.flatten` (`core/src/ops/files/organize/`) plan +moves into new folders from the index and rename in place. `files.archive` +and `files.extract` (`core/src/ops/files/archive/`) write zip or tar.zst to +a temporary name and rename it into place, and plan an extract from the +archive's own directory, refusing an entry that would escape. +`files.set_attributes` and `files.link` plan one row per file and journal +enough to reverse. + A plan is browsable. Every preview whose plan is an `FsPlan` is retained in memory under a handle for a while after its last use, and `files.directory_listing` takes that handle as `overlay`: the listing then diff --git a/docs/plans/2026-09-23-file-operations.md b/docs/plans/2026-09-23-file-operations.md index fb77609a2..bd33a4ee6 100644 --- a/docs/plans/2026-09-23-file-operations.md +++ b/docs/plans/2026-09-23-file-operations.md @@ -1,6 +1,6 @@ # File Operations on Preflight: Rename, Undo, Mirror, Organize, Archive -> Status: proposal. +> Status: landed 2026-09-23; live acceptance on the Expansion drive next. > Captured: 2026-09-23 > Owns: the file operations a file manager is expected to have that > Spacedrive lacks or has without preflight: delete and duplicate in the @@ -254,13 +254,13 @@ have one. | Phase | Scope | Exit proof | |---|---|---| -| F1 | Delete and duplicate in the explorer | ⌘⌫, ⌥⌘⌫ and the menu open the delete dialog on preflight with the last-copy warning and the removed rows; `useDeleteFiles` and its `confirm()` are gone. ⌘D and "Duplicate" write a numbered copy beside the file after preflight | -| F2 | Rename on preflight, batch rename | `validate:files.rename` answers every code above against a case-insensitive and an NTFS fixture; `files.rename_batch` previews new names and collisions for each rule, the job renames a chain like `1→2→3` without overwriting, and the dialog re-previews as rules change | -| F3 | Journal, undo, restore | Every mutating job writes effects. Undoing a move, a copy, a rename, a trash on each platform, and a merge with replacements restores the tree byte for byte; undo refuses a file changed since with `undo.changed`; ⌘Z and the job list both reach it | -| F4 | Mirror | `files.merge` with `remove_extras` previews the extras as flagged deletes, the job removes only what no source holds, and `sd file merge --remove-extras` renders it | -| F5 | Organize and flatten | Organizing a folder by month previews the new folders and moves and leaves records with their identity; flattening a tree with collisions numbers them under `KeepBoth` and prunes the emptied folders | -| F6 | Archive and extract | Archiving and extracting a tree with a folder collision and a path escape previews both, refuses the escape, resumes an interrupted extract at its entry, and leaves no temporary file behind | -| F7 | Attributes, links, trash | Attribute and link plans preview, exFAT refuses a mode, a hard link across volumes is refused, and the trash view restores and empties on macOS, Windows and Linux | +| F1 | Delete and duplicate in the explorer | Landed 2026-09-23. ⌘⌫, ⌥⌘⌫ and the menu open `DeleteModal` on preflight with the last-copy warning and the removed rows; `useDeleteFiles` and its `confirm()` are gone. ⌘D and "Duplicate" write a numbered copy beside the file after validation, and the copy preview plans a keep-both copy at its numbered name | +| F2 | Rename on preflight, batch rename | Landed 2026-09-23. `validate:files.rename` answers every code against the live directory, probing case sensitivity rather than trusting the filesystem's name, with NTFS rules on those volumes and SMB shares; `files.rename_batch` previews new names and collisions per rule, the job renames a chain and a swap without overwriting, and `RenameModal` re-previews as rules change | +| F3 | Journal, undo, restore | Landed 2026-09-23. Every mutating job writes effects; the trash reports locations on macOS through `NSFileManager` and on Windows and Linux through the `trash` crate's listing, with a Spacedrive trash directory on volumes without one; overwrites stash the previous file. Undoing a rename, a copy, a trash and a merge with replacements is tested; a file changed since is `undo.changed` and left; ⌘Z, the job list and the trash view reach it. Trash restore on Windows and Linux is written against the crate and not yet run there | +| F4 | Mirror | Landed 2026-09-23. `files.merge` with `remove_extras` previews the extras as flagged deletes, validation counts the last copies (`merge.last_copies`), the job trashes only what no source holds and prunes the emptied folders, and `sd file merge --remove-extras` and the merge dialog's switch render it | +| F5 | Organize and flatten | Landed 2026-09-23. Organizing by month previews the new folders and moves and each move is a rename, so the inode survives; flattening numbers collisions under keep both, leaves them under skip, and prunes the emptied folders. `sd file organize`, `sd file flatten`, and the two dialogs | +| F6 | Archive and extract | Landed 2026-09-23. Zip and tar.zst round trip with a folder collision and a replacement, an escaping entry refuses the extract, an interrupted extract resumes at its entry, and no temporary file is left behind. `sd file archive`, `sd file extract`, and the two dialogs | +| F7 | Attributes, links, trash | Landed 2026-09-23. Attribute and link plans preview and undo, exFAT refuses a mode, a hard link across volumes or to a directory is refused, and the trash view at `/trash` restores through undo and empties; emptying the platform's trash and restoring from it are written for Windows and Linux and run so far on macOS | F1 first since it is the one gap in the explorer that skips preflight today. F2 before F3 because batch rename produces the chains the journal has to @@ -279,6 +279,16 @@ restore each from the trash view. Measure preview time for a batch rename over the largest folder available, and confirm no operation left a file the journal does not account for. +## Decisions taken + +The seven decisions below were taken as proposed, since the phases were +built in one pass: the OS trash with recorded locations and a Spacedrive +trash directory only where a volume has none; stashing on replace, on; the +five rename rules with regex inside replace; captured dates from the store +with the modification time as the fallback; mirror as a merge option; undo +on both the job list and ⌘Z, plus the trash view; zip and tar with zstd. +Each is a switch of code rather than of design if reversed. + ## Decisions for James 1. Trash mechanism. Proposed: the OS trash with the resulting location diff --git a/packages/interface/src/components/modals/DedupeModal.tsx b/packages/interface/src/components/modals/DeleteModal.tsx similarity index 100% rename from packages/interface/src/components/modals/DedupeModal.tsx rename to packages/interface/src/components/modals/DeleteModal.tsx diff --git a/packages/interface/src/routes/explorer/hooks/useDeleteFiles.ts b/packages/interface/src/routes/explorer/hooks/useDeleteFiles.ts deleted file mode 100644 index 088801b04..000000000 --- a/packages/interface/src/routes/explorer/hooks/useDeleteFiles.ts +++ /dev/null @@ -1,44 +0,0 @@ -import { useCallback } from "react"; -import type { File } from "@sd/ts-client"; -import { useLibraryMutation } from "../../../contexts/SpacedriveContext"; - -/** - * Shared hook for delete file operations. - * Used by both useExplorerKeyboard (DEL key) and useFileContextMenu. - */ -export function useDeleteFiles() { - const mutation = useLibraryMutation("files.delete"); - - const deleteFiles = useCallback( - async (files: File[], permanent: boolean) => { - if (files.length === 0) return false; - if (files.some((f) => !f.sd_path)) return false; - if (mutation.isPending) return false; - - const label = permanent ? "Permanently delete" : "Delete"; - const suffix = permanent ? " This cannot be undone." : ""; - const message = - files.length > 1 - ? `${label} ${files.length} items?${suffix}` - : `${label} "${files[0].name}"?${suffix}`; - - if (!confirm(message)) return false; - - try { - await mutation.mutateAsync({ - targets: { kind: "paths", paths: files.map((f) => f.sd_path) }, - permanent, - recursive: true, - }); - return true; - } catch (err) { - console.error("Failed to delete:", err); - alert(`Failed to delete: ${err}`); - return false; - } - }, - [mutation], - ); - - return { deleteFiles, isPending: mutation.isPending }; -} diff --git a/packages/ts-client/src/generated/types.ts b/packages/ts-client/src/generated/types.ts index 179b71382..ee5b7839a 100644 --- a/packages/ts-client/src/generated/types.ts +++ b/packages/ts-client/src/generated/types.ts @@ -125,11 +125,28 @@ targets_tagged: number; */ targets_pending: number; warnings: string[] }; +export type ArchiveFormat = "zip" | "tar_zstd"; + +/** + * What an attribute change sets. + */ +export type Attributes = { +/** + * Unix permission bits. + */ +mode: number | null; +/** + * Modification time, unix milliseconds. + */ +modified_ms: number | null; hidden: boolean | null }; + /** * Audio metadata extracted from FFmpeg */ export type AudioMediaData = { uuid: string; duration_seconds: number | null; bit_rate: number | null; sample_rate: number | null; channels: string | null; codec: string | null; title: string | null; artist: string | null; album: string | null; album_artist: string | null; genre: string | null; year: number | null; track_number: number | null; disc_number: number | null; composer: string | null; publisher: string | null; copyright: string | null }; +export type CaseRule = "lower" | "upper" | "title" | "keep"; + export type ChangeKind = { type: "create"; size: number } | { type: "create_directory" } | { type: "replace"; existing_size: number; incoming_size: number; reason: ReplaceReason } | /** * A directory on both sides; the plan continues inside it. @@ -138,7 +155,11 @@ export type ChangeKind = { type: "create"; size: number } | { type: "create_dire /** * Nothing the policy resolves. Reported and left alone. */ -{ type: "conflict"; kind: ConflictKind }; +{ type: "conflict"; kind: ConflictKind } | +/** + * The filesystem attributes that change, each absent where it stays. + */ +{ type: "set_attributes"; attributes: Attributes }; /** * Cloud service type identifier @@ -995,6 +1016,25 @@ reclaimable: number; */ sources_queried: number }; +export type Effect = { kind: "created"; path: string; subject: Subject | null } | { kind: "moved"; from: string; to: string; subject: Subject | null } | +/** + * `to` is where the item went, where the platform reports it. + */ +{ kind: "trashed"; from: string; to: string | null; subject: Subject | null } | +/** + * `previous` is where the bytes that were at `path` went, where they + * were kept. + */ +{ kind: "replaced"; path: string; previous: string | null; subject: Subject | null } | +/** + * Gone for good. + */ +{ kind: "removed"; path: string } | +/** + * The filesystem attributes of a path, before and after. + */ +{ kind: "attributes"; path: string; from: Attributes; to: Attributes }; + /** * Type of filesystem entry */ @@ -1155,6 +1195,8 @@ executes_on: string; */ strategy: string | null; estimated_files: number | null; estimated_bytes: number | null; free_space_after: number | null }; +export type ExtensionCase = "lower" | "keep"; + /** * A dependency Spacedrive knows how to discover and use. */ @@ -1236,6 +1278,19 @@ duration_seconds: number | null; */ thumbnail_path?: string | null }; +/** + * Write one archive holding the sources. + */ +export type FileArchiveInput = { sources: SdPath[]; +/** + * The archive file to write. It must not exist. + */ +destination: SdPath; format: ArchiveFormat; +/** + * Trash the sources once the archive is complete. + */ +remove_sources?: boolean }; + /** * Query to get a file by its ID with all related data */ @@ -1302,6 +1357,29 @@ permanent: boolean; */ recursive: boolean }; +/** + * Extract an archive into a folder. + */ +export type FileExtractInput = { archive: SdPath; +/** + * An existing folder the entries are written into. + */ +destination: SdPath; on_conflict: MergeConflictPolicy; +/** + * Leading path components to drop from every entry. + */ +strip_components?: number }; + +/** + * Move every file beneath a folder to the folder itself. + */ +export type FileFlattenInput = { scope: SdPath; on_conflict: FlattenPolicy }; + +/** + * Make a link at `at` to `target`. + */ +export type FileLinkInput = { at: SdPath; target: SdPath; kind: LinkKind }; + /** * Merge one or more folders into an existing directory: recurse into * matching subfolders, skip files whose bytes are proven identical, and @@ -1319,10 +1397,31 @@ destination: SdPath; on_conflict: MergeConflictPolicy; * confirmed identical to the destination's, and prune emptied source * directories. What the merge did not settle stays where it was. */ -consume_sources: boolean }; +consume_sources: boolean; +/** + * Remove from the destination what no source holds, so it ends up + * matching the sources: a mirror. The extras go to the trash. + */ +remove_extras?: boolean }; /** - * Input for renaming a file or directory + * Move the files under a folder into subfolders named by a rule. + */ +export type FileOrganizeInput = { scope: SdPath; rule: OrganizeRule; +/** + * Take the files beneath the folder at any depth, not only those + * directly under it. Every file ends up directly under its new + * subfolder. + */ +recursive?: boolean }; + +/** + * Rename several files by rules applied to each name in order. + */ +export type FileRenameBatchInput = { targets: SdPath[]; rules: RenameRule[] }; + +/** + * Rename one file or directory. */ export type FileRenameInput = { /** @@ -1390,6 +1489,11 @@ available_filters: FilterKind[] }; */ export type FileSearchResult = { file: File; score: number; score_breakdown: ScoreBreakdown; highlights: TextHighlight[]; matched_content: string | null }; +/** + * Set attributes on files; each absent attribute stays as it is. + */ +export type FileSetAttributesInput = { paths: SdPath[]; attributes: Attributes }; + /** * Filesystem type */ @@ -1443,6 +1547,26 @@ export type FileSystem = */ { Other: string }; +/** + * Remove for good what the journals put in the trash, the Spacedrive + * trash directories, and the platform's trash when asked. + */ +export type FileTrashEmptyInput = { +/** + * Empty the platform's own trash as well, everything in it. + */ +os_trash?: boolean }; + +/** + * Reverse what a job did, from its journal. + */ +export type FileUndoInput = { job: string; +/** + * The effects to reverse, by sequence; every effect when absent. The + * trash view restores one item this way. + */ +effects?: number[] | null }; + /** * Indicates which filters are available for a given search type */ @@ -1458,6 +1582,19 @@ export type Finding = { severity: Severity; code: string; message: string; */ path: SdPath | null }; +/** + * What to do with a file whose name is already taken at the root. + */ +export type FlattenPolicy = +/** + * The deeper file is written under a numbered name. + */ +"keep_both" | +/** + * The deeper file stays where it is. + */ +"skip"; + export type FreezeSourceInput = { source_id: string }; export type FreezeSourceOutput = { @@ -1498,7 +1635,7 @@ export type FsPlanSummary = { creates: Tally; directories_created: number; repla * Files at the same path on both sides with different bytes, which the * policy resolved one way or another. */ -collisions: number }; +collisions: number; attributes: number }; /** * Generic progress information that all job types can convert into @@ -1667,6 +1804,20 @@ tag_id: string | null }; export type GetTagChildrenOutput = { tags: Tag[] }; +export type Granularity = +/** + * `2024` + */ +"year" | +/** + * `2024-05` + */ +"year_month" | +/** + * `2024-05-06` + */ +"year_month_day"; + /** * Types of groups that can appear in a space */ @@ -2021,9 +2172,18 @@ export type JobInfoOutput = { id: string; name: string; status: JobStatus; progr export type JobInfoQueryInput = { job_id: string }; +export type JobJournalInput = { job_id: string }; + +export type JobJournalOutput = { effects: Recorded[] }; + export type JobListInput = { status: JobStatus | null }; -export type JobListItem = { id: string; name: string; device_id: string; status: JobStatus; progress: number; action_type: string | null; action_context: ActionContextInfo | null; created_at: string; started_at: string | null; completed_at: string | null }; +export type JobListItem = { id: string; name: string; device_id: string; status: JobStatus; progress: number; action_type: string | null; action_context: ActionContextInfo | null; created_at: string; started_at: string | null; completed_at: string | null; +/** + * What the job's journal amounts to, for jobs that changed the + * filesystem. + */ +journal: JournalSummary | null }; export type JobListOutput = { jobs: JobListItem[] }; @@ -2128,6 +2288,11 @@ export type JobStatus = */ "cancelled"; +/** + * What a journal amounts to, for a list of jobs. + */ +export type JournalSummary = { effects: number; reversible: number }; + export type JsonValue = null | boolean | number | string | JsonValue[] | { [key in string]: JsonValue }; /** @@ -2549,6 +2714,8 @@ devicesRegistered: boolean; */ message: string }; +export type LinkKind = "symlink" | "hardlink"; + export type ListAdaptersInput = Record; export type ListEventsInput = Record; @@ -3027,6 +3194,23 @@ export type OperatingSystem = "MacOS" | "Windows" | "Linux" | "IOs" | "Android" */ export type OperationSnapshot = { broadcasts_sent: number; state_changes_broadcast: number; shared_changes_broadcast: number; broadcast_batches_sent: number; failed_broadcasts: number; changes_received: number; changes_applied: number; changes_rejected: number; buffer_queue_depth: number; active_backfill_sessions: number; backfill_sessions_completed: number; backfill_pagination_rounds: number; retry_queue_depth: number; retry_attempts: number; retry_successes: number }; +export type OrganizeDateField = "modified" | "created" | +/** + * The capture time the store holds for a photo or video, or the + * modification time where it holds none. + */ +"captured"; + +export type OrganizeRule = { kind: "by_date"; field: OrganizeDateField; granularity: Granularity } | +/** + * The content kind the indexer assigned: Images, Videos, Documents. + */ +{ kind: "by_kind" } | +/** + * The extension, lowercased; files without one go under "No extension". + */ +{ kind: "by_extension" }; + /** * One row of a listing as the overlaid plan changes it. */ @@ -3246,7 +3430,15 @@ export type PlanBasis = /** * The stores the plan read, at the revisions it read them. */ -{ kind: "index"; revisions: StoreRevision[] }; +{ kind: "index"; revisions: StoreRevision[] } | +/** + * A job's journal, for an undo. + */ +{ kind: "journal"; job: string } | +/** + * An archive's own directory, for an extract. + */ +{ kind: "archive"; path: SdPath; entries: number }; /** * One source folder of the plan and the folder it is written into. @@ -3368,6 +3560,11 @@ export type ProxyPairingConfigOutput = { auto_accept_vouched: boolean; auto_vouc export type ReadSizeBucket = { range: string; reads: number }; +/** + * One effect as the journal holds it. + */ +export type Recorded = { sequence: number; effect: Effect; recorded_at: string }; + /** * Input for the redundancy summary query */ @@ -3440,6 +3637,21 @@ createdAt: string; */ statistics: LibraryStatistics }; +export type RenameRule = +/** + * Replace `find` with `with`, in the stem or the whole name. + */ +{ kind: "replace"; find: string; with: string; regex?: boolean; whole_name?: boolean } | { kind: "case"; stem: CaseRule; extension?: ExtensionCase } | { kind: "affix"; prefix?: string; suffix?: string } | +/** + * The stem from a pattern holding `{n}`, counted from `start` in + * steps of `step` over the targets in order. + */ +{ kind: "sequence"; pattern: string; start?: number; step?: number } | +/** + * The whole name from a pattern. + */ +{ kind: "template"; pattern: string }; + export type ReorderGroupsInput = { space_id: string; group_ids: string[] }; export type ReorderItemsInput = { group_id: string | null; item_ids: string[] }; @@ -4091,6 +4303,12 @@ export type StateTransition = { from: DeviceSyncState; to: DeviceSyncState; time export type StoreRevision = { source: string; revision: number }; +/** + * The state of an effect's result when it was recorded: what undo checks + * against before touching it. + */ +export type Subject = { size: number; mtime_ms: number; is_dir: boolean }; + /** * Sync activity types for detailed sync monitoring */ @@ -4398,6 +4616,46 @@ volume_uuid: string | null; */ whole_volume: boolean; job_id: string | null }; +export type TrashEmptyOutput = { +/** + * Items the journals named that were removed. + */ +purged: number; +/** + * Items removed from Spacedrive trash directories beyond those. + */ +spacedrive_trash: number; os_trash_emptied: boolean; failed: string[] }; + +export type TrashListInput = { +/** + * The most items to list; every item when absent. + */ +limit: number | null }; + +export type TrashListOutput = { items: TrashedItem[] }; + +/** + * One item a job put in the trash. + */ +export type TrashedItem = { job: string; sequence: number; +/** + * Where it was. + */ +from: string; +/** + * Where it is. + */ +location: string; trashed_at: string; size: number; is_dir: boolean; +/** + * Whether the item is still at its location. + */ +present: boolean; +/** + * Whether it sits in a Spacedrive trash directory rather than the + * platform's trash. + */ +spacedrive_trash: boolean }; + export type UnapplyTagsInput = { targets: TagTargets; tag_ids: string[] }; export type UnapplyTagsOutput = { @@ -5163,11 +5421,20 @@ export type CoreAction = export type LibraryAction = { type: 'adapters.update'; input: UpdateAdapterInput; output: UpdateAdapterOutput } | { type: 'config.library.update'; input: UpdateLibraryConfigInput; output: UpdateLibraryConfigOutput } + | { type: 'files.archive'; input: FileArchiveInput; output: JobReceipt } | { type: 'files.copy'; input: FileCopyInput; output: JobReceipt } | { type: 'files.createFolder'; input: CreateFolderInput; output: CreateFolderOutput } | { type: 'files.delete'; input: FileDeleteInput; output: JobReceipt } + | { type: 'files.extract'; input: FileExtractInput; output: JobReceipt } + | { type: 'files.flatten'; input: FileFlattenInput; output: JobReceipt } + | { type: 'files.link'; input: FileLinkInput; output: JobReceipt } | { type: 'files.merge'; input: FileMergeInput; output: JobReceipt } + | { type: 'files.organize'; input: FileOrganizeInput; output: JobReceipt } | { type: 'files.rename'; input: FileRenameInput; output: JobReceipt } + | { type: 'files.rename_batch'; input: FileRenameBatchInput; output: JobReceipt } + | { type: 'files.set_attributes'; input: FileSetAttributesInput; output: JobReceipt } + | { type: 'files.trash_empty'; input: FileTrashEmptyInput; output: TrashEmptyOutput } + | { type: 'files.undo'; input: FileUndoInput; output: JobReceipt } | { type: 'indexing.start'; input: IndexInput; output: JobReceipt } | { type: 'indexing.startup'; input: StartupIndexingInput; output: StartupIndexingOutput } | { type: 'jobs.cancel'; input: JobCancelInput; output: JobCancelOutput } @@ -5247,9 +5514,11 @@ export type LibraryQuery = | { type: 'files.content_kind_stats'; input: ContentKindStatsInput; output: ContentKindStatsOutput } | { type: 'files.directory_listing'; input: DirectoryListingInput; output: DirectoryListingOutput } | { type: 'files.media_listing'; input: MediaListingInput; output: MediaListingOutput } + | { type: 'files.trash_list'; input: TrashListInput; output: TrashListOutput } | { type: 'jobs.active'; input: ActiveJobsInput; output: ActiveJobsOutput } | { type: 'jobs.get_copy_metadata'; input: CopyMetadataQueryInput; output: CopyMetadataOutput } | { type: 'jobs.info'; input: JobInfoQueryInput; output: JobInfoOutput } + | { type: 'jobs.journal'; input: JobJournalInput; output: JobJournalOutput } | { type: 'jobs.list'; input: JobListInput; output: JobListOutput } | { type: 'libraries.info'; input: LibraryInfoQueryInput; output: Library } | { type: 'paths.compare'; input: PathCompareInput; output: PathCompareOutput } @@ -5281,15 +5550,33 @@ export type LibraryQuery = ; export type LibraryValidate = - { type: 'files.copy'; input: FileCopyInput; output: Validation } + { type: 'files.archive'; input: FileArchiveInput; output: Validation } + | { type: 'files.copy'; input: FileCopyInput; output: Validation } | { type: 'files.delete'; input: FileDeleteInput; output: Validation } + | { type: 'files.extract'; input: FileExtractInput; output: Validation } + | { type: 'files.flatten'; input: FileFlattenInput; output: Validation } + | { type: 'files.link'; input: FileLinkInput; output: Validation } | { type: 'files.merge'; input: FileMergeInput; output: Validation } + | { type: 'files.organize'; input: FileOrganizeInput; output: Validation } + | { type: 'files.rename'; input: FileRenameInput; output: Validation } + | { type: 'files.rename_batch'; input: FileRenameBatchInput; output: Validation } + | { type: 'files.set_attributes'; input: FileSetAttributesInput; output: Validation } + | { type: 'files.undo'; input: FileUndoInput; output: Validation } ; export type LibraryPreview = - { type: 'files.copy'; input: FileCopyInput; output: FsPlan } + { type: 'files.archive'; input: FileArchiveInput; output: FsPlan } + | { type: 'files.copy'; input: FileCopyInput; output: FsPlan } | { type: 'files.delete'; input: FileDeleteInput; output: FsPlan } + | { type: 'files.extract'; input: FileExtractInput; output: FsPlan } + | { type: 'files.flatten'; input: FileFlattenInput; output: FsPlan } + | { type: 'files.link'; input: FileLinkInput; output: FsPlan } | { type: 'files.merge'; input: FileMergeInput; output: FsPlan } + | { type: 'files.organize'; input: FileOrganizeInput; output: FsPlan } + | { type: 'files.rename'; input: FileRenameInput; output: FsPlan } + | { type: 'files.rename_batch'; input: FileRenameBatchInput; output: FsPlan } + | { type: 'files.set_attributes'; input: FileSetAttributesInput; output: FsPlan } + | { type: 'files.undo'; input: FileUndoInput; output: FsPlan } ; // ===== Wire Method Mappings ===== @@ -5330,11 +5617,20 @@ export const WIRE_METHODS = { libraryActions: { 'adapters.update': 'action:adapters.update.input', 'config.library.update': 'action:config.library.update.input', + 'files.archive': 'action:files.archive.input', 'files.copy': 'action:files.copy.input', 'files.createFolder': 'action:files.createFolder.input', 'files.delete': 'action:files.delete.input', + 'files.extract': 'action:files.extract.input', + 'files.flatten': 'action:files.flatten.input', + 'files.link': 'action:files.link.input', 'files.merge': 'action:files.merge.input', + 'files.organize': 'action:files.organize.input', 'files.rename': 'action:files.rename.input', + 'files.rename_batch': 'action:files.rename_batch.input', + 'files.set_attributes': 'action:files.set_attributes.input', + 'files.trash_empty': 'action:files.trash_empty.input', + 'files.undo': 'action:files.undo.input', 'indexing.start': 'action:indexing.start.input', 'indexing.startup': 'action:indexing.startup.input', 'jobs.cancel': 'action:jobs.cancel.input', @@ -5414,9 +5710,11 @@ export const WIRE_METHODS = { 'files.content_kind_stats': 'query:files.content_kind_stats', 'files.directory_listing': 'query:files.directory_listing', 'files.media_listing': 'query:files.media_listing', + 'files.trash_list': 'query:files.trash_list', 'jobs.active': 'query:jobs.active', 'jobs.get_copy_metadata': 'query:jobs.get_copy_metadata', 'jobs.info': 'query:jobs.info', + 'jobs.journal': 'query:jobs.journal', 'jobs.list': 'query:jobs.list', 'libraries.info': 'query:libraries.info', 'paths.compare': 'query:paths.compare', @@ -5448,15 +5746,33 @@ export const WIRE_METHODS = { }, libraryValidates: { + 'files.archive': 'validate:files.archive', 'files.copy': 'validate:files.copy', 'files.delete': 'validate:files.delete', + 'files.extract': 'validate:files.extract', + 'files.flatten': 'validate:files.flatten', + 'files.link': 'validate:files.link', 'files.merge': 'validate:files.merge', + 'files.organize': 'validate:files.organize', + 'files.rename': 'validate:files.rename', + 'files.rename_batch': 'validate:files.rename_batch', + 'files.set_attributes': 'validate:files.set_attributes', + 'files.undo': 'validate:files.undo', }, libraryPreviews: { + 'files.archive': 'preview:files.archive', 'files.copy': 'preview:files.copy', 'files.delete': 'preview:files.delete', + 'files.extract': 'preview:files.extract', + 'files.flatten': 'preview:files.flatten', + 'files.link': 'preview:files.link', 'files.merge': 'preview:files.merge', + 'files.organize': 'preview:files.organize', + 'files.rename': 'preview:files.rename', + 'files.rename_batch': 'preview:files.rename_batch', + 'files.set_attributes': 'preview:files.set_attributes', + 'files.undo': 'preview:files.undo', }, } as const;