diff --git a/core-new/benchmarks/results/shape_large-aggregation-hdd.json b/core-new/benchmarks/results/shape_large-aggregation-hdd.json new file mode 100644 index 000000000..998111c4b --- /dev/null +++ b/core-new/benchmarks/results/shape_large-aggregation-hdd.json @@ -0,0 +1,33 @@ +{ + "runs": [ + { + "dirs": 16507, + "dirs_per_s": 275.1166666666667, + "durations": { + "content_s": 0.0, + "discovery_s": 175.4, + "processing_s": 60.0, + "total_s": 60.0 + }, + "errors": 0, + "files": 100000, + "files_per_s": 1666.6666666666667, + "meta": { + "hardware_label": "External HDD (Seagate)", + "host": { + "cpu_model": "Apple M3 Max", + "cpu_physical_cores": 16, + "memory_total_gb": 48 + }, + "id": "f01681c8-bc82-4e05-a26e-eeaedc3e1239", + "location_paths": [ + "/Volumes/Seagate/benchdata/shape_large" + ], + "recipe_name": "shape_large", + "timestamp_utc": "2025-08-10T09:46:25.346840+00:00" + }, + "scenario": "aggregation", + "total_gb": 11882.38 + } + ] +} \ No newline at end of file diff --git a/core-new/benchmarks/results/shape_large-indexing_discovery-hdd.json b/core-new/benchmarks/results/shape_large-indexing_discovery-hdd.json new file mode 100644 index 000000000..b63a0d234 --- /dev/null +++ b/core-new/benchmarks/results/shape_large-indexing_discovery-hdd.json @@ -0,0 +1,33 @@ +{ + "runs": [ + { + "dirs": 16507, + "dirs_per_s": 281.1, + "durations": { + "content_s": 0.0, + "discovery_s": 58.7, + "processing_s": 51.6, + "total_s": 58.72 + }, + "errors": 0, + "files": 100000, + "files_per_s": 1702.9, + "meta": { + "hardware_label": "External HDD (Seagate)", + "host": { + "cpu_model": "Apple M3 Max", + "cpu_physical_cores": 16, + "memory_total_gb": 48 + }, + "id": "8085806f-d7e3-4cfc-9c66-9b0ec782e3be", + "location_paths": [ + "/Volumes/Seagate/benchdata/shape_large" + ], + "recipe_name": "shape_large", + "timestamp_utc": "2025-08-10T09:42:47.682209+00:00" + }, + "scenario": "indexing-discovery", + "total_gb": 11882.38 + } + ] +} \ No newline at end of file diff --git a/core-new/benchmarks/results/shape_medium-aggregation-hdd.json b/core-new/benchmarks/results/shape_medium-aggregation-hdd.json new file mode 100644 index 000000000..3ffa89fa2 --- /dev/null +++ b/core-new/benchmarks/results/shape_medium-aggregation-hdd.json @@ -0,0 +1,33 @@ +{ + "runs": [ + { + "dirs": 1847, + "dirs_per_s": 302.78688524590166, + "durations": { + "content_s": 0.0, + "discovery_s": 7.4, + "processing_s": 6.1, + "total_s": 6.1 + }, + "errors": 0, + "files": 20000, + "files_per_s": 3278.688524590164, + "meta": { + "hardware_label": "External HDD (Seagate)", + "host": { + "cpu_model": "Apple M3 Max", + "cpu_physical_cores": 16, + "memory_total_gb": 48 + }, + "id": "d209e68e-f366-44cb-8249-d6fecc68e6d8", + "location_paths": [ + "/Volumes/Seagate/benchdata/shape_medium" + ], + "recipe_name": "shape_medium", + "timestamp_utc": "2025-08-10T09:46:36.101659+00:00" + }, + "scenario": "aggregation", + "total_gb": 375.63 + } + ] +} \ No newline at end of file diff --git a/core-new/benchmarks/results/shape_medium-content_identification-ssd.json b/core-new/benchmarks/results/shape_medium-content_identification-ssd.json new file mode 100644 index 000000000..adaa98a52 --- /dev/null +++ b/core-new/benchmarks/results/shape_medium-content_identification-ssd.json @@ -0,0 +1,33 @@ +{ + "runs": [ + { + "dirs": 1847, + "dirs_per_s": 6.948833709556057, + "durations": { + "content_s": 265.8, + "discovery_s": 273.4, + "processing_s": 272.1, + "total_s": 265.8 + }, + "errors": 0, + "files": 20000, + "files_per_s": 75.24454477050413, + "meta": { + "hardware_label": "Internal SSD (Macintosh HD)", + "host": { + "cpu_model": "Apple M3 Max", + "cpu_physical_cores": 16, + "memory_total_gb": 48 + }, + "id": "62872b39-d91b-4195-89e5-d63986c516ea", + "location_paths": [ + "/Users/jamespine/Projects/spacedrive/core-new/benchdata/shape_medium" + ], + "recipe_name": "shape_medium", + "timestamp_utc": "2025-08-10T09:40:28.247578+00:00" + }, + "scenario": "content-identification", + "total_gb": 375.63 + } + ] +} \ No newline at end of file diff --git a/core-new/benchmarks/results/shape_medium-indexing_discovery-hdd.json b/core-new/benchmarks/results/shape_medium-indexing_discovery-hdd.json new file mode 100644 index 000000000..35a4d854c --- /dev/null +++ b/core-new/benchmarks/results/shape_medium-indexing_discovery-hdd.json @@ -0,0 +1,33 @@ +{ + "runs": [ + { + "dirs": 1847, + "dirs_per_s": 81.3, + "durations": { + "content_s": 0.0, + "discovery_s": 22.7, + "processing_s": 6.5, + "total_s": 22.71 + }, + "errors": 0, + "files": 20000, + "files_per_s": 880.9, + "meta": { + "hardware_label": "External HDD (Seagate)", + "host": { + "cpu_model": "Apple M3 Max", + "cpu_physical_cores": 16, + "memory_total_gb": 48 + }, + "id": "0b40d364-b963-436a-88fd-5f9f2ba9de3b", + "location_paths": [ + "/Volumes/Seagate/benchdata/shape_medium" + ], + "recipe_name": "shape_medium", + "timestamp_utc": "2025-08-10T09:43:15.290493+00:00" + }, + "scenario": "indexing-discovery", + "total_gb": 375.63 + } + ] +} \ No newline at end of file diff --git a/core-new/benchmarks/results/shape_small-aggregation-hdd.json b/core-new/benchmarks/results/shape_small-aggregation-hdd.json new file mode 100644 index 000000000..1742cf187 --- /dev/null +++ b/core-new/benchmarks/results/shape_small-aggregation-hdd.json @@ -0,0 +1,33 @@ +{ + "runs": [ + { + "dirs": 72, + "dirs_per_s": 60.0, + "durations": { + "content_s": 0.0, + "discovery_s": 1.7, + "processing_s": 1.2, + "total_s": 1.2 + }, + "errors": 0, + "files": 5000, + "files_per_s": 4166.666666666667, + "meta": { + "hardware_label": "External HDD (Seagate)", + "host": { + "cpu_model": "Apple M3 Max", + "cpu_physical_cores": 16, + "memory_total_gb": 48 + }, + "id": "acc2002e-ac20-4e83-a772-e8e15a46abe7", + "location_paths": [ + "/Volumes/Seagate/benchdata/shape_small" + ], + "recipe_name": "shape_small", + "timestamp_utc": "2025-08-10T09:46:40.825505+00:00" + }, + "scenario": "aggregation", + "total_gb": 8.21 + } + ] +} \ No newline at end of file diff --git a/core-new/benchmarks/results/shape_small-content_identification-ssd.json b/core-new/benchmarks/results/shape_small-content_identification-ssd.json new file mode 100644 index 000000000..82301139a --- /dev/null +++ b/core-new/benchmarks/results/shape_small-content_identification-ssd.json @@ -0,0 +1,33 @@ +{ + "runs": [ + { + "dirs": 72, + "dirs_per_s": 1.0480349344978166, + "durations": { + "content_s": 68.7, + "discovery_s": 70.2, + "processing_s": 69.9, + "total_s": 68.7 + }, + "errors": 0, + "files": 5000, + "files_per_s": 72.78020378457059, + "meta": { + "hardware_label": "Internal SSD (Macintosh HD)", + "host": { + "cpu_model": "Apple M3 Max", + "cpu_physical_cores": 16, + "memory_total_gb": 48 + }, + "id": "cfe4988b-bfa4-4b9f-9632-4bc02d9738c7", + "location_paths": [ + "/Users/jamespine/Projects/spacedrive/core-new/benchdata/shape_small" + ], + "recipe_name": "shape_small", + "timestamp_utc": "2025-08-10T09:41:41.304584+00:00" + }, + "scenario": "content-identification", + "total_gb": 8.21 + } + ] +} \ No newline at end of file diff --git a/core-new/benchmarks/results/shape_small-indexing_discovery-hdd.json b/core-new/benchmarks/results/shape_small-indexing_discovery-hdd.json new file mode 100644 index 000000000..869725133 --- /dev/null +++ b/core-new/benchmarks/results/shape_small-indexing_discovery-hdd.json @@ -0,0 +1,33 @@ +{ + "runs": [ + { + "dirs": 72, + "dirs_per_s": 26.3, + "durations": { + "content_s": 0.0, + "discovery_s": 2.7, + "processing_s": 1.1, + "total_s": 2.73 + }, + "errors": 0, + "files": 5000, + "files_per_s": 1828.2, + "meta": { + "hardware_label": "External HDD (Seagate)", + "host": { + "cpu_model": "Apple M3 Max", + "cpu_physical_cores": 16, + "memory_total_gb": 48 + }, + "id": "edb6828c-3605-4dde-9147-e97d2546a9b3", + "location_paths": [ + "/Volumes/Seagate/benchdata/shape_small" + ], + "recipe_name": "shape_small", + "timestamp_utc": "2025-08-10T09:43:23.958382+00:00" + }, + "scenario": "indexing-discovery", + "total_gb": 8.21 + } + ] +} \ No newline at end of file diff --git a/core-new/benchmarks/results/whitepaper_metrics.csv b/core-new/benchmarks/results/whitepaper_metrics.csv index 1ffa9271e..380df3845 100644 --- a/core-new/benchmarks/results/whitepaper_metrics.csv +++ b/core-new/benchmarks/results/whitepaper_metrics.csv @@ -1,8 +1,10 @@ -Phase,Hardware,Files_per_s,GB_per_s,Files,Dirs,GB,Errors,Recipe -Discovery,Internal SSD (Macintosh HD),1662.5,197.55,100000,16507,11882.38,0,shape_large -Discovery,Internal SSD (Macintosh HD),2720.6,51.11,20000,1847,375.63,0,shape_medium -Discovery,Internal SSD (Macintosh HD),3588.6,5.91,5000,72,8.21,0,shape_small -Processing,Internal SSD (Macintosh HD),1890.4,224.62,100000,16507,11882.38,0,shape_large -Processing,Internal SSD (Macintosh HD),3278.7,61.58,20000,1847,375.63,0,shape_medium -Processing,Internal SSD (Macintosh HD),4166.7,6.84,5000,72,8.21,0,shape_small -Content Identification,Internal SSD (Macintosh HD),74.6,8.86,100000,16507,11882.38,0,shape_large +Phase,Hardware,Files_per_s,GB_per_s,Files,Dirs,GB,Errors,Recipe,Duration_s +Discovery,Internal SSD (Macintosh HD),1662.5,197.38,100000,16507,11882.38,0,shape_large,60.2 +Discovery,Internal SSD (Macintosh HD),2720.6,50.76,20000,1847,375.63,0,shape_medium,7.4 +Discovery,Internal SSD (Macintosh HD),3588.6,5.86,5000,72,8.21,0,shape_small,1.4 +Processing,Internal SSD (Macintosh HD),1890.4,224.62,100000,16507,11882.38,0,shape_large,52.9 +Processing,Internal SSD (Macintosh HD),3278.7,61.58,20000,1847,375.63,0,shape_medium,6.1 +Processing,Internal SSD (Macintosh HD),4166.7,6.84,5000,72,8.21,0,shape_small,1.2 +Content Identification,Internal SSD (Macintosh HD),74.6,8.86,100000,16507,11882.38,0,shape_large,1340.8 +Content Identification,Internal SSD (Macintosh HD),75.2,1.41,20000,1847,375.63,0,shape_medium,265.8 +Content Identification,Internal SSD (Macintosh HD),72.8,0.12,5000,72,8.21,0,shape_small,68.7 diff --git a/core-new/benchmarks/src/reporting/csv.rs b/core-new/benchmarks/src/reporting/csv.rs index 240ec76e2..dd9856d87 100644 --- a/core-new/benchmarks/src/reporting/csv.rs +++ b/core-new/benchmarks/src/reporting/csv.rs @@ -36,7 +36,7 @@ impl Reporter for CsvReporter { fn render(&self, runs: &[BenchmarkRun], dest: &Path) -> Result<()> { // Collect all data rows first so we can sort them - let mut data_rows: Vec<(String, String, f64, f64, u64, u64, f64, u64, String)> = Vec::new(); + let mut data_rows: Vec<(String, String, f64, f64, u64, u64, f64, u64, String, f64)> = Vec::new(); // Process each run individually for run in runs { @@ -110,13 +110,17 @@ impl Reporter for CsvReporter { .clone() .unwrap_or_else(|| "Unknown".to_string()); - // Calculate GB/s - let gb_per_s = if let Some(duration) = durations.total_s { - if duration > 0.0 { - total_gb / duration - } else { - 0.0 - } + // Get phase-specific duration or fall back to total + let phase_duration = match scenario { + "indexing_discovery" => durations.discovery_s.unwrap_or(durations.total_s.unwrap_or(0.0)), + "aggregation" => durations.processing_s.unwrap_or(durations.total_s.unwrap_or(0.0)), + "content_identification" => durations.content_s.unwrap_or(durations.total_s.unwrap_or(0.0)), + _ => durations.total_s.unwrap_or(0.0), + }; + + // Calculate GB/s using phase-specific duration + let gb_per_s = if phase_duration > 0.0 { + total_gb / phase_duration } else { 0.0 }; @@ -131,6 +135,7 @@ impl Reporter for CsvReporter { total_gb, errors, meta.recipe_name.clone(), + phase_duration, )); } @@ -144,14 +149,14 @@ impl Reporter for CsvReporter { // Build CSV output let mut rows = - vec!["Phase,Hardware,Files_per_s,GB_per_s,Files,Dirs,GB,Errors,Recipe".to_string()]; + vec!["Phase,Hardware,Files_per_s,GB_per_s,Files,Dirs,GB,Errors,Recipe,Duration_s".to_string()]; - for (phase, hardware, files_per_s, gb_per_s, files, dirs, total_gb, errors, recipe) in + for (phase, hardware, files_per_s, gb_per_s, files, dirs, total_gb, errors, recipe, duration) in data_rows { rows.push(format!( - "{},{},{:.1},{:.2},{},{},{:.2},{},{}", - phase, hardware, files_per_s, gb_per_s, files, dirs, total_gb, errors, recipe + "{},{},{:.1},{:.2},{},{},{:.2},{},{},{:.1}", + phase, hardware, files_per_s, gb_per_s, files, dirs, total_gb, errors, recipe, duration )); } diff --git a/core-new/whitepaper/spacedrive.pdf b/core-new/whitepaper/spacedrive.pdf index 7743b1492..0b2c0dfa8 100644 Binary files a/core-new/whitepaper/spacedrive.pdf and b/core-new/whitepaper/spacedrive.pdf differ diff --git a/core-new/whitepaper/spacedrive.tex b/core-new/whitepaper/spacedrive.tex index 8377c90db..ee8e23722 100644 --- a/core-new/whitepaper/spacedrive.tex +++ b/core-new/whitepaper/spacedrive.tex @@ -2101,13 +2101,13 @@ Systematic performance testing demonstrates Spacedrive's efficiency across criti \label{tab:performance} \begin{tabular}{llrrrrrrl} \toprule - \textbf{Phase} & \textbf{Hardware} & \textbf{Files/s} & \textbf{GB/s} & \textbf{Files} & \textbf{Dirs} & \textbf{GB} & \textbf{Recipe} \\ + \textbf{Phase} & \textbf{Hardware} & \textbf{Files/s} & \textbf{Duration/s} & \textbf{GB/s} & \textbf{Files} & \textbf{Dirs} & \textbf{GB} & \textbf{Recipe} \\ \midrule \multicolumn{8}{l}{\textit{Indexing}} \\ \csvreader[ late after line=\\ - ]{../benchmarks/results/whitepaper_metrics.csv}{Phase=\Phase,Hardware=\Hardware,Files_per_s=\Fps,GB_per_s=\Gbps,Files=\Files,Dirs=\Dirs,GB=\Gb,Recipe=\Recipe}{% - \Phase & \Hardware & \num{\Fps} & \num{\Gbps} & \num{\Files} & \num{\Dirs} & \num{\Gb} & \url{\Recipe}} + ]{../benchmarks/results/whitepaper_metrics.csv}{Phase=\Phase,Hardware=\Hardware,Files_per_s=\Fps,Duration_s=\Duration,GB_per_s=\Gbps,Files=\Files,Dirs=\Dirs,GB=\Gb,Recipe=\Recipe}{% + \Phase & \Hardware & \num{\Fps} & \num{\Duration} & \num{\Gbps} & \num{\Files} & \num{\Dirs} & \num{\Gb} & \url{\Recipe}} \bottomrule \end{tabular} \end{table*}