From f6e99da024d619e98bb2448015b49feb03aa9f05 Mon Sep 17 00:00:00 2001 From: Jamie Pine <32987599+jamiepine@users.noreply.github.com> Date: Fri, 1 Apr 2022 07:28:09 -0700 Subject: [PATCH] fixed indexer time + added thumbnail generation --- core/src/file/indexer.rs | 15 ++++++----- core/src/file/thumb.rs | 56 +++++++++++++++++++++++++++++++++------- core/src/job/jobs.rs | 1 + core/src/util/mod.rs | 2 +- core/src/util/time.rs | 17 ------------ 5 files changed, 58 insertions(+), 33 deletions(-) delete mode 100644 core/src/util/time.rs diff --git a/core/src/file/indexer.rs b/core/src/file/indexer.rs index 638f300e5..010c20ec6 100644 --- a/core/src/file/indexer.rs +++ b/core/src/file/indexer.rs @@ -1,9 +1,10 @@ use crate::job::jobs::JobReportUpdate; use crate::job::{jobs::Job, worker::WorkerContext}; use crate::sys::locations::{create_location, LocationResource}; -use crate::util::time; + use crate::CoreContext; use anyhow::{anyhow, Result}; +use chrono::{DateTime, SecondsFormat, Utc}; use serde::{Deserialize, Serialize}; use std::ffi::OsStr; use std::{collections::HashMap, fs, path::Path, path::PathBuf, time::Instant}; @@ -167,11 +168,12 @@ pub async fn scan_path( } let raw_sql = format!( r#" - INSERT INTO file_paths (id, is_dir, location_id, materialized_path, name, extension, parent_id, date_indexed, temp_checksum) + INSERT INTO file_paths (id, is_dir, location_id, materialized_path, name, extension, parent_id, date_created, temp_checksum) VALUES {} "#, files.join(", ") ); + // println!("{}", raw_sql); let count = db._execute_raw(&raw_sql).await; println!("Inserted {:?} records", count); } @@ -215,8 +217,11 @@ fn prepare_values( } }; + let date_created: DateTime = metadata.created().unwrap().into(); + let parsed_date_created = date_created.to_rfc3339_opts(SecondsFormat::Millis, true); + let values = format!( - "({}, {}, {}, \"{}\", \"{}\", \"{}\", {}, \"{}\", \"{}\")", + "({}, {}, {}, \"{}\", \"{}\", \"{}\", {},\"{}\", \"{}\")", id, metadata.is_dir(), location.id, @@ -227,9 +232,7 @@ fn prepare_values( .clone() .map(|id| format!("\"{}\"", &id)) .unwrap_or("NULL".to_string()), - &time::system_time_to_date_time(metadata.created()) - .unwrap() - .to_string(), + parsed_date_created, partial_checksum ); diff --git a/core/src/file/thumb.rs b/core/src/file/thumb.rs index f9b615fca..670e77347 100644 --- a/core/src/file/thumb.rs +++ b/core/src/file/thumb.rs @@ -1,5 +1,7 @@ +use crate::job::jobs::JobReportUpdate; use crate::prisma::FilePathData; use crate::state::client; +use crate::sys; use crate::{ job::{jobs::Job, worker::WorkerContext}, prisma::FilePath, @@ -8,6 +10,7 @@ use crate::{ use anyhow::Result; use image::*; use prisma_client_rust::or; +use std::fs; use std::path::Path; use webp::*; @@ -16,26 +19,58 @@ pub struct ThumbnailJob { pub location_id: i32, } -static THUMBNAIL_SIZE_FACTOR: f32 = 1.0; +static THUMBNAIL_SIZE_FACTOR: f32 = 0.2; static THUMBNAIL_QUALITY: f32 = 30.0; static CACHE_DIR_NAME: &str = "thumbnails"; #[async_trait::async_trait] impl Job for ThumbnailJob { async fn run(&self, ctx: WorkerContext) -> Result<()> { + let config = client::get(); let core_ctx = ctx.core_ctx.clone(); + + let location = sys::locations::get_location(&core_ctx, self.location_id).await?; + + fs::create_dir_all( + Path::new(&config.data_path) + .join(CACHE_DIR_NAME) + .join(format!("{}", self.location_id)), + ) + .unwrap(); + + let root_path = location.path.unwrap(); + let image_files = get_images(&core_ctx, self.location_id).await?; + let location_id = location.id.clone(); + println!("Found {:?} files", image_files.len()); - for image_file in image_files { - generate_thumbnail( - &image_file.materialized_path, - &image_file.extension.unwrap(), - self.location_id, - ) - .unwrap(); - } + tokio::task::spawn_blocking(move || { + ctx.progress(vec![ + JobReportUpdate::TaskCount(image_files.len()), + JobReportUpdate::Message(format!( + "Preparing to process {} files", + image_files.len() + )), + ]); + + for (i, image_file) in image_files.iter().enumerate() { + ctx.progress(vec![JobReportUpdate::Message(format!( + "Processing {}", + image_file.materialized_path.clone() + ))]); + let path = format!("{}{}", root_path, image_file.materialized_path); + let checksum = image_file.temp_checksum.as_ref().unwrap(); + + generate_thumbnail(&path, checksum, location_id).unwrap(); + + ctx.progress(vec![JobReportUpdate::CompletedTaskCount(i + 1)]); + } + }) + .await + .unwrap(); + Ok(()) } } @@ -67,6 +102,9 @@ pub fn generate_thumbnail( .join(format!("{}", location_id)) .join(file_hash) .with_extension("webp"); + + println!("Writing to {}", output_path.display()); + std::fs::write(&output_path, &*webp).unwrap(); Ok(()) diff --git a/core/src/job/jobs.rs b/core/src/job/jobs.rs index f04f4c3c7..cf595d8a9 100644 --- a/core/src/job/jobs.rs +++ b/core/src/job/jobs.rs @@ -78,6 +78,7 @@ impl Jobs { } } +#[derive(Debug)] pub enum JobReportUpdate { TaskCount(usize), CompletedTaskCount(usize), diff --git a/core/src/util/mod.rs b/core/src/util/mod.rs index 077885d7b..8b1378917 100644 --- a/core/src/util/mod.rs +++ b/core/src/util/mod.rs @@ -1 +1 @@ -pub mod time; + diff --git a/core/src/util/time.rs b/core/src/util/time.rs deleted file mode 100644 index f96859b8e..000000000 --- a/core/src/util/time.rs +++ /dev/null @@ -1,17 +0,0 @@ -use anyhow::Result; -use chrono::NaiveDateTime; -use std::io; -use std::time::{SystemTime, UNIX_EPOCH}; - -pub fn system_time_to_date_time( - system_time: io::Result, -) -> Result { - // extract system time or resort to current time if failure - let system_time = system_time.unwrap_or(SystemTime::now()); - let std_duration = system_time.duration_since(UNIX_EPOCH)?; - let chrono_duration = chrono::Duration::from_std(std_duration)?; - let unix = NaiveDateTime::from_timestamp(0, 0); - let naive = unix + chrono_duration; - // let date_time: DateTime = Utc.from_local_datetime(&naive).unwrap(); - Ok(naive) -}