From cabc00d5dda36fb0b3798fba6bfe3249be608256 Mon Sep 17 00:00:00 2001 From: Jamie Pine Date: Thu, 14 Apr 2022 13:46:54 -0700 Subject: [PATCH] ffmpeg progress --- .vscode/settings.json | 2 + Cargo.lock | 53 ++++ apps/debug/Cargo.toml | 4 +- apps/debug/src/checksum.rs | 91 ------- apps/debug/src/main.rs | 21 +- apps/debug/src/webp.rs | 71 ------ core/Cargo.toml | 1 + .../migrations/20220414190025_/migration.sql | 29 +++ core/prisma/schema.prisma | 19 +- core/src/db/migrate.rs | 17 +- core/src/encode/metadata.rs | 136 ++++++++++ core/src/encode/mod.rs | 1 + core/src/encode/video_thumb.rs | 0 core/src/encode/video_thumbstrip.rs | 0 core/src/file/{ => cas}/checksum.rs | 13 - core/src/file/cas/identifier.rs | 0 core/src/file/cas/mod.rs | 1 + core/src/file/explorer/mod.rs | 59 +---- core/src/file/explorer/open.rs | 61 +++++ core/src/file/indexer.rs | 8 +- core/src/file/mod.rs | 2 +- core/src/lib.rs | 7 +- packages/config/tsconfig/nextjs.json | 2 +- pnpm-lock.yaml | 241 +++++++++++++++++- 24 files changed, 568 insertions(+), 271 deletions(-) delete mode 100644 apps/debug/src/checksum.rs delete mode 100644 apps/debug/src/webp.rs create mode 100644 core/prisma/migrations/20220414190025_/migration.sql create mode 100644 core/src/encode/metadata.rs create mode 100644 core/src/encode/video_thumb.rs create mode 100644 core/src/encode/video_thumbstrip.rs rename core/src/file/{ => cas}/checksum.rs (85%) create mode 100644 core/src/file/cas/identifier.rs create mode 100644 core/src/file/cas/mod.rs create mode 100644 core/src/file/explorer/open.rs diff --git a/.vscode/settings.json b/.vscode/settings.json index f05a1b362..319a86bb8 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,7 +1,9 @@ { "cSpell.words": [ + "creationdate", "ipfs", "Keepsafe", + "quicktime", "repr", "tailwindcss" ], diff --git a/Cargo.lock b/Cargo.lock index 24b65393e..a514d7d0f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -546,6 +546,12 @@ version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" +[[package]] +name = "bitstream-io" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a429905f63bae528a4afe5e7520089139a7694e910f9a12e89010d738b9cca2" + [[package]] name = "bitvec" version = "0.22.3" @@ -1555,8 +1561,11 @@ name = "debug" version = "0.1.0" dependencies = [ "anyhow", + "chrono", "data-encoding", + "ffmpeg-next", "kamadak-exif", + "matroska", "mime", "ring 0.16.20", "sdcore", @@ -1986,6 +1995,31 @@ dependencies = [ "instant", ] +[[package]] +name = "ffmpeg-next" +version = "5.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "585e5eaf57eceaa199ba6f6a1f46bdad7992930bb7b45b40275d9445b5ba2bc8" +dependencies = [ + "bitflags 1.3.2", + "ffmpeg-sys-next", + "libc", +] + +[[package]] +name = "ffmpeg-sys-next" +version = "5.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ba12dea33516e30c160ce557c7e43dd857276368eb1cd0eef4fce6529f2dee5" +dependencies = [ + "bindgen", + "cc", + "libc", + "num_cpus", + "pkg-config", + "vcpkg", +] + [[package]] name = "field-offset" version = "0.3.4" @@ -4198,6 +4232,17 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a3e378b66a060d48947b590737b30a1be76706c8dd7b8ba0f2fe3989c68a853f" +[[package]] +name = "matroska" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee093e2ef3d0e7a8062daacfb315c8adf19b7dc96fd4bd33a9dff4c9b7e35d77" +dependencies = [ + "bitstream-io", + "phf 0.10.1", + "time 0.3.9", +] + [[package]] name = "maybe-uninit" version = "2.0.0" @@ -6596,6 +6641,7 @@ dependencies = [ "data-encoding", "dyn-clone", "env_logger", + "ffmpeg-next", "futures", "hostname 0.3.1", "hotwatch", @@ -7869,8 +7915,15 @@ checksum = "c2702e08a7a860f005826c6815dcac101b19b5eb330c27fe4a5928fec1d20ddd" dependencies = [ "libc", "num_threads", + "time-macros", ] +[[package]] +name = "time-macros" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42657b1a6f4d817cda8e7a0ace261fe0cc946cf3a80314390b22cc61ae080792" + [[package]] name = "tinyvec" version = "1.5.1" diff --git a/apps/debug/Cargo.toml b/apps/debug/Cargo.toml index d57a34e68..1782be1c3 100644 --- a/apps/debug/Cargo.toml +++ b/apps/debug/Cargo.toml @@ -13,7 +13,9 @@ ring = "0.16.20" thumbnailer = "0.4.0" mime = "0.3.16" sha256 = "1.0.3" -# ffmpeg-next = "5.0.3" +ffmpeg-next = "5.0.3" sdcore = { path = "../../core" } uhlc = "0.4.1" +matroska = "0.11.0" tokio = { version = "1.17.0", features = ["sync", "rt"] } +chrono = "0.4.19" diff --git a/apps/debug/src/checksum.rs b/apps/debug/src/checksum.rs deleted file mode 100644 index 6bb25a9bd..000000000 --- a/apps/debug/src/checksum.rs +++ /dev/null @@ -1,91 +0,0 @@ -#![allow(dead_code)] - -use anyhow::Result; -use data_encoding::HEXLOWER; -use ring::digest::{Context, SHA256}; -use std::fs::{self, File}; -use std::io::{BufReader, Read}; -use std::os::unix::prelude::FileExt; -use std::time::Instant; - -static BIG_FILE: &str = "/Users/jamie/Movies/2022-03-08 06-08-35.mkv"; -static LITTLE_FILE: &str = "/Users/jamie/Movies/client_state.json"; - -pub fn do_thing() { - println!("Generating hash from file {:?}", BIG_FILE); - - let start = Instant::now(); - let checksum = sampled_checksum(BIG_FILE).unwrap(); - println!( - "Sampled checksum completed in {:?} {}", - start.elapsed(), - checksum - ); - - // let start = Instant::now(); - // let checksum = full_checksum(BIG_FILE).unwrap(); - // println!( - // "Full checksum completed in {:?} {}", - // start.elapsed(), - // checksum - // ); -} - -static SAMPLE_COUNT: u64 = 4; -static SAMPLE_SIZE: u64 = 10000; - -pub fn sampled_checksum(path: &str) -> Result { - // get file size - let metadata = fs::metadata(path)?; - let size = metadata.len(); - // open file reference - let file = File::open(path)?; - - let mut context = Context::new(&SHA256); - - // include the file size in the checksum - context.update(&size.to_le_bytes()); - - // if size is small enough, just read the whole thing - if SAMPLE_COUNT * SAMPLE_SIZE > size { - let mut buf = vec![0u8; size.try_into()?]; - file.read_exact_at(&mut buf, 0)?; - context.update(&buf); - } else { - // loop over samples - for i in 0..SAMPLE_COUNT { - let start_point = (size / SAMPLE_COUNT) * i; - let mut buf = vec![0u8; SAMPLE_SIZE.try_into()?]; - file.read_exact_at(&mut buf, start_point)?; - context.update(&buf); - } - - // sample end of file - let mut buf = vec![0u8; SAMPLE_SIZE.try_into()?]; - file.read_exact_at(&mut buf, size - SAMPLE_SIZE)?; - context.update(&buf); - } - let digest = context.finish(); - let hex = HEXLOWER.encode(digest.as_ref()); - - Ok(hex) -} - -pub fn full_checksum(path: &str) -> Result { - // read file as buffer and convert to digest - let mut reader = BufReader::new(File::open(path).unwrap()); - let mut context = Context::new(&SHA256); - let mut buffer = [0; 1024]; - loop { - let count = reader.read(&mut buffer)?; - if count == 0 { - break; - } - context.update(&buffer[..count]); - } - let digest = context.finish(); - // create a lowercase hash from - let hex = HEXLOWER.encode(digest.as_ref()); - - Ok(hex) -} diff --git a/apps/debug/src/main.rs b/apps/debug/src/main.rs index 2bbf73538..580685540 100644 --- a/apps/debug/src/main.rs +++ b/apps/debug/src/main.rs @@ -1,23 +1,6 @@ #![allow(dead_code)] -pub mod checksum; - -use sdcore::{prisma, sync::engine::test, sync::FakeCoreContext}; +// use sdcore::{prisma, sync::engine::test, sync::FakeCoreContext}; #[tokio::main] -async fn main() { - let db = prisma::new_client().await.unwrap(); - let ctx = FakeCoreContext {}; - test(&ctx).await; - // checksum::do_thing(); - - // create an HLC with a generated UUID and relying on SystemTime::now() - // let hlc = HLC::default(); - - // // generate timestamps - // let ts1 = hlc.new_timestamp(); - // let ts2 = hlc.new_timestamp(); - - // println!("ts1 {}", ts1); - // println!("ts2 {}", ts2); -} +async fn main() {} diff --git a/apps/debug/src/webp.rs b/apps/debug/src/webp.rs deleted file mode 100644 index c76622f46..000000000 --- a/apps/debug/src/webp.rs +++ /dev/null @@ -1,71 +0,0 @@ -use anyhow::Result; -use mime; -use std::fs::File; -use std::io::BufReader; -use std::io::Cursor; -use std::io::Write; -use std::path; -use thumbnailer::{create_thumbnails, ThumbnailSize}; - -static FILE: &str = "/Users/jamie/Desktop/FF0QjSoUcAUEciH.jpg"; - -pub fn main() { - let start = std::time::Instant::now(); - let file = File::open(FILE).unwrap(); - let reader = BufReader::new(file); - - let mut thumbnails = - create_thumbnails(reader, mime::IMAGE_JPEG, [ThumbnailSize::Small]).unwrap(); - - let thumbnail = thumbnails.pop().unwrap(); - - let mut buf = Cursor::new(Vec::new()); - - thumbnail.write_png(&mut buf).unwrap(); - - // write buf to file - File::create(path::Path::new( - "/Users/jamie/Desktop/FF0QjSoUcAUEciH_thumb.png", - )) - .unwrap() - .write_all(&buf.get_ref()) - .unwrap(); - println!("done {:?}", start.elapsed()); -} - -use image::*; -use std::path::Path; -use webp::*; - -fn main() { - let _start = std::time::Instant::now(); - let start = std::time::Instant::now(); - // Using `image` crate, open the included .jpg file - let img = image::open("/Users/jamie/Desktop/IMG_5812.jpeg").unwrap(); - // let (w, h) = img.dimensions(); - println!("loaded image {:?}", start.elapsed()); - let start = std::time::Instant::now(); - // Optionally, resize the existing photo and convert back into DynamicImage - // let size_factor = 1.0; - // let img: DynamicImage = image::DynamicImage::ImageRgba8(imageops::resize( - // &img, - // (w as f64 * size_factor) as u32, - // (h as f64 * size_factor) as u32, - // imageops::FilterType::Triangle, - // )); - println!("resized image {:?}", start.elapsed()); - let start = std::time::Instant::now(); - // Create the WebP encoder for the above image - let encoder: Encoder = Encoder::from_image(&img).unwrap(); - - // Encode the image at a specified quality 0-100 - let webp: WebPMemory = encoder.encode(30f32); - println!("encoded image {:?}", start.elapsed()); - let start = std::time::Instant::now(); - // Define and write the WebP-encoded file to a given path - let output_path = Path::new("/Users/jamie/Desktop/IMG_5812").with_extension("webp"); - std::fs::write(&output_path, &*webp).unwrap(); - - println!("written to file webp {:?}", start.elapsed()); - println!("done in {:?}", _start.elapsed()); -} diff --git a/core/Cargo.toml b/core/Cargo.toml index a8dde21b6..4cef9a2a2 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -50,3 +50,4 @@ dyn-clone = "1.0.5" image = "0.24.1" webp = "0.2.2" uhlc = "0.4.1" +ffmpeg-next = "5.0.3" diff --git a/core/prisma/migrations/20220414190025_/migration.sql b/core/prisma/migrations/20220414190025_/migration.sql new file mode 100644 index 000000000..80563d586 --- /dev/null +++ b/core/prisma/migrations/20220414190025_/migration.sql @@ -0,0 +1,29 @@ +/* + Warnings: + + - You are about to drop the column `capture_device` on the `media_data` table. All the data in the column will be lost. + - You are about to drop the column `duration` on the `media_data` table. All the data in the column will be lost. + +*/ +-- RedefineTables +PRAGMA foreign_keys=OFF; +CREATE TABLE "new_media_data" ( + "id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, + "pixel_width" INTEGER, + "pixel_height" INTEGER, + "longitude" REAL, + "latitude" REAL, + "fps" INTEGER, + "capture_device_make" TEXT, + "capture_device_model" TEXT, + "capture_device_software" TEXT, + "duration_seconds" INTEGER, + "codecs" TEXT, + "streams_json" TEXT, + CONSTRAINT "media_data_id_fkey" FOREIGN KEY ("id") REFERENCES "files" ("id") ON DELETE CASCADE ON UPDATE CASCADE +); +INSERT INTO "new_media_data" ("id", "latitude", "longitude", "pixel_height", "pixel_width") SELECT "id", "latitude", "longitude", "pixel_height", "pixel_width" FROM "media_data"; +DROP TABLE "media_data"; +ALTER TABLE "new_media_data" RENAME TO "media_data"; +PRAGMA foreign_key_check; +PRAGMA foreign_keys=ON; diff --git a/core/prisma/schema.prisma b/core/prisma/schema.prisma index aebbd8296..fc574fa90 100644 --- a/core/prisma/schema.prisma +++ b/core/prisma/schema.prisma @@ -162,13 +162,18 @@ model FilePath { } model MediaData { - id Int @id - pixel_width Int? - pixel_height Int? - longitude Float? - latitude Float? - capture_device String? - duration Int? + id Int @id + pixel_width Int? + pixel_height Int? + longitude Float? + latitude Float? + fps Int? + capture_device_make String? // eg: "Apple" + capture_device_model String? // eg: "iPhone 12" + capture_device_software String? // eg: "12.1.1" + duration_seconds Int? + codecs String? // eg: "h264,acc" + streams Int? // change this relation to File after testing files File? @relation(fields: [id], references: [id], onDelete: Cascade, onUpdate: Cascade) diff --git a/core/src/db/migrate.rs b/core/src/db/migrate.rs index cf5e9cb68..54681f011 100644 --- a/core/src/db/migrate.rs +++ b/core/src/db/migrate.rs @@ -1,14 +1,27 @@ -use crate::file::checksum::sha256_digest; use crate::{prisma, prisma::Migration}; use anyhow::Result; use data_encoding::HEXLOWER; use include_dir::{include_dir, Dir}; +use ring::digest::{Context, Digest, SHA256}; use std::ffi::OsStr; -use std::io::BufReader; +use std::io::{BufReader, Read}; const INIT_MIGRATION: &str = include_str!("../../prisma/migrations/migration_table/migration.sql"); static MIGRATIONS_DIR: Dir = include_dir!("$CARGO_MANIFEST_DIR/prisma/migrations"); +pub fn sha256_digest(mut reader: R) -> Result { + let mut context = Context::new(&SHA256); + let mut buffer = [0; 1024]; + loop { + let count = reader.read(&mut buffer)?; + if count == 0 { + break; + } + context.update(&buffer[..count]); + } + Ok(context.finish()) +} + pub async fn run_migrations(db_url: &str) -> Result<()> { let client = prisma::new_client_with_url(&format!("file:{}", &db_url)).await?; diff --git a/core/src/encode/metadata.rs b/core/src/encode/metadata.rs new file mode 100644 index 000000000..4b94238df --- /dev/null +++ b/core/src/encode/metadata.rs @@ -0,0 +1,136 @@ +extern crate ffmpeg_next as ffmpeg; +use chrono::NaiveDateTime; +use ffmpeg::{dictionary::Iter, format}; +use std::{env, ffi::OsStr, fs, path::Path}; + +#[derive(Default, Debug)] +pub struct MediaItem { + pub created_at: Option, + pub brand: Option, + pub model: Option, + pub duration_seconds: f64, + pub best_video_stream_index: usize, + pub best_audio_stream_index: usize, + pub best_subtitle_stream_index: usize, + pub steams: Vec, +} + +#[derive(Debug)] +pub struct Stream { + pub codec: String, + pub frames: f64, + pub duration_seconds: f64, + pub kind: Option, +} + +#[derive(Debug)] +pub enum StreamKind { + Video(VideoStream), + Audio(AudioStream), +} + +#[derive(Debug)] +pub struct VideoStream { + pub width: u32, + pub height: u32, + pub aspect_ratio: String, + pub format: format::Pixel, + pub bitrate: usize, +} + +#[derive(Debug)] +pub struct AudioStream { + pub channels: u16, + pub format: format::Sample, + pub bitrate: usize, + pub rate: u32, +} + +fn extract(iter: &mut Iter, key: &str) -> Option { + iter.find(|k| k.0.contains(key)).map(|k| k.1.to_string()) +} + +pub fn get_video_metadata(path: &str) -> Result<(), ffmpeg::Error> { + ffmpeg::init().unwrap(); + + let mut name = Path::new(path) + .file_name() + .and_then(OsStr::to_str) + .map(ToString::to_string) + .unwrap_or(String::new()); + + // strip to exact potential date length and attempt to parse + name = name.chars().take(19).collect(); + // specifically OBS uses this format for time, other checks could be added + let potential_date = NaiveDateTime::parse_from_str(&name, "%Y-%m-%d %H-%M-%S"); + + match ffmpeg::format::input(&path) { + Ok(context) => { + let mut media_item = MediaItem::default(); + let metadata = context.metadata(); + let mut iter = metadata.iter(); + + // creation_time is usually the creation date of the file + media_item.created_at = extract(&mut iter, "creation_time"); + // apple photos use "com.apple.quicktime.creationdate", which we care more about than the creation_time + media_item.created_at = extract(&mut iter, "creationdate"); + // fallback to potential time if exists + if media_item.created_at.is_none() { + media_item.created_at = potential_date.map(|d| d.to_string()).ok(); + } + // origin metadata + media_item.brand = extract(&mut iter, "major_brand"); + media_item.brand = extract(&mut iter, "make"); + media_item.model = extract(&mut iter, "model"); + + if let Some(stream) = context.streams().best(ffmpeg::media::Type::Video) { + media_item.best_video_stream_index = stream.index(); + } + if let Some(stream) = context.streams().best(ffmpeg::media::Type::Audio) { + media_item.best_audio_stream_index = stream.index(); + } + if let Some(stream) = context.streams().best(ffmpeg::media::Type::Subtitle) { + media_item.best_subtitle_stream_index = stream.index(); + } + media_item.duration_seconds = + context.duration() as f64 / f64::from(ffmpeg::ffi::AV_TIME_BASE); + + for stream in context.streams() { + let codec = ffmpeg::codec::context::Context::from_parameters(stream.parameters())?; + + let mut stream_item = Stream { + codec: codec.id().name().to_string(), + frames: stream.frames() as f64, + duration_seconds: stream.duration() as f64 * f64::from(stream.time_base()), + kind: None, + }; + + if codec.medium() == ffmpeg::media::Type::Video { + if let Ok(video) = codec.decoder().video() { + stream_item.kind = Some(StreamKind::Video(VideoStream { + bitrate: video.bit_rate(), + format: video.format(), + width: video.width(), + height: video.height(), + aspect_ratio: video.aspect_ratio().to_string(), + })); + } + } else if codec.medium() == ffmpeg::media::Type::Audio { + if let Ok(audio) = codec.decoder().audio() { + stream_item.kind = Some(StreamKind::Audio(AudioStream { + channels: audio.channels(), + bitrate: audio.bit_rate(), + rate: audio.rate(), + format: audio.format(), + })); + } + } + media_item.steams.push(stream_item); + } + println!("{:#?}", media_item); + } + + Err(error) => println!("error: {}", error), + } + Ok(()) +} diff --git a/core/src/encode/mod.rs b/core/src/encode/mod.rs index e69de29bb..de5643fda 100644 --- a/core/src/encode/mod.rs +++ b/core/src/encode/mod.rs @@ -0,0 +1 @@ +pub mod metadata; diff --git a/core/src/encode/video_thumb.rs b/core/src/encode/video_thumb.rs new file mode 100644 index 000000000..e69de29bb diff --git a/core/src/encode/video_thumbstrip.rs b/core/src/encode/video_thumbstrip.rs new file mode 100644 index 000000000..e69de29bb diff --git a/core/src/file/checksum.rs b/core/src/file/cas/checksum.rs similarity index 85% rename from core/src/file/checksum.rs rename to core/src/file/cas/checksum.rs index e812c2532..f90ac797c 100644 --- a/core/src/file/checksum.rs +++ b/core/src/file/cas/checksum.rs @@ -61,16 +61,3 @@ pub fn full_checksum(path: &str) -> Result { Ok(hex) } - -pub fn sha256_digest(mut reader: R) -> io::Result { - let mut context = Context::new(&SHA256); - let mut buffer = [0; 1024]; - loop { - let count = reader.read(&mut buffer)?; - if count == 0 { - break; - } - context.update(&buffer[..count]); - } - Ok(context.finish()) -} diff --git a/core/src/file/cas/identifier.rs b/core/src/file/cas/identifier.rs new file mode 100644 index 000000000..e69de29bb diff --git a/core/src/file/cas/mod.rs b/core/src/file/cas/mod.rs new file mode 100644 index 000000000..239eb62db --- /dev/null +++ b/core/src/file/cas/mod.rs @@ -0,0 +1 @@ +pub mod checksum; diff --git a/core/src/file/explorer/mod.rs b/core/src/file/explorer/mod.rs index 7423ad158..9e4b725e5 100644 --- a/core/src/file/explorer/mod.rs +++ b/core/src/file/explorer/mod.rs @@ -1,58 +1 @@ -use std::path::Path; - -use super::{thumb::THUMBNAIL_CACHE_DIR_NAME, FileError, FilePath}; -use crate::{ - file::DirectoryWithContents, prisma, state::client, sys::locations::get_location, CoreContext, -}; - -pub async fn open_dir( - ctx: &CoreContext, - location_id: &i32, - path: &str, -) -> Result { - let db = &ctx.database; - let config = client::get(); - - // get location - let location = get_location(ctx, location_id.clone()).await?; - - let directory = db - .file_path() - .find_first(vec![ - prisma::FilePath::location_id().equals(location.id), - prisma::FilePath::materialized_path().equals(path.into()), - prisma::FilePath::is_dir().equals(true), - ]) - .exec() - .await? - .ok_or(FileError::DirectoryNotFound(path.to_string()))?; - - let files = db - .file_path() - .find_many(vec![prisma::FilePath::parent_id().equals(directory.id)]) - .exec() - .await?; - - let files: Vec = files.into_iter().map(|l| l.into()).collect(); - - let mut contents: Vec = vec![]; - - for mut file in files { - if file.temp_cas_id.is_some() { - let path = Path::new(&config.data_path) - .join(THUMBNAIL_CACHE_DIR_NAME) - .join(format!("{}", location.id)) - .join(file.temp_cas_id.as_ref().unwrap()) - .with_extension("webp"); - - let exists = path.exists(); - file.has_local_thumbnail = exists; - } - contents.push(file); - } - - Ok(DirectoryWithContents { - directory: directory.into(), - contents, - }) -} +pub mod open; diff --git a/core/src/file/explorer/open.rs b/core/src/file/explorer/open.rs new file mode 100644 index 000000000..ad3be6e79 --- /dev/null +++ b/core/src/file/explorer/open.rs @@ -0,0 +1,61 @@ +use std::path::Path; + +use crate::{ + file::{thumb::THUMBNAIL_CACHE_DIR_NAME, DirectoryWithContents, FileError, FilePath}, + prisma, + state::client, + sys::locations::get_location, + CoreContext, +}; + +pub async fn open_dir( + ctx: &CoreContext, + location_id: &i32, + path: &str, +) -> Result { + let db = &ctx.database; + let config = client::get(); + + // get location + let location = get_location(ctx, location_id.clone()).await?; + + let directory = db + .file_path() + .find_first(vec![ + prisma::FilePath::location_id().equals(location.id), + prisma::FilePath::materialized_path().equals(path.into()), + prisma::FilePath::is_dir().equals(true), + ]) + .exec() + .await? + .ok_or(FileError::DirectoryNotFound(path.to_string()))?; + + let files = db + .file_path() + .find_many(vec![prisma::FilePath::parent_id().equals(directory.id)]) + .exec() + .await?; + + let files: Vec = files.into_iter().map(|l| l.into()).collect(); + + let mut contents: Vec = vec![]; + + for mut file in files { + if file.temp_cas_id.is_some() { + let path = Path::new(&config.data_path) + .join(THUMBNAIL_CACHE_DIR_NAME) + .join(format!("{}", location.id)) + .join(file.temp_cas_id.as_ref().unwrap()) + .with_extension("webp"); + + let exists = path.exists(); + file.has_local_thumbnail = exists; + } + contents.push(file); + } + + Ok(DirectoryWithContents { + directory: directory.into(), + contents, + }) +} diff --git a/core/src/file/indexer.rs b/core/src/file/indexer.rs index a4a1370c8..7f7e5571b 100644 --- a/core/src/file/indexer.rs +++ b/core/src/file/indexer.rs @@ -10,7 +10,7 @@ use std::ffi::OsStr; use std::{collections::HashMap, fs, path::Path, path::PathBuf, time::Instant}; use walkdir::{DirEntry, WalkDir}; -use super::checksum::partial_checksum; +use super::cas::checksum::partial_checksum; #[derive(Debug)] pub struct IndexerJob { @@ -197,14 +197,16 @@ fn prepare_values( Some(p) => p .clone() .strip_prefix(&location_path) - // .and_then(|p| p.strip_suffix(format!("{}{}", name, extension).as_str())) + .and_then(|p| p.strip_suffix(format!("{}{}", name, extension).as_str())) .unwrap_or_default(), None => return Err(anyhow!("{}", file_path.to_str().unwrap_or_default())), }; let partial_checksum = { if !metadata.is_dir() { - partial_checksum(&file_path.to_str().unwrap(), metadata.len()).unwrap() + let mut x = partial_checksum(&file_path.to_str().unwrap(), metadata.len()).unwrap(); + x.truncate(16); + x } else { "".to_string() } diff --git a/core/src/file/mod.rs b/core/src/file/mod.rs index 92a5dfe2e..f562f2170 100644 --- a/core/src/file/mod.rs +++ b/core/src/file/mod.rs @@ -8,7 +8,7 @@ use crate::{ prisma::{self, FileData, FilePathData}, sys::SysError, }; -pub mod checksum; +pub mod cas; pub mod explorer; pub mod indexer; pub mod thumb; diff --git a/core/src/lib.rs b/core/src/lib.rs index 0b7fcb772..5ebe525ed 100644 --- a/core/src/lib.rs +++ b/core/src/lib.rs @@ -17,6 +17,7 @@ use crate::file::thumb::ThumbnailJob; pub mod client; pub mod crypto; pub mod db; +pub mod encode; pub mod file; pub mod job; pub mod library; @@ -292,9 +293,9 @@ impl Core { path, location_id, limit: _, - } => { - CoreResponse::LibGetExplorerDir(file::explorer::open_dir(&ctx, &location_id, &path).await?) - } + } => CoreResponse::LibGetExplorerDir( + file::explorer::open::open_dir(&ctx, &location_id, &path).await?, + ), ClientQuery::LibGetTags => todo!(), ClientQuery::JobGetRunning => CoreResponse::JobGetRunning(self.jobs.get_running().await), // TODO: FIX THIS diff --git a/packages/config/tsconfig/nextjs.json b/packages/config/tsconfig/nextjs.json index 91cd404f7..53d1e81d9 100644 --- a/packages/config/tsconfig/nextjs.json +++ b/packages/config/tsconfig/nextjs.json @@ -17,6 +17,6 @@ "isolatedModules": true, "jsx": "preserve" }, - "include": ["src", "next-env.d.ts"], + "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx"], "exclude": ["node_modules"] } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 14d97e8f0..1cb731a07 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -133,6 +133,25 @@ importers: apps/server: specifiers: {} + apps/web: + specifiers: + '@types/node': ^12.12.21 + '@types/react': ^17.0.2 + '@types/react-dom': ^17.0.1 + next: latest + react: ^17.0.2 + react-dom: ^17.0.2 + typescript: '4.0' + dependencies: + next: 12.1.5_react-dom@17.0.2+react@17.0.2 + react: 17.0.2 + react-dom: 17.0.2_react@17.0.2 + devDependencies: + '@types/node': 12.20.47 + '@types/react': 17.0.44 + '@types/react-dom': 17.0.15 + typescript: 4.0.8 + core: specifiers: '@types/node': ^17.0.23 @@ -559,12 +578,124 @@ packages: '@jridgewell/sourcemap-codec': 1.4.11 dev: false + /@next/env/12.1.5: + resolution: {integrity: sha512-+34yUJslfJi7Lyx6ELuN8nWcOzi27izfYnZIC1Dqv7kmmfiBVxgzR3BXhlvEMTKC2IRJhXVs2FkMY+buQe3k7Q==} + dev: false + /@next/eslint-plugin-next/12.1.4: resolution: {integrity: sha512-BRy565KVK6Cdy8LHaHTiwctLqBu/RT84RLpESug70BDJzBlV8QBvODyx/j7wGhvYqp9kvstM05lyb6JaTkSCcQ==} dependencies: glob: 7.1.7 dev: false + /@next/swc-android-arm-eabi/12.1.5: + resolution: {integrity: sha512-SKnGTdYcoN04Y2DvE0/Y7/MjkA+ltsmbuH/y/hR7Ob7tsj+8ZdOYuk+YvW1B8dY20nDPHP58XgDTSm2nA8BzzA==} + engines: {node: '>= 10'} + cpu: [arm] + os: [android] + requiresBuild: true + dev: false + optional: true + + /@next/swc-android-arm64/12.1.5: + resolution: {integrity: sha512-YXiqgQ/9Rxg1dXp6brXbeQM1JDx9SwUY/36JiE+36FXqYEmDYbxld9qkX6GEzkc5rbwJ+RCitargnzEtwGW0mw==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [android] + requiresBuild: true + dev: false + optional: true + + /@next/swc-darwin-arm64/12.1.5: + resolution: {integrity: sha512-y8mhldb/WFZ6lFeowkGfi0cO/lBdiBqDk4T4LZLvCpoQp4Or/NzUN6P5NzBQZ5/b4oUHM/wQICEM+1wKA4qIVw==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [darwin] + requiresBuild: true + dev: false + optional: true + + /@next/swc-darwin-x64/12.1.5: + resolution: {integrity: sha512-wqJ3X7WQdTwSGi0kIDEmzw34QHISRIQ5uvC+VXmsIlCPFcMA+zM5723uh8NfuKGquDMiEMS31a83QgkuHMYbwQ==} + engines: {node: '>= 10'} + cpu: [x64] + os: [darwin] + requiresBuild: true + dev: false + optional: true + + /@next/swc-linux-arm-gnueabihf/12.1.5: + resolution: {integrity: sha512-WnhdM5duONMvt2CncAl+9pim0wBxDS2lHoo7ub/o/i1bRbs11UTzosKzEXVaTDCUkCX2c32lIDi1WcN2ZPkcdw==} + engines: {node: '>= 10'} + cpu: [arm] + os: [linux] + requiresBuild: true + dev: false + optional: true + + /@next/swc-linux-arm64-gnu/12.1.5: + resolution: {integrity: sha512-Jq2H68yQ4bLUhR/XQnbw3LDW0GMQn355qx6rU36BthDLeGue7YV7MqNPa8GKvrpPocEMW77nWx/1yI6w6J07gw==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [linux] + requiresBuild: true + dev: false + optional: true + + /@next/swc-linux-arm64-musl/12.1.5: + resolution: {integrity: sha512-KgPjwdbhDqXI7ghNN8V/WAiLquc9Ebe8KBrNNEL0NQr+yd9CyKJ6KqjayVkmX+hbHzbyvbui/5wh/p3CZQ9xcQ==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [linux] + requiresBuild: true + dev: false + optional: true + + /@next/swc-linux-x64-gnu/12.1.5: + resolution: {integrity: sha512-O2ErUTvCJ6DkNTSr9pbu1n3tcqykqE/ebty1rwClzIYdOgpB3T2MfEPP+K7GhUR87wmN/hlihO9ch7qpVFDGKw==} + engines: {node: '>= 10'} + cpu: [x64] + os: [linux] + requiresBuild: true + dev: false + optional: true + + /@next/swc-linux-x64-musl/12.1.5: + resolution: {integrity: sha512-1eIlZmlO/VRjxxzUBcVosf54AFU3ltAzHi+BJA+9U/lPxCYIsT+R4uO3QksRzRjKWhVQMRjEnlXyyq5SKJm7BA==} + engines: {node: '>= 10'} + cpu: [x64] + os: [linux] + requiresBuild: true + dev: false + optional: true + + /@next/swc-win32-arm64-msvc/12.1.5: + resolution: {integrity: sha512-oromsfokbEuVb0CBLLE7R9qX3KGXucZpsojLpzUh1QJjuy1QkrPJncwr8xmWQnwgtQ6ecMWXgXPB+qtvizT9Tw==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [win32] + requiresBuild: true + dev: false + optional: true + + /@next/swc-win32-ia32-msvc/12.1.5: + resolution: {integrity: sha512-a/51L5KzBpeZSW9LbekMo3I3Cwul+V+QKwbEIMA+Qwb2qrlcn1L9h3lt8cHqNTFt2y72ce6aTwDTw1lyi5oIRA==} + engines: {node: '>= 10'} + cpu: [ia32] + os: [win32] + requiresBuild: true + dev: false + optional: true + + /@next/swc-win32-x64-msvc/12.1.5: + resolution: {integrity: sha512-/SoXW1Ntpmpw3AXAzfDRaQidnd8kbZ2oSni8u5z0yw6t4RwJvmdZy1eOaAADRThWKV+2oU90++LSnXJIwBRWYQ==} + engines: {node: '>= 10'} + cpu: [x64] + os: [win32] + requiresBuild: true + dev: false + optional: true + /@nodelib/fs.scandir/2.1.5: resolution: {integrity: sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==} engines: {node: '>= 8'} @@ -1253,6 +1384,10 @@ packages: resolution: {integrity: sha1-7ihweulOEdK4J7y+UnC86n8+ce4=} dev: false + /@types/node/12.20.47: + resolution: {integrity: sha512-BzcaRsnFuznzOItW1WpQrDHM7plAa7GIDMZ6b5pnMbkqEtM/6WCOhvZar39oeMQP79gwvFUWjjptE7/KGcNqFg==} + dev: true + /@types/node/17.0.23: resolution: {integrity: sha512-UxDxWn7dl97rKVeVS61vErvw086aCYhDLyvRQZ5Rk65rZKepaFdm53GeqXaKBuOhED4e9uWq34IC3TdSdJJ2Gw==} dev: true @@ -1275,6 +1410,12 @@ packages: resolution: {integrity: sha512-L28j2FcJfSZOnL1WBjDYp2vUHCeIFlyYI/53EwD/rKUBQ7MtUUfbQWiyKJGpcnv4/WgrhWsFKrcPstcAt/J0tQ==} dev: false + /@types/react-dom/17.0.15: + resolution: {integrity: sha512-Tr9VU9DvNoHDWlmecmcsE5ZZiUkYx+nKBzum4Oxe1K0yJVyBlfbq7H3eXjxXqJczBKqPGq3EgfTru4MgKb9+Yw==} + dependencies: + '@types/react': 17.0.44 + dev: true + /@types/react-dom/18.0.0: resolution: {integrity: sha512-49897Y0UiCGmxZqpC8Blrf6meL8QUla6eb+BBhn69dTXlmuOlzkfr7HHY/O8J25e1lTUMs+YYxSlVDAaGHCOLg==} dependencies: @@ -1327,7 +1468,6 @@ packages: '@types/prop-types': 15.7.5 '@types/scheduler': 0.16.2 csstype: 3.0.11 - dev: false /@types/react/18.0.0: resolution: {integrity: sha512-7+K7zEQYu7NzOwQGLR91KwWXXDzmTFODRVizJyIALf6RfLv2GDpqpknX64pvRVILXCpXi7O/pua8NGk44dLvJw==} @@ -3053,6 +3193,48 @@ packages: hasBin: true dev: false + /next/12.1.5_react-dom@17.0.2+react@17.0.2: + resolution: {integrity: sha512-YGHDpyfgCfnT5GZObsKepmRnne7Kzp7nGrac07dikhutWQug7hHg85/+sPJ4ZW5Q2pDkb+n0FnmLkmd44htIJQ==} + engines: {node: '>=12.22.0'} + hasBin: true + peerDependencies: + fibers: '>= 3.1.0' + node-sass: ^6.0.0 || ^7.0.0 + react: ^17.0.2 || ^18.0.0-0 + react-dom: ^17.0.2 || ^18.0.0-0 + sass: ^1.3.0 + peerDependenciesMeta: + fibers: + optional: true + node-sass: + optional: true + sass: + optional: true + dependencies: + '@next/env': 12.1.5 + caniuse-lite: 1.0.30001327 + postcss: 8.4.5 + react: 17.0.2 + react-dom: 17.0.2_react@17.0.2 + styled-jsx: 5.0.1_react@17.0.2 + optionalDependencies: + '@next/swc-android-arm-eabi': 12.1.5 + '@next/swc-android-arm64': 12.1.5 + '@next/swc-darwin-arm64': 12.1.5 + '@next/swc-darwin-x64': 12.1.5 + '@next/swc-linux-arm-gnueabihf': 12.1.5 + '@next/swc-linux-arm64-gnu': 12.1.5 + '@next/swc-linux-arm64-musl': 12.1.5 + '@next/swc-linux-x64-gnu': 12.1.5 + '@next/swc-linux-x64-musl': 12.1.5 + '@next/swc-win32-arm64-msvc': 12.1.5 + '@next/swc-win32-ia32-msvc': 12.1.5 + '@next/swc-win32-x64-msvc': 12.1.5 + transitivePeerDependencies: + - '@babel/core' + - babel-plugin-macros + dev: false + /node-fetch/2.6.7: resolution: {integrity: sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==} engines: {node: 4.x || >=6.0.0} @@ -3306,6 +3488,15 @@ packages: source-map-js: 1.0.2 dev: false + /postcss/8.4.5: + resolution: {integrity: sha512-jBDboWM8qpaqwkMwItqTQTiFikhs/67OYVvblFFTM7MrZjt6yMKd6r2kgXizEbTTljacm4NldIlZnhbjr84QYg==} + engines: {node: ^10 || ^12 || >=14} + dependencies: + nanoid: 3.3.2 + picocolors: 1.0.0 + source-map-js: 1.0.2 + dev: false + /prettier/2.6.2: resolution: {integrity: sha512-PkUpF+qoXTqhOeWL9fu7As8LXsIUZ1WYaJiY/a7McAQzxjk82OF0tibkFXVCDImZtWxbvojFjerkiLb0/q8mew==} engines: {node: '>=10.13.0'} @@ -3364,6 +3555,17 @@ packages: pure-color: 1.3.0 dev: false + /react-dom/17.0.2_react@17.0.2: + resolution: {integrity: sha512-s4h96KtLDUQlsENhMn1ar8t2bEa+q/YAtj8pPPdIjPDGBDIVNsrD9aXNWqspUe6AzKCIG0C1HZZLqLV7qpOBGA==} + peerDependencies: + react: 17.0.2 + dependencies: + loose-envify: 1.4.0 + object-assign: 4.1.1 + react: 17.0.2 + scheduler: 0.20.2 + dev: false + /react-dom/18.0.0_react@18.0.0: resolution: {integrity: sha512-XqX7uzmFo0pUceWFCt7Gff6IyIMzFUn7QMZrbrQfGxtaxXZIcGQzoNpRLE3fQLnS4XzLLPMZX2T9TRcSrasicw==} peerDependencies: @@ -3607,6 +3809,14 @@ packages: react: 18.0.0 dev: false + /react/17.0.2: + resolution: {integrity: sha512-gnhPt75i/dq/z3/6q/0asP78D0u592D5L1pd7M8P+dck6Fu/jJeL6iVVK23fptSUZj8Vjf++7wXA8UNclGQcbA==} + engines: {node: '>=0.10.0'} + dependencies: + loose-envify: 1.4.0 + object-assign: 4.1.1 + dev: false + /react/18.0.0: resolution: {integrity: sha512-x+VL6wbT4JRVPm7EGxXhZ8w8LTROaxPXOqhlGyVSrv0sB1jkyFGgXxJ8LVoPRLvPR6/CIZGFmfzqUa2NYeMr2A==} engines: {node: '>=0.10.0'} @@ -3722,6 +3932,13 @@ packages: resolution: {integrity: sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw==} dev: false + /scheduler/0.20.2: + resolution: {integrity: sha512-2eWfGgAqqWFGqtdMmcL5zCMK1U8KlXv8SQFGglL3CEtd0aDVDWgeF/YoCmvln55m5zSk3J/20hTaSBeSObsQDQ==} + dependencies: + loose-envify: 1.4.0 + object-assign: 4.1.1 + dev: false + /scheduler/0.21.0: resolution: {integrity: sha512-1r87x5fz9MXqswA2ERLo0EbOAU74DpIUO090gIasYTqlVoJeMcl+Z1Rg7WHz+qtPujhS/hGIt9kxZOYBV3faRQ==} dependencies: @@ -3842,6 +4059,22 @@ packages: engines: {node: '>=0.10.0'} dev: true + /styled-jsx/5.0.1_react@17.0.2: + resolution: {integrity: sha512-+PIZ/6Uk40mphiQJJI1202b+/dYeTVd9ZnMPR80pgiWbjIwvN2zIp4r9et0BgqBuShh48I0gttPlAXA7WVvBxw==} + engines: {node: '>= 12.0.0'} + peerDependencies: + '@babel/core': '*' + babel-plugin-macros: '*' + react: '>= 16.8.0 || 17.x.x || ^18.0.0-0' + peerDependenciesMeta: + '@babel/core': + optional: true + babel-plugin-macros: + optional: true + dependencies: + react: 17.0.2 + dev: false + /supports-color/5.5.0: resolution: {integrity: sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==} engines: {node: '>=4'} @@ -4148,6 +4381,12 @@ packages: engines: {node: '>=12.20'} dev: false + /typescript/4.0.8: + resolution: {integrity: sha512-oz1765PN+imfz1MlZzSZPtC/tqcwsCyIYA8L47EkRnRW97ztRk83SzMiWLrnChC0vqoYxSU1fcFUDA5gV/ZiPg==} + engines: {node: '>=4.2.0'} + hasBin: true + dev: true + /typescript/4.6.3: resolution: {integrity: sha512-yNIatDa5iaofVozS/uQJEl3JRWLKKGJKh6Yaiv0GLGSuhpFJe7P3SbHZ8/yjAHRQwKRoA6YZqlfjXWmVzoVSMw==} engines: {node: '>=4.2.0'}