mirror of
https://github.com/spacedriveapp/spacedrive.git
synced 2026-10-30 04:49:32 -04:00
Declare what a photo and a video carry
The store's filesystem schema had one model with six fields, so a record could say how big a file is and not what took the picture. The old library had image_media_data and video_media_data and both were lost in the move. Facets are schema-driven, so these are declarations rather than machinery: generate_ddl emits facet_image, facet_video and facet_audio on create, and diff_schemas adds them to a store that predates them. Audio came along with the other two, being the same kind of fact and already shaped in the tree. No writer yet, enrichment is still parked. What this buys is that the shape can only be decided once. The search contract named title = "name" on a model with no name field. A filesystem record's display name is its filename, which the walk writes to record.title, so the contract points at the record instead.
This commit is contained in:
1 parent
8abe325259
commit
df965b851b
4 files changed
+163
-46
No files matched your search
+107
-14
@@ -826,24 +826,115 @@ fn address(
|
||||
}
|
||||
}
|
||||
|
||||
pub fn filesystem_schema() -> DataTypeSchema {
|
||||
let mut fields = IndexMap::new();
|
||||
fields.insert("size".to_string(), FieldType::Integer);
|
||||
fields.insert("mtime".to_string(), FieldType::Integer);
|
||||
fields.insert("atime".to_string(), FieldType::Integer);
|
||||
fields.insert("inode".to_string(), FieldType::Integer);
|
||||
fields.insert("mode".to_string(), FieldType::Integer);
|
||||
fields.insert("extension".to_string(), FieldType::String);
|
||||
fields.insert("is_hidden".to_string(), FieldType::Boolean);
|
||||
/// One facet model from a flat field list, so a declaration reads as the
|
||||
/// columns it produces rather than as builder calls.
|
||||
fn model(fields: &[(&str, FieldType)]) -> ModelDef {
|
||||
ModelDef {
|
||||
fields: fields
|
||||
.iter()
|
||||
.map(|(name, ty)| ((*name).to_string(), *ty))
|
||||
.collect::<IndexMap<_, _>>(),
|
||||
relations: RelationsDef::default(),
|
||||
}
|
||||
}
|
||||
|
||||
/// The image facet. Shape follows what an EXIF read yields: geometry, capture
|
||||
/// time and place, the camera and lens that took it, and the rendering hints a
|
||||
/// viewer needs before it has decoded anything.
|
||||
fn image_model() -> ModelDef {
|
||||
model(&[
|
||||
("width", FieldType::Integer),
|
||||
("height", FieldType::Integer),
|
||||
("blurhash", FieldType::String),
|
||||
("date_taken", FieldType::Datetime),
|
||||
("latitude", FieldType::Float),
|
||||
("longitude", FieldType::Float),
|
||||
("camera_make", FieldType::String),
|
||||
("camera_model", FieldType::String),
|
||||
("lens_model", FieldType::String),
|
||||
("focal_length", FieldType::String),
|
||||
("aperture", FieldType::String),
|
||||
("shutter_speed", FieldType::String),
|
||||
("iso", FieldType::Integer),
|
||||
("orientation", FieldType::Integer),
|
||||
("color_space", FieldType::String),
|
||||
("color_profile", FieldType::String),
|
||||
("bit_depth", FieldType::String),
|
||||
("artist", FieldType::String),
|
||||
("copyright", FieldType::String),
|
||||
("description", FieldType::Text),
|
||||
])
|
||||
}
|
||||
|
||||
/// The video facet. Frame rate is kept as a numerator and denominator because
|
||||
/// 24000/1001 is not 23.976 and the difference accumulates over a timeline.
|
||||
fn video_model() -> ModelDef {
|
||||
model(&[
|
||||
("width", FieldType::Integer),
|
||||
("height", FieldType::Integer),
|
||||
("blurhash", FieldType::String),
|
||||
("duration_seconds", FieldType::Float),
|
||||
("bit_rate", FieldType::Integer),
|
||||
("codec", FieldType::String),
|
||||
("pixel_format", FieldType::String),
|
||||
("color_space", FieldType::String),
|
||||
("color_range", FieldType::String),
|
||||
("color_primaries", FieldType::String),
|
||||
("color_transfer", FieldType::String),
|
||||
("fps_num", FieldType::Integer),
|
||||
("fps_den", FieldType::Integer),
|
||||
("audio_codec", FieldType::String),
|
||||
("audio_channels", FieldType::String),
|
||||
("audio_sample_rate", FieldType::Integer),
|
||||
("audio_bit_rate", FieldType::Integer),
|
||||
("title", FieldType::String),
|
||||
("artist", FieldType::String),
|
||||
("album", FieldType::String),
|
||||
("creation_time", FieldType::Datetime),
|
||||
("date_captured", FieldType::Datetime),
|
||||
])
|
||||
}
|
||||
|
||||
/// The audio facet, carrying the tags a library sorts and groups by alongside
|
||||
/// the stream properties.
|
||||
fn audio_model() -> ModelDef {
|
||||
model(&[
|
||||
("duration_seconds", FieldType::Float),
|
||||
("bit_rate", FieldType::Integer),
|
||||
("sample_rate", FieldType::Integer),
|
||||
("channels", FieldType::String),
|
||||
("codec", FieldType::String),
|
||||
("title", FieldType::String),
|
||||
("artist", FieldType::String),
|
||||
("album", FieldType::String),
|
||||
("album_artist", FieldType::String),
|
||||
("genre", FieldType::String),
|
||||
("year", FieldType::Integer),
|
||||
("track_number", FieldType::Integer),
|
||||
("disc_number", FieldType::Integer),
|
||||
("composer", FieldType::String),
|
||||
("publisher", FieldType::String),
|
||||
("copyright", FieldType::String),
|
||||
])
|
||||
}
|
||||
|
||||
pub fn filesystem_schema() -> DataTypeSchema {
|
||||
let mut models = IndexMap::new();
|
||||
models.insert(
|
||||
"file".to_string(),
|
||||
ModelDef {
|
||||
fields,
|
||||
relations: RelationsDef::default(),
|
||||
},
|
||||
model(&[
|
||||
("size", FieldType::Integer),
|
||||
("mtime", FieldType::Integer),
|
||||
("atime", FieldType::Integer),
|
||||
("inode", FieldType::Integer),
|
||||
("mode", FieldType::Integer),
|
||||
("extension", FieldType::String),
|
||||
("is_hidden", FieldType::Boolean),
|
||||
]),
|
||||
);
|
||||
models.insert("image".to_string(), image_model());
|
||||
models.insert("video".to_string(), video_model());
|
||||
models.insert("audio".to_string(), audio_model());
|
||||
|
||||
DataTypeSchema {
|
||||
data_type: DataTypeMeta {
|
||||
@@ -854,7 +945,9 @@ pub fn filesystem_schema() -> DataTypeSchema {
|
||||
models,
|
||||
search: SearchContract {
|
||||
primary_model: "file".to_string(),
|
||||
title: "name".to_string(),
|
||||
// The display name is the filename, which the walk writes to
|
||||
// `record.title` rather than to any facet column.
|
||||
title: "_derived.title".to_string(),
|
||||
preview: "_derived.none".to_string(),
|
||||
subtitle: None,
|
||||
search_fields: Vec::new(),
|
||||
|
||||
@@ -31,7 +31,7 @@ pub struct ModelDef {
|
||||
}
|
||||
|
||||
/// Supported field types that map to SQLite column types.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
pub enum FieldType {
|
||||
/// TEXT — short string
|
||||
|
||||
@@ -864,3 +864,26 @@ async fn a_fresh_resolution_adopts_an_identity_the_path_already_has() {
|
||||
"an observation cannot rename a record the ledger already holds"
|
||||
);
|
||||
}
|
||||
|
||||
/// The media facets are declared alongside the file facet, so a source gets
|
||||
/// their tables the moment it is created and an existing one gets them on the
|
||||
/// next open rather than needing a hand-written migration.
|
||||
#[tokio::test]
|
||||
async fn a_source_carries_a_table_for_every_declared_facet() {
|
||||
let fixture = Fixture::new().await;
|
||||
let db = fixture.open().await;
|
||||
|
||||
let tables: Vec<String> = sqlx::query_scalar(
|
||||
"SELECT name FROM sqlite_master WHERE type = 'table' AND name LIKE 'facet_%'",
|
||||
)
|
||||
.fetch_all(db.pool())
|
||||
.await
|
||||
.expect("read table list");
|
||||
|
||||
for expected in ["facet_file", "facet_image", "facet_video", "facet_audio"] {
|
||||
assert!(
|
||||
tables.iter().any(|t| t == expected),
|
||||
"{expected} missing from {tables:?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -462,40 +462,45 @@ The last two rows are worth stating plainly: atime and permissions were empty
|
||||
in the old library too, so those are an equal gap rather than a regression.
|
||||
Everything above them is a real loss.
|
||||
|
||||
1. **The walk discards metadata it already has.** `DirEntry`
|
||||
(`ops/indexing/state.rs:79`) carries path, kind, size, mtime and inode, and
|
||||
`run_ephemeral_processing_static` fills the rest with `accessed: None,
|
||||
created: None, permissions: None`. The `fs::Metadata` behind each entry has
|
||||
`created()`, `accessed()` and `mode()` on it already, so the stat is paid
|
||||
for and the fields are dropped on the floor. Birth time is the expensive one:
|
||||
it is the timestamp a photo library sorts by, it is unrecoverable once the
|
||||
file is copied, and every day of indexing without it loses more of them.
|
||||
`EntryMetadata` and `facet_file.mode` both have the columns waiting.
|
||||
1. **Done.** The walk keeps what it already stat'd. `DirEntry`
|
||||
carried path, kind, size, mtime and inode while
|
||||
`run_ephemeral_processing_static` filled the rest with `accessed: None,
|
||||
created: None, permissions: None`, even though the `fs::Metadata` behind
|
||||
each entry has `created()`, `accessed()` and `mode()` on it. Birth time was
|
||||
the expensive one: it is the timestamp a photo library sorts by, it is
|
||||
unrecoverable once the file is copied, and every day of indexing without it
|
||||
lost more of them. `DirEntry` now carries all three and `facet_file.mode`
|
||||
has a writer.
|
||||
|
||||
2. **The bulk walk does not build the tree.** Parent linkage is
|
||||
`parent_of(external_id).and_then(|p| ledger.uuid_of(p))`, so a child is
|
||||
linked only if the ledger has already resolved its directory. The incremental
|
||||
path satisfies that and the walk does not:
|
||||
2. **Done.** The bulk walk builds the tree. Parent linkage was
|
||||
`parent_of(external_id).and_then(|p| ledger.uuid_of(p))`, which links a
|
||||
child only if the ledger has already resolved its directory. The incremental
|
||||
path satisfied that and the walk did not:
|
||||
|
||||
```
|
||||
06:37-06:39 2,102,811 records from the bulk walk 1,349 parented
|
||||
10:48-10:49 19,856 records from browse and watcher 19,769 parented
|
||||
```
|
||||
|
||||
240,122 directories exist as records and 949 of them know their parent. The
|
||||
fix is ordering rather than schema: resolve a directory before the batch
|
||||
carrying its children, or resolve parents within a batch first, which
|
||||
`parents_first` already does one layer down in `apply_files` for exactly this
|
||||
reason.
|
||||
240,122 directories existed as records and 949 of them knew their parent.
|
||||
The fix was ordering rather than schema: `shallowest_first` resolves a
|
||||
directory before the children in the same batch, one layer above where
|
||||
`parents_first` already did it in `apply_files` for the same reason.
|
||||
|
||||
3. **No media facets.** `filesystem_schema()` declares one model with six
|
||||
fields. The old library had `image_media_data` (EXIF, GPS, camera, lens,
|
||||
orientation, blurhash) and `video_media_data` (codec, fps, colour primaries,
|
||||
duration, audio stream). The facet mechanism is schema-driven and already
|
||||
generates `facet_file` from a declaration, so `facet_image` and `facet_video`
|
||||
are a schema entry rather than new machinery. Their writer is enrichment,
|
||||
which is parked, but declaring them costs nothing and stops the shape being
|
||||
invented twice.
|
||||
3. **Done.** `filesystem_schema()` declared one model with six fields against
|
||||
the old library's `image_media_data` (EXIF, GPS, camera, lens, orientation,
|
||||
blurhash) and `video_media_data` (codec, fps, colour primaries, duration,
|
||||
audio stream). The facet mechanism is schema-driven, so `image`, `video` and
|
||||
`audio` are declarations rather than machinery: `generate_ddl` emits their
|
||||
tables on create and `diff_schemas` adds them to a store that predates them.
|
||||
`audio_media_data` came across with the other two, being the same kind of
|
||||
fact and already shaped. Their writer is enrichment, which is parked, but
|
||||
the shape can no longer be invented twice.
|
||||
|
||||
The search contract came with them. It named `title = "name"` on a model
|
||||
with no `name` field; a filesystem record's display name is its filename,
|
||||
written to `record.title` by the walk rather than to any facet column, so
|
||||
the contract now says `_derived.title` and falls through to the record.
|
||||
|
||||
4. **Sidecars belong in the source store.** A sidecar is keyed by content uuid
|
||||
and `content` now lives in the source store, so leaving the row in
|
||||
@@ -533,11 +538,7 @@ Everything above them is a real loss.
|
||||
root it is given. `thumbs.pvcache` is already per-source and already the hot
|
||||
tier, so this is the durable half catching up to where the cache went.
|
||||
|
||||
**Order.** (1) and (2) are bugs against data the walk already holds and should
|
||||
land first; every walk before then produces records that need re-walking. (2)
|
||||
also has to land with P2.6, which makes the tree load-bearing rather than merely
|
||||
missing. (3) is a schema declaration. (4) is behind the content identity job,
|
||||
since a sidecar has nothing to key to while `content` has 0 rows.
|
||||
**Order.** (1), (2) and (3) have landed. (4) is what remains.
|
||||
|
||||
**The content identity job landed.** `ContentIdentityJob` hashes what a source
|
||||
holds and has not identified, dispatched at `LOW` behind the walk that produced
|
||||
|
||||
Reference in new issue
Block a user