Files
spacedrive/core/prisma/schema.prisma
Ericson "Fogo" Soares 5da703af2c [ENG-536] Improve indexer for huge locations (#781)
* WIP materialized_path abstraction revamp

* Optimizing indexer rules loading

* Using a better serialize impl for indexer rules

* New interruptable and faster Walker

* WIP new indexer

* WIP first success compiling after breaking the world

* Fixing some warnings

* Handling some lifetime issues in the walker

* New job completed with errors feature

* Introducing completed with errors to indexer
Removing IOError variant from JobError and using FileIOError instead

* Rust fmt

* Adding missing job status

* Better ergonomics to IsolatedFilePathData
Conversions from db's file_path kinds
Keeping original's relative path data to better conversion to OS's path
First unit tests

* Testing and fixing parent method

* Some error handling

* Rust fmt

* Some small fixes

* Fixing indexer rules decoding

* Bunch of small fixes

* Rust fmt

* Fixing indexer rules

* Updating frontend to new materialized_path format

* Trying to fix windows CI

---------

Co-authored-by: Brendan Allan <brendonovich@outlook.com>
2023-05-16 14:15:51 +00:00

475 lines
14 KiB
Plaintext

datasource db {
provider = "sqlite"
url = "file:dev.db"
}
generator client {
provider = "cargo prisma"
output = "../src/prisma.rs"
}
generator sync {
provider = "cargo prisma-sync"
output = "../src/prisma_sync.rs"
}
//// Sync ////
model OwnedOperation {
id Bytes @id
timestamp BigInt
data Bytes
model String
node_id Int
node Node @relation(fields: [node_id], references: [id])
@@map("owned_operation")
}
model SharedOperation {
id Bytes @id
timestamp BigInt
model String
record_id Bytes
// Enum: ??
kind String
data Bytes
node_id Int
node Node @relation(fields: [node_id], references: [id])
@@map("shared_operation")
}
model Statistics {
id Int @id @default(autoincrement())
date_captured DateTime @default(now())
total_object_count Int @default(0)
library_db_size String @default("0")
total_bytes_used String @default("0")
total_bytes_capacity String @default("0")
total_unique_bytes String @default("0")
total_bytes_free String @default("0")
preview_media_bytes String @default("0")
@@map("statistics")
}
/// @local(id: pub_id)
model Node {
id Int @id @default(autoincrement())
pub_id Bytes @unique
name String
// Enum: sd_core::node::Platform
platform Int @default(0)
version String?
last_seen DateTime @default(now())
timezone String?
date_created DateTime @default(now())
jobs Job[]
Location Location[]
OwnedOperation OwnedOperation[]
SharedOperation SharedOperation[]
@@map("node")
}
model Volume {
id Int @id @default(autoincrement())
node_id Int
name String
mount_point String
total_bytes_capacity String @default("0")
total_bytes_available String @default("0")
disk_type String?
filesystem String?
is_system Boolean @default(false)
date_modified DateTime @default(now())
@@unique([node_id, mount_point, name])
@@map("volume")
}
/// @shared(id: pub_id)
model Location {
id Int @id @default(autoincrement())
pub_id Bytes @unique
node_id Int
name String
path String
total_capacity Int?
available_capacity Int?
is_archived Boolean @default(false)
generate_preview_media Boolean @default(true)
sync_preview_media Boolean @default(true)
hidden Boolean @default(false)
date_created DateTime @default(now())
node Node @relation(fields: [node_id], references: [id])
file_paths FilePath[]
indexer_rules IndexerRulesInLocation[]
@@map("location")
}
/// @shared(id: pub_id)
model FilePath {
id Int @id @default(autoincrement())
pub_id Bytes @unique
is_dir Boolean @default(false)
// content addressable storage id - blake3 sampled checksum
cas_id String?
// full byte contents digested into blake3 checksum
integrity_checksum String? @unique
// location that owns this path
location_id Int
location Location @relation(fields: [location_id], references: [id], onDelete: Cascade, onUpdate: Cascade)
// the path of the file relative to its location
materialized_path String
// the name and extension
name String
extension String // Extension MUST have 'COLLATE NOCASE' in migration
size_in_bytes String @default("0")
inode Bytes // This is actually an unsigned 64 bit integer, but we don't have this type in SQLite
device Bytes // This is actually an unsigned 64 bit integer, but we don't have this type in SQLite
// the unique Object for this file path
object_id Int?
object Object? @relation(fields: [object_id], references: [id], onDelete: Restrict)
key_id Int? // replacement for encryption
// permissions String?
date_created DateTime @default(now())
date_modified DateTime @default(now())
date_indexed DateTime @default(now())
// NOTE: this self relation for the file tree was causing SQLite to go to forever bed, disabling until workaround
// parent FilePath? @relation("directory_file_paths", fields: [parent_id], references: [id], onDelete: NoAction, onUpdate: NoAction)
// children FilePath[] @relation("directory_file_paths")
key Key? @relation(fields: [key_id], references: [id])
@@unique([location_id, materialized_path, name, extension])
@@unique([location_id, inode, device])
@@index([location_id])
@@index([location_id, materialized_path])
@@map("file_path")
}
/// @shared(id: pub_id)
model Object {
id Int @id @default(autoincrement())
pub_id Bytes @unique
// Enum: sd_file_ext::kind::ObjectKind
kind Int @default(0)
key_id Int?
// handy ways to mark an object
hidden Boolean @default(false)
favorite Boolean @default(false)
important Boolean @default(false)
// if we have generated preview media for this object on at least one Node
has_thumbnail Boolean @default(false)
has_thumbstrip Boolean @default(false)
has_video_preview Boolean @default(false)
// TODO: change above to:
// has_generated_thumbnail Boolean @default(false)
// has_generated_thumbstrip Boolean @default(false)
// has_generated_video_preview Boolean @default(false)
// integration with ipfs
ipfs_id String?
// plain text note
note String?
// the original known creation date of this object
date_created DateTime @default(now())
date_accessed DateTime?
tags TagOnObject[]
labels LabelOnObject[]
albums ObjectInAlbum[]
spaces ObjectInSpace[]
file_paths FilePath[]
comments Comment[]
media_data MediaData?
key Key? @relation(fields: [key_id], references: [id])
@@map("object")
}
// if there is a conflicting cas_id, the conficting file should be updated to have a larger cas_id as the field is unique, however this record is kept to tell the indexer (upon discovering this CAS) that there is alternate versions of the file and to check by a full integrity hash to define for which to associate with.
model FileConflict {
original_object_id Int @unique
detactched_object_id Int @unique
@@map("file_conflict")
}
// keys allow us to know exactly which files can be decrypted with a given key
// they can be "mounted" to a client, and then used to decrypt files automatically
/// @shared(id: uuid)
model Key {
id Int @id @default(autoincrement())
// uuid to identify the key
uuid String @unique
version String
key_type String
// the name that the user sets
name String?
// is this key the default for encryption?
// was not tagged as unique as i'm not too sure if PCR will handle it
// can always be tagged as unique, the keys API will need updating to use `find_unique()`
default Boolean @default(false)
// nullable if concealed for security
date_created DateTime? @default(now())
// encryption algorithm used to encrypt the key
algorithm String
// hashing algorithm used for hashing the key with the content salt
hashing_algorithm String
// salt used for encrypting data with this key
content_salt Bytes
// the *encrypted* master key (48 bytes)
master_key Bytes
// the nonce used for encrypting the master key
master_key_nonce Bytes
// the nonce used for encrypting the key
key_nonce Bytes
// the *encrypted* key
key Bytes
// the salt used for deriving the KEK (used for encrypting the master key) from the root key
salt Bytes
automount Boolean @default(false)
objects Object[]
file_paths FilePath[]
@@map("key")
}
model MediaData {
id Int @id
pixel_width Int?
pixel_height Int?
longitude Float?
latitude Float?
fps Int?
capture_device_make String? // eg: "Apple"
capture_device_model String? // eg: "iPhone 12"
capture_device_software String? // eg: "12.1.1"
duration_seconds Int?
codecs String? // eg: "h264,acc"
streams Int?
object Object? @relation(fields: [id], references: [id], onDelete: Cascade, onUpdate: Cascade)
@@map("media_data")
}
//// Tag ////
/// @shared(id: pub_id)
model Tag {
id Int @id @default(autoincrement())
pub_id Bytes @unique
name String?
color String?
total_objects Int? @default(0)
// Enum: ??
redundancy_goal Int? @default(1)
date_created DateTime @default(now())
date_modified DateTime @default(now())
tag_objects TagOnObject[]
@@map("tag")
}
model TagOnObject {
date_created DateTime @default(now())
tag_id Int
tag Tag @relation(fields: [tag_id], references: [id], onDelete: NoAction, onUpdate: NoAction)
object_id Int
object Object @relation(fields: [object_id], references: [id], onDelete: NoAction, onUpdate: NoAction)
@@id([tag_id, object_id])
@@map("tag_on_object")
}
//// Label ////
model Label {
id Int @id @default(autoincrement())
pub_id Bytes @unique
name String?
date_created DateTime @default(now())
date_modified DateTime @default(now())
label_objects LabelOnObject[]
@@map("label")
}
model LabelOnObject {
date_created DateTime @default(now())
label_id Int
label Label @relation(fields: [label_id], references: [id], onDelete: NoAction, onUpdate: NoAction)
object_id Int
object Object @relation(fields: [object_id], references: [id], onDelete: NoAction, onUpdate: NoAction)
@@id([label_id, object_id])
@@map("label_on_object")
}
//// Space ////
model Space {
id Int @id @default(autoincrement())
pub_id Bytes @unique
name String?
description String?
date_created DateTime @default(now())
date_modified DateTime @default(now())
objects ObjectInSpace[]
@@map("space")
}
model ObjectInSpace {
date_created DateTime @default(now())
space_id Int
space Space @relation(fields: [space_id], references: [id], onDelete: NoAction, onUpdate: NoAction)
object_id Int
object Object @relation(fields: [object_id], references: [id], onDelete: NoAction, onUpdate: NoAction)
@@id([space_id, object_id])
@@map("object_in_space")
}
//// Job ////
model Job {
id Bytes @id
name String
node_id Int
action String? // Will be composed of "{action_description}(-{children_order})*"
// Enum: sd_core::job::job_manager:JobStatus
status Int @default(0) // 0 = Queued
// List of errors, separated by "\n\n" in case of failed jobs or completed with errors
errors_text String?
data Bytes? // Serialized data to be used on pause/resume
metadata Bytes? // Serialized metadata field with info about the job after completion
parent_id Bytes?
task_count Int @default(1)
completed_task_count Int @default(0)
date_created DateTime @default(now())
date_started DateTime? @default(now()) // Started execution
date_completed DateTime? @default(now()) // Finished execution
nodes Node @relation(fields: [node_id], references: [id], onDelete: Cascade, onUpdate: Cascade)
parent Job? @relation("jobs_dependency", fields: [parent_id], references: [id], onDelete: Cascade, onUpdate: Cascade)
children Job[] @relation("jobs_dependency")
@@map("job")
}
//// Album ////
/// @shared(id: pub_id)
model Album {
id Int @id @default(autoincrement())
pub_id Bytes @unique
name String
is_hidden Boolean @default(false)
date_created DateTime @default(now())
date_modified DateTime @default(now())
objects ObjectInAlbum[]
@@map("album")
}
model ObjectInAlbum {
date_created DateTime @default(now())
album_id Int
album Album @relation(fields: [album_id], references: [id], onDelete: NoAction, onUpdate: NoAction)
object_id Int
object Object @relation(fields: [object_id], references: [id], onDelete: NoAction, onUpdate: NoAction)
@@id([album_id, object_id])
@@map("object_in_album")
}
//// Comment ////
model Comment {
id Int @id @default(autoincrement())
pub_id Bytes @unique
content String
date_created DateTime @default(now())
date_modified DateTime @default(now())
object_id Int?
object Object? @relation(fields: [object_id], references: [id])
@@map("comment")
}
//// Indexer Rules ////
model IndexerRule {
id Int @id @default(autoincrement())
// Enum: sd_core::location::indexer::rules::RuleKind
kind Int
name String
default Boolean @default(false)
parameters Bytes
date_created DateTime @default(now())
date_modified DateTime @default(now())
locations IndexerRulesInLocation[]
@@map("indexer_rule")
}
model IndexerRulesInLocation {
date_created DateTime @default(now())
location_id Int
location Location @relation(fields: [location_id], references: [id], onDelete: NoAction, onUpdate: NoAction)
indexer_rule_id Int
indexer_rule IndexerRule @relation(fields: [indexer_rule_id], references: [id], onDelete: NoAction, onUpdate: NoAction)
@@id([location_id, indexer_rule_id])
@@map("indexer_rule_in_location")
}