From 29a87a91ceaf6acc566ea7427cb6a58449451bce Mon Sep 17 00:00:00 2001 From: Jamie Pine Date: Sat, 6 Sep 2025 21:00:37 -0400 Subject: [PATCH] huge refactor --- .prettierrc.js | 31 - .../ACT-000-action-system.md | 0 .../ACT-001-action-manager-registry.md | 0 {core-new/.tasks => .tasks}/AI-000-ai-epic.md | 0 .../.tasks => .tasks}/AI-001-ai-agent.md | 0 .../AI-002-create-finetuning-dataset.md | 0 .../CLI-000-command-line-interface.md | 0 .../CLOUD-000-cloud-as-a-peer.md | 0 .../CLOUD-001-design-cloud-core-infra.md | 9 +- .../CLOUD-002-relay-server.md | 0 .../CLOUD-003-cloud-volume.md | 0 .../.tasks => .tasks}/CORE-000-vdfs-core.md | 0 .../CORE-001-entry-centric-model.md | 0 .../CORE-002-sdpath-addressing.md | 0 .../CORE-003-content-identity.md | 0 .../CORE-004-closure-table.md | 0 .../CORE-005-file-type-system.md | 0 .../CORE-006-semantic-tagging-architecture.md | 0 ...m-state-for-on-demand-state-computation.md | 0 .../CORE-008-virtual-sidecar-system.md | 4 +- .../CORE-009-user-managed-collections.md | 0 .../CORE-010-file-ingestion-workflow.md | 0 .../DEV-000-development-validation.md | 0 .../DEV-001-multi-process-test-framework.md | 0 .../FILE-000-file-operations.md | 0 .../FILE-001-file-copy-job.md | 0 .../FILE-002-file-deletion-job.md | 0 .../INDEX-000-indexing-file-management.md | 0 .../INDEX-001-location-watcher-service.md | 0 ...NDEX-002-stale-file-detection-algorithm.md | 0 .../.tasks => .tasks}/JOB-000-job-system.md | 0 .../.tasks => .tasks}/JOB-001-job-manager.md | 0 .../.tasks => .tasks}/JOB-002-job-logging.md | 0 .../LOC-000-location-operations.md | 0 .../LOC-001-location-management-actions.md | 0 ...l-locations-via-pure-hierarchical-model.md | 0 .../LSYNC-000-library-sync.md | 0 .../LSYNC-001-design-library-sync-protocol.md | 0 .../LSYNC-002-metadata-sync.md | 0 .../LSYNC-003-file-op-sync.md | 0 ...C-004-sync-relationship-database-schema.md | 0 .../.tasks => .tasks}/NET-000-networking.md | 0 .../NET-001-iroh-p2p-stack.md | 0 .../NET-002-device-pairing.md | 0 .../NET-003-spacedrop-protocol.md | 0 .../PLUG-000-wasm-plugin-system.md | 0 .../PLUG-001-integrate-wasm-runtime.md | 0 .../PLUG-002-define-vdfs-plugin-api.md | 0 .../PLUG-003-develop-twitter-agent-poc.md | 0 .../RES-000-resource-management.md | 0 .../RES-001-adaptive-throttling.md | 0 .../SEARCH-000-temporal-semantic-search.md | 0 .../SEARCH-001-async-searchjob.md | 0 ...CH-002-two-stage-fts-semantic-reranking.md | 0 .../SEARCH-003-unified-vector-repositories.md | 0 .../SEC-000-security-and-privacy.md | 0 .../SEC-002-database-encryption.md | 0 .../SEC-003-cryptographic-audit-log.md | 0 .../.tasks => .tasks}/SEC-004-rbac-system.md | 0 .../SEC-005-secure-credential-vault.md | 0 .../SEC-006-certificate-pinning.md | 0 ...-encryption-policies-for-public-sharing.md | 0 .../VOL-000-volume-operations.md | 0 ...physicalclass-and-location-logicalclass.md | 0 ...VOL-002-automatic-volume-classification.md | 0 ...elligent-storage-tiering-warning-system.md | 0 ...004-remote-volume-indexing-with-opendal.md | 0 ...e-as-a-virtual-volume-for-direct-import.md | 0 ...ference-sidecars-for-live-photo-support.md | 0 {core-new/.tasks => .tasks}/task.schema.json | 0 README.md | 54 +- spacedrive-cloud => apps/cloud | 0 combine_paths.py | 72 - core-new/Cargo.toml | 161 -- core-new/benchmarks/Cargo.toml | 39 - core-new/build.rs | 13 - core-new/examples/indexing_showcase.rs | 136 -- core-new/src/context.rs | 78 - core-new/src/lib.rs | 499 ----- core-new/src/library/config.rs | 208 --- core-new/src/library/mod.rs | 238 --- core-new/src/location/mod.rs | 563 ------ core-new/src/volume/error.rs | 83 - core-new/src/volume/mod.rs | 92 - core-new/src/volume/speed.rs | 369 ---- core-new/src/volume/types.rs | 685 ------- {core-new => core}/.scripts/combine.sh | 0 {core-new => core/.scripts}/test_daemon.sh | 0 .../.scripts}/update_spacedrive.sh | 2 +- {core-new => core}/AGENTS.md | 0 {core-new => core}/Cargo.lock | Bin 251354 -> 251346 bytes core/Cargo.toml | 303 ++-- {core-new => core}/README.md | 2 +- core/benchmarks/Cargo.toml | 37 + .../benchmarks/recipes/shape_large.yaml | 0 .../benchmarks/recipes/shape_medium.yaml | 0 .../benchmarks/recipes/shape_small.yaml | 0 .../results/shape_large-aggregation-hdd.json | 0 .../results/shape_large-aggregation-ssd.json | 2 +- ...hape_large-content_identification-hdd.json | 0 ...hape_large-content_identification-ssd.json | 2 +- .../shape_large-indexing_discovery-hdd.json | 0 .../shape_large-indexing_discovery-ssd.json | 2 +- .../results/shape_medium-aggregation-hdd.json | 0 .../results/shape_medium-aggregation-ssd.json | 2 +- ...ape_medium-content_identification-hdd.json | 0 ...ape_medium-content_identification-ssd.json | 2 +- .../shape_medium-indexing_discovery-hdd.json | 0 .../shape_medium-indexing_discovery-ssd.json | 2 +- .../results/shape_small-aggregation-hdd.json | 0 .../results/shape_small-aggregation-ssd.json | 2 +- ...hape_small-content_identification-hdd.json | 0 ...hape_small-content_identification-ssd.json | 2 +- .../shape_small-indexing_discovery-hdd.json | 0 .../shape_small-indexing_discovery-ssd.json | 2 +- .../benchmarks/results/whitepaper_metrics.csv | 0 .../benchmarks/src/bin/sd-bench-new.rs | 0 {core-new => core}/benchmarks/src/cli/args.rs | 0 .../benchmarks/src/cli/commands.rs | 0 {core-new => core}/benchmarks/src/cli/mod.rs | 0 .../benchmarks/src/config/mod.rs | 0 .../benchmarks/src/core_boot/mod.rs | 0 .../benchmarks/src/generator/filesystem.rs | 0 .../benchmarks/src/generator/mod.rs | 0 .../benchmarks/src/generator/noop.rs | 0 .../benchmarks/src/generator/registry.rs | 0 {core-new => core}/benchmarks/src/lib.rs | 0 .../benchmarks/src/metrics/mod.rs | 0 .../benchmarks/src/metrics/sources.rs | 0 .../benchmarks/src/recipe/mod.rs | 0 .../benchmarks/src/recipe/schema.rs | 0 .../benchmarks/src/reporting/csv.rs | 0 .../benchmarks/src/reporting/json_summary.rs | 0 .../benchmarks/src/reporting/mod.rs | 0 .../benchmarks/src/reporting/registry.rs | 0 .../benchmarks/src/runner/mod.rs | 0 .../benchmarks/src/runner/monitor.rs | 0 .../benchmarks/src/scenarios/common.rs | 0 .../src/scenarios/content_identification.rs | 0 .../benchmarks/src/scenarios/core_indexing.rs | 0 .../benchmarks/src/scenarios/mod.rs | 0 .../benchmarks/src/scenarios/registry.rs | 0 {core-new => core}/benchmarks/src/util/fs.rs | 0 {core-new => core}/benchmarks/src/util/mod.rs | 0 {core-new => core}/benchmarks/src/util/rng.rs | 0 .../benchmarks/src/util/time.rs | 0 core/build.rs | 20 +- core/crates/cloud-services/Cargo.toml | 57 - core/crates/cloud-services/src/client.rs | 358 ---- core/crates/cloud-services/src/error.rs | 216 --- .../src/key_manager/key_store.rs | 331 ---- .../cloud-services/src/key_manager/mod.rs | 183 -- core/crates/cloud-services/src/lib.rs | 55 - core/crates/cloud-services/src/p2p/mod.rs | 272 --- .../src/p2p/new_sync_messages_notifier.rs | 154 -- core/crates/cloud-services/src/p2p/runner.rs | 1109 ------------ core/crates/cloud-services/src/sync/ingest.rs | 121 -- core/crates/cloud-services/src/sync/mod.rs | 136 -- .../crates/cloud-services/src/sync/receive.rs | 356 ---- core/crates/cloud-services/src/sync/send.rs | 337 ---- .../cloud-services/src/token_refresher.rs | 468 ----- core/crates/file-path-helper/Cargo.toml | 33 - core/crates/file-path-helper/README.md | 3 - .../src/isolated_file_path_data.rs | 799 -------- core/crates/file-path-helper/src/lib.rs | 438 ----- core/crates/heavy-lifting/Cargo.toml | 63 - .../src/file_identifier/cas_id.rs | 80 - .../heavy-lifting/src/file_identifier/job.rs | 1072 ----------- .../heavy-lifting/src/file_identifier/mod.rs | 290 --- .../src/file_identifier/shallow.rs | 238 --- .../src/file_identifier/tasks/identifier.rs | 578 ------ .../src/file_identifier/tasks/mod.rs | 187 -- .../file_identifier/tasks/object_processor.rs | 434 ----- core/crates/heavy-lifting/src/indexer/job.rs | 1163 ------------ core/crates/heavy-lifting/src/indexer/mod.rs | 566 ------ .../heavy-lifting/src/indexer/shallow.rs | 303 ---- .../heavy-lifting/src/indexer/tasks/mod.rs | 7 - .../heavy-lifting/src/indexer/tasks/saver.rs | 309 ---- .../src/indexer/tasks/updater.rs | 307 ---- .../src/indexer/tasks/walker/entry.rs | 93 - .../src/indexer/tasks/walker/metadata.rs | 64 - .../src/indexer/tasks/walker/mod.rs | 1176 ------------ .../src/indexer/tasks/walker/rules.rs | 261 --- .../src/indexer/tasks/walker/save_state.rs | 219 --- .../heavy-lifting/src/job_system/error.rs | 76 - .../heavy-lifting/src/job_system/job.rs | 1189 ------------ .../heavy-lifting/src/job_system/mod.rs | 378 ---- .../heavy-lifting/src/job_system/report.rs | 436 ----- .../heavy-lifting/src/job_system/runner.rs | 713 -------- .../heavy-lifting/src/job_system/store.rs | 234 --- .../heavy-lifting/src/job_system/utils.rs | 35 - core/crates/heavy-lifting/src/lib.rs | 114 -- .../helpers/exif_media_data.rs | 172 -- .../helpers/ffmpeg_media_data.rs | 789 -------- .../src/media_processor/helpers/mod.rs | 12 - .../media_processor/helpers/thumbnailer.rs | 547 ------ .../heavy-lifting/src/media_processor/job.rs | 1096 ----------- .../heavy-lifting/src/media_processor/mod.rs | 193 -- .../src/media_processor/shallow.rs | 271 --- .../tasks/media_data_extractor.rs | 604 ------- .../src/media_processor/tasks/mod.rs | 5 - .../src/media_processor/tasks/thumbnailer.rs | 488 ----- core/crates/heavy-lifting/src/utils/mod.rs | 1 - .../heavy-lifting/src/utils/sub_path.rs | 113 -- core/crates/indexer-rules/Cargo.toml | 33 - core/crates/indexer-rules/src/lib.rs | 947 ---------- core/crates/indexer-rules/src/seed.rs | 346 ---- core/crates/indexer-rules/src/serde_impl.rs | 217 --- core/crates/p2p/Cargo.toml | 70 - core/crates/p2p/src/error.rs | 24 - core/crates/p2p/src/lib.rs | 162 -- .../authorize_new_device_in_sync_group.rs | 22 - .../p2p/src/schema/cloud_services/mod.rs | 25 - .../notify_new_sync_messages.rs | 12 - core/crates/p2p/src/schema/mod.rs | 36 - core/crates/p2p/src/server/mod.rs | 191 -- .../p2p/src/server/router/cloud_services.rs | 47 - core/crates/p2p/src/server/router/mod.rs | 19 - core/crates/prisma-helpers/Cargo.toml | 20 - core/crates/prisma-helpers/src/lib.rs | 576 ------ .../crates}/spacedrive-jobs-derive/Cargo.toml | 0 .../crates}/spacedrive-jobs-derive/src/lib.rs | 0 core/crates/sync/Cargo.toml | 39 - core/crates/sync/README.md | 132 -- core/crates/sync/src/backfill.rs | 669 ------- core/crates/sync/src/db_operation.rs | 103 -- core/crates/sync/src/ingest_utils.rs | 517 ------ core/crates/sync/src/lib.rs | 153 -- core/crates/sync/src/manager.rs | 750 -------- {core-new => core}/crush.json | 0 {core-new => core}/examples/file_type_demo.rs | 0 {core-new => core}/examples/indexing_demo.rs | 0 core/examples/indexing_showcase.rs | 136 ++ .../examples/job_logging_test.rs | 0 {core-new => core}/examples/library_demo.rs | 0 .../examples/location_watcher_demo.rs | 0 .../examples/pause_resume_demo.rs | 0 {core-new => core}/examples/shutdown_demo.rs | 0 .../examples/simple_pause_resume.rs | 0 {core-new => core}/examples/test_migration.rs | 0 {core-new => core}/examples/volume_demo.rs | 0 .../20230616064440_init/migration.sql | 271 --- .../migration.sql | 63 - .../20230619032753_p2p/migration.sql | 17 - .../migration.sql | 33 - .../migration.sql | 2 - .../20230711114013_preferences/migration.sql | 5 - .../migration.sql | 95 - .../migration.sql | 7 - .../migration.sql | 15 - .../20230812141757_added_albums/migration.sql | 23 - .../20230828195811_media_data/migration.sql | 152 -- .../migration.sql | 2 - .../migration.sql | 2 - .../migration.sql | 40 - .../migration.sql | 28 - .../migration.sql | 15 - .../migration.sql | 2 - .../migration.sql | 24 - .../migration.sql | 39 - .../migration.sql | 40 - .../migration.sql | 33 - .../migration.sql | 35 - .../migration.sql | 57 - .../migration.sql | 2 - .../migration.sql | 37 - .../migration.sql | 29 - .../migration.sql | 52 - .../migration.sql | 128 -- .../migration.sql | 29 - .../migration.sql | 12 - .../migration.sql | 2 - .../migration.sql | 6 - .../migration.sql | 2 - .../migration.sql | 15 - .../migration.sql | 226 --- core/prisma/migrations/migration_lock.toml | 3 - core/prisma/schema.prisma | 741 -------- {core-new => core}/rust-toolchain.toml | 0 core/src/api/backups.rs | 499 ----- core/src/api/cloud/devices.rs | 397 ---- core/src/api/cloud/libraries.rs | 140 -- core/src/api/cloud/locations.rs | 112 -- core/src/api/cloud/mod.rs | 391 ---- core/src/api/cloud/sync_groups.rs | 404 ----- core/src/api/cloud/thumbnails.rs | 61 - core/src/api/devices.rs | 64 - core/src/api/ephemeral_files.rs | 661 ------- core/src/api/files.rs | 1053 ----------- core/src/api/jobs.rs | 444 ----- core/src/api/keys.rs | 447 ----- core/src/api/labels.rs | 175 -- core/src/api/libraries.rs | 667 ------- core/src/api/locations.rs | 561 ------ core/src/api/mod.rs | 258 --- core/src/api/models.rs | 23 - core/src/api/nodes.rs | 135 -- core/src/api/notifications.rs | 167 -- core/src/api/p2p.rs | 154 -- core/src/api/preferences.rs | 22 - core/src/api/search/exif_data.rs | 29 - core/src/api/search/file_path.rs | 296 --- core/src/api/search/mod.rs | 424 ----- core/src/api/search/object.rs | 181 -- core/src/api/search/saved.rs | 232 --- core/src/api/search/utils.rs | 125 -- core/src/api/sync.rs | 78 - core/src/api/tags.rs | 396 ---- core/src/api/utils/invalidate.rs | 431 ----- core/src/api/utils/library.rs | 53 - core/src/api/utils/mod.rs | 109 -- core/src/api/volumes.rs | 82 - core/src/api/web_api.rs | 29 - {core-new => core}/src/bin/cli.rs | 0 {core-new => core}/src/config/app_config.rs | 0 {core-new => core}/src/config/migration.rs | 0 {core-new => core}/src/config/mod.rs | 0 core/src/context.rs | 275 +-- core/src/custom_uri/mod.rs | 592 ------ core/src/custom_uri/mpsc_to_async_write.rs | 44 - core/src/custom_uri/serve_file.rs | 150 -- core/src/custom_uri/utils.rs | 84 - {core-new => core}/src/device/config.rs | 0 {core-new => core}/src/device/manager.rs | 0 {core-new => core}/src/device/mod.rs | 0 {core-new => core}/src/domain/addressing.rs | 0 .../src/domain/content_identity.rs | 0 {core-new => core}/src/domain/device.rs | 0 {core-new => core}/src/domain/entry.rs | 0 {core-new => core}/src/domain/location.rs | 0 {core-new => core}/src/domain/mod.rs | 0 .../src/domain/user_metadata.rs | 0 {core-new => core}/src/domain/volume.rs | 0 {core-new => core}/src/file_type/builtin.rs | 0 .../src/file_type/definitions/archives.toml | 0 .../src/file_type/definitions/audio.toml | 0 .../src/file_type/definitions/code.toml | 0 .../src/file_type/definitions/documents.toml | 0 .../src/file_type/definitions/images.toml | 0 .../src/file_type/definitions/misc.toml | 0 .../src/file_type/definitions/video.toml | 0 {core-new => core}/src/file_type/magic.rs | 0 {core-new => core}/src/file_type/mod.rs | 0 {core-new => core}/src/file_type/registry.rs | 0 .../actions/BUILDER_REFACTOR_PLAN.md | 0 .../src/infrastructure/actions/builder.rs | 0 .../src/infrastructure/actions/error.rs | 0 .../src/infrastructure/actions/handler.rs | 0 .../src/infrastructure/actions/manager.rs | 0 .../src/infrastructure/actions/mod.rs | 0 .../src/infrastructure/actions/output.rs | 0 .../src/infrastructure/actions/receipt.rs | 0 .../src/infrastructure/actions/registry.rs | 0 .../src/infrastructure/actions/tests.rs | 0 .../src/infrastructure/cli/README.md | 0 .../src/infrastructure/cli/adapters/copy.rs | 0 .../src/infrastructure/cli/adapters/mod.rs | 0 .../src/infrastructure/cli/commands/daemon.rs | 0 .../src/infrastructure/cli/commands/file.rs | 0 .../src/infrastructure/cli/commands/job.rs | 0 .../infrastructure/cli/commands/library.rs | 0 .../infrastructure/cli/commands/location.rs | 0 .../src/infrastructure/cli/commands/mod.rs | 0 .../infrastructure/cli/commands/network.rs | 0 .../src/infrastructure/cli/commands/system.rs | 0 .../src/infrastructure/cli/commands/volume.rs | 0 .../src/infrastructure/cli/daemon/client.rs | 0 .../src/infrastructure/cli/daemon/config.rs | 0 .../cli/daemon/handlers/core.rs | 0 .../cli/daemon/handlers/file.rs | 0 .../infrastructure/cli/daemon/handlers/job.rs | 0 .../cli/daemon/handlers/library.rs | 0 .../cli/daemon/handlers/location.rs | 0 .../infrastructure/cli/daemon/handlers/mod.rs | 0 .../cli/daemon/handlers/network.rs | 0 .../cli/daemon/handlers/system.rs | 0 .../cli/daemon/handlers/volume.rs | 0 .../src/infrastructure/cli/daemon/mod.rs | 0 .../cli/daemon/services/helpers.rs | 0 .../infrastructure/cli/daemon/services/mod.rs | 0 .../cli/daemon/services/state.rs | 0 .../cli/daemon/types/commands.rs | 0 .../infrastructure/cli/daemon/types/common.rs | 0 .../infrastructure/cli/daemon/types/mod.rs | 0 .../cli/daemon/types/responses.rs | 0 .../src/infrastructure/cli/mod.rs | 0 .../src/infrastructure/cli/output/context.rs | 0 .../infrastructure/cli/output/formatters.rs | 0 .../src/infrastructure/cli/output/messages.rs | 0 .../src/infrastructure/cli/output/mod.rs | 0 .../src/infrastructure/cli/output/section.rs | 0 .../src/infrastructure/cli/output/tests.rs | 0 .../src/infrastructure/cli/pairing_ui.rs | 0 .../src/infrastructure/cli/state.rs | 0 .../src/infrastructure/cli/tui.rs | 0 .../src/infrastructure/cli/utils.rs | 0 .../database/entities/audit_log.rs | 0 .../database/entities/collection.rs | 0 .../database/entities/collection_entry.rs | 0 .../database/entities/content_identity.rs | 0 .../database/entities/content_kind.rs | 0 .../database/entities/device.rs | 0 .../database/entities/directory_paths.rs | 0 .../infrastructure/database/entities/entry.rs | 0 .../database/entities/entry_closure.rs | 0 .../database/entities/indexer_rule.rs | 0 .../infrastructure/database/entities/label.rs | 0 .../database/entities/location.rs | 0 .../database/entities/metadata_label.rs | 0 .../database/entities/metadata_tag.rs | 0 .../database/entities/mime_type.rs | 0 .../infrastructure/database/entities/mod.rs | 0 .../database/entities/sidecar.rs | 0 .../database/entities/sidecar_availability.rs | 0 .../infrastructure/database/entities/tag.rs | 0 .../database/entities/user_metadata.rs | 0 .../database/entities/volume.rs | 0 .../m20240101_000001_initial_schema.rs | 0 .../m20240102_000001_populate_lookups.rs | 0 .../m20240107_000001_create_collections.rs | 0 .../m20250109_000001_create_sidecars.rs | 0 ...m20250110_000001_refactor_volumes_table.rs | 0 .../m20250112_000001_create_indexer_rules.rs | 0 .../infrastructure/database/migration/mod.rs | 0 .../src/infrastructure/database/mod.rs | 0 .../src/infrastructure/events/mod.rs | 0 .../src/infrastructure/jobs/context.rs | 0 .../src/infrastructure/jobs/database.rs | 0 .../src/infrastructure/jobs/error.rs | 0 .../src/infrastructure/jobs/executor.rs | 0 .../infrastructure/jobs/generic_progress.rs | 0 .../src/infrastructure/jobs/handle.rs | 0 .../src/infrastructure/jobs/logger.rs | 0 .../src/infrastructure/jobs/manager.rs | 0 .../src/infrastructure/jobs/mod.rs | 0 .../src/infrastructure/jobs/output.rs | 0 .../src/infrastructure/jobs/progress.rs | 0 .../src/infrastructure/jobs/registry.rs | 0 .../src/infrastructure/jobs/traits.rs | 0 .../src/infrastructure/jobs/types.rs | 0 {core-new => core}/src/infrastructure/mod.rs | 0 .../src/keys/device_key_manager.rs | 0 .../src/keys/library_key_manager.rs | 0 {core-new => core}/src/keys/mod.rs | 0 core/src/lib.rs | 848 +++++---- core/src/library/config.rs | 647 ++----- {core-new => core}/src/library/error.rs | 0 core/src/library/library.rs | 218 --- {core-new => core}/src/library/lock.rs | 0 {core-new => core}/src/library/manager.rs | 0 core/src/library/manager/error.rs | 65 - core/src/library/manager/mod.rs | 697 ------- core/src/library/manager/pragmas.rs | 109 -- core/src/library/mod.rs | 244 ++- core/src/library/name.rs | 61 - core/src/library/statistics.rs | 122 -- core/src/location/archive/archive_job.rs | 0 core/src/location/archive/mod.rs | 0 core/src/location/error.rs | 115 -- {core-new => core}/src/location/manager.rs | 0 core/src/location/manager/mod.rs | 451 ----- core/src/location/manager/runner.rs | 459 ----- core/src/location/manager/watcher/android.rs | 303 ---- core/src/location/manager/watcher/ios.rs | 452 ----- core/src/location/manager/watcher/linux.rs | 309 ---- core/src/location/manager/watcher/macos.rs | 478 ----- core/src/location/manager/watcher/mod.rs | 885 --------- core/src/location/manager/watcher/utils.rs | 1241 ------------- core/src/location/manager/watcher/windows.rs | 419 ----- core/src/location/metadata.rs | 295 --- core/src/location/mod.rs | 1602 ++++++----------- core/src/location/non_indexed.rs | 414 ----- core/src/node/config.rs | 572 ------ core/src/node/hardware.rs | 210 --- core/src/node/mod.rs | 6 - core/src/node/platform.rs | 62 - core/src/notifications.rs | 34 - core/src/object/fs/archive.rs | 0 core/src/object/fs/convert.rs | 0 core/src/object/fs/error.rs | 51 - core/src/object/fs/mod.rs | 222 --- core/src/object/fs/old_copy.rs | 669 ------- core/src/object/fs/old_cut.rs | 142 -- core/src/object/fs/old_delete.rs | 122 -- core/src/object/fs/old_erase.rs | 205 --- core/src/object/fs/sync.rs | 0 .../object/media/old_thumbnail/clean_up.rs | 183 -- .../object/media/old_thumbnail/directory.rs | 385 ---- core/src/object/mod.rs | 3 - core/src/object/old_orphan_remover.rs | 106 -- core/src/object/tag/mod.rs | 47 - core/src/object/tag/seed.rs | 34 - core/src/object/validation/hash.rs | 25 - core/src/object/validation/mod.rs | 23 - .../object/validation/old_validator_job.rs | 203 --- core/src/old_job/error.rs | 119 -- core/src/old_job/manager.rs | 416 ----- core/src/old_job/mod.rs | 1148 ------------ core/src/old_job/report.rs | 504 ------ core/src/old_job/worker.rs | 660 ------- core/src/old_p2p/events.rs | 159 -- core/src/old_p2p/libraries.rs | 139 -- core/src/old_p2p/manager.rs | 466 ----- core/src/old_p2p/metadata.rs | 132 -- core/src/old_p2p/mod.rs | 17 - core/src/old_p2p/operations/library.rs | 134 -- core/src/old_p2p/operations/mod.rs | 8 - core/src/old_p2p/operations/ping.rs | 37 - core/src/old_p2p/operations/rspc.rs | 63 - core/src/old_p2p/operations/spacedrop.rs | 382 ---- core/src/old_p2p/protocol.rs | 131 -- core/src/old_p2p/sync/mod.rs | 262 --- core/src/old_p2p/sync/proto.rs | 43 - .../src/operations/addressing.rs | 0 .../src/operations/content/action.rs | 0 .../src/operations/content/mod.rs | 0 .../src/operations/devices/mod.rs | 0 .../src/operations/devices/revoke/action.rs | 0 .../src/operations/devices/revoke/mod.rs | 0 .../src/operations/devices/revoke/output.rs | 0 .../src/operations/entries/mod.rs | 0 .../src/operations/entries/state.rs | 0 .../src/operations/files/copy/action.rs | 0 .../src/operations/files/copy/database.rs | 0 .../operations/files/copy/docs/ANALYSIS.md | 0 .../files/copy/docs/FILE_SYNC_OVERLAP.md | 0 .../copy/docs/PROGRESSIVE_COPY_DESIGN.md | 0 .../copy/docs/RESUME_VALIDATION_DESIGN.md | 0 .../src/operations/files/copy/input.rs | 0 .../src/operations/files/copy/job.rs | 0 .../src/operations/files/copy/mod.rs | 0 .../src/operations/files/copy/output.rs | 0 .../src/operations/files/copy/routing.rs | 0 .../src/operations/files/copy/strategy.rs | 0 .../src/operations/files/delete/action.rs | 0 .../src/operations/files/delete/job.rs | 0 .../src/operations/files/delete/mod.rs | 0 .../src/operations/files/delete/output.rs | 0 .../files/duplicate_detection/action.rs | 0 .../files/duplicate_detection/job.rs | 0 .../files/duplicate_detection/mod.rs | 0 .../src/operations/files/mod.rs | 0 .../src/operations/files/validation/action.rs | 0 .../src/operations/files/validation/job.rs | 0 .../src/operations/files/validation/mod.rs | 0 .../src/operations/indexing/action.rs | 0 .../indexing/change_detection/mod.rs | 0 .../src/operations/indexing/entry.rs | 0 .../src/operations/indexing/hierarchy.rs | 0 .../src/operations/indexing/job.rs | 0 .../src/operations/indexing/metrics.rs | 0 .../src/operations/indexing/mod.rs | 0 .../src/operations/indexing/path_resolver.rs | 0 .../src/operations/indexing/persistence.rs | 0 .../operations/indexing/phases/aggregation.rs | 0 .../src/operations/indexing/phases/content.rs | 0 .../operations/indexing/phases/discovery.rs | 0 .../src/operations/indexing/phases/mod.rs | 0 .../operations/indexing/phases/processing.rs | 0 .../src/operations/indexing/progress.rs | 0 .../src/operations/indexing/rules.rs | 0 .../src/operations/indexing/state.rs | 0 .../src/operations/indexing/tests/mod.rs | 0 .../src/operations/libraries/create/action.rs | 0 .../src/operations/libraries/create/mod.rs | 0 .../src/operations/libraries/create/output.rs | 0 .../src/operations/libraries/delete/action.rs | 0 .../src/operations/libraries/delete/mod.rs | 0 .../src/operations/libraries/delete/output.rs | 0 .../src/operations/libraries/export/action.rs | 0 .../src/operations/libraries/export/mod.rs | 0 .../src/operations/libraries/export/output.rs | 0 .../src/operations/libraries/mod.rs | 0 .../src/operations/libraries/rename/action.rs | 0 .../src/operations/libraries/rename/mod.rs | 0 .../src/operations/libraries/rename/output.rs | 0 .../src/operations/locations/add/action.rs | 0 .../src/operations/locations/add/mod.rs | 0 .../src/operations/locations/add/output.rs | 0 .../src/operations/locations/index/action.rs | 0 .../src/operations/locations/index/mod.rs | 0 .../src/operations/locations/mod.rs | 0 .../src/operations/locations/remove/action.rs | 0 .../src/operations/locations/remove/mod.rs | 0 .../src/operations/locations/remove/output.rs | 0 .../src/operations/locations/rescan/action.rs | 0 .../src/operations/locations/rescan/mod.rs | 0 .../src/operations/locations/rescan/output.rs | 0 .../src/operations/media/live_photo.rs | 0 .../src/operations/media/live_photo_query.rs | 0 .../src/operations/media/mod.rs | 0 .../src/operations/media/thumbnail/action.rs | 0 .../src/operations/media/thumbnail/error.rs | 0 .../operations/media/thumbnail/generator.rs | 0 .../src/operations/media/thumbnail/job.rs | 0 .../src/operations/media/thumbnail/mod.rs | 0 .../src/operations/media/thumbnail/state.rs | 0 .../src/operations/media/thumbnail/utils.rs | 0 .../src/operations/metadata/action.rs | 0 .../src/operations/metadata/mod.rs | 0 {core-new => core}/src/operations/mod.rs | 0 .../src/operations/sidecar/mod.rs | 0 .../src/operations/sidecar/path.rs | 0 .../src/operations/sidecar/types.rs | 0 .../src/operations/volumes/mod.rs | 0 .../operations/volumes/speed_test/action.rs | 0 .../operations/volumes/speed_test/handler.rs | 0 .../src/operations/volumes/speed_test/mod.rs | 0 .../src/operations/volumes/track/action.rs | 0 .../src/operations/volumes/track/handler.rs | 0 .../src/operations/volumes/track/mod.rs | 0 .../src/operations/volumes/untrack/action.rs | 0 .../src/operations/volumes/untrack/handler.rs | 0 .../src/operations/volumes/untrack/mod.rs | 0 core/src/preferences/kv.rs | 161 -- core/src/preferences/library.rs | 129 -- core/src/preferences/mod.rs | 56 - core/src/search/mod.rs | 1 - {core-new => core}/src/services/device.rs | 0 .../src/services/entry_state_service.rs | 0 .../src/services/file_sharing.rs | 0 .../location_watcher/event_handler.rs | 0 .../src/services/location_watcher/mod.rs | 0 .../location_watcher/platform/linux.rs | 0 .../location_watcher/platform/macos.rs | 0 .../services/location_watcher/platform/mod.rs | 0 .../location_watcher/platform/windows.rs | 0 .../src/services/location_watcher/utils.rs | 0 {core-new => core}/src/services/mod.rs | 0 .../services/networking/core/event_loop.rs | 0 .../src/services/networking/core/mod.rs | 0 .../services/networking/device/connection.rs | 0 .../src/services/networking/device/mod.rs | 0 .../services/networking/device/persistence.rs | 0 .../services/networking/device/registry.rs | 0 .../src/services/networking/mod.rs | 0 .../networking/protocols/file_transfer.rs | 0 .../networking/protocols/messaging.rs | 0 .../src/services/networking/protocols/mod.rs | 0 .../networking/protocols/pairing/initiator.rs | 0 .../networking/protocols/pairing/joiner.rs | 0 .../networking/protocols/pairing/messages.rs | 0 .../networking/protocols/pairing/mod.rs | 0 .../protocols/pairing/persistence.rs | 0 .../networking/protocols/pairing/security.rs | 0 .../networking/protocols/pairing/types.rs | 0 .../services/networking/protocols/registry.rs | 0 .../src/services/networking/utils/identity.rs | 0 .../src/services/networking/utils/logging.rs | 0 .../src/services/networking/utils/mod.rs | 0 .../src/services/sidecar_manager.rs | 0 .../src/services/volume_monitor.rs | 0 {core-new => core}/src/shared/errors.rs | 0 {core-new => core}/src/shared/mod.rs | 0 {core-new => core}/src/shared/types.rs | 0 {core-new => core}/src/shared/utils.rs | 0 {core-new => core}/src/test_framework/mod.rs | 0 .../src/test_framework/runner.rs | 0 core/src/util/abort_on_drop.rs | 39 - core/src/util/batched_stream.rs | 67 - core/src/util/debug_initializer.rs | 198 -- core/src/util/infallible_request.rs | 28 - core/src/util/maybe_undefined.rs | 88 - core/src/util/mod.rs | 17 - core/src/util/mpscrr.rs | 326 ---- core/src/util/observable.rs | 101 -- core/src/util/unsafe_streamed_query.rs | 31 - core/src/util/version_manager.rs | 224 --- core/src/volume/actor.rs | 521 ------ .../src/volume/classification.rs | 0 core/src/volume/error.rs | 342 +--- {core-new => core}/src/volume/manager.rs | 0 core/src/volume/mod.rs | 113 +- core/src/volume/os.rs | 502 ------ {core-new => core}/src/volume/os_detection.rs | 0 core/src/volume/speed.rs | 443 +++-- core/src/volume/state.rs | 237 --- core/src/volume/types.rs | 755 +++++--- core/src/volume/volumes.rs | 130 -- core/src/volume/watcher.rs | 262 --- {core-new => core}/tests/copy_action_test.rs | 0 .../tests/copy_progress_test.rs | 0 .../tests/cross_device_copy_test.rs | 0 .../tests/database_migration_test.rs | 0 .../tests/device_pairing_test.rs | 0 .../tests/device_persistence_test.rs | 0 {core-new => core}/tests/event_system_test.rs | 0 .../tests/file_transfer_test.rs | 0 {core-new => core}/tests/helpers/mod.rs | 0 .../tests/helpers/test_volumes.rs | 0 .../tests/indexing_rules_test.rs | 0 {core-new => core}/tests/indexing_test.rs | 0 .../tests/job_pause_resume_test.rs | 0 .../tests/job_registration_test.rs | 0 {core-new => core}/tests/job_shutdown_test.rs | 0 {core-new => core}/tests/library_test.rs | 0 .../tests/volume_tracking_test.rs | 0 .../volume_tracking_with_test_volumes.rs | 0 .../whitepaper/architecture.png | Bin {core-new => core}/whitepaper/comment.cut | 0 {core-new => core}/whitepaper/grok-changes.md | 0 .../whitepaper/grok-critique.md | 0 .../whitepaper/grok-critique2.md | 0 .../whitepaper/proposed-changes-v2.md | 0 .../whitepaper/proposed-changes.md | 0 {core-new => core}/whitepaper/references.bib | 0 {core-new => core}/whitepaper/spacedrive.bbl | 0 {core-new => core}/whitepaper/spacedrive.blg | 0 {core-new => core}/whitepaper/spacedrive.out | 0 {core-new => core}/whitepaper/spacedrive.pdf | Bin {core-new => core}/whitepaper/spacedrive.tex | 2 +- crates/ai/Cargo.toml | 79 - crates/ai/README.md | 3 - crates/ai/src/lib.rs | 86 - crates/ai/src/old_image_labeler/mod.rs | 56 - crates/ai/src/old_image_labeler/model/mod.rs | 259 --- .../ai/src/old_image_labeler/model/yolov8.rs | 168 -- crates/ai/src/old_image_labeler/old_actor.rs | 612 ------- crates/ai/src/old_image_labeler/process.rs | 507 ------ crates/ai/src/utils/mod.rs | 27 - crates/file-ext/Cargo.toml | 20 - crates/file-ext/src/extensions.rs | 566 ------ crates/file-ext/src/kind.rs | 62 - crates/file-ext/src/lib.rs | 4 - crates/file-ext/src/magic.rs | 232 --- crates/file-ext/src/text.rs | 297 --- crates/old-p2p/Cargo.toml | 51 - crates/old-p2p/README.md | 9 - crates/old-p2p/crates/block/Cargo.toml | 18 - crates/old-p2p/crates/block/src/block.rs | 97 - crates/old-p2p/crates/block/src/block_size.rs | 120 -- crates/old-p2p/crates/block/src/lib.rs | 469 ----- crates/old-p2p/crates/block/src/sb_request.rs | 268 --- crates/old-p2p/crates/proto/Cargo.toml | 14 - crates/old-p2p/crates/proto/src/lib.rs | 88 - crates/old-p2p/crates/tunnel/Cargo.toml | 17 - crates/old-p2p/crates/tunnel/src/lib.rs | 143 -- crates/old-p2p/src/hook.rs | 141 -- crates/old-p2p/src/hooks.rs | 9 - crates/old-p2p/src/hooks/mdns.rs | 234 --- crates/old-p2p/src/hooks/quic.rs | 10 - crates/old-p2p/src/hooks/quic/handle.rs | 122 -- crates/old-p2p/src/hooks/quic/transport.rs | 928 ---------- crates/old-p2p/src/hooks/quic/utils.rs | 53 - crates/old-p2p/src/identity.rs | 172 -- crates/old-p2p/src/lib.rs | 34 - crates/old-p2p/src/p2p.rs | 407 ----- crates/old-p2p/src/peer.rs | 331 ---- crates/old-p2p/src/smart_guards.rs | 55 - crates/old-p2p/src/stream.rs | 81 - crates/prisma-cli/Cargo.toml | 18 - crates/prisma-cli/src/bin/prisma.rs | 3 - crates/prisma-cli/src/bin/sync.rs | 3 - crates/prisma/.gitignore | 1 - crates/prisma/Cargo.toml | 16 - crates/prisma/src/lib.rs | 15 - crates/sync-generator/Cargo.toml | 16 - crates/sync-generator/src/attribute/mod.rs | 55 - crates/sync-generator/src/attribute/parser.rs | 155 -- crates/sync-generator/src/lib.rs | 195 -- crates/sync-generator/src/model.rs | 214 --- crates/sync-generator/src/sync_data.rs | 351 ---- crates/sync/Cargo.toml | 16 - crates/sync/src/compressed.rs | 359 ---- crates/sync/src/crdt.rs | 76 - crates/sync/src/factory.rs | 163 -- crates/sync/src/lib.rs | 44 - crates/sync/src/model_traits.rs | 27 - .../task-validator/Cargo.toml | 0 .../task-validator/src/main.rs | 0 crates/utils/Cargo.toml | 23 - crates/utils/src/db.rs | 163 -- crates/utils/src/error.rs | 59 - crates/utils/src/lib.rs | 135 -- {core-new/docs => docs}/README.md | 0 {core-new/docs => docs}/benchmarks.md | 0 {core-new/docs => docs/core}/architecture.md | 0 .../docs => docs/core}/cli-multi-instance.md | 0 {core-new/docs => docs/core}/cli.md | 0 {core-new/docs => docs/core}/database.md | 0 {core-new/docs => docs/core}/domain-models.md | 0 {core-new/docs => docs/core}/examples.md | 0 {core-new/docs => docs/core}/indexing.md | 0 {core-new/docs => docs/core}/job-system.md | 0 {core-new/docs => docs/core}/library.md | 0 {core-new/docs => docs/core}/locations.md | 0 {core-new/docs => docs/core}/networking.md | 0 {core-new/docs => docs/core}/pairing.md | 0 {core-new/docs => docs/core}/task-tracking.md | 0 {core-new/docs => docs/core}/testing.md | 0 .../docs => docs/core}/virtual_sidecars.md | 0 {core-new/docs => docs/core}/volume-system.md | 0 .../docs => docs}/design/ACTIONS_REFACTOR.md | 0 .../design/ACTION_SYSTEM_DESIGN.md | 0 .../design/AGENT_MANAGER_DESIGN.md | 0 .../docs => docs}/design/API_COMPARISON.md | 0 .../design/ARCHITECTURE_DECISIONS.md | 0 .../design/AT_REST_LIBRARY_ENCRYPTION.md | 0 .../design/BENCHMARKING_SUITE_DESIGN.md | 0 .../design/CLOSURE_TABLE_INDEXING_PROPOSAL.md | 0 ...OSS_DEVICE_FILE_TRANSFER_IMPLEMENTATION.md | 0 ...ROSS_PLATFORM_COPY_AND_VOLUME_AWARENESS.md | 0 .../docs => docs}/design/DAEMON_REFACTOR.md | 0 .../design/DESIGN_CORE_LIFECYCLE.md | 0 .../design/DESIGN_DEVICE_MANAGEMENT.md | 0 .../design/DESIGN_FILE_DATA_MODEL.md | 0 .../design/DESIGN_FILE_DATA_MODEL_VISUAL.md | 0 .../design/DESIGN_FILE_TYPE_SYSTEM.md | 0 .../design/DESIGN_LIBRARY_IMPLEMENTATION.md | 0 .../design/DESIGN_LIBRARY_ORGANIZATION.md | 0 .../DESIGN_LIBRARY_ORGANIZATION_VISUAL.md | 0 .../design/DESIGN_OPTIMIZED_STORAGE.md | 0 .../design/DESIGN_VDFS_INTEGRATION.md | 0 .../design/DEVICE_PAIRING_PROTOCOL.md | 0 .../design/DOMAIN_MODELS_README.md | 0 .../design/ENTITY_REFACTOR_DESIGN.md | 0 .../design/FFMPEG_BUNDLING_DESIGN.md | 57 +- .../design/FILE_SHARING_DESIGN.md | 0 .../FUNCTION_BASED_TEST_FRAMEWORK_DESIGN.md | 0 .../design/IDEA_FOR_BETTER_INDEXING.md | 0 .../design/IMPLEMENTATION_STATUS.md | 0 .../docs => docs}/design/INDEXER_ANALYSIS.md | 0 .../design/INDEXER_ANALYSIS_2.md | 0 .../design/INDEXER_JOB_EXAMPLE.md | 0 .../docs => docs}/design/INDEXER_PROGRESS.md | 6 +- .../design/INDEXER_RULES_SYSTEM.md | 71 +- .../INDEXING_DISCOVERY_OPTIMIZATION_PLAN.md | 0 .../design/INTEGRATION_SYSTEM_DESIGN.md | 0 .../INTEGRATION_SYSTEM_DESIGN_GEMINI.md | 0 .../design/IPHONE_AS_VOLUME_DESIGN.md | 0 .../design/IROH_MIGRATION_DESIGN.md | 0 .../docs => docs}/design/JOB_SYSTEM_DESIGN.md | 0 .../design/JOB_SYSTEM_MACRO_EXAMPLE.md | 0 .../docs => docs}/design/JOB_SYSTEM_README.md | 0 .../design/LIBP2P_INTEGRATION_DESIGN.md | 0 .../design/LIBRARY_LEADERSHIP.md | 0 .../design/NETWORKING_SYSTEM_DESIGN.md | 228 +-- .../design/OLD_SPACEDRIVE_ANALYSIS.md | 0 .../design/OPERATIONS_REFACTOR_PLAN.md | 0 .../PERSISTENT_DEVICE_CONNECTIONS_DESIGN.md | 0 .../design/REFERENCE_SIDECARS.md | 0 .../docs => docs}/design/REWRITE_PLAN.MD | 2 +- .../docs => docs}/design/SDPATH_REFACTOR.md | 0 .../design/SDPATH_REFACTOR_COVERAGE.md | 0 .../docs => docs}/design/SEARCH_DESIGN.md | 95 +- .../design/SIMULATION_ENGINE_DESIGN.md | 0 .../design/SPACEDRIVE_COMPLETE_OVERVIEW.md | 143 +- .../docs => docs}/design/SPACEDROP_DESIGN.md | 0 .../design/SPACEDROP_IMPLEMENTATION_PLAN.md | 24 +- {core-new/docs => docs}/design/STRUCTURE.md | 6 +- {core-new/docs => docs}/design/SYNC_DESIGN.md | 0 .../design/SYNC_DESIGN_2025_08_19.md | 0 .../design/SYNC_FIRST_DRAFT_DESIGN.md | 0 .../design/SYNC_INTEGRATION_NOTES.md | 0 .../design/THUMBNAIL_SYSTEM_DESIGN.md | 14 +- {core-new/docs => docs}/design/UI_DESIGN.md | 0 .../docs => docs}/design/VDFS_MODEL_VISUAL.md | 0 .../design/VIRTUAL_LOCATIONS_DESIGN.md | 0 .../design/VIRTUAL_SIDECAR_SYSTEM.md | 0 .../design/VOLUME_CLASSIFICATION_DESIGN.md | 0 .../VOLUME_TRACKING_IMPLEMENTATION_PLAN.md | 0 .../design/WATCHER_VDFS_INTEGRATION.md | 190 +- .../design/WHITEPAPER_IMPL_ROADMAP.md | 0 .../design/cli-output-refactor.md | 0 .../design/frontend_graphql_usage.tsx | 0 .../design/indexer-scope-upgrade.md | 0 .../docs => docs}/design/landing-page-idea.md | 0 .../networking_implementation_summary.md | 0 {core-new/docs => docs}/history.md | 0 {core-new/docs => docs}/philosophy.md | 0 {core-new/docs => docs}/roadmap.md | 0 {core-new/docs => docs}/whitepaper.md | 0 error.txt | 1024 ----------- interface/.eslintrc.js | 13 - interface/.gitignore | 1 - interface/ErrorFallback.tsx | 200 -- interface/RoutingContext.tsx | 30 - interface/TabsContext.tsx | 18 - interface/app/$libraryId/404.tsx | 29 - interface/app/$libraryId/Explorer/Context.tsx | 29 - .../ContextMenu/AssignTagMenuItems.tsx | 258 --- .../Explorer/ContextMenu/ConditionalItem.tsx | 45 - .../ContextMenu/FilePath/CutCopyItems.tsx | 56 - .../Explorer/ContextMenu/FilePath/Items.tsx | 299 --- .../Explorer/ContextMenu/Object/Items.tsx | 112 -- .../Explorer/ContextMenu/OpenWith.tsx | 143 -- .../Explorer/ContextMenu/SharedItems.tsx | 236 --- .../Explorer/ContextMenu/context.tsx | 43 - .../$libraryId/Explorer/ContextMenu/index.tsx | 95 - .../app/$libraryId/Explorer/CopyAsPath.tsx | 36 - .../$libraryId/Explorer/DismissibleNotice.tsx | 79 - .../app/$libraryId/Explorer/DragOverlay.tsx | 96 - .../$libraryId/Explorer/ExplorerDraggable.tsx | 28 - .../$libraryId/Explorer/ExplorerDroppable.tsx | 36 - .../$libraryId/Explorer/ExplorerPathBar.tsx | 257 --- .../$libraryId/Explorer/ExplorerTagBar.tsx | 369 ---- .../Explorer/FilePath/DeleteDialog.tsx | 130 -- .../Explorer/FilePath/ErrorBarrier.tsx | 40 - .../$libraryId/Explorer/FilePath/Image.tsx | 14 - .../Explorer/FilePath/LayeredFileIcon.tsx | 62 - .../$libraryId/Explorer/FilePath/Original.tsx | 263 --- .../Explorer/FilePath/RenameTextBox.tsx | 328 ---- .../Explorer/FilePath/Thumb.module.scss | 25 - .../$libraryId/Explorer/FilePath/Thumb.tsx | 439 ----- .../$libraryId/Explorer/FilePath/useFrame.tsx | 15 - .../app/$libraryId/Explorer/FilePath/utils.ts | 107 -- .../Explorer/Inspector/FavoriteButton.tsx | 36 - .../Explorer/Inspector/MediaData.tsx | 249 --- .../$libraryId/Explorer/Inspector/Note.tsx | 62 - .../$libraryId/Explorer/Inspector/index.tsx | 672 ------- .../$libraryId/Explorer/Inspector/store.tsx | 4 - .../OptionsPanel/ListView/IconSize.tsx | 37 - .../OptionsPanel/ListView/TextSize.tsx | 37 - .../Explorer/OptionsPanel/ListView/index.tsx | 11 - .../Explorer/OptionsPanel/ListView/util.ts | 18 - .../Explorer/OptionsPanel/index.tsx | 236 --- .../$libraryId/Explorer/ParentContextMenu.tsx | 297 --- .../Explorer/QuickPreview/Context.tsx | 26 - .../Explorer/QuickPreview/ImageSlider.tsx | 111 -- .../Explorer/QuickPreview/index.tsx | 778 -------- .../$libraryId/Explorer/QuickPreview/store.ts | 9 - .../Explorer/RevealInNativeExplorer.tsx | 35 - .../app/$libraryId/Explorer/TopBarOptions.tsx | 184 -- .../app/$libraryId/Explorer/View/Context.tsx | 26 - .../Explorer/View/DragScrollable.tsx | 33 - .../$libraryId/Explorer/View/EmptyNotice.tsx | 41 - .../View/Grid/DragSelect/DragSelectable.tsx | 17 - .../Explorer/View/Grid/DragSelect/context.tsx | 20 - .../Explorer/View/Grid/DragSelect/index.tsx | 600 ------ .../Grid/DragSelect/useDragSelectable.tsx | 58 - .../Grid/DragSelect/useSelectedTargets.tsx | 35 - .../Explorer/View/Grid/DragSelect/util.ts | 16 - .../$libraryId/Explorer/View/Grid/Item.tsx | 72 - .../Explorer/View/Grid/useKeySelection.tsx | 151 -- .../Explorer/View/GridView/Item/Context.tsx | 13 - .../Explorer/View/GridView/Item/index.tsx | 218 --- .../Explorer/View/GridView/index.tsx | 83 - .../Explorer/View/ListView/Item.tsx | 92 - .../Explorer/View/ListView/TableRow.tsx | 57 - .../Explorer/View/ListView/context.tsx | 16 - .../Explorer/View/ListView/index.tsx | 1006 ----------- .../Explorer/View/ListView/useRanges.tsx | 158 -- .../Explorer/View/ListView/useTable.tsx | 236 --- .../Explorer/View/MediaView/DateHeader.tsx | 60 - .../Explorer/View/MediaView/Item.tsx | 68 - .../Explorer/View/MediaView/index.tsx | 235 --- .../Explorer/View/MediaView/util.ts | 41 - .../Explorer/View/RenamableItemText.tsx | 180 -- .../app/$libraryId/Explorer/View/ViewItem.tsx | 244 --- .../app/$libraryId/Explorer/View/index.tsx | 265 --- .../Explorer/View/useActiveItem.tsx | 124 -- .../Explorer/View/useDragScrollable.tsx | 66 - .../Explorer/hooks/useExplorerCopyPaste.tsx | 160 -- interface/app/$libraryId/Explorer/index.tsx | 158 -- interface/app/$libraryId/Explorer/store.ts | 196 -- .../app/$libraryId/Explorer/useExplorer.ts | 223 --- .../$libraryId/Explorer/useExplorerDnd.tsx | 194 -- .../Explorer/useExplorerDraggable.tsx | 67 - .../Explorer/useExplorerDroppable.tsx | 210 --- .../Explorer/useExplorerItemData.tsx | 130 -- .../Explorer/useExplorerOperatingSystem.tsx | 29 - .../Explorer/useExplorerPreferences.ts | 69 - .../$libraryId/Explorer/useKeyRevealFinder.ts | 61 - interface/app/$libraryId/Explorer/util.ts | 176 -- interface/app/$libraryId/KeyManager/Key.tsx | 251 --- interface/app/$libraryId/KeyManager/List.tsx | 60 - .../app/$libraryId/KeyManager/Mounter.tsx | 163 -- .../app/$libraryId/KeyManager/NotSetup.tsx | 136 -- .../app/$libraryId/KeyManager/NotUnlocked.tsx | 57 - interface/app/$libraryId/KeyManager/index.tsx | 26 - interface/app/$libraryId/Layout/CMDK/CMDK.css | 676 ------- .../app/$libraryId/Layout/CMDK/CMDK.scss | 68 - .../app/$libraryId/Layout/CMDK/index.tsx | 279 --- .../Layout/CMDK/pages/CMDKLocations.tsx | 46 - .../$libraryId/Layout/CMDK/pages/CMDKTags.tsx | 32 - interface/app/$libraryId/Layout/Context.tsx | 14 - .../app/$libraryId/Layout/DndContext.tsx | 24 - .../Layout/Sidebar/DebugPopover.tsx | 272 --- .../Sidebar/JobManager/IsRunningJob.tsx | 8 - .../Layout/Sidebar/JobManager/Job.module.scss | 46 - .../Layout/Sidebar/JobManager/Job.tsx | 116 -- .../Sidebar/JobManager/JobContainer.tsx | 169 -- .../Layout/Sidebar/JobManager/JobGroup.tsx | 306 ---- .../Layout/Sidebar/JobManager/index.tsx | 184 -- .../Layout/Sidebar/SidebarLayout/Context.tsx | 18 - .../Sidebar/SidebarLayout/FeedbackPopover.tsx | 130 -- .../Layout/Sidebar/SidebarLayout/Footer.tsx | 91 - .../Layout/Sidebar/SidebarLayout/Icon.tsx | 5 - .../SidebarLayout/JobManagerPopover.tsx | 75 - .../SidebarLayout/LibrariesDropdown.tsx | 74 - .../Layout/Sidebar/SidebarLayout/Link.tsx | 74 - .../Layout/Sidebar/SidebarLayout/Section.tsx | 22 - .../Layout/Sidebar/SidebarLayout/SeeMore.tsx | 29 - .../Sidebar/SidebarLayout/WindowControls.tsx | 26 - .../Layout/Sidebar/SidebarLayout/index.tsx | 276 --- .../app/$libraryId/Layout/Sidebar/helpers.ts | 4 - .../app/$libraryId/Layout/Sidebar/index.tsx | 40 - .../Sidebar/sections/Categories/index.tsx | 69 - .../Layout/Sidebar/sections/Debug/index.tsx | 23 - .../sections/Devices/AddDeviceDialog.tsx | 46 - .../Layout/Sidebar/sections/Devices/index.tsx | 49 - .../Layout/Sidebar/sections/Library/index.tsx | 42 - .../Layout/Sidebar/sections/Local/index.tsx | 218 --- .../sections/Locations/ContextMenu.tsx | 80 - .../Sidebar/sections/Locations/index.tsx | 163 -- .../Layout/Sidebar/sections/Peers/index.tsx | 40 - .../Sidebar/sections/SavedSearches/index.tsx | 106 -- .../Sidebar/sections/Tags/ContextMenu.tsx | 56 - .../Layout/Sidebar/sections/Tags/index.tsx | 70 - .../Layout/Sidebar/sections/Tools/index.tsx | 79 - .../app/$libraryId/Layout/Sidebar/store.ts | 11 - interface/app/$libraryId/Layout/auth.tsx | 97 - interface/app/$libraryId/Layout/index.tsx | 170 -- interface/app/$libraryId/Layout/store.ts | 10 - .../app/$libraryId/PageLayout/Context.tsx | 14 - interface/app/$libraryId/PageLayout/index.tsx | 31 - interface/app/$libraryId/Spacedrop/index.tsx | 190 -- interface/app/$libraryId/Spacedrop/toast.tsx | 118 -- interface/app/$libraryId/TopBar/Context.tsx | 36 - interface/app/$libraryId/TopBar/Layout.tsx | 24 - .../$libraryId/TopBar/NavigationButtons.tsx | 92 - interface/app/$libraryId/TopBar/Portal.tsx | 22 - .../app/$libraryId/TopBar/TopBarButton.tsx | 53 - .../app/$libraryId/TopBar/TopBarMobile.tsx | 86 - .../app/$libraryId/TopBar/TopBarOptions.tsx | 232 --- interface/app/$libraryId/TopBar/index.tsx | 207 --- interface/app/$libraryId/debug/dnd.tsx | 34 - interface/app/$libraryId/ephemeral.tsx | 266 --- interface/app/$libraryId/favorites.tsx | 83 - interface/app/$libraryId/index.tsx | 81 - interface/app/$libraryId/labels.tsx | 75 - interface/app/$libraryId/location/$id.tsx | 269 --- .../$libraryId/location/LocationOptions.tsx | 146 -- interface/app/$libraryId/media.tsx | 7 - interface/app/$libraryId/network.tsx | 334 ---- interface/app/$libraryId/node/$id.tsx | 61 - .../app/$libraryId/overview/FileKindStats.tsx | 320 ---- .../overview/Layout/HorizontalScroll.tsx | 98 - .../$libraryId/overview/Layout/Section.tsx | 41 - .../app/$libraryId/overview/LibraryStats.tsx | 243 --- .../app/$libraryId/overview/LocationCard.tsx | 49 - interface/app/$libraryId/overview/NewCard.tsx | 71 - .../app/$libraryId/overview/StatCard.tsx | 114 -- .../app/$libraryId/overview/StorageBar.tsx | 85 - interface/app/$libraryId/overview/index.tsx | 138 -- interface/app/$libraryId/peer/$id.tsx | 55 - .../app/$libraryId/peer/StarfieldEffect.tsx | 335 ---- interface/app/$libraryId/people.tsx | 7 - interface/app/$libraryId/recents.tsx | 81 - interface/app/$libraryId/saved-search/$id.tsx | 148 -- .../search/Filters/FilterRegistry.tsx | 31 - .../Filters/components/AppliedFilters.tsx | 191 -- .../components/FilterOptionBoolean.tsx | 43 - .../components/FilterOptionDateRange.tsx | 45 - .../Filters/components/FilterOptionList.tsx | 51 - .../Filters/factories/createBooleanFilter.ts | 43 - .../factories/createDateRangeFilter.ts | 93 - .../factories/createInOrNotInFilter.ts | 81 - .../factories/createTextMatchFilter.ts | 59 - .../Filters/hooks/useToggleOptionSelected.tsx | 36 - .../app/$libraryId/search/Filters/index.tsx | 112 -- .../Filters/registry/BooleanFilters.tsx | 31 - .../search/Filters/registry/DateFilters.tsx | 169 -- .../search/Filters/registry/KindFilter.tsx | 47 - .../Filters/registry/LocationFilter.tsx | 45 - .../search/Filters/registry/TagsFilter.tsx | 56 - .../search/Filters/registry/TextFilters.tsx | 33 - .../app/$libraryId/search/Filters/store.ts | 115 -- .../$libraryId/search/Filters/typeGuards.ts | 78 - .../app/$libraryId/search/FiltersOld.tsx | 890 --------- interface/app/$libraryId/search/SearchBar.tsx | 147 -- .../app/$libraryId/search/SearchOptions.tsx | 385 ---- interface/app/$libraryId/search/context.tsx | 22 - interface/app/$libraryId/search/index.tsx | 81 - interface/app/$libraryId/search/store.tsx | 27 - interface/app/$libraryId/search/useSearch.ts | 209 --- .../search/useSearchExplorerQuery.ts | 36 - interface/app/$libraryId/search/util.tsx | 54 - interface/app/$libraryId/settings/Layout.tsx | 56 - .../app/$libraryId/settings/ModalLayout.tsx | 45 - .../$libraryId/settings/OverviewLayout.tsx | 10 - interface/app/$libraryId/settings/Setting.tsx | 51 - interface/app/$libraryId/settings/Sidebar.tsx | 152 -- .../settings/client/SpacedriveAccount.tsx | 31 - .../settings/client/account/Profile.tsx | 204 --- .../settings/client/account/ShowPassword.tsx | 27 - .../client/account/handlers/cookieHandler.ts | 148 -- .../client/account/handlers/windowHandler.ts | 54 - .../settings/client/account/index.tsx | 184 -- .../$libraryId/settings/client/appearance.tsx | 350 ---- .../$libraryId/settings/client/backups.tsx | 88 - .../$libraryId/settings/client/extensions.tsx | 80 - .../$libraryId/settings/client/general.tsx | 292 --- .../app/$libraryId/settings/client/index.ts | 14 - .../settings/client/keybindings.tsx | 233 --- .../settings/client/network/debug.tsx | 36 - .../settings/client/network/index.tsx | 502 ------ .../$libraryId/settings/client/privacy.tsx | 48 - .../app/$libraryId/settings/client/usage.tsx | 149 -- interface/app/$libraryId/settings/index.tsx | 28 - .../$libraryId/settings/library/contacts.tsx | 13 - .../$libraryId/settings/library/general.tsx | 146 -- .../app/$libraryId/settings/library/index.tsx | 19 - .../settings/library/locations/$id.tsx | 250 --- .../library/locations/AddLocationButton.tsx | 100 - .../library/locations/AddLocationDialog.tsx | 276 --- .../library/locations/DeleteDialog.tsx | 38 - .../IndexerRuleEditor/RuleButton.tsx | 79 - .../locations/IndexerRuleEditor/RuleInput.tsx | 161 -- .../locations/IndexerRuleEditor/RulesForm.tsx | 295 --- .../locations/IndexerRuleEditor/index.tsx | 150 -- .../settings/library/locations/ListItem.tsx | 144 -- .../settings/library/locations/PathInput.tsx | 37 - .../settings/library/locations/index.tsx | 51 - .../locations/openDirectoryPickerDialog.ts | 13 - .../settings/library/saved-searches/index.tsx | 133 -- .../$libraryId/settings/library/security.tsx | 12 - .../$libraryId/settings/library/sharing.tsx | 12 - .../settings/library/tags/CreateDialog.tsx | 103 -- .../settings/library/tags/DeleteDialog.tsx | 36 - .../settings/library/tags/EditForm.tsx | 92 - .../settings/library/tags/index.tsx | 82 - .../app/$libraryId/settings/node/index.tsx | 6 - .../settings/node/libraries/CreateDialog.tsx | 73 - .../node/libraries/DeleteDeviceDialog.tsx | 91 - .../settings/node/libraries/DeleteDialog.tsx | 56 - .../settings/node/libraries/DeviceItem.tsx | 68 - .../settings/node/libraries/ListItem.tsx | 92 - .../settings/node/libraries/index.tsx | 55 - .../app/$libraryId/settings/node/p2p.tsx | 39 - .../$libraryId/settings/resources/about.tsx | 91 - .../settings/resources/changelog.tsx | 63 - .../$libraryId/settings/resources/index.tsx | 7 - interface/app/$libraryId/spaces.tsx | 3 - interface/app/$libraryId/tag/$id.tsx | 114 -- interface/app/I18n.ts | 29 - interface/app/RootContext.tsx | 18 - interface/app/demo.react.tsx | 73 - interface/app/demo.solid.tsx | 84 - interface/app/index.tsx | 349 ---- interface/app/onboarding/Layout.tsx | 87 - interface/app/onboarding/Progress.tsx | 52 - interface/app/onboarding/components.tsx | 6 - interface/app/onboarding/context.tsx | 153 -- interface/app/onboarding/creating-library.tsx | 17 - interface/app/onboarding/full-disk.tsx | 65 - interface/app/onboarding/index.tsx | 44 - interface/app/onboarding/joining-library.tsx | 14 - interface/app/onboarding/locations.tsx | 190 -- interface/app/onboarding/login.tsx | 101 -- interface/app/onboarding/new-library.tsx | 74 - interface/app/onboarding/prerelease.tsx | 49 - interface/app/onboarding/privacy.tsx | 54 - interface/app/p2p/index.tsx | 76 - interface/app/route-schemas.ts | 47 - interface/app/style.scss | 425 ----- interface/components/Accordion.tsx | 64 - interface/components/AlertDialog.tsx | 58 - interface/components/AuthCheck.tsx | 10 - interface/components/AuthRequiredOverlay.tsx | 20 - interface/components/Authentication.tsx | 170 -- interface/components/Codeblock.tsx | 22 - interface/components/ColorPicker.tsx | 33 - interface/components/Devtools.tsx | 14 - interface/components/DismissibleNotice.tsx | 76 - interface/components/DragRegion.tsx | 19 - interface/components/Folder.tsx | 45 - interface/components/Icon.tsx | 25 - interface/components/Loader.tsx | 21 - interface/components/Login.tsx | 352 ---- interface/components/LoginButton.tsx | 49 - interface/components/Menu.tsx | 48 - interface/components/MultiCheckbox.tsx | 109 -- interface/components/PDFViewer.tsx | 65 - interface/components/PasswordMeter.tsx | 62 - interface/components/Register.tsx | 222 --- interface/components/RequestAddDialog.tsx | 120 -- interface/components/ShowPassword.tsx | 27 - interface/components/Sparkles.tsx | 100 - interface/components/SubtleButton.tsx | 15 - interface/components/TextViewer/index.tsx | 160 -- interface/components/TextViewer/one-dark.scss | 445 ----- .../components/TextViewer/one-light.scss | 433 ----- interface/components/TextViewer/prism-lazy.ts | 62 - interface/components/TextViewer/prism.tsx | 42 - interface/components/TrafficLights.tsx | 78 - interface/components/TruncatedText.tsx | 21 - interface/components/index.ts | 16 - interface/hooks/index.ts | 36 - interface/hooks/useAccessToken.ts | 8 - interface/hooks/useCallbackToWatchForm.ts | 65 - interface/hooks/useCallbackToWatchResize.ts | 77 - interface/hooks/useClickOutside.ts | 35 - interface/hooks/useCounter.ts | 90 - interface/hooks/useDebouncedForm.ts | 19 - interface/hooks/useDeeplinkEventHandler.ts | 37 - interface/hooks/useDismissibleNoticeStore.tsx | 18 - interface/hooks/useDragAndDropState.ts | 180 -- interface/hooks/useFileDropEventHandler.ts | 44 - interface/hooks/useFocusState.tsx | 17 - interface/hooks/useHomeDir.ts | 14 - interface/hooks/useInputState.tsx | 10 - interface/hooks/useIsDark.ts | 25 - interface/hooks/useIsLocationIndexing.ts | 36 - interface/hooks/useIsTextTruncated.ts | 23 - interface/hooks/useKeyDeleteFile.tsx | 45 - interface/hooks/useKeyMatcher.ts | 86 - interface/hooks/useKeybind.ts | 35 - interface/hooks/useKeybindEventHandler.ts | 73 - interface/hooks/useKeybindFactory.ts | 5 - interface/hooks/useLocale.ts | 22 - interface/hooks/useMouseItemResize.ts | 65 - interface/hooks/useOperatingSystem.ts | 31 - interface/hooks/usePrefersReducedMotion.ts | 37 - interface/hooks/useQuickRescan.ts | 48 - interface/hooks/useRandomInterval.ts | 38 - interface/hooks/useRedirectToNewLocation.ts | 46 - interface/hooks/useRouteTitle.ts | 19 - interface/hooks/useScrolled.tsx | 24 - interface/hooks/useShortcut.ts | 207 --- interface/hooks/useShowControls.ts | 11 - interface/hooks/useTheme.ts | 57 - interface/hooks/useWindowSize.ts | 39 - interface/hooks/useWindowState.tsx | 7 - interface/hooks/useZodParams.ts | 10 - interface/hooks/useZodRouteParams.ts | 29 - interface/hooks/useZodSearchParams.ts | 170 -- interface/index.tsx | 126 -- interface/locales/README.md | 42 - interface/locales/ar/common.json | 774 -------- interface/locales/be/common.json | 848 --------- interface/locales/cs/common.json | 813 --------- interface/locales/de/common.json | 748 -------- interface/locales/en/common.json | 819 --------- interface/locales/es/common.json | 750 -------- interface/locales/fr/common.json | 749 -------- interface/locales/i18nnext.d.ts | 1 - interface/locales/it/common.json | 749 -------- interface/locales/ja/common.json | 744 -------- interface/locales/nl/common.json | 748 -------- interface/locales/ru/common.json | 848 --------- interface/locales/tr/common.json | 748 -------- interface/locales/uk/common.json | 881 --------- interface/locales/zh-CN/common.json | 740 -------- interface/locales/zh-TW/common.json | 740 -------- interface/package.json | 90 - interface/tailwind.config.js | 1 - interface/tsconfig.json | 13 - interface/types/declarations.d.ts | 7 - interface/util/Platform.tsx | 108 -- interface/util/events.ts | 37 - interface/util/hardware.ts | 22 - interface/util/index.tsx | 32 - interface/util/keybinds.ts | 45 - interface/util/pdfViewer.tsx | 24 - interface/util/useTraceUpdate.tsx | 28 - interface/util/uuid.ts | 1 - tsconfig.json | 6 +- 1258 files changed, 3195 insertions(+), 123076 deletions(-) delete mode 100644 .prettierrc.js rename {core-new/.tasks => .tasks}/ACT-000-action-system.md (100%) rename {core-new/.tasks => .tasks}/ACT-001-action-manager-registry.md (100%) rename {core-new/.tasks => .tasks}/AI-000-ai-epic.md (100%) rename {core-new/.tasks => .tasks}/AI-001-ai-agent.md (100%) rename {core-new/.tasks => .tasks}/AI-002-create-finetuning-dataset.md (100%) rename {core-new/.tasks => .tasks}/CLI-000-command-line-interface.md (100%) rename {core-new/.tasks => .tasks}/CLOUD-000-cloud-as-a-peer.md (100%) rename {core-new/.tasks => .tasks}/CLOUD-001-design-cloud-core-infra.md (63%) rename {core-new/.tasks => .tasks}/CLOUD-002-relay-server.md (100%) rename {core-new/.tasks => .tasks}/CLOUD-003-cloud-volume.md (100%) rename {core-new/.tasks => .tasks}/CORE-000-vdfs-core.md (100%) rename {core-new/.tasks => .tasks}/CORE-001-entry-centric-model.md (100%) rename {core-new/.tasks => .tasks}/CORE-002-sdpath-addressing.md (100%) rename {core-new/.tasks => .tasks}/CORE-003-content-identity.md (100%) rename {core-new/.tasks => .tasks}/CORE-004-closure-table.md (100%) rename {core-new/.tasks => .tasks}/CORE-005-file-type-system.md (100%) rename {core-new/.tasks => .tasks}/CORE-006-semantic-tagging-architecture.md (100%) rename {core-new/.tasks => .tasks}/CORE-007-quantum-state-for-on-demand-state-computation.md (100%) rename {core-new/.tasks => .tasks}/CORE-008-virtual-sidecar-system.md (96%) rename {core-new/.tasks => .tasks}/CORE-009-user-managed-collections.md (100%) rename {core-new/.tasks => .tasks}/CORE-010-file-ingestion-workflow.md (100%) rename {core-new/.tasks => .tasks}/DEV-000-development-validation.md (100%) rename {core-new/.tasks => .tasks}/DEV-001-multi-process-test-framework.md (100%) rename {core-new/.tasks => .tasks}/FILE-000-file-operations.md (100%) rename {core-new/.tasks => .tasks}/FILE-001-file-copy-job.md (100%) rename {core-new/.tasks => .tasks}/FILE-002-file-deletion-job.md (100%) rename {core-new/.tasks => .tasks}/INDEX-000-indexing-file-management.md (100%) rename {core-new/.tasks => .tasks}/INDEX-001-location-watcher-service.md (100%) rename {core-new/.tasks => .tasks}/INDEX-002-stale-file-detection-algorithm.md (100%) rename {core-new/.tasks => .tasks}/JOB-000-job-system.md (100%) rename {core-new/.tasks => .tasks}/JOB-001-job-manager.md (100%) rename {core-new/.tasks => .tasks}/JOB-002-job-logging.md (100%) rename {core-new/.tasks => .tasks}/LOC-000-location-operations.md (100%) rename {core-new/.tasks => .tasks}/LOC-001-location-management-actions.md (100%) rename {core-new/.tasks => .tasks}/LOC-005-virtual-locations-via-pure-hierarchical-model.md (100%) rename {core-new/.tasks => .tasks}/LSYNC-000-library-sync.md (100%) rename {core-new/.tasks => .tasks}/LSYNC-001-design-library-sync-protocol.md (100%) rename {core-new/.tasks => .tasks}/LSYNC-002-metadata-sync.md (100%) rename {core-new/.tasks => .tasks}/LSYNC-003-file-op-sync.md (100%) rename {core-new/.tasks => .tasks}/LSYNC-004-sync-relationship-database-schema.md (100%) rename {core-new/.tasks => .tasks}/NET-000-networking.md (100%) rename {core-new/.tasks => .tasks}/NET-001-iroh-p2p-stack.md (100%) rename {core-new/.tasks => .tasks}/NET-002-device-pairing.md (100%) rename {core-new/.tasks => .tasks}/NET-003-spacedrop-protocol.md (100%) rename {core-new/.tasks => .tasks}/PLUG-000-wasm-plugin-system.md (100%) rename {core-new/.tasks => .tasks}/PLUG-001-integrate-wasm-runtime.md (100%) rename {core-new/.tasks => .tasks}/PLUG-002-define-vdfs-plugin-api.md (100%) rename {core-new/.tasks => .tasks}/PLUG-003-develop-twitter-agent-poc.md (100%) rename {core-new/.tasks => .tasks}/RES-000-resource-management.md (100%) rename {core-new/.tasks => .tasks}/RES-001-adaptive-throttling.md (100%) rename {core-new/.tasks => .tasks}/SEARCH-000-temporal-semantic-search.md (100%) rename {core-new/.tasks => .tasks}/SEARCH-001-async-searchjob.md (100%) rename {core-new/.tasks => .tasks}/SEARCH-002-two-stage-fts-semantic-reranking.md (100%) rename {core-new/.tasks => .tasks}/SEARCH-003-unified-vector-repositories.md (100%) rename {core-new/.tasks => .tasks}/SEC-000-security-and-privacy.md (100%) rename {core-new/.tasks => .tasks}/SEC-002-database-encryption.md (100%) rename {core-new/.tasks => .tasks}/SEC-003-cryptographic-audit-log.md (100%) rename {core-new/.tasks => .tasks}/SEC-004-rbac-system.md (100%) rename {core-new/.tasks => .tasks}/SEC-005-secure-credential-vault.md (100%) rename {core-new/.tasks => .tasks}/SEC-006-certificate-pinning.md (100%) rename {core-new/.tasks => .tasks}/SEC-007-per-library-encryption-policies-for-public-sharing.md (100%) rename {core-new/.tasks => .tasks}/VOL-000-volume-operations.md (100%) rename {core-new/.tasks => .tasks}/VOL-001-volume-physicalclass-and-location-logicalclass.md (100%) rename {core-new/.tasks => .tasks}/VOL-002-automatic-volume-classification.md (100%) rename {core-new/.tasks => .tasks}/VOL-003-intelligent-storage-tiering-warning-system.md (100%) rename {core-new/.tasks => .tasks}/VOL-004-remote-volume-indexing-with-opendal.md (100%) rename {core-new/.tasks => .tasks}/VOL-005-treat-connected-iphone-as-a-virtual-volume-for-direct-import.md (100%) rename {core-new/.tasks => .tasks}/VSS-003-reference-sidecars-for-live-photo-support.md (100%) rename {core-new/.tasks => .tasks}/task.schema.json (100%) rename spacedrive-cloud => apps/cloud (100%) delete mode 100755 combine_paths.py delete mode 100644 core-new/Cargo.toml delete mode 100644 core-new/benchmarks/Cargo.toml delete mode 100644 core-new/build.rs delete mode 100644 core-new/examples/indexing_showcase.rs delete mode 100644 core-new/src/context.rs delete mode 100644 core-new/src/lib.rs delete mode 100644 core-new/src/library/config.rs delete mode 100644 core-new/src/library/mod.rs delete mode 100644 core-new/src/location/mod.rs delete mode 100644 core-new/src/volume/error.rs delete mode 100644 core-new/src/volume/mod.rs delete mode 100644 core-new/src/volume/speed.rs delete mode 100644 core-new/src/volume/types.rs rename {core-new => core}/.scripts/combine.sh (100%) rename {core-new => core/.scripts}/test_daemon.sh (100%) rename {core-new => core/.scripts}/update_spacedrive.sh (95%) rename {core-new => core}/AGENTS.md (100%) rename {core-new => core}/Cargo.lock (99%) rename {core-new => core}/README.md (99%) create mode 100644 core/benchmarks/Cargo.toml rename {core-new => core}/benchmarks/recipes/shape_large.yaml (100%) rename {core-new => core}/benchmarks/recipes/shape_medium.yaml (100%) rename {core-new => core}/benchmarks/recipes/shape_small.yaml (100%) rename {core-new => core}/benchmarks/results/shape_large-aggregation-hdd.json (100%) rename {core-new => core}/benchmarks/results/shape_large-aggregation-ssd.json (90%) rename {core-new => core}/benchmarks/results/shape_large-content_identification-hdd.json (100%) rename {core-new => core}/benchmarks/results/shape_large-content_identification-ssd.json (90%) rename {core-new => core}/benchmarks/results/shape_large-indexing_discovery-hdd.json (100%) rename {core-new => core}/benchmarks/results/shape_large-indexing_discovery-ssd.json (90%) rename {core-new => core}/benchmarks/results/shape_medium-aggregation-hdd.json (100%) rename {core-new => core}/benchmarks/results/shape_medium-aggregation-ssd.json (90%) rename {core-new => core}/benchmarks/results/shape_medium-content_identification-hdd.json (100%) rename {core-new => core}/benchmarks/results/shape_medium-content_identification-ssd.json (90%) rename {core-new => core}/benchmarks/results/shape_medium-indexing_discovery-hdd.json (100%) rename {core-new => core}/benchmarks/results/shape_medium-indexing_discovery-ssd.json (90%) rename {core-new => core}/benchmarks/results/shape_small-aggregation-hdd.json (100%) rename {core-new => core}/benchmarks/results/shape_small-aggregation-ssd.json (90%) rename {core-new => core}/benchmarks/results/shape_small-content_identification-hdd.json (100%) rename {core-new => core}/benchmarks/results/shape_small-content_identification-ssd.json (90%) rename {core-new => core}/benchmarks/results/shape_small-indexing_discovery-hdd.json (100%) rename {core-new => core}/benchmarks/results/shape_small-indexing_discovery-ssd.json (90%) rename {core-new => core}/benchmarks/results/whitepaper_metrics.csv (100%) rename {core-new => core}/benchmarks/src/bin/sd-bench-new.rs (100%) rename {core-new => core}/benchmarks/src/cli/args.rs (100%) rename {core-new => core}/benchmarks/src/cli/commands.rs (100%) rename {core-new => core}/benchmarks/src/cli/mod.rs (100%) rename {core-new => core}/benchmarks/src/config/mod.rs (100%) rename {core-new => core}/benchmarks/src/core_boot/mod.rs (100%) rename {core-new => core}/benchmarks/src/generator/filesystem.rs (100%) rename {core-new => core}/benchmarks/src/generator/mod.rs (100%) rename {core-new => core}/benchmarks/src/generator/noop.rs (100%) rename {core-new => core}/benchmarks/src/generator/registry.rs (100%) rename {core-new => core}/benchmarks/src/lib.rs (100%) rename {core-new => core}/benchmarks/src/metrics/mod.rs (100%) rename {core-new => core}/benchmarks/src/metrics/sources.rs (100%) rename {core-new => core}/benchmarks/src/recipe/mod.rs (100%) rename {core-new => core}/benchmarks/src/recipe/schema.rs (100%) rename {core-new => core}/benchmarks/src/reporting/csv.rs (100%) rename {core-new => core}/benchmarks/src/reporting/json_summary.rs (100%) rename {core-new => core}/benchmarks/src/reporting/mod.rs (100%) rename {core-new => core}/benchmarks/src/reporting/registry.rs (100%) rename {core-new => core}/benchmarks/src/runner/mod.rs (100%) rename {core-new => core}/benchmarks/src/runner/monitor.rs (100%) rename {core-new => core}/benchmarks/src/scenarios/common.rs (100%) rename {core-new => core}/benchmarks/src/scenarios/content_identification.rs (100%) rename {core-new => core}/benchmarks/src/scenarios/core_indexing.rs (100%) rename {core-new => core}/benchmarks/src/scenarios/mod.rs (100%) rename {core-new => core}/benchmarks/src/scenarios/registry.rs (100%) rename {core-new => core}/benchmarks/src/util/fs.rs (100%) rename {core-new => core}/benchmarks/src/util/mod.rs (100%) rename {core-new => core}/benchmarks/src/util/rng.rs (100%) rename {core-new => core}/benchmarks/src/util/time.rs (100%) delete mode 100644 core/crates/cloud-services/Cargo.toml delete mode 100644 core/crates/cloud-services/src/client.rs delete mode 100644 core/crates/cloud-services/src/error.rs delete mode 100644 core/crates/cloud-services/src/key_manager/key_store.rs delete mode 100644 core/crates/cloud-services/src/key_manager/mod.rs delete mode 100644 core/crates/cloud-services/src/lib.rs delete mode 100644 core/crates/cloud-services/src/p2p/mod.rs delete mode 100644 core/crates/cloud-services/src/p2p/new_sync_messages_notifier.rs delete mode 100644 core/crates/cloud-services/src/p2p/runner.rs delete mode 100644 core/crates/cloud-services/src/sync/ingest.rs delete mode 100644 core/crates/cloud-services/src/sync/mod.rs delete mode 100644 core/crates/cloud-services/src/sync/receive.rs delete mode 100644 core/crates/cloud-services/src/sync/send.rs delete mode 100644 core/crates/cloud-services/src/token_refresher.rs delete mode 100644 core/crates/file-path-helper/Cargo.toml delete mode 100644 core/crates/file-path-helper/README.md delete mode 100644 core/crates/file-path-helper/src/isolated_file_path_data.rs delete mode 100644 core/crates/file-path-helper/src/lib.rs delete mode 100644 core/crates/heavy-lifting/Cargo.toml delete mode 100644 core/crates/heavy-lifting/src/file_identifier/cas_id.rs delete mode 100644 core/crates/heavy-lifting/src/file_identifier/job.rs delete mode 100644 core/crates/heavy-lifting/src/file_identifier/mod.rs delete mode 100644 core/crates/heavy-lifting/src/file_identifier/shallow.rs delete mode 100644 core/crates/heavy-lifting/src/file_identifier/tasks/identifier.rs delete mode 100644 core/crates/heavy-lifting/src/file_identifier/tasks/mod.rs delete mode 100644 core/crates/heavy-lifting/src/file_identifier/tasks/object_processor.rs delete mode 100644 core/crates/heavy-lifting/src/indexer/job.rs delete mode 100644 core/crates/heavy-lifting/src/indexer/mod.rs delete mode 100644 core/crates/heavy-lifting/src/indexer/shallow.rs delete mode 100644 core/crates/heavy-lifting/src/indexer/tasks/mod.rs delete mode 100644 core/crates/heavy-lifting/src/indexer/tasks/saver.rs delete mode 100644 core/crates/heavy-lifting/src/indexer/tasks/updater.rs delete mode 100644 core/crates/heavy-lifting/src/indexer/tasks/walker/entry.rs delete mode 100644 core/crates/heavy-lifting/src/indexer/tasks/walker/metadata.rs delete mode 100644 core/crates/heavy-lifting/src/indexer/tasks/walker/mod.rs delete mode 100644 core/crates/heavy-lifting/src/indexer/tasks/walker/rules.rs delete mode 100644 core/crates/heavy-lifting/src/indexer/tasks/walker/save_state.rs delete mode 100644 core/crates/heavy-lifting/src/job_system/error.rs delete mode 100644 core/crates/heavy-lifting/src/job_system/job.rs delete mode 100644 core/crates/heavy-lifting/src/job_system/mod.rs delete mode 100644 core/crates/heavy-lifting/src/job_system/report.rs delete mode 100644 core/crates/heavy-lifting/src/job_system/runner.rs delete mode 100644 core/crates/heavy-lifting/src/job_system/store.rs delete mode 100644 core/crates/heavy-lifting/src/job_system/utils.rs delete mode 100644 core/crates/heavy-lifting/src/lib.rs delete mode 100644 core/crates/heavy-lifting/src/media_processor/helpers/exif_media_data.rs delete mode 100644 core/crates/heavy-lifting/src/media_processor/helpers/ffmpeg_media_data.rs delete mode 100644 core/crates/heavy-lifting/src/media_processor/helpers/mod.rs delete mode 100644 core/crates/heavy-lifting/src/media_processor/helpers/thumbnailer.rs delete mode 100644 core/crates/heavy-lifting/src/media_processor/job.rs delete mode 100644 core/crates/heavy-lifting/src/media_processor/mod.rs delete mode 100644 core/crates/heavy-lifting/src/media_processor/shallow.rs delete mode 100644 core/crates/heavy-lifting/src/media_processor/tasks/media_data_extractor.rs delete mode 100644 core/crates/heavy-lifting/src/media_processor/tasks/mod.rs delete mode 100644 core/crates/heavy-lifting/src/media_processor/tasks/thumbnailer.rs delete mode 100644 core/crates/heavy-lifting/src/utils/mod.rs delete mode 100644 core/crates/heavy-lifting/src/utils/sub_path.rs delete mode 100644 core/crates/indexer-rules/Cargo.toml delete mode 100644 core/crates/indexer-rules/src/lib.rs delete mode 100644 core/crates/indexer-rules/src/seed.rs delete mode 100644 core/crates/indexer-rules/src/serde_impl.rs delete mode 100644 core/crates/p2p/Cargo.toml delete mode 100644 core/crates/p2p/src/error.rs delete mode 100644 core/crates/p2p/src/lib.rs delete mode 100644 core/crates/p2p/src/schema/cloud_services/authorize_new_device_in_sync_group.rs delete mode 100644 core/crates/p2p/src/schema/cloud_services/mod.rs delete mode 100644 core/crates/p2p/src/schema/cloud_services/notify_new_sync_messages.rs delete mode 100644 core/crates/p2p/src/schema/mod.rs delete mode 100644 core/crates/p2p/src/server/mod.rs delete mode 100644 core/crates/p2p/src/server/router/cloud_services.rs delete mode 100644 core/crates/p2p/src/server/router/mod.rs delete mode 100644 core/crates/prisma-helpers/Cargo.toml delete mode 100644 core/crates/prisma-helpers/src/lib.rs rename {core-new => core/crates}/spacedrive-jobs-derive/Cargo.toml (100%) rename {core-new => core/crates}/spacedrive-jobs-derive/src/lib.rs (100%) delete mode 100644 core/crates/sync/Cargo.toml delete mode 100644 core/crates/sync/README.md delete mode 100644 core/crates/sync/src/backfill.rs delete mode 100644 core/crates/sync/src/db_operation.rs delete mode 100644 core/crates/sync/src/ingest_utils.rs delete mode 100644 core/crates/sync/src/lib.rs delete mode 100644 core/crates/sync/src/manager.rs rename {core-new => core}/crush.json (100%) rename {core-new => core}/examples/file_type_demo.rs (100%) rename {core-new => core}/examples/indexing_demo.rs (100%) create mode 100644 core/examples/indexing_showcase.rs rename {core-new => core}/examples/job_logging_test.rs (100%) rename {core-new => core}/examples/library_demo.rs (100%) rename {core-new => core}/examples/location_watcher_demo.rs (100%) rename {core-new => core}/examples/pause_resume_demo.rs (100%) rename {core-new => core}/examples/shutdown_demo.rs (100%) rename {core-new => core}/examples/simple_pause_resume.rs (100%) rename {core-new => core}/examples/test_migration.rs (100%) rename {core-new => core}/examples/volume_demo.rs (100%) delete mode 100644 core/prisma/migrations/20230616064440_init/migration.sql delete mode 100644 core/prisma/migrations/20230618234117_remove_owned_operations/migration.sql delete mode 100644 core/prisma/migrations/20230619032753_p2p/migration.sql delete mode 100644 core/prisma/migrations/20230619144125_collate_nocase/migration.sql delete mode 100644 core/prisma/migrations/20230621173906_size_in_bytes_bytes/migration.sql delete mode 100644 core/prisma/migrations/20230711114013_preferences/migration.sql delete mode 100644 core/prisma/migrations/20230712050046_library_instance/migration.sql delete mode 100644 core/prisma/migrations/20230712063345_notifications/migration.sql delete mode 100644 core/prisma/migrations/20230724131659_relation_operation/migration.sql delete mode 100644 core/prisma/migrations/20230812141757_added_albums/migration.sql delete mode 100644 core/prisma/migrations/20230828195811_media_data/migration.sql delete mode 100644 core/prisma/migrations/20230912065026_file_path_hidden/migration.sql delete mode 100644 core/prisma/migrations/20230921203938_add_size_bytes_in_location_table/migration.sql delete mode 100644 core/prisma/migrations/20230926180932_remove_device_from_file_path_table/migration.sql delete mode 100644 core/prisma/migrations/20231005202254_sort_by_image_taken/migration.sql delete mode 100644 core/prisma/migrations/20231022013532_saved_searches/migration.sql delete mode 100644 core/prisma/migrations/20231113140411_tag_date_created/migration.sql delete mode 100644 core/prisma/migrations/20231121173834_filters_string/migration.sql delete mode 100644 core/prisma/migrations/20231204174640_update_tags_and_labels/migration.sql delete mode 100644 core/prisma/migrations/20231219105608_crdt_operation/migration.sql delete mode 100644 core/prisma/migrations/20240221044741_drop_node_peer_id/migration.sql delete mode 100644 core/prisma/migrations/20240314064009_cache_instance_metadata/migration.sql delete mode 100644 core/prisma/migrations/20240319065147_remove_remoteidentity/migration.sql delete mode 100644 core/prisma/migrations/20240405185748_add_target_to_saved_search/migration.sql delete mode 100644 core/prisma/migrations/20240408081222_numeric_sync_model_id/migration.sql delete mode 100644 core/prisma/migrations/20240408215355_add_location_scan_state/migration.sql delete mode 100644 core/prisma/migrations/20240409202941_rename_media_data_to_exif_data/migration.sql delete mode 100644 core/prisma/migrations/20240508074615_ffmpeg_media_data/migration.sql delete mode 100644 core/prisma/migrations/20240512011221_better_statistics/migration.sql delete mode 100644 core/prisma/migrations/20240517045456_change_indexer_rule_name_to_unique/migration.sql delete mode 100644 core/prisma/migrations/20240531034707_add_node_remote_identity_to_instance/migration.sql delete mode 100644 core/prisma/migrations/20240705205916_create_object_kind_statistics_table/migration.sql delete mode 100644 core/prisma/migrations/20240710015044_add_info_colum_to_job/migration.sql delete mode 100644 core/prisma/migrations/20240712155557_create_storage_statistics_table/migration.sql delete mode 100644 core/prisma/migrations/20250130041311_adding_devices/migration.sql delete mode 100644 core/prisma/migrations/migration_lock.toml delete mode 100644 core/prisma/schema.prisma rename {core-new => core}/rust-toolchain.toml (100%) delete mode 100644 core/src/api/backups.rs delete mode 100644 core/src/api/cloud/devices.rs delete mode 100644 core/src/api/cloud/libraries.rs delete mode 100644 core/src/api/cloud/locations.rs delete mode 100644 core/src/api/cloud/mod.rs delete mode 100644 core/src/api/cloud/sync_groups.rs delete mode 100644 core/src/api/cloud/thumbnails.rs delete mode 100644 core/src/api/devices.rs delete mode 100644 core/src/api/ephemeral_files.rs delete mode 100644 core/src/api/files.rs delete mode 100644 core/src/api/jobs.rs delete mode 100644 core/src/api/keys.rs delete mode 100644 core/src/api/labels.rs delete mode 100644 core/src/api/libraries.rs delete mode 100644 core/src/api/locations.rs delete mode 100644 core/src/api/mod.rs delete mode 100644 core/src/api/models.rs delete mode 100644 core/src/api/nodes.rs delete mode 100644 core/src/api/notifications.rs delete mode 100644 core/src/api/p2p.rs delete mode 100644 core/src/api/preferences.rs delete mode 100644 core/src/api/search/exif_data.rs delete mode 100644 core/src/api/search/file_path.rs delete mode 100644 core/src/api/search/mod.rs delete mode 100644 core/src/api/search/object.rs delete mode 100644 core/src/api/search/saved.rs delete mode 100644 core/src/api/search/utils.rs delete mode 100644 core/src/api/sync.rs delete mode 100644 core/src/api/tags.rs delete mode 100644 core/src/api/utils/invalidate.rs delete mode 100644 core/src/api/utils/library.rs delete mode 100644 core/src/api/utils/mod.rs delete mode 100644 core/src/api/volumes.rs delete mode 100644 core/src/api/web_api.rs rename {core-new => core}/src/bin/cli.rs (100%) rename {core-new => core}/src/config/app_config.rs (100%) rename {core-new => core}/src/config/migration.rs (100%) rename {core-new => core}/src/config/mod.rs (100%) delete mode 100644 core/src/custom_uri/mod.rs delete mode 100644 core/src/custom_uri/mpsc_to_async_write.rs delete mode 100644 core/src/custom_uri/serve_file.rs delete mode 100644 core/src/custom_uri/utils.rs rename {core-new => core}/src/device/config.rs (100%) rename {core-new => core}/src/device/manager.rs (100%) rename {core-new => core}/src/device/mod.rs (100%) rename {core-new => core}/src/domain/addressing.rs (100%) rename {core-new => core}/src/domain/content_identity.rs (100%) rename {core-new => core}/src/domain/device.rs (100%) rename {core-new => core}/src/domain/entry.rs (100%) rename {core-new => core}/src/domain/location.rs (100%) rename {core-new => core}/src/domain/mod.rs (100%) rename {core-new => core}/src/domain/user_metadata.rs (100%) rename {core-new => core}/src/domain/volume.rs (100%) rename {core-new => core}/src/file_type/builtin.rs (100%) rename {core-new => core}/src/file_type/definitions/archives.toml (100%) rename {core-new => core}/src/file_type/definitions/audio.toml (100%) rename {core-new => core}/src/file_type/definitions/code.toml (100%) rename {core-new => core}/src/file_type/definitions/documents.toml (100%) rename {core-new => core}/src/file_type/definitions/images.toml (100%) rename {core-new => core}/src/file_type/definitions/misc.toml (100%) rename {core-new => core}/src/file_type/definitions/video.toml (100%) rename {core-new => core}/src/file_type/magic.rs (100%) rename {core-new => core}/src/file_type/mod.rs (100%) rename {core-new => core}/src/file_type/registry.rs (100%) rename {core-new => core}/src/infrastructure/actions/BUILDER_REFACTOR_PLAN.md (100%) rename {core-new => core}/src/infrastructure/actions/builder.rs (100%) rename {core-new => core}/src/infrastructure/actions/error.rs (100%) rename {core-new => core}/src/infrastructure/actions/handler.rs (100%) rename {core-new => core}/src/infrastructure/actions/manager.rs (100%) rename {core-new => core}/src/infrastructure/actions/mod.rs (100%) rename {core-new => core}/src/infrastructure/actions/output.rs (100%) rename {core-new => core}/src/infrastructure/actions/receipt.rs (100%) rename {core-new => core}/src/infrastructure/actions/registry.rs (100%) rename {core-new => core}/src/infrastructure/actions/tests.rs (100%) rename {core-new => core}/src/infrastructure/cli/README.md (100%) rename {core-new => core}/src/infrastructure/cli/adapters/copy.rs (100%) rename {core-new => core}/src/infrastructure/cli/adapters/mod.rs (100%) rename {core-new => core}/src/infrastructure/cli/commands/daemon.rs (100%) rename {core-new => core}/src/infrastructure/cli/commands/file.rs (100%) rename {core-new => core}/src/infrastructure/cli/commands/job.rs (100%) rename {core-new => core}/src/infrastructure/cli/commands/library.rs (100%) rename {core-new => core}/src/infrastructure/cli/commands/location.rs (100%) rename {core-new => core}/src/infrastructure/cli/commands/mod.rs (100%) rename {core-new => core}/src/infrastructure/cli/commands/network.rs (100%) rename {core-new => core}/src/infrastructure/cli/commands/system.rs (100%) rename {core-new => core}/src/infrastructure/cli/commands/volume.rs (100%) rename {core-new => core}/src/infrastructure/cli/daemon/client.rs (100%) rename {core-new => core}/src/infrastructure/cli/daemon/config.rs (100%) rename {core-new => core}/src/infrastructure/cli/daemon/handlers/core.rs (100%) rename {core-new => core}/src/infrastructure/cli/daemon/handlers/file.rs (100%) rename {core-new => core}/src/infrastructure/cli/daemon/handlers/job.rs (100%) rename {core-new => core}/src/infrastructure/cli/daemon/handlers/library.rs (100%) rename {core-new => core}/src/infrastructure/cli/daemon/handlers/location.rs (100%) rename {core-new => core}/src/infrastructure/cli/daemon/handlers/mod.rs (100%) rename {core-new => core}/src/infrastructure/cli/daemon/handlers/network.rs (100%) rename {core-new => core}/src/infrastructure/cli/daemon/handlers/system.rs (100%) rename {core-new => core}/src/infrastructure/cli/daemon/handlers/volume.rs (100%) rename {core-new => core}/src/infrastructure/cli/daemon/mod.rs (100%) rename {core-new => core}/src/infrastructure/cli/daemon/services/helpers.rs (100%) rename {core-new => core}/src/infrastructure/cli/daemon/services/mod.rs (100%) rename {core-new => core}/src/infrastructure/cli/daemon/services/state.rs (100%) rename {core-new => core}/src/infrastructure/cli/daemon/types/commands.rs (100%) rename {core-new => core}/src/infrastructure/cli/daemon/types/common.rs (100%) rename {core-new => core}/src/infrastructure/cli/daemon/types/mod.rs (100%) rename {core-new => core}/src/infrastructure/cli/daemon/types/responses.rs (100%) rename {core-new => core}/src/infrastructure/cli/mod.rs (100%) rename {core-new => core}/src/infrastructure/cli/output/context.rs (100%) rename {core-new => core}/src/infrastructure/cli/output/formatters.rs (100%) rename {core-new => core}/src/infrastructure/cli/output/messages.rs (100%) rename {core-new => core}/src/infrastructure/cli/output/mod.rs (100%) rename {core-new => core}/src/infrastructure/cli/output/section.rs (100%) rename {core-new => core}/src/infrastructure/cli/output/tests.rs (100%) rename {core-new => core}/src/infrastructure/cli/pairing_ui.rs (100%) rename {core-new => core}/src/infrastructure/cli/state.rs (100%) rename {core-new => core}/src/infrastructure/cli/tui.rs (100%) rename {core-new => core}/src/infrastructure/cli/utils.rs (100%) rename {core-new => core}/src/infrastructure/database/entities/audit_log.rs (100%) rename {core-new => core}/src/infrastructure/database/entities/collection.rs (100%) rename {core-new => core}/src/infrastructure/database/entities/collection_entry.rs (100%) rename {core-new => core}/src/infrastructure/database/entities/content_identity.rs (100%) rename {core-new => core}/src/infrastructure/database/entities/content_kind.rs (100%) rename {core-new => core}/src/infrastructure/database/entities/device.rs (100%) rename {core-new => core}/src/infrastructure/database/entities/directory_paths.rs (100%) rename {core-new => core}/src/infrastructure/database/entities/entry.rs (100%) rename {core-new => core}/src/infrastructure/database/entities/entry_closure.rs (100%) rename {core-new => core}/src/infrastructure/database/entities/indexer_rule.rs (100%) rename {core-new => core}/src/infrastructure/database/entities/label.rs (100%) rename {core-new => core}/src/infrastructure/database/entities/location.rs (100%) rename {core-new => core}/src/infrastructure/database/entities/metadata_label.rs (100%) rename {core-new => core}/src/infrastructure/database/entities/metadata_tag.rs (100%) rename {core-new => core}/src/infrastructure/database/entities/mime_type.rs (100%) rename {core-new => core}/src/infrastructure/database/entities/mod.rs (100%) rename {core-new => core}/src/infrastructure/database/entities/sidecar.rs (100%) rename {core-new => core}/src/infrastructure/database/entities/sidecar_availability.rs (100%) rename {core-new => core}/src/infrastructure/database/entities/tag.rs (100%) rename {core-new => core}/src/infrastructure/database/entities/user_metadata.rs (100%) rename {core-new => core}/src/infrastructure/database/entities/volume.rs (100%) rename {core-new => core}/src/infrastructure/database/migration/m20240101_000001_initial_schema.rs (100%) rename {core-new => core}/src/infrastructure/database/migration/m20240102_000001_populate_lookups.rs (100%) rename {core-new => core}/src/infrastructure/database/migration/m20240107_000001_create_collections.rs (100%) rename {core-new => core}/src/infrastructure/database/migration/m20250109_000001_create_sidecars.rs (100%) rename {core-new => core}/src/infrastructure/database/migration/m20250110_000001_refactor_volumes_table.rs (100%) rename {core-new => core}/src/infrastructure/database/migration/m20250112_000001_create_indexer_rules.rs (100%) rename {core-new => core}/src/infrastructure/database/migration/mod.rs (100%) rename {core-new => core}/src/infrastructure/database/mod.rs (100%) rename {core-new => core}/src/infrastructure/events/mod.rs (100%) rename {core-new => core}/src/infrastructure/jobs/context.rs (100%) rename {core-new => core}/src/infrastructure/jobs/database.rs (100%) rename {core-new => core}/src/infrastructure/jobs/error.rs (100%) rename {core-new => core}/src/infrastructure/jobs/executor.rs (100%) rename {core-new => core}/src/infrastructure/jobs/generic_progress.rs (100%) rename {core-new => core}/src/infrastructure/jobs/handle.rs (100%) rename {core-new => core}/src/infrastructure/jobs/logger.rs (100%) rename {core-new => core}/src/infrastructure/jobs/manager.rs (100%) rename {core-new => core}/src/infrastructure/jobs/mod.rs (100%) rename {core-new => core}/src/infrastructure/jobs/output.rs (100%) rename {core-new => core}/src/infrastructure/jobs/progress.rs (100%) rename {core-new => core}/src/infrastructure/jobs/registry.rs (100%) rename {core-new => core}/src/infrastructure/jobs/traits.rs (100%) rename {core-new => core}/src/infrastructure/jobs/types.rs (100%) rename {core-new => core}/src/infrastructure/mod.rs (100%) rename {core-new => core}/src/keys/device_key_manager.rs (100%) rename {core-new => core}/src/keys/library_key_manager.rs (100%) rename {core-new => core}/src/keys/mod.rs (100%) rename {core-new => core}/src/library/error.rs (100%) delete mode 100644 core/src/library/library.rs rename {core-new => core}/src/library/lock.rs (100%) rename {core-new => core}/src/library/manager.rs (100%) delete mode 100644 core/src/library/manager/error.rs delete mode 100644 core/src/library/manager/mod.rs delete mode 100644 core/src/library/manager/pragmas.rs delete mode 100644 core/src/library/name.rs delete mode 100644 core/src/library/statistics.rs delete mode 100644 core/src/location/archive/archive_job.rs delete mode 100644 core/src/location/archive/mod.rs delete mode 100644 core/src/location/error.rs rename {core-new => core}/src/location/manager.rs (100%) delete mode 100644 core/src/location/manager/mod.rs delete mode 100644 core/src/location/manager/runner.rs delete mode 100644 core/src/location/manager/watcher/android.rs delete mode 100644 core/src/location/manager/watcher/ios.rs delete mode 100644 core/src/location/manager/watcher/linux.rs delete mode 100644 core/src/location/manager/watcher/macos.rs delete mode 100644 core/src/location/manager/watcher/mod.rs delete mode 100644 core/src/location/manager/watcher/utils.rs delete mode 100644 core/src/location/manager/watcher/windows.rs delete mode 100644 core/src/location/metadata.rs delete mode 100644 core/src/location/non_indexed.rs delete mode 100644 core/src/node/config.rs delete mode 100644 core/src/node/hardware.rs delete mode 100644 core/src/node/mod.rs delete mode 100644 core/src/node/platform.rs delete mode 100644 core/src/notifications.rs delete mode 100644 core/src/object/fs/archive.rs delete mode 100644 core/src/object/fs/convert.rs delete mode 100644 core/src/object/fs/error.rs delete mode 100644 core/src/object/fs/mod.rs delete mode 100644 core/src/object/fs/old_copy.rs delete mode 100644 core/src/object/fs/old_cut.rs delete mode 100644 core/src/object/fs/old_delete.rs delete mode 100644 core/src/object/fs/old_erase.rs delete mode 100644 core/src/object/fs/sync.rs delete mode 100644 core/src/object/media/old_thumbnail/clean_up.rs delete mode 100644 core/src/object/media/old_thumbnail/directory.rs delete mode 100644 core/src/object/mod.rs delete mode 100644 core/src/object/old_orphan_remover.rs delete mode 100644 core/src/object/tag/mod.rs delete mode 100644 core/src/object/tag/seed.rs delete mode 100644 core/src/object/validation/hash.rs delete mode 100644 core/src/object/validation/mod.rs delete mode 100644 core/src/object/validation/old_validator_job.rs delete mode 100644 core/src/old_job/error.rs delete mode 100644 core/src/old_job/manager.rs delete mode 100644 core/src/old_job/mod.rs delete mode 100644 core/src/old_job/report.rs delete mode 100644 core/src/old_job/worker.rs delete mode 100644 core/src/old_p2p/events.rs delete mode 100644 core/src/old_p2p/libraries.rs delete mode 100644 core/src/old_p2p/manager.rs delete mode 100644 core/src/old_p2p/metadata.rs delete mode 100644 core/src/old_p2p/mod.rs delete mode 100644 core/src/old_p2p/operations/library.rs delete mode 100644 core/src/old_p2p/operations/mod.rs delete mode 100644 core/src/old_p2p/operations/ping.rs delete mode 100644 core/src/old_p2p/operations/rspc.rs delete mode 100644 core/src/old_p2p/operations/spacedrop.rs delete mode 100644 core/src/old_p2p/protocol.rs delete mode 100644 core/src/old_p2p/sync/mod.rs delete mode 100644 core/src/old_p2p/sync/proto.rs rename {core-new => core}/src/operations/addressing.rs (100%) rename {core-new => core}/src/operations/content/action.rs (100%) rename {core-new => core}/src/operations/content/mod.rs (100%) rename {core-new => core}/src/operations/devices/mod.rs (100%) rename {core-new => core}/src/operations/devices/revoke/action.rs (100%) rename {core-new => core}/src/operations/devices/revoke/mod.rs (100%) rename {core-new => core}/src/operations/devices/revoke/output.rs (100%) rename {core-new => core}/src/operations/entries/mod.rs (100%) rename {core-new => core}/src/operations/entries/state.rs (100%) rename {core-new => core}/src/operations/files/copy/action.rs (100%) rename {core-new => core}/src/operations/files/copy/database.rs (100%) rename {core-new => core}/src/operations/files/copy/docs/ANALYSIS.md (100%) rename {core-new => core}/src/operations/files/copy/docs/FILE_SYNC_OVERLAP.md (100%) rename {core-new => core}/src/operations/files/copy/docs/PROGRESSIVE_COPY_DESIGN.md (100%) rename {core-new => core}/src/operations/files/copy/docs/RESUME_VALIDATION_DESIGN.md (100%) rename {core-new => core}/src/operations/files/copy/input.rs (100%) rename {core-new => core}/src/operations/files/copy/job.rs (100%) rename {core-new => core}/src/operations/files/copy/mod.rs (100%) rename {core-new => core}/src/operations/files/copy/output.rs (100%) rename {core-new => core}/src/operations/files/copy/routing.rs (100%) rename {core-new => core}/src/operations/files/copy/strategy.rs (100%) rename {core-new => core}/src/operations/files/delete/action.rs (100%) rename {core-new => core}/src/operations/files/delete/job.rs (100%) rename {core-new => core}/src/operations/files/delete/mod.rs (100%) rename {core-new => core}/src/operations/files/delete/output.rs (100%) rename {core-new => core}/src/operations/files/duplicate_detection/action.rs (100%) rename {core-new => core}/src/operations/files/duplicate_detection/job.rs (100%) rename {core-new => core}/src/operations/files/duplicate_detection/mod.rs (100%) rename {core-new => core}/src/operations/files/mod.rs (100%) rename {core-new => core}/src/operations/files/validation/action.rs (100%) rename {core-new => core}/src/operations/files/validation/job.rs (100%) rename {core-new => core}/src/operations/files/validation/mod.rs (100%) rename {core-new => core}/src/operations/indexing/action.rs (100%) rename {core-new => core}/src/operations/indexing/change_detection/mod.rs (100%) rename {core-new => core}/src/operations/indexing/entry.rs (100%) rename {core-new => core}/src/operations/indexing/hierarchy.rs (100%) rename {core-new => core}/src/operations/indexing/job.rs (100%) rename {core-new => core}/src/operations/indexing/metrics.rs (100%) rename {core-new => core}/src/operations/indexing/mod.rs (100%) rename {core-new => core}/src/operations/indexing/path_resolver.rs (100%) rename {core-new => core}/src/operations/indexing/persistence.rs (100%) rename {core-new => core}/src/operations/indexing/phases/aggregation.rs (100%) rename {core-new => core}/src/operations/indexing/phases/content.rs (100%) rename {core-new => core}/src/operations/indexing/phases/discovery.rs (100%) rename {core-new => core}/src/operations/indexing/phases/mod.rs (100%) rename {core-new => core}/src/operations/indexing/phases/processing.rs (100%) rename {core-new => core}/src/operations/indexing/progress.rs (100%) rename {core-new => core}/src/operations/indexing/rules.rs (100%) rename {core-new => core}/src/operations/indexing/state.rs (100%) rename {core-new => core}/src/operations/indexing/tests/mod.rs (100%) rename {core-new => core}/src/operations/libraries/create/action.rs (100%) rename {core-new => core}/src/operations/libraries/create/mod.rs (100%) rename {core-new => core}/src/operations/libraries/create/output.rs (100%) rename {core-new => core}/src/operations/libraries/delete/action.rs (100%) rename {core-new => core}/src/operations/libraries/delete/mod.rs (100%) rename {core-new => core}/src/operations/libraries/delete/output.rs (100%) rename {core-new => core}/src/operations/libraries/export/action.rs (100%) rename {core-new => core}/src/operations/libraries/export/mod.rs (100%) rename {core-new => core}/src/operations/libraries/export/output.rs (100%) rename {core-new => core}/src/operations/libraries/mod.rs (100%) rename {core-new => core}/src/operations/libraries/rename/action.rs (100%) rename {core-new => core}/src/operations/libraries/rename/mod.rs (100%) rename {core-new => core}/src/operations/libraries/rename/output.rs (100%) rename {core-new => core}/src/operations/locations/add/action.rs (100%) rename {core-new => core}/src/operations/locations/add/mod.rs (100%) rename {core-new => core}/src/operations/locations/add/output.rs (100%) rename {core-new => core}/src/operations/locations/index/action.rs (100%) rename {core-new => core}/src/operations/locations/index/mod.rs (100%) rename {core-new => core}/src/operations/locations/mod.rs (100%) rename {core-new => core}/src/operations/locations/remove/action.rs (100%) rename {core-new => core}/src/operations/locations/remove/mod.rs (100%) rename {core-new => core}/src/operations/locations/remove/output.rs (100%) rename {core-new => core}/src/operations/locations/rescan/action.rs (100%) rename {core-new => core}/src/operations/locations/rescan/mod.rs (100%) rename {core-new => core}/src/operations/locations/rescan/output.rs (100%) rename {core-new => core}/src/operations/media/live_photo.rs (100%) rename {core-new => core}/src/operations/media/live_photo_query.rs (100%) rename {core-new => core}/src/operations/media/mod.rs (100%) rename {core-new => core}/src/operations/media/thumbnail/action.rs (100%) rename {core-new => core}/src/operations/media/thumbnail/error.rs (100%) rename {core-new => core}/src/operations/media/thumbnail/generator.rs (100%) rename {core-new => core}/src/operations/media/thumbnail/job.rs (100%) rename {core-new => core}/src/operations/media/thumbnail/mod.rs (100%) rename {core-new => core}/src/operations/media/thumbnail/state.rs (100%) rename {core-new => core}/src/operations/media/thumbnail/utils.rs (100%) rename {core-new => core}/src/operations/metadata/action.rs (100%) rename {core-new => core}/src/operations/metadata/mod.rs (100%) rename {core-new => core}/src/operations/mod.rs (100%) rename {core-new => core}/src/operations/sidecar/mod.rs (100%) rename {core-new => core}/src/operations/sidecar/path.rs (100%) rename {core-new => core}/src/operations/sidecar/types.rs (100%) rename {core-new => core}/src/operations/volumes/mod.rs (100%) rename {core-new => core}/src/operations/volumes/speed_test/action.rs (100%) rename {core-new => core}/src/operations/volumes/speed_test/handler.rs (100%) rename {core-new => core}/src/operations/volumes/speed_test/mod.rs (100%) rename {core-new => core}/src/operations/volumes/track/action.rs (100%) rename {core-new => core}/src/operations/volumes/track/handler.rs (100%) rename {core-new => core}/src/operations/volumes/track/mod.rs (100%) rename {core-new => core}/src/operations/volumes/untrack/action.rs (100%) rename {core-new => core}/src/operations/volumes/untrack/handler.rs (100%) rename {core-new => core}/src/operations/volumes/untrack/mod.rs (100%) delete mode 100644 core/src/preferences/kv.rs delete mode 100644 core/src/preferences/library.rs delete mode 100644 core/src/preferences/mod.rs delete mode 100644 core/src/search/mod.rs rename {core-new => core}/src/services/device.rs (100%) rename {core-new => core}/src/services/entry_state_service.rs (100%) rename {core-new => core}/src/services/file_sharing.rs (100%) rename {core-new => core}/src/services/location_watcher/event_handler.rs (100%) rename {core-new => core}/src/services/location_watcher/mod.rs (100%) rename {core-new => core}/src/services/location_watcher/platform/linux.rs (100%) rename {core-new => core}/src/services/location_watcher/platform/macos.rs (100%) rename {core-new => core}/src/services/location_watcher/platform/mod.rs (100%) rename {core-new => core}/src/services/location_watcher/platform/windows.rs (100%) rename {core-new => core}/src/services/location_watcher/utils.rs (100%) rename {core-new => core}/src/services/mod.rs (100%) rename {core-new => core}/src/services/networking/core/event_loop.rs (100%) rename {core-new => core}/src/services/networking/core/mod.rs (100%) rename {core-new => core}/src/services/networking/device/connection.rs (100%) rename {core-new => core}/src/services/networking/device/mod.rs (100%) rename {core-new => core}/src/services/networking/device/persistence.rs (100%) rename {core-new => core}/src/services/networking/device/registry.rs (100%) rename {core-new => core}/src/services/networking/mod.rs (100%) rename {core-new => core}/src/services/networking/protocols/file_transfer.rs (100%) rename {core-new => core}/src/services/networking/protocols/messaging.rs (100%) rename {core-new => core}/src/services/networking/protocols/mod.rs (100%) rename {core-new => core}/src/services/networking/protocols/pairing/initiator.rs (100%) rename {core-new => core}/src/services/networking/protocols/pairing/joiner.rs (100%) rename {core-new => core}/src/services/networking/protocols/pairing/messages.rs (100%) rename {core-new => core}/src/services/networking/protocols/pairing/mod.rs (100%) rename {core-new => core}/src/services/networking/protocols/pairing/persistence.rs (100%) rename {core-new => core}/src/services/networking/protocols/pairing/security.rs (100%) rename {core-new => core}/src/services/networking/protocols/pairing/types.rs (100%) rename {core-new => core}/src/services/networking/protocols/registry.rs (100%) rename {core-new => core}/src/services/networking/utils/identity.rs (100%) rename {core-new => core}/src/services/networking/utils/logging.rs (100%) rename {core-new => core}/src/services/networking/utils/mod.rs (100%) rename {core-new => core}/src/services/sidecar_manager.rs (100%) rename {core-new => core}/src/services/volume_monitor.rs (100%) rename {core-new => core}/src/shared/errors.rs (100%) rename {core-new => core}/src/shared/mod.rs (100%) rename {core-new => core}/src/shared/types.rs (100%) rename {core-new => core}/src/shared/utils.rs (100%) rename {core-new => core}/src/test_framework/mod.rs (100%) rename {core-new => core}/src/test_framework/runner.rs (100%) delete mode 100644 core/src/util/abort_on_drop.rs delete mode 100644 core/src/util/batched_stream.rs delete mode 100644 core/src/util/debug_initializer.rs delete mode 100644 core/src/util/infallible_request.rs delete mode 100644 core/src/util/maybe_undefined.rs delete mode 100644 core/src/util/mod.rs delete mode 100644 core/src/util/mpscrr.rs delete mode 100644 core/src/util/observable.rs delete mode 100644 core/src/util/unsafe_streamed_query.rs delete mode 100644 core/src/util/version_manager.rs delete mode 100644 core/src/volume/actor.rs rename {core-new => core}/src/volume/classification.rs (100%) rename {core-new => core}/src/volume/manager.rs (100%) delete mode 100644 core/src/volume/os.rs rename {core-new => core}/src/volume/os_detection.rs (100%) delete mode 100644 core/src/volume/state.rs delete mode 100644 core/src/volume/volumes.rs delete mode 100644 core/src/volume/watcher.rs rename {core-new => core}/tests/copy_action_test.rs (100%) rename {core-new => core}/tests/copy_progress_test.rs (100%) rename {core-new => core}/tests/cross_device_copy_test.rs (100%) rename {core-new => core}/tests/database_migration_test.rs (100%) rename {core-new => core}/tests/device_pairing_test.rs (100%) rename {core-new => core}/tests/device_persistence_test.rs (100%) rename {core-new => core}/tests/event_system_test.rs (100%) rename {core-new => core}/tests/file_transfer_test.rs (100%) rename {core-new => core}/tests/helpers/mod.rs (100%) rename {core-new => core}/tests/helpers/test_volumes.rs (100%) rename {core-new => core}/tests/indexing_rules_test.rs (100%) rename {core-new => core}/tests/indexing_test.rs (100%) rename {core-new => core}/tests/job_pause_resume_test.rs (100%) rename {core-new => core}/tests/job_registration_test.rs (100%) rename {core-new => core}/tests/job_shutdown_test.rs (100%) rename {core-new => core}/tests/library_test.rs (100%) rename {core-new => core}/tests/volume_tracking_test.rs (100%) rename {core-new => core}/tests/volume_tracking_with_test_volumes.rs (100%) rename {core-new => core}/whitepaper/architecture.png (100%) rename {core-new => core}/whitepaper/comment.cut (100%) rename {core-new => core}/whitepaper/grok-changes.md (100%) rename {core-new => core}/whitepaper/grok-critique.md (100%) rename {core-new => core}/whitepaper/grok-critique2.md (100%) rename {core-new => core}/whitepaper/proposed-changes-v2.md (100%) rename {core-new => core}/whitepaper/proposed-changes.md (100%) rename {core-new => core}/whitepaper/references.bib (100%) rename {core-new => core}/whitepaper/spacedrive.bbl (100%) rename {core-new => core}/whitepaper/spacedrive.blg (100%) rename {core-new => core}/whitepaper/spacedrive.out (100%) rename {core-new => core}/whitepaper/spacedrive.pdf (100%) rename {core-new => core}/whitepaper/spacedrive.tex (99%) delete mode 100644 crates/ai/Cargo.toml delete mode 100644 crates/ai/README.md delete mode 100644 crates/ai/src/lib.rs delete mode 100644 crates/ai/src/old_image_labeler/mod.rs delete mode 100644 crates/ai/src/old_image_labeler/model/mod.rs delete mode 100644 crates/ai/src/old_image_labeler/model/yolov8.rs delete mode 100644 crates/ai/src/old_image_labeler/old_actor.rs delete mode 100644 crates/ai/src/old_image_labeler/process.rs delete mode 100644 crates/ai/src/utils/mod.rs delete mode 100644 crates/file-ext/Cargo.toml delete mode 100644 crates/file-ext/src/extensions.rs delete mode 100644 crates/file-ext/src/kind.rs delete mode 100644 crates/file-ext/src/lib.rs delete mode 100644 crates/file-ext/src/magic.rs delete mode 100644 crates/file-ext/src/text.rs delete mode 100644 crates/old-p2p/Cargo.toml delete mode 100644 crates/old-p2p/README.md delete mode 100644 crates/old-p2p/crates/block/Cargo.toml delete mode 100644 crates/old-p2p/crates/block/src/block.rs delete mode 100644 crates/old-p2p/crates/block/src/block_size.rs delete mode 100644 crates/old-p2p/crates/block/src/lib.rs delete mode 100644 crates/old-p2p/crates/block/src/sb_request.rs delete mode 100644 crates/old-p2p/crates/proto/Cargo.toml delete mode 100644 crates/old-p2p/crates/proto/src/lib.rs delete mode 100644 crates/old-p2p/crates/tunnel/Cargo.toml delete mode 100644 crates/old-p2p/crates/tunnel/src/lib.rs delete mode 100644 crates/old-p2p/src/hook.rs delete mode 100644 crates/old-p2p/src/hooks.rs delete mode 100644 crates/old-p2p/src/hooks/mdns.rs delete mode 100644 crates/old-p2p/src/hooks/quic.rs delete mode 100644 crates/old-p2p/src/hooks/quic/handle.rs delete mode 100644 crates/old-p2p/src/hooks/quic/transport.rs delete mode 100644 crates/old-p2p/src/hooks/quic/utils.rs delete mode 100644 crates/old-p2p/src/identity.rs delete mode 100644 crates/old-p2p/src/lib.rs delete mode 100644 crates/old-p2p/src/p2p.rs delete mode 100644 crates/old-p2p/src/peer.rs delete mode 100644 crates/old-p2p/src/smart_guards.rs delete mode 100644 crates/old-p2p/src/stream.rs delete mode 100644 crates/prisma-cli/Cargo.toml delete mode 100644 crates/prisma-cli/src/bin/prisma.rs delete mode 100644 crates/prisma-cli/src/bin/sync.rs delete mode 100644 crates/prisma/.gitignore delete mode 100644 crates/prisma/Cargo.toml delete mode 100644 crates/prisma/src/lib.rs delete mode 100644 crates/sync-generator/Cargo.toml delete mode 100644 crates/sync-generator/src/attribute/mod.rs delete mode 100644 crates/sync-generator/src/attribute/parser.rs delete mode 100644 crates/sync-generator/src/lib.rs delete mode 100644 crates/sync-generator/src/model.rs delete mode 100644 crates/sync-generator/src/sync_data.rs delete mode 100644 crates/sync/Cargo.toml delete mode 100644 crates/sync/src/compressed.rs delete mode 100644 crates/sync/src/crdt.rs delete mode 100644 crates/sync/src/factory.rs delete mode 100644 crates/sync/src/lib.rs delete mode 100644 crates/sync/src/model_traits.rs rename {core-new => crates}/task-validator/Cargo.toml (100%) rename {core-new => crates}/task-validator/src/main.rs (100%) delete mode 100644 crates/utils/Cargo.toml delete mode 100644 crates/utils/src/db.rs delete mode 100644 crates/utils/src/error.rs delete mode 100644 crates/utils/src/lib.rs rename {core-new/docs => docs}/README.md (100%) rename {core-new/docs => docs}/benchmarks.md (100%) rename {core-new/docs => docs/core}/architecture.md (100%) rename {core-new/docs => docs/core}/cli-multi-instance.md (100%) rename {core-new/docs => docs/core}/cli.md (100%) rename {core-new/docs => docs/core}/database.md (100%) rename {core-new/docs => docs/core}/domain-models.md (100%) rename {core-new/docs => docs/core}/examples.md (100%) rename {core-new/docs => docs/core}/indexing.md (100%) rename {core-new/docs => docs/core}/job-system.md (100%) rename {core-new/docs => docs/core}/library.md (100%) rename {core-new/docs => docs/core}/locations.md (100%) rename {core-new/docs => docs/core}/networking.md (100%) rename {core-new/docs => docs/core}/pairing.md (100%) rename {core-new/docs => docs/core}/task-tracking.md (100%) rename {core-new/docs => docs/core}/testing.md (100%) rename {core-new/docs => docs/core}/virtual_sidecars.md (100%) rename {core-new/docs => docs/core}/volume-system.md (100%) rename {core-new/docs => docs}/design/ACTIONS_REFACTOR.md (100%) rename {core-new/docs => docs}/design/ACTION_SYSTEM_DESIGN.md (100%) rename {core-new/docs => docs}/design/AGENT_MANAGER_DESIGN.md (100%) rename {core-new/docs => docs}/design/API_COMPARISON.md (100%) rename {core-new/docs => docs}/design/ARCHITECTURE_DECISIONS.md (100%) rename {core-new/docs => docs}/design/AT_REST_LIBRARY_ENCRYPTION.md (100%) rename {core-new/docs => docs}/design/BENCHMARKING_SUITE_DESIGN.md (100%) rename {core-new/docs => docs}/design/CLOSURE_TABLE_INDEXING_PROPOSAL.md (100%) rename {core-new/docs => docs}/design/CROSS_DEVICE_FILE_TRANSFER_IMPLEMENTATION.md (100%) rename {core-new/docs => docs}/design/CROSS_PLATFORM_COPY_AND_VOLUME_AWARENESS.md (100%) rename {core-new/docs => docs}/design/DAEMON_REFACTOR.md (100%) rename {core-new/docs => docs}/design/DESIGN_CORE_LIFECYCLE.md (100%) rename {core-new/docs => docs}/design/DESIGN_DEVICE_MANAGEMENT.md (100%) rename {core-new/docs => docs}/design/DESIGN_FILE_DATA_MODEL.md (100%) rename {core-new/docs => docs}/design/DESIGN_FILE_DATA_MODEL_VISUAL.md (100%) rename {core-new/docs => docs}/design/DESIGN_FILE_TYPE_SYSTEM.md (100%) rename {core-new/docs => docs}/design/DESIGN_LIBRARY_IMPLEMENTATION.md (100%) rename {core-new/docs => docs}/design/DESIGN_LIBRARY_ORGANIZATION.md (100%) rename {core-new/docs => docs}/design/DESIGN_LIBRARY_ORGANIZATION_VISUAL.md (100%) rename {core-new/docs => docs}/design/DESIGN_OPTIMIZED_STORAGE.md (100%) rename {core-new/docs => docs}/design/DESIGN_VDFS_INTEGRATION.md (100%) rename {core-new/docs => docs}/design/DEVICE_PAIRING_PROTOCOL.md (100%) rename {core-new/docs => docs}/design/DOMAIN_MODELS_README.md (100%) rename {core-new/docs => docs}/design/ENTITY_REFACTOR_DESIGN.md (100%) rename {core-new/docs => docs}/design/FFMPEG_BUNDLING_DESIGN.md (94%) rename {core-new/docs => docs}/design/FILE_SHARING_DESIGN.md (100%) rename {core-new/docs => docs}/design/FUNCTION_BASED_TEST_FRAMEWORK_DESIGN.md (100%) rename {core-new/docs => docs}/design/IDEA_FOR_BETTER_INDEXING.md (100%) rename {core-new/docs => docs}/design/IMPLEMENTATION_STATUS.md (100%) rename {core-new/docs => docs}/design/INDEXER_ANALYSIS.md (100%) rename {core-new/docs => docs}/design/INDEXER_ANALYSIS_2.md (100%) rename {core-new/docs => docs}/design/INDEXER_JOB_EXAMPLE.md (100%) rename {core-new/docs => docs}/design/INDEXER_PROGRESS.md (97%) rename {core-new/docs => docs}/design/INDEXER_RULES_SYSTEM.md (90%) rename {core-new/docs => docs}/design/INDEXING_DISCOVERY_OPTIMIZATION_PLAN.md (100%) rename {core-new/docs => docs}/design/INTEGRATION_SYSTEM_DESIGN.md (100%) rename {core-new/docs => docs}/design/INTEGRATION_SYSTEM_DESIGN_GEMINI.md (100%) rename {core-new/docs => docs}/design/IPHONE_AS_VOLUME_DESIGN.md (100%) rename {core-new/docs => docs}/design/IROH_MIGRATION_DESIGN.md (100%) rename {core-new/docs => docs}/design/JOB_SYSTEM_DESIGN.md (100%) rename {core-new/docs => docs}/design/JOB_SYSTEM_MACRO_EXAMPLE.md (100%) rename {core-new/docs => docs}/design/JOB_SYSTEM_README.md (100%) rename {core-new/docs => docs}/design/LIBP2P_INTEGRATION_DESIGN.md (100%) rename {core-new/docs => docs}/design/LIBRARY_LEADERSHIP.md (100%) rename {core-new/docs => docs}/design/NETWORKING_SYSTEM_DESIGN.md (87%) rename {core-new/docs => docs}/design/OLD_SPACEDRIVE_ANALYSIS.md (100%) rename {core-new/docs => docs}/design/OPERATIONS_REFACTOR_PLAN.md (100%) rename {core-new/docs => docs}/design/PERSISTENT_DEVICE_CONNECTIONS_DESIGN.md (100%) rename {core-new/docs => docs}/design/REFERENCE_SIDECARS.md (100%) rename {core-new/docs => docs}/design/REWRITE_PLAN.MD (99%) rename {core-new/docs => docs}/design/SDPATH_REFACTOR.md (100%) rename {core-new/docs => docs}/design/SDPATH_REFACTOR_COVERAGE.md (100%) rename {core-new/docs => docs}/design/SEARCH_DESIGN.md (93%) rename {core-new/docs => docs}/design/SIMULATION_ENGINE_DESIGN.md (100%) rename SPACEDRIVE_COMPLETE_OVERVIEW.md => docs/design/SPACEDRIVE_COMPLETE_OVERVIEW.md (89%) rename {core-new/docs => docs}/design/SPACEDROP_DESIGN.md (100%) rename {core-new/docs => docs}/design/SPACEDROP_IMPLEMENTATION_PLAN.md (98%) rename {core-new/docs => docs}/design/STRUCTURE.md (97%) rename {core-new/docs => docs}/design/SYNC_DESIGN.md (100%) rename {core-new/docs => docs}/design/SYNC_DESIGN_2025_08_19.md (100%) rename {core-new/docs => docs}/design/SYNC_FIRST_DRAFT_DESIGN.md (100%) rename {core-new/docs => docs}/design/SYNC_INTEGRATION_NOTES.md (100%) rename {core-new/docs => docs}/design/THUMBNAIL_SYSTEM_DESIGN.md (95%) rename {core-new/docs => docs}/design/UI_DESIGN.md (100%) rename {core-new/docs => docs}/design/VDFS_MODEL_VISUAL.md (100%) rename {core-new/docs => docs}/design/VIRTUAL_LOCATIONS_DESIGN.md (100%) rename {core-new/docs => docs}/design/VIRTUAL_SIDECAR_SYSTEM.md (100%) rename {core-new/docs => docs}/design/VOLUME_CLASSIFICATION_DESIGN.md (100%) rename {core-new/docs => docs}/design/VOLUME_TRACKING_IMPLEMENTATION_PLAN.md (100%) rename {core-new/docs => docs}/design/WATCHER_VDFS_INTEGRATION.md (93%) rename {core-new/docs => docs}/design/WHITEPAPER_IMPL_ROADMAP.md (100%) rename {core-new/docs => docs}/design/cli-output-refactor.md (100%) rename {core-new/docs => docs}/design/frontend_graphql_usage.tsx (100%) rename {core-new/docs => docs}/design/indexer-scope-upgrade.md (100%) rename {core-new/docs => docs}/design/landing-page-idea.md (100%) rename {core-new/docs => docs}/design/networking_implementation_summary.md (100%) rename {core-new/docs => docs}/history.md (100%) rename {core-new/docs => docs}/philosophy.md (100%) rename {core-new/docs => docs}/roadmap.md (100%) rename {core-new/docs => docs}/whitepaper.md (100%) delete mode 100644 error.txt delete mode 100644 interface/.eslintrc.js delete mode 100644 interface/.gitignore delete mode 100644 interface/ErrorFallback.tsx delete mode 100644 interface/RoutingContext.tsx delete mode 100644 interface/TabsContext.tsx delete mode 100644 interface/app/$libraryId/404.tsx delete mode 100644 interface/app/$libraryId/Explorer/Context.tsx delete mode 100644 interface/app/$libraryId/Explorer/ContextMenu/AssignTagMenuItems.tsx delete mode 100644 interface/app/$libraryId/Explorer/ContextMenu/ConditionalItem.tsx delete mode 100644 interface/app/$libraryId/Explorer/ContextMenu/FilePath/CutCopyItems.tsx delete mode 100644 interface/app/$libraryId/Explorer/ContextMenu/FilePath/Items.tsx delete mode 100644 interface/app/$libraryId/Explorer/ContextMenu/Object/Items.tsx delete mode 100644 interface/app/$libraryId/Explorer/ContextMenu/OpenWith.tsx delete mode 100644 interface/app/$libraryId/Explorer/ContextMenu/SharedItems.tsx delete mode 100644 interface/app/$libraryId/Explorer/ContextMenu/context.tsx delete mode 100644 interface/app/$libraryId/Explorer/ContextMenu/index.tsx delete mode 100644 interface/app/$libraryId/Explorer/CopyAsPath.tsx delete mode 100644 interface/app/$libraryId/Explorer/DismissibleNotice.tsx delete mode 100644 interface/app/$libraryId/Explorer/DragOverlay.tsx delete mode 100644 interface/app/$libraryId/Explorer/ExplorerDraggable.tsx delete mode 100644 interface/app/$libraryId/Explorer/ExplorerDroppable.tsx delete mode 100644 interface/app/$libraryId/Explorer/ExplorerPathBar.tsx delete mode 100644 interface/app/$libraryId/Explorer/ExplorerTagBar.tsx delete mode 100644 interface/app/$libraryId/Explorer/FilePath/DeleteDialog.tsx delete mode 100644 interface/app/$libraryId/Explorer/FilePath/ErrorBarrier.tsx delete mode 100644 interface/app/$libraryId/Explorer/FilePath/Image.tsx delete mode 100644 interface/app/$libraryId/Explorer/FilePath/LayeredFileIcon.tsx delete mode 100644 interface/app/$libraryId/Explorer/FilePath/Original.tsx delete mode 100644 interface/app/$libraryId/Explorer/FilePath/RenameTextBox.tsx delete mode 100644 interface/app/$libraryId/Explorer/FilePath/Thumb.module.scss delete mode 100644 interface/app/$libraryId/Explorer/FilePath/Thumb.tsx delete mode 100644 interface/app/$libraryId/Explorer/FilePath/useFrame.tsx delete mode 100644 interface/app/$libraryId/Explorer/FilePath/utils.ts delete mode 100644 interface/app/$libraryId/Explorer/Inspector/FavoriteButton.tsx delete mode 100644 interface/app/$libraryId/Explorer/Inspector/MediaData.tsx delete mode 100644 interface/app/$libraryId/Explorer/Inspector/Note.tsx delete mode 100644 interface/app/$libraryId/Explorer/Inspector/index.tsx delete mode 100644 interface/app/$libraryId/Explorer/Inspector/store.tsx delete mode 100644 interface/app/$libraryId/Explorer/OptionsPanel/ListView/IconSize.tsx delete mode 100644 interface/app/$libraryId/Explorer/OptionsPanel/ListView/TextSize.tsx delete mode 100644 interface/app/$libraryId/Explorer/OptionsPanel/ListView/index.tsx delete mode 100644 interface/app/$libraryId/Explorer/OptionsPanel/ListView/util.ts delete mode 100644 interface/app/$libraryId/Explorer/OptionsPanel/index.tsx delete mode 100644 interface/app/$libraryId/Explorer/ParentContextMenu.tsx delete mode 100644 interface/app/$libraryId/Explorer/QuickPreview/Context.tsx delete mode 100644 interface/app/$libraryId/Explorer/QuickPreview/ImageSlider.tsx delete mode 100644 interface/app/$libraryId/Explorer/QuickPreview/index.tsx delete mode 100644 interface/app/$libraryId/Explorer/QuickPreview/store.ts delete mode 100644 interface/app/$libraryId/Explorer/RevealInNativeExplorer.tsx delete mode 100644 interface/app/$libraryId/Explorer/TopBarOptions.tsx delete mode 100644 interface/app/$libraryId/Explorer/View/Context.tsx delete mode 100644 interface/app/$libraryId/Explorer/View/DragScrollable.tsx delete mode 100644 interface/app/$libraryId/Explorer/View/EmptyNotice.tsx delete mode 100644 interface/app/$libraryId/Explorer/View/Grid/DragSelect/DragSelectable.tsx delete mode 100644 interface/app/$libraryId/Explorer/View/Grid/DragSelect/context.tsx delete mode 100644 interface/app/$libraryId/Explorer/View/Grid/DragSelect/index.tsx delete mode 100644 interface/app/$libraryId/Explorer/View/Grid/DragSelect/useDragSelectable.tsx delete mode 100644 interface/app/$libraryId/Explorer/View/Grid/DragSelect/useSelectedTargets.tsx delete mode 100644 interface/app/$libraryId/Explorer/View/Grid/DragSelect/util.ts delete mode 100644 interface/app/$libraryId/Explorer/View/Grid/Item.tsx delete mode 100644 interface/app/$libraryId/Explorer/View/Grid/useKeySelection.tsx delete mode 100644 interface/app/$libraryId/Explorer/View/GridView/Item/Context.tsx delete mode 100644 interface/app/$libraryId/Explorer/View/GridView/Item/index.tsx delete mode 100644 interface/app/$libraryId/Explorer/View/GridView/index.tsx delete mode 100644 interface/app/$libraryId/Explorer/View/ListView/Item.tsx delete mode 100644 interface/app/$libraryId/Explorer/View/ListView/TableRow.tsx delete mode 100644 interface/app/$libraryId/Explorer/View/ListView/context.tsx delete mode 100644 interface/app/$libraryId/Explorer/View/ListView/index.tsx delete mode 100644 interface/app/$libraryId/Explorer/View/ListView/useRanges.tsx delete mode 100644 interface/app/$libraryId/Explorer/View/ListView/useTable.tsx delete mode 100644 interface/app/$libraryId/Explorer/View/MediaView/DateHeader.tsx delete mode 100644 interface/app/$libraryId/Explorer/View/MediaView/Item.tsx delete mode 100644 interface/app/$libraryId/Explorer/View/MediaView/index.tsx delete mode 100644 interface/app/$libraryId/Explorer/View/MediaView/util.ts delete mode 100644 interface/app/$libraryId/Explorer/View/RenamableItemText.tsx delete mode 100644 interface/app/$libraryId/Explorer/View/ViewItem.tsx delete mode 100644 interface/app/$libraryId/Explorer/View/index.tsx delete mode 100644 interface/app/$libraryId/Explorer/View/useActiveItem.tsx delete mode 100644 interface/app/$libraryId/Explorer/View/useDragScrollable.tsx delete mode 100644 interface/app/$libraryId/Explorer/hooks/useExplorerCopyPaste.tsx delete mode 100644 interface/app/$libraryId/Explorer/index.tsx delete mode 100644 interface/app/$libraryId/Explorer/store.ts delete mode 100644 interface/app/$libraryId/Explorer/useExplorer.ts delete mode 100644 interface/app/$libraryId/Explorer/useExplorerDnd.tsx delete mode 100644 interface/app/$libraryId/Explorer/useExplorerDraggable.tsx delete mode 100644 interface/app/$libraryId/Explorer/useExplorerDroppable.tsx delete mode 100644 interface/app/$libraryId/Explorer/useExplorerItemData.tsx delete mode 100644 interface/app/$libraryId/Explorer/useExplorerOperatingSystem.tsx delete mode 100644 interface/app/$libraryId/Explorer/useExplorerPreferences.ts delete mode 100644 interface/app/$libraryId/Explorer/useKeyRevealFinder.ts delete mode 100644 interface/app/$libraryId/Explorer/util.ts delete mode 100644 interface/app/$libraryId/KeyManager/Key.tsx delete mode 100644 interface/app/$libraryId/KeyManager/List.tsx delete mode 100644 interface/app/$libraryId/KeyManager/Mounter.tsx delete mode 100644 interface/app/$libraryId/KeyManager/NotSetup.tsx delete mode 100644 interface/app/$libraryId/KeyManager/NotUnlocked.tsx delete mode 100644 interface/app/$libraryId/KeyManager/index.tsx delete mode 100644 interface/app/$libraryId/Layout/CMDK/CMDK.css delete mode 100644 interface/app/$libraryId/Layout/CMDK/CMDK.scss delete mode 100644 interface/app/$libraryId/Layout/CMDK/index.tsx delete mode 100644 interface/app/$libraryId/Layout/CMDK/pages/CMDKLocations.tsx delete mode 100644 interface/app/$libraryId/Layout/CMDK/pages/CMDKTags.tsx delete mode 100644 interface/app/$libraryId/Layout/Context.tsx delete mode 100644 interface/app/$libraryId/Layout/DndContext.tsx delete mode 100644 interface/app/$libraryId/Layout/Sidebar/DebugPopover.tsx delete mode 100644 interface/app/$libraryId/Layout/Sidebar/JobManager/IsRunningJob.tsx delete mode 100644 interface/app/$libraryId/Layout/Sidebar/JobManager/Job.module.scss delete mode 100644 interface/app/$libraryId/Layout/Sidebar/JobManager/Job.tsx delete mode 100644 interface/app/$libraryId/Layout/Sidebar/JobManager/JobContainer.tsx delete mode 100644 interface/app/$libraryId/Layout/Sidebar/JobManager/JobGroup.tsx delete mode 100644 interface/app/$libraryId/Layout/Sidebar/JobManager/index.tsx delete mode 100644 interface/app/$libraryId/Layout/Sidebar/SidebarLayout/Context.tsx delete mode 100644 interface/app/$libraryId/Layout/Sidebar/SidebarLayout/FeedbackPopover.tsx delete mode 100644 interface/app/$libraryId/Layout/Sidebar/SidebarLayout/Footer.tsx delete mode 100644 interface/app/$libraryId/Layout/Sidebar/SidebarLayout/Icon.tsx delete mode 100644 interface/app/$libraryId/Layout/Sidebar/SidebarLayout/JobManagerPopover.tsx delete mode 100644 interface/app/$libraryId/Layout/Sidebar/SidebarLayout/LibrariesDropdown.tsx delete mode 100644 interface/app/$libraryId/Layout/Sidebar/SidebarLayout/Link.tsx delete mode 100644 interface/app/$libraryId/Layout/Sidebar/SidebarLayout/Section.tsx delete mode 100644 interface/app/$libraryId/Layout/Sidebar/SidebarLayout/SeeMore.tsx delete mode 100644 interface/app/$libraryId/Layout/Sidebar/SidebarLayout/WindowControls.tsx delete mode 100644 interface/app/$libraryId/Layout/Sidebar/SidebarLayout/index.tsx delete mode 100644 interface/app/$libraryId/Layout/Sidebar/helpers.ts delete mode 100644 interface/app/$libraryId/Layout/Sidebar/index.tsx delete mode 100644 interface/app/$libraryId/Layout/Sidebar/sections/Categories/index.tsx delete mode 100644 interface/app/$libraryId/Layout/Sidebar/sections/Debug/index.tsx delete mode 100644 interface/app/$libraryId/Layout/Sidebar/sections/Devices/AddDeviceDialog.tsx delete mode 100644 interface/app/$libraryId/Layout/Sidebar/sections/Devices/index.tsx delete mode 100644 interface/app/$libraryId/Layout/Sidebar/sections/Library/index.tsx delete mode 100644 interface/app/$libraryId/Layout/Sidebar/sections/Local/index.tsx delete mode 100644 interface/app/$libraryId/Layout/Sidebar/sections/Locations/ContextMenu.tsx delete mode 100644 interface/app/$libraryId/Layout/Sidebar/sections/Locations/index.tsx delete mode 100644 interface/app/$libraryId/Layout/Sidebar/sections/Peers/index.tsx delete mode 100644 interface/app/$libraryId/Layout/Sidebar/sections/SavedSearches/index.tsx delete mode 100644 interface/app/$libraryId/Layout/Sidebar/sections/Tags/ContextMenu.tsx delete mode 100644 interface/app/$libraryId/Layout/Sidebar/sections/Tags/index.tsx delete mode 100644 interface/app/$libraryId/Layout/Sidebar/sections/Tools/index.tsx delete mode 100644 interface/app/$libraryId/Layout/Sidebar/store.ts delete mode 100644 interface/app/$libraryId/Layout/auth.tsx delete mode 100644 interface/app/$libraryId/Layout/index.tsx delete mode 100644 interface/app/$libraryId/Layout/store.ts delete mode 100644 interface/app/$libraryId/PageLayout/Context.tsx delete mode 100644 interface/app/$libraryId/PageLayout/index.tsx delete mode 100644 interface/app/$libraryId/Spacedrop/index.tsx delete mode 100644 interface/app/$libraryId/Spacedrop/toast.tsx delete mode 100644 interface/app/$libraryId/TopBar/Context.tsx delete mode 100644 interface/app/$libraryId/TopBar/Layout.tsx delete mode 100644 interface/app/$libraryId/TopBar/NavigationButtons.tsx delete mode 100644 interface/app/$libraryId/TopBar/Portal.tsx delete mode 100644 interface/app/$libraryId/TopBar/TopBarButton.tsx delete mode 100644 interface/app/$libraryId/TopBar/TopBarMobile.tsx delete mode 100644 interface/app/$libraryId/TopBar/TopBarOptions.tsx delete mode 100644 interface/app/$libraryId/TopBar/index.tsx delete mode 100644 interface/app/$libraryId/debug/dnd.tsx delete mode 100644 interface/app/$libraryId/ephemeral.tsx delete mode 100644 interface/app/$libraryId/favorites.tsx delete mode 100644 interface/app/$libraryId/index.tsx delete mode 100644 interface/app/$libraryId/labels.tsx delete mode 100644 interface/app/$libraryId/location/$id.tsx delete mode 100644 interface/app/$libraryId/location/LocationOptions.tsx delete mode 100644 interface/app/$libraryId/media.tsx delete mode 100644 interface/app/$libraryId/network.tsx delete mode 100644 interface/app/$libraryId/node/$id.tsx delete mode 100644 interface/app/$libraryId/overview/FileKindStats.tsx delete mode 100644 interface/app/$libraryId/overview/Layout/HorizontalScroll.tsx delete mode 100644 interface/app/$libraryId/overview/Layout/Section.tsx delete mode 100644 interface/app/$libraryId/overview/LibraryStats.tsx delete mode 100644 interface/app/$libraryId/overview/LocationCard.tsx delete mode 100644 interface/app/$libraryId/overview/NewCard.tsx delete mode 100644 interface/app/$libraryId/overview/StatCard.tsx delete mode 100644 interface/app/$libraryId/overview/StorageBar.tsx delete mode 100644 interface/app/$libraryId/overview/index.tsx delete mode 100644 interface/app/$libraryId/peer/$id.tsx delete mode 100644 interface/app/$libraryId/peer/StarfieldEffect.tsx delete mode 100644 interface/app/$libraryId/people.tsx delete mode 100644 interface/app/$libraryId/recents.tsx delete mode 100644 interface/app/$libraryId/saved-search/$id.tsx delete mode 100644 interface/app/$libraryId/search/Filters/FilterRegistry.tsx delete mode 100644 interface/app/$libraryId/search/Filters/components/AppliedFilters.tsx delete mode 100644 interface/app/$libraryId/search/Filters/components/FilterOptionBoolean.tsx delete mode 100644 interface/app/$libraryId/search/Filters/components/FilterOptionDateRange.tsx delete mode 100644 interface/app/$libraryId/search/Filters/components/FilterOptionList.tsx delete mode 100644 interface/app/$libraryId/search/Filters/factories/createBooleanFilter.ts delete mode 100644 interface/app/$libraryId/search/Filters/factories/createDateRangeFilter.ts delete mode 100644 interface/app/$libraryId/search/Filters/factories/createInOrNotInFilter.ts delete mode 100644 interface/app/$libraryId/search/Filters/factories/createTextMatchFilter.ts delete mode 100644 interface/app/$libraryId/search/Filters/hooks/useToggleOptionSelected.tsx delete mode 100644 interface/app/$libraryId/search/Filters/index.tsx delete mode 100644 interface/app/$libraryId/search/Filters/registry/BooleanFilters.tsx delete mode 100644 interface/app/$libraryId/search/Filters/registry/DateFilters.tsx delete mode 100644 interface/app/$libraryId/search/Filters/registry/KindFilter.tsx delete mode 100644 interface/app/$libraryId/search/Filters/registry/LocationFilter.tsx delete mode 100644 interface/app/$libraryId/search/Filters/registry/TagsFilter.tsx delete mode 100644 interface/app/$libraryId/search/Filters/registry/TextFilters.tsx delete mode 100644 interface/app/$libraryId/search/Filters/store.ts delete mode 100644 interface/app/$libraryId/search/Filters/typeGuards.ts delete mode 100644 interface/app/$libraryId/search/FiltersOld.tsx delete mode 100644 interface/app/$libraryId/search/SearchBar.tsx delete mode 100644 interface/app/$libraryId/search/SearchOptions.tsx delete mode 100644 interface/app/$libraryId/search/context.tsx delete mode 100644 interface/app/$libraryId/search/index.tsx delete mode 100644 interface/app/$libraryId/search/store.tsx delete mode 100644 interface/app/$libraryId/search/useSearch.ts delete mode 100644 interface/app/$libraryId/search/useSearchExplorerQuery.ts delete mode 100644 interface/app/$libraryId/search/util.tsx delete mode 100644 interface/app/$libraryId/settings/Layout.tsx delete mode 100644 interface/app/$libraryId/settings/ModalLayout.tsx delete mode 100644 interface/app/$libraryId/settings/OverviewLayout.tsx delete mode 100644 interface/app/$libraryId/settings/Setting.tsx delete mode 100644 interface/app/$libraryId/settings/Sidebar.tsx delete mode 100644 interface/app/$libraryId/settings/client/SpacedriveAccount.tsx delete mode 100644 interface/app/$libraryId/settings/client/account/Profile.tsx delete mode 100644 interface/app/$libraryId/settings/client/account/ShowPassword.tsx delete mode 100644 interface/app/$libraryId/settings/client/account/handlers/cookieHandler.ts delete mode 100644 interface/app/$libraryId/settings/client/account/handlers/windowHandler.ts delete mode 100644 interface/app/$libraryId/settings/client/account/index.tsx delete mode 100644 interface/app/$libraryId/settings/client/appearance.tsx delete mode 100644 interface/app/$libraryId/settings/client/backups.tsx delete mode 100644 interface/app/$libraryId/settings/client/extensions.tsx delete mode 100644 interface/app/$libraryId/settings/client/general.tsx delete mode 100644 interface/app/$libraryId/settings/client/index.ts delete mode 100644 interface/app/$libraryId/settings/client/keybindings.tsx delete mode 100644 interface/app/$libraryId/settings/client/network/debug.tsx delete mode 100644 interface/app/$libraryId/settings/client/network/index.tsx delete mode 100644 interface/app/$libraryId/settings/client/privacy.tsx delete mode 100644 interface/app/$libraryId/settings/client/usage.tsx delete mode 100644 interface/app/$libraryId/settings/index.tsx delete mode 100644 interface/app/$libraryId/settings/library/contacts.tsx delete mode 100644 interface/app/$libraryId/settings/library/general.tsx delete mode 100644 interface/app/$libraryId/settings/library/index.tsx delete mode 100644 interface/app/$libraryId/settings/library/locations/$id.tsx delete mode 100644 interface/app/$libraryId/settings/library/locations/AddLocationButton.tsx delete mode 100644 interface/app/$libraryId/settings/library/locations/AddLocationDialog.tsx delete mode 100644 interface/app/$libraryId/settings/library/locations/DeleteDialog.tsx delete mode 100644 interface/app/$libraryId/settings/library/locations/IndexerRuleEditor/RuleButton.tsx delete mode 100644 interface/app/$libraryId/settings/library/locations/IndexerRuleEditor/RuleInput.tsx delete mode 100644 interface/app/$libraryId/settings/library/locations/IndexerRuleEditor/RulesForm.tsx delete mode 100644 interface/app/$libraryId/settings/library/locations/IndexerRuleEditor/index.tsx delete mode 100644 interface/app/$libraryId/settings/library/locations/ListItem.tsx delete mode 100644 interface/app/$libraryId/settings/library/locations/PathInput.tsx delete mode 100644 interface/app/$libraryId/settings/library/locations/index.tsx delete mode 100644 interface/app/$libraryId/settings/library/locations/openDirectoryPickerDialog.ts delete mode 100644 interface/app/$libraryId/settings/library/saved-searches/index.tsx delete mode 100644 interface/app/$libraryId/settings/library/security.tsx delete mode 100644 interface/app/$libraryId/settings/library/sharing.tsx delete mode 100644 interface/app/$libraryId/settings/library/tags/CreateDialog.tsx delete mode 100644 interface/app/$libraryId/settings/library/tags/DeleteDialog.tsx delete mode 100644 interface/app/$libraryId/settings/library/tags/EditForm.tsx delete mode 100644 interface/app/$libraryId/settings/library/tags/index.tsx delete mode 100644 interface/app/$libraryId/settings/node/index.tsx delete mode 100644 interface/app/$libraryId/settings/node/libraries/CreateDialog.tsx delete mode 100644 interface/app/$libraryId/settings/node/libraries/DeleteDeviceDialog.tsx delete mode 100644 interface/app/$libraryId/settings/node/libraries/DeleteDialog.tsx delete mode 100644 interface/app/$libraryId/settings/node/libraries/DeviceItem.tsx delete mode 100644 interface/app/$libraryId/settings/node/libraries/ListItem.tsx delete mode 100644 interface/app/$libraryId/settings/node/libraries/index.tsx delete mode 100644 interface/app/$libraryId/settings/node/p2p.tsx delete mode 100644 interface/app/$libraryId/settings/resources/about.tsx delete mode 100644 interface/app/$libraryId/settings/resources/changelog.tsx delete mode 100644 interface/app/$libraryId/settings/resources/index.tsx delete mode 100644 interface/app/$libraryId/spaces.tsx delete mode 100644 interface/app/$libraryId/tag/$id.tsx delete mode 100644 interface/app/I18n.ts delete mode 100644 interface/app/RootContext.tsx delete mode 100644 interface/app/demo.react.tsx delete mode 100644 interface/app/demo.solid.tsx delete mode 100644 interface/app/index.tsx delete mode 100644 interface/app/onboarding/Layout.tsx delete mode 100644 interface/app/onboarding/Progress.tsx delete mode 100644 interface/app/onboarding/components.tsx delete mode 100644 interface/app/onboarding/context.tsx delete mode 100644 interface/app/onboarding/creating-library.tsx delete mode 100644 interface/app/onboarding/full-disk.tsx delete mode 100644 interface/app/onboarding/index.tsx delete mode 100644 interface/app/onboarding/joining-library.tsx delete mode 100644 interface/app/onboarding/locations.tsx delete mode 100644 interface/app/onboarding/login.tsx delete mode 100644 interface/app/onboarding/new-library.tsx delete mode 100644 interface/app/onboarding/prerelease.tsx delete mode 100644 interface/app/onboarding/privacy.tsx delete mode 100644 interface/app/p2p/index.tsx delete mode 100644 interface/app/route-schemas.ts delete mode 100644 interface/app/style.scss delete mode 100644 interface/components/Accordion.tsx delete mode 100644 interface/components/AlertDialog.tsx delete mode 100644 interface/components/AuthCheck.tsx delete mode 100644 interface/components/AuthRequiredOverlay.tsx delete mode 100644 interface/components/Authentication.tsx delete mode 100644 interface/components/Codeblock.tsx delete mode 100644 interface/components/ColorPicker.tsx delete mode 100644 interface/components/Devtools.tsx delete mode 100644 interface/components/DismissibleNotice.tsx delete mode 100644 interface/components/DragRegion.tsx delete mode 100644 interface/components/Folder.tsx delete mode 100644 interface/components/Icon.tsx delete mode 100644 interface/components/Loader.tsx delete mode 100644 interface/components/Login.tsx delete mode 100644 interface/components/LoginButton.tsx delete mode 100644 interface/components/Menu.tsx delete mode 100644 interface/components/MultiCheckbox.tsx delete mode 100644 interface/components/PDFViewer.tsx delete mode 100644 interface/components/PasswordMeter.tsx delete mode 100644 interface/components/Register.tsx delete mode 100644 interface/components/RequestAddDialog.tsx delete mode 100644 interface/components/ShowPassword.tsx delete mode 100644 interface/components/Sparkles.tsx delete mode 100644 interface/components/SubtleButton.tsx delete mode 100644 interface/components/TextViewer/index.tsx delete mode 100644 interface/components/TextViewer/one-dark.scss delete mode 100644 interface/components/TextViewer/one-light.scss delete mode 100644 interface/components/TextViewer/prism-lazy.ts delete mode 100644 interface/components/TextViewer/prism.tsx delete mode 100644 interface/components/TrafficLights.tsx delete mode 100644 interface/components/TruncatedText.tsx delete mode 100644 interface/components/index.ts delete mode 100644 interface/hooks/index.ts delete mode 100644 interface/hooks/useAccessToken.ts delete mode 100644 interface/hooks/useCallbackToWatchForm.ts delete mode 100644 interface/hooks/useCallbackToWatchResize.ts delete mode 100644 interface/hooks/useClickOutside.ts delete mode 100644 interface/hooks/useCounter.ts delete mode 100644 interface/hooks/useDebouncedForm.ts delete mode 100644 interface/hooks/useDeeplinkEventHandler.ts delete mode 100644 interface/hooks/useDismissibleNoticeStore.tsx delete mode 100644 interface/hooks/useDragAndDropState.ts delete mode 100644 interface/hooks/useFileDropEventHandler.ts delete mode 100644 interface/hooks/useFocusState.tsx delete mode 100644 interface/hooks/useHomeDir.ts delete mode 100644 interface/hooks/useInputState.tsx delete mode 100644 interface/hooks/useIsDark.ts delete mode 100644 interface/hooks/useIsLocationIndexing.ts delete mode 100644 interface/hooks/useIsTextTruncated.ts delete mode 100644 interface/hooks/useKeyDeleteFile.tsx delete mode 100644 interface/hooks/useKeyMatcher.ts delete mode 100644 interface/hooks/useKeybind.ts delete mode 100644 interface/hooks/useKeybindEventHandler.ts delete mode 100644 interface/hooks/useKeybindFactory.ts delete mode 100644 interface/hooks/useLocale.ts delete mode 100644 interface/hooks/useMouseItemResize.ts delete mode 100644 interface/hooks/useOperatingSystem.ts delete mode 100644 interface/hooks/usePrefersReducedMotion.ts delete mode 100644 interface/hooks/useQuickRescan.ts delete mode 100644 interface/hooks/useRandomInterval.ts delete mode 100644 interface/hooks/useRedirectToNewLocation.ts delete mode 100644 interface/hooks/useRouteTitle.ts delete mode 100644 interface/hooks/useScrolled.tsx delete mode 100644 interface/hooks/useShortcut.ts delete mode 100644 interface/hooks/useShowControls.ts delete mode 100644 interface/hooks/useTheme.ts delete mode 100644 interface/hooks/useWindowSize.ts delete mode 100644 interface/hooks/useWindowState.tsx delete mode 100644 interface/hooks/useZodParams.ts delete mode 100644 interface/hooks/useZodRouteParams.ts delete mode 100644 interface/hooks/useZodSearchParams.ts delete mode 100644 interface/index.tsx delete mode 100644 interface/locales/README.md delete mode 100644 interface/locales/ar/common.json delete mode 100644 interface/locales/be/common.json delete mode 100644 interface/locales/cs/common.json delete mode 100644 interface/locales/de/common.json delete mode 100644 interface/locales/en/common.json delete mode 100644 interface/locales/es/common.json delete mode 100644 interface/locales/fr/common.json delete mode 100644 interface/locales/i18nnext.d.ts delete mode 100644 interface/locales/it/common.json delete mode 100644 interface/locales/ja/common.json delete mode 100644 interface/locales/nl/common.json delete mode 100644 interface/locales/ru/common.json delete mode 100644 interface/locales/tr/common.json delete mode 100644 interface/locales/uk/common.json delete mode 100644 interface/locales/zh-CN/common.json delete mode 100644 interface/locales/zh-TW/common.json delete mode 100644 interface/package.json delete mode 100644 interface/tailwind.config.js delete mode 100644 interface/tsconfig.json delete mode 100644 interface/types/declarations.d.ts delete mode 100644 interface/util/Platform.tsx delete mode 100644 interface/util/events.ts delete mode 100644 interface/util/hardware.ts delete mode 100644 interface/util/index.tsx delete mode 100644 interface/util/keybinds.ts delete mode 100644 interface/util/pdfViewer.tsx delete mode 100644 interface/util/useTraceUpdate.tsx delete mode 100644 interface/util/uuid.ts diff --git a/.prettierrc.js b/.prettierrc.js deleted file mode 100644 index f57e12019..000000000 --- a/.prettierrc.js +++ /dev/null @@ -1,31 +0,0 @@ -const path = require('node:path'); - -/** - * {@type require('prettier').Config} - */ -module.exports = { - useTabs: true, - printWidth: 100, - singleQuote: true, - trailingComma: 'none', - bracketSameLine: false, - semi: true, - quoteProps: 'consistent', - importOrder: [ - // external packages - '', - // spacedrive packages - '^@sd/(interface|client|ui)(/.*)?$', - // internal packages - '^@/', - '^~/', - '', - // relative - '^[../]', - '^[./]' - ], - importOrderParserPlugins: ['typescript', 'jsx', 'decorators'], - importOrderTypeScriptVersion: '5.0.0', - tailwindConfig: path.resolve(path.join(__dirname, 'packages/ui/tailwind.config.js')), - plugins: ['@ianvs/prettier-plugin-sort-imports', 'prettier-plugin-tailwindcss'] -}; diff --git a/core-new/.tasks/ACT-000-action-system.md b/.tasks/ACT-000-action-system.md similarity index 100% rename from core-new/.tasks/ACT-000-action-system.md rename to .tasks/ACT-000-action-system.md diff --git a/core-new/.tasks/ACT-001-action-manager-registry.md b/.tasks/ACT-001-action-manager-registry.md similarity index 100% rename from core-new/.tasks/ACT-001-action-manager-registry.md rename to .tasks/ACT-001-action-manager-registry.md diff --git a/core-new/.tasks/AI-000-ai-epic.md b/.tasks/AI-000-ai-epic.md similarity index 100% rename from core-new/.tasks/AI-000-ai-epic.md rename to .tasks/AI-000-ai-epic.md diff --git a/core-new/.tasks/AI-001-ai-agent.md b/.tasks/AI-001-ai-agent.md similarity index 100% rename from core-new/.tasks/AI-001-ai-agent.md rename to .tasks/AI-001-ai-agent.md diff --git a/core-new/.tasks/AI-002-create-finetuning-dataset.md b/.tasks/AI-002-create-finetuning-dataset.md similarity index 100% rename from core-new/.tasks/AI-002-create-finetuning-dataset.md rename to .tasks/AI-002-create-finetuning-dataset.md diff --git a/core-new/.tasks/CLI-000-command-line-interface.md b/.tasks/CLI-000-command-line-interface.md similarity index 100% rename from core-new/.tasks/CLI-000-command-line-interface.md rename to .tasks/CLI-000-command-line-interface.md diff --git a/core-new/.tasks/CLOUD-000-cloud-as-a-peer.md b/.tasks/CLOUD-000-cloud-as-a-peer.md similarity index 100% rename from core-new/.tasks/CLOUD-000-cloud-as-a-peer.md rename to .tasks/CLOUD-000-cloud-as-a-peer.md diff --git a/core-new/.tasks/CLOUD-001-design-cloud-core-infra.md b/.tasks/CLOUD-001-design-cloud-core-infra.md similarity index 63% rename from core-new/.tasks/CLOUD-001-design-cloud-core-infra.md rename to .tasks/CLOUD-001-design-cloud-core-infra.md index 1ca33c95d..d159add50 100644 --- a/core-new/.tasks/CLOUD-001-design-cloud-core-infra.md +++ b/.tasks/CLOUD-001-design-cloud-core-infra.md @@ -11,7 +11,7 @@ whitepaper: Section 5.1 ## Description -Design the infrastructure for provisioning and running isolated `sd-core-new` instances for users in a cloud environment. This involves creating a scalable and secure architecture, likely using containerization and orchestration technologies like Kubernetes. +Design the infrastructure for provisioning and running isolated `sd-core` instances for users in a cloud environment. This involves creating a scalable and secure architecture, likely using containerization and orchestration technologies like Kubernetes. ## Implementation Steps @@ -21,6 +21,7 @@ Design the infrastructure for provisioning and running isolated `sd-core-new` in 4. Specify the security and networking policies for the cloud environment. ## Acceptance Criteria -- [ ] A detailed architecture document is created. -- [ ] The design addresses scalability, security, and cost-effectiveness. -- [ ] The design is approved and ready for implementation. + +- [ ] A detailed architecture document is created. +- [ ] The design addresses scalability, security, and cost-effectiveness. +- [ ] The design is approved and ready for implementation. diff --git a/core-new/.tasks/CLOUD-002-relay-server.md b/.tasks/CLOUD-002-relay-server.md similarity index 100% rename from core-new/.tasks/CLOUD-002-relay-server.md rename to .tasks/CLOUD-002-relay-server.md diff --git a/core-new/.tasks/CLOUD-003-cloud-volume.md b/.tasks/CLOUD-003-cloud-volume.md similarity index 100% rename from core-new/.tasks/CLOUD-003-cloud-volume.md rename to .tasks/CLOUD-003-cloud-volume.md diff --git a/core-new/.tasks/CORE-000-vdfs-core.md b/.tasks/CORE-000-vdfs-core.md similarity index 100% rename from core-new/.tasks/CORE-000-vdfs-core.md rename to .tasks/CORE-000-vdfs-core.md diff --git a/core-new/.tasks/CORE-001-entry-centric-model.md b/.tasks/CORE-001-entry-centric-model.md similarity index 100% rename from core-new/.tasks/CORE-001-entry-centric-model.md rename to .tasks/CORE-001-entry-centric-model.md diff --git a/core-new/.tasks/CORE-002-sdpath-addressing.md b/.tasks/CORE-002-sdpath-addressing.md similarity index 100% rename from core-new/.tasks/CORE-002-sdpath-addressing.md rename to .tasks/CORE-002-sdpath-addressing.md diff --git a/core-new/.tasks/CORE-003-content-identity.md b/.tasks/CORE-003-content-identity.md similarity index 100% rename from core-new/.tasks/CORE-003-content-identity.md rename to .tasks/CORE-003-content-identity.md diff --git a/core-new/.tasks/CORE-004-closure-table.md b/.tasks/CORE-004-closure-table.md similarity index 100% rename from core-new/.tasks/CORE-004-closure-table.md rename to .tasks/CORE-004-closure-table.md diff --git a/core-new/.tasks/CORE-005-file-type-system.md b/.tasks/CORE-005-file-type-system.md similarity index 100% rename from core-new/.tasks/CORE-005-file-type-system.md rename to .tasks/CORE-005-file-type-system.md diff --git a/core-new/.tasks/CORE-006-semantic-tagging-architecture.md b/.tasks/CORE-006-semantic-tagging-architecture.md similarity index 100% rename from core-new/.tasks/CORE-006-semantic-tagging-architecture.md rename to .tasks/CORE-006-semantic-tagging-architecture.md diff --git a/core-new/.tasks/CORE-007-quantum-state-for-on-demand-state-computation.md b/.tasks/CORE-007-quantum-state-for-on-demand-state-computation.md similarity index 100% rename from core-new/.tasks/CORE-007-quantum-state-for-on-demand-state-computation.md rename to .tasks/CORE-007-quantum-state-for-on-demand-state-computation.md diff --git a/core-new/.tasks/CORE-008-virtual-sidecar-system.md b/.tasks/CORE-008-virtual-sidecar-system.md similarity index 96% rename from core-new/.tasks/CORE-008-virtual-sidecar-system.md rename to .tasks/CORE-008-virtual-sidecar-system.md index 9249810a2..79b269057 100644 --- a/core-new/.tasks/CORE-008-virtual-sidecar-system.md +++ b/.tasks/CORE-008-virtual-sidecar-system.md @@ -1,8 +1,8 @@ --- id: CORE-008 title: Virtual Sidecar System -status: To Do -assignee: unassigned +status: In Progress +assignee: james parent: CORE-000 priority: High tags: [core, vdfs, sidecars, derivatives] diff --git a/core-new/.tasks/CORE-009-user-managed-collections.md b/.tasks/CORE-009-user-managed-collections.md similarity index 100% rename from core-new/.tasks/CORE-009-user-managed-collections.md rename to .tasks/CORE-009-user-managed-collections.md diff --git a/core-new/.tasks/CORE-010-file-ingestion-workflow.md b/.tasks/CORE-010-file-ingestion-workflow.md similarity index 100% rename from core-new/.tasks/CORE-010-file-ingestion-workflow.md rename to .tasks/CORE-010-file-ingestion-workflow.md diff --git a/core-new/.tasks/DEV-000-development-validation.md b/.tasks/DEV-000-development-validation.md similarity index 100% rename from core-new/.tasks/DEV-000-development-validation.md rename to .tasks/DEV-000-development-validation.md diff --git a/core-new/.tasks/DEV-001-multi-process-test-framework.md b/.tasks/DEV-001-multi-process-test-framework.md similarity index 100% rename from core-new/.tasks/DEV-001-multi-process-test-framework.md rename to .tasks/DEV-001-multi-process-test-framework.md diff --git a/core-new/.tasks/FILE-000-file-operations.md b/.tasks/FILE-000-file-operations.md similarity index 100% rename from core-new/.tasks/FILE-000-file-operations.md rename to .tasks/FILE-000-file-operations.md diff --git a/core-new/.tasks/FILE-001-file-copy-job.md b/.tasks/FILE-001-file-copy-job.md similarity index 100% rename from core-new/.tasks/FILE-001-file-copy-job.md rename to .tasks/FILE-001-file-copy-job.md diff --git a/core-new/.tasks/FILE-002-file-deletion-job.md b/.tasks/FILE-002-file-deletion-job.md similarity index 100% rename from core-new/.tasks/FILE-002-file-deletion-job.md rename to .tasks/FILE-002-file-deletion-job.md diff --git a/core-new/.tasks/INDEX-000-indexing-file-management.md b/.tasks/INDEX-000-indexing-file-management.md similarity index 100% rename from core-new/.tasks/INDEX-000-indexing-file-management.md rename to .tasks/INDEX-000-indexing-file-management.md diff --git a/core-new/.tasks/INDEX-001-location-watcher-service.md b/.tasks/INDEX-001-location-watcher-service.md similarity index 100% rename from core-new/.tasks/INDEX-001-location-watcher-service.md rename to .tasks/INDEX-001-location-watcher-service.md diff --git a/core-new/.tasks/INDEX-002-stale-file-detection-algorithm.md b/.tasks/INDEX-002-stale-file-detection-algorithm.md similarity index 100% rename from core-new/.tasks/INDEX-002-stale-file-detection-algorithm.md rename to .tasks/INDEX-002-stale-file-detection-algorithm.md diff --git a/core-new/.tasks/JOB-000-job-system.md b/.tasks/JOB-000-job-system.md similarity index 100% rename from core-new/.tasks/JOB-000-job-system.md rename to .tasks/JOB-000-job-system.md diff --git a/core-new/.tasks/JOB-001-job-manager.md b/.tasks/JOB-001-job-manager.md similarity index 100% rename from core-new/.tasks/JOB-001-job-manager.md rename to .tasks/JOB-001-job-manager.md diff --git a/core-new/.tasks/JOB-002-job-logging.md b/.tasks/JOB-002-job-logging.md similarity index 100% rename from core-new/.tasks/JOB-002-job-logging.md rename to .tasks/JOB-002-job-logging.md diff --git a/core-new/.tasks/LOC-000-location-operations.md b/.tasks/LOC-000-location-operations.md similarity index 100% rename from core-new/.tasks/LOC-000-location-operations.md rename to .tasks/LOC-000-location-operations.md diff --git a/core-new/.tasks/LOC-001-location-management-actions.md b/.tasks/LOC-001-location-management-actions.md similarity index 100% rename from core-new/.tasks/LOC-001-location-management-actions.md rename to .tasks/LOC-001-location-management-actions.md diff --git a/core-new/.tasks/LOC-005-virtual-locations-via-pure-hierarchical-model.md b/.tasks/LOC-005-virtual-locations-via-pure-hierarchical-model.md similarity index 100% rename from core-new/.tasks/LOC-005-virtual-locations-via-pure-hierarchical-model.md rename to .tasks/LOC-005-virtual-locations-via-pure-hierarchical-model.md diff --git a/core-new/.tasks/LSYNC-000-library-sync.md b/.tasks/LSYNC-000-library-sync.md similarity index 100% rename from core-new/.tasks/LSYNC-000-library-sync.md rename to .tasks/LSYNC-000-library-sync.md diff --git a/core-new/.tasks/LSYNC-001-design-library-sync-protocol.md b/.tasks/LSYNC-001-design-library-sync-protocol.md similarity index 100% rename from core-new/.tasks/LSYNC-001-design-library-sync-protocol.md rename to .tasks/LSYNC-001-design-library-sync-protocol.md diff --git a/core-new/.tasks/LSYNC-002-metadata-sync.md b/.tasks/LSYNC-002-metadata-sync.md similarity index 100% rename from core-new/.tasks/LSYNC-002-metadata-sync.md rename to .tasks/LSYNC-002-metadata-sync.md diff --git a/core-new/.tasks/LSYNC-003-file-op-sync.md b/.tasks/LSYNC-003-file-op-sync.md similarity index 100% rename from core-new/.tasks/LSYNC-003-file-op-sync.md rename to .tasks/LSYNC-003-file-op-sync.md diff --git a/core-new/.tasks/LSYNC-004-sync-relationship-database-schema.md b/.tasks/LSYNC-004-sync-relationship-database-schema.md similarity index 100% rename from core-new/.tasks/LSYNC-004-sync-relationship-database-schema.md rename to .tasks/LSYNC-004-sync-relationship-database-schema.md diff --git a/core-new/.tasks/NET-000-networking.md b/.tasks/NET-000-networking.md similarity index 100% rename from core-new/.tasks/NET-000-networking.md rename to .tasks/NET-000-networking.md diff --git a/core-new/.tasks/NET-001-iroh-p2p-stack.md b/.tasks/NET-001-iroh-p2p-stack.md similarity index 100% rename from core-new/.tasks/NET-001-iroh-p2p-stack.md rename to .tasks/NET-001-iroh-p2p-stack.md diff --git a/core-new/.tasks/NET-002-device-pairing.md b/.tasks/NET-002-device-pairing.md similarity index 100% rename from core-new/.tasks/NET-002-device-pairing.md rename to .tasks/NET-002-device-pairing.md diff --git a/core-new/.tasks/NET-003-spacedrop-protocol.md b/.tasks/NET-003-spacedrop-protocol.md similarity index 100% rename from core-new/.tasks/NET-003-spacedrop-protocol.md rename to .tasks/NET-003-spacedrop-protocol.md diff --git a/core-new/.tasks/PLUG-000-wasm-plugin-system.md b/.tasks/PLUG-000-wasm-plugin-system.md similarity index 100% rename from core-new/.tasks/PLUG-000-wasm-plugin-system.md rename to .tasks/PLUG-000-wasm-plugin-system.md diff --git a/core-new/.tasks/PLUG-001-integrate-wasm-runtime.md b/.tasks/PLUG-001-integrate-wasm-runtime.md similarity index 100% rename from core-new/.tasks/PLUG-001-integrate-wasm-runtime.md rename to .tasks/PLUG-001-integrate-wasm-runtime.md diff --git a/core-new/.tasks/PLUG-002-define-vdfs-plugin-api.md b/.tasks/PLUG-002-define-vdfs-plugin-api.md similarity index 100% rename from core-new/.tasks/PLUG-002-define-vdfs-plugin-api.md rename to .tasks/PLUG-002-define-vdfs-plugin-api.md diff --git a/core-new/.tasks/PLUG-003-develop-twitter-agent-poc.md b/.tasks/PLUG-003-develop-twitter-agent-poc.md similarity index 100% rename from core-new/.tasks/PLUG-003-develop-twitter-agent-poc.md rename to .tasks/PLUG-003-develop-twitter-agent-poc.md diff --git a/core-new/.tasks/RES-000-resource-management.md b/.tasks/RES-000-resource-management.md similarity index 100% rename from core-new/.tasks/RES-000-resource-management.md rename to .tasks/RES-000-resource-management.md diff --git a/core-new/.tasks/RES-001-adaptive-throttling.md b/.tasks/RES-001-adaptive-throttling.md similarity index 100% rename from core-new/.tasks/RES-001-adaptive-throttling.md rename to .tasks/RES-001-adaptive-throttling.md diff --git a/core-new/.tasks/SEARCH-000-temporal-semantic-search.md b/.tasks/SEARCH-000-temporal-semantic-search.md similarity index 100% rename from core-new/.tasks/SEARCH-000-temporal-semantic-search.md rename to .tasks/SEARCH-000-temporal-semantic-search.md diff --git a/core-new/.tasks/SEARCH-001-async-searchjob.md b/.tasks/SEARCH-001-async-searchjob.md similarity index 100% rename from core-new/.tasks/SEARCH-001-async-searchjob.md rename to .tasks/SEARCH-001-async-searchjob.md diff --git a/core-new/.tasks/SEARCH-002-two-stage-fts-semantic-reranking.md b/.tasks/SEARCH-002-two-stage-fts-semantic-reranking.md similarity index 100% rename from core-new/.tasks/SEARCH-002-two-stage-fts-semantic-reranking.md rename to .tasks/SEARCH-002-two-stage-fts-semantic-reranking.md diff --git a/core-new/.tasks/SEARCH-003-unified-vector-repositories.md b/.tasks/SEARCH-003-unified-vector-repositories.md similarity index 100% rename from core-new/.tasks/SEARCH-003-unified-vector-repositories.md rename to .tasks/SEARCH-003-unified-vector-repositories.md diff --git a/core-new/.tasks/SEC-000-security-and-privacy.md b/.tasks/SEC-000-security-and-privacy.md similarity index 100% rename from core-new/.tasks/SEC-000-security-and-privacy.md rename to .tasks/SEC-000-security-and-privacy.md diff --git a/core-new/.tasks/SEC-002-database-encryption.md b/.tasks/SEC-002-database-encryption.md similarity index 100% rename from core-new/.tasks/SEC-002-database-encryption.md rename to .tasks/SEC-002-database-encryption.md diff --git a/core-new/.tasks/SEC-003-cryptographic-audit-log.md b/.tasks/SEC-003-cryptographic-audit-log.md similarity index 100% rename from core-new/.tasks/SEC-003-cryptographic-audit-log.md rename to .tasks/SEC-003-cryptographic-audit-log.md diff --git a/core-new/.tasks/SEC-004-rbac-system.md b/.tasks/SEC-004-rbac-system.md similarity index 100% rename from core-new/.tasks/SEC-004-rbac-system.md rename to .tasks/SEC-004-rbac-system.md diff --git a/core-new/.tasks/SEC-005-secure-credential-vault.md b/.tasks/SEC-005-secure-credential-vault.md similarity index 100% rename from core-new/.tasks/SEC-005-secure-credential-vault.md rename to .tasks/SEC-005-secure-credential-vault.md diff --git a/core-new/.tasks/SEC-006-certificate-pinning.md b/.tasks/SEC-006-certificate-pinning.md similarity index 100% rename from core-new/.tasks/SEC-006-certificate-pinning.md rename to .tasks/SEC-006-certificate-pinning.md diff --git a/core-new/.tasks/SEC-007-per-library-encryption-policies-for-public-sharing.md b/.tasks/SEC-007-per-library-encryption-policies-for-public-sharing.md similarity index 100% rename from core-new/.tasks/SEC-007-per-library-encryption-policies-for-public-sharing.md rename to .tasks/SEC-007-per-library-encryption-policies-for-public-sharing.md diff --git a/core-new/.tasks/VOL-000-volume-operations.md b/.tasks/VOL-000-volume-operations.md similarity index 100% rename from core-new/.tasks/VOL-000-volume-operations.md rename to .tasks/VOL-000-volume-operations.md diff --git a/core-new/.tasks/VOL-001-volume-physicalclass-and-location-logicalclass.md b/.tasks/VOL-001-volume-physicalclass-and-location-logicalclass.md similarity index 100% rename from core-new/.tasks/VOL-001-volume-physicalclass-and-location-logicalclass.md rename to .tasks/VOL-001-volume-physicalclass-and-location-logicalclass.md diff --git a/core-new/.tasks/VOL-002-automatic-volume-classification.md b/.tasks/VOL-002-automatic-volume-classification.md similarity index 100% rename from core-new/.tasks/VOL-002-automatic-volume-classification.md rename to .tasks/VOL-002-automatic-volume-classification.md diff --git a/core-new/.tasks/VOL-003-intelligent-storage-tiering-warning-system.md b/.tasks/VOL-003-intelligent-storage-tiering-warning-system.md similarity index 100% rename from core-new/.tasks/VOL-003-intelligent-storage-tiering-warning-system.md rename to .tasks/VOL-003-intelligent-storage-tiering-warning-system.md diff --git a/core-new/.tasks/VOL-004-remote-volume-indexing-with-opendal.md b/.tasks/VOL-004-remote-volume-indexing-with-opendal.md similarity index 100% rename from core-new/.tasks/VOL-004-remote-volume-indexing-with-opendal.md rename to .tasks/VOL-004-remote-volume-indexing-with-opendal.md diff --git a/core-new/.tasks/VOL-005-treat-connected-iphone-as-a-virtual-volume-for-direct-import.md b/.tasks/VOL-005-treat-connected-iphone-as-a-virtual-volume-for-direct-import.md similarity index 100% rename from core-new/.tasks/VOL-005-treat-connected-iphone-as-a-virtual-volume-for-direct-import.md rename to .tasks/VOL-005-treat-connected-iphone-as-a-virtual-volume-for-direct-import.md diff --git a/core-new/.tasks/VSS-003-reference-sidecars-for-live-photo-support.md b/.tasks/VSS-003-reference-sidecars-for-live-photo-support.md similarity index 100% rename from core-new/.tasks/VSS-003-reference-sidecars-for-live-photo-support.md rename to .tasks/VSS-003-reference-sidecars-for-live-photo-support.md diff --git a/core-new/.tasks/task.schema.json b/.tasks/task.schema.json similarity index 100% rename from core-new/.tasks/task.schema.json rename to .tasks/task.schema.json diff --git a/README.md b/README.md index c986b5f21..259bb00bb 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ What started as an ambitious vision became an engineering lesson. Now we're ship
> **The Revolution** -> +> > Copy files between your iPhone and MacBook as easily as moving between folders. Search across all your devices with a single query. Organize photos that live anywhere. **Device boundaries disappear.**

@@ -50,9 +50,9 @@ What started as an ambitious vision became an engineering lesson. Now we're ship ## The Vision Realized -**Copy iPhone video to MacBook storage?** Done. -**Search across all devices instantly?** Built-in. -**Organize files that live everywhere?** Native. +**Copy iPhone video to MacBook storage?** Done. +**Search across all devices instantly?** Built-in. +**Organize files that live everywhere?** Native. **Keep it private and lightning fast?** Always. The original Spacedrive captured imaginations with a bold promise: the **Virtual Distributed File System**. Manage all your files across all your devices as if they were one giant drive. We delivered impressive file management, but the revolutionary cross-device magic remained just out of reach. @@ -64,21 +64,25 @@ The original Spacedrive captured imaginations with a bold promise: the **Virtual Your files are scattered across devices, cloud services, and external drives. Traditional file managers trap you in local boundaries. Spacedrive makes those boundaries disappear: **🌐 Universal File Access** + - Browse files on any device from any device - External drives, cloud storage, remote servers - all unified - Offline files show up with cached metadata **⚡ Lightning Search** + - Find files across all locations with a single search - Content search inside documents, PDFs, and media - AI-powered semantic search: "find sunset photos from vacation" **🔄 Seamless Operations** + - Copy, move, and organize files between any devices - Drag and drop across device boundaries - Batch operations on distributed collections **🔒 Privacy First** + - Your data stays on your devices - Optional cloud sync, never required - End-to-end encryption for all transfers @@ -88,12 +92,14 @@ Your files are scattered across devices, cloud services, and external drives. Tr The original Spacedrive got 500,000 installs because the vision was right. Development paused because the execution was flawed: ### The Problems (2022-2024) + - **Split personality**: Couldn't copy between different location types - **Search limitations**: Basic filename matching, not true content discovery - **Technical debt**: Built on foundations that couldn't scale - **Feature paralysis**: Perfect became the enemy of good ### The Breakthrough (2024-2025) + - **Unified experience**: Every operation works everywhere - **Real search**: Content indexing, semantic understanding, instant results - **Modern foundation**: Built for performance and extensibility @@ -118,6 +124,7 @@ We kept the revolutionary vision. We rebuilt the foundation to deliver it. ``` **Cross-device operations made simple:** + - Drag photos from your iPhone to external storage - Search finds files regardless of which device they're on - Organize distributed media collections as if they were local @@ -138,8 +145,9 @@ spacedrive server --host 0.0.0.0 --port 8080 ``` **Perfect for:** + - **Creators**: Manage media across multiple workstations -- **Developers**: Sync projects between dev environments +- **Developers**: Sync projects between dev environments - **Families**: Shared photo organization across devices - **Self-hosters**: Private cloud with true file management @@ -150,6 +158,7 @@ Access your files from any browser, anywhere. Full Spacedrive functionality with ## Architecture: Built to Last ### Self-Contained Libraries + ``` My Photos.sdlibrary/ ├── library.json # Configuration & device registry @@ -159,11 +168,13 @@ My Photos.sdlibrary/ ``` **Portable by design:** + - **Backup** = copy the folder -- **Share** = send the folder +- **Share** = send the folder - **Migrate** = move the folder ### Unified Operations + No more confusion between "indexed" and "direct" files. Every file operation works the same way: - **Indexed locations**: Rich metadata, lightning search, smart organization @@ -171,16 +182,18 @@ No more confusion between "indexed" and "direct" files. Every file operation wor - **Hybrid mode**: Best of both worlds automatically ### Real Search Engine + ``` 🔍 Search: "sunset photos from vacation" Results across all devices: 📱 iPhone/Photos/Vacation2024/sunset_beach.jpg -💾 External/Backup/2024/vacation_sunset.mov +💾 External/Backup/2024/vacation_sunset.mov ☁️ iCloud/Memories/golden_hour_sunset.heic ``` **Beyond filename matching:** + - Full-text content search in documents - Image recognition and scene detection - Vector search for semantic queries @@ -189,24 +202,28 @@ Results across all devices: ## What's Shipping: The VDFS Roadmap ### Q1 2025: Foundation + - ✅ **Core rewrite** with unified file system - ✅ **Working CLI** with daemon architecture - 🚧 **Desktop app** rebuilt on new foundation - 🚧 **Real search** with content indexing -### Q2 2025: Device Communication +### Q2 2025: Device Communication + - 🔄 **P2P discovery** and secure connections - 🔄 **Cross-device operations** (copy, move, sync) - 🔄 **Mobile apps** with desktop feature parity - 🔄 **Web interface** for universal access ### Q3 2025: Intelligence + - 🎯 **AI-powered organization** with local models - 🎯 **Smart collections** and auto-tagging - 🎯 **Cloud integrations** (iCloud, Google Drive, etc.) - 🎯 **Advanced media analysis** ### Q4 2025: Ecosystem + - 🚀 **Extension system** for community features - 🚀 **Professional tools** for creators and teams - 🚀 **Enterprise features** and compliance @@ -236,6 +253,7 @@ spacedrive job monitor ``` **Working today:** + - ✅ Multi-location management - ✅ Smart indexing with progress tracking - ✅ Content-aware search @@ -245,18 +263,21 @@ spacedrive job monitor ## Sustainable Open Source ### Always Free & Open + - **Core file management** and VDFS operations - **Local search** and organization features - **P2P sync** between your own devices - **Privacy-first** architecture ### Premium Value-Adds + - **Spacedrive Cloud**: Cross-internet sync and backup - **Advanced AI**: Professional media analysis and organization - **Team features**: Shared libraries and collaboration - **Enterprise**: SSO, compliance, and enterprise deployment ### Community First + - **Weekly dev streams** showing real progress - **Open roadmap** with community voting - **Contributor rewards** and recognition program @@ -265,20 +286,26 @@ spacedrive job monitor ## Why It Will Work This Time ### Technical Maturity + From 500k installs and 34k stars, we learned what users actually need: + - **Performance first**: Sub-second search, responsive UI, efficient sync - **Reliability**: Robust error handling, data integrity, graceful failures - **Simplicity**: Complex features with simple interfaces ### Market Reality + The world has changed since 2022: + - **Privacy concerns** have intensified with cloud services - **AI expectations** for semantic search and smart organization - **Multi-device life** is now universal, not niche - **Creator economy** needs professional file management tools ### Execution Discipline + No more feature paralysis: + - **Ship working features**, enhance over time - **Measure real usage**, not just code metrics - **Community feedback** drives priority decisions @@ -287,18 +314,21 @@ No more feature paralysis: ## Get Involved ### For Users + - ⭐ **Star the repo** to follow development - 💬 **Join Discord** for updates and early access - 🐛 **Report issues** and request features - 📖 **Beta testing** as features ship ### For Developers + - 🔧 **Contribute code** to the core rewrite - 📚 **Improve docs** and tutorials - 🧪 **Write tests** and benchmarks - 🎨 **Design interfaces** for new features ### For Organizations + - 💼 **Early access** to enterprise features - 🤝 **Partnership** opportunities - 💰 **Sponsorship** and development funding @@ -320,12 +350,12 @@ The future of file management isn't about better folder hierarchies or cloud sto

Follow the comeback
- Website · - Discord · + Website · + Discord · Twitter · - Core Development + Core Development

The file manager that should exist. Finally being built right. -

\ No newline at end of file +

diff --git a/spacedrive-cloud b/apps/cloud similarity index 100% rename from spacedrive-cloud rename to apps/cloud diff --git a/combine_paths.py b/combine_paths.py deleted file mode 100755 index 9f8e32b3c..000000000 --- a/combine_paths.py +++ /dev/null @@ -1,72 +0,0 @@ -#!/usr/bin/env python3 -import os -import sys -import argparse -from pathlib import Path - -def combine_paths(paths, output_file): - """Combine multiple paths into a single text file.""" - with open(output_file, 'w', encoding='utf-8') as out: - for path_str in paths: - path = Path(path_str) - - if not path.exists(): - print(f"Warning: {path} does not exist, skipping...") - continue - - out.write(f"\n{'='*80}\n") - out.write(f"PATH: {path}\n") - out.write(f"{'='*80}\n\n") - - if path.is_file(): - try: - with open(path, 'r', encoding='utf-8') as f: - out.write(f.read()) - out.write('\n\n') - except UnicodeDecodeError: - out.write(f"[Binary file - skipped]\n\n") - except Exception as e: - out.write(f"[Error reading file: {e}]\n\n") - - elif path.is_dir(): - for root, dirs, files in os.walk(path): - # Skip common ignored directories - dirs[:] = [d for d in dirs if not d.startswith('.') and d not in ['node_modules', '__pycache__', 'target', 'dist', 'build']] - - for file in files: - if file.startswith('.'): - continue - - file_path = Path(root) / file - out.write(f"\n{'-'*60}\n") - out.write(f"FILE: {file_path}\n") - out.write(f"{'-'*60}\n\n") - - try: - with open(file_path, 'r', encoding='utf-8') as f: - out.write(f.read()) - out.write('\n\n') - except UnicodeDecodeError: - out.write(f"[Binary file - skipped]\n\n") - except Exception as e: - out.write(f"[Error reading file: {e}]\n\n") - -def main(): - parser = argparse.ArgumentParser(description='Combine multiple paths into a single text file') - parser.add_argument('paths', nargs='+', help='Paths to combine') - parser.add_argument('-o', '--output', help='Output file name (default: foldername.txt based on first path)') - - args = parser.parse_args() - - if args.output: - output_file = args.output - else: - first_path = Path(args.paths[0]) - folder_name = first_path.name if first_path.is_dir() else first_path.stem - output_file = f"{folder_name}.txt" - - combine_paths(args.paths, output_file) - print(f"Combined {len(args.paths)} paths into {output_file}") - -if __name__ == "__main__": - main() \ No newline at end of file diff --git a/core-new/Cargo.toml b/core-new/Cargo.toml deleted file mode 100644 index 817954111..000000000 --- a/core-new/Cargo.toml +++ /dev/null @@ -1,161 +0,0 @@ -[package] -name = "sd-core-new" -version = "0.1.0" -edition = "2021" - -[features] -default = [] -# FFmpeg support for video thumbnails -ffmpeg = ["dep:sd-ffmpeg"] - -[workspace] -members = [ - "benchmarks", - "task-validator", -] - -[dependencies] -# Async runtime -tokio = { version = "1.40", features = ["full"] } -futures = "0.3" -async-trait = "0.1" - -# Database -sea-orm = { version = "1.1", features = ["sqlx-sqlite", "runtime-tokio-rustls", "macros", "uuid", "with-json", "with-chrono"] } -sea-orm-migration = { version = "1.1", features = ["runtime-tokio-rustls", "sqlx-sqlite"] } -sqlx = { version = "0.8", features = ["runtime-tokio-rustls", "sqlite"] } - -# API (temporarily disabled) -# axum = "0.7" -# async-graphql = "7.0" -# async-graphql-axum = "7.0" - -# Serialization -serde = { version = "1.0", features = ["derive"] } -serde_json = "1.0" -toml = "0.8" -int-enum = "1.1" -strum = { version = "0.26", features = ["derive"] } - -# Error handling -thiserror = "1.0" -anyhow = "1.0" - - -# File operations -notify = "6.1" # File system watching -blake3 = "1.5" # Content addressing -sha2 = "0.10" # SHA-256 hashing for CAS IDs -hex = "0.4" # Hex encoding for volume fingerprints - -# Logging -tracing = "0.1" -tracing-subscriber = { version = "0.3", features = ["env-filter"] } - -# Indexer rules engine -globset = { version = "0.4", features = ["serde1"] } -gix-ignore = { version = "0.11", features = ["serde"] } -futures-concurrency = "7.6" - -# Job system dependencies -rmp = "0.8" # MessagePack core types -rmp-serde = "1.3" # MessagePack serialization for job state -inventory = "0.3" # Automatic job registration -sd-task-system = { path = "../crates/task-system" } -spacedrive-jobs-derive = { path = "spacedrive-jobs-derive" } # Job derive macros - -# Media processing dependencies -sd-images = { path = "../crates/images" } -sd-ffmpeg = { path = "../crates/ffmpeg", optional = true } -webp = "0.3" -image = "0.25" -tokio-rustls = "0.26" - -# Networking -# Iroh P2P networking -iroh = "0.28" -iroh-net = "0.28" -iroh-blobs = "0.28" -iroh-gossip = "0.28" - -# Serialization for protocols -serde_cbor = "0.11" - -# Cryptography for signing (backward compatibility) -ed25519-dalek = "2.1" - -# Legacy networking (kept for compatibility during transition) -mdns-sd = "0.13" # mDNS service discovery (DEPRECATED - use libp2p DHT) -snow = "0.9" # Noise Protocol encryption (DEPRECATED - use libp2p noise) -ring = "0.16" # Crypto primitives -argon2 = "0.5" # Password derivation -aes-gcm = "0.10" # AES-GCM encryption for secure storage -async-stream = "0.3" # File streaming -backoff = "0.4" # Retry logic -bincode = "2.0.0-rc.3" # Efficient encoding -# futures-util = "0.3" # WebSocket utilities (disabled for now) -rustls = { version = "0.23", features = ["aws_lc_rs"] } # TLS implementation (DEPRECATED - use libp2p noise) -rcgen = "0.11" # Certificate generation (DEPRECATED - use libp2p noise) -tokio-stream = "0.1" # Async streams - -# BIP39 wordlist support -bip39 = "2.0" - -# Additional cryptography -chacha20poly1305 = "0.10" # Authenticated encryption for chunk-level security -hkdf = "0.12" # Key derivation function for session keys -x25519-dalek = "2.0" -hmac = "0.12" - -# Network utilities -if-watch = "3.0" -local-ip-address = "0.5" -# colored already defined above - -# Utils -uuid = { version = "1.11", features = ["v4", "v5", "v7", "serde"] } -chrono = { version = "0.4", features = ["serde"] } -once_cell = "1.20" -dirs = "5.0" -whoami = "1.5" -rand = "0.8" # Random number generation for secure delete -tempfile = "3.14" # Temporary directories for testing - -# Secure storage -keyring = "3.6" - -# CLI dependencies -clap = { version = "4.5", features = ["derive", "env"] } -comfy-table = "7.1" -dialoguer = "0.11" -indicatif = "0.17" -owo-colors = "4.1" -supports-color = "3.0" -console = "0.15" -colored = "2.1" - -[build-dependencies] -vergen = { version = "8", features = ["git", "gitcl", "cargo"] } -ratatui = "0.29" -crossterm = "0.28" -indicatif = "0.17" -console = "0.15" -dialoguer = "0.11" -colored = "2.1" -comfy-table = "7.1" -owo-colors = "4.1" -supports-color = "3.0" - -# Platform specific -[target.'cfg(unix)'.dependencies] -libc = "0.2" - -[[bin]] -name = "spacedrive" -path = "src/bin/cli.rs" - - - -[dev-dependencies] -tempfile = "3.14" -pretty_assertions = "1.4" diff --git a/core-new/benchmarks/Cargo.toml b/core-new/benchmarks/Cargo.toml deleted file mode 100644 index 86a11c859..000000000 --- a/core-new/benchmarks/Cargo.toml +++ /dev/null @@ -1,39 +0,0 @@ -[package] -name = "sd-bench" -version = "0.1.0" -edition = "2021" - -[dependencies] -anyhow = "1.0" -serde = { version = "1.0", features = ["derive"] } -serde_yaml = "0.9" -serde_json = "1.0" -clap = { version = "4.5", features = ["derive", "env"] } -tokio = { version = "1.40", features = ["full"] } -tracing = "0.1" -tracing-subscriber = { version = "0.3", features = ["env-filter"] } -rand = "0.8" -regex = "1.10" -walkdir = "2.5" -indicatif = "0.17" -humantime-serde = "1.1" -humantime = "2.1" -chrono = { version = "0.4", features = ["serde"] } -dirs = "5.0" -uuid = { version = "1.11", features = ["v4", "serde"] } -tempfile = "3.14" -sd-core-new = { path = ".." } -serde_with = { version = "3.9", features = ["json"] } -async-trait = "0.1" -blake3 = "1.5" -sysinfo = { version = "0.30", default-features = false, features = ["multithread"] } - -[lib] -name = "sd_bench" -path = "src/lib.rs" - -[[bin]] -name = "sd-bench" -path = "src/bin/sd-bench-new.rs" - - diff --git a/core-new/build.rs b/core-new/build.rs deleted file mode 100644 index 4b32dbe9b..000000000 --- a/core-new/build.rs +++ /dev/null @@ -1,13 +0,0 @@ -use vergen::EmitBuilder; - -fn main() -> Result<(), Box> { - // Emit the instructions - EmitBuilder::builder() - .git_sha(true) - .git_commit_timestamp() - .git_branch() - .cargo_opt_level() - .cargo_target_triple() - .emit()?; - Ok(()) -} diff --git a/core-new/examples/indexing_showcase.rs b/core-new/examples/indexing_showcase.rs deleted file mode 100644 index 18a6e0218..000000000 --- a/core-new/examples/indexing_showcase.rs +++ /dev/null @@ -1,136 +0,0 @@ -//! Showcase of the production-ready indexer implementation -//! -//! This example demonstrates the sophisticated features of our new indexer: -//! - Multi-phase processing (Discovery → Processing → Content) -//! - Hardcoded filtering with should_skip_path -//! - Incremental indexing with inode tracking -//! - Performance metrics and reporting -//! - Full resumability with checkpoints - -use std::path::Path; - -fn main() { - println!("🚀 Spacedrive Production Indexer Showcase\n"); - - // Demonstrate the filtering system - showcase_filtering(); - - // Show the modular architecture - showcase_architecture(); - - // Display sample metrics output - showcase_metrics(); -} - -fn showcase_filtering() { - println!("📁 Smart Filtering System"); - println!("========================\n"); - - // Import the actual function from our implementation - use sd_core_new::operations::indexing::filters::should_skip_path; - - let test_paths = vec![ - // Files that should be skipped - (".DS_Store", true, "macOS system file"), - ("Thumbs.db", true, "Windows thumbnail cache"), - ("node_modules", true, "npm packages directory"), - (".git", true, "Git repository data"), - ("target", true, "Rust build directory"), - ("__pycache__", true, "Python cache"), - (".mypy_cache", true, "Python type checker cache"), - - // Files that should NOT be skipped - ("document.pdf", false, "Regular document"), - ("photo.jpg", false, "Image file"), - ("src", false, "Source code directory"), - (".config", false, "User config directory (allowed)"), - ("project.rs", false, "Rust source file"), - ]; - - println!("Testing path filtering:"); - for (path_str, should_skip, description) in test_paths { - let path = Path::new(path_str); - let skipped = should_skip_path(path); - let result = if skipped == should_skip { "✅" } else { "❌" }; - println!(" {} {:20} -> {:8} ({})", - result, - path_str, - if skipped { "SKIP" } else { "INDEX" }, - description - ); - } - - println!("\n💡 Note: This is where the future IndexerRuleEngine will integrate!"); - println!(" The should_skip_path function has a clear TODO marker for rules system.\n"); -} - -fn showcase_architecture() { - println!("🏗️ Modular Architecture"); - println!("=======================\n"); - - println!("core-new/src/operations/indexing/"); - println!("├── mod.rs # Module exports and documentation"); - println!("├── job.rs # Main IndexerJob with state machine"); - println!("├── state.rs # Resumable state management"); - println!("├── entry.rs # Entry processing with inode support"); - println!("├── filters.rs # Hardcoded filtering (→ future rules)"); - println!("├── metrics.rs # Performance tracking"); - println!("├── change_detection/ # Incremental indexing"); - println!("│ └── mod.rs # Inode-based change detection"); - println!("└── phases/ # Multi-phase processing"); - println!(" ├── discovery.rs # Directory walking"); - println!(" ├── processing.rs # Database operations"); - println!(" └── content.rs # CAS ID generation\n"); - - println!("Key Features:"); - println!("✅ Full resumability with checkpoint system"); - println!("✅ Inode tracking for move/rename detection"); - println!("✅ Batch processing (1000 items per batch)"); - println!("✅ Non-critical error collection"); - println!("✅ Path prefix optimization"); - println!("✅ Content deduplication ready\n"); -} - -fn showcase_metrics() { - println!("📊 Performance Metrics"); - println!("=====================\n"); - - // Show what metrics output looks like - let sample_output = r#"Indexing completed in 12.5s: -- Files: 10,234 (818.7/s) -- Directories: 1,523 (121.8/s) -- Total size: 2.34 GB (191.23 MB/s) -- Database writes: 10,234 in 11 batches (avg 930.4 items/batch) -- Errors: 5 (skipped 1,523 paths) -- Phase timing: discovery 5.2s, processing 6.1s, content 1.2s"#; - - println!("Sample metrics output:"); - println!("{}\n", sample_output); - - // Show the indexer progress phases - println!("Progress Tracking Phases:"); - println!("1️⃣ Discovery: 'Found 245 entries in /Users/demo/Documents'"); - println!("2️⃣ Processing: 'Batch 3/11' (database operations)"); - println!("3️⃣ Content: 'Generating content identities (456/1234)'"); - println!("4️⃣ Finalizing: 'Cleaning up and saving final state'\n"); - - // Show change detection in action - println!("🔄 Incremental Indexing Example:"); - println!("First run: Indexed 5,000 files"); - println!("Second run: Detected 3 new, 5 modified, 2 moved files"); - println!(" Only processed 10 files instead of 5,000!"); - println!(" Used inode tracking to detect moves efficiently\n"); -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_showcase_runs() { - // Just verify our showcase compiles and runs - showcase_filtering(); - showcase_architecture(); - showcase_metrics(); - } -} \ No newline at end of file diff --git a/core-new/src/context.rs b/core-new/src/context.rs deleted file mode 100644 index bb656e2a0..000000000 --- a/core-new/src/context.rs +++ /dev/null @@ -1,78 +0,0 @@ -//! Shared context providing access to core application components. - -//! Shared context providing access to core application components. - -use crate::{ - config::JobLoggingConfig, - device::DeviceManager, infrastructure::events::EventBus, - keys::library_key_manager::LibraryKeyManager, library::LibraryManager, - infrastructure::actions::manager::ActionManager, - services::networking::NetworkingService, volume::VolumeManager, -}; -use std::{path::PathBuf, sync::Arc}; -use tokio::sync::RwLock; - -/// Shared context providing access to core application components. -#[derive(Clone)] -pub struct CoreContext { - pub events: Arc, - pub device_manager: Arc, - pub library_manager: Arc, - pub volume_manager: Arc, - pub library_key_manager: Arc, - // This is wrapped in an RwLock to allow it to be set after initialization - pub action_manager: Arc>>>, - pub networking: Arc>>>, - // Job logging configuration - pub job_logging_config: Option, - pub job_logs_dir: Option, -} - -impl CoreContext { - /// Create a new context with the given components - pub fn new( - events: Arc, - device_manager: Arc, - library_manager: Arc, - volume_manager: Arc, - library_key_manager: Arc, - ) -> Self { - Self { - events, - device_manager, - library_manager, - volume_manager, - library_key_manager, - action_manager: Arc::new(RwLock::new(None)), - networking: Arc::new(RwLock::new(None)), - job_logging_config: None, - job_logs_dir: None, - } - } - - /// Set job logging configuration - pub fn set_job_logging(&mut self, config: JobLoggingConfig, logs_dir: PathBuf) { - self.job_logging_config = Some(config); - self.job_logs_dir = Some(logs_dir); - } - - /// Helper method for services to get the networking service - pub async fn get_networking(&self) -> Option> { - self.networking.read().await.clone() - } - - /// Method for Core to set networking after it's initialized - pub async fn set_networking(&self, networking: Arc) { - *self.networking.write().await = Some(networking); - } - - /// Helper method to get the action manager - pub async fn get_action_manager(&self) -> Option> { - self.action_manager.read().await.clone() - } - - /// Method for Core to set action manager after it's initialized - pub async fn set_action_manager(&self, action_manager: Arc) { - *self.action_manager.write().await = Some(action_manager); - } -} diff --git a/core-new/src/lib.rs b/core-new/src/lib.rs deleted file mode 100644 index 649adfe4d..000000000 --- a/core-new/src/lib.rs +++ /dev/null @@ -1,499 +0,0 @@ -#![allow(warnings)] -//! Spacedrive Core v2 -//! -//! A unified, simplified architecture for cross-platform file management. - -pub mod config; -pub mod context; -pub mod device; -pub mod domain; -pub mod file_type; -pub mod infrastructure; -pub mod keys; -pub mod library; -pub mod location; -pub mod operations; -pub mod services; -pub mod shared; -pub mod test_framework; -pub mod volume; - -use services::networking::protocols::PairingProtocolHandler; -use services::networking::utils::logging::NetworkLogger; - -// Compatibility module for legacy networking references -pub mod networking { - pub use crate::services::networking::*; -} - -use crate::config::AppConfig; -use crate::context::CoreContext; -use crate::device::DeviceManager; -use crate::infrastructure::actions::manager::ActionManager; -use crate::infrastructure::events::{Event, EventBus}; -use crate::library::LibraryManager; -use crate::services::Services; -use crate::volume::{VolumeDetectionConfig, VolumeManager}; -use std::path::PathBuf; -use std::sync::Arc; -use tokio::sync::{mpsc, RwLock}; -use tracing::{error, info}; - -/// Pending pairing request information -#[derive(Debug, Clone)] -pub struct PendingPairingRequest { - pub request_id: uuid::Uuid, - pub device_id: uuid::Uuid, - pub device_name: String, - pub received_at: chrono::DateTime, -} - -/// Spacedrop request message -#[derive(serde::Serialize, serde::Deserialize)] -struct SpacedropRequest { - transfer_id: uuid::Uuid, - file_path: String, - sender_name: String, - message: Option, - file_size: u64, -} - -// NOTE: SimplePairingUI has been moved to CLI infrastructure -// See: src/infrastructure/cli/pairing_ui.rs for CLI-specific implementations - -/// Bridge between networking events and core events -pub struct NetworkEventBridge { - network_events: mpsc::UnboundedReceiver, - core_events: Arc, -} - -impl NetworkEventBridge { - pub fn new( - network_events: mpsc::UnboundedReceiver, - core_events: Arc, - ) -> Self { - Self { - network_events, - core_events, - } - } - - pub async fn run(mut self) { - while let Some(event) = self.network_events.recv().await { - if let Some(core_event) = self.translate_event(event) { - self.core_events.emit(core_event); - } - } - } - - fn translate_event(&self, event: networking::NetworkEvent) -> Option { - match event { - networking::NetworkEvent::ConnectionEstablished { device_id, .. } => { - Some(Event::DeviceConnected { - device_id, - device_name: "Connected Device".to_string(), - }) - } - networking::NetworkEvent::ConnectionLost { device_id, .. } => { - Some(Event::DeviceDisconnected { device_id }) - } - networking::NetworkEvent::PairingCompleted { - device_id, - device_info, - } => Some(Event::DeviceConnected { - device_id, - device_name: device_info.device_name, - }), - _ => None, // Some events don't map to core events - } - } -} - -/// The main context for all core operations -pub struct Core { - /// Application configuration - pub config: Arc>, - - /// Device manager - pub device: Arc, - - /// Library manager - pub libraries: Arc, - - /// Volume manager - pub volumes: Arc, - - /// Event bus for state changes - pub events: Arc, - - /// Container for high-level services - pub services: Services, - - /// Shared context for core components - pub context: Arc, -} - -impl Core { - /// Initialize a new Core instance with default data directory - pub async fn new() -> Result> { - let data_dir = crate::config::default_data_dir()?; - Self::new_with_config(data_dir).await - } - - /// Initialize a new Core instance with custom data directory - pub async fn new_with_config(data_dir: PathBuf) -> Result> { - info!("Initializing Spacedrive Core at {:?}", data_dir); - - // 1. Load or create app config - let config = AppConfig::load_or_create(&data_dir)?; - config.ensure_directories()?; - let config = Arc::new(RwLock::new(config)); - - // 2. Initialize device manager - let device = Arc::new(DeviceManager::init_with_path(&data_dir)?); - // Set the global device ID for legacy compatibility - shared::utils::set_current_device_id(device.device_id()?); - - // 3. Create event bus - let events = Arc::new(EventBus::default()); - - // 4. Initialize volume manager - let volume_config = VolumeDetectionConfig::default(); - let device_id = device.device_id()?; - let volumes = Arc::new(VolumeManager::new(device_id, volume_config, events.clone())); - - // 5. Initialize volume detection - // info!("Initializing volume detection..."); - // match volumes.initialize().await { - // Ok(()) => info!("Volume manager initialized"), - // Err(e) => error!("Failed to initialize volume manager: {}", e), - // } - - // 6. Initialize library manager with libraries directory - let libraries_dir = config.read().await.libraries_dir(); - let libraries = Arc::new(LibraryManager::new_with_dir(libraries_dir, events.clone())); - - // 7. Initialize library key manager - let library_key_manager = - Arc::new(crate::keys::library_key_manager::LibraryKeyManager::new()?); - - // 8. Register all job types - info!("Registering job types..."); - crate::operations::register_all_jobs(); - info!("Job types registered"); - - // 9. Create the context that will be shared with services - let mut context_inner = CoreContext::new( - events.clone(), - device.clone(), - libraries.clone(), - volumes.clone(), - library_key_manager.clone(), - ); - - // Set job logging configuration if enabled - let app_config = config.read().await; - if app_config.job_logging.enabled { - context_inner - .set_job_logging(app_config.job_logging.clone(), app_config.job_logs_dir()); - } - drop(app_config); - - let context = Arc::new(context_inner); - - // 10. Initialize services first, passing them the context - let services = Services::new(context.clone()); - - // 11. Auto-load all libraries with context for job manager initialization - info!("Loading existing libraries..."); - let loaded_libraries: Vec> = - match libraries.load_all_with_context(context.clone()).await { - Ok(count) => { - info!("Loaded {} libraries", count); - libraries.list().await - } - Err(e) => { - error!("Failed to load libraries: {}", e); - vec![] - } - }; - - // Initialize sidecar manager for each loaded library - for library in &loaded_libraries { - info!("Initializing sidecar manager for library {}", library.id()); - if let Err(e) = services.sidecar_manager.init_library(&library).await { - error!( - "Failed to initialize sidecar manager for library {}: {}", - library.id(), - e - ); - } else { - // Run bootstrap scan - if let Err(e) = services.sidecar_manager.bootstrap_scan(&library).await { - error!( - "Failed to run sidecar bootstrap scan for library {}: {}", - library.id(), - e - ); - } - } - } - - info!("Starting background services..."); - match services.start_all().await { - Ok(()) => info!("Background services started"), - Err(e) => error!("Failed to start services: {}", e), - } - - // 12. Initialize ActionManager and set it in context - let action_manager = Arc::new(crate::infrastructure::actions::manager::ActionManager::new( - context.clone(), - )); - context.set_action_manager(action_manager).await; - - // 13. Emit startup event - events.emit(Event::CoreStarted); - - Ok(Self { - config, - device, - libraries, - volumes, - events, - services, - context, - }) - } - - /// Get the application configuration - pub fn config(&self) -> Arc> { - self.config.clone() - } - - /// Initialize networking using master key - pub async fn init_networking(&mut self) -> Result<(), Box> { - self.init_networking_with_logger(Arc::new(networking::SilentLogger)) - .await - } - - /// Initialize networking with custom logger - pub async fn init_networking_with_logger( - &mut self, - logger: Arc, - ) -> Result<(), Box> { - logger.info("Initializing networking...").await; - - // Initialize networking service through the services container - let data_dir = self.config.read().await.data_dir.clone(); - self.services - .init_networking( - self.device.clone(), - self.services.library_key_manager.clone(), - data_dir, - ) - .await?; - - // Start the networking service - self.services.start_networking().await?; - - // Get the networking service for protocol registration - if let Some(networking_service) = self.services.networking() { - // Register default protocol handlers - self.register_default_protocols(&networking_service).await?; - - // Set up event bridge to integrate with core event system - let event_bridge = NetworkEventBridge::new( - networking_service - .subscribe_events() - .await - .unwrap_or_else(|| { - let (_, rx) = tokio::sync::mpsc::unbounded_channel(); - rx - }), - self.events.clone(), - ); - tokio::spawn(event_bridge.run()); - - // Make networking service available to the context for other services - self.context.set_networking(networking_service).await; - } - - logger.info("Networking initialized successfully").await; - Ok(()) - } - - /// Register default protocol handlers - async fn register_default_protocols( - &self, - networking: &networking::NetworkingService, - ) -> Result<(), Box> { - let logger = std::sync::Arc::new(networking::utils::logging::ConsoleLogger); - - // Get command sender for the pairing handler's state machine - let command_sender = networking - .command_sender() - .ok_or("NetworkingEventLoop command sender not available")? - .clone(); - - // Get data directory from config - let data_dir = { - let config = self.config.read().await; - config.data_dir.clone() - }; - - let pairing_handler = Arc::new(networking::protocols::PairingProtocolHandler::new_with_persistence( - networking.identity().clone(), - networking.device_registry(), - logger.clone(), - command_sender, - data_dir, - )); - - // Try to load persisted sessions, but don't fail if there's an error - if let Err(e) = pairing_handler.load_persisted_sessions().await { - logger.warn(&format!("Failed to load persisted pairing sessions: {}. Starting with empty sessions.", e)).await; - } - - // Start the state machine task for pairing - networking::protocols::PairingProtocolHandler::start_state_machine_task( - pairing_handler.clone(), - ); - - // Start cleanup task for expired sessions - networking::protocols::PairingProtocolHandler::start_cleanup_task(pairing_handler.clone()); - - let messaging_handler = networking::protocols::MessagingProtocolHandler::new(); - let mut file_transfer_handler = - networking::protocols::FileTransferProtocolHandler::new_default(logger.clone()); - - // Inject device registry into file transfer handler for encryption - file_transfer_handler.set_device_registry(networking.device_registry()); - - let protocol_registry = networking.protocol_registry(); - { - let mut registry = protocol_registry.write().await; - registry.register_handler(pairing_handler)?; - registry.register_handler(Arc::new(messaging_handler))?; - registry.register_handler(Arc::new(file_transfer_handler))?; - } - - Ok(()) - } - - /// Initialize networking from Arc - for daemon use - pub async fn init_networking_shared( - core: Arc, - ) -> Result, Box> { - info!("Initializing networking for shared core..."); - - // Create a new Core with networking enabled - let mut new_core = - Core::new_with_config(core.config().read().await.data_dir.clone()).await?; - - // Initialize networking on the new core - new_core.init_networking().await?; - - info!("Networking initialized successfully for shared core"); - Ok(Arc::new(new_core)) - } - - /// Get the networking service (if initialized) - pub fn networking(&self) -> Option> { - self.services.networking() - } - - /// Get list of connected devices - pub async fn get_connected_devices( - &self, - ) -> Result, Box> { - Ok(self.services.device.get_connected_devices().await?) - } - - /// Get detailed information about connected devices - pub async fn get_connected_devices_info( - &self, - ) -> Result, Box> { - Ok(self.services.device.get_connected_devices_info().await?) - } - - /// Add a location to the file system watcher - pub async fn add_watched_location( - &self, - location_id: uuid::Uuid, - library_id: uuid::Uuid, - path: std::path::PathBuf, - enabled: bool, - ) -> Result<(), Box> { - use crate::services::location_watcher::WatchedLocation; - - let watched_location = WatchedLocation { - id: location_id, - library_id, - path, - enabled, - }; - - Ok(self - .services - .location_watcher - .add_location(watched_location) - .await?) - } - - /// Remove a location from the file system watcher - pub async fn remove_watched_location( - &self, - location_id: uuid::Uuid, - ) -> Result<(), Box> { - Ok(self - .services - .location_watcher - .remove_location(location_id) - .await?) - } - - /// Update file watching settings for a location - pub async fn update_watched_location( - &self, - location_id: uuid::Uuid, - enabled: bool, - ) -> Result<(), Box> { - Ok(self - .services - .location_watcher - .update_location(location_id, enabled) - .await?) - } - - /// Get all currently watched locations - pub async fn get_watched_locations( - &self, - ) -> Vec { - self.services.location_watcher.get_watched_locations().await - } - - /// Shutdown the core gracefully - pub async fn shutdown(&self) -> Result<(), Box> { - info!("Shutting down Spacedrive Core..."); - - // Networking service is stopped by services.stop_all() - - // Stop all services - self.services.stop_all().await?; - - // Stop volume monitoring - self.volumes.stop_monitoring().await; - - // Close all libraries - self.libraries.close_all().await?; - - // Save configuration - self.config.write().await.save()?; - - // Emit shutdown event - self.events.emit(Event::CoreShutdown); - - info!("Spacedrive Core shutdown complete"); - Ok(()) - } -} diff --git a/core-new/src/library/config.rs b/core-new/src/library/config.rs deleted file mode 100644 index 207483d25..000000000 --- a/core-new/src/library/config.rs +++ /dev/null @@ -1,208 +0,0 @@ -//! Library configuration types - -use chrono::{DateTime, Utc}; -use serde::{Deserialize, Serialize}; -use uuid::Uuid; - -/// Library configuration stored in library.json -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct LibraryConfig { - /// Version of the configuration format - pub version: u32, - - /// Unique identifier for this library - pub id: Uuid, - - /// Human-readable name - pub name: String, - - /// Optional description - pub description: Option, - - /// When the library was created - pub created_at: DateTime, - - /// When the library was last modified - pub updated_at: DateTime, - - /// Library-specific settings - pub settings: LibrarySettings, - - /// Library statistics - pub statistics: LibraryStatistics, -} - -/// Library-specific settings -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct LibrarySettings { - /// Whether to generate thumbnails for media files - pub generate_thumbnails: bool, - - /// Thumbnail quality (0-100) - pub thumbnail_quality: u8, - - /// Whether to enable AI-powered tagging - pub enable_ai_tagging: bool, - - /// Whether sync is enabled for this library - pub sync_enabled: bool, - - /// Whether the library is encrypted at rest - pub encryption_enabled: bool, - - /// Custom thumbnail sizes to generate - pub thumbnail_sizes: Vec, - - /// File extensions to ignore during indexing - pub ignored_extensions: Vec, - - /// Maximum file size to index (in bytes) - pub max_file_size: Option, - - /// Whether to automatically track system volumes - pub auto_track_system_volumes: bool, - - /// Whether to automatically track external volumes when connected - pub auto_track_external_volumes: bool, - - /// Indexer settings (rule toggles and related) - #[serde(default)] - pub indexer: IndexerSettings, -} - -impl LibraryConfig { - /// Load library configuration from a JSON file - pub async fn load(path: &std::path::Path) -> Result { - let config_data = tokio::fs::read_to_string(path) - .await - .map_err(|e| super::error::LibraryError::IoError(e))?; - let config: LibraryConfig = serde_json::from_str(&config_data) - .map_err(|e| super::error::LibraryError::JsonError(e))?; - Ok(config) - } -} - -impl Default for LibrarySettings { - fn default() -> Self { - Self { - generate_thumbnails: true, - thumbnail_quality: 85, - enable_ai_tagging: false, - sync_enabled: false, - encryption_enabled: false, - thumbnail_sizes: vec![128, 256, 512], - ignored_extensions: vec![ - ".tmp".to_string(), - ".temp".to_string(), - ".cache".to_string(), - ".part".to_string(), - ], - max_file_size: Some(100 * 1024 * 1024 * 1024), // 100GB - auto_track_system_volumes: true, // Default to true for user convenience - auto_track_external_volumes: false, // Default to false for privacy - indexer: IndexerSettings::default(), - } - } -} - -/// Indexer settings controlling rule toggles -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct IndexerSettings { - #[serde(default = "IndexerSettings::default_true")] - pub no_system_files: bool, - #[serde(default = "IndexerSettings::default_true")] - pub no_git: bool, - #[serde(default = "IndexerSettings::default_true")] - pub no_dev_dirs: bool, - #[serde(default)] - pub no_hidden: bool, - #[serde(default = "IndexerSettings::default_true")] - pub gitignore: bool, - #[serde(default)] - pub only_images: bool, -} - -impl IndexerSettings { - fn default_true() -> bool { - true - } -} - -impl Default for IndexerSettings { - fn default() -> Self { - Self { - no_system_files: true, - no_git: true, - no_dev_dirs: true, - no_hidden: false, - gitignore: true, - only_images: false, - } - } -} - -/// Library statistics -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct LibraryStatistics { - /// Total number of files indexed - pub total_files: u64, - - /// Total size of all files in bytes - pub total_size: u64, - - /// Number of locations in this library - pub location_count: u32, - - /// Number of tags created - pub tag_count: u32, - - /// Number of thumbnails generated - pub thumbnail_count: u64, - - /// Last time the library was fully indexed - pub last_indexed: Option>, - - /// When these statistics were last updated - pub updated_at: DateTime, -} - -impl Default for LibraryStatistics { - fn default() -> Self { - Self { - total_files: 0, - total_size: 0, - location_count: 0, - tag_count: 0, - thumbnail_count: 0, - last_indexed: None, - updated_at: Utc::now(), - } - } -} - -/// Thumbnail generation metadata -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct ThumbnailMetadata { - /// Version of the thumbnail format - pub version: u32, - - /// Quality setting used for generation - pub quality: u8, - - /// Sizes that were generated - pub sizes: Vec, - - /// When this metadata was created - pub created_at: DateTime, -} - -impl Default for ThumbnailMetadata { - fn default() -> Self { - Self { - version: 1, - quality: 85, - sizes: vec![128, 256, 512], - created_at: Utc::now(), - } - } -} diff --git a/core-new/src/library/mod.rs b/core-new/src/library/mod.rs deleted file mode 100644 index 6e8740e21..000000000 --- a/core-new/src/library/mod.rs +++ /dev/null @@ -1,238 +0,0 @@ -//! Library management system -//! -//! This module provides the core library functionality for Spacedrive. -//! Each library is a self-contained directory with its own database, -//! thumbnails, and other data. - -mod config; -mod error; -mod lock; -mod manager; - -pub use config::{LibraryConfig, LibrarySettings, LibraryStatistics}; -pub use error::{LibraryError, Result}; -pub use lock::LibraryLock; -pub use manager::{LibraryManager, DiscoveredLibrary}; - -use crate::infrastructure::{ - database::Database, - jobs::manager::JobManager, -}; -use std::path::{Path, PathBuf}; -use std::sync::Arc; -use tokio::sync::RwLock; -use uuid::Uuid; - -/// Represents an open Spacedrive library -pub struct Library { - /// Root directory of the library (the .sdlibrary folder) - path: PathBuf, - - /// Library configuration - config: RwLock, - - /// Database connection - db: Arc, - - /// Job manager for this library - jobs: Arc, - - /// Lock preventing concurrent access - _lock: LibraryLock, -} - -impl Library { - /// Get the library ID - pub fn id(&self) -> Uuid { - // Config is immutable for ID, so we can use try_read - self.config.try_read().map(|c| c.id).unwrap_or_else(|_| { - // This should never happen in practice - panic!("Failed to read library config for ID") - }) - } - - /// Get the library name - pub async fn name(&self) -> String { - self.config.read().await.name.clone() - } - - /// Get the library path - pub fn path(&self) -> &Path { - &self.path - } - - /// Get the database - pub fn db(&self) -> &Arc { - &self.db - } - - /// Get the job manager - pub fn jobs(&self) -> &Arc { - &self.jobs - } - - /// Get a copy of the current configuration - pub async fn config(&self) -> LibraryConfig { - self.config.read().await.clone() - } - - /// Update library configuration - pub async fn update_config(&self, f: F) -> Result<()> - where - F: FnOnce(&mut LibraryConfig), - { - let mut config = self.config.write().await; - f(&mut config); - config.updated_at = chrono::Utc::now(); - - // Save to disk - let config_path = self.path.join("library.json"); - let json = serde_json::to_string_pretty(&*config)?; - tokio::fs::write(config_path, json).await?; - - Ok(()) - } - - /// Save library configuration to disk - pub async fn save_config(&self, config: &LibraryConfig) -> Result<()> { - let config_path = self.path.join("library.json"); - let json = serde_json::to_string_pretty(config)?; - tokio::fs::write(config_path, json).await?; - Ok(()) - } - - /// Get the thumbnail directory for this library - pub fn thumbnails_dir(&self) -> PathBuf { - self.path.join("thumbnails") - } - - /// Get the path for a specific thumbnail with size - pub fn thumbnail_path(&self, cas_id: &str, size: u32) -> PathBuf { - if cas_id.len() < 4 { - // Fallback for short IDs - return self.thumbnails_dir().join(format!("{}_{}.webp", cas_id, size)); - } - - // Two-level sharding based on first four characters - let shard1 = &cas_id[0..2]; - let shard2 = &cas_id[2..4]; - - self.thumbnails_dir() - .join(shard1) - .join(shard2) - .join(format!("{}_{}.webp", cas_id, size)) - } - - /// Get the path for any thumbnail size (legacy compatibility) - pub fn thumbnail_path_legacy(&self, cas_id: &str) -> PathBuf { - self.thumbnail_path(cas_id, 256) // Default to 256px - } - - /// Save a thumbnail with specific size - pub async fn save_thumbnail(&self, cas_id: &str, size: u32, data: &[u8]) -> Result<()> { - let path = self.thumbnail_path(cas_id, size); - - // Ensure parent directory exists - if let Some(parent) = path.parent() { - tokio::fs::create_dir_all(parent).await?; - } - - // Write thumbnail - tokio::fs::write(path, data).await?; - - Ok(()) - } - - /// Check if a thumbnail exists for a specific size - pub async fn has_thumbnail(&self, cas_id: &str, size: u32) -> bool { - tokio::fs::metadata(self.thumbnail_path(cas_id, size)) - .await - .is_ok() - } - - /// Shutdown the library, gracefully stopping all jobs - pub async fn shutdown(&self) -> Result<()> { - // Shutdown the job manager, which will pause all running jobs - self.jobs.shutdown().await?; - - // Save config to ensure any updates are persisted - let config = self.config.read().await; - self.save_config(&*config).await?; - - Ok(()) - } - - /// Check if thumbnails exist for all specified sizes - pub async fn has_all_thumbnails(&self, cas_id: &str, sizes: &[u32]) -> bool { - for &size in sizes { - if !self.has_thumbnail(cas_id, size).await { - return false; - } - } - true - } - - /// Get thumbnail data for specific size - pub async fn get_thumbnail(&self, cas_id: &str, size: u32) -> Result> { - let path = self.thumbnail_path(cas_id, size); - Ok(tokio::fs::read(path).await?) - } - - /// Get the best available thumbnail (largest size available) - pub async fn get_best_thumbnail(&self, cas_id: &str, preferred_sizes: &[u32]) -> Result)>> { - // Try sizes in descending order - let mut sizes = preferred_sizes.to_vec(); - sizes.sort_by(|a, b| b.cmp(a)); - - for &size in &sizes { - if self.has_thumbnail(cas_id, size).await { - let data = self.get_thumbnail(cas_id, size).await?; - return Ok(Some((size, data))); - } - } - - Ok(None) - } - - /// Start thumbnail generation job - pub async fn generate_thumbnails(&self, entry_ids: Option>) -> Result { - use crate::operations::media::thumbnail::{ThumbnailJob, ThumbnailJobConfig}; - - let config = ThumbnailJobConfig { - sizes: self.config().await.settings.thumbnail_sizes.clone(), - quality: self.config().await.settings.thumbnail_quality, - regenerate: false, - batch_size: 50, - max_concurrent: 4, - }; - - let job = if let Some(ids) = entry_ids { - ThumbnailJob::for_entries(ids, config) - } else { - ThumbnailJob::new(config) - }; - - self.jobs().dispatch(job).await - .map_err(|e| LibraryError::JobError(e)) - } - - /// Update library statistics - pub async fn update_statistics(&self, f: F) -> Result<()> - where - F: FnOnce(&mut LibraryStatistics), - { - self.update_config(|config| { - f(&mut config.statistics); - config.statistics.updated_at = chrono::Utc::now(); - }).await - } -} - -// Note: Library does not implement Clone due to the exclusive lock -// Use Arc when you need shared access - -/// Current library configuration version -pub const LIBRARY_CONFIG_VERSION: u32 = 2; - -/// Library directory extension -pub const LIBRARY_EXTENSION: &str = "sdlibrary"; \ No newline at end of file diff --git a/core-new/src/location/mod.rs b/core-new/src/location/mod.rs deleted file mode 100644 index 983d53c72..000000000 --- a/core-new/src/location/mod.rs +++ /dev/null @@ -1,563 +0,0 @@ -//! Location management - simplified implementation matching core patterns - -pub mod manager; - -use crate::{ - infrastructure::{ - database::entities::{self, entry::EntryKind}, - events::{Event, EventBus}, - jobs::{handle::JobHandle, output::IndexedOutput, types::JobStatus}, - }, - library::Library, - operations::indexing::{IndexMode as JobIndexMode, IndexerJob, IndexerJobConfig, PathResolver, rules::RuleToggles}, - domain::addressing::SdPath, -}; - -use sea_orm::{ActiveModelTrait, ActiveValue::Set, ColumnTrait, EntityTrait, QueryFilter, TransactionTrait}; -use serde::{Deserialize, Serialize}; -use std::{path::PathBuf, sync::Arc}; -use tokio::fs; -use tracing::{error, info, warn}; -use uuid::Uuid; - -pub use manager::LocationManager; - -/// Location creation arguments (simplified from production version) -#[derive(Debug, Serialize, Deserialize)] -pub struct LocationCreateArgs { - pub path: PathBuf, - pub name: Option, - pub index_mode: IndexMode, -} - -/// Location indexing mode -#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] -pub enum IndexMode { - /// Only scan file/directory structure - Shallow, - /// Quick scan (metadata only) - Quick, - /// Include content hashing for deduplication - Content, - /// Full indexing with content analysis and metadata - Deep, - /// Full indexing with all features - Full, -} - -impl From for JobIndexMode { - fn from(mode: IndexMode) -> Self { - match mode { - IndexMode::Shallow => JobIndexMode::Shallow, - IndexMode::Quick => JobIndexMode::Content, - IndexMode::Content => JobIndexMode::Content, - IndexMode::Deep => JobIndexMode::Deep, - IndexMode::Full => JobIndexMode::Deep, - } - } -} - -impl From<&str> for IndexMode { - fn from(s: &str) -> Self { - match s.to_lowercase().as_str() { - "shallow" => IndexMode::Shallow, - "quick" => IndexMode::Quick, - "content" => IndexMode::Content, - "deep" => IndexMode::Deep, - "full" => IndexMode::Full, - _ => IndexMode::Full, - } - } -} - -impl std::fmt::Display for IndexMode { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - IndexMode::Shallow => write!(f, "shallow"), - IndexMode::Quick => write!(f, "quick"), - IndexMode::Content => write!(f, "content"), - IndexMode::Deep => write!(f, "deep"), - IndexMode::Full => write!(f, "full"), - } - } -} - -/// Managed location representation -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct ManagedLocation { - pub id: Uuid, - pub name: String, - pub path: PathBuf, - pub device_id: i32, - pub library_id: Uuid, - pub indexing_enabled: bool, - pub index_mode: IndexMode, - pub watch_enabled: bool, -} - -/// Location management errors -#[derive(Debug, thiserror::Error)] -pub enum LocationError { - #[error("Database error: {0}")] - Database(#[from] sea_orm::DbErr), - #[error("Database error: {0}")] - DatabaseError(String), - #[error("Path does not exist: {path}")] - PathNotFound { path: PathBuf }, - #[error("Path not accessible: {path}")] - PathNotAccessible { path: PathBuf }, - #[error("Location already exists: {path}")] - LocationExists { path: PathBuf }, - #[error("Location not found: {id}")] - LocationNotFound { id: Uuid }, - #[error("IO error: {0}")] - Io(#[from] std::io::Error), - #[error("Invalid path: {0}")] - InvalidPath(String), - #[error("Job error: {0}")] - Job(#[from] crate::infrastructure::jobs::error::JobError), - #[error("Other error: {0}")] - Other(String), -} - -pub type LocationResult = Result; - -/// Create a new location and start indexing (production pattern) -pub async fn create_location( - library: Arc, - events: &EventBus, - args: LocationCreateArgs, - device_id: i32, -) -> LocationResult { - let path_str = args - .path - .to_str() - .ok_or_else(|| LocationError::InvalidPath("Non-UTF8 path".to_string()))?; - - // Validate path exists - if !args.path.exists() { - return Err(LocationError::PathNotFound { path: args.path }); - } - - if !args.path.is_dir() { - return Err(LocationError::InvalidPath( - "Path must be a directory".to_string(), - )); - } - - // Begin transaction to ensure atomicity - let txn = library.db().conn().begin().await - .map_err(|e| LocationError::DatabaseError(e.to_string()))?; - - // First, check if an entry already exists for this path - // We need to create a root entry for the location directory - let directory_name = args.path - .file_name() - .and_then(|n| n.to_str()) - .unwrap_or("Unknown") - .to_string(); - - // Create entry for the location directory - let entry_model = entities::entry::ActiveModel { - uuid: Set(Some(Uuid::new_v4())), - name: Set(directory_name.clone()), - kind: Set(EntryKind::Directory as i32), - extension: Set(None), - metadata_id: Set(None), - content_id: Set(None), - size: Set(0), - aggregate_size: Set(0), - child_count: Set(0), - file_count: Set(0), - created_at: Set(chrono::Utc::now()), - modified_at: Set(chrono::Utc::now()), - accessed_at: Set(None), - permissions: Set(None), - inode: Set(None), - parent_id: Set(None), // Location root has no parent - ..Default::default() - }; - - let entry_record = entry_model.insert(&txn).await - .map_err(|e| LocationError::DatabaseError(e.to_string()))?; - let entry_id = entry_record.id; - - // Add self-reference to closure table - let self_closure = entities::entry_closure::ActiveModel { - ancestor_id: Set(entry_id), - descendant_id: Set(entry_id), - depth: Set(0), - ..Default::default() - }; - self_closure.insert(&txn).await - .map_err(|e| LocationError::DatabaseError(e.to_string()))?; - - // Add to directory_paths table - let dir_path_entry = entities::directory_paths::ActiveModel { - entry_id: Set(entry_id), - path: Set(path_str.to_string()), - ..Default::default() - }; - dir_path_entry.insert(&txn).await - .map_err(|e| LocationError::DatabaseError(e.to_string()))?; - - // Check if a location already exists for this entry - let existing = entities::location::Entity::find() - .filter(entities::location::Column::EntryId.eq(entry_id)) - .one(&txn) - .await - .map_err(|e| LocationError::DatabaseError(e.to_string()))?; - - if existing.is_some() { - // Rollback transaction - txn.rollback().await - .map_err(|e| LocationError::DatabaseError(e.to_string()))?; - return Err(LocationError::LocationExists { path: args.path }); - } - - // Create location record - let location_id = Uuid::new_v4(); - let name = args.name.unwrap_or_else(|| { - args.path - .file_name() - .and_then(|n| n.to_str()) - .unwrap_or("Unknown") - .to_string() - }); - - let location_model = entities::location::ActiveModel { - id: Set(0), // Auto-increment - uuid: Set(location_id), - device_id: Set(device_id), - entry_id: Set(entry_id), - name: Set(Some(name.clone())), - index_mode: Set(args.index_mode.to_string()), - scan_state: Set("pending".to_string()), - last_scan_at: Set(None), - error_message: Set(None), - total_file_count: Set(0), - total_byte_size: Set(0), - created_at: Set(chrono::Utc::now()), - updated_at: Set(chrono::Utc::now()), - }; - - let location_record = location_model.insert(&txn).await - .map_err(|e| LocationError::DatabaseError(e.to_string()))?; - let location_db_id = location_record.id; - - // Commit transaction - txn.commit().await - .map_err(|e| LocationError::DatabaseError(e.to_string()))?; - - info!("Created location '{}' with ID: {}", name, location_db_id); - - // Emit location added event - events.emit(Event::LocationAdded { - library_id: library.id(), - location_id, - path: args.path.clone(), - }); - - // Start indexing (simplified - in production this goes through proper job manager) - start_location_indexing( - library.clone(), - events, - location_db_id, - location_id, - args.path, - args.index_mode, - ) - .await?; - - Ok(location_db_id) -} - -/// Start indexing for a location (production implementation) -async fn start_location_indexing( - library: Arc, - events: &EventBus, - location_db_id: i32, - location_uuid: Uuid, - path: PathBuf, - index_mode: IndexMode, -) -> LocationResult<()> { - info!("Starting indexing for location: {}", path.display()); - - // Update scan state to "running" - update_location_scan_state(library.clone(), location_db_id, "running", None).await?; - - // Emit indexing started event - events.emit(Event::IndexingStarted { - location_id: location_uuid, - }); - - // Get device UUID for SdPath - let device_uuid = get_device_uuid(library.clone()).await?; - let location_sd_path = SdPath::new(device_uuid, path.clone()); - - // Create and dispatch indexer job through the proper job manager - let lib_cfg = library.config().await; - let idx_cfg = lib_cfg.settings.indexer; - let mut config = IndexerJobConfig::new(location_uuid, location_sd_path, index_mode.into()); - config.rule_toggles = RuleToggles { - no_system_files: idx_cfg.no_system_files, - no_hidden: idx_cfg.no_hidden, - no_git: idx_cfg.no_git, - gitignore: idx_cfg.gitignore, - only_images: idx_cfg.only_images, - no_dev_dirs: idx_cfg.no_dev_dirs, - }; - let indexer_job = IndexerJob::new(config); - - match library.jobs().dispatch(indexer_job).await { - Ok(job_handle) => { - info!( - "Successfully dispatched indexer job {} for location: {}", - job_handle.id(), - path.display() - ); - - // Monitor job progress asynchronously - let events_clone = events.clone(); - let library_clone = library.clone(); - let handle_clone = job_handle.clone(); - - tokio::spawn(async move { - monitor_indexing_job( - handle_clone, - events_clone, - library_clone, - location_db_id, - location_uuid, - path, - ) - .await; - }); - } - Err(e) => { - error!( - "Failed to dispatch indexer job for {}: {}", - path.display(), - e - ); - - // Update scan state to failed - if let Err(update_err) = update_location_scan_state( - library.clone(), - location_db_id, - "failed", - Some(e.to_string()), - ) - .await - { - error!("Failed to update scan state: {}", update_err); - } - - events.emit(Event::IndexingFailed { - location_id: location_uuid, - error: e.to_string(), - }); - - return Err(LocationError::Other(format!( - "Failed to start indexing: {}", - e - ))); - } - } - - Ok(()) -} - -/// Monitor indexing job progress and update location state accordingly -async fn monitor_indexing_job( - job_handle: JobHandle, - events: EventBus, - library: Arc, - location_db_id: i32, - location_uuid: Uuid, - path: PathBuf, -) { - info!( - "Monitoring indexer job {} for location: {}", - job_handle.id(), - path.display() - ); - - // Wait for job completion - let job_result = job_handle.wait().await; - - match job_result { - Ok(output) => { - info!( - "Indexing completed successfully for location: {}", - path.display() - ); - - // Parse output to get statistics - if let Some(indexer_output) = output.as_indexed() { - // Update location stats - if let Err(e) = update_location_stats( - library.clone(), - location_db_id, - indexer_output.total_files, - indexer_output.total_bytes, - ) - .await - { - error!("Failed to update location stats: {}", e); - } - - // Update scan state to completed - if let Err(e) = - update_location_scan_state(library.clone(), location_db_id, "completed", None) - .await - { - error!("Failed to update scan state: {}", e); - } - - // Emit completion events - events.emit(Event::IndexingCompleted { - location_id: location_uuid, - total_files: indexer_output.total_files, - total_dirs: indexer_output.total_dirs, - }); - - events.emit(Event::FilesIndexed { - library_id: library.id(), - location_id: location_uuid, - count: indexer_output.total_files as usize, - }); - - info!( - "Location indexing completed: {} ({} files, {} dirs, {} bytes)", - path.display(), - indexer_output.total_files, - indexer_output.total_dirs, - indexer_output.total_bytes - ); - } else { - warn!("Job completed but output format was unexpected"); - - // Update scan state to completed anyway - if let Err(e) = - update_location_scan_state(library.clone(), location_db_id, "completed", None) - .await - { - error!("Failed to update scan state: {}", e); - } - } - } - Err(e) => { - error!("Indexing failed for {}: {}", path.display(), e); - - // Update scan state to failed - if let Err(update_err) = update_location_scan_state( - library.clone(), - location_db_id, - "failed", - Some(e.to_string()), - ) - .await - { - error!("Failed to update scan state: {}", update_err); - } - - events.emit(Event::IndexingFailed { - location_id: location_uuid, - error: e.to_string(), - }); - } - } -} - -/// Scan directory to get basic stats -async fn scan_directory_stats(path: &PathBuf) -> Result<(u64, u64), std::io::Error> { - let mut file_count = 0u64; - let mut total_size = 0u64; - - let mut stack = vec![path.clone()]; - - while let Some(current_path) = stack.pop() { - if let Ok(mut entries) = fs::read_dir(¤t_path).await { - while let Ok(Some(entry)) = entries.next_entry().await { - if let Ok(metadata) = entry.metadata().await { - if metadata.is_file() { - file_count += 1; - total_size += metadata.len(); - } else if metadata.is_dir() { - stack.push(entry.path()); - } - } - } - } - } - - Ok((file_count, total_size)) -} - -/// Update location scan state -async fn update_location_scan_state( - library: Arc, - location_id: i32, - state: &str, - error_message: Option, -) -> LocationResult<()> { - let location = entities::location::Entity::find_by_id(location_id) - .one(library.db().conn()) - .await? - .ok_or_else(|| LocationError::LocationNotFound { id: Uuid::nil() })?; - - let mut active_location: entities::location::ActiveModel = location.into(); - active_location.scan_state = Set(state.to_string()); - active_location.error_message = Set(error_message); - active_location.updated_at = Set(chrono::Utc::now()); - - if state == "running" { - active_location.last_scan_at = Set(Some(chrono::Utc::now())); - } - - active_location.update(library.db().conn()).await?; - Ok(()) -} - -/// Update location statistics -async fn update_location_stats( - library: Arc, - location_id: i32, - file_count: u64, - total_size: u64, -) -> LocationResult<()> { - let location = entities::location::Entity::find_by_id(location_id) - .one(library.db().conn()) - .await? - .ok_or_else(|| LocationError::LocationNotFound { id: Uuid::nil() })?; - - let mut active_location: entities::location::ActiveModel = location.into(); - active_location.total_file_count = Set(file_count as i64); - active_location.total_byte_size = Set(total_size as i64); - active_location.updated_at = Set(chrono::Utc::now()); - - active_location.update(library.db().conn()).await?; - Ok(()) -} - -/// Get device UUID for current device -async fn get_device_uuid(_library: Arc) -> LocationResult { - // Get the current device ID from the global state - let device_uuid = crate::shared::utils::get_current_device_id(); - - if device_uuid.is_nil() { - return Err(LocationError::InvalidPath("Current device ID not initialized".to_string())); - } - - Ok(device_uuid) -} - -/// List all locations for a library -pub async fn list_locations( - library: Arc, -) -> LocationResult> { - Ok(entities::location::Entity::find() - .all(library.db().conn()) - .await?) -} diff --git a/core-new/src/volume/error.rs b/core-new/src/volume/error.rs deleted file mode 100644 index cc2564ff0..000000000 --- a/core-new/src/volume/error.rs +++ /dev/null @@ -1,83 +0,0 @@ -//! Volume-related error types - -use thiserror::Error; - -/// Errors that can occur during volume operations -#[derive(Error, Debug)] -pub enum VolumeError { - /// IO error during volume operations - #[error("IO error: {0}")] - Io(#[from] std::io::Error), - - /// Platform-specific error - #[error("Platform error: {0}")] - Platform(String), - - /// Volume not found - #[error("Volume not found: {0}")] - NotFound(String), - - /// Volume is not mounted - #[error("Volume is not mounted: {0}")] - NotMounted(String), - - /// Volume is read-only - #[error("Volume is read-only: {0}")] - ReadOnly(String), - - /// Insufficient space on volume - #[error("Insufficient space on volume: required {required}, available {available}")] - InsufficientSpace { required: u64, available: u64 }, - - /// Speed test was cancelled or failed - #[error("Speed test cancelled or failed")] - SpeedTestFailed, - - /// Volume detection failed - #[error("Volume detection failed: {0}")] - DetectionFailed(String), - - /// Permission denied - #[error("Permission denied: {0}")] - PermissionDenied(String), - - /// Operation timed out - #[error("Operation timed out")] - Timeout, - - /// Invalid volume data - #[error("Invalid volume data: {0}")] - InvalidData(String), - - /// Database operation failed - #[error("Database error: {0}")] - Database(String), - - /// Volume is already tracked - #[error("Volume is already tracked: {0}")] - AlreadyTracked(String), - - /// Volume is not tracked - #[error("Volume is not tracked: {0}")] - NotTracked(String), -} - -impl VolumeError { - /// Create a platform-specific error - pub fn platform(msg: impl Into) -> Self { - Self::Platform(msg.into()) - } - - /// Create a detection failed error - pub fn detection_failed(msg: impl Into) -> Self { - Self::DetectionFailed(msg.into()) - } - - /// Create an insufficient space error - pub fn insufficient_space(required: u64, available: u64) -> Self { - Self::InsufficientSpace { required, available } - } -} - -/// Result type for volume operations -pub type VolumeResult = Result; \ No newline at end of file diff --git a/core-new/src/volume/mod.rs b/core-new/src/volume/mod.rs deleted file mode 100644 index 5464bad8c..000000000 --- a/core-new/src/volume/mod.rs +++ /dev/null @@ -1,92 +0,0 @@ -//! Volume management for Spacedrive Core v2 -//! -//! This module provides functionality for detecting, monitoring, and managing storage volumes -//! across different platforms. It's designed to integrate with the copy system for optimal -//! file operation routing. - -pub mod classification; -pub mod error; -pub mod manager; -pub mod os_detection; -pub mod speed; -pub mod types; - -pub use error::VolumeError; -pub use manager::VolumeManager; -pub use types::{ - DiskType, FileSystem, MountType, Volume, VolumeDetectionConfig, VolumeEvent, VolumeFingerprint, - VolumeInfo, -}; - -// Re-export platform-specific detection -pub use os_detection::detect_volumes; - -/// Extension trait for Volume operations -pub trait VolumeExt { - /// Checks if volume is mounted and accessible - async fn is_available(&self) -> bool; - - /// Checks if volume has enough free space - fn has_space(&self, required_bytes: u64) -> bool; - - /// Check if path is on this volume - fn contains_path(&self, path: &std::path::Path) -> bool; -} - -impl VolumeExt for Volume { - async fn is_available(&self) -> bool { - self.is_mounted && tokio::fs::metadata(&self.mount_point).await.is_ok() - } - - fn has_space(&self, required_bytes: u64) -> bool { - self.total_bytes_available >= required_bytes - } - - fn contains_path(&self, path: &std::path::Path) -> bool { - // Check primary mount point - if path.starts_with(&self.mount_point) { - return true; - } - - // Check additional mount points (for APFS volumes) - self.mount_points.iter().any(|mp| path.starts_with(mp)) - } -} - -/// Utilities for volume operations -pub mod util { - use super::*; - use std::path::Path; - - /// Check if a path is on the specified volume - pub fn is_path_on_volume(path: &Path, volume: &Volume) -> bool { - volume.contains_path(&path.to_path_buf()) - } - - /// Calculate relative path from volume mount point - pub fn relative_path_on_volume(path: &Path, volume: &Volume) -> Option { - // Try primary mount point first - if let Ok(relative) = path.strip_prefix(&volume.mount_point) { - return Some(relative.to_path_buf()); - } - - // Try additional mount points - for mount_point in &volume.mount_points { - if let Ok(relative) = path.strip_prefix(mount_point) { - return Some(relative.to_path_buf()); - } - } - - None - } - - /// Find the volume that contains the given path - pub fn find_volume_for_path<'a>( - path: &Path, - volumes: impl Iterator, - ) -> Option<&'a Volume> { - volumes - .filter(|vol| vol.contains_path(&path.to_path_buf())) - .max_by_key(|vol| vol.mount_point.as_os_str().len()) // Prefer most specific mount - } -} diff --git a/core-new/src/volume/speed.rs b/core-new/src/volume/speed.rs deleted file mode 100644 index 670fc2ece..000000000 --- a/core-new/src/volume/speed.rs +++ /dev/null @@ -1,369 +0,0 @@ -//! Volume speed testing functionality - -use crate::volume::{ - error::{VolumeError, VolumeResult}, - types::{MountType, Volume, VolumeType}, -}; -use std::time::Instant; -use tokio::{ - fs::{File, OpenOptions}, - io::{AsyncReadExt, AsyncWriteExt}, - time::{timeout, Duration}, -}; -use tracing::{debug, instrument, warn}; - -/// Configuration for speed tests -#[derive(Debug, Clone)] -pub struct SpeedTestConfig { - /// Size of the test file in megabytes - pub file_size_mb: usize, - /// Timeout for the test in seconds - pub timeout_secs: u64, - /// Number of test iterations for averaging - pub iterations: usize, -} - -impl Default for SpeedTestConfig { - fn default() -> Self { - Self { - file_size_mb: 10, - timeout_secs: 30, - iterations: 1, - } - } -} - -/// Result of a speed test -#[derive(Debug, Clone)] -pub struct SpeedTestResult { - /// Write speed in MB/s - pub write_speed_mbps: f64, - /// Read speed in MB/s - pub read_speed_mbps: f64, - /// Total time taken for the test - pub duration_secs: f64, -} - -/// Run a speed test on the given volume -#[instrument(skip(volume), fields(volume_name = %volume.name))] -pub async fn run_speed_test(volume: &Volume) -> VolumeResult<(u64, u64)> { - run_speed_test_with_config(volume, SpeedTestConfig::default()).await -} - -/// Run a speed test with custom configuration -#[instrument(skip(volume, config), fields(volume_name = %volume.name))] -pub async fn run_speed_test_with_config( - volume: &Volume, - config: SpeedTestConfig, -) -> VolumeResult<(u64, u64)> { - if !volume.is_mounted { - return Err(VolumeError::NotMounted(volume.name.clone())); - } - - if volume.read_only { - return Err(VolumeError::ReadOnly(volume.name.clone())); - } - - debug!("Starting speed test with config: {:?}", config); - - let test_location = TestLocation::new(&volume.mount_point, &volume.mount_type).await?; - let result = perform_speed_test(&test_location, &config).await?; - - // Cleanup - test_location.cleanup().await?; - - debug!( - "Speed test completed: {:.2} MB/s write, {:.2} MB/s read", - result.write_speed_mbps, result.read_speed_mbps - ); - - Ok(( - result.read_speed_mbps as u64, - result.write_speed_mbps as u64, - )) -} - -/// Helper for managing test files and directories -struct TestLocation { - test_file: std::path::PathBuf, - created_dir: Option, -} - -impl TestLocation { - /// Create a new test location - async fn new(volume_path: &std::path::Path, mount_type: &MountType) -> VolumeResult { - let (dir, created_dir) = get_writable_directory(volume_path, mount_type).await?; - let test_file = dir.join("spacedrive_speed_test.tmp"); - - Ok(Self { - test_file, - created_dir, - }) - } - - /// Clean up test files and directories - async fn cleanup(&self) -> VolumeResult<()> { - // Remove test file - if self.test_file.exists() { - if let Err(e) = tokio::fs::remove_file(&self.test_file).await { - warn!("Failed to remove test file: {}", e); - } - } - - // Remove created directory if we created it - if let Some(ref dir) = self.created_dir { - if let Err(e) = tokio::fs::remove_dir_all(dir).await { - warn!("Failed to remove test directory: {}", e); - } - } - - Ok(()) - } -} - -/// Perform the actual speed test -async fn perform_speed_test( - location: &TestLocation, - config: &SpeedTestConfig, -) -> VolumeResult { - let test_data = generate_test_data(config.file_size_mb); - let timeout_duration = Duration::from_secs(config.timeout_secs); - - let mut write_speeds = Vec::new(); - let mut read_speeds = Vec::new(); - let overall_start = Instant::now(); - - for iteration in 0..config.iterations { - debug!( - "Speed test iteration {}/{}", - iteration + 1, - config.iterations - ); - - // Write test - let write_speed = timeout( - timeout_duration, - perform_write_test(&location.test_file, &test_data), - ) - .await - .map_err(|_| VolumeError::Timeout)??; - - write_speeds.push(write_speed); - - // Read test - let read_speed = timeout( - timeout_duration, - perform_read_test(&location.test_file, test_data.len()), - ) - .await - .map_err(|_| VolumeError::Timeout)??; - - read_speeds.push(read_speed); - - // Clean up test file between iterations - if iteration < config.iterations - 1 { - let _ = tokio::fs::remove_file(&location.test_file).await; - } - } - - let avg_write_speed = write_speeds.iter().sum::() / write_speeds.len() as f64; - let avg_read_speed = read_speeds.iter().sum::() / read_speeds.len() as f64; - - Ok(SpeedTestResult { - write_speed_mbps: avg_write_speed, - read_speed_mbps: avg_read_speed, - duration_secs: overall_start.elapsed().as_secs_f64(), - }) -} - -/// Generate test data for speed testing -fn generate_test_data(size_mb: usize) -> Vec { - let size_bytes = size_mb * 1024 * 1024; - - // Use a pattern instead of zeros to avoid compression optimizations - let pattern = b"SpacedriveSpeedTest0123456789ABCDEF"; - let mut data = Vec::with_capacity(size_bytes); - - for i in 0..size_bytes { - data.push(pattern[i % pattern.len()]); - } - - data -} - -/// Perform write speed test -async fn perform_write_test(file_path: &std::path::Path, data: &[u8]) -> VolumeResult { - let start = Instant::now(); - - let mut file = OpenOptions::new() - .write(true) - .create(true) - .truncate(true) - .open(file_path) - .await?; - - file.write_all(data).await?; - file.sync_all().await?; // Ensure data is written to disk - - let duration = start.elapsed(); - let speed_mbps = (data.len() as f64 / 1024.0 / 1024.0) / duration.as_secs_f64(); - - Ok(speed_mbps) -} - -/// Perform read speed test -async fn perform_read_test(file_path: &std::path::Path, expected_size: usize) -> VolumeResult { - let start = Instant::now(); - - let mut file = File::open(file_path).await?; - let mut buffer = Vec::with_capacity(expected_size); - file.read_to_end(&mut buffer).await?; - - let duration = start.elapsed(); - let speed_mbps = (buffer.len() as f64 / 1024.0 / 1024.0) / duration.as_secs_f64(); - - Ok(speed_mbps) -} - -/// Get a writable directory within the volume -async fn get_writable_directory( - volume_path: &std::path::Path, - mount_type: &MountType, -) -> VolumeResult<(std::path::PathBuf, Option)> { - match mount_type { - MountType::System => { - // For system volumes, prefer using temp directory - let temp_dir = std::env::temp_dir(); - Ok((temp_dir, None)) - } - _ => { - // For external volumes, try to write in the root or create a temp directory - let candidates = [ - volume_path.join("tmp"), - volume_path.join(".spacedrive_temp"), - volume_path.to_path_buf(), - ]; - - for candidate in &candidates { - // Try to create the directory - if let Ok(()) = tokio::fs::create_dir_all(candidate).await { - // Test if we can write to it - let test_file = candidate.join("test_write_permissions"); - if tokio::fs::write(&test_file, b"test").await.is_ok() { - let _ = tokio::fs::remove_file(&test_file).await; - - // If we created a directory specifically for this test, mark it for cleanup - let created_dir = if candidate - .file_name() - .map_or(false, |name| name == "tmp" || name == ".spacedrive_temp") - { - Some(candidate.clone()) - } else { - None - }; - - return Ok((candidate.clone(), created_dir)); - } - } - } - - Err(VolumeError::PermissionDenied(format!( - "No writable directory found in volume: {}", - volume_path.display() - ))) - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::volume::{ - types::{DiskType, FileSystem}, - VolumeFingerprint, - }; - use tempfile::TempDir; - - #[tokio::test] - async fn test_speed_test_config() { - let config = SpeedTestConfig::default(); - assert_eq!(config.file_size_mb, 10); - assert_eq!(config.timeout_secs, 30); - assert_eq!(config.iterations, 1); - } - - #[tokio::test] - async fn test_generate_test_data() { - let data = generate_test_data(1); // 1MB - assert_eq!(data.len(), 1024 * 1024); - - // Verify pattern is not all zeros - assert!(data.iter().any(|&b| b != 0)); - } - - #[tokio::test] - async fn test_writable_directory_external() { - let temp_dir = TempDir::new().unwrap(); - let volume_path = temp_dir.path(); - - let (writable_dir, created_dir) = get_writable_directory(volume_path, &MountType::External) - .await - .unwrap(); - - assert!(writable_dir.exists()); - - // Cleanup if we created a directory - if let Some(dir) = created_dir { - let _ = tokio::fs::remove_dir_all(dir).await; - } - } - - #[tokio::test] - async fn test_writable_directory_system() { - let (writable_dir, created_dir) = - get_writable_directory(&std::path::PathBuf::from("/"), &MountType::System) - .await - .unwrap(); - - assert!(writable_dir.exists()); - assert!(created_dir.is_none()); // Should use system temp, not create new dir - } - - #[tokio::test] - async fn test_full_speed_test() { - let temp_dir = TempDir::new().unwrap(); - - let volume = Volume::new( - uuid::Uuid::new_v4(), // Test device ID - "Test Volume".to_string(), - MountType::External, - VolumeType::External, - temp_dir.path().to_path_buf(), - vec![], - DiskType::Unknown, - FileSystem::Other("test".to_string()), - 1000000000, // 1GB capacity - 500000000, // 500MB available - false, // Not read-only - None, - VolumeFingerprint::new( - "Test Volume", - 1000000000, - "test", - ), - ); - - let config = SpeedTestConfig { - file_size_mb: 1, // Small test file - timeout_secs: 10, - iterations: 1, - }; - - let result = run_speed_test_with_config(&volume, config).await; - assert!(result.is_ok()); - - let (read_speed, write_speed) = result.unwrap(); - assert!(read_speed > 0); - assert!(write_speed > 0); - } -} diff --git a/core-new/src/volume/types.rs b/core-new/src/volume/types.rs deleted file mode 100644 index f4b26fbbe..000000000 --- a/core-new/src/volume/types.rs +++ /dev/null @@ -1,685 +0,0 @@ -//! Volume type definitions - -use serde::{Deserialize, Serialize}; -use std::fmt; -use std::path::PathBuf; -use uuid::Uuid; - -/// Spacedrive volume identifier file content -/// This file is created in the root of writable volumes for persistent identification -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct SpacedriveVolumeId { - /// Unique identifier for this volume - pub id: Uuid, - /// When this identifier was created - pub created: chrono::DateTime, - /// Name of the device that created this identifier - pub device_name: Option, - /// Original volume name when identifier was created - pub volume_name: String, - /// Device ID that created this identifier - pub device_id: Uuid, -} - -/// Unique fingerprint for a storage volume -#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, Hash)] -pub struct VolumeFingerprint(pub String); - -impl VolumeFingerprint { - /// Create a new volume fingerprint from volume properties - /// Uses intrinsic volume characteristics for cross-device portable identification - pub fn new(name: &str, total_bytes: u64, file_system: &str) -> Self { - let mut hasher = blake3::Hasher::new(); - hasher.update(b"content_based:"); - hasher.update(name.as_bytes()); - hasher.update(&total_bytes.to_be_bytes()); - hasher.update(file_system.as_bytes()); - hasher.update(&(name.len() as u64).to_be_bytes()); - - Self(hasher.finalize().to_hex().to_string()) - } - - /// Create a fingerprint from a Spacedrive identifier UUID (preferred method) - /// This provides stable identification across devices, renames and remounts - pub fn from_spacedrive_id(spacedrive_id: Uuid) -> Self { - let mut hasher = blake3::Hasher::new(); - hasher.update(b"spacedrive_id:"); - hasher.update(spacedrive_id.as_bytes()); - - Self(hasher.finalize().to_hex().to_string()) - } - - /// Generate 8-character short ID for CLI display and commands - pub fn short_id(&self) -> String { - self.0.chars().take(8).collect() - } - - /// Generate 12-character medium ID for disambiguation - pub fn medium_id(&self) -> String { - self.0.chars().take(12).collect() - } - - /// Create fingerprint from hex string - pub fn from_hex(hex: impl Into) -> Self { - Self(hex.into()) - } - - /// Create fingerprint from string (alias for from_hex) - pub fn from_string(s: &str) -> Result { - Ok(Self(s.to_string())) - } - - /// Check if a string could be a short ID (8 chars, hex) - pub fn is_short_id(s: &str) -> bool { - s.len() == 8 && s.chars().all(|c| c.is_ascii_hexdigit()) - } - - /// Check if a string could be a medium ID (12 chars, hex) - pub fn is_medium_id(s: &str) -> bool { - s.len() == 12 && s.chars().all(|c| c.is_ascii_hexdigit()) - } - - /// Check if this fingerprint matches a short or medium ID - pub fn matches_short_id(&self, short_id: &str) -> bool { - if Self::is_short_id(short_id) { - self.short_id() == short_id - } else if Self::is_medium_id(short_id) { - self.medium_id() == short_id - } else { - false - } - } -} - -impl fmt::Display for VolumeFingerprint { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{}", self.0) - } -} - -/// Classification of volume types for UX and auto-tracking decisions -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -pub enum VolumeType { - /// Primary system drive containing OS and user data - /// Examples: C:\ on Windows, / on Linux, Macintosh HD on macOS - Primary, - - /// Dedicated user data volumes (separate from OS) - /// Examples: /System/Volumes/Data on macOS, separate /home on Linux - UserData, - - /// External or removable storage devices - /// Examples: USB drives, external HDDs, /Volumes/* on macOS - External, - - /// Secondary internal storage (additional drives/partitions) - /// Examples: D:, E: drives on Windows, additional mounted drives - Secondary, - - /// System/OS internal volumes (hidden from normal view) - /// Examples: /System/Volumes/* on macOS, Recovery partitions - System, - - /// Network attached storage - /// Examples: SMB mounts, NFS, cloud storage - Network, - - /// Unknown or unclassified volumes - Unknown, -} - -impl VolumeType { - /// Should this volume type be auto-tracked by default? - pub fn auto_track_by_default(&self) -> bool { - match self { - // Only auto-track the primary system volume - // Users should explicitly choose to track other volumes - VolumeType::Primary => true, - VolumeType::UserData - | VolumeType::External - | VolumeType::Secondary - | VolumeType::Network - | VolumeType::System - | VolumeType::Unknown => false, - } - } - - /// Should this volume be shown in the default UI view? - pub fn show_by_default(&self) -> bool { - !matches!(self, VolumeType::System | VolumeType::Unknown) - } - - /// User-friendly display name for the volume type - pub fn display_name(&self) -> &'static str { - match self { - VolumeType::Primary => "Primary Drive", - VolumeType::UserData => "User Data", - VolumeType::External => "External Drive", - VolumeType::Secondary => "Secondary Drive", - VolumeType::System => "System Volume", - VolumeType::Network => "Network Drive", - VolumeType::Unknown => "Unknown", - } - } - - /// Icon/indicator for CLI display - pub fn icon(&self) -> &'static str { - match self { - VolumeType::Primary => "[PRI]", - VolumeType::UserData => "[USR]", - VolumeType::External => "[EXT]", - VolumeType::Secondary => "[SEC]", - VolumeType::System => "[SYS]", - VolumeType::Network => "[NET]", - VolumeType::Unknown => "[UNK]", - } - } -} - -/// Events emitted by the Volume Manager when volume state changes -#[derive(Debug, Clone, Serialize, Deserialize)] -pub enum VolumeEvent { - /// Emitted when a new volume is discovered - VolumeAdded(Volume), - /// Emitted when a volume is removed/unmounted - VolumeRemoved { fingerprint: VolumeFingerprint }, - /// Emitted when a volume's properties are updated - VolumeUpdated { - fingerprint: VolumeFingerprint, - old: VolumeInfo, - new: VolumeInfo, - }, - /// Emitted when a volume's speed test completes - VolumeSpeedTested { - fingerprint: VolumeFingerprint, - read_speed_mbps: u64, - write_speed_mbps: u64, - }, - /// Emitted when a volume's mount status changes - VolumeMountChanged { - fingerprint: VolumeFingerprint, - is_mounted: bool, - }, - /// Emitted when a volume encounters an error - VolumeError { - fingerprint: VolumeFingerprint, - error: String, - }, -} - -/// Represents a physical or virtual storage volume in the system -#[derive(Serialize, Deserialize, Debug, Clone)] -pub struct Volume { - /// Unique fingerprint for this volume - pub fingerprint: VolumeFingerprint, - - /// Device this volume belongs to - pub device_id: uuid::Uuid, - - /// Human-readable volume name - pub name: String, - /// Type of mount (system, external, etc) - pub mount_type: MountType, - /// Classification of this volume for UX decisions - pub volume_type: VolumeType, - /// Primary path where the volume is mounted - pub mount_point: PathBuf, - /// Additional mount points (for APFS volumes, etc.) - pub mount_points: Vec, - /// Whether the volume is currently mounted - pub is_mounted: bool, - - /// Type of storage device (SSD, HDD, etc) - pub disk_type: DiskType, - /// Filesystem type (NTFS, EXT4, etc) - pub file_system: FileSystem, - /// Whether the volume is mounted read-only - pub read_only: bool, - - /// Hardware identifier (platform-specific) - pub hardware_id: Option, - /// Current error status if any - pub error_status: Option, - - // Storage information - /// Total storage capacity in bytes - pub total_bytes_capacity: u64, - /// Available storage space in bytes - pub total_bytes_available: u64, - - // Performance metrics (populated by speed tests) - /// Read speed in megabytes per second - pub read_speed_mbps: Option, - /// Write speed in megabytes per second - pub write_speed_mbps: Option, - - /// Whether this volume should be visible in default views - pub is_user_visible: bool, - - /// Whether this volume should be auto-tracked - pub auto_track_eligible: bool, - - /// When this volume information was last updated - pub last_updated: chrono::DateTime, -} - -/// Summary information about a volume (for updates and caching) -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct VolumeInfo { - pub is_mounted: bool, - pub total_bytes_available: u64, - pub read_speed_mbps: Option, - pub write_speed_mbps: Option, - pub error_status: Option, -} - -/// Information about a tracked volume in the database -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct TrackedVolume { - pub id: i32, - pub uuid: uuid::Uuid, - pub device_id: uuid::Uuid, - pub fingerprint: VolumeFingerprint, - pub display_name: Option, - pub tracked_at: chrono::DateTime, - pub last_seen_at: chrono::DateTime, - pub is_online: bool, - pub total_capacity: Option, - pub available_capacity: Option, - pub read_speed_mbps: Option, - pub write_speed_mbps: Option, - pub last_speed_test_at: Option>, - pub file_system: Option, - pub mount_point: Option, - pub is_removable: Option, - pub is_network_drive: Option, - pub device_model: Option, - pub volume_type: String, - pub is_user_visible: Option, - pub auto_track_eligible: Option, -} - -impl From<&Volume> for VolumeInfo { - fn from(volume: &Volume) -> Self { - Self { - is_mounted: volume.is_mounted, - total_bytes_available: volume.total_bytes_available, - read_speed_mbps: volume.read_speed_mbps, - write_speed_mbps: volume.write_speed_mbps, - error_status: volume.error_status.clone(), - } - } -} - -impl TrackedVolume { - /// Convert a TrackedVolume back to a Volume for display purposes - /// This is used for offline volumes that aren't currently detected - pub fn to_offline_volume(&self) -> Volume { - use std::path::PathBuf; - - Volume { - fingerprint: self.fingerprint.clone(), - device_id: self.device_id, - name: self - .display_name - .clone() - .unwrap_or_else(|| "Unknown".to_string()), - mount_type: crate::volume::types::MountType::External, // Default for tracked volumes - volume_type: match self.volume_type.as_str() { - "Primary" => VolumeType::Primary, - "UserData" => VolumeType::UserData, - "External" => VolumeType::External, - "Secondary" => VolumeType::Secondary, - "System" => VolumeType::System, - "Network" => VolumeType::Network, - _ => VolumeType::Unknown, - }, - mount_point: PathBuf::from( - self.mount_point - .clone() - .unwrap_or_else(|| "Not connected".to_string()), - ), - mount_points: vec![], // Not available for offline volumes - disk_type: crate::volume::types::DiskType::Unknown, - file_system: crate::volume::types::FileSystem::from_string( - &self - .file_system - .clone() - .unwrap_or_else(|| "Unknown".to_string()), - ), - total_bytes_capacity: self.total_capacity.unwrap_or(0), - total_bytes_available: self.available_capacity.unwrap_or(0), - read_only: false, // Assume not read-only for tracked volumes - hardware_id: self.device_model.clone(), - is_mounted: false, // Offline volumes are not mounted - error_status: None, - read_speed_mbps: self.read_speed_mbps.map(|s| s as u64), - write_speed_mbps: self.write_speed_mbps.map(|s| s as u64), - last_updated: self.last_seen_at, - is_user_visible: self.is_user_visible.unwrap_or(true), - auto_track_eligible: self.auto_track_eligible.unwrap_or(false), - } - } -} - -impl Volume { - /// Create a new Volume instance - pub fn new( - device_id: uuid::Uuid, - name: String, - mount_type: MountType, - volume_type: VolumeType, - mount_point: PathBuf, - additional_mount_points: Vec, - disk_type: DiskType, - file_system: FileSystem, - total_bytes_capacity: u64, - total_bytes_available: u64, - read_only: bool, - hardware_id: Option, - fingerprint: VolumeFingerprint, // Accept pre-computed fingerprint - ) -> Self { - Self { - fingerprint, - device_id, - name, - mount_type, - volume_type, - mount_point, - mount_points: additional_mount_points, - is_mounted: true, - disk_type, - file_system, - total_bytes_capacity, - total_bytes_available, - read_only, - hardware_id, - error_status: None, - read_speed_mbps: None, - write_speed_mbps: None, - auto_track_eligible: volume_type.auto_track_by_default(), - is_user_visible: volume_type.show_by_default(), - last_updated: chrono::Utc::now(), - } - } - - /// Update volume information - pub fn update_info(&mut self, info: VolumeInfo) { - self.is_mounted = info.is_mounted; - self.total_bytes_available = info.total_bytes_available; - self.read_speed_mbps = info.read_speed_mbps; - self.write_speed_mbps = info.write_speed_mbps; - self.error_status = info.error_status; - self.last_updated = chrono::Utc::now(); - } - - /// Check if this volume supports fast copy operations (CoW) - pub fn supports_fast_copy(&self) -> bool { - matches!( - self.file_system, - FileSystem::APFS | FileSystem::Btrfs | FileSystem::ZFS | FileSystem::ReFS - ) - } - - /// Get the optimal chunk size for copying to/from this volume - pub fn optimal_chunk_size(&self) -> usize { - match self.disk_type { - DiskType::SSD => 1024 * 1024, // 1MB for SSDs - DiskType::HDD => 256 * 1024, // 256KB for HDDs - DiskType::Unknown => 64 * 1024, // 64KB default - } - } - - /// Estimate copy speed between this and another volume - pub fn estimate_copy_speed(&self, other: &Volume) -> Option { - let self_read = self.read_speed_mbps?; - let other_write = other.write_speed_mbps?; - - // Bottleneck is the slower of read or write speed - Some(self_read.min(other_write)) - } - - /// Check if a path is contained within this volume - pub fn contains_path(&self, path: &PathBuf) -> bool { - // Check primary mount point - if path.starts_with(&self.mount_point) { - return true; - } - - // Check additional mount points - for mount_point in &self.mount_points { - if path.starts_with(mount_point) { - return true; - } - } - - false - } -} - -/// Represents the type of physical storage device -#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, Hash)] -pub enum DiskType { - /// Solid State Drive - SSD, - /// Hard Disk Drive - HDD, - /// Unknown or virtual disk type - Unknown, -} - -impl fmt::Display for DiskType { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - DiskType::SSD => write!(f, "SSD"), - DiskType::HDD => write!(f, "HDD"), - DiskType::Unknown => write!(f, "Unknown"), - } - } -} - -impl DiskType { - pub fn from_string(disk_type: &str) -> Self { - match disk_type.to_uppercase().as_str() { - "SSD" => Self::SSD, - "HDD" => Self::HDD, - _ => Self::Unknown, - } - } -} - -/// Represents the filesystem type of the volume -#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, Hash)] -pub enum FileSystem { - /// Windows NTFS filesystem - NTFS, - /// FAT32 filesystem - FAT32, - /// Linux EXT4 filesystem - EXT4, - /// Apple APFS filesystem - APFS, - /// ExFAT filesystem - ExFAT, - /// Btrfs filesystem (Linux) - Btrfs, - /// ZFS filesystem - ZFS, - /// Windows ReFS filesystem - ReFS, - /// Other/unknown filesystem type - Other(String), -} - -impl fmt::Display for FileSystem { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - FileSystem::NTFS => write!(f, "NTFS"), - FileSystem::FAT32 => write!(f, "FAT32"), - FileSystem::EXT4 => write!(f, "EXT4"), - FileSystem::APFS => write!(f, "APFS"), - FileSystem::ExFAT => write!(f, "ExFAT"), - FileSystem::Btrfs => write!(f, "Btrfs"), - FileSystem::ZFS => write!(f, "ZFS"), - FileSystem::ReFS => write!(f, "ReFS"), - FileSystem::Other(name) => write!(f, "{}", name), - } - } -} - -impl FileSystem { - pub fn from_string(fs: &str) -> Self { - match fs.to_uppercase().as_str() { - "NTFS" => Self::NTFS, - "FAT32" => Self::FAT32, - "EXT4" => Self::EXT4, - "APFS" => Self::APFS, - "EXFAT" => Self::ExFAT, - "BTRFS" => Self::Btrfs, - "ZFS" => Self::ZFS, - "REFS" => Self::ReFS, - other => Self::Other(other.to_string()), - } - } - - /// Check if this filesystem supports reflinks/clones - pub fn supports_reflink(&self) -> bool { - matches!(self, Self::APFS | Self::Btrfs | Self::ZFS | Self::ReFS) - } - - /// Check if this filesystem supports sendfile optimization - pub fn supports_sendfile(&self) -> bool { - matches!(self, Self::EXT4 | Self::Btrfs | Self::ZFS | Self::NTFS) - } -} - -/// Represents how the volume is mounted in the system -#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, Hash)] -pub enum MountType { - /// System/boot volume - System, - /// External/removable volume - External, - /// Network-attached volume - Network, - /// Virtual/container volume - Virtual, -} - -impl fmt::Display for MountType { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - MountType::System => write!(f, "System"), - MountType::External => write!(f, "External"), - MountType::Network => write!(f, "Network"), - MountType::Virtual => write!(f, "Virtual"), - } - } -} - -impl MountType { - pub fn from_string(mount_type: &str) -> Self { - match mount_type.to_uppercase().as_str() { - "SYSTEM" => Self::System, - "EXTERNAL" => Self::External, - "NETWORK" => Self::Network, - "VIRTUAL" => Self::Virtual, - _ => Self::System, - } - } -} - -/// Configuration for volume detection and monitoring -#[derive(Debug, Clone)] -pub struct VolumeDetectionConfig { - /// Whether to include system volumes - pub include_system: bool, - /// Whether to include virtual volumes - pub include_virtual: bool, - /// Whether to run speed tests on discovery - pub run_speed_test: bool, - /// How often to refresh volume information (in seconds) - pub refresh_interval_secs: u64, -} - -impl Default for VolumeDetectionConfig { - fn default() -> Self { - Self { - include_system: true, - include_virtual: false, - run_speed_test: false, // Expensive operation, off by default - refresh_interval_secs: 30, - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_volume_fingerprint() { - let volume = Volume::new( - uuid::Uuid::new_v4(), - "Test Volume".to_string(), - MountType::External, - VolumeType::External, - PathBuf::from("/mnt/test"), - vec![], - DiskType::SSD, - FileSystem::EXT4, - 1000000000, - 500000000, - false, - Some("test-hw-id".to_string()), - VolumeFingerprint::new("Test", 500000000, "ext4"), - ); - - // Test basic fingerprint creation - let fingerprint = VolumeFingerprint::new( - "Test Volume", - 1000000000, // 1GB - "ext4", - ); - assert!(!fingerprint.0.is_empty()); - - // Test Spacedrive ID fingerprint - let spacedrive_id = Uuid::new_v4(); - let spacedrive_fingerprint = VolumeFingerprint::from_spacedrive_id(spacedrive_id); - assert!(!spacedrive_fingerprint.0.is_empty()); - assert_ne!(fingerprint, spacedrive_fingerprint); - } - - #[test] - fn test_volume_contains_path() { - let volume = Volume::new( - uuid::Uuid::new_v4(), - "Test".to_string(), - MountType::System, - VolumeType::System, - PathBuf::from("/home"), - vec![PathBuf::from("/home"), PathBuf::from("/mnt/home")], - DiskType::SSD, - FileSystem::EXT4, - 1000000, - 500000, - false, - None, - VolumeFingerprint::new("Test", 1000000, "ext4"), - ); - - assert!(volume.contains_path(&PathBuf::from("/home/user/file.txt"))); - assert!(volume.contains_path(&PathBuf::from("/mnt/home/user/file.txt"))); - assert!(!volume.contains_path(&PathBuf::from("/var/log/file.txt"))); - } - - #[test] - fn test_filesystem_capabilities() { - assert!(FileSystem::APFS.supports_reflink()); - assert!(FileSystem::Btrfs.supports_reflink()); - assert!(!FileSystem::FAT32.supports_reflink()); - - assert!(FileSystem::EXT4.supports_sendfile()); - assert!(!FileSystem::FAT32.supports_sendfile()); - } -} diff --git a/core-new/.scripts/combine.sh b/core/.scripts/combine.sh similarity index 100% rename from core-new/.scripts/combine.sh rename to core/.scripts/combine.sh diff --git a/core-new/test_daemon.sh b/core/.scripts/test_daemon.sh similarity index 100% rename from core-new/test_daemon.sh rename to core/.scripts/test_daemon.sh diff --git a/core-new/update_spacedrive.sh b/core/.scripts/update_spacedrive.sh similarity index 95% rename from core-new/update_spacedrive.sh rename to core/.scripts/update_spacedrive.sh index 74a51920a..fe97a921e 100644 --- a/core-new/update_spacedrive.sh +++ b/core/.scripts/update_spacedrive.sh @@ -1,7 +1,7 @@ #!/bin/bash # --- Configuration --- -PROJECT_ROOT="/Users/jamespine/Projects/spacedrive/core-new" +PROJECT_ROOT="/Users/jamespine/Projects/spacedrive/core" INSTALL_DIR="/usr/local/bin" BINARY_NAME="spacedrive" diff --git a/core-new/AGENTS.md b/core/AGENTS.md similarity index 100% rename from core-new/AGENTS.md rename to core/AGENTS.md diff --git a/core-new/Cargo.lock b/core/Cargo.lock similarity index 99% rename from core-new/Cargo.lock rename to core/Cargo.lock index 0576dfabd14dabab381d2dd43182349b9cac0d7d..fa1f43e960374f5fbee9c0fce478e5f6f6ec6eac 100644 GIT binary patch delta 48 zcmV-00MGy0s}Itv4}gRLv;u{dmrD;0S}k0l>(2KVVeRcmrB+G42N@;0=ILP G17|<<(Gy?* delta 53 zcmcb#n*Y{n{)QIDEljObS# diff --git a/core/Cargo.toml b/core/Cargo.toml index d40d2d121..cbb8fc0ef 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -1,161 +1,166 @@ [package] +edition = "2021" name = "sd-core" -version = "0.5.0" - -authors = ["Spacedrive Technology Inc "] -description = "Virtual distributed filesystem engine that powers Spacedrive." -edition.workspace = true -license.workspace = true -repository.workspace = true -rust-version.workspace = true +version = "0.1.0" [features] default = [] -# This feature allows features to be disabled when the Core is running on mobile. -mobile = [] -# This feature controls whether the Spacedrive Core contains functionality which requires FFmpeg. -ai = ["dep:sd-ai"] -ffmpeg = ["sd-core-heavy-lifting/ffmpeg", "sd-media-metadata/ffmpeg"] -heif = ["sd-images/heif"] +# FFmpeg support for video thumbnails +ffmpeg = ["dep:sd-ffmpeg"] + +[workspace] +members = ["benchmarks", "crates/*"] [dependencies] -# Inner Core Sub-crates -sd-core-cloud-services = { path = "./crates/cloud-services" } -sd-core-file-path-helper = { path = "./crates/file-path-helper" } -sd-core-heavy-lifting = { path = "./crates/heavy-lifting" } -sd-core-indexer-rules = { path = "./crates/indexer-rules" } -sd-core-prisma-helpers = { path = "./crates/prisma-helpers" } -sd-core-sync = { path = "./crates/sync" } +# Async runtime +async-trait = "0.1" +futures = "0.3" +tokio = { version = "1.40", features = ["full"] } -# Spacedrive Sub-crates -sd-actors = { path = "../crates/actors" } -sd-ai = { path = "../crates/ai", optional = true } -sd-crypto = { path = "../crates/crypto" } -sd-ffmpeg = { path = "../crates/ffmpeg", optional = true } -sd-file-ext = { path = "../crates/file-ext" } -sd-images = { path = "../crates/images", features = ["rspc", "serde", "specta"] } -sd-media-metadata = { path = "../crates/media-metadata" } -sd-old-p2p = { path = "../crates/old-p2p", features = ["specta"] } -sd-old-p2p-block = { path = "../crates/old-p2p/crates/block" } -sd-old-p2p-proto = { path = "../crates/old-p2p/crates/proto" } -sd-old-p2p-tunnel = { path = "../crates/old-p2p/crates/tunnel" } -sd-prisma = { path = "../crates/prisma" } -sd-sync = { path = "../crates/sync" } -sd-task-system = { path = "../crates/task-system" } -sd-utils = { path = "../crates/utils" } - -# Workspace dependencies -async-channel = { workspace = true } -async-stream = { workspace = true } -async-trait = { workspace = true } -axum = { workspace = true, features = ["ws"] } -base64 = { workspace = true } -blake3 = { workspace = true } -bytes = { workspace = true } -chrono = { workspace = true, features = ["serde"] } -futures = { workspace = true } -futures-concurrency = { workspace = true } -hyper = { workspace = true, features = ["client", "http1", "server"] } -image = { workspace = true } -itertools = { workspace = true } -libc = { workspace = true } -normpath = { workspace = true, features = ["localization"] } -pin-project-lite = { workspace = true } -prisma-client-rust = { workspace = true, features = ["rspc"] } -regex = { workspace = true } -reqwest = { workspace = true, features = ["json", "native-tls-vendored"] } -rmp-serde = { workspace = true } -rmpv = { workspace = true } -rspc = { workspace = true, features = ["alpha", "axum", "chrono", "unstable", "uuid"] } -sd-cloud-schema = { workspace = true } -serde = { workspace = true, features = ["derive", "rc"] } -serde_json = { workspace = true } -specta = { workspace = true } -strum = { workspace = true, features = ["derive"] } -strum_macros = { workspace = true } -tempfile = { workspace = true } -thiserror = { workspace = true } -tokio-stream = { workspace = true, features = ["fs"] } -tokio-util = { workspace = true, features = ["io"] } -tracing = { workspace = true } -tracing-subscriber = { workspace = true, features = ["env-filter"] } -uuid = { workspace = true, features = ["serde", "v4", "v7"] } - -# Specific Core dependencies -async-recursion = "1.1" -base91 = "0.1.0" -ctor = "0.2.8" -directories = "5.0" -flate2 = "1.0" -fsevent = "2.1.2" -hex = "0.4.3" -hostname = "0.4.0" -http-body = "1.0" -http-range = "0.1.5" -hyper-util = { version = "0.1.9", features = ["tokio"] } -int-enum = "0.5" # Update blocked due to API breaking changes -mini-moka = "0.10.3" -once_cell = "1.19.0" -serde-hashkey = "0.4.5" -serde_repr = "0.1.19" -serde_with = "3.8" -slotmap = "1.0" -sysinfo = "0.29.11" # Update blocked due to API breaking changes -tar = "0.4.41" -tower-service = "0.3.2" -tracing-appender = "0.2.3" -whoami = "1.5.2" - -[dependencies.tokio] -features = ["io-util", "macros", "process", "rt-multi-thread", "sync", "time"] -workspace = true - -[dependencies.notify] -default-features = false -features = ["macos_fsevent"] -git = "https://github.com/notify-rs/notify.git" -rev = "c3929ed114" - -# Override features of transitive dependencies -[dependencies.openssl] -features = ["vendored"] -version = "0.10.66" -[dependencies.openssl-sys] -features = ["vendored"] -version = "0.9.103" - -# Platform-specific dependencies -[target.'cfg(target_os = "macos")'.dependencies] -plist = "1.6" -trash = "5.1" - -[target.'cfg(target_os = "linux")'.dependencies] -inotify = "0.11.0" -trash = "5.1" - -[target.'cfg(target_os = "windows")'.dependencies] -trash = "5.1" -windows = { features = [ - "Win32_Storage_FileSystem", - "Win32_System_IO", - "Win32_System_Ioctl", - "Win32_System_WindowsProgramming" -], version = "0.58" } - -[target.'cfg(target_os = "ios")'.dependencies] -icrate = { version = "0.1.2", features = [ - "Foundation", - "Foundation_NSFileManager", - "Foundation_NSNumber", - "Foundation_NSString" +# Database +sea-orm = { version = "1.1", features = [ + "macros", + "runtime-tokio-rustls", + "sqlx-sqlite", + "uuid", + "with-chrono", + "with-json" ] } +sea-orm-migration = { version = "1.1", features = ["runtime-tokio-rustls", "sqlx-sqlite"] } +sqlx = { version = "0.8", features = ["runtime-tokio-rustls", "sqlite"] } + +# API (temporarily disabled) +# axum = "0.7" +# async-graphql = "7.0" +# async-graphql-axum = "7.0" + +# Serialization +int-enum = "1.1" +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" +strum = { version = "0.26", features = ["derive"] } +toml = "0.8" + +# Error handling +anyhow = "1.0" +thiserror = "1.0" + + +# File operations +blake3 = "1.5" # Content addressing +hex = "0.4" # Hex encoding for volume fingerprints +notify = "6.1" # File system watching +sha2 = "0.10" # SHA-256 hashing for CAS IDs + +# Logging +tracing = "0.1" +tracing-subscriber = { version = "0.3", features = ["env-filter"] } + +# Indexer rules engine +futures-concurrency = "7.6" +gix-ignore = { version = "0.11", features = ["serde"] } +globset = { version = "0.4", features = ["serde1"] } + +# Job system dependencies +inventory = "0.3" # Automatic job registration +rmp = "0.8" # MessagePack core types +rmp-serde = "1.3" # MessagePack serialization for job state +sd-task-system = { path = "crates/task-system" } +spacedrive-jobs-derive = { path = "crates/spacedrive-jobs-derive" } # Job derive macros + +# Media processing dependencies +image = "0.25" +sd-ffmpeg = { path = "crates/ffmpeg", optional = true } +sd-images = { path = "crates/images" } +tokio-rustls = "0.26" +webp = "0.3" + +# Networking +# Iroh P2P networking +iroh = "0.28" +iroh-blobs = "0.28" +iroh-gossip = "0.28" +iroh-net = "0.28" + +# Serialization for protocols +serde_cbor = "0.11" + +# Cryptography for signing (backward compatibility) +ed25519-dalek = "2.1" + +# Legacy networking (kept for compatibility during transition) +aes-gcm = "0.10" # AES-GCM encryption for secure storage +argon2 = "0.5" # Password derivation +async-stream = "0.3" # File streaming +backoff = "0.4" # Retry logic +bincode = "2.0.0-rc.3" # Efficient encoding +mdns-sd = "0.13" # mDNS service discovery (DEPRECATED - use libp2p DHT) +ring = "0.16" # Crypto primitives +snow = "0.9" # Noise Protocol encryption (DEPRECATED - use libp2p noise) +# futures-util = "0.3" # WebSocket utilities (disabled for now) +rcgen = "0.11" # Certificate generation (DEPRECATED - use libp2p noise) +rustls = { version = "0.23", features = [ + "aws_lc_rs" +] } # TLS implementation (DEPRECATED - use libp2p noise) +tokio-stream = "0.1" # Async streams + +# BIP39 wordlist support +bip39 = "2.0" + +# Additional cryptography +chacha20poly1305 = "0.10" # Authenticated encryption for chunk-level security +hkdf = "0.12" # Key derivation function for session keys +hmac = "0.12" +x25519-dalek = "2.0" + +# Network utilities +if-watch = "3.0" +local-ip-address = "0.5" +# colored already defined above + +# Utils +chrono = { version = "0.4", features = ["serde"] } +dirs = "5.0" +once_cell = "1.20" +rand = "0.8" # Random number generation for secure delete +tempfile = "3.14" # Temporary directories for testing +uuid = { version = "1.11", features = ["serde", "v4", "v5", "v7"] } +whoami = "1.5" + +# Secure storage +keyring = "3.6" + +# CLI dependencies +clap = { version = "4.5", features = ["derive", "env"] } +colored = "2.1" +comfy-table = "7.1" +console = "0.15" +dialoguer = "0.11" +indicatif = "0.17" +owo-colors = "4.1" +supports-color = "3.0" + +[build-dependencies] +colored = "2.1" +comfy-table = "7.1" +console = "0.15" +crossterm = "0.28" +dialoguer = "0.11" +indicatif = "0.17" +owo-colors = "4.1" +ratatui = "0.29" +supports-color = "3.0" +vergen = { version = "8", features = ["cargo", "git", "gitcl"] } + +# Platform specific +[target.'cfg(unix)'.dependencies] +libc = "0.2" + +[[bin]] +name = "spacedrive" +path = "src/bin/cli.rs" -[target.'cfg(target_os = "android")'.dependencies] -tracing-android = "0.2.0" [dev-dependencies] -# Workspace dependencies -tracing-test = { workspace = true } - -# Specific Core dependencies -boxcar = "0.2.5" +pretty_assertions = "1.4" +tempfile = "3.14" diff --git a/core-new/README.md b/core/README.md similarity index 99% rename from core-new/README.md rename to core/README.md index c14b61e47..2898235e3 100644 --- a/core-new/README.md +++ b/core/README.md @@ -293,7 +293,7 @@ Currently working features: ```bash # Clone and build git clone https://github.com/spacedriveapp/spacedrive -cd spacedrive/core-new +cd spacedrive/core cargo build --release # Try the CLI diff --git a/core/benchmarks/Cargo.toml b/core/benchmarks/Cargo.toml new file mode 100644 index 000000000..1fe95233b --- /dev/null +++ b/core/benchmarks/Cargo.toml @@ -0,0 +1,37 @@ +[package] +edition = "2021" +name = "sd-bench" +version = "0.1.0" + +[dependencies] +anyhow = "1.0" +async-trait = "0.1" +blake3 = "1.5" +chrono = { version = "0.4", features = ["serde"] } +clap = { version = "4.5", features = ["derive", "env"] } +dirs = "5.0" +humantime = "2.1" +humantime-serde = "1.1" +indicatif = "0.17" +rand = "0.8" +regex = "1.10" +sd-core = { path = ".." } +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" +serde_with = { version = "3.9", features = ["json"] } +serde_yaml = "0.9" +sysinfo = { version = "0.30", default-features = false, features = ["multithread"] } +tempfile = "3.14" +tokio = { version = "1.40", features = ["full"] } +tracing = "0.1" +tracing-subscriber = { version = "0.3", features = ["env-filter"] } +uuid = { version = "1.11", features = ["serde", "v4"] } +walkdir = "2.5" + +[lib] +name = "sd_bench" +path = "src/lib.rs" + +[[bin]] +name = "sd-bench" +path = "src/bin/sd-bench-new.rs" diff --git a/core-new/benchmarks/recipes/shape_large.yaml b/core/benchmarks/recipes/shape_large.yaml similarity index 100% rename from core-new/benchmarks/recipes/shape_large.yaml rename to core/benchmarks/recipes/shape_large.yaml diff --git a/core-new/benchmarks/recipes/shape_medium.yaml b/core/benchmarks/recipes/shape_medium.yaml similarity index 100% rename from core-new/benchmarks/recipes/shape_medium.yaml rename to core/benchmarks/recipes/shape_medium.yaml diff --git a/core-new/benchmarks/recipes/shape_small.yaml b/core/benchmarks/recipes/shape_small.yaml similarity index 100% rename from core-new/benchmarks/recipes/shape_small.yaml rename to core/benchmarks/recipes/shape_small.yaml diff --git a/core-new/benchmarks/results/shape_large-aggregation-hdd.json b/core/benchmarks/results/shape_large-aggregation-hdd.json similarity index 100% rename from core-new/benchmarks/results/shape_large-aggregation-hdd.json rename to core/benchmarks/results/shape_large-aggregation-hdd.json diff --git a/core-new/benchmarks/results/shape_large-aggregation-ssd.json b/core/benchmarks/results/shape_large-aggregation-ssd.json similarity index 90% rename from core-new/benchmarks/results/shape_large-aggregation-ssd.json rename to core/benchmarks/results/shape_large-aggregation-ssd.json index 51e98565f..7cb6dc483 100644 --- a/core-new/benchmarks/results/shape_large-aggregation-ssd.json +++ b/core/benchmarks/results/shape_large-aggregation-ssd.json @@ -21,7 +21,7 @@ }, "id": "6a501e72-da12-43bc-a4f6-e336ec7a1d56", "location_paths": [ - "/Users/jamespine/Projects/spacedrive/core-new/benchdata/shape_large" + "/Users/jamespine/Projects/spacedrive/core/benchdata/shape_large" ], "recipe_name": "shape_large", "timestamp_utc": "2025-08-10T09:12:02.347142+00:00" diff --git a/core-new/benchmarks/results/shape_large-content_identification-hdd.json b/core/benchmarks/results/shape_large-content_identification-hdd.json similarity index 100% rename from core-new/benchmarks/results/shape_large-content_identification-hdd.json rename to core/benchmarks/results/shape_large-content_identification-hdd.json diff --git a/core-new/benchmarks/results/shape_large-content_identification-ssd.json b/core/benchmarks/results/shape_large-content_identification-ssd.json similarity index 90% rename from core-new/benchmarks/results/shape_large-content_identification-ssd.json rename to core/benchmarks/results/shape_large-content_identification-ssd.json index e5bec83d3..5a9156209 100644 --- a/core-new/benchmarks/results/shape_large-content_identification-ssd.json +++ b/core/benchmarks/results/shape_large-content_identification-ssd.json @@ -21,7 +21,7 @@ }, "id": "8d825ffa-d866-4fe7-b8fb-52559e63191e", "location_paths": [ - "/Users/jamespine/Projects/spacedrive/core-new/benchdata/shape_large" + "/Users/jamespine/Projects/spacedrive/core/benchdata/shape_large" ], "recipe_name": "shape_large", "timestamp_utc": "2025-08-10T09:35:52.292540+00:00" diff --git a/core-new/benchmarks/results/shape_large-indexing_discovery-hdd.json b/core/benchmarks/results/shape_large-indexing_discovery-hdd.json similarity index 100% rename from core-new/benchmarks/results/shape_large-indexing_discovery-hdd.json rename to core/benchmarks/results/shape_large-indexing_discovery-hdd.json diff --git a/core-new/benchmarks/results/shape_large-indexing_discovery-ssd.json b/core/benchmarks/results/shape_large-indexing_discovery-ssd.json similarity index 90% rename from core-new/benchmarks/results/shape_large-indexing_discovery-ssd.json rename to core/benchmarks/results/shape_large-indexing_discovery-ssd.json index a82d6d74d..489cdf5d9 100644 --- a/core-new/benchmarks/results/shape_large-indexing_discovery-ssd.json +++ b/core/benchmarks/results/shape_large-indexing_discovery-ssd.json @@ -21,7 +21,7 @@ }, "id": "942a9485-bfc7-4af6-9e98-9544f55b9642", "location_paths": [ - "/Users/jamespine/Projects/spacedrive/core-new/benchdata/shape_large" + "/Users/jamespine/Projects/spacedrive/core/benchdata/shape_large" ], "recipe_name": "shape_large", "timestamp_utc": "2025-08-10T09:10:43.892804+00:00" diff --git a/core-new/benchmarks/results/shape_medium-aggregation-hdd.json b/core/benchmarks/results/shape_medium-aggregation-hdd.json similarity index 100% rename from core-new/benchmarks/results/shape_medium-aggregation-hdd.json rename to core/benchmarks/results/shape_medium-aggregation-hdd.json diff --git a/core-new/benchmarks/results/shape_medium-aggregation-ssd.json b/core/benchmarks/results/shape_medium-aggregation-ssd.json similarity index 90% rename from core-new/benchmarks/results/shape_medium-aggregation-ssd.json rename to core/benchmarks/results/shape_medium-aggregation-ssd.json index ee3adf3fc..34001b9ba 100644 --- a/core-new/benchmarks/results/shape_medium-aggregation-ssd.json +++ b/core/benchmarks/results/shape_medium-aggregation-ssd.json @@ -21,7 +21,7 @@ }, "id": "7cb7172d-a51a-4783-a3c8-ff5c82024ba3", "location_paths": [ - "/Users/jamespine/Projects/spacedrive/core-new/benchdata/shape_medium" + "/Users/jamespine/Projects/spacedrive/core/benchdata/shape_medium" ], "recipe_name": "shape_medium", "timestamp_utc": "2025-08-10T09:12:12.291950+00:00" diff --git a/core-new/benchmarks/results/shape_medium-content_identification-hdd.json b/core/benchmarks/results/shape_medium-content_identification-hdd.json similarity index 100% rename from core-new/benchmarks/results/shape_medium-content_identification-hdd.json rename to core/benchmarks/results/shape_medium-content_identification-hdd.json diff --git a/core-new/benchmarks/results/shape_medium-content_identification-ssd.json b/core/benchmarks/results/shape_medium-content_identification-ssd.json similarity index 90% rename from core-new/benchmarks/results/shape_medium-content_identification-ssd.json rename to core/benchmarks/results/shape_medium-content_identification-ssd.json index adaa98a52..4afc7e824 100644 --- a/core-new/benchmarks/results/shape_medium-content_identification-ssd.json +++ b/core/benchmarks/results/shape_medium-content_identification-ssd.json @@ -21,7 +21,7 @@ }, "id": "62872b39-d91b-4195-89e5-d63986c516ea", "location_paths": [ - "/Users/jamespine/Projects/spacedrive/core-new/benchdata/shape_medium" + "/Users/jamespine/Projects/spacedrive/core/benchdata/shape_medium" ], "recipe_name": "shape_medium", "timestamp_utc": "2025-08-10T09:40:28.247578+00:00" diff --git a/core-new/benchmarks/results/shape_medium-indexing_discovery-hdd.json b/core/benchmarks/results/shape_medium-indexing_discovery-hdd.json similarity index 100% rename from core-new/benchmarks/results/shape_medium-indexing_discovery-hdd.json rename to core/benchmarks/results/shape_medium-indexing_discovery-hdd.json diff --git a/core-new/benchmarks/results/shape_medium-indexing_discovery-ssd.json b/core/benchmarks/results/shape_medium-indexing_discovery-ssd.json similarity index 90% rename from core-new/benchmarks/results/shape_medium-indexing_discovery-ssd.json rename to core/benchmarks/results/shape_medium-indexing_discovery-ssd.json index 836a8b12b..5dca3979c 100644 --- a/core-new/benchmarks/results/shape_medium-indexing_discovery-ssd.json +++ b/core/benchmarks/results/shape_medium-indexing_discovery-ssd.json @@ -21,7 +21,7 @@ }, "id": "e3a3c71e-1850-4f71-9d4f-5ef8b9a94df5", "location_paths": [ - "/Users/jamespine/Projects/spacedrive/core-new/benchdata/shape_medium" + "/Users/jamespine/Projects/spacedrive/core/benchdata/shape_medium" ], "recipe_name": "shape_medium", "timestamp_utc": "2025-08-10T09:10:54.220721+00:00" diff --git a/core-new/benchmarks/results/shape_small-aggregation-hdd.json b/core/benchmarks/results/shape_small-aggregation-hdd.json similarity index 100% rename from core-new/benchmarks/results/shape_small-aggregation-hdd.json rename to core/benchmarks/results/shape_small-aggregation-hdd.json diff --git a/core-new/benchmarks/results/shape_small-aggregation-ssd.json b/core/benchmarks/results/shape_small-aggregation-ssd.json similarity index 90% rename from core-new/benchmarks/results/shape_small-aggregation-ssd.json rename to core/benchmarks/results/shape_small-aggregation-ssd.json index 109e072c6..cb16c99c2 100644 --- a/core-new/benchmarks/results/shape_small-aggregation-ssd.json +++ b/core/benchmarks/results/shape_small-aggregation-ssd.json @@ -21,7 +21,7 @@ }, "id": "28911e81-983c-4680-9cfd-287b91589650", "location_paths": [ - "/Users/jamespine/Projects/spacedrive/core-new/benchdata/shape_small" + "/Users/jamespine/Projects/spacedrive/core/benchdata/shape_small" ], "recipe_name": "shape_small", "timestamp_utc": "2025-08-10T09:12:16.491885+00:00" diff --git a/core-new/benchmarks/results/shape_small-content_identification-hdd.json b/core/benchmarks/results/shape_small-content_identification-hdd.json similarity index 100% rename from core-new/benchmarks/results/shape_small-content_identification-hdd.json rename to core/benchmarks/results/shape_small-content_identification-hdd.json diff --git a/core-new/benchmarks/results/shape_small-content_identification-ssd.json b/core/benchmarks/results/shape_small-content_identification-ssd.json similarity index 90% rename from core-new/benchmarks/results/shape_small-content_identification-ssd.json rename to core/benchmarks/results/shape_small-content_identification-ssd.json index 82301139a..d9cc4b67e 100644 --- a/core-new/benchmarks/results/shape_small-content_identification-ssd.json +++ b/core/benchmarks/results/shape_small-content_identification-ssd.json @@ -21,7 +21,7 @@ }, "id": "cfe4988b-bfa4-4b9f-9632-4bc02d9738c7", "location_paths": [ - "/Users/jamespine/Projects/spacedrive/core-new/benchdata/shape_small" + "/Users/jamespine/Projects/spacedrive/core/benchdata/shape_small" ], "recipe_name": "shape_small", "timestamp_utc": "2025-08-10T09:41:41.304584+00:00" diff --git a/core-new/benchmarks/results/shape_small-indexing_discovery-hdd.json b/core/benchmarks/results/shape_small-indexing_discovery-hdd.json similarity index 100% rename from core-new/benchmarks/results/shape_small-indexing_discovery-hdd.json rename to core/benchmarks/results/shape_small-indexing_discovery-hdd.json diff --git a/core-new/benchmarks/results/shape_small-indexing_discovery-ssd.json b/core/benchmarks/results/shape_small-indexing_discovery-ssd.json similarity index 90% rename from core-new/benchmarks/results/shape_small-indexing_discovery-ssd.json rename to core/benchmarks/results/shape_small-indexing_discovery-ssd.json index 1199a0474..cc8416da0 100644 --- a/core-new/benchmarks/results/shape_small-indexing_discovery-ssd.json +++ b/core/benchmarks/results/shape_small-indexing_discovery-ssd.json @@ -21,7 +21,7 @@ }, "id": "1c95ba05-cc82-4947-966a-b4944cc030d8", "location_paths": [ - "/Users/jamespine/Projects/spacedrive/core-new/benchdata/shape_small" + "/Users/jamespine/Projects/spacedrive/core/benchdata/shape_small" ], "recipe_name": "shape_small", "timestamp_utc": "2025-08-10T09:10:58.068059+00:00" diff --git a/core-new/benchmarks/results/whitepaper_metrics.csv b/core/benchmarks/results/whitepaper_metrics.csv similarity index 100% rename from core-new/benchmarks/results/whitepaper_metrics.csv rename to core/benchmarks/results/whitepaper_metrics.csv diff --git a/core-new/benchmarks/src/bin/sd-bench-new.rs b/core/benchmarks/src/bin/sd-bench-new.rs similarity index 100% rename from core-new/benchmarks/src/bin/sd-bench-new.rs rename to core/benchmarks/src/bin/sd-bench-new.rs diff --git a/core-new/benchmarks/src/cli/args.rs b/core/benchmarks/src/cli/args.rs similarity index 100% rename from core-new/benchmarks/src/cli/args.rs rename to core/benchmarks/src/cli/args.rs diff --git a/core-new/benchmarks/src/cli/commands.rs b/core/benchmarks/src/cli/commands.rs similarity index 100% rename from core-new/benchmarks/src/cli/commands.rs rename to core/benchmarks/src/cli/commands.rs diff --git a/core-new/benchmarks/src/cli/mod.rs b/core/benchmarks/src/cli/mod.rs similarity index 100% rename from core-new/benchmarks/src/cli/mod.rs rename to core/benchmarks/src/cli/mod.rs diff --git a/core-new/benchmarks/src/config/mod.rs b/core/benchmarks/src/config/mod.rs similarity index 100% rename from core-new/benchmarks/src/config/mod.rs rename to core/benchmarks/src/config/mod.rs diff --git a/core-new/benchmarks/src/core_boot/mod.rs b/core/benchmarks/src/core_boot/mod.rs similarity index 100% rename from core-new/benchmarks/src/core_boot/mod.rs rename to core/benchmarks/src/core_boot/mod.rs diff --git a/core-new/benchmarks/src/generator/filesystem.rs b/core/benchmarks/src/generator/filesystem.rs similarity index 100% rename from core-new/benchmarks/src/generator/filesystem.rs rename to core/benchmarks/src/generator/filesystem.rs diff --git a/core-new/benchmarks/src/generator/mod.rs b/core/benchmarks/src/generator/mod.rs similarity index 100% rename from core-new/benchmarks/src/generator/mod.rs rename to core/benchmarks/src/generator/mod.rs diff --git a/core-new/benchmarks/src/generator/noop.rs b/core/benchmarks/src/generator/noop.rs similarity index 100% rename from core-new/benchmarks/src/generator/noop.rs rename to core/benchmarks/src/generator/noop.rs diff --git a/core-new/benchmarks/src/generator/registry.rs b/core/benchmarks/src/generator/registry.rs similarity index 100% rename from core-new/benchmarks/src/generator/registry.rs rename to core/benchmarks/src/generator/registry.rs diff --git a/core-new/benchmarks/src/lib.rs b/core/benchmarks/src/lib.rs similarity index 100% rename from core-new/benchmarks/src/lib.rs rename to core/benchmarks/src/lib.rs diff --git a/core-new/benchmarks/src/metrics/mod.rs b/core/benchmarks/src/metrics/mod.rs similarity index 100% rename from core-new/benchmarks/src/metrics/mod.rs rename to core/benchmarks/src/metrics/mod.rs diff --git a/core-new/benchmarks/src/metrics/sources.rs b/core/benchmarks/src/metrics/sources.rs similarity index 100% rename from core-new/benchmarks/src/metrics/sources.rs rename to core/benchmarks/src/metrics/sources.rs diff --git a/core-new/benchmarks/src/recipe/mod.rs b/core/benchmarks/src/recipe/mod.rs similarity index 100% rename from core-new/benchmarks/src/recipe/mod.rs rename to core/benchmarks/src/recipe/mod.rs diff --git a/core-new/benchmarks/src/recipe/schema.rs b/core/benchmarks/src/recipe/schema.rs similarity index 100% rename from core-new/benchmarks/src/recipe/schema.rs rename to core/benchmarks/src/recipe/schema.rs diff --git a/core-new/benchmarks/src/reporting/csv.rs b/core/benchmarks/src/reporting/csv.rs similarity index 100% rename from core-new/benchmarks/src/reporting/csv.rs rename to core/benchmarks/src/reporting/csv.rs diff --git a/core-new/benchmarks/src/reporting/json_summary.rs b/core/benchmarks/src/reporting/json_summary.rs similarity index 100% rename from core-new/benchmarks/src/reporting/json_summary.rs rename to core/benchmarks/src/reporting/json_summary.rs diff --git a/core-new/benchmarks/src/reporting/mod.rs b/core/benchmarks/src/reporting/mod.rs similarity index 100% rename from core-new/benchmarks/src/reporting/mod.rs rename to core/benchmarks/src/reporting/mod.rs diff --git a/core-new/benchmarks/src/reporting/registry.rs b/core/benchmarks/src/reporting/registry.rs similarity index 100% rename from core-new/benchmarks/src/reporting/registry.rs rename to core/benchmarks/src/reporting/registry.rs diff --git a/core-new/benchmarks/src/runner/mod.rs b/core/benchmarks/src/runner/mod.rs similarity index 100% rename from core-new/benchmarks/src/runner/mod.rs rename to core/benchmarks/src/runner/mod.rs diff --git a/core-new/benchmarks/src/runner/monitor.rs b/core/benchmarks/src/runner/monitor.rs similarity index 100% rename from core-new/benchmarks/src/runner/monitor.rs rename to core/benchmarks/src/runner/monitor.rs diff --git a/core-new/benchmarks/src/scenarios/common.rs b/core/benchmarks/src/scenarios/common.rs similarity index 100% rename from core-new/benchmarks/src/scenarios/common.rs rename to core/benchmarks/src/scenarios/common.rs diff --git a/core-new/benchmarks/src/scenarios/content_identification.rs b/core/benchmarks/src/scenarios/content_identification.rs similarity index 100% rename from core-new/benchmarks/src/scenarios/content_identification.rs rename to core/benchmarks/src/scenarios/content_identification.rs diff --git a/core-new/benchmarks/src/scenarios/core_indexing.rs b/core/benchmarks/src/scenarios/core_indexing.rs similarity index 100% rename from core-new/benchmarks/src/scenarios/core_indexing.rs rename to core/benchmarks/src/scenarios/core_indexing.rs diff --git a/core-new/benchmarks/src/scenarios/mod.rs b/core/benchmarks/src/scenarios/mod.rs similarity index 100% rename from core-new/benchmarks/src/scenarios/mod.rs rename to core/benchmarks/src/scenarios/mod.rs diff --git a/core-new/benchmarks/src/scenarios/registry.rs b/core/benchmarks/src/scenarios/registry.rs similarity index 100% rename from core-new/benchmarks/src/scenarios/registry.rs rename to core/benchmarks/src/scenarios/registry.rs diff --git a/core-new/benchmarks/src/util/fs.rs b/core/benchmarks/src/util/fs.rs similarity index 100% rename from core-new/benchmarks/src/util/fs.rs rename to core/benchmarks/src/util/fs.rs diff --git a/core-new/benchmarks/src/util/mod.rs b/core/benchmarks/src/util/mod.rs similarity index 100% rename from core-new/benchmarks/src/util/mod.rs rename to core/benchmarks/src/util/mod.rs diff --git a/core-new/benchmarks/src/util/rng.rs b/core/benchmarks/src/util/rng.rs similarity index 100% rename from core-new/benchmarks/src/util/rng.rs rename to core/benchmarks/src/util/rng.rs diff --git a/core-new/benchmarks/src/util/time.rs b/core/benchmarks/src/util/time.rs similarity index 100% rename from core-new/benchmarks/src/util/time.rs rename to core/benchmarks/src/util/time.rs diff --git a/core/build.rs b/core/build.rs index af19d274e..4b32dbe9b 100644 --- a/core/build.rs +++ b/core/build.rs @@ -1,11 +1,13 @@ -use std::process::Command; +use vergen::EmitBuilder; -fn main() { - let output = Command::new("git") - .args(["rev-parse", "--short", "HEAD"]) - .output() - .expect("error getting git hash. Does `git rev-parse --short HEAD` work for you?"); - let git_hash = String::from_utf8(output.stdout) - .expect("Error passing output of `git rev-parse --short HEAD`"); - println!("cargo:rustc-env=GIT_HASH={git_hash}"); +fn main() -> Result<(), Box> { + // Emit the instructions + EmitBuilder::builder() + .git_sha(true) + .git_commit_timestamp() + .git_branch() + .cargo_opt_level() + .cargo_target_triple() + .emit()?; + Ok(()) } diff --git a/core/crates/cloud-services/Cargo.toml b/core/crates/cloud-services/Cargo.toml deleted file mode 100644 index 1dc8b6d71..000000000 --- a/core/crates/cloud-services/Cargo.toml +++ /dev/null @@ -1,57 +0,0 @@ -[package] -name = "sd-core-cloud-services" -version = "0.1.0" - -edition = "2021" - -[dependencies] -# Core Spacedrive Sub-crates -sd-core-heavy-lifting = { path = "../heavy-lifting" } -sd-core-prisma-helpers = { path = "../prisma-helpers" } -sd-core-sync = { path = "../sync" } - -# Spacedrive Sub-crates -sd-actors = { path = "../../../crates/actors" } -sd-cloud-schema = { workspace = true } -sd-crypto = { path = "../../../crates/crypto" } -sd-prisma = { path = "../../../crates/prisma" } -sd-utils = { path = "../../../crates/utils" } - -# Workspace dependencies -anyhow = { workspace = true } -async-stream = { workspace = true } -base64 = { workspace = true } -blake3 = { workspace = true } -chrono = { workspace = true, features = ["serde"] } -flume = { workspace = true } -futures = { workspace = true } -futures-concurrency = { workspace = true } -iroh = { workspace = true, features = ["discovery-local-network"] } -quic-rpc = { workspace = true, features = ["iroh-transport", "quinn-transport"] } -rmp-serde = { workspace = true } -rspc = { workspace = true } -serde = { workspace = true, features = ["derive"] } -serde_json = { workspace = true } -specta = { workspace = true } -thiserror = { workspace = true } -tokio = { workspace = true, features = ["sync", "time"] } -tokio-stream = { workspace = true } -tokio-util = { workspace = true } -tracing = { workspace = true } -uuid = { workspace = true, features = ["serde"] } -zeroize = { workspace = true } - -# External dependencies -dashmap = "6.1.0" -paste = "=1.0.15" -quinn = { package = "iroh-quinn", version = "0.12" } -# Using whatever version of reqwest that reqwest-middleware uses, just putting here to enable some features -reqwest = { version = "0.12", features = ["json", "native-tls-vendored", "stream"] } -reqwest-middleware = { version = "0.4", features = ["json"] } -reqwest-retry = "0.7" -rustls = { version = "=0.23.19", default-features = false, features = ["brotli", "ring", "std"] } -rustls-platform-verifier = "0.4.0" - - -[dev-dependencies] -tokio = { workspace = true, features = ["rt", "sync", "time"] } diff --git a/core/crates/cloud-services/src/client.rs b/core/crates/cloud-services/src/client.rs deleted file mode 100644 index d28037357..000000000 --- a/core/crates/cloud-services/src/client.rs +++ /dev/null @@ -1,358 +0,0 @@ -use crate::p2p::{NotifyUser, UserResponse}; - -use sd_cloud_schema::{Client, Service, ServicesALPN}; - -use std::{net::SocketAddr, sync::Arc, time::Duration}; - -use futures::Stream; -use iroh::relay::RelayUrl; -use quic_rpc::{client::QuinnConnector, RpcClient}; -use quinn::{crypto::rustls::QuicClientConfig, ClientConfig, Endpoint}; -use reqwest::{IntoUrl, Url}; -use reqwest_middleware::{reqwest, ClientBuilder, ClientWithMiddleware}; -// use reqwest_retry::{policies::ExponentialBackoff, RetryTransientMiddleware}; -use tokio::sync::{Mutex, RwLock}; -use tracing::warn; - -use super::{ - error::Error, key_manager::KeyManager, p2p::CloudP2P, token_refresher::TokenRefresher, -}; - -pub type CloudServicesClient = Client>; - -#[derive(Debug, Default, Clone)] -enum ClientState { - #[default] - NotConnected, - Connected(CloudServicesClient), -} - -/// Cloud services are a optional feature that allows you to interact with the cloud services -/// of Spacedrive. -/// They're optional in two different ways: -/// - The cloud services depends on a user being logged in with our server. -/// - The user being connected to the internet to begin with. -/// -/// As we don't want to force the user to be connected to the internet, we have to make sure -/// that core can always operate without the cloud services. -#[derive(Debug)] -pub struct CloudServices { - client_state: Arc>, - get_cloud_api_address: Url, - http_client: ClientWithMiddleware, - domain_name: String, - pub cloud_p2p_dns_origin_name: String, - pub cloud_p2p_relay_url: RelayUrl, - pub cloud_p2p_dns_pkarr_url: Url, - pub token_refresher: TokenRefresher, - key_manager: Arc>>>, - cloud_p2p: Arc>>>, - pub(crate) notify_user_tx: flume::Sender, - notify_user_rx: flume::Receiver, - user_response_tx: flume::Sender, - pub(crate) user_response_rx: flume::Receiver, - pub has_bootstrapped: Arc>, -} - -impl CloudServices { - /// Creates a new cloud services client that can be used to interact with the cloud services. - /// The client will try to connect to the cloud services on a best effort basis, as the user - /// might not be connected to the internet. - /// If the client fails to connect, it will try again the next time it's used. - pub async fn new( - get_cloud_api_address: impl IntoUrl + Send, - cloud_p2p_relay_url: impl IntoUrl + Send, - cloud_p2p_dns_pkarr_url: impl IntoUrl + Send, - cloud_p2p_dns_origin_name: String, - domain_name: String, - ) -> Result { - let mut http_client_builder = reqwest::Client::builder().timeout(Duration::from_secs(3)); - - #[cfg(not(debug_assertions))] - { - http_client_builder = http_client_builder.https_only(true); - } - - let cloud_p2p_relay_url = cloud_p2p_relay_url - .into_url() - .map_err(Error::InvalidUrl)? - .into(); - - let cloud_p2p_dns_pkarr_url = cloud_p2p_dns_pkarr_url - .into_url() - .map_err(Error::InvalidUrl)?; - - let http_client = - ClientBuilder::new(http_client_builder.build().map_err(Error::HttpClientInit)?) - // TODO: Re-enable retry middleware. It's currently disabled because it's causing blocking issues on mobile core initialization. - // .with(RetryTransientMiddleware::new_with_policy( - // ExponentialBackoff::builder().build_with_max_retries(3), - // )) - .build(); - let get_cloud_api_address = get_cloud_api_address - .into_url() - .map_err(Error::InvalidUrl)?; - - let client_state = match Self::init_client( - &http_client, - get_cloud_api_address.clone(), - domain_name.clone(), - ) - .await - { - Ok(client) => Arc::new(RwLock::new(ClientState::Connected(client))), - Err(e) => { - warn!( - ?e, - "Failed to initialize cloud services client; \ - This is a best effort and we will continue in Not Connected mode" - ); - Arc::new(RwLock::new(ClientState::NotConnected)) - } - }; - - let (notify_user_tx, notify_user_rx) = flume::bounded(16); - let (user_response_tx, user_response_rx) = flume::bounded(16); - - Ok(Self { - client_state, - token_refresher: TokenRefresher::new( - http_client.clone(), - get_cloud_api_address.clone(), - ), - get_cloud_api_address, - http_client, - cloud_p2p_dns_origin_name, - cloud_p2p_relay_url, - cloud_p2p_dns_pkarr_url, - domain_name, - key_manager: Arc::default(), - cloud_p2p: Arc::default(), - notify_user_tx, - notify_user_rx, - user_response_tx, - user_response_rx, - has_bootstrapped: Arc::default(), - }) - } - - pub fn stream_user_notifications(&self) -> impl Stream + '_ { - self.notify_user_rx.stream() - } - - #[must_use] - pub const fn http_client(&self) -> &ClientWithMiddleware { - &self.http_client - } - - /// Send back a user response to the Cloud P2P actor - /// - /// # Panics - /// Will panic if the channel is closed, which should never happen - pub async fn send_user_response(&self, response: UserResponse) { - self.user_response_tx - .send_async(response) - .await - .expect("user response channel must never close"); - } - - async fn init_client( - http_client: &ClientWithMiddleware, - get_cloud_api_address: Url, - domain_name: String, - ) -> Result { - let cloud_api_address = http_client - .get(get_cloud_api_address) - .send() - .await - .map_err(Error::FailedToRequestApiAddress)? - .error_for_status() - .map_err(Error::AuthServerError)? - .text() - .await - .map_err(Error::FailedToExtractApiAddress)? - .parse::()?; - - let mut crypto_config = { - #[cfg(debug_assertions)] - { - #[derive(Debug)] - struct SkipServerVerification; - impl rustls::client::danger::ServerCertVerifier for SkipServerVerification { - fn verify_server_cert( - &self, - _end_entity: &rustls::pki_types::CertificateDer<'_>, - _intermediates: &[rustls::pki_types::CertificateDer<'_>], - _server_name: &rustls::pki_types::ServerName<'_>, - _ocsp_response: &[u8], - _now: rustls::pki_types::UnixTime, - ) -> Result { - Ok(rustls::client::danger::ServerCertVerified::assertion()) - } - - fn verify_tls12_signature( - &self, - _message: &[u8], - _cert: &rustls::pki_types::CertificateDer<'_>, - _dss: &rustls::DigitallySignedStruct, - ) -> Result - { - Ok(rustls::client::danger::HandshakeSignatureValid::assertion()) - } - - fn verify_tls13_signature( - &self, - _message: &[u8], - _cert: &rustls::pki_types::CertificateDer<'_>, - _dss: &rustls::DigitallySignedStruct, - ) -> Result - { - Ok(rustls::client::danger::HandshakeSignatureValid::assertion()) - } - - fn supported_verify_schemes(&self) -> Vec { - vec![ - rustls::SignatureScheme::RSA_PKCS1_SHA1, - rustls::SignatureScheme::ECDSA_SHA1_Legacy, - rustls::SignatureScheme::RSA_PKCS1_SHA256, - rustls::SignatureScheme::ECDSA_NISTP256_SHA256, - rustls::SignatureScheme::RSA_PKCS1_SHA384, - rustls::SignatureScheme::ECDSA_NISTP384_SHA384, - rustls::SignatureScheme::RSA_PKCS1_SHA512, - rustls::SignatureScheme::ECDSA_NISTP521_SHA512, - rustls::SignatureScheme::RSA_PSS_SHA256, - rustls::SignatureScheme::RSA_PSS_SHA384, - rustls::SignatureScheme::RSA_PSS_SHA512, - rustls::SignatureScheme::ED25519, - rustls::SignatureScheme::ED448, - ] - } - } - - rustls::ClientConfig::builder_with_protocol_versions(&[&rustls::version::TLS13]) - .dangerous() - .with_custom_certificate_verifier(Arc::new(SkipServerVerification)) - .with_no_client_auth() - } - - #[cfg(not(debug_assertions))] - { - rustls::ClientConfig::builder_with_protocol_versions(&[&rustls::version::TLS13]) - .dangerous() - .with_custom_certificate_verifier(Arc::new( - rustls_platform_verifier::Verifier::new(), - )) - .with_no_client_auth() - } - }; - - crypto_config - .alpn_protocols - .extend([ServicesALPN::LATEST.to_vec()]); - - let client_config = ClientConfig::new(Arc::new( - QuicClientConfig::try_from(crypto_config) - .expect("misconfigured TLS client config, this is a bug and should crash"), - )); - - let mut endpoint = Endpoint::client("[::]:0".parse().expect("hardcoded address")) - .map_err(Error::FailedToCreateEndpoint)?; - endpoint.set_default_client_config(client_config); - - Ok(Client::new(RpcClient::new(QuinnConnector::::new( - endpoint, - cloud_api_address, - domain_name, - )))) - } - - /// Returns a client to the cloud services. - /// - /// If the client is not connected, it will try to connect to the cloud services. - /// Available routes documented in - /// [`sd_cloud_schema::Service`](https://github.com/spacedriveapp/cloud-services-schema). - pub async fn client(&self) -> Result { - if let ClientState::Connected(client) = { &*self.client_state.read().await } { - return Ok(client.clone()); - } - - // If we're not connected, we need to try to connect. - let client = Self::init_client( - &self.http_client, - self.get_cloud_api_address.clone(), - self.domain_name.clone(), - ) - .await?; - *self.client_state.write().await = ClientState::Connected(client.clone()); - - Ok(client) - } - - pub async fn set_key_manager(&self, key_manager: KeyManager) { - self.key_manager - .write() - .await - .replace(Arc::new(key_manager)); - } - - pub async fn key_manager(&self) -> Result, Error> { - self.key_manager - .read() - .await - .as_ref() - .map_or(Err(Error::KeyManagerNotInitialized), |key_manager| { - Ok(Arc::clone(key_manager)) - }) - } - - pub async fn set_cloud_p2p(&self, cloud_p2p: CloudP2P) { - self.cloud_p2p.write().await.replace(Arc::new(cloud_p2p)); - } - - pub async fn cloud_p2p(&self) -> Result, Error> { - self.cloud_p2p - .read() - .await - .as_ref() - .map_or(Err(Error::CloudP2PNotInitialized), |cloud_p2p| { - Ok(Arc::clone(cloud_p2p)) - }) - } -} - -#[cfg(test)] -mod tests { - use sd_cloud_schema::{auth, devices}; - - use super::*; - - #[ignore] - #[tokio::test] - async fn test_client() { - let response = CloudServices::new( - "http://localhost:9420/cloud-api-address", - "http://relay.localhost:9999/", - "http://pkarr.localhost:9999/", - "dns.localhost:9999".to_string(), - "localhost".to_string(), - ) - .await - .unwrap() - .client() - .await - .unwrap() - .devices() - .list(devices::list::Request { - access_token: auth::AccessToken("invalid".to_string()), - }) - .await - .unwrap(); - - assert!(matches!( - response, - Err(sd_cloud_schema::Error::Client( - sd_cloud_schema::error::ClientSideError::Unauthorized - )) - )); - } -} diff --git a/core/crates/cloud-services/src/error.rs b/core/crates/cloud-services/src/error.rs deleted file mode 100644 index 8fe175a3a..000000000 --- a/core/crates/cloud-services/src/error.rs +++ /dev/null @@ -1,216 +0,0 @@ -use sd_cloud_schema::{ - cloud_p2p, - sync::{self, groups}, - Request, Response, -}; -use sd_utils::error::FileIOError; - -use std::{io, net::AddrParseError}; - -use quic_rpc::{ - pattern::{bidi_streaming, rpc, server_streaming}, - transport::{mapped::MappedConnector, quinn::QuinnConnector}, -}; - -#[derive(thiserror::Error, Debug)] -pub enum Error { - // Setup errors - #[error("Couldn't parse Cloud Services API address URL: {0}")] - InvalidUrl(reqwest::Error), - #[error("Failed to parse Cloud Services API address URL")] - FailedToParseRelayUrl, - #[error("Failed to initialize http client: {0}")] - HttpClientInit(reqwest::Error), - #[error("Failed to request Cloud Services API address from Auth Server route: {0}")] - FailedToRequestApiAddress(reqwest_middleware::Error), - #[error("Auth Server's Cloud Services API address route returned an error: {0}")] - AuthServerError(reqwest::Error), - #[error( - "Failed to extract response body from Auth Server's Cloud Services API address route: {0}" - )] - FailedToExtractApiAddress(reqwest::Error), - #[error("Failed to parse auth server's Cloud Services API address: {0}")] - FailedToParseApiAddress(#[from] AddrParseError), - #[error("Failed to create endpoint: {0}")] - FailedToCreateEndpoint(io::Error), - - // Token refresher errors - #[error("Invalid token format, missing claims")] - MissingClaims, - #[error("Failed to decode access token data: {0}")] - DecodeAccessTokenData(#[from] base64::DecodeError), - #[error("Failed to deserialize access token json data: {0}")] - DeserializeAccessTokenData(#[from] serde_json::Error), - #[error("Token expired")] - TokenExpired, - #[error("Failed to request refresh token: {0}")] - RefreshTokenRequest(reqwest_middleware::Error), - #[error("Missing tokens on refresh response")] - MissingTokensOnRefreshResponse, - #[error("Failed to parse token header value to string: {0}")] - FailedToParseTokenHeaderValueToString(#[from] reqwest::header::ToStrError), - - // Key Manager errors - #[error("Failed to handle File on KeyManager: {0}")] - FileIO(#[from] FileIOError), - #[error("Failed to handle key store serialization: {0}")] - KeyStoreSerialization(rmp_serde::encode::Error), - #[error("Failed to handle key store deserialization: {0}")] - KeyStoreDeserialization(rmp_serde::decode::Error), - #[error("Key store encryption related error: {{context: \"{context}\", source: {source}}}")] - KeyStoreCrypto { - #[source] - source: sd_crypto::Error, - context: &'static str, - }, - #[error("Key manager not initialized")] - KeyManagerNotInitialized, - - // Cloud P2P errors - #[error("Failed to create Cloud P2P endpoint: {0}")] - CreateCloudP2PEndpoint(anyhow::Error), - #[error("Failed to connect to Cloud P2P node: {0}")] - ConnectToCloudP2PNode(anyhow::Error), - #[error("Communication error with Cloud P2P node: {0}")] - CloudP2PRpcCommunication( - #[from] rpc::Error>, - ), - #[error("Cloud P2P not initialized")] - CloudP2PNotInitialized, - #[error("Failed to initialize LocalSwarmDiscovery: {0}")] - LocalSwarmDiscoveryInit(anyhow::Error), - #[error("Failed to initialize DhtDiscovery: {0}")] - DhtDiscoveryInit(anyhow::Error), - - // Communication errors - #[error("Failed to communicate with RPC backend: {0}")] - RpcCommunication(#[from] rpc::Error>), - #[error("Failed to communicate with RPC sync backend: {0}")] - RpcSyncCommunication( - #[from] - rpc::Error< - MappedConnector>, - >, - ), - #[error("Failed to communicate with Server Streaming RPC backend: {0}")] - ServerStreamCommunication(#[from] server_streaming::Error>), - #[error("Failed to communicate with Server Streaming RPC sync backend: {0}")] - ServerStreamSyncCommunication( - #[from] - server_streaming::Error< - MappedConnector>, - >, - ), - #[error("Failed to receive next response from Server Streaming RPC backend: {0}")] - ServerStreamRecv(#[from] server_streaming::ItemError>), - #[error("Failed to receive next response from Server Streaming RPC sync backend: {0}")] - ServerStreamSyncRecv( - #[from] - server_streaming::ItemError< - MappedConnector>, - >, - ), - #[error("Failed to communicate with Bidi Streaming RPC backend: {0}")] - BidiStreamCommunication(#[from] bidi_streaming::Error>), - #[error("Failed to communicate with Bidi Streaming RPC sync backend: {0}")] - BidiStreamSyncCommunication( - #[from] - bidi_streaming::Error< - MappedConnector>, - >, - ), - #[error("Failed to receive next response from Bidi Streaming RPC backend: {0}")] - BidiStreamRecv(#[from] bidi_streaming::ItemError>), - #[error("Error from backend: {0}")] - Backend(#[from] sd_cloud_schema::Error), - #[error("Failed to get access token from refresher: {0}")] - GetToken(#[from] GetTokenError), - #[error("Unexpected empty response from backend, context: {0}")] - EmptyResponse(&'static str), - #[error("Unexpected response from backend, context: {0}")] - UnexpectedResponse(&'static str), - - // Sync error - #[error("Sync error: {0}")] - Sync(#[from] sd_core_sync::Error), - #[error("Tried to sync messages with a group without having needed key")] - MissingSyncGroupKey(groups::PubId), - #[error("Failed to encrypt sync messages: {0}")] - Encrypt(sd_crypto::Error), - #[error("Failed to decrypt sync messages: {0}")] - Decrypt(sd_crypto::Error), - #[error("Failed to upload sync messages: {0}")] - UploadSyncMessages(reqwest_middleware::Error), - #[error("Failed to download sync messages: {0}")] - DownloadSyncMessages(reqwest_middleware::Error), - #[error("Received an error response from uploading sync messages: {0}")] - ErrorResponseUploadSyncMessages(reqwest::Error), - #[error("Received an error response from downloading sync messages: {0}")] - ErrorResponseDownloadSyncMessages(reqwest::Error), - #[error( - "Received an error response from downloading sync messages while reading its bytes: {0}" - )] - ErrorResponseDownloadReadBytesSyncMessages(reqwest::Error), - #[error("Critical error while uploading sync messages")] - CriticalErrorWhileUploadingSyncMessages, - #[error("Failed to send End update to push sync messages")] - EndUpdatePushSyncMessages(io::Error), - #[error("Unexpected end of stream while encrypting sync messages")] - UnexpectedEndOfStream, - #[error("Failed to create directory to store timestamp keeper files")] - FailedToCreateTimestampKeepersDirectory(io::Error), - #[error("Failed to read last timestamp keeper for pulling sync messages: {0}")] - FailedToReadLastTimestampKeeper(io::Error), - #[error("Failed to handle last timestamp keeper serialization: {0}")] - LastTimestampKeeperSerialization(rmp_serde::encode::Error), - #[error("Failed to handle last timestamp keeper deserialization: {0}")] - LastTimestampKeeperDeserialization(rmp_serde::decode::Error), - #[error("Failed to write last timestamp keeper for pulling sync messages: {0}")] - FailedToWriteLastTimestampKeeper(io::Error), - #[error("Sync messages download and decrypt task panicked")] - SyncMessagesDownloadAndDecryptTaskPanicked, - #[error("Serialization failure to push sync messages: {0}")] - SerializationFailureToPushSyncMessages(rmp_serde::encode::Error), - #[error("Deserialization failure to pull sync messages: {0}")] - DeserializationFailureToPullSyncMessages(rmp_serde::decode::Error), - #[error("Read nonce stream decryption: {0}")] - ReadNonceStreamDecryption(io::Error), - #[error("Incomplete download bytes sync messages")] - IncompleteDownloadBytesSyncMessages, - #[error("Timed out while waiting to recive thumbnail data")] - ThumbnailRequestTimeout, - - // Temporary errors - #[error("Device missing secret key for decrypting sync messages")] - MissingKeyHash, - #[error("Not Implemented yet")] - NotImplemented, - #[error("Device not found")] - DeviceNotFound, - #[error("Invalid CAS ID")] - InvalidCasId, - #[error("Internal Error")] - InternalError, - #[error("Remote Device Error")] - RemoteDeviceError, -} - -#[derive(thiserror::Error, Debug)] -pub enum GetTokenError { - #[error("Token refresher not initialized")] - RefresherNotInitialized, - #[error("Token refresher failed to refresh and need to be initialized again")] - FailedToRefresh, -} - -impl From for rspc::Error { - fn from(e: Error) -> Self { - Self::with_cause(rspc::ErrorCode::InternalServerError, e.to_string(), e) - } -} - -impl From for rspc::Error { - fn from(e: GetTokenError) -> Self { - Self::with_cause(rspc::ErrorCode::InternalServerError, e.to_string(), e) - } -} diff --git a/core/crates/cloud-services/src/key_manager/key_store.rs b/core/crates/cloud-services/src/key_manager/key_store.rs deleted file mode 100644 index acf97dad9..000000000 --- a/core/crates/cloud-services/src/key_manager/key_store.rs +++ /dev/null @@ -1,331 +0,0 @@ -use crate::Error; - -use sd_cloud_schema::{ - sync::{groups, KeyHash}, - NodeId, SecretKey as IrohSecretKey, -}; -use sd_crypto::{ - cloud::{decrypt, encrypt, secret_key::SecretKey}, - primitives::{EncryptedBlock, OneShotNonce, StreamNonce}, - CryptoRng, -}; -use sd_utils::error::FileIOError; -use tracing::debug; - -use std::{ - collections::{BTreeMap, VecDeque}, - fs::Metadata, - path::PathBuf, - pin::pin, -}; - -use futures::StreamExt; -use serde::{Deserialize, Serialize}; -use tokio::{ - fs, - io::{AsyncReadExt, AsyncWriteExt, BufWriter}, -}; -use zeroize::{Zeroize, ZeroizeOnDrop}; - -type KeyStack = VecDeque<(KeyHash, SecretKey)>; - -#[derive(Serialize, Deserialize)] -pub struct KeyStore { - iroh_secret_key: IrohSecretKey, - keys: BTreeMap, -} - -impl KeyStore { - pub const fn new(iroh_secret_key: IrohSecretKey) -> Self { - Self { - iroh_secret_key, - keys: BTreeMap::new(), - } - } - - pub fn add_key(&mut self, group_pub_id: groups::PubId, key: SecretKey) { - self.keys.entry(group_pub_id).or_default().push_front(( - KeyHash(blake3::hash(key.as_ref()).to_hex().to_string()), - key, - )); - } - - pub fn add_key_with_hash( - &mut self, - group_pub_id: groups::PubId, - key: SecretKey, - key_hash: KeyHash, - ) { - debug!( - key_hash = key_hash.0, - ?group_pub_id, - "Added single cloud sync key to key manager" - ); - - self.keys - .entry(group_pub_id) - .or_default() - .push_front((key_hash, key)); - } - - pub fn add_many_keys( - &mut self, - group_pub_id: groups::PubId, - keys: impl IntoIterator>, - ) { - let group_entry = self.keys.entry(group_pub_id).or_default(); - - // We reverse the secret keys as a implementation detail to - // keep the keys in the same order as they were added as a stack - for key in keys.into_iter().rev() { - let key_hash = blake3::hash(key.as_ref()).to_hex().to_string(); - - debug!( - key_hash, - ?group_pub_id, - "Added cloud sync key to key manager" - ); - - group_entry.push_front((KeyHash(key_hash), key)); - } - } - - pub fn remove_group(&mut self, group_pub_id: groups::PubId) { - self.keys.remove(&group_pub_id); - } - - pub fn iroh_secret_key(&self) -> IrohSecretKey { - self.iroh_secret_key.clone() - } - - pub fn node_id(&self) -> NodeId { - self.iroh_secret_key.public() - } - - pub fn get_key(&self, group_pub_id: groups::PubId, hash: &KeyHash) -> Option { - self.keys.get(&group_pub_id).and_then(|group| { - group - .iter() - .find_map(|(key_hash, key)| (key_hash == hash).then(|| key.clone())) - }) - } - - pub fn get_latest_key(&self, group_pub_id: groups::PubId) -> Option<(KeyHash, SecretKey)> { - self.keys - .get(&group_pub_id) - .and_then(|group| group.front().cloned()) - } - - pub fn get_group_keys(&self, group_pub_id: groups::PubId) -> Vec { - self.keys - .get(&group_pub_id) - .map(|group| group.iter().map(|(_key_hash, key)| key.clone()).collect()) - .unwrap_or_default() - } - - pub async fn encrypt( - &self, - key: &SecretKey, - rng: &mut CryptoRng, - keys_file_path: &PathBuf, - ) -> Result<(), Error> { - let plain_text_bytes = - rmp_serde::to_vec_named(self).map_err(Error::KeyStoreSerialization)?; - let mut file = BufWriter::with_capacity( - EncryptedBlock::CIPHER_TEXT_SIZE, - fs::OpenOptions::new() - .create(true) - .write(true) - .truncate(true) - .open(&keys_file_path) - .await - .map_err(|e| { - FileIOError::from(( - &keys_file_path, - e, - "Failed to open space keys file to encrypt", - )) - })?, - ); - - if plain_text_bytes.len() < EncryptedBlock::PLAIN_TEXT_SIZE { - use encrypt::OneShotEncryption; - - let EncryptedBlock { nonce, cipher_text } = key - .encrypt(&plain_text_bytes, rng) - .map_err(|e| Error::KeyStoreCrypto { - source: e, - context: "Failed to oneshot encrypt key store", - })?; - - file.write_all(nonce.as_slice()).await.map_err(|e| { - FileIOError::from(( - &keys_file_path, - e, - "Failed to write space keys file oneshot nonce", - )) - })?; - - file.write_all(cipher_text.as_slice()).await.map_err(|e| { - FileIOError::from(( - &keys_file_path, - e, - "Failed to write space keys file oneshot cipher text", - )) - })?; - } else { - use encrypt::StreamEncryption; - - let (nonce, stream) = key.encrypt(plain_text_bytes.as_slice(), rng); - - file.write_all(nonce.as_slice()).await.map_err(|e| { - FileIOError::from(( - &keys_file_path, - e, - "Failed to write space keys file stream nonce", - )) - })?; - - let mut stream = pin!(stream); - while let Some(res) = stream.next().await { - file.write_all(&res.map_err(|e| Error::KeyStoreCrypto { - source: e, - context: "Failed to stream encrypt key store", - })?) - .await - .map_err(|e| { - FileIOError::from(( - &keys_file_path, - e, - "Failed to write space keys file stream cipher text", - )) - })?; - } - }; - - file.flush().await.map_err(|e| { - FileIOError::from((&keys_file_path, e, "Failed to flush space keys file")).into() - }) - } - - pub async fn decrypt( - key: &SecretKey, - metadata: Metadata, - keys_file_path: &PathBuf, - ) -> Result { - let mut file = fs::File::open(&keys_file_path).await.map_err(|e| { - FileIOError::from(( - keys_file_path, - e, - "Failed to open space keys file to decrypt", - )) - })?; - - let usize_file_len = - usize::try_from(metadata.len()).expect("Failed to convert metadata length to usize"); - - let key_store_bytes = - if usize_file_len <= EncryptedBlock::CIPHER_TEXT_SIZE + size_of::() { - use decrypt::OneShotDecryption; - - let mut nonce = OneShotNonce::default(); - - file.read_exact(&mut nonce).await.map_err(|e| { - FileIOError::from(( - keys_file_path, - e, - "Failed to read space keys file oneshot nonce", - )) - })?; - - let mut cipher_text = vec![0u8; usize_file_len - size_of::()]; - - file.read_exact(&mut cipher_text).await.map_err(|e| { - FileIOError::from(( - keys_file_path, - e, - "Failed to read space keys file oneshot cipher text", - )) - })?; - - key.decrypt_owned(&EncryptedBlock { nonce, cipher_text }) - .map_err(|e| Error::KeyStoreCrypto { - source: e, - context: "Failed to oneshot decrypt space keys file", - })? - } else { - use decrypt::StreamDecryption; - - let mut nonce = StreamNonce::default(); - - let mut key_store_bytes = Vec::with_capacity( - (usize_file_len - size_of::()) / EncryptedBlock::CIPHER_TEXT_SIZE - * EncryptedBlock::PLAIN_TEXT_SIZE, - ); - - file.read_exact(&mut nonce).await.map_err(|e| { - FileIOError::from(( - keys_file_path, - e, - "Failed to read space keys file stream nonce", - )) - })?; - - key.decrypt(&nonce, &mut file, &mut key_store_bytes) - .await - .map_err(|e| Error::KeyStoreCrypto { - source: e, - context: "Failed to stream decrypt space keys file", - })?; - - key_store_bytes - }; - - let this = rmp_serde::from_slice::(&key_store_bytes) - .map_err(Error::KeyStoreDeserialization)?; - - #[cfg(debug_assertions)] - { - use std::fmt::Write; - let mut key_hashes_log = String::new(); - - this.keys.iter().for_each(|(group_pub_id, key_stack)| { - writeln!( - key_hashes_log, - "Group: {group_pub_id:?} => KeyHashes: {:?}", - key_stack - .iter() - .map(|(KeyHash(key_hash), _)| key_hash) - .collect::>() - ) - .expect("Failed to write to key hashes log"); - }); - - tracing::info!("Loaded key hashes: {key_hashes_log}"); - } - - Ok(this) - } -} - -/// Zeroize our secret keys and scrambles up iroh's secret key that doesn't implement zeroize -impl Zeroize for KeyStore { - fn zeroize(&mut self) { - self.iroh_secret_key = IrohSecretKey::generate(); - self.keys.values_mut().for_each(|group| { - group - .iter_mut() - .map(|(_key_hash, key)| key) - .for_each(Zeroize::zeroize); - }); - self.keys = BTreeMap::new(); - } -} - -impl Drop for KeyStore { - fn drop(&mut self) { - self.zeroize(); - } -} - -impl ZeroizeOnDrop for KeyStore {} diff --git a/core/crates/cloud-services/src/key_manager/mod.rs b/core/crates/cloud-services/src/key_manager/mod.rs deleted file mode 100644 index 64007a190..000000000 --- a/core/crates/cloud-services/src/key_manager/mod.rs +++ /dev/null @@ -1,183 +0,0 @@ -use crate::Error; - -use sd_cloud_schema::{ - sync::{groups, KeyHash}, - NodeId, SecretKey as IrohSecretKey, -}; -use sd_crypto::{cloud::secret_key::SecretKey, CryptoRng}; -use sd_utils::error::FileIOError; - -use std::{ - fmt, - path::{Path, PathBuf}, -}; - -use tokio::{fs, sync::RwLock}; - -mod key_store; - -use key_store::KeyStore; - -const KEY_FILE_NAME: &str = "space.keys"; - -pub struct KeyManager { - master_key: SecretKey, - keys_file_path: PathBuf, - store: RwLock, -} - -impl KeyManager { - pub async fn new( - master_key: SecretKey, - iroh_secret_key: IrohSecretKey, - data_directory: impl AsRef + Send, - rng: &mut CryptoRng, - ) -> Result { - async fn inner( - master_key: SecretKey, - iroh_secret_key: IrohSecretKey, - keys_file_path: PathBuf, - rng: &mut CryptoRng, - ) -> Result { - let store = KeyStore::new(iroh_secret_key); - store.encrypt(&master_key, rng, &keys_file_path).await?; - - Ok(KeyManager { - master_key, - keys_file_path, - store: RwLock::new(store), - }) - } - - inner( - master_key, - iroh_secret_key, - data_directory.as_ref().join(KEY_FILE_NAME), - rng, - ) - .await - } - - pub async fn load( - master_key: SecretKey, - data_directory: impl AsRef + Send, - ) -> Result { - async fn inner( - master_key: SecretKey, - keys_file_path: PathBuf, - ) -> Result { - Ok(KeyManager { - store: RwLock::new( - KeyStore::decrypt( - &master_key, - fs::metadata(&keys_file_path).await.map_err(|e| { - FileIOError::from(( - &keys_file_path, - e, - "Failed to read space keys file", - )) - })?, - &keys_file_path, - ) - .await?, - ), - master_key, - keys_file_path, - }) - } - - inner(master_key, data_directory.as_ref().join(KEY_FILE_NAME)).await - } - - pub async fn iroh_secret_key(&self) -> IrohSecretKey { - self.store.read().await.iroh_secret_key() - } - - pub async fn node_id(&self) -> NodeId { - self.store.read().await.node_id() - } - - pub async fn add_key( - &self, - group_pub_id: groups::PubId, - key: SecretKey, - rng: &mut CryptoRng, - ) -> Result<(), Error> { - let mut store = self.store.write().await; - store.add_key(group_pub_id, key); - // Keeping the write lock here, this way we ensure that we can't corrupt the file - store - .encrypt(&self.master_key, rng, &self.keys_file_path) - .await - } - - pub async fn add_key_with_hash( - &self, - group_pub_id: groups::PubId, - key: SecretKey, - key_hash: KeyHash, - rng: &mut CryptoRng, - ) -> Result<(), Error> { - let mut store = self.store.write().await; - store.add_key_with_hash(group_pub_id, key, key_hash); - // Keeping the write lock here, this way we ensure that we can't corrupt the file - store - .encrypt(&self.master_key, rng, &self.keys_file_path) - .await - } - - pub async fn remove_group( - &self, - group_pub_id: groups::PubId, - rng: &mut CryptoRng, - ) -> Result<(), Error> { - let mut store = self.store.write().await; - store.remove_group(group_pub_id); - // Keeping the write lock here, this way we ensure that we can't corrupt the file - store - .encrypt(&self.master_key, rng, &self.keys_file_path) - .await - } - - pub async fn add_many_keys( - &self, - group_pub_id: groups::PubId, - keys: impl IntoIterator< - Item = SecretKey, - IntoIter = impl DoubleEndedIterator + Send, - > + Send, - rng: &mut CryptoRng, - ) -> Result<(), Error> { - let mut store = self.store.write().await; - store.add_many_keys(group_pub_id, keys); - // Keeping the write lock here, this way we ensure that we can't corrupt the file - store - .encrypt(&self.master_key, rng, &self.keys_file_path) - .await - } - - pub async fn get_latest_key( - &self, - group_pub_id: groups::PubId, - ) -> Option<(KeyHash, SecretKey)> { - self.store.read().await.get_latest_key(group_pub_id) - } - - pub async fn get_key(&self, group_pub_id: groups::PubId, hash: &KeyHash) -> Option { - self.store.read().await.get_key(group_pub_id, hash) - } - - pub async fn get_group_keys(&self, group_pub_id: groups::PubId) -> Vec { - self.store.read().await.get_group_keys(group_pub_id) - } -} - -impl fmt::Debug for KeyManager { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("KeyManager") - .field("master_key", &"[REDACTED]") - .field("keys_file_path", &self.keys_file_path) - .field("store", &"[REDACTED]") - .finish() - } -} diff --git a/core/crates/cloud-services/src/lib.rs b/core/crates/cloud-services/src/lib.rs deleted file mode 100644 index 064634fac..000000000 --- a/core/crates/cloud-services/src/lib.rs +++ /dev/null @@ -1,55 +0,0 @@ -#![recursion_limit = "256"] -#![warn( - clippy::all, - clippy::pedantic, - clippy::correctness, - clippy::perf, - clippy::style, - clippy::suspicious, - clippy::complexity, - clippy::nursery, - clippy::unwrap_used, - unused_qualifications, - rust_2018_idioms, - trivial_casts, - trivial_numeric_casts, - unused_allocation, - clippy::unnecessary_cast, - clippy::cast_lossless, - clippy::cast_possible_truncation, - clippy::cast_possible_wrap, - clippy::cast_precision_loss, - clippy::cast_sign_loss, - clippy::dbg_macro, - clippy::deprecated_cfg_attr, - clippy::separated_literal_suffix, - deprecated -)] -#![forbid(deprecated_in_future)] -#![allow(clippy::missing_errors_doc, clippy::module_name_repetitions)] - -mod error; - -mod client; -mod key_manager; -mod p2p; -mod sync; -mod token_refresher; - -pub use client::CloudServices; -pub use error::{Error, GetTokenError}; -pub use key_manager::KeyManager; -pub use p2p::{ - CloudP2P, JoinSyncGroupResponse, JoinedLibraryCreateArgs, NotifyUser, Ticket, UserResponse, -}; -pub use sync::{ - declare_actors as declare_cloud_sync, SyncActors as CloudSyncActors, - SyncActorsState as CloudSyncActorsState, -}; - -// Re-exports -pub use quic_rpc::transport::quinn::QuinnConnector; - -// Export URL for the auth server -pub const AUTH_SERVER_URL: &str = "https://auth.spacedrive.com"; -// pub const AUTH_SERVER_URL: &str = "http://localhost:9420"; diff --git a/core/crates/cloud-services/src/p2p/mod.rs b/core/crates/cloud-services/src/p2p/mod.rs deleted file mode 100644 index 1b390205d..000000000 --- a/core/crates/cloud-services/src/p2p/mod.rs +++ /dev/null @@ -1,272 +0,0 @@ -use crate::{sync::ReceiveAndIngestNotifiers, CloudServices, Error}; - -use sd_cloud_schema::{ - cloud_p2p::{authorize_new_device_in_sync_group, CloudP2PALPN, CloudP2PError}, - devices::{self, Device}, - libraries, - sync::groups::{self, GroupWithDevices}, - SecretKey as IrohSecretKey, -}; -use sd_crypto::{CryptoRng, SeedableRng}; -use sd_prisma::prisma::file_path::cas_id; - -use std::{path::PathBuf, sync::Arc, time::Duration}; - -use iroh::{ - discovery::{ - dns::DnsDiscovery, local_swarm_discovery::LocalSwarmDiscovery, pkarr::dht::DhtDiscovery, - ConcurrentDiscovery, Discovery, - }, - Endpoint, NodeId, RelayMap, RelayMode, RelayUrl, -}; -use reqwest::Url; -use serde::{Deserialize, Serialize}; -use tokio::{spawn, sync::oneshot, time::sleep}; -use tracing::{debug, error, warn}; - -mod new_sync_messages_notifier; -mod runner; - -use runner::Runner; - -#[derive(Debug)] -pub struct JoinedLibraryCreateArgs { - pub pub_id: libraries::PubId, - pub name: String, - pub description: Option, -} - -#[derive(Debug)] -pub struct RecivedGetThumbnailArgs { - pub cas_id: cas_id::Type, - pub error: Option, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize, specta::Type)] -#[serde(transparent)] -#[repr(transparent)] -#[specta(rename = "CloudP2PTicket")] -pub struct Ticket(u64); - -#[derive(Debug, Serialize, specta::Type)] -#[serde(tag = "kind", content = "data")] -#[specta(rename = "CloudP2PNotifyUser")] -pub enum NotifyUser { - ReceivedJoinSyncGroupRequest { - ticket: Ticket, - asking_device: Device, - sync_group: GroupWithDevices, - }, - ReceivedJoinSyncGroupResponse { - response: JoinSyncGroupResponse, - sync_group: GroupWithDevices, - }, - SendingJoinSyncGroupResponseError { - error: JoinSyncGroupError, - sync_group: GroupWithDevices, - }, - TimedOutJoinRequest { - device: Device, - succeeded: bool, - }, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize, specta::Type)] -pub enum JoinSyncGroupError { - Communication, - InternalServer, - Auth, -} - -#[derive(Debug, Serialize, specta::Type)] -pub enum JoinSyncGroupResponse { - Accepted { authorizor_device: Device }, - Failed(CloudP2PError), - CriticalError, -} - -#[derive(Debug, Clone, Serialize, Deserialize, specta::Type)] -pub struct BasicLibraryCreationArgs { - pub id: libraries::PubId, - pub name: String, - pub description: Option, -} - -#[derive(Debug, Deserialize, specta::Type)] -#[serde(tag = "kind", content = "data")] -#[specta(rename = "CloudP2PUserResponse")] -pub enum UserResponse { - AcceptDeviceInSyncGroup { - ticket: Ticket, - accepted: Option, - }, -} -#[derive(Debug, Clone)] -pub struct CloudP2P { - msgs_tx: flume::Sender, -} - -impl CloudP2P { - pub async fn new( - current_device_pub_id: devices::PubId, - cloud_services: &CloudServices, - mut rng: CryptoRng, - iroh_secret_key: IrohSecretKey, - dns_origin_domain: String, - dns_pkarr_url: Url, - relay_url: RelayUrl, - data_directory: PathBuf, - ) -> Result { - let dht_discovery = DhtDiscovery::builder() - .secret_key(iroh_secret_key.clone()) - .pkarr_relay(dns_pkarr_url) - .build() - .map_err(Error::DhtDiscoveryInit)?; - - let endpoint = Endpoint::builder() - .alpns(vec![CloudP2PALPN::LATEST.to_vec()]) - .discovery(Box::new(ConcurrentDiscovery::from_services(vec![ - Box::new(DnsDiscovery::new(dns_origin_domain)), - Box::new( - LocalSwarmDiscovery::new(iroh_secret_key.public()) - .map_err(Error::LocalSwarmDiscoveryInit)?, - ), - Box::new(dht_discovery.clone()), - ]))) - .secret_key(iroh_secret_key) - .relay_mode(RelayMode::Custom(RelayMap::from_url(relay_url))) - .bind() - .await - .map_err(Error::CreateCloudP2PEndpoint)?; - - spawn({ - let endpoint = endpoint.clone(); - async move { - loop { - let Ok(node_addr) = endpoint.node_addr().await.map_err(|e| { - warn!(?e, "Failed to get direct addresses to force publish on DHT"); - }) else { - sleep(Duration::from_secs(5)).await; - continue; - }; - - debug!("Force publishing peer on DHT"); - return dht_discovery.publish(&node_addr.info); - } - } - }); - - let (msgs_tx, msgs_rx) = flume::bounded(16); - - spawn({ - let runner = Runner::new( - current_device_pub_id, - cloud_services, - msgs_tx.clone(), - endpoint, - data_directory, - ) - .await?; - let user_response_rx = cloud_services.user_response_rx.clone(); - - async move { - // All cloned runners share a single state with internal mutability - while let Err(e) = spawn(runner.clone().run( - msgs_rx.clone(), - user_response_rx.clone(), - CryptoRng::from_seed(rng.generate_fixed()), - )) - .await - { - if e.is_panic() { - error!("Cloud P2P runner panicked"); - } else { - break; - } - } - } - }); - - Ok(Self { msgs_tx }) - } - - /// Requests the device with the given connection ID asking for permission to the current device - /// to join the sync group - /// - /// # Panics - /// Will panic if the actor channel is closed, which should never happen - pub async fn request_join_sync_group( - &self, - devices_in_group: Vec<(devices::PubId, NodeId)>, - req: authorize_new_device_in_sync_group::Request, - tx: oneshot::Sender, - ) { - self.msgs_tx - .send_async(runner::Message::Request(runner::Request::JoinSyncGroup { - req, - devices_in_group, - tx, - })) - .await - .expect("Channel closed"); - } - - /// Register a notifier for the desired sync group, which will notify the receiver actor when - /// new sync messages arrive through cloud p2p notification requests. - /// - /// # Panics - /// Will panic if the actor channel is closed, which should never happen - pub async fn register_sync_messages_receiver_notifier( - &self, - sync_group_pub_id: groups::PubId, - notifier: Arc, - ) { - self.msgs_tx - .send_async(runner::Message::RegisterSyncMessageNotifier(( - sync_group_pub_id, - notifier, - ))) - .await - .expect("Channel closed"); - } - - /// Emit a notification that new sync messages were sent to cloud, so other devices should pull - /// them as soon as possible. - /// - /// # Panics - /// Will panic if the actor channel is closed, which should never happen - pub async fn notify_new_sync_messages(&self, group_pub_id: groups::PubId) { - self.msgs_tx - .send_async(runner::Message::NotifyPeersSyncMessages(group_pub_id)) - .await - .expect("Channel closed"); - } - - /// Requests the binary of a thumbnail from a specific device endpoint - /// - /// # Panics - /// Will panic if the actor channel is closed, which should never happen - pub async fn request_thumbnail_data( - &self, - device_pub_id: devices::PubId, - cas_id: cas_id::Type, - library_pub_id: libraries::PubId, - tx: oneshot::Sender, - ) { - self.msgs_tx - .send_async(runner::Message::Request(runner::Request::GetThumbnail { - device_pub_id, - cas_id, - library_pub_id, - tx, - })) - .await - .expect("Channel closed"); - } -} - -impl Drop for CloudP2P { - fn drop(&mut self) { - self.msgs_tx.send(runner::Message::Stop).ok(); - } -} diff --git a/core/crates/cloud-services/src/p2p/new_sync_messages_notifier.rs b/core/crates/cloud-services/src/p2p/new_sync_messages_notifier.rs deleted file mode 100644 index 1d754cc73..000000000 --- a/core/crates/cloud-services/src/p2p/new_sync_messages_notifier.rs +++ /dev/null @@ -1,154 +0,0 @@ -use crate::{token_refresher::TokenRefresher, Error}; - -use sd_cloud_schema::{ - cloud_p2p::{Client, CloudP2PALPN}, - devices, - sync::groups, -}; - -use std::time::Duration; - -use futures_concurrency::future::Join; -use iroh::{Endpoint, NodeId}; -use quic_rpc::{transport::quinn::QuinnConnector, RpcClient}; -use tokio::time::Instant; -use tracing::{debug, error, instrument, warn}; - -use super::runner::Message; - -const CACHED_MAX_DURATION: Duration = Duration::from_secs(60 * 5); - -pub async fn dispatch_notifier( - group_pub_id: groups::PubId, - device_pub_id: devices::PubId, - devices: Option<(Instant, Vec<(devices::PubId, NodeId)>)>, - msgs_tx: flume::Sender, - cloud_services: sd_cloud_schema::Client< - QuinnConnector, - >, - token_refresher: TokenRefresher, - endpoint: Endpoint, -) { - match notify_peers( - group_pub_id, - device_pub_id, - devices, - cloud_services, - token_refresher, - endpoint, - ) - .await - { - Ok((true, devices)) => { - if msgs_tx - .send_async(Message::UpdateCachedDevices((group_pub_id, devices))) - .await - .is_err() - { - warn!("Failed to send update cached devices message to update cached devices"); - } - } - - Ok((false, _)) => {} - - Err(e) => { - error!(?e, "Failed to notify peers"); - } - } -} - -#[instrument(skip(cloud_services, token_refresher, endpoint))] -async fn notify_peers( - group_pub_id: groups::PubId, - device_pub_id: devices::PubId, - devices: Option<(Instant, Vec<(devices::PubId, NodeId)>)>, - cloud_services: sd_cloud_schema::Client< - QuinnConnector, - >, - token_refresher: TokenRefresher, - endpoint: Endpoint, -) -> Result<(bool, Vec<(devices::PubId, NodeId)>), Error> { - let (devices, update_cache) = match devices { - Some((when, devices)) if when.elapsed() < CACHED_MAX_DURATION => (devices, false), - _ => { - debug!("Fetching devices connection ids for group"); - let groups::get::Response(groups::get::ResponseKind::DevicesConnectionIds(devices)) = - cloud_services - .sync() - .groups() - .get(groups::get::Request { - access_token: token_refresher.get_access_token().await?, - pub_id: group_pub_id, - kind: groups::get::RequestKind::DevicesConnectionIds, - }) - .await?? - else { - unreachable!("Only DevicesConnectionIds response is expected, as we requested it"); - }; - - (devices, true) - } - }; - - send_notifications(group_pub_id, device_pub_id, &devices, &endpoint).await; - - Ok((update_cache, devices)) -} - -async fn send_notifications( - group_pub_id: groups::PubId, - device_pub_id: devices::PubId, - devices: &[(devices::PubId, NodeId)], - endpoint: &Endpoint, -) { - devices - .iter() - .filter(|(peer_device_pub_id, _)| *peer_device_pub_id != device_pub_id) - .map(|(peer_device_pub_id, connection_id)| async move { - if let Err(e) = - connect_and_send_notification(group_pub_id, device_pub_id, connection_id, endpoint) - .await - { - // Using just a debug log here because we don't want to spam the logs with - // every single notification failure, as this is more a nice to have feature than a - // critical one - debug!(?e, %peer_device_pub_id, "Failed to send new sync messages notification to peer"); - } else { - debug!(%peer_device_pub_id, "Sent new sync messages notification to peer"); - } - }) - .collect::>() - .join() - .await; -} - -async fn connect_and_send_notification( - group_pub_id: groups::PubId, - device_pub_id: devices::PubId, - connection_id: &NodeId, - endpoint: &Endpoint, -) -> Result<(), Error> { - let client = Client::new(RpcClient::new(QuinnConnector::from_connection( - endpoint - .connect(*connection_id, CloudP2PALPN::LATEST) - .await - .map_err(Error::ConnectToCloudP2PNode)?, - ))); - - if let Err(e) = client - .notify_new_sync_messages( - sd_cloud_schema::cloud_p2p::notify_new_sync_messages::Request { - sync_group_pub_id: group_pub_id, - device_pub_id, - }, - ) - .await? - { - warn!( - ?e, - "This route shouldn't return an error, it's just a notification", - ); - }; - - Ok(()) -} diff --git a/core/crates/cloud-services/src/p2p/runner.rs b/core/crates/cloud-services/src/p2p/runner.rs deleted file mode 100644 index b7f0b5c07..000000000 --- a/core/crates/cloud-services/src/p2p/runner.rs +++ /dev/null @@ -1,1109 +0,0 @@ -use crate::{ - p2p::JoinSyncGroupError, sync::ReceiveAndIngestNotifiers, token_refresher::TokenRefresher, - CloudServices, Error, KeyManager, -}; - -use sd_cloud_schema::{ - cloud_p2p::{ - self, authorize_new_device_in_sync_group, notify_new_sync_messages, Client, CloudP2PALPN, - CloudP2PError, Service, - }, - devices::{self, Device}, - libraries::{self}, - sync::groups, -}; -use sd_crypto::{CryptoRng, SeedableRng}; -use sd_prisma::prisma::file_path::cas_id; - -use std::{ - collections::HashMap, - path::PathBuf, - pin::pin, - sync::{ - atomic::{AtomicU64, Ordering}, - Arc, - }, - time::Duration, -}; - -use dashmap::DashMap; -use flume::SendError; -use futures::StreamExt; -use futures_concurrency::stream::Merge; -use iroh::{key::PublicKey, Endpoint, NodeId}; -use quic_rpc::{ - server::{Accepting, RpcChannel, RpcServerError}, - transport::quinn::{QuinnConnector, QuinnListener}, - RpcClient, RpcServer, -}; -use tokio::{ - spawn, - sync::{oneshot, Mutex}, - task::JoinHandle, - time::{interval, Instant, MissedTickBehavior}, -}; -use tokio_stream::wrappers::IntervalStream; -use tracing::{debug, error, warn}; - -use super::{ - new_sync_messages_notifier::dispatch_notifier, BasicLibraryCreationArgs, JoinSyncGroupResponse, - JoinedLibraryCreateArgs, NotifyUser, RecivedGetThumbnailArgs, Ticket, UserResponse, -}; - -const TEN_SECONDS: Duration = Duration::from_secs(10); -const FIVE_MINUTES: Duration = Duration::from_secs(60 * 5); - -#[allow(clippy::large_enum_variant)] // Ignoring because the enum Stop variant will only happen a single time ever -pub enum Message { - Request(Request), - RegisterSyncMessageNotifier((groups::PubId, Arc)), - NotifyPeersSyncMessages(groups::PubId), - UpdateCachedDevices((groups::PubId, Vec<(devices::PubId, NodeId)>)), - Stop, -} - -pub enum Request { - JoinSyncGroup { - req: authorize_new_device_in_sync_group::Request, - devices_in_group: Vec<(devices::PubId, NodeId)>, - tx: oneshot::Sender, - }, - GetThumbnail { - device_pub_id: devices::PubId, - cas_id: cas_id::Type, - library_pub_id: libraries::PubId, - tx: oneshot::Sender, - }, -} - -/// We use internal mutability here, but don't worry because there will always be a single -/// [`Runner`] running at a time, so the lock is never contended -pub struct Runner { - current_device_pub_id: devices::PubId, - token_refresher: TokenRefresher, - cloud_services: sd_cloud_schema::Client< - QuinnConnector, - >, - msgs_tx: flume::Sender, - endpoint: Endpoint, - key_manager: Arc, - ticketer: Arc, - notify_user_tx: flume::Sender, - sync_messages_receiver_notifiers_map: - Arc>>, - pending_sync_group_join_requests: Arc>>, - cached_devices_per_group: HashMap)>, - timeout_checker_buffer: Vec<(Ticket, PendingSyncGroupJoin)>, - data_directory: PathBuf, -} - -impl Clone for Runner { - fn clone(&self) -> Self { - Self { - current_device_pub_id: self.current_device_pub_id, - token_refresher: self.token_refresher.clone(), - cloud_services: self.cloud_services.clone(), - msgs_tx: self.msgs_tx.clone(), - endpoint: self.endpoint.clone(), - key_manager: Arc::clone(&self.key_manager), - ticketer: Arc::clone(&self.ticketer), - notify_user_tx: self.notify_user_tx.clone(), - sync_messages_receiver_notifiers_map: Arc::clone( - &self.sync_messages_receiver_notifiers_map, - ), - pending_sync_group_join_requests: Arc::clone(&self.pending_sync_group_join_requests), - // Just cache the devices and their node_ids per group - cached_devices_per_group: HashMap::new(), - // This one is a temporary buffer only used for timeout checker - timeout_checker_buffer: vec![], - data_directory: self.data_directory.clone(), - } - } -} - -struct PendingSyncGroupJoin { - channel: RpcChannel>, - request: authorize_new_device_in_sync_group::Request, - this_device: Device, - since: Instant, -} - -type P2PServerEndpoint = QuinnListener; - -impl Runner { - pub async fn new( - current_device_pub_id: devices::PubId, - cloud_services: &CloudServices, - msgs_tx: flume::Sender, - endpoint: Endpoint, - data_directory: PathBuf, - ) -> Result { - Ok(Self { - current_device_pub_id, - token_refresher: cloud_services.token_refresher.clone(), - cloud_services: cloud_services.client().await?, - msgs_tx, - endpoint, - key_manager: cloud_services.key_manager().await?, - ticketer: Arc::default(), - notify_user_tx: cloud_services.notify_user_tx.clone(), - sync_messages_receiver_notifiers_map: Arc::default(), - pending_sync_group_join_requests: Arc::default(), - cached_devices_per_group: HashMap::new(), - timeout_checker_buffer: vec![], - data_directory, - }) - } - - pub async fn run( - mut self, - msgs_rx: flume::Receiver, - user_response_rx: flume::Receiver, - mut rng: CryptoRng, - ) { - // Ignoring because this is only used internally and I think that boxing will be more expensive than wasting - // some extra bytes for smaller variants - #[allow(clippy::large_enum_variant)] - enum StreamMessage { - AcceptResult( - Result, RpcServerError>, - ), - Message(Message), - UserResponse(UserResponse), - Tick, - } - - let mut ticker = interval(TEN_SECONDS); - ticker.set_missed_tick_behavior(MissedTickBehavior::Skip); - - // FIXME(@fogodev): Update this function to use iroh-net transport instead of quinn - // when it's implemented - let (server, server_handle) = setup_server_endpoint(self.endpoint.clone()); - - let mut msg_stream = pin!(( - async_stream::stream! { - loop { - yield StreamMessage::AcceptResult(server.accept().await); - } - }, - msgs_rx.stream().map(StreamMessage::Message), - user_response_rx.stream().map(StreamMessage::UserResponse), - IntervalStream::new(ticker).map(|_| StreamMessage::Tick), - ) - .merge()); - - while let Some(msg) = msg_stream.next().await { - match msg { - StreamMessage::AcceptResult(Ok(accepting)) => { - let Ok((request, channel)) = accepting.read_first().await.map_err(|e| { - error!(?e, "Failed to read first request from a new connection;"); - }) else { - continue; - }; - - self.handle_request(request, channel).await; - } - - StreamMessage::AcceptResult(Err(e)) => { - // TODO(@fogodev): Maybe report this error to the user on a toast? - error!(?e, "Error accepting connection;"); - } - - StreamMessage::Message(Message::Request(Request::JoinSyncGroup { - req, - devices_in_group, - tx, - })) => self.dispatch_join_requests(req, devices_in_group, &mut rng, tx), - - StreamMessage::Message(Message::Request(Request::GetThumbnail { - device_pub_id, - cas_id, - library_pub_id, - tx, - })) => self.dispatch_get_thumbnail(device_pub_id, cas_id, library_pub_id, tx), - - StreamMessage::Message(Message::RegisterSyncMessageNotifier(( - group_pub_id, - notifier, - ))) => { - self.sync_messages_receiver_notifiers_map - .insert(group_pub_id, notifier); - } - - StreamMessage::Message(Message::NotifyPeersSyncMessages(group_pub_id)) => { - spawn(dispatch_notifier( - group_pub_id, - self.current_device_pub_id, - self.cached_devices_per_group.get(&group_pub_id).cloned(), - self.msgs_tx.clone(), - self.cloud_services.clone(), - self.token_refresher.clone(), - self.endpoint.clone(), - )); - } - - StreamMessage::Message(Message::UpdateCachedDevices(( - group_pub_id, - devices_connections_ids, - ))) => { - self.cached_devices_per_group - .insert(group_pub_id, (Instant::now(), devices_connections_ids)); - } - - StreamMessage::UserResponse(UserResponse::AcceptDeviceInSyncGroup { - ticket, - accepted, - }) => { - self.handle_join_response(ticket, accepted).await; - } - - StreamMessage::Tick => self.tick().await, - - StreamMessage::Message(Message::Stop) => { - server_handle.abort(); - break; - } - } - } - } - - fn dispatch_join_requests( - &self, - req: authorize_new_device_in_sync_group::Request, - devices_in_group: Vec<(devices::PubId, NodeId)>, - rng: &mut CryptoRng, - tx: oneshot::Sender, - ) { - async fn inner( - key_manager: Arc, - endpoint: Endpoint, - mut rng: CryptoRng, - req: authorize_new_device_in_sync_group::Request, - devices_in_group: Vec<(devices::PubId, NodeId)>, - tx: oneshot::Sender, - ) -> Result { - let group_pub_id = req.sync_group.pub_id; - loop { - let client = - match connect_to_first_available_client(&endpoint, &devices_in_group).await { - Ok(client) => client, - Err(e) => { - return Ok(JoinSyncGroupResponse::Failed(e)); - } - }; - - match client - .authorize_new_device_in_sync_group(req.clone()) - .await? - { - Ok(authorize_new_device_in_sync_group::Response { - authorizor_device, - keys, - library_pub_id, - library_name, - library_description, - }) => { - debug!( - device_pub_id = %authorizor_device.pub_id, - %group_pub_id, - keys_count = keys.len(), - %library_pub_id, - library_name, - "Received join sync group response" - ); - - key_manager - .add_many_keys( - group_pub_id, - keys.into_iter().map(|key| { - key.as_slice() - .try_into() - .expect("critical error, backend has invalid secret keys") - }), - &mut rng, - ) - .await?; - - if tx - .send(JoinedLibraryCreateArgs { - pub_id: library_pub_id, - name: library_name, - description: library_description, - }) - .is_err() - { - error!("Failed to handle library creation locally from received library data"); - return Ok(JoinSyncGroupResponse::CriticalError); - } - - return Ok(JoinSyncGroupResponse::Accepted { authorizor_device }); - } - - // In case of timeout, we will try again - Err(CloudP2PError::TimedOut) => continue, - - Err(e) => return Ok(JoinSyncGroupResponse::Failed(e)), - } - } - } - - spawn({ - let endpoint = self.endpoint.clone(); - let notify_user_tx = self.notify_user_tx.clone(); - let key_manager = Arc::clone(&self.key_manager); - let rng = CryptoRng::from_seed(rng.generate_fixed()); - async move { - let sync_group = req.sync_group.clone(); - - if let Err(SendError(response)) = notify_user_tx - .send_async(NotifyUser::ReceivedJoinSyncGroupResponse { - response: inner(key_manager, endpoint, rng, req, devices_in_group, tx) - .await - .unwrap_or_else(|e| { - error!( - ?e, - "Failed to issue authorize new device in sync group request;" - ); - JoinSyncGroupResponse::CriticalError - }), - sync_group, - }) - .await - { - error!(?response, "Failed to send response to user;"); - } - } - }); - } - - #[allow(clippy::too_many_lines)] - fn dispatch_get_thumbnail( - &self, - device_pub_id: devices::PubId, - cas_id: cas_id::Type, - library_pub_id: libraries::PubId, - tx: oneshot::Sender, - ) { - debug!(?device_pub_id, ?cas_id, "Received request for thumbnail"); - let current_device_pub_id = self.current_device_pub_id; - let cas_id_clone = cas_id.clone(); - - // Put tx in an Arc to allow multiple references to it - let tx = Arc::new(Mutex::new(Some(tx))); - - let device_connection = self - .cached_devices_per_group - .values() - .find(|(_, devices)| devices.iter().any(|(pub_id, _)| pub_id == &device_pub_id)) - .and_then(|(_, devices)| devices.iter().find(|(pub_id, _)| pub_id == &device_pub_id)) - .ok_or_else(|| { - error!("Failed to find device in the cached devices list"); - - // Use a clone of the channel to send the error response - let tx_clone = tx.clone(); - spawn(async move { - if let Some(tx) = tx_clone.lock().await.take() { - if tx - .send(RecivedGetThumbnailArgs { - cas_id: cas_id_clone.clone(), - error: Some(Error::DeviceNotFound), - }) - .is_err() - { - error!("Failed to send response to user;"); - } - } - }); - }) - .expect("Device must be in the cached devices list"); - - let (_, device_connection_id) = device_connection; - - debug!("Device Connection ID: {:?}", device_connection_id); - let data_dir_clone = self.data_directory.clone(); - - // Spawn a separate task to avoid blocking the runner - spawn({ - let endpoint = self.endpoint.clone(); - let device_connection_id = *device_connection_id; - let tx = tx.clone(); - let cas_id_clone_clone = cas_id.clone(); - - async move { - // Connect to the device - let client = - match connect_to_specific_client(&endpoint, &device_connection_id).await { - Ok(client) => client, - Err(e) => { - error!(?e, "Failed to connect to device"); - // Send the error through the channel - if let Some(tx) = tx.lock().await.take() { - if tx - .send(RecivedGetThumbnailArgs { - cas_id: cas_id_clone_clone, - error: Some(Error::DeviceNotFound), - }) - .is_err() - { - error!("Failed to send response to user;"); - } - } - return; - } - }; - - // Create the request - let request = cloud_p2p::get_thumbnail::Request { - cas_id: cas_id_clone_clone.clone().unwrap_or_default(), - device_pub_id: current_device_pub_id, - library_pub_id, - }; - - // Send the request - match client.get_thumbnail(request).await { - Ok(Ok(cloud_p2p::get_thumbnail::Response { thumbnail })) => { - debug!(?cas_id, "Successfully received thumbnail"); - - // Convert cas_id to a string - let cas_id_str = cas_id_clone_clone.clone().unwrap_or_default(); - - // If we received a thumbnail, try to save it locally - if let Some(thumbnail_data) = &thumbnail { - // Try to save the thumbnail, but don't fail if saving fails - if let Err(e) = save_remote_thumbnail( - &cas_id_str, - thumbnail_data, - data_dir_clone, - library_pub_id, - ) - .await - { - error!(?e, "Failed to save remote thumbnail locally, but continuing with response"); - } - } - - // Send the response via the oneshot channel - if let Some(tx) = tx.lock().await.take() { - if tx - .send(RecivedGetThumbnailArgs { - cas_id: cas_id_clone_clone.clone(), - error: None, - }) - .is_err() - { - error!("Failed to send thumbnail response to user"); - } - } - } - Ok(Err(e)) => { - error!(?e, "Remote device returned error for thumbnail request"); - // Send the error through the channel - if let Some(tx) = tx.lock().await.take() { - if tx - .send(RecivedGetThumbnailArgs { - cas_id: cas_id_clone_clone.clone(), - error: Some(Error::RemoteDeviceError), - }) - .is_err() - { - error!("Failed to send response to user;"); - } - } - } - Err(e) => { - error!(?e, "Failed to send thumbnail request to remote device"); - // Send the error through the channel - if let Some(tx) = tx.lock().await.take() { - if tx - .send(RecivedGetThumbnailArgs { - cas_id: cas_id_clone_clone.clone(), - error: Some(Error::InternalError), - }) - .is_err() - { - error!("Failed to send response to user;"); - } - } - } - } - } - }); - } - - #[allow(clippy::too_many_lines)] - async fn handle_request( - &self, - request: cloud_p2p::Request, - channel: RpcChannel, - ) { - match request { - cloud_p2p::Request::AuthorizeNewDeviceInSyncGroup( - authorize_new_device_in_sync_group::Request { - sync_group, - asking_device, - }, - ) => { - let ticket = Ticket(self.ticketer.fetch_add(1, Ordering::Relaxed)); - let this_device = sync_group - .devices - .iter() - .find(|device| device.pub_id == self.current_device_pub_id) - .expect( - "current device must be in the sync group, otherwise we wouldn't be here", - ) - .clone(); - - self.notify_user_tx - .send_async(NotifyUser::ReceivedJoinSyncGroupRequest { - ticket, - asking_device: asking_device.clone(), - sync_group: sync_group.clone(), - }) - .await - .expect("notify_user_tx must never closes!"); - - self.pending_sync_group_join_requests.lock().await.insert( - ticket, - PendingSyncGroupJoin { - channel, - request: authorize_new_device_in_sync_group::Request { - sync_group, - asking_device, - }, - this_device, - since: Instant::now(), - }, - ); - } - - cloud_p2p::Request::NotifyNewSyncMessages(req) => { - if let Err(e) = channel - .rpc( - req, - (), - |(), - notify_new_sync_messages::Request { - sync_group_pub_id, - device_pub_id, - }| async move { - debug!(%sync_group_pub_id, %device_pub_id, "Received new sync messages notification"); - if let Some(notifier) = self - .sync_messages_receiver_notifiers_map - .get(&sync_group_pub_id) - { - notifier.notify_receiver(); - } else { - warn!("Received new sync messages notification for unknown sync group"); - } - - Ok(notify_new_sync_messages::Response) - }, - ) - .await - { - error!( - ?e, - "Failed to reply to new sync messages notification request" - ); - } - } - - cloud_p2p::Request::GetThumbnail(req) => { - if let Err(e) = channel - .rpc( - req, - (), - |(), - cloud_p2p::get_thumbnail::Request { - cas_id, - device_pub_id, - library_pub_id, - }| async move { - debug!( - ?cas_id, - "Received thumbnail request from device {:?}", device_pub_id - ); - - match fetch_local_thumbnail( - Some(cas_id.clone()), - self.data_directory.clone(), - library_pub_id, - ) - .await - { - Ok(Some(thumbnail_data)) => { - debug!(?cas_id, "Found thumbnail locally"); - Ok(cloud_p2p::get_thumbnail::Response { - thumbnail: Some(thumbnail_data), - }) - } - Ok(None) => { - debug!(?cas_id, "Thumbnail not found locally"); - Err(CloudP2PError::Rejected) - } - Err(e) => { - error!(?e, ?cas_id, "Error fetching thumbnail"); - Err(CloudP2PError::Rejected) - } - } - }, - ) - .await - { - error!(?e, "Failed to send get thumbnail response;"); - } - } - } - } - - async fn handle_join_response( - &self, - ticket: Ticket, - accepted: Option, - ) { - let Some(PendingSyncGroupJoin { - channel, - request, - this_device, - .. - }) = self - .pending_sync_group_join_requests - .lock() - .await - .remove(&ticket) - else { - warn!("Received join response for unknown ticket; We probably timed out this request already"); - return; - }; - - let sync_group = request.sync_group.clone(); - let asking_device_pub_id = request.asking_device.pub_id; - - let was_accepted = accepted.is_some(); - - let response = if let Some(BasicLibraryCreationArgs { - id: library_pub_id, - name: library_name, - description: library_description, - }) = accepted - { - Ok(authorize_new_device_in_sync_group::Response { - authorizor_device: this_device, - keys: self - .key_manager - .get_group_keys(request.sync_group.pub_id) - .await - .into_iter() - .map(Into::into) - .collect(), - library_pub_id, - library_name, - library_description, - }) - } else { - Err(CloudP2PError::Rejected) - }; - - if let Err(e) = channel - .rpc(request, (), |(), _req| async move { response }) - .await - { - error!(?e, "Failed to send response to user;"); - self.notify_join_error(sync_group, JoinSyncGroupError::Communication) - .await; - - return; - } - - if was_accepted { - let Ok(access_token) = self - .token_refresher - .get_access_token() - .await - .map_err(|e| error!(?e, "Failed to get access token;")) - else { - self.notify_join_error(sync_group, JoinSyncGroupError::Auth) - .await; - return; - }; - - match self - .cloud_services - .sync() - .groups() - .reply_join_request(groups::reply_join_request::Request { - access_token, - group_pub_id: sync_group.pub_id, - authorized_device_pub_id: asking_device_pub_id, - authorizor_device_pub_id: self.current_device_pub_id, - }) - .await - { - Ok(Ok(groups::reply_join_request::Response)) => { - // Everything is Awesome! - } - Ok(Err(e)) => { - error!(?e, "Failed to reply to join request"); - self.notify_join_error(sync_group, JoinSyncGroupError::InternalServer) - .await; - } - Err(e) => { - error!(?e, "Failed to send reply to join request"); - self.notify_join_error(sync_group, JoinSyncGroupError::Communication) - .await; - } - } - } - } - - async fn notify_join_error( - &self, - sync_group: groups::GroupWithDevices, - error: JoinSyncGroupError, - ) { - self.notify_user_tx - .send_async(NotifyUser::SendingJoinSyncGroupResponseError { error, sync_group }) - .await - .expect("notify_user_tx must never closes!"); - } - - async fn tick(&mut self) { - self.timeout_checker_buffer.clear(); - - let mut pending_sync_group_join_requests = - self.pending_sync_group_join_requests.lock().await; - - for (ticket, pending_sync_group_join) in pending_sync_group_join_requests.drain() { - if pending_sync_group_join.since.elapsed() > FIVE_MINUTES { - let PendingSyncGroupJoin { - channel, request, .. - } = pending_sync_group_join; - - let asking_device = request.asking_device.clone(); - - let notify_message = match channel - .rpc(request, (), |(), _req| async move { - Err(CloudP2PError::TimedOut) - }) - .await - { - Ok(()) => NotifyUser::TimedOutJoinRequest { - device: asking_device, - succeeded: true, - }, - Err(e) => { - error!(?e, "Failed to send timed out response to user;"); - NotifyUser::TimedOutJoinRequest { - device: asking_device, - succeeded: false, - } - } - }; - - self.notify_user_tx - .send_async(notify_message) - .await - .expect("notify_user_tx must never closes!"); - } else { - self.timeout_checker_buffer - .push((ticket, pending_sync_group_join)); - } - } - - pending_sync_group_join_requests.extend(self.timeout_checker_buffer.drain(..)); - } -} - -async fn connect_to_first_available_client( - endpoint: &Endpoint, - devices_in_group: &[(devices::PubId, NodeId)], -) -> Result>, CloudP2PError> { - for (device_pub_id, device_connection_id) in devices_in_group { - if let Ok(connection) = endpoint - .connect(*device_connection_id, CloudP2PALPN::LATEST) - .await - .map_err( - |e| error!(?e, %device_pub_id, "Failed to connect to authorizor device candidate"), - ) { - debug!(%device_pub_id, "Connected to authorizor device candidate"); - - return Ok(Client::new(RpcClient::new( - QuinnConnector::from_connection(connection), - ))); - } - } - - Err(CloudP2PError::UnableToConnect) -} - -async fn connect_to_specific_client( - endpoint: &Endpoint, - device_connection_id: &PublicKey, -) -> Result>, CloudP2PError> { - // Get the connection id by fetching using the device pub id - let connection = endpoint - .connect(*device_connection_id, CloudP2PALPN::LATEST) - .await - .map_err(|e| { - error!(?e, "Failed to connect to authorizor device candidate"); - CloudP2PError::UnableToConnect - })?; - debug!(%device_connection_id, "Connected to authorizor device candidate"); - Ok(Client::new(RpcClient::new( - QuinnConnector::from_connection(connection), - ))) -} - -fn setup_server_endpoint( - endpoint: Endpoint, -) -> (RpcServer, JoinHandle<()>) { - let local_addr = { - let (ipv4_addr, maybe_ipv6_addr) = endpoint.bound_sockets(); - // Trying to give preference to IPv6 addresses because it's 2024 - maybe_ipv6_addr.unwrap_or(ipv4_addr) - }; - - let (connections_tx, connections_rx) = flume::bounded(16); - - ( - RpcServer::new(QuinnListener::handle_connections( - connections_rx, - local_addr, - )), - spawn(async move { - while let Some(connecting) = endpoint.accept().await { - if let Ok(connection) = connecting.await.map_err(|e| { - warn!(?e, "Cloud P2P failed to accept connection"); - }) { - if connections_tx.send_async(connection).await.is_err() { - warn!("Connection receiver dropped"); - break; - } - } - } - }), - ) -} - -async fn fetch_local_thumbnail( - cas_id: cas_id::Type, - data_directory: PathBuf, - library_pub_id: libraries::PubId, -) -> Result>, Error> { - use tokio::fs; - use tracing::{debug, error}; - - debug!(?cas_id, "Fetching thumbnail from local storage"); - - // Convert cas_id to a string - let cas_id = cas_id.unwrap_or_default(); - - let cas_id = sd_core_prisma_helpers::CasId::from(cas_id); - - let thumbnails_directory = - sd_core_heavy_lifting::media_processor::get_thumbnails_directory(data_directory); - - // Get the shard hex for the cas_id - let shard_hex = sd_core_heavy_lifting::media_processor::get_shard_hex(&cas_id); - - // First try to find the thumbnail in the specific library folder - let library_path = thumbnails_directory.join(library_pub_id.to_string()); - let shard_path = library_path.join(shard_hex); - let thumbnail_path = shard_path.join(format!("{}.webp", cas_id.as_str())); - - debug!("Checking for thumbnail at {:?}", thumbnail_path); - - // If the thumbnail exists in the specific library folder, read it - if fs::metadata(&thumbnail_path).await.is_ok() { - match fs::read(&thumbnail_path).await { - Ok(data) => { - debug!("Found thumbnail at {:?}", thumbnail_path); - return Ok(Some(data)); - } - Err(e) => { - error!(?e, "Failed to read thumbnail file"); - return Err(Error::InternalError); - } - } - } - - // If not found in the specific library, try the ephemeral directory - let ephemeral_dir = thumbnails_directory.join("ephemeral"); - let ephemeral_shard_path = ephemeral_dir.join(shard_hex); - let ephemeral_thumbnail_path = ephemeral_shard_path.join(format!("{}.webp", cas_id.as_str())); - - debug!( - "Checking for thumbnail in ephemeral at {:?}", - ephemeral_thumbnail_path - ); - - // If the thumbnail exists in ephemeral, read it - if fs::metadata(&ephemeral_thumbnail_path).await.is_ok() { - match fs::read(&ephemeral_thumbnail_path).await { - Ok(data) => { - debug!("Found thumbnail at {:?}", ephemeral_thumbnail_path); - return Ok(Some(data)); - } - Err(e) => { - error!(?e, "Failed to read thumbnail file"); - return Err(Error::InternalError); - } - } - } - - // If we still don't have the thumbnail, search all library folders as a fallback - // This is to handle cases where the library ID might have changed - let Ok(mut directories) = fs::read_dir(&thumbnails_directory).await else { - debug!("No thumbnails directory found"); - return Ok(None); - }; - - // Try to find the thumbnail in any other library directories - while let Ok(Some(entry)) = directories.next_entry().await { - let dir_path = entry.path(); - - // Skip files and already checked directories - if !dir_path.is_dir() || dir_path == library_path || dir_path == ephemeral_dir { - continue; - } - - // Check if thumbnail exists in this directory - let other_shard_path = dir_path.join(shard_hex); - let other_thumbnail_path = other_shard_path.join(format!("{}.webp", cas_id.as_str())); - - debug!("Checking for thumbnail at {:?}", other_thumbnail_path); - - if fs::metadata(&other_thumbnail_path).await.is_ok() { - match fs::read(&other_thumbnail_path).await { - Ok(data) => { - debug!("Found thumbnail at {:?}", other_thumbnail_path); - return Ok(Some(data)); - } - Err(e) => { - error!(?e, "Failed to read thumbnail file"); - return Err(Error::InternalError); - } - } - } - } - - // If we get here, the thumbnail doesn't exist anywhere - debug!("Thumbnail not found for {}", cas_id.as_str()); - Ok(None) -} - -async fn save_remote_thumbnail( - cas_id: &str, - thumbnail_data: &[u8], - data_directory: PathBuf, - library_pub_id: libraries::PubId, -) -> Result { - use tokio::fs; - use tracing::{debug, error}; - - debug!(?cas_id, "Saving remote thumbnail to local storage"); - - // Convert to CasId for path computation - let cas_id = sd_core_prisma_helpers::CasId::from(cas_id); - - // Get the thumbnails directory - let thumbnails_directory = - sd_core_heavy_lifting::media_processor::get_thumbnails_directory(data_directory); - let library_dir = thumbnails_directory.join(library_pub_id.to_string()); - - // Get the shard hex for organizing thumbnails - let shard_hex = sd_core_heavy_lifting::media_processor::get_shard_hex(&cas_id); - - // Create the full directory path - let shard_dir = library_dir.join(shard_hex); - - // Create the directories if they don't exist - if let Err(e) = fs::create_dir_all(&shard_dir).await { - error!(?e, "Failed to create thumbnail directory structure in library folder, falling back to ephemeral"); - - // If we can't create in library folder, fall back to ephemeral - let ephemeral_dir = thumbnails_directory.join("ephemeral"); - let ephemeral_shard_dir = ephemeral_dir.join(shard_hex); - - if let Err(e) = fs::create_dir_all(&ephemeral_shard_dir).await { - error!( - ?e, - "Failed to create thumbnail directory structure in ephemeral folder" - ); - return Err(Error::InternalError); - } - - // Create the full path for the thumbnail in ephemeral - let thumbnail_path = ephemeral_shard_dir.join(format!("{}.webp", cas_id.as_str())); - - // Write the thumbnail data to disk - match fs::write(&thumbnail_path, thumbnail_data).await { - Ok(()) => { - debug!( - "Successfully saved remote thumbnail to ephemeral: {:?}", - thumbnail_path - ); - return Ok(thumbnail_path); - } - Err(e) => { - error!( - ?e, - "Failed to write remote thumbnail to disk in ephemeral folder" - ); - return Err(Error::InternalError); - } - } - } - - // Create the full path for the thumbnail in the library folder - let thumbnail_path = shard_dir.join(format!("{}.webp", cas_id.as_str())); - - // Write the thumbnail data to disk - match fs::write(&thumbnail_path, thumbnail_data).await { - Ok(()) => { - debug!( - "Successfully saved remote thumbnail to library folder: {:?}", - thumbnail_path - ); - Ok(thumbnail_path) - } - Err(e) => { - error!( - ?e, - "Failed to write remote thumbnail to disk in library folder" - ); - - // If writing to library folder fails, try ephemeral as a fallback - let ephemeral_dir = thumbnails_directory.join("ephemeral"); - let ephemeral_shard_dir = ephemeral_dir.join(shard_hex); - - if let Err(e) = fs::create_dir_all(&ephemeral_shard_dir).await { - error!( - ?e, - "Failed to create thumbnail directory structure in ephemeral folder" - ); - return Err(Error::InternalError); - } - - let ephemeral_thumbnail_path = - ephemeral_shard_dir.join(format!("{}.webp", cas_id.as_str())); - - match fs::write(&ephemeral_thumbnail_path, thumbnail_data).await { - Ok(()) => { - debug!( - "Successfully saved remote thumbnail to ephemeral fallback: {:?}", - ephemeral_thumbnail_path - ); - Ok(ephemeral_thumbnail_path) - } - Err(e) => { - error!( - ?e, - "Failed to write remote thumbnail to disk in ephemeral fallback folder" - ); - Err(Error::InternalError) - } - } - } - } -} diff --git a/core/crates/cloud-services/src/sync/ingest.rs b/core/crates/cloud-services/src/sync/ingest.rs deleted file mode 100644 index a7dd65af3..000000000 --- a/core/crates/cloud-services/src/sync/ingest.rs +++ /dev/null @@ -1,121 +0,0 @@ -use crate::Error; - -use sd_core_sync::SyncManager; - -use sd_actors::{Actor, Stopper}; - -use std::{ - future::IntoFuture, - sync::{ - atomic::{AtomicBool, Ordering}, - Arc, - }, -}; - -use futures::FutureExt; -use futures_concurrency::future::Race; -use tokio::{ - sync::Notify, - time::{sleep, Instant}, -}; -use tracing::{debug, error}; - -use super::{ReceiveAndIngestNotifiers, SyncActors, ONE_MINUTE}; - -/// Responsible for taking sync operations received from the cloud, -/// and applying them to the local database via the sync system's ingest actor. - -pub struct Ingester { - sync: SyncManager, - notifiers: Arc, - active: Arc, - active_notify: Arc, -} - -impl Actor for Ingester { - const IDENTIFIER: SyncActors = SyncActors::Ingester; - - async fn run(&mut self, stop: Stopper) { - enum Race { - Notified, - Stopped, - } - - loop { - self.active.store(true, Ordering::Relaxed); - self.active_notify.notify_waiters(); - - if let Err(e) = self.run_loop_iteration().await { - error!(?e, "Error during cloud sync ingester actor iteration"); - sleep(ONE_MINUTE).await; - continue; - } - - self.active.store(false, Ordering::Relaxed); - self.active_notify.notify_waiters(); - - if matches!( - ( - self.notifiers - .wait_notification_to_ingest() - .map(|()| Race::Notified), - stop.into_future().map(|()| Race::Stopped), - ) - .race() - .await, - Race::Stopped - ) { - break; - } - } - } -} - -impl Ingester { - pub const fn new( - sync: SyncManager, - notifiers: Arc, - active: Arc, - active_notify: Arc, - ) -> Self { - Self { - sync, - notifiers, - active, - active_notify, - } - } - - async fn run_loop_iteration(&self) -> Result<(), Error> { - let start = Instant::now(); - - let operations_to_ingest_count = self - .sync - .db - .cloud_crdt_operation() - .count(vec![]) - .exec() - .await - .map_err(sd_core_sync::Error::from)?; - - if operations_to_ingest_count == 0 { - debug!("Nothing to ingest, early finishing ingester loop"); - return Ok(()); - } - - debug!( - operations_to_ingest_count, - "Starting sync messages cloud ingestion loop" - ); - - let ingested_count = self.sync.ingest_ops().await?; - - debug!( - ingested_count, - "Finished sync messages cloud ingestion loop in {:?}", - start.elapsed() - ); - - Ok(()) - } -} diff --git a/core/crates/cloud-services/src/sync/mod.rs b/core/crates/cloud-services/src/sync/mod.rs deleted file mode 100644 index b694befb4..000000000 --- a/core/crates/cloud-services/src/sync/mod.rs +++ /dev/null @@ -1,136 +0,0 @@ -use crate::{CloudServices, Error}; - -use sd_core_sync::SyncManager; - -use sd_actors::{ActorsCollection, IntoActor}; -use sd_cloud_schema::sync::groups; -use sd_crypto::CryptoRng; - -use std::{ - fmt, - path::Path, - sync::{atomic::AtomicBool, Arc}, - time::Duration, -}; - -use futures_concurrency::future::TryJoin; -use tokio::sync::Notify; - -mod ingest; -mod receive; -mod send; - -use ingest::Ingester; -use receive::Receiver; -use send::Sender; - -const ONE_MINUTE: Duration = Duration::from_secs(60); - -#[derive(Default)] -pub struct SyncActorsState { - pub send_active: Arc, - pub receive_active: Arc, - pub ingest_active: Arc, - pub state_change_notifier: Arc, - receiver_and_ingester_notifiers: Arc, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, specta::Type)] -#[specta(rename = "CloudSyncActors")] -pub enum SyncActors { - Ingester, - Sender, - Receiver, -} - -impl fmt::Display for SyncActors { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::Ingester => write!(f, "Cloud Sync Ingester"), - Self::Sender => write!(f, "Cloud Sync Sender"), - Self::Receiver => write!(f, "Cloud Sync Receiver"), - } - } -} - -#[derive(Debug, Default)] -pub struct ReceiveAndIngestNotifiers { - ingester: Notify, - receiver: Notify, -} - -impl ReceiveAndIngestNotifiers { - pub fn notify_receiver(&self) { - self.receiver.notify_one(); - } - - async fn wait_notification_to_receive(&self) { - self.receiver.notified().await; - } - - fn notify_ingester(&self) { - self.ingester.notify_one(); - } - - async fn wait_notification_to_ingest(&self) { - self.ingester.notified().await; - } -} - -pub async fn declare_actors( - data_dir: Box, - cloud_services: Arc, - actors: &ActorsCollection, - actors_state: &SyncActorsState, - sync_group_pub_id: groups::PubId, - sync: SyncManager, - rng: CryptoRng, -) -> Result, Error> { - let (sender, receiver) = ( - Sender::new( - sync_group_pub_id, - sync.clone(), - Arc::clone(&cloud_services), - Arc::clone(&actors_state.send_active), - Arc::clone(&actors_state.state_change_notifier), - rng, - ), - Receiver::new( - data_dir, - sync_group_pub_id, - cloud_services.clone(), - sync.clone(), - Arc::clone(&actors_state.receiver_and_ingester_notifiers), - Arc::clone(&actors_state.receive_active), - Arc::clone(&actors_state.state_change_notifier), - ), - ) - .try_join() - .await?; - - let ingester = Ingester::new( - sync, - Arc::clone(&actors_state.receiver_and_ingester_notifiers), - Arc::clone(&actors_state.ingest_active), - Arc::clone(&actors_state.state_change_notifier), - ); - - actors - .declare_many_boxed([ - sender.into_actor(), - receiver.into_actor(), - ingester.into_actor(), - ]) - .await; - - cloud_services - .cloud_p2p() - .await? - .register_sync_messages_receiver_notifier( - sync_group_pub_id, - Arc::clone(&actors_state.receiver_and_ingester_notifiers), - ) - .await; - - Ok(Arc::clone(&actors_state.receiver_and_ingester_notifiers)) -} diff --git a/core/crates/cloud-services/src/sync/receive.rs b/core/crates/cloud-services/src/sync/receive.rs deleted file mode 100644 index 53a8230a3..000000000 --- a/core/crates/cloud-services/src/sync/receive.rs +++ /dev/null @@ -1,356 +0,0 @@ -use crate::{CloudServices, Error, KeyManager}; - -use sd_cloud_schema::{ - devices, - sync::{ - groups, - messages::{pull, MessagesCollection}, - }, - Client, Request, Response, -}; -use sd_core_sync::{ - cloud_crdt_op_db, CRDTOperation, CompressedCRDTOperationsPerModel, SyncManager, -}; - -use sd_actors::{Actor, Stopper}; -use sd_crypto::{ - cloud::{OneShotDecryption, SecretKey, StreamDecryption}, - primitives::{EncryptedBlock, StreamNonce}, -}; -use sd_prisma::prisma::PrismaClient; - -use std::{ - collections::{hash_map::Entry, HashMap}, - future::IntoFuture, - path::Path, - sync::{ - atomic::{AtomicBool, Ordering}, - Arc, - }, -}; - -use chrono::{DateTime, Utc}; -use futures::{FutureExt, StreamExt}; -use futures_concurrency::future::{Race, TryJoin}; -use quic_rpc::transport::quinn::QuinnConnector; -use serde::{Deserialize, Serialize}; -use tokio::{fs, io, sync::Notify, time::sleep}; -use tracing::{debug, error, instrument, warn}; -use uuid::Uuid; - -use super::{ReceiveAndIngestNotifiers, SyncActors, ONE_MINUTE}; - -const CLOUD_SYNC_DATA_KEEPER_DIRECTORY: &str = "cloud_sync_data_keeper"; - -/// Responsible for downloading sync operations from the cloud to be processed by the ingester - -pub struct Receiver { - keeper: LastTimestampKeeper, - sync_group_pub_id: groups::PubId, - device_pub_id: devices::PubId, - cloud_services: Arc, - cloud_client: Client>, - key_manager: Arc, - sync: SyncManager, - notifiers: Arc, - active: Arc, - active_notifier: Arc, -} - -impl Actor for Receiver { - const IDENTIFIER: SyncActors = SyncActors::Receiver; - - async fn run(&mut self, stop: Stopper) { - enum Race { - Continue, - Stop, - } - - loop { - self.active.store(true, Ordering::Relaxed); - self.active_notifier.notify_waiters(); - - let res = self.run_loop_iteration().await; - - self.active.store(false, Ordering::Relaxed); - - if let Err(e) = res { - error!(?e, "Error during cloud sync receiver actor iteration"); - sleep(ONE_MINUTE).await; - continue; - } - - self.active_notifier.notify_waiters(); - - if matches!( - ( - sleep(ONE_MINUTE).map(|()| Race::Continue), - self.notifiers - .wait_notification_to_receive() - .map(|()| Race::Continue), - stop.into_future().map(|()| Race::Stop), - ) - .race() - .await, - Race::Stop - ) { - break; - } - } - } -} - -impl Receiver { - pub async fn new( - data_dir: impl AsRef + Send, - sync_group_pub_id: groups::PubId, - cloud_services: Arc, - sync: SyncManager, - notifiers: Arc, - active: Arc, - active_notify: Arc, - ) -> Result { - let (keeper, cloud_client, key_manager) = ( - LastTimestampKeeper::load(data_dir.as_ref(), sync_group_pub_id), - cloud_services.client(), - cloud_services.key_manager(), - ) - .try_join() - .await?; - - Ok(Self { - keeper, - sync_group_pub_id, - device_pub_id: devices::PubId(Uuid::from(&sync.device_pub_id)), - cloud_services, - cloud_client, - key_manager, - sync, - notifiers, - active, - active_notifier: active_notify, - }) - } - - async fn run_loop_iteration(&mut self) -> Result<(), Error> { - let mut responses_stream = self - .cloud_client - .sync() - .messages() - .pull(pull::Request { - access_token: self - .cloud_services - .token_refresher - .get_access_token() - .await?, - group_pub_id: self.sync_group_pub_id, - current_device_pub_id: self.device_pub_id, - start_time_per_device: self - .keeper - .timestamps - .iter() - .map(|(device_pub_id, timestamp)| (*device_pub_id, *timestamp)) - .collect(), - }) - .await?; - - while let Some(new_messages_res) = responses_stream.next().await { - let pull::Response(new_messages) = new_messages_res??; - if new_messages.is_empty() { - break; - } - - self.handle_new_messages(new_messages).await?; - } - - debug!("Finished sync messages receiver actor iteration"); - - self.keeper.save().await - } - - async fn handle_new_messages( - &mut self, - new_messages: Vec, - ) -> Result<(), Error> { - debug!( - new_messages_collections_count = new_messages.len(), - start_time = ?new_messages.first().map(|c| c.start_time), - end_time = ?new_messages.first().map(|c| c.end_time), - "Handling new sync messages collections", - ); - - for message in new_messages.into_iter().filter(|message| { - if message.original_device_pub_id == self.device_pub_id { - warn!("Received sync message from the current device, need to check backend, this is a bug!"); - false - } else { - true - } - }) { - debug!( - new_messages_count = message.operations_count, - start_time = ?message.start_time, - end_time = ?message.end_time, - "Handling new sync messages", - ); - - let (device_pub_id, timestamp) = handle_single_message( - self.sync_group_pub_id, - message, - &self.key_manager, - &self.sync, - ) - .await?; - - match self.keeper.timestamps.entry(device_pub_id) { - Entry::Occupied(mut entry) => { - if entry.get() < ×tamp { - *entry.get_mut() = timestamp; - } - } - - Entry::Vacant(entry) => { - entry.insert(timestamp); - } - } - - // To ingest after each sync message collection is received, we MUST download and - // store the messages SEQUENTIALLY, otherwise we might ingest messages out of order - // due to parallel downloads - self.notifiers.notify_ingester(); - } - - Ok(()) - } -} - -#[instrument( - skip_all, - fields(%sync_group_pub_id, %original_device_pub_id, operations_count, ?key_hash, %end_time), -)] -async fn handle_single_message( - sync_group_pub_id: groups::PubId, - MessagesCollection { - original_device_pub_id, - end_time, - operations_count, - key_hash, - encrypted_messages, - .. - }: MessagesCollection, - key_manager: &KeyManager, - sync: &SyncManager, -) -> Result<(devices::PubId, DateTime), Error> { - // FIXME(@fogodev): If we don't have the key hash, we need to fetch it from another device in the group if possible - let Some(secret_key) = key_manager.get_key(sync_group_pub_id, &key_hash).await else { - return Err(Error::MissingKeyHash); - }; - - debug!( - size = encrypted_messages.len(), - "Received encrypted sync messages collection" - ); - - let crdt_ops = decrypt_messages(encrypted_messages, secret_key, original_device_pub_id).await?; - - assert_eq!( - crdt_ops.len(), - operations_count as usize, - "Sync messages count mismatch" - ); - - write_cloud_ops_to_db(crdt_ops, &sync.db).await?; - - Ok((original_device_pub_id, end_time)) -} - -#[instrument(skip(encrypted_messages, secret_key), fields(messages_size = %encrypted_messages.len()), err)] -async fn decrypt_messages( - encrypted_messages: Vec, - secret_key: SecretKey, - devices::PubId(device_pub_id): devices::PubId, -) -> Result, Error> { - let plain_text = if encrypted_messages.len() <= EncryptedBlock::CIPHER_TEXT_SIZE { - OneShotDecryption::decrypt(&secret_key, encrypted_messages.as_slice().into()) - .map_err(Error::Decrypt)? - } else { - let (nonce, cipher_text) = encrypted_messages.split_at(size_of::()); - - let mut plain_text = Vec::with_capacity(cipher_text.len()); - - StreamDecryption::decrypt( - &secret_key, - nonce.try_into().expect("we split the correct amount"), - cipher_text, - &mut plain_text, - ) - .await - .map_err(Error::Decrypt)?; - - plain_text - }; - - rmp_serde::from_slice::(&plain_text) - .map(|compressed_ops| compressed_ops.into_ops(device_pub_id)) - .map_err(Error::DeserializationFailureToPullSyncMessages) -} - -#[instrument(skip_all, err)] -pub async fn write_cloud_ops_to_db( - ops: Vec, - db: &PrismaClient, -) -> Result<(), sd_core_sync::Error> { - db._batch( - ops.into_iter() - .map(|op| cloud_crdt_op_db(&op).map(|op| op.to_query(db))) - .collect::, _>>()?, - ) - .await?; - - Ok(()) -} - -#[derive(Serialize, Deserialize, Debug)] -struct LastTimestampKeeper { - timestamps: HashMap>, - file_path: Box, -} - -impl LastTimestampKeeper { - async fn load(data_dir: &Path, sync_group_pub_id: groups::PubId) -> Result { - let cloud_sync_data_directory = data_dir.join(CLOUD_SYNC_DATA_KEEPER_DIRECTORY); - - fs::create_dir_all(&cloud_sync_data_directory) - .await - .map_err(Error::FailedToCreateTimestampKeepersDirectory)?; - - let file_path = cloud_sync_data_directory - .join(format!("{sync_group_pub_id}.bin")) - .into_boxed_path(); - - match fs::read(&file_path).await { - Ok(bytes) => Ok(Self { - timestamps: rmp_serde::from_slice(&bytes) - .map_err(Error::LastTimestampKeeperDeserialization)?, - file_path, - }), - - Err(e) if e.kind() == io::ErrorKind::NotFound => Ok(Self { - timestamps: HashMap::new(), - file_path, - }), - - Err(e) => Err(Error::FailedToReadLastTimestampKeeper(e)), - } - } - - async fn save(&self) -> Result<(), Error> { - fs::write( - &self.file_path, - &rmp_serde::to_vec_named(&self.timestamps) - .map_err(Error::LastTimestampKeeperSerialization)?, - ) - .await - .map_err(Error::FailedToWriteLastTimestampKeeper) - } -} diff --git a/core/crates/cloud-services/src/sync/send.rs b/core/crates/cloud-services/src/sync/send.rs deleted file mode 100644 index cf7f3a544..000000000 --- a/core/crates/cloud-services/src/sync/send.rs +++ /dev/null @@ -1,337 +0,0 @@ -use crate::{CloudServices, Error, KeyManager}; - -use sd_core_sync::{CompressedCRDTOperationsPerModelPerDevice, SyncEvent, SyncManager, NTP64}; - -use sd_actors::{Actor, Stopper}; -use sd_cloud_schema::{ - devices, - error::{ClientSideError, NotFoundError}, - sync::{groups, messages}, - Client, Request, Response, -}; -use sd_crypto::{ - cloud::{OneShotEncryption, SecretKey, StreamEncryption}, - primitives::EncryptedBlock, - CryptoRng, SeedableRng, -}; -use sd_utils::{datetime_to_timestamp, timestamp_to_datetime}; - -use std::{ - future::IntoFuture, - pin::pin, - sync::{ - atomic::{AtomicBool, Ordering}, - Arc, - }, - time::{Duration, UNIX_EPOCH}, -}; - -use chrono::{DateTime, Utc}; -use futures::{FutureExt, StreamExt, TryStreamExt}; -use futures_concurrency::future::{Race, TryJoin}; -use quic_rpc::transport::quinn::QuinnConnector; -use tokio::{ - sync::{broadcast, Notify}, - time::sleep, -}; -use tracing::{debug, error}; -use uuid::Uuid; - -use super::{SyncActors, ONE_MINUTE}; - -const TEN_SECONDS: Duration = Duration::from_secs(10); - -const MESSAGES_COLLECTION_SIZE: u32 = 10_000; - -enum RaceNotifiedOrStopped { - Notified, - Stopped, -} - -enum LoopStatus { - SentMessages, - Idle, -} - -type LatestTimestamp = NTP64; - -#[derive(Debug)] -pub struct Sender { - sync_group_pub_id: groups::PubId, - sync: SyncManager, - cloud_services: Arc, - cloud_client: Client>, - key_manager: Arc, - is_active: Arc, - state_notify: Arc, - rng: CryptoRng, - maybe_latest_timestamp: Option, -} - -impl Actor for Sender { - const IDENTIFIER: SyncActors = SyncActors::Sender; - - async fn run(&mut self, stop: Stopper) { - loop { - self.is_active.store(true, Ordering::Relaxed); - self.state_notify.notify_waiters(); - - let res = self.run_loop_iteration().await; - - self.is_active.store(false, Ordering::Relaxed); - - match res { - Ok(LoopStatus::SentMessages) => { - if let Ok(cloud_p2p) = self.cloud_services.cloud_p2p().await.map_err(|e| { - error!(?e, "Failed to get cloud p2p client on sender actor"); - }) { - cloud_p2p - .notify_new_sync_messages(self.sync_group_pub_id) - .await; - } - } - - Ok(LoopStatus::Idle) => {} - - Err(e) => { - error!(?e, "Error during cloud sync sender actor iteration"); - sleep(ONE_MINUTE).await; - continue; - } - } - - self.state_notify.notify_waiters(); - - if matches!( - ( - // recreate subscription each time so that existing messages are dropped - wait_notification(self.sync.subscribe()), - stop.into_future().map(|()| RaceNotifiedOrStopped::Stopped), - ) - .race() - .await, - RaceNotifiedOrStopped::Stopped - ) { - break; - } - - sleep(TEN_SECONDS).await; - } - } -} - -impl Sender { - pub async fn new( - sync_group_pub_id: groups::PubId, - sync: SyncManager, - cloud_services: Arc, - is_active: Arc, - state_notify: Arc, - rng: CryptoRng, - ) -> Result { - let (cloud_client, key_manager) = (cloud_services.client(), cloud_services.key_manager()) - .try_join() - .await?; - - Ok(Self { - sync_group_pub_id, - sync, - cloud_services, - cloud_client, - key_manager, - is_active, - state_notify, - rng, - maybe_latest_timestamp: None, - }) - } - - async fn run_loop_iteration(&mut self) -> Result { - debug!("Starting cloud sender actor loop iteration"); - - let current_device_pub_id = devices::PubId(Uuid::from(&self.sync.device_pub_id)); - - let (key_hash, secret_key) = self - .key_manager - .get_latest_key(self.sync_group_pub_id) - .await - .ok_or(Error::MissingSyncGroupKey(self.sync_group_pub_id))?; - - let current_latest_timestamp = self.get_latest_timestamp(current_device_pub_id).await?; - - let mut crdt_ops_stream = pin!(self.sync.stream_device_ops( - &self.sync.device_pub_id, - MESSAGES_COLLECTION_SIZE, - current_latest_timestamp - )); - - let mut status = LoopStatus::Idle; - - let mut new_latest_timestamp = current_latest_timestamp; - - debug!( - chunk_size = MESSAGES_COLLECTION_SIZE, - "Trying to fetch chunk of sync messages from the database" - ); - while let Some(ops_res) = crdt_ops_stream.next().await { - let ops = ops_res?; - - let (Some(first), Some(last)) = (ops.first(), ops.last()) else { - break; - }; - - debug!("Got first and last sync messages"); - - #[allow(clippy::cast_possible_truncation)] - let operations_count = ops.len() as u32; - - debug!(operations_count, "Got chunk of sync messages"); - - new_latest_timestamp = last.timestamp; - - let start_time = timestamp_to_datetime(first.timestamp); - let end_time = timestamp_to_datetime(last.timestamp); - - // Ignoring this device_pub_id here as we already know it - let (_device_pub_id, compressed_ops) = - CompressedCRDTOperationsPerModelPerDevice::new_single_device(ops); - - let messages_bytes = rmp_serde::to_vec_named(&compressed_ops) - .map_err(Error::SerializationFailureToPushSyncMessages)?; - - let encrypted_messages = - encrypt_messages(&secret_key, &mut self.rng, messages_bytes).await?; - - let encrypted_messages_size = encrypted_messages.len(); - - debug!( - operations_count, - encrypted_messages_size, "Sending sync messages to cloud", - ); - - self.cloud_client - .sync() - .messages() - .push(messages::push::Request { - access_token: self - .cloud_services - .token_refresher - .get_access_token() - .await?, - group_pub_id: self.sync_group_pub_id, - device_pub_id: current_device_pub_id, - key_hash: key_hash.clone(), - operations_count, - time_range: (start_time, end_time), - encrypted_messages, - }) - .await??; - - debug!( - operations_count, - encrypted_messages_size, "Sent sync messages to cloud", - ); - - status = LoopStatus::SentMessages; - } - - self.maybe_latest_timestamp = Some(new_latest_timestamp); - - debug!("Finished cloud sender actor loop iteration"); - - Ok(status) - } - - async fn get_latest_timestamp( - &self, - current_device_pub_id: devices::PubId, - ) -> Result { - if let Some(latest_timestamp) = &self.maybe_latest_timestamp { - Ok(*latest_timestamp) - } else { - let latest_time = match self - .cloud_client - .sync() - .messages() - .get_latest_time(messages::get_latest_time::Request { - access_token: self - .cloud_services - .token_refresher - .get_access_token() - .await?, - group_pub_id: self.sync_group_pub_id, - kind: messages::get_latest_time::Kind::ForCurrentDevice(current_device_pub_id), - }) - .await? - { - Ok(messages::get_latest_time::Response { - latest_time, - latest_device_pub_id, - }) => { - assert_eq!(latest_device_pub_id, current_device_pub_id); - latest_time - } - - Err(sd_cloud_schema::Error::Client(ClientSideError::NotFound( - NotFoundError::LatestSyncMessageTime, - ))) => DateTime::::from(UNIX_EPOCH), - - Err(e) => return Err(e.into()), - }; - - Ok(datetime_to_timestamp(latest_time)) - } - } -} - -async fn encrypt_messages( - secret_key: &SecretKey, - rng: &mut CryptoRng, - messages_bytes: Vec, -) -> Result, Error> { - if messages_bytes.len() <= EncryptedBlock::PLAIN_TEXT_SIZE { - let mut nonce_and_cipher_text = Vec::with_capacity(OneShotEncryption::cipher_text_size( - secret_key, - messages_bytes.len(), - )); - - let EncryptedBlock { nonce, cipher_text } = - OneShotEncryption::encrypt(secret_key, messages_bytes.as_slice(), rng) - .map_err(Error::Encrypt)?; - - nonce_and_cipher_text.extend_from_slice(nonce.as_slice()); - nonce_and_cipher_text.extend(&cipher_text); - - Ok(nonce_and_cipher_text) - } else { - let mut rng = CryptoRng::from_seed(rng.generate_fixed()); - let mut nonce_and_cipher_text = Vec::with_capacity(StreamEncryption::cipher_text_size( - secret_key, - messages_bytes.len(), - )); - - let (nonce, cipher_stream) = - StreamEncryption::encrypt(secret_key, messages_bytes.as_slice(), &mut rng); - - nonce_and_cipher_text.extend_from_slice(nonce.as_slice()); - - let mut cipher_stream = pin!(cipher_stream); - - while let Some(ciphered_chunk) = cipher_stream.try_next().await.map_err(Error::Encrypt)? { - nonce_and_cipher_text.extend(ciphered_chunk); - } - - Ok(nonce_and_cipher_text) - } -} - -async fn wait_notification(mut rx: broadcast::Receiver) -> RaceNotifiedOrStopped { - // wait until Created message comes in - loop { - if matches!(rx.recv().await, Ok(SyncEvent::Created)) { - break; - }; - } - - RaceNotifiedOrStopped::Notified -} diff --git a/core/crates/cloud-services/src/token_refresher.rs b/core/crates/cloud-services/src/token_refresher.rs deleted file mode 100644 index ae11e15db..000000000 --- a/core/crates/cloud-services/src/token_refresher.rs +++ /dev/null @@ -1,468 +0,0 @@ -use sd_cloud_schema::auth::{AccessToken, RefreshToken}; - -use std::{pin::pin, time::Duration}; - -use base64::prelude::{Engine, BASE64_URL_SAFE_NO_PAD}; -use chrono::{DateTime, Utc}; -use futures::StreamExt; -use futures_concurrency::stream::Merge; -use reqwest::Url; -use reqwest_middleware::{reqwest::header, ClientWithMiddleware}; -use tokio::{ - spawn, - sync::oneshot, - time::{interval, sleep, MissedTickBehavior}, -}; -use tokio_stream::wrappers::IntervalStream; -use tracing::{error, warn}; - -use super::{Error, GetTokenError}; - -const ONE_MINUTE: Duration = Duration::from_secs(60); -const TEN_SECONDS: Duration = Duration::from_secs(10); - -enum Message { - Init( - ( - AccessToken, - RefreshToken, - oneshot::Sender>, - ), - ), - CheckInitialization(oneshot::Sender>), - RequestToken(oneshot::Sender>), - RefreshTime, - Tick, -} - -#[derive(Debug, Clone)] -pub struct TokenRefresher { - tx: flume::Sender, -} - -impl TokenRefresher { - pub(crate) fn new(http_client: ClientWithMiddleware, auth_server_url: Url) -> Self { - let (tx, rx) = flume::bounded(8); - - spawn(async move { - let refresh_url = auth_server_url - .join("/api/auth/session/refresh") - .expect("hardcoded refresh url path"); - - while let Err(e) = spawn(Runner::run( - http_client.clone(), - refresh_url.clone(), - rx.clone(), - )) - .await - { - if e.is_panic() { - if let Some(msg) = e.into_panic().downcast_ref::<&str>() { - error!(?msg, "Panic in request handler!"); - } else { - error!("Some unknown panic in request handler!"); - } - } - } - }); - - Self { tx } - } - - pub async fn init( - &self, - access_token: AccessToken, - refresh_token: RefreshToken, - ) -> Result<(), Error> { - let (tx, rx) = oneshot::channel(); - self.tx - .send_async(Message::Init((access_token, refresh_token, tx))) - .await - .expect("Token refresher channel closed"); - - rx.await.expect("Token refresher channel closed") - } - - pub async fn check_initialization(&self) -> Result<(), GetTokenError> { - let (tx, rx) = oneshot::channel(); - self.tx - .send_async(Message::CheckInitialization(tx)) - .await - .expect("Token refresher channel closed"); - - rx.await.expect("Token refresher channel closed") - } - - pub async fn get_access_token(&self) -> Result { - let (tx, rx) = oneshot::channel(); - self.tx - .send_async(Message::RequestToken(tx)) - .await - .expect("Token refresher channel closed"); - - rx.await.expect("Token refresher channel closed") - } -} - -struct Runner { - initialized: bool, - http_client: ClientWithMiddleware, - refresh_url: Url, - current_token: Option, - current_refresh_token: Option, - token_decoding_buffer: Vec, - refresh_tx: flume::Sender, -} - -impl Runner { - async fn run( - http_client: ClientWithMiddleware, - refresh_url: Url, - msgs_rx: flume::Receiver, - ) { - let (refresh_tx, refresh_rx) = flume::bounded(1); - - let mut ticker = interval(TEN_SECONDS); - ticker.set_missed_tick_behavior(MissedTickBehavior::Skip); - - let mut msg_stream = pin!(( - msgs_rx.into_stream(), - refresh_rx.into_stream(), - IntervalStream::new(ticker).map(|_| Message::Tick) - ) - .merge()); - - let mut runner = Self { - initialized: false, - http_client, - refresh_url, - current_token: None, - current_refresh_token: None, - token_decoding_buffer: Vec::new(), - refresh_tx, - }; - - while let Some(msg) = msg_stream.next().await { - match msg { - Message::Init((access_token, refresh_token, ack)) => { - if ack - .send(runner.init(access_token, refresh_token).await) - .is_err() - { - error!("Failed to send init token refresher response, receiver dropped;"); - } - } - - Message::CheckInitialization(ack) => runner.check_initialization(ack), - - Message::RequestToken(ack) => runner.reply_token(ack), - - Message::RefreshTime => { - if let Err(e) = runner.refresh().await { - error!(?e, "Failed to refresh token: {e}"); - } - } - - Message::Tick => runner.tick().await, - } - } - } - - async fn init( - &mut self, - access_token: AccessToken, - refresh_token: RefreshToken, - ) -> Result<(), Error> { - let access_token_duration = - Self::extract_access_token_duration(&mut self.token_decoding_buffer, &access_token)?; - - self.initialized = true; - self.current_token = Some(access_token); - self.current_refresh_token = Some(refresh_token); - - // If the token has an expiration smaller than a minute, we need to refresh it immediately. - if access_token_duration < ONE_MINUTE { - self.refresh_tx - .send_async(Message::RefreshTime) - .await - .expect("refresh channel never closes"); - } else { - // This task will be mostly parked waiting a sleep - spawn(Self::schedule_refresh( - self.refresh_tx.clone(), - access_token_duration - ONE_MINUTE, - )); - } - - Ok(()) - } - - fn reply_token(&self, ack: oneshot::Sender>) { - if ack - .send(self.current_token.clone().ok_or({ - if self.initialized { - GetTokenError::FailedToRefresh - } else { - GetTokenError::RefresherNotInitialized - } - })) - .is_err() - { - warn!("Failed to send access token response, receiver dropped;"); - } - } - - async fn refresh(&mut self) -> Result<(), Error> { - let RefreshToken(refresh_token) = self - .current_refresh_token - .clone() - .expect("refresh token is set otherwise we wouldn't be here"); - - let response = self - .http_client - .post(self.refresh_url.clone()) - .header("rid", "session") - .header(header::AUTHORIZATION, format!("Bearer {refresh_token}")) - .send() - .await - .map_err(Error::RefreshTokenRequest)? - .error_for_status() - .map_err(Error::AuthServerError)?; - - if let (Some(access_token), Some(refresh_token)) = ( - response.headers().get("st-access-token"), - response.headers().get("st-refresh-token"), - ) { - // Only set values if we can parse both of them to strings - let (access_token, refresh_token) = ( - Self::token_header_value_to_string(access_token)?, - Self::token_header_value_to_string(refresh_token)?, - ); - - self.current_token = Some(AccessToken(access_token)); - self.current_refresh_token = Some(RefreshToken(refresh_token)); - } else { - return Err(Error::MissingTokensOnRefreshResponse); - } - - Ok(()) - } - - fn extract_access_token_duration( - token_decoding_buffer: &mut Vec, - AccessToken(token): &AccessToken, - ) -> Result { - #[derive(serde::Deserialize)] - struct Token { - #[serde(with = "chrono::serde::ts_seconds")] - exp: DateTime, - } - - token_decoding_buffer.clear(); - - // The format of a JWT token is simple: - // ".." - BASE64_URL_SAFE_NO_PAD.decode_vec( - token.split('.').nth(1).ok_or(Error::MissingClaims)?, - token_decoding_buffer, - )?; - - serde_json::from_slice::(token_decoding_buffer)? - .exp - .signed_duration_since(Utc::now()) - .to_std() - .map_err(|_| Error::TokenExpired) - } - - async fn schedule_refresh(refresh_tx: flume::Sender, wait_time: Duration) { - sleep(wait_time).await; - refresh_tx - .send_async(Message::RefreshTime) - .await - .expect("Refresh channel closed"); - } - - fn token_header_value_to_string(token: &header::HeaderValue) -> Result { - token.to_str().map(str::to_string).map_err(Into::into) - } - - fn check_initialization(&self, ack: oneshot::Sender>) { - if ack - .send(if self.initialized { - Ok(()) - } else { - Err(GetTokenError::RefresherNotInitialized) - }) - .is_err() - { - warn!("Failed to send access token response, receiver dropped;"); - } - } - - /// This method is a safeguard to make sure we try to keep refreshing tokens even if they - /// already expired, as the refresh token has a bigger expiration than the access token. - async fn tick(&mut self) { - if let Some(access_token) = &self.current_token { - if matches!( - Self::extract_access_token_duration(&mut self.token_decoding_buffer, access_token), - Err(Error::TokenExpired) - ) { - if let Err(e) = self.refresh().await { - error!(?e, "Failed to refresh expired token on tick method;"); - } - } - } - } -} - -/// This test is here for documentation purposes only, they are not meant to be run. -/// They're just examples of how to sign-up/sign-in and refresh tokens -#[cfg(test)] -mod tests { - use reqwest::header; - use reqwest_middleware::ClientBuilder; - use reqwest_retry::{policies::ExponentialBackoff, RetryTransientMiddleware}; - use serde_json::json; - - use crate::AUTH_SERVER_URL; - - use super::*; - - async fn get_tokens() -> (AccessToken, RefreshToken) { - let client = reqwest::Client::new(); - - let req_body = json!({ - "formFields": [ - { - "id": "email", - "value": "johndoe@gmail.com" - }, - { - "id": "password", - "value": "testPass123" - } - ] - }); - - let response = client - .post(format!("{AUTH_SERVER_URL}/api/auth/public/signup")) - .header("rid", "emailpassword") - .header("st-auth-mode", "header") - .json(&req_body) - .send() - .await - .unwrap(); - - assert_eq!(response.status(), 200); - - if let (Some(access_token), Some(refresh_token)) = ( - response.headers().get("st-access-token"), - response.headers().get("st-refresh-token"), - ) { - ( - AccessToken(access_token.to_str().unwrap().to_string()), - RefreshToken(refresh_token.to_str().unwrap().to_string()), - ) - } else { - let response = client - .post(format!("{AUTH_SERVER_URL}/api/auth/public/signin")) - .header("rid", "emailpassword") - .header("st-auth-mode", "header") - .json(&req_body) - .send() - .await - .unwrap(); - - assert_eq!(response.status(), 200); - - ( - AccessToken( - response - .headers() - .get("st-access-token") - .unwrap() - .to_str() - .unwrap() - .to_string(), - ), - RefreshToken( - response - .headers() - .get("st-refresh-token") - .unwrap() - .to_str() - .unwrap() - .to_string(), - ), - ) - } - } - - #[ignore = "Documentation only"] - #[tokio::test] - async fn test_refresh_token() { - let (AccessToken(access_token), RefreshToken(refresh_token)) = get_tokens().await; - - let client = reqwest::Client::new(); - let response = client - .post(format!("{AUTH_SERVER_URL}/api/auth/session/refresh")) - .header("rid", "session") - .header(header::AUTHORIZATION, format!("Bearer {refresh_token}")) - .send() - .await - .unwrap(); - - assert_eq!(response.status(), 200); - - assert_ne!( - response - .headers() - .get("st-access-token") - .unwrap() - .to_str() - .unwrap(), - access_token.as_str() - ); - - assert_ne!( - response - .headers() - .get("st-refresh-token") - .unwrap() - .to_str() - .unwrap(), - refresh_token.as_str() - ); - } - - #[ignore = "Needs an actual SuperTokens auth server running"] - #[tokio::test] - async fn test_refresher_runner() { - let http_client_builder = reqwest::Client::builder().timeout(Duration::from_secs(3)); - - let http_client = ClientBuilder::new(http_client_builder.build().unwrap()) - .with(RetryTransientMiddleware::new_with_policy( - ExponentialBackoff::builder().build_with_max_retries(3), - )) - .build(); - - let (refresh_tx, _refresh_rx) = flume::bounded(1); - - let mut runner = Runner { - initialized: false, - http_client, - refresh_url: Url::parse(&format!("{AUTH_SERVER_URL}/api/auth/session/refresh")) - .unwrap(), - current_token: None, - current_refresh_token: None, - token_decoding_buffer: Vec::new(), - refresh_tx, - }; - - let (access_token, refresh_token) = get_tokens().await; - - runner.init(access_token, refresh_token).await.unwrap(); - - runner.refresh().await.unwrap(); - } -} diff --git a/core/crates/file-path-helper/Cargo.toml b/core/crates/file-path-helper/Cargo.toml deleted file mode 100644 index 87f79968e..000000000 --- a/core/crates/file-path-helper/Cargo.toml +++ /dev/null @@ -1,33 +0,0 @@ -[package] -name = "sd-core-file-path-helper" -version = "0.1.0" - -authors = ["Ericson Soares "] -edition.workspace = true -license.workspace = true -readme = "README.md" -repository.workspace = true -rust-version.workspace = true - -[dependencies] -# Inner Core Sub-crates -sd-core-prisma-helpers = { path = "../prisma-helpers" } -sd-core-sync = { path = "../sync" } - -# Spacedrive Sub-crates -sd-prisma = { path = "../../../crates/prisma" } -sd-utils = { path = "../../../crates/utils" } - -# Workspace dependencies -chrono = { workspace = true, features = ["serde"] } -prisma-client-rust = { workspace = true } -regex = { workspace = true } -serde = { workspace = true, features = ["derive"] } -thiserror = { workspace = true } -tokio = { workspace = true, features = ["fs"] } -tracing = { workspace = true } - -# Specific File Path Helper dependencies -[target.'cfg(target_os = "windows")'.dependencies.windows] -features = ["Win32_Security", "Win32_Storage_FileSystem"] -version = "0.58" diff --git a/core/crates/file-path-helper/README.md b/core/crates/file-path-helper/README.md deleted file mode 100644 index 46e7d51ab..000000000 --- a/core/crates/file-path-helper/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# Spacedrive FilePathHelper - -A bunch of file_path related abstractions. diff --git a/core/crates/file-path-helper/src/isolated_file_path_data.rs b/core/crates/file-path-helper/src/isolated_file_path_data.rs deleted file mode 100644 index fe83bbee9..000000000 --- a/core/crates/file-path-helper/src/isolated_file_path_data.rs +++ /dev/null @@ -1,799 +0,0 @@ -use sd_core_prisma_helpers::{ - file_path_for_file_identifier, file_path_for_media_processor, file_path_for_object_validator, - file_path_to_full_path, file_path_to_handle_custom_uri, file_path_to_handle_p2p_serve_file, - file_path_to_isolate, file_path_to_isolate_with_id, file_path_to_isolate_with_pub_id, - file_path_walker, file_path_watcher_remove, file_path_with_object, -}; - -use sd_prisma::prisma::{file_path, location}; -use sd_utils::error::NonUtf8PathError; - -use std::{ - borrow::Cow, - fmt, - path::{Path, PathBuf, MAIN_SEPARATOR, MAIN_SEPARATOR_STR}, - sync::OnceLock, -}; - -use regex::RegexSet; -use serde::{Deserialize, Serialize}; - -use super::FilePathError; - -static FORBIDDEN_FILE_NAMES: OnceLock = OnceLock::new(); - -#[derive(Debug)] -pub struct IsolatedFilePathDataParts<'a> { - pub location_id: location::id::Type, - pub materialized_path: &'a str, - pub is_dir: bool, - pub name: &'a str, - pub extension: &'a str, - relative_path: &'a str, -} - -#[derive(Serialize, Deserialize, Debug, Hash, Eq, PartialEq, Clone, Default)] -#[non_exhaustive] -pub struct IsolatedFilePathData<'a> { - // WARN! These fields MUST NOT be changed outside the location module, that's why they have this visibility - // and are not public. They have some specific logic on them and should not be written to directly. - // If you wanna access one of them outside from location module, write yourself an accessor method - // to have read only access to them. - pub(super) location_id: location::id::Type, - pub(super) materialized_path: Cow<'a, str>, - pub(super) is_dir: bool, - pub(super) name: Cow<'a, str>, - pub(super) extension: Cow<'a, str>, - relative_path: Cow<'a, str>, -} - -impl IsolatedFilePathData<'static> { - pub fn new( - location_id: location::id::Type, - location_path: impl AsRef, - full_path: impl AsRef, - is_dir: bool, - ) -> Result { - let full_path = full_path.as_ref(); - let location_path = location_path.as_ref(); - - let extension = (!is_dir) - .then(|| { - full_path - .extension() - .and_then(|ext| ext.to_str().map(str::to_string)) - .unwrap_or_default() - }) - .unwrap_or_default(); - - Ok(Self { - is_dir, - location_id, - materialized_path: Cow::Owned(extract_normalized_materialized_path_str( - location_id, - location_path, - full_path, - )?), - name: Cow::Owned( - (location_path != full_path) - .then(|| Self::prepare_name(full_path, is_dir).to_string()) - .unwrap_or_default(), - ), - extension: Cow::Owned(extension), - relative_path: Cow::Owned(extract_relative_path( - location_id, - location_path, - full_path, - )?), - }) - } -} - -impl<'a> IsolatedFilePathData<'a> { - #[must_use] - pub const fn location_id(&self) -> location::id::Type { - self.location_id - } - - #[must_use] - pub fn extension(&self) -> &str { - self.extension.as_ref() - } - - #[must_use] - pub fn to_owned(self) -> IsolatedFilePathData<'static> { - IsolatedFilePathData { - location_id: self.location_id, - materialized_path: Cow::Owned(self.materialized_path.to_string()), - is_dir: self.is_dir, - name: Cow::Owned(self.name.to_string()), - extension: Cow::Owned(self.extension.to_string()), - relative_path: Cow::Owned(self.relative_path.to_string()), - } - } - - #[must_use] - pub const fn is_dir(&self) -> bool { - self.is_dir - } - - #[must_use] - pub fn is_root(&self) -> bool { - self.is_dir - && self.materialized_path == "/" - && self.name.is_empty() - && self.relative_path.is_empty() - } - - #[must_use] - pub fn to_parts(&self) -> IsolatedFilePathDataParts<'_> { - IsolatedFilePathDataParts { - location_id: self.location_id, - materialized_path: &self.materialized_path, - is_dir: self.is_dir, - name: &self.name, - extension: &self.extension, - relative_path: &self.relative_path, - } - } - - /// Return the `IsolatedFilePath` for the parent of the current file or directory. - /// - /// # Panics - /// May panic if the materialized path was malformed, without a slash for the parent directory. - /// Considering that the parent can be just `/` for the root directory. - #[must_use] - pub fn parent(&'a self) -> Self { - let (parent_path_str, name, relative_path) = if self.materialized_path == "/" { - ("/", "", "") - } else { - let trailing_slash_idx = self.materialized_path.len() - 1; - let last_slash_idx = self.materialized_path[..trailing_slash_idx] - .rfind('/') - .expect("malformed materialized path at `parent` method"); - - ( - &self.materialized_path[..=last_slash_idx], - &self.materialized_path[last_slash_idx + 1..trailing_slash_idx], - &self.materialized_path[1..trailing_slash_idx], - ) - }; - - Self { - is_dir: true, - location_id: self.location_id, - relative_path: Cow::Borrowed(relative_path), - materialized_path: Cow::Borrowed(parent_path_str), - name: Cow::Borrowed(name), - extension: Cow::Borrowed(""), - } - } - - pub fn from_relative_str( - location_id: location::id::Type, - relative_file_path_str: &'a str, - ) -> Self { - let is_dir = relative_file_path_str.ends_with('/'); - - let (materialized_path, maybe_name, maybe_extension) = - Self::separate_path_name_and_extension_from_str(relative_file_path_str, is_dir); - - Self { - location_id, - materialized_path: Cow::Borrowed(materialized_path), - is_dir, - name: maybe_name.map(Cow::Borrowed).unwrap_or_default(), - extension: maybe_extension.map(Cow::Borrowed).unwrap_or_default(), - relative_path: Cow::Borrowed(relative_file_path_str), - } - } - - #[must_use] - pub fn full_name(&self) -> String { - if self.extension.is_empty() { - self.name.to_string() - } else { - format!("{}.{}", self.name, self.extension) - } - } - - #[must_use] - pub fn materialized_path_for_children(&self) -> Option { - if self.materialized_path == "/" && self.name.is_empty() && self.is_dir { - // We're at the root file_path - Some("/".to_string()) - } else { - self.is_dir - .then(|| format!("{}{}/", self.materialized_path, self.name)) - } - } - - pub fn separate_name_and_extension_from_str( - source: &'a str, - ) -> Result<(&'a str, &'a str), FilePathError> { - if source.contains(MAIN_SEPARATOR) { - return Err(FilePathError::InvalidFilenameAndExtension( - source.to_string(), - )); - } - - source.rfind('.').map_or_else( - || Ok((source, "")), // It's a file without extension - |last_dot_idx| { - if last_dot_idx == 0 { - // The dot is the first character, so it's a hidden file - Ok((source, "")) - } else { - Ok((&source[..last_dot_idx], &source[last_dot_idx + 1..])) - } - }, - ) - } - - #[allow(clippy::missing_panics_doc)] // Don't actually panic as the regexes are hardcoded - #[must_use] - pub fn accept_file_name(name: &str) -> bool { - let reg = { - // Maybe we should enforce windows more restrictive rules on all platforms? - #[cfg(target_os = "windows")] - { - FORBIDDEN_FILE_NAMES.get_or_init(|| { - RegexSet::new([ - r"(?i)^(CON|PRN|AUX|NUL|COM[1-9]|LPT[1-9])(\.\w+)*$", - r#"[<>:"/\\|?*\u0000-\u0031]"#, - ]) - .expect("this regex should always be valid") - }) - } - - #[cfg(not(target_os = "windows"))] - { - FORBIDDEN_FILE_NAMES.get_or_init(|| { - RegexSet::new([r"/|\x00"]).expect("this regex should always be valid") - }) - } - }; - - !reg.is_match(name) - } - - #[must_use] - pub fn separate_path_name_and_extension_from_str( - source: &'a str, - is_dir: bool, - ) -> ( - &'a str, // Materialized path - Option<&'a str>, // Maybe a name - Option<&'a str>, // Maybe an extension - ) { - let length = source.len(); - - if length == 1 { - // The case for the root path - (source, None, None) - } else if is_dir { - let last_char_idx = if source.ends_with('/') { - length - 1 - } else { - length - }; - - let first_name_char_idx = source[..last_char_idx].rfind('/').unwrap_or(0) + 1; - ( - &source[..first_name_char_idx], - Some(&source[first_name_char_idx..last_char_idx]), - None, - ) - } else { - let first_name_char_idx = source.rfind('/').unwrap_or(0) + 1; - let end_idx = first_name_char_idx - 1; - source[first_name_char_idx..].rfind('.').map_or_else( - || { - ( - &source[..end_idx], - Some(&source[first_name_char_idx..]), - None, - ) - }, - |last_dot_relative_idx| { - let last_dot_idx = first_name_char_idx + last_dot_relative_idx; - ( - &source[..end_idx], - Some(&source[first_name_char_idx..last_dot_idx]), - Some(&source[last_dot_idx + 1..]), - ) - }, - ) - } - } - - fn prepare_name(path: &Path, is_dir: bool) -> &str { - // Not using `impl AsRef` here because it's an private method - if is_dir { - path.file_name() - } else { - path.file_stem() - } - .unwrap_or_default() - .to_str() - .unwrap_or_default() - } - - #[must_use] - pub fn from_db_data( - location_id: location::id::Type, - is_dir: bool, - materialized_path: Cow<'a, str>, - name: Cow<'a, str>, - extension: Cow<'a, str>, - ) -> Self { - Self { - relative_path: Cow::Owned(assemble_relative_path( - &materialized_path, - &name, - &extension, - is_dir, - )), - location_id, - materialized_path, - is_dir, - name, - extension, - } - } -} - -impl AsRef for IsolatedFilePathData<'_> { - fn as_ref(&self) -> &Path { - Path::new(self.relative_path.as_ref()) - } -} - -impl From> for file_path::UniqueWhereParam { - fn from(path: IsolatedFilePathData<'static>) -> Self { - Self::LocationIdMaterializedPathNameExtensionEquals( - path.location_id, - path.materialized_path.into_owned(), - path.name.into_owned(), - path.extension.into_owned(), - ) - } -} - -impl From> for file_path::WhereParam { - fn from(path: IsolatedFilePathData<'static>) -> Self { - Self::And(vec![ - file_path::location_id::equals(Some(path.location_id)), - file_path::materialized_path::equals(Some(path.materialized_path.into_owned())), - file_path::name::equals(Some(path.name.into_owned())), - file_path::extension::equals(Some(path.extension.into_owned())), - ]) - } -} - -impl From<&IsolatedFilePathData<'_>> for file_path::UniqueWhereParam { - fn from(path: &IsolatedFilePathData<'_>) -> Self { - Self::LocationIdMaterializedPathNameExtensionEquals( - path.location_id, - path.materialized_path.to_string(), - path.name.to_string(), - path.extension.to_string(), - ) - } -} - -impl From<&IsolatedFilePathData<'_>> for file_path::WhereParam { - fn from(path: &IsolatedFilePathData<'_>) -> Self { - Self::And(vec![ - file_path::location_id::equals(Some(path.location_id)), - file_path::materialized_path::equals(Some(path.materialized_path.to_string())), - file_path::name::equals(Some(path.name.to_string())), - file_path::extension::equals(Some(path.extension.to_string())), - ]) - } -} - -impl fmt::Display for IsolatedFilePathData<'_> { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{}", self.relative_path) - } -} - -impl fmt::Display for IsolatedFilePathDataParts<'_> { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{}", self.relative_path) - } -} - -#[macro_use] -mod macros { - macro_rules! impl_from_db { - ($($file_path_kind:ident),+ $(,)?) => { - $( - impl ::std::convert::TryFrom<$file_path_kind::Data> for $crate:: - isolated_file_path_data:: - IsolatedFilePathData<'static> - { - type Error = ::sd_utils::db::MissingFieldError; - - fn try_from(path: $file_path_kind::Data) -> Result { - use ::sd_utils::db::maybe_missing; - use ::std::borrow::Cow; - - Ok(Self::from_db_data( - maybe_missing(path.location_id, "file_path.location_id")?, - maybe_missing(path.is_dir, "file_path.is_dir")?, - Cow::Owned(maybe_missing(path.materialized_path, "file_path.materialized_path")?), - Cow::Owned(maybe_missing(path.name, "file_path.name")?), - Cow::Owned(maybe_missing(path.extension, "file_path.extension")?) - )) - } - } - - impl<'a> ::std::convert::TryFrom<&'a $file_path_kind::Data> for $crate:: - isolated_file_path_data:: - IsolatedFilePathData<'a> - { - type Error =::sd_utils::db::MissingFieldError; - - fn try_from(path: &'a $file_path_kind::Data) -> Result { - use ::sd_utils::db::maybe_missing; - use ::std::borrow::Cow; - - Ok(Self::from_db_data( - maybe_missing(path.location_id, "file_path.location_id")?, - maybe_missing(path.is_dir, "file_path.is_dir")?, - Cow::Borrowed(maybe_missing(&path.materialized_path, "file_path.materialized_path")?), - Cow::Borrowed(maybe_missing(&path.name, "file_path.name")?), - Cow::Borrowed(maybe_missing(&path.extension, "file_path.extension")?) - )) - } - } - )+ - }; - } - - macro_rules! impl_from_db_without_location_id { - ($($file_path_kind:ident),+ $(,)?) => { - $( - impl ::std::convert::TryFrom<(::sd_prisma::prisma::location::id::Type, $file_path_kind::Data)> for $crate:: - isolated_file_path_data:: - IsolatedFilePathData<'static> - { - type Error = ::sd_utils::db::MissingFieldError; - - fn try_from((location_id, path): (::sd_prisma::prisma::location::id::Type, $file_path_kind::Data)) -> Result { - use ::sd_utils::db::maybe_missing; - use ::std::borrow::Cow; - - Ok(Self::from_db_data( - location_id, - maybe_missing(path.is_dir, "file_path.is_dir")?, - Cow::Owned(maybe_missing(path.materialized_path, "file_path.materialized_path")?), - Cow::Owned(maybe_missing(path.name, "file_path.name")?), - Cow::Owned(maybe_missing(path.extension, "file_path.extension")?) - )) - } - } - - impl<'a> ::std::convert::TryFrom<(::sd_prisma::prisma::location::id::Type, &'a $file_path_kind::Data)> for $crate:: - isolated_file_path_data:: - IsolatedFilePathData<'a> - { - type Error = ::sd_utils::db::MissingFieldError; - - fn try_from((location_id, path): (::sd_prisma::prisma::location::id::Type, &'a $file_path_kind::Data)) -> Result { - use ::sd_utils::db::maybe_missing; - use ::std::borrow::Cow; - - Ok(Self::from_db_data( - location_id, - maybe_missing(path.is_dir, "file_path.is_dir")?, - Cow::Borrowed(maybe_missing(&path.materialized_path, "file_path.materialized_path")?), - Cow::Borrowed(maybe_missing(&path.name, "file_path.name")?), - Cow::Borrowed(maybe_missing(&path.extension, "file_path.extension")?) - )) - } - } - )+ - }; - } -} - -impl_from_db!( - file_path, - file_path_to_isolate, - file_path_to_isolate_with_pub_id, - file_path_walker, - file_path_to_isolate_with_id, - file_path_with_object, - file_path_watcher_remove -); - -impl_from_db_without_location_id!( - file_path_for_file_identifier, - file_path_to_full_path, - file_path_for_media_processor, - file_path_for_object_validator, - file_path_to_handle_custom_uri, - file_path_to_handle_p2p_serve_file -); - -fn extract_relative_path( - location_id: location::id::Type, - location_path: impl AsRef, - path: impl AsRef, -) -> Result { - let path = path.as_ref(); - - path.strip_prefix(location_path) - .map_err(|_| FilePathError::UnableToExtractMaterializedPath { - location_id, - path: path.into(), - }) - .and_then(|relative| { - relative - .to_str() - .map(|relative_str| relative_str.replace('\\', "/")) - .ok_or_else(|| NonUtf8PathError(path.into()).into()) - }) -} - -/// This function separates a file path from a location path, and normalizes replacing '\' with '/' -/// to be consistent between Windows and Unix like systems -pub fn extract_normalized_materialized_path_str( - location_id: location::id::Type, - location_path: impl AsRef, - path: impl AsRef, -) -> Result { - let path = path.as_ref(); - - path.strip_prefix(location_path) - .map_err(|_| FilePathError::UnableToExtractMaterializedPath { - location_id, - path: path.into(), - })? - .parent() - .map_or_else( - || Ok("/".to_string()), - |materialized_path| { - materialized_path - .to_str() - .map(|materialized_path_str| { - if materialized_path_str.is_empty() { - "/".to_string() - } else { - format!("/{}/", materialized_path_str.replace('\\', "/")) - } - }) - .ok_or_else(|| NonUtf8PathError(path.into())) - }, - ) - .map_err(Into::into) -} - -fn assemble_relative_path( - materialized_path: &str, - name: &str, - extension: &str, - is_dir: bool, -) -> String { - if !is_dir && !extension.is_empty() { - format!("{}{}.{}", &materialized_path[1..], name, extension) - } else { - format!("{}{}", &materialized_path[1..], name) - } -} - -pub fn join_location_relative_path( - location_path: impl AsRef, - relative_path: impl AsRef, -) -> PathBuf { - let relative_path = relative_path.as_ref(); - let relative_path = relative_path - .strip_prefix(MAIN_SEPARATOR_STR) - .unwrap_or(relative_path); - - location_path.as_ref().join(relative_path) -} - -pub fn push_location_relative_path( - mut location_path: PathBuf, - relative_path: impl AsRef, -) -> PathBuf { - let relative_path = relative_path.as_ref(); - - let relative_path = relative_path - .strip_prefix(MAIN_SEPARATOR_STR) - .unwrap_or(relative_path); - location_path.push(relative_path); - - location_path -} - -#[cfg(test)] -mod tests { - use super::*; - - fn expected( - materialized_path: &'static str, - is_dir: bool, - name: &'static str, - extension: &'static str, - relative_path: &'static str, - ) -> IsolatedFilePathData<'static> { - IsolatedFilePathData { - location_id: 1, - materialized_path: materialized_path.into(), - is_dir, - name: name.into(), - extension: extension.into(), - relative_path: relative_path.into(), - } - } - - #[test] - fn new_method() { - let tester = |full_path, is_dir, expected, msg| { - let actual = - IsolatedFilePathData::new(1, "/spacedrive/location", full_path, is_dir).unwrap(); - assert_eq!(actual, expected, "{msg}"); - }; - - tester( - "/spacedrive/location", - true, - expected("/", true, "", "", ""), - "the location root directory", - ); - - tester( - "/spacedrive/location/file.txt", - false, - expected("/", false, "file", "txt", "file.txt"), - "a file in the root directory", - ); - - tester( - "/spacedrive/location/dir", - true, - expected("/", true, "dir", "", "dir"), - "a directory in the root directory", - ); - - tester( - "/spacedrive/location/dir/file.txt", - false, - expected("/dir/", false, "file", "txt", "dir/file.txt"), - "a directory with a file inside", - ); - - tester( - "/spacedrive/location/dir/dir2", - true, - expected("/dir/", true, "dir2", "", "dir/dir2"), - "a directory in a directory", - ); - - tester( - "/spacedrive/location/dir/dir2/dir3", - true, - expected("/dir/dir2/", true, "dir3", "", "dir/dir2/dir3"), - "3 level of directories", - ); - - tester( - "/spacedrive/location/dir/dir2/dir3/file.txt", - false, - expected( - "/dir/dir2/dir3/", - false, - "file", - "txt", - "dir/dir2/dir3/file.txt", - ), - "a file inside a third level directory", - ); - } - - #[test] - fn parent_method() { - let tester = |full_path, is_dir, expected, msg| { - let child = - IsolatedFilePathData::new(1, "/spacedrive/location", full_path, is_dir).unwrap(); - - let actual = child.parent(); - assert_eq!(actual, expected, "{msg}"); - }; - - tester( - "/spacedrive/location", - true, - expected("/", true, "", "", ""), - "the location root directory", - ); - - tester( - "/spacedrive/location/file.txt", - false, - expected("/", true, "", "", ""), - "a file in the root directory", - ); - - tester( - "/spacedrive/location/dir", - true, - expected("/", true, "", "", ""), - "a directory in the root directory", - ); - - tester( - "/spacedrive/location/dir/file.txt", - false, - expected("/", true, "dir", "", "dir"), - "a directory with a file inside", - ); - - tester( - "/spacedrive/location/dir/dir2", - true, - expected("/", true, "dir", "", "dir"), - "a directory in a directory", - ); - - tester( - "/spacedrive/location/dir/dir2/dir3", - true, - expected("/dir/", true, "dir2", "", "dir/dir2"), - "3 level of directories", - ); - - tester( - "/spacedrive/location/dir/dir2/dir3/file.txt", - false, - expected("/dir/dir2/", true, "dir3", "", "dir/dir2/dir3"), - "a file inside a third level directory", - ); - } - - #[test] - fn extract_normalized_materialized_path() { - let tester = |path, expected, msg| { - let actual = - extract_normalized_materialized_path_str(1, "/spacedrive/location", path).unwrap(); - assert_eq!(actual, expected, "{msg}"); - }; - - tester("/spacedrive/location", "/", "the location root directory"); - tester( - "/spacedrive/location/file.txt", - "/", - "a file in the root directory", - ); - tester( - "/spacedrive/location/dir", - "/", - "a directory in the root directory", - ); - tester( - "/spacedrive/location/dir/file.txt", - "/dir/", - "a directory with a file inside", - ); - tester( - "/spacedrive/location/dir/dir2", - "/dir/", - "a directory in a directory", - ); - tester( - "/spacedrive/location/dir/dir2/dir3", - "/dir/dir2/", - "3 level of directories", - ); - tester( - "/spacedrive/location/dir/dir2/dir3/file.txt", - "/dir/dir2/dir3/", - "a file inside a third level directory", - ); - } -} diff --git a/core/crates/file-path-helper/src/lib.rs b/core/crates/file-path-helper/src/lib.rs deleted file mode 100644 index d582cb16e..000000000 --- a/core/crates/file-path-helper/src/lib.rs +++ /dev/null @@ -1,438 +0,0 @@ -#![warn( - clippy::all, - clippy::pedantic, - clippy::correctness, - clippy::perf, - clippy::style, - clippy::suspicious, - clippy::complexity, - clippy::nursery, - clippy::unwrap_used, - unused_qualifications, - rust_2018_idioms, - trivial_casts, - trivial_numeric_casts, - unused_allocation, - clippy::unnecessary_cast, - clippy::cast_lossless, - clippy::cast_possible_truncation, - clippy::cast_possible_wrap, - clippy::cast_precision_loss, - clippy::cast_sign_loss, - clippy::dbg_macro, - clippy::deprecated_cfg_attr, - clippy::separated_literal_suffix, - deprecated -)] -#![forbid(deprecated_in_future)] -#![allow(clippy::missing_errors_doc, clippy::module_name_repetitions)] - -use sd_prisma::prisma::{file_path, location, PrismaClient}; -use sd_utils::error::{FileIOError, NonUtf8PathError}; - -use std::{ - fs::Metadata, - path::{Path, PathBuf, MAIN_SEPARATOR_STR}, - time::SystemTime, -}; - -use chrono::{DateTime, Utc}; -use prisma_client_rust::QueryError; -use serde::{Deserialize, Serialize}; -use thiserror::Error; -use tokio::{fs, io}; -use tracing::error; - -pub mod isolated_file_path_data; - -pub use isolated_file_path_data::{ - join_location_relative_path, push_location_relative_path, IsolatedFilePathData, - IsolatedFilePathDataParts, -}; - -#[derive(Clone, Copy, Debug, Serialize, Deserialize)] -pub struct FilePathMetadata { - pub inode: u64, - pub size_in_bytes: u64, - pub created_at: DateTime, - pub modified_at: DateTime, - pub hidden: bool, -} - -pub fn path_is_hidden(path: impl AsRef, metadata: &Metadata) -> bool { - #[cfg(target_family = "unix")] - { - use std::ffi::OsStr; - let _ = metadata; // just to avoid warnings on Linux - if path - .as_ref() - .file_name() - .and_then(OsStr::to_str) - .is_some_and(|s| s.starts_with('.')) - { - return true; - } - } - - #[cfg(target_os = "macos")] - { - use std::os::macos::fs::MetadataExt; - - // https://developer.apple.com/library/archive/documentation/FileManagement/Conceptual/FileSystemProgrammingGuide/FileSystemDetails/FileSystemDetails.html#:~:text=UF_HIDDEN - const UF_HIDDEN: u32 = 0x8000; - - if (metadata.st_flags() & UF_HIDDEN) == UF_HIDDEN { - return true; - } - } - - #[cfg(target_family = "windows")] - { - use std::os::windows::fs::MetadataExt; - - const FILE_ATTRIBUTE_HIDDEN: u32 = 0x2; - - let _ = path; // just to avoid warnings on Windows - - if (metadata.file_attributes() & FILE_ATTRIBUTE_HIDDEN) == FILE_ATTRIBUTE_HIDDEN { - return true; - } - } - - false -} - -#[cfg(target_family = "windows")] -// TODO(matheus-consoli): rewrite this function using a safe API once `MetadataExt::file_index` got stabilized -// see: `https://doc.rust-lang.org/std/os/windows/fs/trait.MetadataExt.html#tymethod.file_index` -fn get_inode_windows>(path: P) -> Result { - use std::ptr::null_mut; - use windows::{ - core::HSTRING, - Win32::Foundation::{CloseHandle, HANDLE}, - Win32::Storage::FileSystem::{ - CreateFileW, GetFileInformationByHandle, BY_HANDLE_FILE_INFORMATION, - FILE_ATTRIBUTE_NORMAL, FILE_FLAG_BACKUP_SEMANTICS, FILE_SHARE_READ, OPEN_EXISTING, - }, - }; - - let handle = unsafe { - CreateFileW( - &HSTRING::from(path.as_ref()), - 0, - FILE_SHARE_READ, - None, - OPEN_EXISTING, - FILE_ATTRIBUTE_NORMAL | FILE_FLAG_BACKUP_SEMANTICS, - HANDLE(null_mut()), - ) - }?; - - let mut file_info = BY_HANDLE_FILE_INFORMATION::default(); - let res = unsafe { GetFileInformationByHandle(handle, &mut file_info) }; - _ = unsafe { CloseHandle(handle) }; - res?; - - Ok(file_info.nFileIndexLow as u64 | ((file_info.nFileIndexHigh as u64) << 32)) -} - -impl FilePathMetadata { - pub fn from_path( - path: impl AsRef + Copy, - metadata: &Metadata, - ) -> Result { - let inode = { - #[cfg(target_family = "unix")] - { - get_inode(metadata) - } - - #[cfg(target_family = "windows")] - { - tokio::task::block_in_place(|| { - get_inode_windows(path.as_ref()).map_err(|e| FileIOError::from((path, e))) - })? - } - }; - - Ok(Self { - inode, - hidden: path_is_hidden(path, metadata), - size_in_bytes: metadata.len(), - created_at: metadata.created_or_now().into(), - modified_at: metadata.modified_or_now().into(), - }) - } -} - -#[derive(Error, Debug)] -pub enum FilePathError { - #[error("file path not found: ")] - IdNotFound(file_path::id::Type), - #[error("file Path not found: ", .0.display())] - NotFound(Box), - #[error("location '{0}' not found")] - LocationNotFound(location::id::Type), - #[error("received an invalid sub path: ", .location_path.display(), .sub_path.display())] - InvalidSubPath { - location_path: Box, - sub_path: Box, - }, - #[error("sub path is not a directory: ", .0.display())] - SubPathNotDirectory(Box), - #[error( - "the parent directory of the received sub path isn't indexed in the location: ", - .location_id, - .sub_path.display() - )] - SubPathParentNotInLocation { - location_id: location::id::Type, - sub_path: Box, - }, - #[error("unable to extract materialized path from location: ", .location_id, .path.display())] - UnableToExtractMaterializedPath { - location_id: location::id::Type, - path: Box, - }, - #[error("database error: {0}")] - Database(#[from] QueryError), - - #[error(transparent)] - FileIO(#[from] FileIOError), - #[error(transparent)] - NonUtf8Path(#[from] NonUtf8PathError), - #[error("received an invalid filename and extension: ")] - InvalidFilenameAndExtension(String), - #[error(transparent)] - Sync(#[from] sd_core_sync::Error), -} - -#[must_use] -pub fn filter_existing_file_path_params( - IsolatedFilePathData { - materialized_path, - is_dir, - location_id, - name, - extension, - .. - }: &IsolatedFilePathData<'_>, -) -> Vec { - vec![ - file_path::location_id::equals(Some(*location_id)), - file_path::materialized_path::equals(Some(materialized_path.to_string())), - file_path::is_dir::equals(Some(*is_dir)), - file_path::name::equals(Some(name.to_string())), - file_path::extension::equals(Some(extension.to_string())), - ] -} - -/// With this function we try to do a loose filtering of file paths, to avoid having to do check -/// twice for directories and for files. This is because directories have a trailing `/` or `\` in -/// the materialized path -pub fn loose_find_existing_file_path_params( - location_id: location::id::Type, - location_path: impl AsRef, - full_path: impl AsRef, -) -> Result, FilePathError> { - let location_path = location_path.as_ref(); - let full_path = full_path.as_ref(); - - let file_iso_file_path = - IsolatedFilePathData::new(location_id, location_path, full_path, false)?; - - let dir_iso_file_path = IsolatedFilePathData::new(location_id, location_path, full_path, true)?; - - Ok(vec![ - file_path::location_id::equals(Some(location_id)), - file_path::materialized_path::equals(Some( - file_iso_file_path.materialized_path.to_string(), - )), - file_path::name::in_vec(vec![ - file_iso_file_path.name.to_string(), - dir_iso_file_path.name.to_string(), - ]), - file_path::extension::in_vec(vec![ - file_iso_file_path.extension.to_string(), - dir_iso_file_path.extension.to_string(), - ]), - ]) -} - -#[allow(clippy::missing_panics_doc)] // Don't actually panic -pub async fn ensure_sub_path_is_in_location( - location_path: impl AsRef + Send, - sub_path: impl AsRef + Send, -) -> Result { - let mut sub_path = sub_path.as_ref(); - let location_path = location_path.as_ref(); - if sub_path.starts_with(MAIN_SEPARATOR_STR) { - if sub_path == Path::new(MAIN_SEPARATOR_STR) { - // We're dealing with the location root path here - return Ok(location_path.to_path_buf()); - } - // SAFETY: we just checked that it starts with the separator - sub_path = sub_path - .strip_prefix(MAIN_SEPARATOR_STR) - .expect("we just checked that it starts with the separator"); - } - - if sub_path.starts_with(location_path) { - Ok(sub_path.to_path_buf()) - } else { - // If the sub_path doesn't start with the location_path, we have to check if it's a - // materialized path received from the frontend, then we check if the full path exists - let full_path = location_path.join(sub_path); - - match fs::metadata(&full_path).await { - Ok(_) => Ok(full_path), - Err(e) if e.kind() == io::ErrorKind::NotFound => Err(FilePathError::InvalidSubPath { - sub_path: sub_path.into(), - location_path: location_path.into(), - }), - Err(e) => Err(FileIOError::from((full_path, e)).into()), - } - } -} - -pub async fn ensure_file_path_exists( - sub_path: impl AsRef + Send, - iso_file_path: &IsolatedFilePathData<'_>, - db: &PrismaClient, - error_fn: impl FnOnce(Box) -> E + Send, -) -> Result<(), E> -where - E: From, -{ - if check_file_path_exists(iso_file_path, db).await? { - Ok(()) - } else { - Err(error_fn(sub_path.as_ref().into())) - } -} - -pub async fn check_file_path_exists( - iso_file_path: &IsolatedFilePathData<'_>, - db: &PrismaClient, -) -> Result -where - E: From, -{ - Ok(iso_file_path.is_root() - || db - .file_path() - .count(filter_existing_file_path_params(iso_file_path)) - .exec() - .await? > 0) -} - -#[allow(clippy::missing_panics_doc)] // Don't actually panic -pub async fn ensure_sub_path_is_directory( - location_path: impl AsRef + Send, - sub_path: impl AsRef + Send, -) -> Result<(), FilePathError> { - let mut sub_path = sub_path.as_ref(); - - if sub_path == Path::new(MAIN_SEPARATOR_STR) { - // Sub path for the location root path is always a directory - return Ok(()); - } - - match fs::metadata(sub_path).await { - Ok(meta) => { - if meta.is_file() { - Err(FilePathError::SubPathNotDirectory(sub_path.into())) - } else { - Ok(()) - } - } - Err(e) if e.kind() == io::ErrorKind::NotFound => { - if sub_path.starts_with("/") { - // SAFETY: we just checked that it starts with the separator - sub_path = sub_path - .strip_prefix("/") - .expect("we just checked that it starts with the separator"); - } - - let location_path = location_path.as_ref(); - let full_path = location_path.join(sub_path); - match fs::metadata(&full_path).await { - Ok(meta) => { - if meta.is_file() { - Err(FilePathError::SubPathNotDirectory(sub_path.into())) - } else { - Ok(()) - } - } - Err(e) if e.kind() == io::ErrorKind::NotFound => { - Err(FilePathError::InvalidSubPath { - sub_path: sub_path.into(), - location_path: location_path.into(), - }) - } - Err(e) => Err(FileIOError::from((full_path, e)).into()), - } - } - Err(e) => Err(FileIOError::from((sub_path, e)).into()), - } -} - -#[must_use] -pub fn get_inode(metadata: &Metadata) -> u64 { - #[cfg(target_family = "unix")] - { - use std::os::unix::fs::MetadataExt; - - metadata.ino() - } - - #[cfg(target_family = "windows")] - { - // TODO use this when it's stable and remove winapi-utils dependency - - // use std::os::windows::fs::MetadataExt; - - // - // metadata - // .file_index() - // .expect("This function must not be called from a `DirEntry`'s `Metadata") - // - - todo!("Use metadata: {:#?}", metadata) - } -} - -pub async fn get_inode_from_path(path: impl AsRef + Send) -> Result { - #[cfg(target_family = "unix")] - { - // TODO use this when it's stable and remove winapi-utils dependency - let metadata = fs::metadata(path.as_ref()) - .await - .map_err(|e| FileIOError::from((path, e)))?; - - Ok(get_inode(&metadata)) - } - - #[cfg(target_family = "windows")] - { - Ok(tokio::task::block_in_place(|| { - get_inode_windows(path.as_ref()).map_err(|e| FileIOError::from((path, e))) - })?) - } -} - -pub trait MetadataExt { - fn created_or_now(&self) -> SystemTime; - - fn modified_or_now(&self) -> SystemTime; -} - -impl MetadataExt for Metadata { - fn created_or_now(&self) -> SystemTime { - self.created().unwrap_or_else(|_| SystemTime::now()) - } - - fn modified_or_now(&self) -> SystemTime { - self.modified().unwrap_or_else(|_| SystemTime::now()) - } -} diff --git a/core/crates/heavy-lifting/Cargo.toml b/core/crates/heavy-lifting/Cargo.toml deleted file mode 100644 index 20743e9fa..000000000 --- a/core/crates/heavy-lifting/Cargo.toml +++ /dev/null @@ -1,63 +0,0 @@ -[package] -name = "sd-core-heavy-lifting" -version = "0.1.0" - -authors = ["Ericson Soares "] -edition.workspace = true -license.workspace = true -repository.workspace = true - -[features] -default = [] -# This feature controls whether the Spacedrive Heavy Lifting contains functionality which requires FFmpeg. -ffmpeg = ["dep:sd-ffmpeg"] - -[dependencies] -# Inner Core Sub-crates -sd-core-file-path-helper = { path = "../file-path-helper" } -sd-core-indexer-rules = { path = "../indexer-rules" } -sd-core-prisma-helpers = { path = "../prisma-helpers" } -sd-core-sync = { path = "../sync" } - -# Spacedrive Sub-crates -sd-ffmpeg = { path = "../../../crates/ffmpeg", optional = true } -sd-file-ext = { path = "../../../crates/file-ext" } -sd-images = { path = "../../../crates/images" } -sd-media-metadata = { path = "../../../crates/media-metadata" } -sd-prisma = { path = "../../../crates/prisma" } -sd-sync = { path = "../../../crates/sync" } -sd-task-system = { path = "../../../crates/task-system" } -sd-utils = { path = "../../../crates/utils" } - -# Workspace dependencies -async-channel = { workspace = true } -async-trait = { workspace = true } -blake3 = { workspace = true } -chrono = { workspace = true, features = ["serde"] } -futures = { workspace = true } -futures-concurrency = { workspace = true } -globset = { workspace = true } -image = { workspace = true } -itertools = { workspace = true } -lending-stream = { workspace = true } -prisma-client-rust = { workspace = true } -rmp-serde = { workspace = true } -rmpv = { workspace = true } -rspc = { workspace = true } -serde = { workspace = true, features = ["derive", "rc"] } -serde_json = { workspace = true } -specta = { workspace = true } -strum = { workspace = true, features = ["derive", "phf"] } -thiserror = { workspace = true } -tokio = { workspace = true, features = ["fs", "parking_lot", "sync"] } -tokio-stream = { workspace = true, features = ["fs"] } -tracing = { workspace = true } -uuid = { workspace = true, features = ["serde", "v4"] } -webp = { workspace = true } - -# Specific Heavy Lifting dependencies -static_assertions = "1.1" - -[dev-dependencies] -tempfile = { workspace = true } -tracing-test = { workspace = true } diff --git a/core/crates/heavy-lifting/src/file_identifier/cas_id.rs b/core/crates/heavy-lifting/src/file_identifier/cas_id.rs deleted file mode 100644 index 7240c3483..000000000 --- a/core/crates/heavy-lifting/src/file_identifier/cas_id.rs +++ /dev/null @@ -1,80 +0,0 @@ -use sd_core_prisma_helpers::CasId; - -use std::path::Path; - -use blake3::Hasher; -use static_assertions::const_assert; -use tokio::{ - fs::{self, File}, - io::{self, AsyncReadExt, AsyncSeekExt, SeekFrom}, -}; -use tracing::{instrument, trace, Level}; - -const SAMPLE_COUNT: u64 = 4; -const SAMPLE_SIZE: u64 = 1024 * 10; -const HEADER_OR_FOOTER_SIZE: u64 = 1024 * 8; - -// minimum file size of 100KiB, to avoid sample hashing for small files as they can be smaller than the total sample size -const MINIMUM_FILE_SIZE: u64 = 1024 * 100; - -// Asserting that nobody messed up our consts -const_assert!((HEADER_OR_FOOTER_SIZE * 2 + SAMPLE_COUNT * SAMPLE_SIZE) < MINIMUM_FILE_SIZE); - -// Asserting that the sample size is larger than header/footer size, as the same buffer is used for both -const_assert!(SAMPLE_SIZE > HEADER_OR_FOOTER_SIZE); - -#[instrument( - skip(path), - ret(level = Level::TRACE), - err, - fields(path = %path.as_ref().display() -))] -// SAFETY: Casts here are safe, they're hardcoded values we have some const assertions above to make sure they're correct -#[allow(clippy::cast_possible_truncation)] -#[allow(clippy::cast_possible_wrap)] -pub async fn generate_cas_id( - path: impl AsRef + Send, - size: u64, -) -> Result, io::Error> { - let mut hasher = Hasher::new(); - hasher.update(&size.to_le_bytes()); - - if size <= MINIMUM_FILE_SIZE { - trace!("File is small, hashing the whole file"); - // For small files, we hash the whole file - hasher.update(&fs::read(path).await?); - } else { - trace!("File bigger than threshold, hashing samples"); - - let mut file = File::open(path).await?; - let mut buf = vec![0; SAMPLE_SIZE as usize].into_boxed_slice(); - - // Hashing the header - let mut current_pos = file - .read_exact(&mut buf[..HEADER_OR_FOOTER_SIZE as usize]) - .await? as u64; - hasher.update(&buf[..HEADER_OR_FOOTER_SIZE as usize]); - - // Sample hashing the inner content of the file - let seek_jump = (size - HEADER_OR_FOOTER_SIZE * 2) / SAMPLE_COUNT; - loop { - file.read_exact(&mut buf).await?; - hasher.update(&buf); - - if current_pos >= (HEADER_OR_FOOTER_SIZE + seek_jump * (SAMPLE_COUNT - 1)) { - break; - } - - current_pos = file.seek(SeekFrom::Start(current_pos + seek_jump)).await?; - } - - // Hashing the footer - file.seek(SeekFrom::End(-(HEADER_OR_FOOTER_SIZE as i64))) - .await?; - file.read_exact(&mut buf[..HEADER_OR_FOOTER_SIZE as usize]) - .await?; - hasher.update(&buf[..HEADER_OR_FOOTER_SIZE as usize]); - } - - Ok(hasher.finalize().to_hex()[..16].to_string().into()) -} diff --git a/core/crates/heavy-lifting/src/file_identifier/job.rs b/core/crates/heavy-lifting/src/file_identifier/job.rs deleted file mode 100644 index dc2d6866c..000000000 --- a/core/crates/heavy-lifting/src/file_identifier/job.rs +++ /dev/null @@ -1,1072 +0,0 @@ -use crate::{ - file_identifier, - job_system::{ - job::{Job, JobReturn, JobTaskDispatcher, ReturnStatus}, - report::ReportOutputMetadata, - utils::cancel_pending_tasks, - DispatcherError, JobErrorOrDispatcherError, SerializableJob, SerializedTasks, - }, - utils::sub_path::maybe_get_iso_file_path_from_sub_path, - Error, JobContext, JobName, LocationScanState, NonCriticalError, OuterContext, ProgressUpdate, - UpdateEvent, -}; - -use sd_core_file_path_helper::IsolatedFilePathData; -use sd_core_prisma_helpers::{file_path_for_file_identifier, CasId}; - -use sd_prisma::{ - prisma::{device, file_path, location, SortOrder}, - prisma_sync, -}; -use sd_sync::{sync_db_not_null_entry, OperationFactory}; -use sd_task_system::{ - AnyTaskOutput, IntoTask, SerializableTask, Task, TaskDispatcher, TaskHandle, TaskId, - TaskOutput, TaskStatus, -}; -use sd_utils::{db::maybe_missing, u64_to_frontend}; - -use std::{ - collections::{HashMap, HashSet}, - fmt, - hash::{Hash, Hasher}, - mem, - path::PathBuf, - sync::Arc, - time::Duration, -}; - -use futures::{stream::FuturesUnordered, StreamExt}; -use futures_concurrency::future::TryJoin; -use serde::{Deserialize, Serialize}; -use serde_json::json; -use tokio::time::Instant; -use tracing::{debug, instrument, trace, warn, Level}; - -use super::{ - accumulate_file_paths_by_cas_id, dispatch_object_processor_tasks, orphan_path_filters_deep, - orphan_path_filters_shallow, - tasks::{self, identifier, object_processor, FilePathToCreateOrLinkObject}, - CHUNK_SIZE, -}; - -#[derive(Debug, Serialize, Deserialize, Clone, Copy)] -enum Phase { - SearchingOrphansWithPriority, - SearchingOrphans, - IdentifyingFiles, - ProcessingObjects, -} - -impl Default for Phase { - fn default() -> Self { - Self::SearchingOrphansWithPriority - } -} - -impl fmt::Display for Phase { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::SearchingOrphans | Self::SearchingOrphansWithPriority => { - write!(f, "searching_orphans") - } - Self::IdentifyingFiles => write!(f, "identifying_files"), - Self::ProcessingObjects => write!(f, "processing_objects"), - } - } -} - -impl From for String { - fn from(phase: Phase) -> Self { - phase.to_string() - } -} - -#[derive(Debug)] -pub struct FileIdentifier { - // Received arguments - location: Arc, - location_path: Arc, - sub_path: Option, - - // Inner state - file_paths_accumulator: HashMap, Vec>, - file_paths_ids_with_priority: HashSet, - last_orphan_file_path_id: Option, - - // Job control - phase: Phase, - - // Run data - metadata: Metadata, - errors: Vec, - - // On shutdown data - pending_tasks_on_resume: Vec>, - tasks_for_shutdown: Vec>>, -} - -impl Hash for FileIdentifier { - fn hash(&self, state: &mut H) { - self.location.id.hash(state); - if let Some(ref sub_path) = self.sub_path { - sub_path.hash(state); - } - } -} - -impl Job for FileIdentifier { - const NAME: JobName = JobName::FileIdentifier; - - async fn resume_tasks( - &mut self, - dispatcher: &JobTaskDispatcher, - ctx: &impl JobContext, - SerializedTasks(serialized_tasks): SerializedTasks, - ) -> Result<(), Error> { - if let Ok(tasks) = dispatcher - .dispatch_many_boxed( - rmp_serde::from_slice::)>>(&serialized_tasks) - .map_err(file_identifier::Error::from)? - .into_iter() - .map(|(task_kind, task_bytes)| async move { - match task_kind { - TaskKind::Identifier => tasks::Identifier::deserialize( - &task_bytes, - (Arc::clone(ctx.db()), ctx.sync().clone()), - ) - .await - .map(IntoTask::into_task), - - TaskKind::ObjectProcessor => tasks::ObjectProcessor::deserialize( - &task_bytes, - (Arc::clone(ctx.db()), ctx.sync().clone()), - ) - .await - .map(IntoTask::into_task), - } - }) - .collect::>() - .try_join() - .await - .map_err(file_identifier::Error::from)?, - ) - .await - { - self.pending_tasks_on_resume = tasks; - } else { - warn!("Failed to dispatch tasks to resume as job was already canceled"); - } - - Ok(()) - } - - #[instrument( - skip_all, - fields( - location_id = self.location.id, - location_path = %self.location_path.display(), - sub_path = ?self.sub_path.as_ref().map(|path| path.display()), - ), - ret(level = Level::TRACE), - err, - )] - async fn run( - mut self, - dispatcher: JobTaskDispatcher, - ctx: impl JobContext, - ) -> Result { - let mut pending_running_tasks = FuturesUnordered::new(); - - let device_pub_id = &ctx.sync().device_pub_id; - let device_id = ctx - .db() - .device() - .find_unique(device::pub_id::equals(device_pub_id.to_db())) - .exec() - .await - .map_err(file_identifier::Error::from)? - .ok_or(file_identifier::Error::DeviceNotFound( - device_pub_id.clone(), - ))? - .id; - - match self - .init_or_resume(&mut pending_running_tasks, &ctx, device_id, &dispatcher) - .await - { - Ok(()) => { /* Everything is awesome! */ } - Err(JobErrorOrDispatcherError::JobError(e)) => { - return Err(e.into()); - } - Err(JobErrorOrDispatcherError::Dispatcher(DispatcherError::JobCanceled(_))) => { - return Ok(self.cancel_job(&mut pending_running_tasks).await); - } - Err(JobErrorOrDispatcherError::Dispatcher(DispatcherError::Shutdown(tasks))) => { - self.tasks_for_shutdown.extend(tasks); - - if pending_running_tasks.is_empty() { - // If no task managed to be dispatched, we can just shutdown - // otherwise we have to process handles below and wait for them to be shutdown too - return Ok(ReturnStatus::Shutdown( - SerializableJob::::serialize(self).await, - )); - } - } - } - - while let Some(task) = pending_running_tasks.next().await { - match task { - Ok(TaskStatus::Done((task_id, TaskOutput::Out(out)))) => { - match self - .process_task_output(task_id, out, &ctx, device_id, &dispatcher) - .await - { - Ok(tasks) => pending_running_tasks.extend(tasks), - Err(DispatcherError::JobCanceled(_)) => { - return Ok(self.cancel_job(&mut pending_running_tasks).await); - } - Err(DispatcherError::Shutdown(tasks)) => { - self.tasks_for_shutdown.extend(tasks); - } - } - } - - Ok(TaskStatus::Done((task_id, TaskOutput::Empty))) => { - warn!(%task_id, "Task returned an empty output"); - } - - Ok(TaskStatus::Shutdown(task)) => { - self.tasks_for_shutdown.push(task); - } - - Ok(TaskStatus::Error(e)) => { - cancel_pending_tasks(&mut pending_running_tasks).await; - - return Err(e); - } - - Ok(TaskStatus::Canceled | TaskStatus::ForcedAbortion) => { - return Ok(self.cancel_job(&mut pending_running_tasks).await); - } - - Err(e) => { - cancel_pending_tasks(&mut pending_running_tasks).await; - - return Err(e.into()); - } - } - } - - if !self.tasks_for_shutdown.is_empty() { - return Ok(ReturnStatus::Shutdown( - SerializableJob::::serialize(self).await, - )); - } - - // From this point onward, we are done with the job and it can't be interrupted anymore - let Self { - location, - metadata, - errors, - .. - } = self; - - let (sync_param, db_param) = sync_db_not_null_entry!( - LocationScanState::FilesIdentified as i32, - location::scan_state - ); - - ctx.sync() - .write_op( - ctx.db(), - ctx.sync().shared_update( - prisma_sync::location::SyncId { - pub_id: location.pub_id.clone(), - }, - [sync_param], - ), - ctx.db() - .location() - .update(location::id::equals(location.id), vec![db_param]) - .select(location::select!({ id })), - ) - .await - .map_err(file_identifier::Error::from)?; - - Ok(ReturnStatus::Completed( - JobReturn::builder() - .with_metadata(metadata) - .with_non_critical_errors(errors) - .build(), - )) - } -} - -impl FileIdentifier { - pub fn new( - location: location::Data, - sub_path: Option, - ) -> Result { - Ok(Self { - location_path: maybe_missing(&location.path, "location.path") - .map(PathBuf::from) - .map(Arc::new)?, - location: Arc::new(location), - sub_path, - file_paths_accumulator: HashMap::new(), - file_paths_ids_with_priority: HashSet::new(), - last_orphan_file_path_id: None, - phase: Phase::default(), - metadata: Metadata::default(), - errors: Vec::new(), - pending_tasks_on_resume: Vec::new(), - tasks_for_shutdown: Vec::new(), - }) - } - - #[allow(clippy::too_many_lines)] - async fn init_or_resume( - &mut self, - pending_running_tasks: &mut FuturesUnordered>, - ctx: &impl JobContext, - device_id: device::id::Type, - dispatcher: &JobTaskDispatcher, - ) -> Result<(), JobErrorOrDispatcherError> { - // if we don't have any pending task, then this is a fresh job - let db = ctx.db(); - let maybe_sub_iso_file_path = - maybe_get_iso_file_path_from_sub_path::( - self.location.id, - self.sub_path.as_ref(), - &*self.location_path, - db, - ) - .await?; - - let start = Instant::now(); - - let location_root_iso_file_path = IsolatedFilePathData::new( - self.location.id, - &*self.location_path, - &*self.location_path, - true, - ) - .map_err(file_identifier::Error::from)?; - - if self.pending_tasks_on_resume.is_empty() { - ctx.progress([ProgressUpdate::phase(self.phase)]).await; - - // First we dispatch some shallow priority tasks to quickly identify orphans in the location - // root directory or in the desired sub-path - self.dispatch_priority_identifier_tasks( - maybe_sub_iso_file_path - .as_ref() - .unwrap_or(&location_root_iso_file_path), - ctx, - device_id, - dispatcher, - pending_running_tasks, - ) - .await?; - - self.phase = Phase::SearchingOrphans; - // Resetting the last orphan file path id for deep search - self.last_orphan_file_path_id = None; - - self.dispatch_deep_identifier_tasks( - maybe_sub_iso_file_path.as_ref(), - ctx, - device_id, - dispatcher, - pending_running_tasks, - ) - .await?; - - self.last_orphan_file_path_id = None; - self.phase = Phase::IdentifyingFiles; - - ctx.progress(vec![ - ProgressUpdate::TaskCount(u64::from(self.metadata.total_identifier_tasks)), - ProgressUpdate::phase(self.phase), - ProgressUpdate::Message(format!( - "{} files to be identified", - self.metadata.total_found_orphans - )), - ]) - .await; - - self.metadata.seeking_orphans_time = start.elapsed(); - } else { - pending_running_tasks.extend(mem::take(&mut self.pending_tasks_on_resume)); - - // For these 2 phases, we need to keep dispatching tasks until we have no more orphans to identify - // as we could have receive a shutdown command before being able to run through all orphans - match self.phase { - Phase::SearchingOrphansWithPriority => { - self.dispatch_priority_identifier_tasks( - maybe_sub_iso_file_path - .as_ref() - .unwrap_or(&location_root_iso_file_path), - ctx, - device_id, - dispatcher, - pending_running_tasks, - ) - .await?; - - self.phase = Phase::SearchingOrphans; - // Resetting the last orphan file path id for deep search - self.last_orphan_file_path_id = None; - - self.dispatch_deep_identifier_tasks( - maybe_sub_iso_file_path.as_ref(), - ctx, - device_id, - dispatcher, - pending_running_tasks, - ) - .await?; - - self.last_orphan_file_path_id = None; - self.phase = Phase::IdentifyingFiles; - } - - Phase::SearchingOrphans => { - self.dispatch_deep_identifier_tasks( - maybe_sub_iso_file_path.as_ref(), - ctx, - device_id, - dispatcher, - pending_running_tasks, - ) - .await?; - - self.last_orphan_file_path_id = None; - self.phase = Phase::IdentifyingFiles; - } - - _ => {} - } - - ctx.progress(vec![ - ProgressUpdate::TaskCount(if matches!(self.phase, Phase::IdentifyingFiles) { - u64::from(self.metadata.total_identifier_tasks) - } else { - u64::from(self.metadata.total_object_processor_tasks) - }), - ProgressUpdate::phase(self.phase), - ProgressUpdate::Message(format!( - "{} files to be identified", - self.metadata.total_found_orphans - )), - ]) - .await; - debug!( - resuming_tasks_count = self.pending_tasks_on_resume.len(), - "Resuming tasks for FileIdentifier job;", - ); - } - - Ok(()) - } - - /// Process output of tasks, according to the downcasted output type - /// - /// # Panics - /// Will panic if another task type is added in the job, but this function wasn't updated to handle it - /// - async fn process_task_output( - &mut self, - task_id: TaskId, - any_task_output: Box, - ctx: &impl JobContext, - device_id: device::id::Type, - dispatcher: &JobTaskDispatcher, - ) -> Result>, DispatcherError> { - if any_task_output.is::() { - return self - .process_identifier_output( - task_id, - *any_task_output - .downcast::() - .expect("just checked"), - ctx, - device_id, - dispatcher, - ) - .await; - } else if any_task_output.is::() { - self.process_object_processor_output( - task_id, - *any_task_output - .downcast::() - .expect("just checked"), - ctx, - ) - .await; - } else { - unreachable!("Unexpected task output type: "); - } - - Ok(vec![]) - } - - #[instrument( - skip_all, - fields( - %task_id, - ?extract_metadata_time, - ?save_db_time, - file_paths_to_object_processor_count = file_paths_by_cas_id.len(), - created_objects_count, - total_identified_files, - errors_count = errors.len() - ) - )] - async fn process_identifier_output( - &mut self, - task_id: TaskId, - identifier::Output { - file_path_ids_with_new_object, - file_paths_by_cas_id, - extract_metadata_time, - save_db_time, - created_objects_count, - total_identified_files, - errors, - }: identifier::Output, - ctx: &impl JobContext, - device_id: device::id::Type, - dispatcher: &JobTaskDispatcher, - ) -> Result>, DispatcherError> { - self.metadata.mean_extract_metadata_time += extract_metadata_time; - self.metadata.mean_save_db_time_on_identifier_tasks += save_db_time; - self.metadata.total_identified_files += total_identified_files; - self.metadata.created_objects_count += created_objects_count; - - let file_paths_with_new_object_to_report = file_path_ids_with_new_object - .into_iter() - .filter_map(|id| self.file_paths_ids_with_priority.take(&id)) - .collect::>(); - - if !file_paths_with_new_object_to_report.is_empty() { - ctx.report_update(UpdateEvent::NewIdentifiedObjects { - file_path_ids: file_paths_with_new_object_to_report, - }); - } - - if !errors.is_empty() { - warn!(?errors, "Non critical errors while extracting metadata;"); - self.errors.extend(errors); - } - - accumulate_file_paths_by_cas_id(file_paths_by_cas_id, &mut self.file_paths_accumulator); - - self.metadata.completed_identifier_tasks += 1; - - ctx.progress(vec![ - ProgressUpdate::CompletedTaskCount(u64::from(self.metadata.completed_identifier_tasks)), - ProgressUpdate::Message(format!( - "Identified {} of {} files", - self.metadata.total_identified_files, self.metadata.total_found_orphans - )), - ]) - .await; - - debug!( - "Processed ({}/{}) identifier tasks, took: {extract_metadata_time:?};", - self.metadata.completed_identifier_tasks, self.metadata.total_identifier_tasks, - ); - - // If we completed all identifier tasks, then we dispatch the object processor tasks - if self.metadata.completed_identifier_tasks == self.metadata.total_identifier_tasks { - self.phase = Phase::ProcessingObjects; - let (tasks_count, res) = match dispatch_object_processor_tasks( - self.file_paths_accumulator.drain(), - ctx, - device_id, - dispatcher, - false, - ) - .await - { - Ok(task_handles) => (task_handles.len(), Ok(task_handles)), - Err(DispatcherError::Shutdown(tasks)) => { - (tasks.len(), Err(DispatcherError::Shutdown(tasks))) - } - Err(e) => return Err(e), - }; - - #[allow(clippy::cast_possible_truncation)] - { - // SAFETY: we know that `tasks.len()` is a valid u32 as we wouldn't dispatch more than `u32::MAX` tasks - self.metadata.total_object_processor_tasks = tasks_count as u32; - } - - ctx.progress(vec![ - ProgressUpdate::TaskCount(u64::from(self.metadata.total_object_processor_tasks)), - ProgressUpdate::CompletedTaskCount(0), - ProgressUpdate::phase(self.phase), - ]) - .await; - - res - } else { - Ok(vec![]) - } - } - - #[instrument(skip(self, file_path_ids_with_new_object, ctx))] - async fn process_object_processor_output( - &mut self, - task_id: TaskId, - object_processor::Output { - file_path_ids_with_new_object, - fetch_existing_objects_time, - assign_to_existing_object_time, - create_object_time, - created_objects_count, - linked_objects_count, - }: object_processor::Output, - ctx: &impl JobContext, - ) { - self.metadata.mean_fetch_existing_objects_time += fetch_existing_objects_time; - self.metadata.mean_assign_to_existing_object_time += assign_to_existing_object_time; - self.metadata.mean_create_object_time += create_object_time; - self.metadata.created_objects_count += created_objects_count; - self.metadata.linked_objects_count += linked_objects_count; - - self.metadata.completed_object_processor_tasks += 1; - - ctx.progress(vec![ - ProgressUpdate::CompletedTaskCount(u64::from( - self.metadata.completed_object_processor_tasks, - )), - ProgressUpdate::Message(format!( - "Processed {} of {} objects", - self.metadata.created_objects_count + self.metadata.linked_objects_count, - self.metadata.total_found_orphans - )), - ]) - .await; - - let file_paths_with_new_object_to_report = file_path_ids_with_new_object - .into_iter() - .filter_map(|id| self.file_paths_ids_with_priority.take(&id)) - .collect::>(); - - if !file_paths_with_new_object_to_report.is_empty() { - ctx.report_update(UpdateEvent::NewIdentifiedObjects { - file_path_ids: file_paths_with_new_object_to_report, - }); - } - - debug!( - "Processed ({}/{}) object processor tasks, took: {:?};", - self.metadata.completed_object_processor_tasks, - self.metadata.total_object_processor_tasks, - fetch_existing_objects_time + assign_to_existing_object_time + create_object_time, - ); - } - - async fn dispatch_priority_identifier_tasks( - &mut self, - sub_iso_file_path: &IsolatedFilePathData<'static>, - ctx: &impl JobContext, - device_id: device::id::Type, - dispatcher: &JobTaskDispatcher, - pending_running_tasks: &FuturesUnordered>, - ) -> Result<(), JobErrorOrDispatcherError> { - let db = ctx.db(); - - loop { - let start = Instant::now(); - - #[allow(clippy::cast_possible_wrap)] - // SAFETY: we know that CHUNK_SIZE is a valid i64 - let orphan_paths = db - .file_path() - .find_many(orphan_path_filters_shallow( - self.location.id, - self.last_orphan_file_path_id, - sub_iso_file_path, - )) - .order_by(file_path::id::order(SortOrder::Asc)) - .take(CHUNK_SIZE as i64) - .select(file_path_for_file_identifier::select()) - .exec() - .await - .map_err(file_identifier::Error::from)?; - - trace!(orphans_count = orphan_paths.len(), "Found orphan paths;"); - - if orphan_paths.is_empty() { - break; - } - - self.file_paths_ids_with_priority.extend( - orphan_paths - .iter() - .map(|file_path_for_file_identifier::Data { id, .. }| *id), - ); - - self.metadata.total_found_orphans += orphan_paths.len() as u64; - self.last_orphan_file_path_id = - Some(orphan_paths.last().expect("orphan_paths is not empty").id); - - self.metadata.total_identifier_tasks += 1; - - ctx.progress(vec![ - ProgressUpdate::TaskCount(u64::from(self.metadata.total_identifier_tasks)), - ProgressUpdate::Message(format!( - "Found {} orphan paths", - self.metadata.total_found_orphans - )), - ]) - .await; - - debug!( - "Dispatched ({}/{}) identifier tasks, took: {:?};", - self.metadata.completed_identifier_tasks, - self.metadata.total_identifier_tasks, - start.elapsed(), - ); - - pending_running_tasks.push( - dispatcher - .dispatch(tasks::Identifier::new( - Arc::clone(&self.location), - Arc::clone(&self.location_path), - orphan_paths, - true, - Arc::clone(ctx.db()), - ctx.sync().clone(), - device_id, - )) - .await?, - ); - } - - Ok(()) - } - - async fn dispatch_deep_identifier_tasks( - &mut self, - maybe_sub_iso_file_path: Option<&IsolatedFilePathData<'static>>, - ctx: &impl JobContext, - device_id: device::id::Type, - dispatcher: &JobTaskDispatcher, - pending_running_tasks: &FuturesUnordered>, - ) -> Result<(), JobErrorOrDispatcherError> { - let db = ctx.db(); - - loop { - let start = Instant::now(); - - #[allow(clippy::cast_possible_wrap)] - // SAFETY: we know that CHUNK_SIZE is a valid i64 - let mut orphan_paths = db - .file_path() - .find_many(orphan_path_filters_deep( - self.location.id, - self.last_orphan_file_path_id, - maybe_sub_iso_file_path, - )) - .order_by(file_path::id::order(SortOrder::Asc)) - .take(CHUNK_SIZE as i64) - .select(file_path_for_file_identifier::select()) - .exec() - .await - .map_err(file_identifier::Error::from)?; - - // No other orphans to identify, we can break the loop - if orphan_paths.is_empty() { - break; - } - - // We grab the last id to use as a starting point for the next iteration, in case we skip this one - self.last_orphan_file_path_id = - Some(orphan_paths.last().expect("orphan_paths is not empty").id); - - orphan_paths.retain(|file_path_for_file_identifier::Data { id, .. }| { - !self.file_paths_ids_with_priority.contains(id) - }); - - // If we don't have any new orphan paths after filtering out, we can skip this iteration - if orphan_paths.is_empty() { - continue; - } - - self.metadata.total_found_orphans += orphan_paths.len() as u64; - - self.metadata.total_identifier_tasks += 1; - - ctx.progress(vec![ - ProgressUpdate::TaskCount(u64::from(self.metadata.total_identifier_tasks)), - ProgressUpdate::Message(format!( - "Found {} orphan paths", - self.metadata.total_found_orphans - )), - ]) - .await; - - debug!( - "Dispatched ({}/{}) identifier tasks, took: {:?};", - self.metadata.completed_identifier_tasks, - self.metadata.total_identifier_tasks, - start.elapsed(), - ); - - pending_running_tasks.push( - dispatcher - .dispatch(tasks::Identifier::new( - Arc::clone(&self.location), - Arc::clone(&self.location_path), - orphan_paths, - false, - Arc::clone(ctx.db()), - ctx.sync().clone(), - device_id, - )) - .await?, - ); - } - - Ok(()) - } - - async fn cancel_job( - &mut self, - pending_running_tasks: &mut FuturesUnordered>, - ) -> ReturnStatus { - cancel_pending_tasks(pending_running_tasks).await; - - ReturnStatus::Canceled( - JobReturn::builder() - .with_metadata(mem::take(&mut self.metadata)) - .with_non_critical_errors(mem::take(&mut self.errors)) - .build(), - ) - } -} - -#[derive(Debug, Clone, Copy, Serialize, Deserialize)] -enum TaskKind { - Identifier, - ObjectProcessor, -} - -#[derive(Serialize, Deserialize)] -struct SaveState { - location: Arc, - location_path: Arc, - sub_path: Option, - - file_paths_accumulator: HashMap, Vec>, - file_paths_ids_with_priority: HashSet, - last_orphan_file_path_id: Option, - - phase: Phase, - metadata: Metadata, - - errors: Vec, - - tasks_for_shutdown_bytes: Option, -} - -#[derive(Debug, Clone, Serialize, Deserialize, Default)] -pub struct Metadata { - mean_extract_metadata_time: Duration, - mean_save_db_time_on_identifier_tasks: Duration, - mean_fetch_existing_objects_time: Duration, - mean_assign_to_existing_object_time: Duration, - mean_create_object_time: Duration, - seeking_orphans_time: Duration, - total_found_orphans: u64, - total_identified_files: u64, - created_objects_count: u64, - linked_objects_count: u64, - total_identifier_tasks: u32, - completed_identifier_tasks: u32, - total_object_processor_tasks: u32, - completed_object_processor_tasks: u32, -} - -impl From for Vec { - fn from( - Metadata { - mut mean_extract_metadata_time, - mut mean_save_db_time_on_identifier_tasks, - mut mean_fetch_existing_objects_time, - mut mean_assign_to_existing_object_time, - mut mean_create_object_time, - seeking_orphans_time, - total_found_orphans, - total_identified_files, - created_objects_count, - linked_objects_count, - total_identifier_tasks, - completed_identifier_tasks, - total_object_processor_tasks, - completed_object_processor_tasks, - }: Metadata, - ) -> Self { - // To avoid division by zero - mean_extract_metadata_time /= u32::max(total_identifier_tasks, 1); - mean_save_db_time_on_identifier_tasks /= u32::max(total_identifier_tasks, 1); - - mean_fetch_existing_objects_time /= u32::max(total_object_processor_tasks, 1); - mean_assign_to_existing_object_time /= u32::max(total_object_processor_tasks, 1); - mean_create_object_time /= u32::max(total_object_processor_tasks, 1); - - vec![ - ReportOutputMetadata::FileIdentifier { - total_orphan_paths: u64_to_frontend(total_found_orphans), - total_objects_created: u64_to_frontend(created_objects_count), - total_objects_linked: u64_to_frontend(linked_objects_count), - }, - ReportOutputMetadata::Metrics(HashMap::from([ - ( - "mean_extract_metadata_time".into(), - json!(mean_extract_metadata_time), - ), - ( - "mean_save_db_time_on_identifier_tasks".into(), - json!(mean_save_db_time_on_identifier_tasks), - ), - ( - "mean_fetch_existing_objects_time".into(), - json!(mean_fetch_existing_objects_time), - ), - ( - "mean_assign_to_existing_object_time".into(), - json!(mean_assign_to_existing_object_time), - ), - ( - "mean_create_object_time".into(), - json!(mean_create_object_time), - ), - ("seeking_orphans_time".into(), json!(seeking_orphans_time)), - ("total_found_orphans".into(), json!(total_found_orphans)), - ( - "total_identified_files".into(), - json!(total_identified_files), - ), - ("created_objects_count".into(), json!(created_objects_count)), - ("linked_objects_count".into(), json!(linked_objects_count)), - ( - "total_identifier_tasks".into(), - json!(total_identifier_tasks), - ), - ( - "completed_identifier_tasks".into(), - json!(completed_identifier_tasks), - ), - ( - "total_object_processor_tasks".into(), - json!(total_object_processor_tasks), - ), - ( - "completed_object_processor_tasks".into(), - json!(completed_object_processor_tasks), - ), - ])), - ] - } -} - -impl SerializableJob for FileIdentifier { - async fn serialize(self) -> Result>, rmp_serde::encode::Error> { - let Self { - location, - location_path, - sub_path, - file_paths_accumulator, - file_paths_ids_with_priority, - last_orphan_file_path_id, - phase, - metadata, - errors, - tasks_for_shutdown, - .. - } = self; - - let serialized_tasks = tasks_for_shutdown - .into_iter() - .map(|task| async move { - if task.is::() { - SerializableTask::serialize( - *task.downcast::().expect("just checked"), - ) - .await - .map(|bytes| (TaskKind::Identifier, bytes)) - } else if task.is::() { - task.downcast::() - .expect("just checked") - .serialize() - .await - .map(|bytes| (TaskKind::ObjectProcessor, bytes)) - } else { - unreachable!("Unexpected task type") - } - }) - .collect::>() - .try_join() - .await?; - - let tasks_for_shutdown_bytes = if serialized_tasks.is_empty() { - None - } else { - Some(SerializedTasks(rmp_serde::to_vec_named(&serialized_tasks)?)) - }; - - rmp_serde::to_vec_named(&SaveState { - location, - location_path, - sub_path, - file_paths_accumulator, - file_paths_ids_with_priority, - last_orphan_file_path_id, - phase, - metadata, - errors, - tasks_for_shutdown_bytes, - }) - .map(Some) - } - - async fn deserialize( - serialized_job: &[u8], - _: &OuterCtx, - ) -> Result)>, rmp_serde::decode::Error> { - let SaveState { - location, - location_path, - sub_path, - file_paths_accumulator, - file_paths_ids_with_priority, - last_orphan_file_path_id, - phase, - metadata, - errors, - tasks_for_shutdown_bytes, - } = rmp_serde::from_slice::(serialized_job)?; - - Ok(Some(( - Self { - location, - location_path, - sub_path, - file_paths_accumulator, - file_paths_ids_with_priority, - last_orphan_file_path_id, - phase, - metadata, - errors, - pending_tasks_on_resume: Vec::new(), - tasks_for_shutdown: Vec::new(), - }, - tasks_for_shutdown_bytes, - ))) - } -} diff --git a/core/crates/heavy-lifting/src/file_identifier/mod.rs b/core/crates/heavy-lifting/src/file_identifier/mod.rs deleted file mode 100644 index f777c118d..000000000 --- a/core/crates/heavy-lifting/src/file_identifier/mod.rs +++ /dev/null @@ -1,290 +0,0 @@ -use crate::{utils::sub_path, OuterContext}; - -use sd_core_file_path_helper::{FilePathError, IsolatedFilePathData}; -use sd_core_prisma_helpers::CasId; -use sd_core_sync::DevicePubId; - -use sd_file_ext::{extensions::Extension, kind::ObjectKind}; -use sd_prisma::prisma::{device, file_path, location}; -use sd_task_system::{TaskDispatcher, TaskHandle}; -use sd_utils::{db::MissingFieldError, error::FileIOError}; - -use std::{ - collections::{hash_map::Entry, HashMap}, - fs::Metadata, - mem, - path::Path, - sync::Arc, -}; - -use prisma_client_rust::QueryError; -use rspc::ErrorCode; -use serde::{Deserialize, Serialize}; -use specta::Type; -use tokio::fs; -use tracing::trace; -use uuid::Uuid; - -mod cas_id; -pub mod job; -mod shallow; -mod tasks; - -pub use cas_id::generate_cas_id; - -pub use job::FileIdentifier; -pub use shallow::shallow; - -use tasks::FilePathToCreateOrLinkObject; - -// we break these tasks into chunks of 100 to improve performance -const CHUNK_SIZE: usize = 100; - -#[derive(thiserror::Error, Debug)] -pub enum Error { - #[error("device not found: for rspc::Error { - fn from(e: Error) -> Self { - match e { - Error::SubPath(sub_path_err) => sub_path_err.into(), - - _ => Self::with_cause(ErrorCode::InternalServerError, e.to_string(), e), - } - } -} - -#[derive(thiserror::Error, Debug, Serialize, Deserialize, Type, Clone)] -#[serde(rename_all = "snake_case")] -pub enum NonCriticalFileIdentifierError { - #[error("failed to extract file metadata: {0}")] - FailedToExtractFileMetadata(String), - #[cfg(target_os = "windows")] - #[error("failed to extract metadata from on-demand file: {0}")] - FailedToExtractMetadataFromOnDemandFile(String), - #[error( - "failed to extract isolated file path data: : {error}" - )] - FailedToExtractIsolatedFilePathData { - file_path_pub_id: Uuid, - error: String, - }, - #[error("file path without is_dir field: ")] - FilePathWithoutIsDirField(file_path::id::Type), -} - -#[derive(Debug, Clone)] -pub struct FileMetadata { - pub cas_id: Option>, - pub kind: ObjectKind, - pub fs_metadata: Metadata, -} - -impl FileMetadata { - /// Fetch metadata from the file system and generate a cas id for the file - /// if it's not empty. - /// - /// # Panics - /// Will panic if the file is a directory. - pub async fn new( - location_path: impl AsRef + Send, - iso_file_path: &IsolatedFilePathData<'_>, - ) -> Result { - let path = location_path.as_ref().join(iso_file_path); - - let fs_metadata = fs::metadata(&path) - .await - .map_err(|e| FileIOError::from((&path, e)))?; - - if fs_metadata.is_dir() { - trace!(path = %path.display(), "Skipping directory;"); - return Ok(Self { - cas_id: None, - kind: ObjectKind::Folder, - fs_metadata, - }); - } - - // derive Object kind - let kind = Extension::resolve_conflicting(&path, false) - .await - .map_or(ObjectKind::Unknown, Into::into); - - let cas_id = if fs_metadata.len() != 0 { - generate_cas_id(&path, fs_metadata.len()) - .await - .map_err(|e| FileIOError::from((&path, e)))? - } else { - // We can't do shit with empty files - trace!(path = %path.display(), %kind, "Skipping empty file;"); - return Ok(Self { - cas_id: None, - kind, - fs_metadata, - }); - }; - - trace!( - path = %path.display(), - ?cas_id, - %kind, - "Analyzed file;", - ); - - Ok(Self { - cas_id: Some(cas_id), - kind, - fs_metadata, - }) - } -} - -fn orphan_path_filters_shallow( - location_id: location::id::Type, - file_path_id: Option, - sub_iso_file_path: &IsolatedFilePathData<'_>, -) -> Vec { - sd_utils::chain_optional_iter( - [ - file_path::object_id::equals(None), - file_path::location_id::equals(Some(location_id)), - file_path::materialized_path::equals(Some( - sub_iso_file_path - .materialized_path_for_children() - .expect("sub path for shallow identifier must be a directory"), - )), - ], - [file_path_id.map(file_path::id::gt)], - ) -} - -fn orphan_path_filters_deep( - location_id: location::id::Type, - file_path_id: Option, - maybe_sub_iso_file_path: Option<&IsolatedFilePathData<'_>>, -) -> Vec { - sd_utils::chain_optional_iter( - [ - file_path::object_id::equals(None), - file_path::location_id::equals(Some(location_id)), - ], - [ - // this is a workaround for the cursor not working properly - file_path_id.map(file_path::id::gt), - maybe_sub_iso_file_path.as_ref().map(|sub_iso_file_path| { - file_path::materialized_path::starts_with( - sub_iso_file_path - .materialized_path_for_children() - .expect("sub path iso_file_path must be a directory"), - ) - }), - ], - ) -} - -async fn dispatch_object_processor_tasks( - file_paths_by_cas_id: Iter, - ctx: &impl OuterContext, - device_id: device::id::Type, - dispatcher: &Dispatcher, - with_priority: bool, -) -> Result>, Dispatcher::DispatchError> -where - Iter: IntoIterator, Vec)> + Send, - Iter::IntoIter: Send, - Dispatcher: TaskDispatcher, -{ - let mut current_batch = HashMap::<_, Vec<_>>::new(); - let mut tasks = vec![]; - - let mut current_batch_size = 0; - - for (cas_id, objects_to_create_or_link) in file_paths_by_cas_id { - if objects_to_create_or_link.len() >= CHUNK_SIZE { - tasks.push( - dispatcher - .dispatch(tasks::ObjectProcessor::new( - HashMap::from([(cas_id, objects_to_create_or_link)]), - Arc::clone(ctx.db()), - ctx.sync().clone(), - device_id, - with_priority, - )) - .await?, - ); - } else { - current_batch_size += objects_to_create_or_link.len(); - match current_batch.entry(cas_id) { - Entry::Occupied(entry) => { - entry.into_mut().extend(objects_to_create_or_link); - } - Entry::Vacant(entry) => { - entry.insert(objects_to_create_or_link); - } - } - - if current_batch_size >= CHUNK_SIZE { - tasks.push( - dispatcher - .dispatch(tasks::ObjectProcessor::new( - mem::take(&mut current_batch), - Arc::clone(ctx.db()), - ctx.sync().clone(), - device_id, - with_priority, - )) - .await?, - ); - - current_batch_size = 0; - } - } - } - - if !current_batch.is_empty() { - tasks.push( - dispatcher - .dispatch(tasks::ObjectProcessor::new( - current_batch, - Arc::clone(ctx.db()), - ctx.sync().clone(), - device_id, - with_priority, - )) - .await?, - ); - } - - Ok(tasks) -} - -fn accumulate_file_paths_by_cas_id( - input: HashMap, Vec>, - accumulator: &mut HashMap, Vec>, -) { - for (cas_id, file_paths) in input { - match accumulator.entry(cas_id) { - Entry::<_, Vec<_>>::Occupied(entry) => { - entry.into_mut().extend(file_paths); - } - Entry::Vacant(entry) => { - entry.insert(file_paths); - } - } - } -} diff --git a/core/crates/heavy-lifting/src/file_identifier/shallow.rs b/core/crates/heavy-lifting/src/file_identifier/shallow.rs deleted file mode 100644 index 4c00882da..000000000 --- a/core/crates/heavy-lifting/src/file_identifier/shallow.rs +++ /dev/null @@ -1,238 +0,0 @@ -use crate::{ - file_identifier, utils::sub_path::maybe_get_iso_file_path_from_sub_path, Error, - NonCriticalError, OuterContext, UpdateEvent, -}; - -use sd_core_file_path_helper::IsolatedFilePathData; -use sd_core_prisma_helpers::file_path_for_file_identifier; - -use sd_prisma::prisma::{device, file_path, location, SortOrder}; -use sd_task_system::{ - BaseTaskDispatcher, CancelTaskOnDrop, TaskDispatcher, TaskHandle, TaskOutput, TaskStatus, -}; -use sd_utils::db::maybe_missing; - -use std::{ - collections::HashMap, - path::{Path, PathBuf}, - sync::Arc, -}; - -use futures::{stream::FuturesUnordered, StreamExt}; -use tracing::{debug, instrument, trace, warn}; - -use super::{ - accumulate_file_paths_by_cas_id, dispatch_object_processor_tasks, orphan_path_filters_shallow, - tasks::{self, identifier, object_processor}, - CHUNK_SIZE, -}; - -#[instrument( - skip_all, - fields( - location_id = location.id, - location_path = ?location.path, - sub_path = %sub_path.as_ref().display() - ) - err, -)] -pub async fn shallow( - location: location::Data, - sub_path: impl AsRef + Send, - dispatcher: &BaseTaskDispatcher, - ctx: &impl OuterContext, -) -> Result, Error> { - let db = ctx.db(); - - let location_path = maybe_missing(&location.path, "location.path") - .map(PathBuf::from) - .map(Arc::new) - .map_err(file_identifier::Error::from)?; - - let location = Arc::new(location); - - let sub_iso_file_path = maybe_get_iso_file_path_from_sub_path::( - location.id, - Some(sub_path.as_ref()), - &*location_path, - db, - ) - .await? - .map_or_else( - || { - IsolatedFilePathData::new(location.id, &*location_path, &*location_path, true) - .map_err(file_identifier::Error::from) - }, - Ok, - )?; - - let device_pub_id = &ctx.sync().device_pub_id; - let device_id = ctx - .db() - .device() - .find_unique(device::pub_id::equals(device_pub_id.to_db())) - .exec() - .await - .map_err(file_identifier::Error::from)? - .ok_or(file_identifier::Error::DeviceNotFound( - device_pub_id.clone(), - ))? - .id; - - let mut orphans_count = 0; - let mut last_orphan_file_path_id = None; - - let mut identifier_tasks = vec![]; - - loop { - #[allow(clippy::cast_possible_wrap)] - // SAFETY: we know that CHUNK_SIZE is a valid i64 - let orphan_paths = db - .file_path() - .find_many(orphan_path_filters_shallow( - location.id, - last_orphan_file_path_id, - &sub_iso_file_path, - )) - .order_by(file_path::id::order(SortOrder::Asc)) - .take(CHUNK_SIZE as i64) - .select(file_path_for_file_identifier::select()) - .exec() - .await - .map_err(file_identifier::Error::from)?; - - let Some(last_orphan) = orphan_paths.last() else { - // No orphans here! - break; - }; - - orphans_count += orphan_paths.len() as u64; - last_orphan_file_path_id = Some(last_orphan.id); - - let Ok(tasks) = dispatcher - .dispatch(tasks::Identifier::new( - Arc::clone(&location), - Arc::clone(&location_path), - orphan_paths, - true, - Arc::clone(ctx.db()), - ctx.sync().clone(), - device_id, - )) - .await - else { - debug!("Task system is shutting down while a shallow file identifier was in progress"); - return Ok(vec![]); - }; - - identifier_tasks.push(tasks); - } - - if orphans_count == 0 { - trace!("No orphans found"); - return Ok(vec![]); - } - - process_tasks(identifier_tasks, dispatcher, ctx, device_id).await -} - -async fn process_tasks( - identifier_tasks: Vec>, - dispatcher: &BaseTaskDispatcher, - ctx: &impl OuterContext, - device_id: device::id::Type, -) -> Result, Error> { - let total_identifier_tasks = identifier_tasks.len(); - - let mut pending_running_tasks = identifier_tasks - .into_iter() - .map(CancelTaskOnDrop::new) - .collect::>(); - - let mut errors = vec![]; - let mut completed_identifier_tasks = 0; - let mut file_paths_accumulator = HashMap::new(); - - while let Some(task_result) = pending_running_tasks.next().await { - match task_result { - Ok(TaskStatus::Done((_, TaskOutput::Out(any_task_output)))) => { - // We only care about ExtractFileMetadataTaskOutput because we need to dispatch further tasks - // and the ObjectProcessorTask only gives back some metrics not much important for - // shallow file identifier - if any_task_output.is::() { - let identifier::Output { - file_path_ids_with_new_object, - file_paths_by_cas_id, - errors: more_errors, - .. - } = *any_task_output.downcast().expect("just checked"); - - completed_identifier_tasks += 1; - - ctx.report_update(UpdateEvent::NewIdentifiedObjects { - file_path_ids: file_path_ids_with_new_object, - }); - - accumulate_file_paths_by_cas_id( - file_paths_by_cas_id, - &mut file_paths_accumulator, - ); - - errors.extend(more_errors); - - if total_identifier_tasks == completed_identifier_tasks { - let Ok(tasks) = dispatch_object_processor_tasks( - file_paths_accumulator.drain(), - ctx, - device_id, - dispatcher, - true, - ) - .await - else { - debug!("Task system is shutting down while a shallow file identifier was in progress"); - continue; - }; - - pending_running_tasks.extend(tasks.into_iter().map(CancelTaskOnDrop::new)); - } - } else { - let object_processor::Output { - file_path_ids_with_new_object, - .. - } = *any_task_output.downcast().expect("just checked"); - - ctx.report_update(UpdateEvent::NewIdentifiedObjects { - file_path_ids: file_path_ids_with_new_object, - }); - } - } - - Ok(TaskStatus::Done((task_id, TaskOutput::Empty))) => { - warn!(%task_id, "Task returned an empty output"); - } - - Ok(TaskStatus::Shutdown(_)) => { - debug!( - "Spacedrive is shutting down while a shallow file identifier was in progress" - ); - continue; - } - - Ok(TaskStatus::Error(e)) => { - return Err(e); - } - - Ok(TaskStatus::Canceled | TaskStatus::ForcedAbortion) => { - warn!("Task was cancelled or aborted on shallow file identifier"); - return Ok(errors); - } - - Err(e) => { - return Err(e.into()); - } - } - } - - Ok(errors) -} diff --git a/core/crates/heavy-lifting/src/file_identifier/tasks/identifier.rs b/core/crates/heavy-lifting/src/file_identifier/tasks/identifier.rs deleted file mode 100644 index 125a72713..000000000 --- a/core/crates/heavy-lifting/src/file_identifier/tasks/identifier.rs +++ /dev/null @@ -1,578 +0,0 @@ -use crate::{ - file_identifier::{self, FileMetadata}, - Error, NonCriticalError, -}; - -use sd_core_file_path_helper::IsolatedFilePathData; -use sd_core_prisma_helpers::{file_path_for_file_identifier, CasId, FilePathPubId}; -use sd_core_sync::SyncManager; - -use sd_file_ext::kind::ObjectKind; -use sd_prisma::{ - prisma::{device, file_path, location, PrismaClient}, - prisma_sync, -}; -use sd_sync::{sync_db_entry, OperationFactory}; -use sd_task_system::{ - ExecStatus, Interrupter, InterruptionKind, IntoAnyTaskOutput, SerializableTask, Task, TaskId, -}; -use sd_utils::error::FileIOError; - -use std::{ - collections::HashMap, convert::identity, future::IntoFuture, mem, path::PathBuf, pin::pin, - sync::Arc, time::Duration, -}; - -use futures::stream::{self, FuturesUnordered, StreamExt}; -use futures_concurrency::{future::TryJoin, stream::Merge}; -use serde::{Deserialize, Serialize}; -use tokio::time::Instant; -use tracing::{error, instrument, trace, warn, Level}; - -use super::{create_objects_and_update_file_paths, FilePathToCreateOrLinkObject}; - -#[derive(Debug, Serialize, Deserialize)] -struct IdentifiedFile { - file_path: file_path_for_file_identifier::Data, - cas_id: CasId<'static>, - kind: ObjectKind, -} - -impl IdentifiedFile { - pub fn new( - file_path: file_path_for_file_identifier::Data, - cas_id: impl Into>, - kind: ObjectKind, - ) -> Self { - Self { - file_path, - cas_id: cas_id.into(), - kind, - } - } -} - -#[derive(Debug)] -pub struct Identifier { - // Task control - id: TaskId, - with_priority: bool, - - // Received input args - location: Arc, - location_path: Arc, - file_paths_by_id: HashMap, - - // Inner state - device_id: device::id::Type, - identified_files: HashMap, - file_paths_without_cas_id: Vec, - - // Out collector - output: Output, - - // Dependencies - db: Arc, - sync: SyncManager, -} - -/// Output from the `[Identifier]` task -#[derive(Debug, Default, Serialize, Deserialize)] -pub struct Output { - /// To send to frontend for priority reporting of new objects - pub file_path_ids_with_new_object: Vec, - - /// Files that need to be aggregate between many identifier tasks to be processed by the - /// object processor tasks - pub file_paths_by_cas_id: HashMap, Vec>, - - /// Collected metric about time elapsed extracting metadata from file system - pub extract_metadata_time: Duration, - - /// Collected metric about time spent saving objects on disk - pub save_db_time: Duration, - - /// Total number of objects already created as they didn't have `cas_id`, like directories or empty files - pub created_objects_count: u64, - - /// Total number of files that we were able to identify - pub total_identified_files: u64, - - /// Non critical errors that happened during the task execution - pub errors: Vec, -} - -#[async_trait::async_trait] -impl Task for Identifier { - fn id(&self) -> TaskId { - self.id - } - - fn with_priority(&self) -> bool { - self.with_priority - } - - #[instrument( - skip(self, interrupter), - fields( - task_id = %self.id, - location_id = %self.location.id, - location_path = %self.location_path.display(), - files_count = %self.file_paths_by_id.len(), - ), - ret(level = Level::TRACE), - err, - )] - #[allow(clippy::blocks_in_conditions)] // Due to `err` on `instrument` macro above - async fn run(&mut self, interrupter: &Interrupter) -> Result { - // `Processed` is larger than `Interrupt`, but it's much more common - // so we ignore the size difference to optimize for usage - #[allow(clippy::large_enum_variant)] - enum StreamMessage { - Processed(FilePathPubId, Result), - Interrupt(InterruptionKind), - } - - let Self { - location, - location_path, - device_id, - file_paths_by_id, - file_paths_without_cas_id, - identified_files, - output, - .. - } = self; - - if !file_paths_by_id.is_empty() { - let start_time = Instant::now(); - - let extraction_futures = file_paths_by_id - .iter() - .filter_map(|(file_path_id, file_path)| { - try_iso_file_path_extraction( - location.id, - file_path_id.clone(), - file_path, - Arc::clone(location_path), - &mut output.errors, - ) - }) - .map(|(file_path_id, iso_file_path, location_path)| async move { - StreamMessage::Processed( - file_path_id, - FileMetadata::new(&*location_path, &iso_file_path).await, - ) - }) - .collect::>(); - - let mut msg_stream = pin!(( - extraction_futures, - stream::once(interrupter.into_future()).map(StreamMessage::Interrupt) - ) - .merge()); - - while let Some(msg) = msg_stream.next().await { - match msg { - StreamMessage::Processed(file_path_pub_id, res) => { - let file_path = file_paths_by_id - .remove(&file_path_pub_id) - .expect("file_path must be here"); - - trace!( - files_remaining = file_paths_by_id.len(), - %file_path_pub_id, - "Processed file;", - ); - - match res { - Ok(FileMetadata { - cas_id: Some(cas_id), - kind, - .. - }) => { - identified_files.insert( - file_path_pub_id, - IdentifiedFile::new(file_path, cas_id, kind), - ); - } - Ok(FileMetadata { - cas_id: None, kind, .. - }) => { - let file_path_for_file_identifier::Data { - id, - pub_id, - date_created, - .. - } = file_path; - file_paths_without_cas_id.push(FilePathToCreateOrLinkObject { - id, - file_path_pub_id: pub_id.into(), - kind, - created_at: date_created, - }); - } - Err(e) => { - handle_non_critical_errors( - file_path_pub_id, - &e, - &mut output.errors, - ); - } - } - - if file_paths_by_id.is_empty() { - trace!("All files have been processed"); - // All files have been processed so we can end this merged stream - // and don't keep waiting an interrupt signal - break; - } - } - - StreamMessage::Interrupt(kind) => { - trace!(?kind, "Interrupted;"); - output.extract_metadata_time += start_time.elapsed(); - return Ok(match kind { - InterruptionKind::Pause => ExecStatus::Paused, - InterruptionKind::Cancel => ExecStatus::Canceled, - }); - } - } - } - - output.extract_metadata_time = start_time.elapsed(); - - output.total_identified_files = - identified_files.len() as u64 + file_paths_without_cas_id.len() as u64; - - trace!( - identified_files_count = identified_files.len(), - "All files have been processed, saving cas_ids to db...;" - ); - let start_time = Instant::now(); - // Assign cas_id to each file path - let ((), file_path_ids_with_new_object) = ( - assign_cas_id_to_file_paths(identified_files, &self.db, &self.sync), - create_objects_and_update_file_paths( - file_paths_without_cas_id.drain(..), - &self.db, - &self.sync, - *device_id, - ), - ) - .try_join() - .await?; - - output.save_db_time = start_time.elapsed(); - output.created_objects_count = file_path_ids_with_new_object.len() as u64; - output.file_path_ids_with_new_object = - file_path_ids_with_new_object.into_keys().collect(); - - output.file_paths_by_cas_id = identified_files.drain().fold( - HashMap::new(), - |mut map, - ( - file_path_pub_id, - IdentifiedFile { - cas_id, - kind, - file_path: - file_path_for_file_identifier::Data { - id, date_created, .. - }, - }, - )| { - map.entry(cas_id) - .or_insert_with(|| Vec::with_capacity(1)) - .push(FilePathToCreateOrLinkObject { - id, - file_path_pub_id, - kind, - created_at: date_created, - }); - - map - }, - ); - - trace!(save_db_time = ?output.save_db_time, "Cas_ids saved to db;"); - } else if !file_paths_without_cas_id.is_empty() { - let start_time = Instant::now(); - - // Assign objects to directories - let file_path_ids_with_new_object = create_objects_and_update_file_paths( - file_paths_without_cas_id.drain(..), - &self.db, - &self.sync, - *device_id, - ) - .await?; - - output.save_db_time = start_time.elapsed(); - output.created_objects_count = file_path_ids_with_new_object.len() as u64; - output.file_path_ids_with_new_object = - file_path_ids_with_new_object.into_keys().collect(); - - trace!(save_db_time = ?output.save_db_time, "Directories objects saved to db;"); - } - - Ok(ExecStatus::Done(mem::take(output).into_output())) - } -} - -impl Identifier { - #[must_use] - pub fn new( - location: Arc, - location_path: Arc, - file_paths: Vec, - with_priority: bool, - db: Arc, - sync: SyncManager, - device_id: device::id::Type, - ) -> Self { - let mut output = Output::default(); - - let file_paths_count = file_paths.len(); - let directories_count = file_paths - .iter() - .filter(|file_path| file_path.is_dir.is_some_and(identity)) - .count(); - - let (file_paths_by_id, file_paths_without_cas_id) = file_paths.into_iter().fold( - ( - HashMap::with_capacity(file_paths_count - directories_count), - Vec::with_capacity(directories_count), - ), - |(mut file_paths_by_id, mut directory_file_paths), file_path| { - match file_path.is_dir { - Some(true) => { - let file_path_for_file_identifier::Data { - id, - pub_id, - date_created, - .. - } = file_path; - directory_file_paths.push(FilePathToCreateOrLinkObject { - id, - file_path_pub_id: pub_id.into(), - kind: ObjectKind::Folder, - created_at: date_created, - }); - } - Some(false) => { - file_paths_by_id.insert(file_path.pub_id.as_slice().into(), file_path); - } - None => { - warn!(%file_path.id, "file path without is_dir field, skipping;"); - output.errors.push( - file_identifier::NonCriticalFileIdentifierError::FilePathWithoutIsDirField( - file_path.id, - ) - .into(), - ); - } - }; - - (file_paths_by_id, directory_file_paths) - }, - ); - - Self { - id: TaskId::new_v4(), - location, - location_path, - device_id, - identified_files: HashMap::with_capacity(file_paths_count - directories_count), - file_paths_without_cas_id, - file_paths_by_id, - output, - with_priority, - db, - sync, - } - } -} - -#[instrument(skip_all, err, fields(identified_files_count = identified_files.len()))] -async fn assign_cas_id_to_file_paths( - identified_files: &HashMap, - db: &PrismaClient, - sync: &SyncManager, -) -> Result<(), file_identifier::Error> { - let (ops, queries) = identified_files - .iter() - .map(|(pub_id, IdentifiedFile { cas_id, .. })| { - let (sync_param, db_param) = sync_db_entry!(cas_id, file_path::cas_id); - - ( - sync.shared_update( - prisma_sync::file_path::SyncId { - pub_id: pub_id.to_db(), - }, - [sync_param], - ), - db.file_path() - .update(file_path::pub_id::equals(pub_id.to_db()), vec![db_param]) - // We don't need any data here, just the id avoids receiving the entire object - // as we can't pass an empty select macro call - .select(file_path::select!({ id })), - ) - }) - .unzip::<_, _, Vec<_>, Vec<_>>(); - - if !ops.is_empty() && !queries.is_empty() { - // Assign cas_id to each file path - sync.write_ops(db, (ops, queries)).await?; - } - - Ok(()) -} - -#[instrument(skip(errors))] -fn handle_non_critical_errors( - file_path_pub_id: FilePathPubId, - e: &FileIOError, - errors: &mut Vec, -) { - let formatted_error = format!(""); - - #[cfg(target_os = "windows")] - { - // Handle case where file is on-demand (NTFS only) - if e.source.raw_os_error().map_or(false, |code| code == 362) { - errors.push( - file_identifier::NonCriticalFileIdentifierError::FailedToExtractMetadataFromOnDemandFile( - formatted_error, - ) - .into(), - ); - } else { - errors.push( - file_identifier::NonCriticalFileIdentifierError::FailedToExtractFileMetadata( - formatted_error, - ) - .into(), - ); - } - } - - #[cfg(not(target_os = "windows"))] - { - errors.push( - file_identifier::NonCriticalFileIdentifierError::FailedToExtractFileMetadata( - formatted_error, - ) - .into(), - ); - } -} - -#[instrument( - skip(location_id, file_path, location_path, errors), - fields( - file_path_id = file_path.id, - materialized_path = ?file_path.materialized_path, - name = ?file_path.name, - extension = ?file_path.extension, - ) -)] -fn try_iso_file_path_extraction( - location_id: location::id::Type, - file_path_pub_id: FilePathPubId, - file_path: &file_path_for_file_identifier::Data, - location_path: Arc, - errors: &mut Vec, -) -> Option<(FilePathPubId, IsolatedFilePathData<'static>, Arc)> { - match IsolatedFilePathData::try_from((location_id, file_path)) - .map(IsolatedFilePathData::to_owned) - { - Ok(iso_file_path) => Some((file_path_pub_id, iso_file_path, location_path)), - Err(e) => { - error!(?e, %file_path_pub_id, "Failed to extract isolated file path data;"); - errors.push( - file_identifier::NonCriticalFileIdentifierError::FailedToExtractIsolatedFilePathData { file_path_pub_id: file_path_pub_id.into(), error: e.to_string() }.into(), - - ); - None - } - } -} - -#[derive(Serialize, Deserialize)] -struct SaveState { - id: TaskId, - location: Arc, - location_path: Arc, - device_id: device::id::Type, - file_paths_by_id: HashMap, - identified_files: HashMap, - file_paths_without_cas_id: Vec, - output: Output, - with_priority: bool, -} - -impl SerializableTask for Identifier { - type SerializeError = rmp_serde::encode::Error; - - type DeserializeError = rmp_serde::decode::Error; - - type DeserializeCtx = (Arc, SyncManager); - - async fn serialize(self) -> Result, Self::SerializeError> { - let Self { - id, - location, - location_path, - device_id, - file_paths_by_id, - identified_files, - file_paths_without_cas_id, - output, - with_priority, - .. - } = self; - rmp_serde::to_vec_named(&SaveState { - id, - location, - location_path, - device_id, - file_paths_by_id, - identified_files, - file_paths_without_cas_id, - output, - with_priority, - }) - } - - async fn deserialize( - data: &[u8], - (db, sync): Self::DeserializeCtx, - ) -> Result { - rmp_serde::from_slice::(data).map( - |SaveState { - id, - location, - location_path, - device_id, - file_paths_by_id, - identified_files, - file_paths_without_cas_id, - output, - with_priority, - }| Self { - id, - with_priority, - location, - location_path, - file_paths_by_id, - device_id, - identified_files, - file_paths_without_cas_id, - output, - db, - sync, - }, - ) - } -} diff --git a/core/crates/heavy-lifting/src/file_identifier/tasks/mod.rs b/core/crates/heavy-lifting/src/file_identifier/tasks/mod.rs deleted file mode 100644 index 59f75d0a9..000000000 --- a/core/crates/heavy-lifting/src/file_identifier/tasks/mod.rs +++ /dev/null @@ -1,187 +0,0 @@ -use crate::file_identifier; - -use sd_core_prisma_helpers::{file_path_id, FilePathPubId, ObjectPubId}; -use sd_core_sync::SyncManager; - -use sd_file_ext::kind::ObjectKind; -use sd_prisma::{ - prisma::{device, file_path, object, PrismaClient}, - prisma_sync, -}; -use sd_sync::{option_sync_db_entry, sync_db_entry, sync_entry, CRDTOperation, OperationFactory}; -use sd_utils::chain_optional_iter; - -use std::collections::{HashMap, HashSet}; - -use chrono::{DateTime, FixedOffset}; -use prisma_client_rust::Select; -use serde::{Deserialize, Serialize}; -use tracing::{instrument, trace, Level}; - -pub mod identifier; -pub mod object_processor; - -pub use identifier::Identifier; -pub use object_processor::ObjectProcessor; - -/// This object has all needed data to create a new `object` for a `file_path` or link an existing one. -#[derive(Debug, Serialize, Deserialize)] -pub(super) struct FilePathToCreateOrLinkObject { - id: file_path::id::Type, - file_path_pub_id: FilePathPubId, - kind: ObjectKind, - created_at: Option>, -} - -#[instrument(skip(sync, db))] -fn connect_file_path_to_object<'db>( - file_path_pub_id: &FilePathPubId, - object_pub_id: &ObjectPubId, - db: &'db PrismaClient, - sync: &SyncManager, -) -> (CRDTOperation, Select<'db, file_path_id::Data>) { - trace!("Connecting"); - - ( - sync.shared_update( - prisma_sync::file_path::SyncId { - pub_id: file_path_pub_id.to_db(), - }, - [sync_entry!( - prisma_sync::object::SyncId { - pub_id: object_pub_id.to_db(), - }, - file_path::object - )], - ), - db.file_path() - .update( - file_path::pub_id::equals(file_path_pub_id.to_db()), - vec![file_path::object::connect(object::pub_id::equals( - object_pub_id.to_db(), - ))], - ) - // selecting just id to avoid fetching the whole object - .select(file_path_id::select()), - ) -} - -#[instrument(skip_all, ret(level = Level::TRACE), err)] -async fn create_objects_and_update_file_paths( - files_and_kinds: impl IntoIterator + Send, - db: &PrismaClient, - sync: &SyncManager, - device_id: device::id::Type, -) -> Result, file_identifier::Error> { - trace!("Preparing objects"); - let (object_create_args, file_path_args) = files_and_kinds - .into_iter() - .map( - |FilePathToCreateOrLinkObject { - id, - file_path_pub_id, - kind, - created_at, - }| { - let object_pub_id = ObjectPubId::new(); - - let kind = kind as i32; - - let device_pub_id = sync.device_pub_id.to_db(); - - let (sync_params, db_params) = chain_optional_iter( - [ - ( - sync_entry!( - prisma_sync::device::SyncId { - pub_id: device_pub_id, - }, - object::device - ), - object::device_id::set(Some(device_id)), - ), - sync_db_entry!(kind, object::kind), - ], - [option_sync_db_entry!(created_at, object::date_created)], - ) - .into_iter() - .unzip::<_, _, Vec<_>, Vec<_>>(); - - ( - ( - sync.shared_create( - prisma_sync::object::SyncId { - pub_id: object_pub_id.to_db(), - }, - sync_params, - ), - object::create_unchecked(object_pub_id.to_db(), db_params), - ), - ( - (id, object_pub_id.clone()), - connect_file_path_to_object(&file_path_pub_id, &object_pub_id, db, sync), - ), - ) - }, - ) - .unzip::<_, _, Vec<_>, Vec<_>>(); - - let (mut object_pub_id_by_file_path_id, file_path_update_args) = file_path_args - .into_iter() - .unzip::<_, _, HashMap<_, _>, Vec<_>>( - ); - - let new_objects_count = object_create_args.len(); - if new_objects_count > 0 { - trace!(new_objects_count, "Creating new Objects!;",); - - // create new object records with assembled values - let created_objects_count = sync - .write_ops(db, { - let (sync, db_params) = object_create_args - .into_iter() - .unzip::<_, _, Vec<_>, Vec<_>>(); - - (sync, db.object().create_many(db_params)) - }) - .await?; - - trace!(%created_objects_count, "Created new Objects;"); - - if created_objects_count > 0 { - let file_paths_to_update_count = file_path_update_args.len(); - if file_paths_to_update_count > 0 { - trace!( - file_paths_to_update_count, - "Updating file paths with created objects" - ); - - let updated_file_path_ids = sync - .write_ops( - db, - file_path_update_args - .into_iter() - .unzip::<_, _, Vec<_>, Vec<_>>(), - ) - .await - .map(|file_paths| { - file_paths - .into_iter() - .map(|file_path_id::Data { id }| id) - .collect::>() - })?; - - object_pub_id_by_file_path_id - .retain(|file_path_id, _| updated_file_path_ids.contains(file_path_id)); - } - - Ok(object_pub_id_by_file_path_id) - } else { - trace!("No objects created, skipping file path updates"); - Ok(HashMap::new()) - } - } else { - trace!("No objects to create, skipping file path updates"); - Ok(HashMap::new()) - } -} diff --git a/core/crates/heavy-lifting/src/file_identifier/tasks/object_processor.rs b/core/crates/heavy-lifting/src/file_identifier/tasks/object_processor.rs deleted file mode 100644 index a99d89d8d..000000000 --- a/core/crates/heavy-lifting/src/file_identifier/tasks/object_processor.rs +++ /dev/null @@ -1,434 +0,0 @@ -use crate::{file_identifier, Error}; - -use sd_core_prisma_helpers::{file_path_id, object_for_file_identifier, CasId, ObjectPubId}; -use sd_core_sync::SyncManager; - -use sd_prisma::prisma::{device, file_path, object, PrismaClient}; -use sd_task_system::{ - check_interruption, ExecStatus, Interrupter, IntoAnyTaskOutput, SerializableTask, Task, TaskId, -}; - -use std::{collections::HashMap, mem, sync::Arc, time::Duration}; - -use serde::{Deserialize, Serialize}; -use tokio::time::Instant; -use tracing::{instrument, trace, Level}; - -use super::{ - connect_file_path_to_object, create_objects_and_update_file_paths, FilePathToCreateOrLinkObject, -}; - -#[derive(Debug)] -pub struct ObjectProcessor { - // Task control - id: TaskId, - with_priority: bool, - - // Received input args - file_paths_by_cas_id: HashMap, Vec>, - - // Inner state - stage: Stage, - device_id: device::id::Type, - - // Out collector - output: Output, - - // Dependencies - db: Arc, - sync: SyncManager, -} - -#[derive(Debug, Serialize, Deserialize)] -enum Stage { - Starting, - AssignFilePathsToExistingObjects { - existing_objects_by_cas_id: HashMap, ObjectPubId>, - }, - CreateObjects, -} - -/// Output from the `[ObjectProcessor]` task -#[derive(Debug, Serialize, Deserialize, Default)] -pub struct Output { - /// To send to frontend for priority reporting of new objects - pub file_path_ids_with_new_object: Vec, - - /// Time elapsed fetching existing `objects` from db to be linked to `file_paths` - pub fetch_existing_objects_time: Duration, - - /// Time spent linking `file_paths` to already existing `objects` - pub assign_to_existing_object_time: Duration, - - /// Time spent creating new `objects` - pub create_object_time: Duration, - - /// Number of new `objects` created - pub created_objects_count: u64, - - /// Number of `objects` that were linked to `file_paths` - pub linked_objects_count: u64, -} - -#[async_trait::async_trait] -impl Task for ObjectProcessor { - fn id(&self) -> TaskId { - self.id - } - - fn with_priority(&self) -> bool { - self.with_priority - } - - #[instrument( - skip(self, interrupter), - fields( - task_id = %self.id, - cas_ids_count = %self.file_paths_by_cas_id.len(), - ), - ret(level = Level::TRACE), - err, - )] - #[allow(clippy::blocks_in_conditions)] // Due to `err` on `instrument` macro above - async fn run(&mut self, interrupter: &Interrupter) -> Result { - let Self { - db, - sync, - device_id, - file_paths_by_cas_id, - stage, - output: - Output { - file_path_ids_with_new_object, - fetch_existing_objects_time, - assign_to_existing_object_time, - create_object_time, - created_objects_count, - linked_objects_count, - }, - .. - } = self; - - loop { - match stage { - Stage::Starting => { - trace!("Starting object processor task"); - let start = Instant::now(); - let existing_objects_by_cas_id = - fetch_existing_objects_by_cas_id(file_paths_by_cas_id.keys(), db).await?; - *fetch_existing_objects_time = start.elapsed(); - - trace!( - elapsed_time = ?fetch_existing_objects_time, - existing_objects_count = existing_objects_by_cas_id.len(), - "Fetched existing Objects;", - ); - *stage = Stage::AssignFilePathsToExistingObjects { - existing_objects_by_cas_id, - }; - } - - Stage::AssignFilePathsToExistingObjects { - existing_objects_by_cas_id, - } => { - trace!( - existing_objects_to_link = existing_objects_by_cas_id.len(), - "Assigning file paths to existing Objects;", - ); - let start = Instant::now(); - let more_file_path_ids_with_new_object = assign_existing_objects_to_file_paths( - file_paths_by_cas_id, - existing_objects_by_cas_id, - db, - sync, - ) - .await?; - *assign_to_existing_object_time = start.elapsed(); - file_path_ids_with_new_object.extend(more_file_path_ids_with_new_object); - *linked_objects_count += file_path_ids_with_new_object.len() as u64; - - trace!( - existing_objects_to_link = existing_objects_by_cas_id.len(), - %linked_objects_count, - "Found existing Objects, linked file paths to them;", - ); - - *stage = Stage::CreateObjects; - - if file_paths_by_cas_id.is_empty() { - trace!("No more objects to be created, finishing task"); - // No objects to be created, we're good to finish already - break; - } - } - - Stage::CreateObjects => { - trace!( - creating_count = file_paths_by_cas_id.len(), - "Creating new Objects;" - ); - let start = Instant::now(); - let (more_file_paths_with_new_object, more_linked_objects_count) = - assign_objects_to_duplicated_orphans( - file_paths_by_cas_id, - db, - sync, - *device_id, - ) - .await?; - *create_object_time = start.elapsed(); - file_path_ids_with_new_object.extend(more_file_paths_with_new_object); - *linked_objects_count += more_linked_objects_count; - - *created_objects_count = - file_path_ids_with_new_object.len() as u64 - *linked_objects_count; - - trace!(%created_objects_count, ?create_object_time, "Created new Objects;"); - - break; - } - } - - check_interruption!(interrupter); - } - - Ok(ExecStatus::Done(mem::take(&mut self.output).into_output())) - } -} - -impl ObjectProcessor { - #[must_use] - pub fn new( - file_paths_by_cas_id: HashMap, Vec>, - db: Arc, - sync: SyncManager, - device_id: device::id::Type, - with_priority: bool, - ) -> Self { - Self { - id: TaskId::new_v4(), - db, - sync, - file_paths_by_cas_id, - device_id, - stage: Stage::Starting, - output: Output::default(), - with_priority, - } - } -} - -/// Retrieves objects that are already connected to file paths with the same cas_id -#[instrument(skip_all, err)] -async fn fetch_existing_objects_by_cas_id<'cas_id, Iter>( - cas_ids: Iter, - db: &PrismaClient, -) -> Result, ObjectPubId>, file_identifier::Error> -where - Iter: IntoIterator> + Send, - Iter::IntoIter: Send, -{ - async fn inner( - stringed_cas_ids: Vec, - db: &PrismaClient, - ) -> Result, ObjectPubId>, file_identifier::Error> { - db.object() - .find_many(vec![object::file_paths::some(vec![ - file_path::cas_id::in_vec(stringed_cas_ids), - file_path::object_id::not(None), - ])]) - .select(object_for_file_identifier::select()) - .exec() - .await - .map_err(Into::into) - .map(|objects| { - objects - .into_iter() - .filter_map(|object_for_file_identifier::Data { pub_id, file_paths }| { - file_paths - .first() - .and_then(|file_path| { - file_path - .cas_id - .as_ref() - .map(CasId::from) - .map(CasId::into_owned) - }) - .map(|cas_id| (cas_id, pub_id.into())) - }) - .collect() - }) - } - - let stringed_cas_ids = cas_ids.into_iter().map(Into::into).collect::>(); - - trace!( - cas_ids_count = stringed_cas_ids.len(), - "Fetching existing objects by cas_ids;", - ); - - inner(stringed_cas_ids, db).await -} - -/// Attempt to associate each file path with an object that has been -/// connected to file paths with the same cas_id -#[instrument(skip_all, err, fields(identified_files_count = file_paths_by_cas_id.len()))] -async fn assign_existing_objects_to_file_paths( - file_paths_by_cas_id: &mut HashMap, Vec>, - objects_by_cas_id: &HashMap, ObjectPubId>, - db: &PrismaClient, - sync: &SyncManager, -) -> Result, file_identifier::Error> { - let (ops, queries) = objects_by_cas_id - .iter() - .flat_map(|(cas_id, object_pub_id)| { - file_paths_by_cas_id - .remove(cas_id) - .map(|file_paths| { - file_paths.into_iter().map( - |FilePathToCreateOrLinkObject { - file_path_pub_id, .. - }| { - connect_file_path_to_object(&file_path_pub_id, object_pub_id, db, sync) - }, - ) - }) - .expect("must be here") - }) - .unzip::<_, _, Vec<_>, Vec<_>>(); - - if ops.is_empty() && queries.is_empty() { - return Ok(vec![]); - } - - sync.write_ops(db, (ops, queries)) - .await - .map(|file_paths| { - file_paths - .into_iter() - .map(|file_path_id::Data { id }| id) - .collect() - }) - .map_err(Into::into) -} - -async fn assign_objects_to_duplicated_orphans( - file_paths_by_cas_id: &mut HashMap, Vec>, - db: &PrismaClient, - sync: &SyncManager, - device_id: device::id::Type, -) -> Result<(Vec, u64), file_identifier::Error> { - // at least 1 file path per cas_id - let mut selected_file_paths = Vec::with_capacity(file_paths_by_cas_id.len()); - let mut cas_ids_by_file_path_id = HashMap::with_capacity(file_paths_by_cas_id.len()); - - file_paths_by_cas_id.retain(|cas_id, file_paths| { - let file_path = file_paths.pop().expect("file_paths can't be empty"); - let has_more_file_paths = !file_paths.is_empty(); - - if has_more_file_paths { - cas_ids_by_file_path_id.insert(file_path.id, cas_id.clone()); - } - selected_file_paths.push(file_path); - - has_more_file_paths - }); - - let (mut file_paths_with_new_object, objects_by_cas_id) = - create_objects_and_update_file_paths(selected_file_paths, db, sync, device_id) - .await? - .into_iter() - .map(|(file_path_id, object_pub_id)| { - ( - file_path_id, - cas_ids_by_file_path_id - .remove(&file_path_id) - .map(|cas_id| (cas_id, object_pub_id)), - ) - }) - .unzip::<_, _, Vec<_>, Vec<_>>(); - - let more_file_paths_ids_with_new_object = assign_existing_objects_to_file_paths( - file_paths_by_cas_id, - &objects_by_cas_id.into_iter().flatten().collect(), - db, - sync, - ) - .await?; - - // Sanity check - assert!( - file_paths_by_cas_id.is_empty(), - "We MUST have processed all pending `file_paths` by now" - ); - - let linked_objects_count = more_file_paths_ids_with_new_object.len() as u64; - - file_paths_with_new_object.extend(more_file_paths_ids_with_new_object); - - Ok((file_paths_with_new_object, linked_objects_count)) -} - -#[derive(Debug, Serialize, Deserialize)] -pub struct SaveState { - id: TaskId, - file_paths_by_cas_id: HashMap, Vec>, - device_id: device::id::Type, - stage: Stage, - output: Output, - with_priority: bool, -} - -impl SerializableTask for ObjectProcessor { - type SerializeError = rmp_serde::encode::Error; - - type DeserializeError = rmp_serde::decode::Error; - - type DeserializeCtx = (Arc, SyncManager); - - async fn serialize(self) -> Result, Self::SerializeError> { - let Self { - id, - file_paths_by_cas_id, - device_id, - stage, - output, - with_priority, - .. - } = self; - - rmp_serde::to_vec_named(&SaveState { - id, - file_paths_by_cas_id, - device_id, - stage, - output, - with_priority, - }) - } - - async fn deserialize( - data: &[u8], - (db, sync): Self::DeserializeCtx, - ) -> Result { - rmp_serde::from_slice(data).map( - |SaveState { - id, - file_paths_by_cas_id, - device_id, - stage, - output, - with_priority, - }| Self { - id, - with_priority, - file_paths_by_cas_id, - stage, - device_id, - output, - db, - sync, - }, - ) - } -} diff --git a/core/crates/heavy-lifting/src/indexer/job.rs b/core/crates/heavy-lifting/src/indexer/job.rs deleted file mode 100644 index cf19fbb90..000000000 --- a/core/crates/heavy-lifting/src/indexer/job.rs +++ /dev/null @@ -1,1163 +0,0 @@ -use crate::{ - indexer, - job_system::{ - job::{ - Job, JobContext, JobName, JobReturn, JobTaskDispatcher, ProgressUpdate, ReturnStatus, - }, - report::ReportOutputMetadata, - utils::cancel_pending_tasks, - DispatcherError, JobErrorOrDispatcherError, SerializableJob, SerializedTasks, - }, - utils::sub_path::get_full_path_from_sub_path, - Error, LocationScanState, NonCriticalError, OuterContext, -}; - -use sd_core_file_path_helper::IsolatedFilePathData; -use sd_core_indexer_rules::{IndexerRule, IndexerRuler}; -use sd_core_prisma_helpers::location_with_indexer_rules; - -use sd_prisma::{ - prisma::{device, location}, - prisma_sync, -}; -use sd_sync::{sync_db_not_null_entry, OperationFactory}; -use sd_task_system::{ - AnyTaskOutput, IntoTask, SerializableTask, Task, TaskDispatcher, TaskHandle, TaskId, - TaskOutput, TaskStatus, -}; -use sd_utils::{db::maybe_missing, u64_to_frontend}; - -use std::{ - collections::{HashMap, HashSet, VecDeque}, - hash::{Hash, Hasher}, - mem, - path::PathBuf, - sync::Arc, - time::Duration, -}; - -use futures::{stream::FuturesUnordered, StreamExt}; -use futures_concurrency::future::TryJoin; -use itertools::Itertools; -use serde::{Deserialize, Serialize}; -use serde_json::json; -use tokio::time::Instant; -use tracing::{debug, instrument, trace, warn, Level}; - -use super::{ - remove_non_existing_file_paths, reverse_update_directories_sizes, - tasks::{ - self, saver, updater, - walker::{self, WalkedEntry}, - }, - update_directory_sizes, update_location_size, IsoFilePathFactory, WalkerDBProxy, BATCH_SIZE, -}; - -#[derive(Debug)] -pub struct Indexer { - // Received arguments - location: location_with_indexer_rules::Data, - sub_path: Option, - - // Derived from received arguments - iso_file_path_factory: IsoFilePathFactory, - indexer_ruler: IndexerRuler, - walker_root_path: Option>, - - // Inner state - ancestors_needing_indexing: HashSet, - ancestors_already_indexed: HashSet>, - iso_paths_and_sizes: HashMap, u64>, - - // Optimizations - processing_first_directory: bool, - to_create_buffer: VecDeque, - to_update_buffer: VecDeque, - - // Run data - metadata: Metadata, - errors: Vec, - - // On shutdown data - pending_tasks_on_resume: Vec>, - tasks_for_shutdown: Vec>>, -} - -impl Job for Indexer { - const NAME: JobName = JobName::Indexer; - - async fn resume_tasks( - &mut self, - dispatcher: &JobTaskDispatcher, - ctx: &impl JobContext, - SerializedTasks(serialized_tasks): SerializedTasks, - ) -> Result<(), Error> { - let location_id = self.location.id; - - if let Ok(tasks) = dispatcher - .dispatch_many_boxed( - rmp_serde::from_slice::)>>(&serialized_tasks) - .map_err(indexer::Error::from)? - .into_iter() - .map(|(task_kind, task_bytes)| { - let indexer_ruler = self.indexer_ruler.clone(); - let iso_file_path_factory = self.iso_file_path_factory.clone(); - async move { - match task_kind { - TaskKind::Walk => tasks::Walker::deserialize( - &task_bytes, - ( - indexer_ruler.clone(), - WalkerDBProxy { - location_id, - db: Arc::clone(ctx.db()), - }, - iso_file_path_factory.clone(), - ), - ) - .await - .map(IntoTask::into_task), - - TaskKind::Save => tasks::Saver::deserialize( - &task_bytes, - (Arc::clone(ctx.db()), ctx.sync().clone()), - ) - .await - .map(IntoTask::into_task), - TaskKind::Update => tasks::Updater::deserialize( - &task_bytes, - (Arc::clone(ctx.db()), ctx.sync().clone()), - ) - .await - .map(IntoTask::into_task), - } - } - }) - .collect::>() - .try_join() - .await - .map_err(indexer::Error::from)?, - ) - .await - { - self.pending_tasks_on_resume = tasks; - } else { - warn!("Failed to dispatch tasks to resume as job was already canceled"); - } - - Ok(()) - } - - #[instrument( - skip_all, - fields( - location_id = self.location.id, - location_path = ?self.location.path, - sub_path = ?self.sub_path.as_ref().map(|path| path.display()), - ), - ret(level = Level::TRACE), - err, - )] - async fn run( - mut self, - dispatcher: JobTaskDispatcher, - ctx: impl JobContext, - ) -> Result { - let mut pending_running_tasks = FuturesUnordered::new(); - - let device_pub_id = &ctx.sync().device_pub_id; - let device_id = ctx - .db() - .device() - .find_unique(device::pub_id::equals(device_pub_id.to_db())) - .exec() - .await - .map_err(indexer::Error::from)? - .ok_or(indexer::Error::DeviceNotFound(device_pub_id.clone()))? - .id; - - match self - .init_or_resume(&mut pending_running_tasks, &ctx, &dispatcher) - .await - { - Ok(()) => { /* Everything is awesome! */ } - Err(JobErrorOrDispatcherError::JobError(e)) => { - return Err(e.into()); - } - Err(JobErrorOrDispatcherError::Dispatcher(DispatcherError::JobCanceled(_))) => { - return Ok(self.cancel_job(&mut pending_running_tasks).await); - } - Err(JobErrorOrDispatcherError::Dispatcher(DispatcherError::Shutdown(tasks))) => { - self.tasks_for_shutdown.extend(tasks); - - if pending_running_tasks.is_empty() { - assert_eq!( - self.tasks_for_shutdown.len() as u64, - self.metadata.total_tasks - self.metadata.completed_tasks, - "Shutting down a job without collecting all pending tasks" - ); - // If no task managed to be dispatched, we can just shutdown - // otherwise we have to process handles below and wait for them to be shutdown too - return Ok(ReturnStatus::Shutdown( - SerializableJob::::serialize(self).await, - )); - } - } - } - - if let Some(res) = self - .process_handles(&mut pending_running_tasks, &ctx, device_id, &dispatcher) - .await - { - return res; - } - - if let Some(res) = self - .dispatch_last_save_and_update_tasks( - &mut pending_running_tasks, - &ctx, - device_id, - &dispatcher, - ) - .await - { - return res; - } - - if let Some(res) = self - .index_pending_ancestors(&mut pending_running_tasks, &ctx, device_id, &dispatcher) - .await - { - return res; - } - - if !self.tasks_for_shutdown.is_empty() { - assert_eq!( - self.tasks_for_shutdown.len() as u64, - self.metadata.total_tasks - self.metadata.completed_tasks, - "Shutting down a job without collecting all pending tasks" - ); - return Ok(ReturnStatus::Shutdown( - SerializableJob::::serialize(self).await, - )); - } - - // From here onward, job will not be interrupted anymore - - let Self { - location, - mut metadata, - iso_file_path_factory, - walker_root_path, - iso_paths_and_sizes, - mut errors, - tasks_for_shutdown, - .. - } = self; - - if metadata.indexed_count > 0 || metadata.removed_count > 0 || metadata.updated_count > 0 { - let start_size_update_time = Instant::now(); - - update_directory_sizes(iso_paths_and_sizes, ctx.db(), ctx.sync()).await?; - - let root_path = walker_root_path.expect("must be set"); - if root_path != iso_file_path_factory.location_path { - reverse_update_directories_sizes( - &*root_path, - location.id, - &*iso_file_path_factory.location_path, - ctx.db(), - ctx.sync(), - &mut errors, - ) - .await?; - } - - update_location_size(location.id, location.pub_id.clone(), &ctx).await?; - - metadata.mean_db_write_time += start_size_update_time.elapsed(); - } - - if metadata.removed_count > 0 { - // TODO: Dispatch a task to remove orphan objects - } - - if metadata.indexed_count > 0 || metadata.removed_count > 0 { - ctx.invalidate_query("search.paths"); - } - - assert!( - tasks_for_shutdown.is_empty(), - "all tasks must be completed here" - ); - - let (sync_param, db_param) = - sync_db_not_null_entry!(LocationScanState::Indexed as i32, location::scan_state); - - ctx.sync() - .write_op( - ctx.db(), - ctx.sync().shared_update( - prisma_sync::location::SyncId { - pub_id: location.pub_id.clone(), - }, - [sync_param], - ), - ctx.db() - .location() - .update(location::id::equals(location.id), vec![db_param]) - .select(location::select!({ id })), - ) - .await - .map_err(indexer::Error::from)?; - - Ok(ReturnStatus::Completed( - JobReturn::builder() - .with_metadata(metadata) - .with_non_critical_errors(errors) - .build(), - )) - } -} - -impl Indexer { - pub fn new( - location: location_with_indexer_rules::Data, - sub_path: Option, - ) -> Result { - Ok(Self { - indexer_ruler: location - .indexer_rules - .iter() - .map(|rule| IndexerRule::try_from(&rule.indexer_rule)) - .collect::, _>>() - .map(IndexerRuler::new)?, - iso_file_path_factory: IsoFilePathFactory { - location_id: location.id, - location_path: maybe_missing(&location.path, "location.path") - .map(PathBuf::from) - .map(Arc::new)?, - }, - walker_root_path: None, - ancestors_needing_indexing: HashSet::new(), - ancestors_already_indexed: HashSet::new(), - iso_paths_and_sizes: HashMap::new(), - location, - sub_path, - metadata: Metadata::default(), - - processing_first_directory: true, - - to_create_buffer: VecDeque::new(), - to_update_buffer: VecDeque::new(), - - errors: Vec::new(), - - pending_tasks_on_resume: Vec::new(), - tasks_for_shutdown: Vec::new(), - }) - } - - /// Process output of tasks, according to the downcasted output type - /// - /// # Panics - /// Will panic if another task type is added in the job, but this function wasn't updated to handle it - /// - async fn process_task_output( - &mut self, - task_id: TaskId, - any_task_output: Box, - ctx: &impl JobContext, - device_id: device::id::Type, - dispatcher: &JobTaskDispatcher, - ) -> Result>, JobErrorOrDispatcherError> { - self.metadata.completed_tasks += 1; - - if any_task_output.is::>() { - return self - .process_walk_output( - *any_task_output - .downcast::>() - .expect("just checked"), - ctx, - device_id, - dispatcher, - ) - .await; - } else if any_task_output.is::() { - self.process_save_output( - *any_task_output - .downcast::() - .expect("just checked"), - ctx, - ) - .await; - } else if any_task_output.is::() { - self.process_update_output( - *any_task_output - .downcast::() - .expect("just checked"), - ctx, - ) - .await; - } else { - unreachable!("Unexpected task output type: "); - } - - Ok(Vec::new()) - } - - #[instrument( - skip_all, - fields( - to_create_count = to_create.len(), - to_update_count = to_update.len(), - to_remove_count = to_remove.len(), - accepted_ancestors_count = accepted_ancestors.len(), - directory_iso_file_path = %directory_iso_file_path.as_ref().display(), - more_walker_tasks_count = keep_walking_tasks.len(), - %total_size, - ?scan_time, - ) - )] - async fn process_walk_output( - &mut self, - walker::Output { - to_create, - to_update, - to_remove, - accepted_ancestors, - errors, - directory_iso_file_path, - total_size, - keep_walking_tasks, - scan_time, - .. - }: walker::Output, - ctx: &impl JobContext, - device_id: device::id::Type, - dispatcher: &JobTaskDispatcher, - ) -> Result>, JobErrorOrDispatcherError> { - self.metadata.mean_scan_read_time += scan_time; - #[allow(clippy::cast_possible_truncation)] - // SAFETY: we know that `keep_walking_tasks.len()` is a valid u32 as we wouldn't dispatch more than `u32::MAX` tasks - { - self.metadata.total_walk_tasks += keep_walking_tasks.len() as u32; - } - - let (to_create_count, to_update_count) = (to_create.len(), to_update.len()); - - *self - .iso_paths_and_sizes - .entry(directory_iso_file_path) - .or_default() += total_size; - - for ancestor_iso_file_path in accepted_ancestors - .iter() - .map(|ancestor_entry| &ancestor_entry.iso_file_path) - { - if self - .iso_paths_and_sizes - .contains_key(ancestor_iso_file_path) - { - *self - .iso_paths_and_sizes - .get_mut(ancestor_iso_file_path) - .expect("we just checked") += total_size; - } else { - self.iso_paths_and_sizes - .insert(ancestor_iso_file_path.clone(), total_size); - } - } - - // First we add ancestors, filtering out ancestors already indexed in previous iterations - self.ancestors_needing_indexing - .extend(accepted_ancestors.into_iter().filter(|ancestor_entry| { - !self - .ancestors_already_indexed - .contains(&ancestor_entry.iso_file_path) - })); - - // Then we add new directories to be indexed as they can be received as ancestors in coming iterations - self.ancestors_already_indexed.extend( - to_create - .iter() - .filter(|&WalkedEntry { iso_file_path, .. }| iso_file_path.is_dir()) - .map(|WalkedEntry { iso_file_path, .. }| iso_file_path.clone()), - ); - - if !errors.is_empty() { - warn!(?errors, "Non critical errors while indexing;"); - self.errors.extend(errors); - } - - if !to_remove.is_empty() { - let db_delete_time = Instant::now(); - self.metadata.removed_count += - remove_non_existing_file_paths(to_remove, ctx.db(), ctx.sync()).await?; - self.metadata.mean_db_write_time += db_delete_time.elapsed(); - } - let (save_tasks, update_tasks) = - self.prepare_save_and_update_tasks(to_create, to_update, ctx, device_id); - - ctx.progress(vec![ - ProgressUpdate::TaskCount(self.metadata.total_tasks), - ProgressUpdate::CompletedTaskCount(self.metadata.completed_tasks), - ProgressUpdate::message(format!( - "Found {to_create_count} new files and {to_update_count} to update" - )), - ]) - .await; - - self.metadata.total_tasks += - (keep_walking_tasks.len() + save_tasks.len() + update_tasks.len()) as u64; - - debug!( - "Dispatching more ({}W/{}S/{}U) tasks, completed ({}/{});", - keep_walking_tasks.len(), - save_tasks.len(), - update_tasks.len(), - self.metadata.completed_tasks, - self.metadata.total_tasks - ); - - dispatcher - .dispatch_many_boxed( - keep_walking_tasks - .into_iter() - .map(IntoTask::into_task) - .chain(save_tasks.into_iter().map(IntoTask::into_task)) - .chain(update_tasks.into_iter().map(IntoTask::into_task)), - ) - .await - .map_err(Into::into) - } - - #[instrument(skip(self, ctx))] - async fn process_save_output( - &mut self, - saver::Output { - saved_count, - save_duration, - }: saver::Output, - ctx: &impl JobContext, - ) { - self.metadata.indexed_count += saved_count; - self.metadata.mean_db_write_time += save_duration; - - ctx.progress(vec![ - ProgressUpdate::CompletedTaskCount(self.metadata.completed_tasks), - ProgressUpdate::message(format!("Saved {} files", self.metadata.indexed_count)), - ]) - .await; - - debug!( - "Processed save task in the indexer ({}/{});", - self.metadata.completed_tasks, self.metadata.total_tasks - ); - } - - #[instrument(skip(self, ctx))] - async fn process_update_output( - &mut self, - updater::Output { - updated_count, - update_duration, - }: updater::Output, - ctx: &impl JobContext, - ) { - self.metadata.updated_count += updated_count; - self.metadata.mean_db_write_time += update_duration; - - ctx.progress(vec![ - ProgressUpdate::CompletedTaskCount(self.metadata.completed_tasks), - ProgressUpdate::message(format!("Updated {} files", self.metadata.updated_count)), - ]) - .await; - - debug!( - "Processed update task in the indexer ({}/{});", - self.metadata.completed_tasks, self.metadata.total_tasks - ); - } - - async fn process_handles( - &mut self, - pending_running_tasks: &mut FuturesUnordered>, - ctx: &impl JobContext, - device_id: device::id::Type, - dispatcher: &JobTaskDispatcher, - ) -> Option> { - while let Some(task) = pending_running_tasks.next().await { - match task { - Ok(TaskStatus::Done((task_id, TaskOutput::Out(out)))) => { - match self - .process_task_output(task_id, out, ctx, device_id, dispatcher) - .await - { - Ok(more_handles) => pending_running_tasks.extend(more_handles), - Err(JobErrorOrDispatcherError::JobError(e)) => { - cancel_pending_tasks(pending_running_tasks).await; - - return Some(Err(e.into())); - } - Err(JobErrorOrDispatcherError::Dispatcher( - DispatcherError::JobCanceled(_), - )) => return Some(Ok(self.cancel_job(pending_running_tasks).await)), - - Err(JobErrorOrDispatcherError::Dispatcher(DispatcherError::Shutdown( - tasks, - ))) => self.tasks_for_shutdown.extend(tasks), - }; - } - - Ok(TaskStatus::Done((task_id, TaskOutput::Empty))) => { - warn!(%task_id, "Task returned an empty output"); - } - - Ok(TaskStatus::Shutdown(task)) => self.tasks_for_shutdown.push(task), - - Ok(TaskStatus::Error(e)) => { - cancel_pending_tasks(pending_running_tasks).await; - - return Some(Err(e)); - } - - Ok(TaskStatus::Canceled | TaskStatus::ForcedAbortion) => { - return Some(Ok(self.cancel_job(pending_running_tasks).await)); - } - - Err(e) => { - cancel_pending_tasks(pending_running_tasks).await; - - return Some(Err(e.into())); - } - } - } - - None - } - - async fn init_or_resume( - &mut self, - pending_running_tasks: &mut FuturesUnordered>, - ctx: &impl JobContext, - dispatcher: &JobTaskDispatcher, - ) -> Result<(), JobErrorOrDispatcherError> { - // if we don't have any pending task, then this is a fresh job - let updates = if self.pending_tasks_on_resume.is_empty() { - let walker_root_path = Arc::new( - get_full_path_from_sub_path::( - self.location.id, - self.sub_path.as_ref(), - &*self.iso_file_path_factory.location_path, - ctx.db(), - ) - .await?, - ); - - pending_running_tasks.push( - dispatcher - .dispatch(tasks::Walker::new_deep( - walker_root_path.as_ref(), - Arc::clone(&walker_root_path), - self.indexer_ruler.clone(), - self.iso_file_path_factory.clone(), - WalkerDBProxy { - location_id: self.location.id, - db: Arc::clone(ctx.db()), - }, - )?) - .await?, - ); - - self.metadata.total_tasks = 1; - self.metadata.total_walk_tasks = 1; - - let updates = vec![ - ProgressUpdate::TaskCount(self.metadata.total_tasks), - ProgressUpdate::Message(format!("Indexing {}", walker_root_path.display())), - ]; - - self.walker_root_path = Some(walker_root_path); - - updates - } else { - pending_running_tasks.extend(mem::take(&mut self.pending_tasks_on_resume)); - - vec![ - ProgressUpdate::TaskCount(self.metadata.total_tasks), - ProgressUpdate::Message("Resuming tasks".to_string()), - ] - }; - - ctx.progress(updates).await; - - Ok(()) - } - - async fn dispatch_last_save_and_update_tasks( - &mut self, - pending_running_tasks: &mut FuturesUnordered>, - ctx: &impl JobContext, - device_id: device::id::Type, - dispatcher: &JobTaskDispatcher, - ) -> Option> { - if !self.to_create_buffer.is_empty() || !self.to_update_buffer.is_empty() { - let mut tasks = Vec::with_capacity(2); - - if !self.to_create_buffer.is_empty() { - assert!( - self.to_create_buffer.len() <= BATCH_SIZE, - "last save task must be less than BATCH_SIZE paths" - ); - - self.metadata.total_tasks += 1; - self.metadata.total_paths += self.to_create_buffer.len() as u64; - self.metadata.total_save_tasks += 1; - - tasks.push( - tasks::Saver::new_deep( - self.location.id, - self.location.pub_id.clone(), - self.to_create_buffer.drain(..).collect(), - Arc::clone(ctx.db()), - ctx.sync().clone(), - device_id, - ) - .into_task(), - ); - } - - if !self.to_update_buffer.is_empty() { - assert!( - self.to_update_buffer.len() <= BATCH_SIZE, - "last update task must be less than BATCH_SIZE paths" - ); - - self.metadata.total_tasks += 1; - self.metadata.total_updated_paths += self.to_update_buffer.len() as u64; - self.metadata.total_update_tasks += 1; - - tasks.push( - tasks::Updater::new_deep( - self.to_update_buffer.drain(..).collect(), - Arc::clone(ctx.db()), - ctx.sync().clone(), - ) - .into_task(), - ); - } - - ctx.progress(vec![ProgressUpdate::TaskCount(self.metadata.total_tasks)]) - .await; - - match dispatcher.dispatch_many_boxed(tasks).await { - Ok(task_handles) => pending_running_tasks.extend(task_handles), - Err(DispatcherError::JobCanceled(_)) => { - return Some(Ok(self.cancel_job(pending_running_tasks).await)); - } - Err(DispatcherError::Shutdown(tasks)) => { - self.tasks_for_shutdown.extend(tasks); - } - } - - self.process_handles(pending_running_tasks, ctx, device_id, dispatcher) - .await - } else { - None - } - } - - async fn index_pending_ancestors( - &mut self, - pending_running_tasks: &mut FuturesUnordered>, - ctx: &impl JobContext, - device_id: device::id::Type, - dispatcher: &JobTaskDispatcher, - ) -> Option> { - if self.ancestors_needing_indexing.is_empty() { - return None; - } - - let save_tasks = self - .ancestors_needing_indexing - .drain() - .chunks(BATCH_SIZE) - .into_iter() - .map(|chunk| { - let chunked_saves = chunk.collect::>(); - - self.metadata.total_paths += chunked_saves.len() as u64; - self.metadata.total_save_tasks += 1; - - tasks::Saver::new_deep( - self.location.id, - self.location.pub_id.clone(), - chunked_saves, - Arc::clone(ctx.db()), - ctx.sync().clone(), - device_id, - ) - }) - .collect::>(); - - self.metadata.total_tasks += save_tasks.len() as u64; - - match dispatcher.dispatch_many(save_tasks).await { - Ok(task_handles) => pending_running_tasks.extend(task_handles), - Err(DispatcherError::JobCanceled(_)) => { - return Some(Ok(self.cancel_job(pending_running_tasks).await)); - } - Err(DispatcherError::Shutdown(tasks)) => { - self.tasks_for_shutdown.extend(tasks); - } - } - - self.process_handles(pending_running_tasks, ctx, device_id, dispatcher) - .await - } - - fn prepare_save_and_update_tasks( - &mut self, - to_create: Vec, - to_update: Vec, - ctx: &impl JobContext, - device_id: device::id::Type, - ) -> (Vec, Vec) { - if self.processing_first_directory { - // If we are processing the first directory, we dispatch shallow tasks with higher priority - // this way we provide a faster feedback loop to the user - self.processing_first_directory = false; - - let save_tasks = to_create - .into_iter() - .chunks(BATCH_SIZE) - .into_iter() - .map(|chunk| { - let chunked_saves = chunk.collect::>(); - - self.metadata.total_paths += chunked_saves.len() as u64; - self.metadata.total_save_tasks += 1; - - tasks::Saver::new_shallow( - self.location.id, - self.location.pub_id.clone(), - chunked_saves, - Arc::clone(ctx.db()), - ctx.sync().clone(), - device_id, - ) - }) - .collect::>(); - - let update_tasks = to_update - .into_iter() - .chunks(BATCH_SIZE) - .into_iter() - .map(|chunk| { - let chunked_updates = chunk.collect::>(); - - self.metadata.total_updated_paths += chunked_updates.len() as u64; - self.metadata.total_update_tasks += 1; - - tasks::Updater::new_shallow( - chunked_updates, - Arc::clone(ctx.db()), - ctx.sync().clone(), - ) - }) - .collect::>(); - - (save_tasks, update_tasks) - } else { - self.to_create_buffer.extend(to_create); - - let save_tasks = if self.to_create_buffer.len() > BATCH_SIZE { - let chunks_count = self.to_create_buffer.len() / BATCH_SIZE; - let mut save_tasks = Vec::with_capacity(chunks_count); - - for _ in 0..chunks_count { - let chunked_saves = self - .to_create_buffer - .drain(..BATCH_SIZE) - .collect::>(); - - self.metadata.total_paths += chunked_saves.len() as u64; - self.metadata.total_save_tasks += 1; - - save_tasks.push(tasks::Saver::new_deep( - self.location.id, - self.location.pub_id.clone(), - chunked_saves, - Arc::clone(ctx.db()), - ctx.sync().clone(), - device_id, - )); - } - save_tasks - } else { - trace!("Not enough entries to dispatch a new saver task;"); - vec![] - }; - - self.to_update_buffer.extend(to_update); - - let update_tasks = if self.to_update_buffer.len() > BATCH_SIZE { - let chunks_count = self.to_update_buffer.len() / BATCH_SIZE; - let mut update_tasks = Vec::with_capacity(chunks_count); - - for _ in 0..chunks_count { - let chunked_updates = self - .to_update_buffer - .drain(..BATCH_SIZE) - .collect::>(); - - self.metadata.total_updated_paths += chunked_updates.len() as u64; - self.metadata.total_update_tasks += 1; - - update_tasks.push(tasks::Updater::new_deep( - chunked_updates, - Arc::clone(ctx.db()), - ctx.sync().clone(), - )); - } - update_tasks - } else { - trace!("Not enough entries to dispatch a new updater task;"); - vec![] - }; - - (save_tasks, update_tasks) - } - } - - async fn cancel_job( - &mut self, - pending_running_tasks: &mut FuturesUnordered>, - ) -> ReturnStatus { - cancel_pending_tasks(pending_running_tasks).await; - - ReturnStatus::Canceled( - JobReturn::builder() - .with_metadata(mem::take(&mut self.metadata)) - .with_non_critical_errors(mem::take(&mut self.errors)) - .build(), - ) - } -} - -#[derive(Debug, Clone, Serialize, Deserialize, Default)] -pub struct Metadata { - mean_db_write_time: Duration, - mean_scan_read_time: Duration, - total_tasks: u64, - completed_tasks: u64, - total_paths: u64, - total_updated_paths: u64, - total_walk_tasks: u32, - total_save_tasks: u32, - total_update_tasks: u32, - indexed_count: u64, - updated_count: u64, - removed_count: u64, -} - -impl From for Vec { - fn from( - Metadata { - mut mean_db_write_time, - mut mean_scan_read_time, - total_tasks, - completed_tasks, - total_paths, - total_updated_paths, - total_walk_tasks, - total_save_tasks, - total_update_tasks, - indexed_count, - updated_count, - removed_count, - }: Metadata, - ) -> Self { - mean_scan_read_time /= u32::max(total_walk_tasks, 1); // To avoid division by zero - mean_db_write_time /= total_save_tasks + total_update_tasks + 1; // +1 to update directories sizes - - vec![ - ReportOutputMetadata::Indexer { - total_paths: u64_to_frontend(total_paths), - }, - ReportOutputMetadata::Metrics(HashMap::from([ - ("mean_scan_read_time".into(), json!(mean_scan_read_time)), - ("mean_db_write_time".into(), json!(mean_db_write_time)), - ("total_tasks".into(), json!(total_tasks)), - ("completed_tasks".into(), json!(completed_tasks)), - ("total_paths".into(), json!(total_paths)), - ("total_updated_paths".into(), json!(total_updated_paths)), - ("total_walk_tasks".into(), json!(total_walk_tasks)), - ("total_save_tasks".into(), json!(total_save_tasks)), - ("total_update_tasks".into(), json!(total_update_tasks)), - ("indexed_count".into(), json!(indexed_count)), - ("updated_count".into(), json!(updated_count)), - ("removed_count".into(), json!(removed_count)), - ])), - ] - } -} - -#[derive(Debug, Clone, Copy, Serialize, Deserialize)] -enum TaskKind { - Walk, - Save, - Update, -} - -#[derive(Serialize, Deserialize)] -struct SaveState { - location: location_with_indexer_rules::Data, - sub_path: Option, - - iso_file_path_factory: IsoFilePathFactory, - indexer_ruler: IndexerRuler, - walker_root_path: Option>, - - ancestors_needing_indexing: HashSet, - ancestors_already_indexed: HashSet>, - iso_paths_and_sizes: HashMap, u64>, - - processing_first_directory: bool, - to_create_buffer: VecDeque, - to_update_buffer: VecDeque, - - metadata: Metadata, - errors: Vec, - - tasks_for_shutdown_bytes: Option, -} - -impl SerializableJob for Indexer { - async fn serialize(self) -> Result>, rmp_serde::encode::Error> { - let Self { - location, - sub_path, - metadata, - iso_file_path_factory, - indexer_ruler, - walker_root_path, - ancestors_needing_indexing, - ancestors_already_indexed, - iso_paths_and_sizes, - processing_first_directory, - to_create_buffer, - to_update_buffer, - errors, - tasks_for_shutdown, - .. - } = self; - - let serialized_tasks = tasks_for_shutdown - .into_iter() - .map(|task| async move { - if task.is::>() { - task.downcast::>() - .expect("just checked") - .serialize() - .await - .map(|bytes| (TaskKind::Walk, bytes)) - } else if task.is::() { - task.downcast::() - .expect("just checked") - .serialize() - .await - .map(|bytes| (TaskKind::Save, bytes)) - } else if task.is::() { - task.downcast::() - .expect("just checked") - .serialize() - .await - .map(|bytes| (TaskKind::Update, bytes)) - } else { - unreachable!("Unexpected task type") - } - }) - .collect::>() - .try_join() - .await?; - - let tasks_for_shutdown_bytes = if serialized_tasks.is_empty() { - None - } else { - Some(SerializedTasks(rmp_serde::to_vec_named(&serialized_tasks)?)) - }; - - rmp_serde::to_vec_named(&SaveState { - location, - sub_path, - iso_file_path_factory, - indexer_ruler, - walker_root_path, - ancestors_needing_indexing, - ancestors_already_indexed, - iso_paths_and_sizes, - processing_first_directory, - to_create_buffer, - to_update_buffer, - metadata, - errors, - tasks_for_shutdown_bytes, - }) - .map(Some) - } - - async fn deserialize( - serialized_job: &[u8], - _: &OuterCtx, - ) -> Result)>, rmp_serde::decode::Error> { - let SaveState { - location, - sub_path, - iso_file_path_factory, - indexer_ruler, - walker_root_path, - ancestors_needing_indexing, - ancestors_already_indexed, - iso_paths_and_sizes, - processing_first_directory, - to_create_buffer, - to_update_buffer, - metadata, - errors, - tasks_for_shutdown_bytes, - } = rmp_serde::from_slice::(serialized_job)?; - - Ok(Some(( - Self { - location, - sub_path, - metadata, - iso_file_path_factory, - indexer_ruler, - walker_root_path, - ancestors_needing_indexing, - ancestors_already_indexed, - iso_paths_and_sizes, - processing_first_directory, - to_create_buffer, - to_update_buffer, - errors, - pending_tasks_on_resume: Vec::new(), - tasks_for_shutdown: Vec::new(), - }, - tasks_for_shutdown_bytes, - ))) - } -} - -impl Hash for Indexer { - fn hash(&self, state: &mut H) { - self.location.id.hash(state); - if let Some(ref sub_path) = self.sub_path { - sub_path.hash(state); - } - } -} diff --git a/core/crates/heavy-lifting/src/indexer/mod.rs b/core/crates/heavy-lifting/src/indexer/mod.rs deleted file mode 100644 index 35801617a..000000000 --- a/core/crates/heavy-lifting/src/indexer/mod.rs +++ /dev/null @@ -1,566 +0,0 @@ -use crate::{utils::sub_path, OuterContext}; - -use sd_core_file_path_helper::{FilePathError, IsolatedFilePathData}; -use sd_core_prisma_helpers::{ - file_path_pub_and_cas_ids, file_path_to_isolate_with_pub_id, file_path_walker, -}; -use sd_core_sync::{DevicePubId, SyncManager}; - -use sd_prisma::{ - prisma::{file_path, indexer_rule, location, PrismaClient, SortOrder}, - prisma_sync, -}; -use sd_sync::{sync_db_entry, OperationFactory}; -use sd_utils::{ - db::{size_in_bytes_from_db, size_in_bytes_to_db, MissingFieldError}, - error::{FileIOError, NonUtf8PathError}, - from_bytes_to_uuid, -}; - -use std::{ - collections::{HashMap, HashSet}, - hash::BuildHasher, - mem, - path::{Path, PathBuf}, - sync::Arc, -}; - -use itertools::Itertools; -use prisma_client_rust::{operator::or, QueryError, Select}; -use rspc::ErrorCode; -use serde::{Deserialize, Serialize}; -use specta::Type; -use tracing::{instrument, warn}; - -pub mod job; -mod shallow; -mod tasks; - -pub use shallow::shallow; - -use tasks::walker; - -/// `BATCH_SIZE` is the number of files to index at each task, writing the chunk of files metadata in the database. -const BATCH_SIZE: usize = 1000; - -#[derive(thiserror::Error, Debug)] -pub enum Error { - // Not Found errors - #[error("indexer rule not found: ")] - IndexerRuleNotFound(indexer_rule::id::Type), - #[error(transparent)] - SubPath(#[from] sub_path::Error), - #[error("device not found: for rspc::Error { - fn from(e: Error) -> Self { - match e { - Error::IndexerRuleNotFound(_) => { - Self::with_cause(ErrorCode::NotFound, e.to_string(), e) - } - - Error::SubPath(sub_path_err) => sub_path_err.into(), - - Error::Rules(rule_err) => rule_err.into(), - - _ => Self::with_cause(ErrorCode::InternalServerError, e.to_string(), e), - } - } -} - -#[derive(thiserror::Error, Debug, Serialize, Deserialize, Type, Clone)] -#[serde(rename_all = "snake_case")] -pub enum NonCriticalIndexerError { - #[error("failed to read directory entry: {0}")] - FailedDirectoryEntry(String), - #[error("failed to fetch metadata: {0}")] - Metadata(String), - #[error("error applying indexer rule: {0}")] - IndexerRule(String), - #[error("error trying to extract file path metadata from a file: {0}")] - FilePathMetadata(String), - #[error("failed to fetch file paths ids from existing files on database: {0}")] - FetchAlreadyExistingFilePathIds(String), - #[error("failed to fetch file paths to be removed from database: {0}")] - FetchFilePathsToRemove(String), - #[error("error constructing isolated file path: {0}")] - IsoFilePath(String), - #[error("failed to dispatch new task to keep walking a directory: {0}")] - DispatchKeepWalking(String), - #[error("missing file_path data on database: {0}")] - MissingFilePathData(String), -} - -fn chunk_db_queries<'db, 'iso>( - iso_file_paths: impl IntoIterator>, - db: &'db PrismaClient, -) -> Vec>> { - iso_file_paths - .into_iter() - .chunks(200) - .into_iter() - .map(|paths_chunk| { - db.file_path() - .find_many(vec![or(paths_chunk - .into_iter() - .map(file_path::WhereParam::from) - .collect())]) - .select(file_path_to_isolate_with_pub_id::select()) - }) - .collect::>() -} - -#[allow(clippy::missing_panics_doc)] // Can't actually panic as we use the hashmap to fetch entries from db -async fn update_directory_sizes( - iso_paths_and_sizes: HashMap, u64, impl BuildHasher + Send>, - db: &PrismaClient, - sync: &SyncManager, -) -> Result<(), Error> { - let (ops, queries) = db - ._batch(chunk_db_queries(iso_paths_and_sizes.keys(), db)) - .await? - .into_iter() - .flatten() - .map(|file_path| { - let size_bytes = iso_paths_and_sizes - .get(&IsolatedFilePathData::try_from(&file_path)?) - .map(|size| size_in_bytes_to_db(*size)) - .expect("must be here"); - - let (sync_param, db_param) = sync_db_entry!(size_bytes, file_path::size_in_bytes_bytes); - - Ok(( - sync.shared_update( - prisma_sync::file_path::SyncId { - pub_id: file_path.pub_id.clone(), - }, - [sync_param], - ), - db.file_path() - .update(file_path::pub_id::equals(file_path.pub_id), vec![db_param]) - .select(file_path::select!({ id })), - )) - }) - .collect::, Error>>()? - .into_iter() - .unzip::<_, _, Vec<_>, Vec<_>>(); - - if !ops.is_empty() && !queries.is_empty() { - sync.write_ops(db, (ops, queries)).await?; - } - - Ok(()) -} - -async fn update_location_size( - location_id: location::id::Type, - location_pub_id: location::pub_id::Type, - ctx: &impl OuterContext, -) -> Result<(), Error> { - let db = ctx.db(); - let sync = ctx.sync(); - - let total_size = size_in_bytes_to_db( - db.file_path() - .find_many(vec![ - file_path::location_id::equals(Some(location_id)), - file_path::materialized_path::equals(Some("/".to_string())), - ]) - .select(file_path::select!({ size_in_bytes_bytes })) - .exec() - .await? - .into_iter() - .filter_map(|file_path| { - file_path - .size_in_bytes_bytes - .map(|size_in_bytes_bytes| size_in_bytes_from_db(&size_in_bytes_bytes)) - }) - .sum::(), - ); - - let (sync_param, db_param) = sync_db_entry!(total_size, location::size_in_bytes); - - sync.write_op( - db, - sync.shared_update( - prisma_sync::location::SyncId { - pub_id: location_pub_id, - }, - [sync_param], - ), - db.location() - .update(location::id::equals(location_id), vec![db_param]) - .select(location::select!({ id })), - ) - .await?; - - ctx.invalidate_query("locations.list"); - ctx.invalidate_query("locations.get"); - - Ok(()) -} - -async fn remove_non_existing_file_paths( - to_remove: Vec, - db: &PrismaClient, - sync: &SyncManager, -) -> Result { - #[allow(clippy::cast_sign_loss)] - let (sync_params, db_params): (Vec<_>, Vec<_>) = to_remove - .into_iter() - .map(|file_path| { - ( - sync.shared_delete(prisma_sync::file_path::SyncId { - pub_id: file_path.pub_id, - }), - file_path.id, - ) - }) - .unzip(); - - if sync_params.is_empty() { - return Ok(0); - } - - sync.write_ops( - db, - ( - sync_params, - db.file_path() - .delete_many(vec![file_path::id::in_vec(db_params)]), - ), - ) - .await - .map( - #[allow(clippy::cast_sign_loss)] - |count| count as u64, - ) - .map_err(Into::into) -} - -#[instrument( - skip(base_path, location_path, db, sync, errors), - fields( - base_path = %base_path.as_ref().display(), - location_path = %location_path.as_ref().display(), - ), - err, -)] -#[allow(clippy::missing_panics_doc)] // Can't actually panic as we only deal with directories -pub async fn reverse_update_directories_sizes( - base_path: impl AsRef + Send, - location_id: location::id::Type, - location_path: impl AsRef + Send, - db: &PrismaClient, - sync: &SyncManager, - errors: &mut Vec, -) -> Result<(), Error> { - let location_path = location_path.as_ref(); - - let ancestors = base_path - .as_ref() - .ancestors() - .take_while(|&ancestor| ancestor != location_path) - .map(|ancestor| { - IsolatedFilePathData::new(location_id, location_path, ancestor, true).map( - |iso_file_path| { - let materialized_path = iso_file_path - .materialized_path_for_children() - .expect("each ancestor is a directory"); - - (iso_file_path, materialized_path) - }, - ) - }) - .collect::, _>>()?; - - let mut pub_id_by_ancestor_materialized_path = db - ._batch(chunk_db_queries(ancestors.keys(), db)) - .await? - .into_iter() - .flatten() - .filter_map(|mut file_path| { - let pub_id = mem::take(&mut file_path.pub_id); - IsolatedFilePathData::try_from(file_path) - .map_err(|e| { - errors.push( - NonCriticalIndexerError::MissingFilePathData(format!( - "Found a file_path missing data: , error: {e:#?}", - from_bytes_to_uuid(&pub_id) - )) - .into(), - ); - }) - .map(|iso_file_path| { - ( - iso_file_path - .materialized_path_for_children() - .expect("we know it's a directory"), - (pub_id, 0), - ) - }) - .ok() - }) - .collect::>(); - - compute_sizes( - location_id, - ancestors.values().cloned().collect(), - &mut pub_id_by_ancestor_materialized_path, - db, - errors, - ) - .await?; - - let (sync_ops, update_queries) = ancestors - .into_values() - .filter_map(|materialized_path| { - if let Some((pub_id, size)) = - pub_id_by_ancestor_materialized_path.remove(&materialized_path) - { - let size_bytes = size_in_bytes_to_db(size); - - let (sync_param, db_param) = - sync_db_entry!(size_bytes, file_path::size_in_bytes_bytes); - - Some(( - sync.shared_update( - prisma_sync::file_path::SyncId { - pub_id: pub_id.clone(), - }, - [sync_param], - ), - db.file_path() - .update(file_path::pub_id::equals(pub_id), vec![db_param]) - .select(file_path::select!({ id })), - )) - } else { - warn!("Got a missing ancestor for a file_path in the database, ignoring..."); - None - } - }) - .unzip::<_, _, Vec<_>, Vec<_>>(); - - if !sync_ops.is_empty() && !update_queries.is_empty() { - sync.write_ops(db, (sync_ops, update_queries)).await?; - } - - Ok(()) -} - -async fn compute_sizes( - location_id: location::id::Type, - materialized_paths: Vec, - pub_id_by_ancestor_materialized_path: &mut HashMap, - db: &PrismaClient, - errors: &mut Vec, -) -> Result<(), QueryError> { - for file_path in db - .file_path() - .find_many(vec![ - file_path::location_id::equals(Some(location_id)), - file_path::materialized_path::in_vec(materialized_paths), - ]) - .select(file_path::select!({ pub_id materialized_path size_in_bytes_bytes })) - .exec() - .await? - { - if let Some(materialized_path) = file_path.materialized_path { - if let Some((_, size)) = - pub_id_by_ancestor_materialized_path.get_mut(&materialized_path) - { - *size += file_path.size_in_bytes_bytes.map_or_else( - || { - warn!("Got a directory missing its size in bytes"); - 0 - }, - |size_in_bytes_bytes| size_in_bytes_from_db(&size_in_bytes_bytes), - ); - } - } else { - errors.push( - NonCriticalIndexerError::MissingFilePathData(format!( - "Corrupt database possessing a file_path entry without materialized_path: ", - from_bytes_to_uuid(&file_path.pub_id) - )) - .into(), - ); - } - } - - Ok(()) -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -struct IsoFilePathFactory { - pub location_id: location::id::Type, - pub location_path: Arc, -} - -impl walker::IsoFilePathFactory for IsoFilePathFactory { - fn build( - &self, - path: impl AsRef, - is_dir: bool, - ) -> Result, FilePathError> { - IsolatedFilePathData::new(self.location_id, self.location_path.as_ref(), path, is_dir) - } -} - -#[derive(Debug, Clone)] -struct WalkerDBProxy { - location_id: location::id::Type, - db: Arc, -} - -impl walker::WalkerDBProxy for WalkerDBProxy { - async fn fetch_file_paths( - &self, - found_paths: Vec, - ) -> Result, Error> { - // Each found path is a AND with 4 terms, and SQLite has a expression tree limit of 1000 terms - // so we will use chunks of 200 just to be safe - self.db - ._batch( - found_paths - .into_iter() - .chunks(200) - .into_iter() - .map(|founds| { - self.db - .file_path() - .find_many(vec![or(founds.collect::>())]) - .select(file_path_walker::select()) - }) - .collect::>(), - ) - .await - .map(|fetched| fetched.into_iter().flatten().collect::>()) - .map_err(Into::into) - } - - async fn fetch_file_paths_to_remove( - &self, - parent_iso_file_path: &IsolatedFilePathData<'_>, - mut existing_inodes: HashSet>, - unique_location_id_materialized_path_name_extension_params: Vec, - ) -> Result, NonCriticalIndexerError> { - // NOTE: This batch size can be increased if we wish to trade memory for more performance - const BATCH_SIZE: i64 = 1000; - - let founds_ids = { - let found_chunks = self - .db - ._batch( - unique_location_id_materialized_path_name_extension_params - .into_iter() - .chunks(200) - .into_iter() - .map(|unique_params| { - self.db - .file_path() - .find_many(vec![or(unique_params.collect())]) - .select(file_path::select!({ id inode })) - }) - .collect::>(), - ) - .await - .map_err(|e| { - NonCriticalIndexerError::FetchAlreadyExistingFilePathIds(e.to_string()) - })?; - - found_chunks - .into_iter() - .flatten() - .map(|file_path| { - if let Some(inode) = file_path.inode { - existing_inodes.remove(&inode); - } - file_path.id - }) - .collect::>() - }; - - let mut to_remove = vec![]; - let mut cursor = 1; - - loop { - let materialized_path_param = file_path::materialized_path::equals(Some( - parent_iso_file_path - .materialized_path_for_children() - .expect("the received isolated file path must be from a directory"), - )); - - let found = self - .db - .file_path() - .find_many(vec![ - file_path::location_id::equals(Some(self.location_id)), - file_path::id::gte(cursor), - if existing_inodes.is_empty() { - materialized_path_param - } else { - or(vec![ - materialized_path_param, - file_path::inode::in_vec(existing_inodes.iter().cloned().collect()), - ]) - }, - ]) - .order_by(file_path::id::order(SortOrder::Asc)) - .take(BATCH_SIZE) - .select(file_path::select!({ id pub_id cas_id inode })) - .exec() - .await - .map_err(|e| NonCriticalIndexerError::FetchFilePathsToRemove(e.to_string()))?; - - #[allow(clippy::cast_possible_truncation)] // Safe because we are using a constant - let should_stop = found.len() < BATCH_SIZE as usize; - - if let Some(last) = found.last() { - cursor = last.id; - } else { - break; - } - - to_remove.extend(found.into_iter().filter_map(|file_path| { - if let Some(inode) = file_path.inode { - existing_inodes.remove(&inode); - } - - (!founds_ids.contains(&file_path.id)).then_some(file_path_pub_and_cas_ids::Data { - id: file_path.id, - pub_id: file_path.pub_id, - cas_id: file_path.cas_id, - }) - })); - - if should_stop { - break; - } - } - - Ok(to_remove) - } -} diff --git a/core/crates/heavy-lifting/src/indexer/shallow.rs b/core/crates/heavy-lifting/src/indexer/shallow.rs deleted file mode 100644 index 1bc55b556..000000000 --- a/core/crates/heavy-lifting/src/indexer/shallow.rs +++ /dev/null @@ -1,303 +0,0 @@ -use crate::{ - indexer, utils::sub_path::get_full_path_from_sub_path, Error, NonCriticalError, OuterContext, -}; - -use sd_core_indexer_rules::{IndexerRule, IndexerRuler}; -use sd_core_prisma_helpers::location_with_indexer_rules; -use sd_core_sync::SyncManager; - -use sd_prisma::prisma::{device, PrismaClient}; -use sd_task_system::{BaseTaskDispatcher, CancelTaskOnDrop, IntoTask, TaskDispatcher, TaskOutput}; -use sd_utils::db::maybe_missing; - -use std::{ - collections::HashMap, - path::{Path, PathBuf}, - sync::Arc, -}; - -use futures_concurrency::future::TryJoin; -use itertools::Itertools; -use tracing::{debug, instrument, warn}; - -use super::{ - remove_non_existing_file_paths, reverse_update_directories_sizes, - tasks::{ - self, saver, updater, - walker::{self, ToWalkEntry, WalkedEntry}, - }, - update_directory_sizes, update_location_size, IsoFilePathFactory, WalkerDBProxy, BATCH_SIZE, -}; - -#[instrument( - skip_all, - fields( - location_id = location.id, - location_path = ?location.path, - sub_path = %sub_path.as_ref().display() - ) - err, -)] -pub async fn shallow( - location: location_with_indexer_rules::Data, - sub_path: impl AsRef + Send, - dispatcher: &BaseTaskDispatcher, - ctx: &impl OuterContext, -) -> Result, Error> { - let db = ctx.db(); - let sync = ctx.sync(); - - let location_path = maybe_missing(&location.path, "location.path") - .map(PathBuf::from) - .map(Arc::new) - .map_err(indexer::Error::from)?; - - let to_walk_path = Arc::new( - get_full_path_from_sub_path::( - location.id, - Some(sub_path.as_ref()), - &*location_path, - db, - ) - .await?, - ); - - let device_pub_id = &ctx.sync().device_pub_id; - let device_id = ctx - .db() - .device() - .find_unique(device::pub_id::equals(device_pub_id.to_db())) - .exec() - .await - .map_err(indexer::Error::from)? - .ok_or(indexer::Error::DeviceNotFound(device_pub_id.clone()))? - .id; - - let Some(walker::Output { - to_create, - to_update, - to_remove, - non_indexed_paths, - mut errors, - directory_iso_file_path, - total_size, - .. - }) = walk( - &location, - Arc::clone(&location_path), - Arc::clone(&to_walk_path), - Arc::clone(db), - dispatcher, - ) - .await? - else { - return Ok(vec![]); - }; - - // TODO use non_indexed_paths here in the future, sending it to frontend, showing then alongside the indexed files from db - debug!(non_indexed_paths_count = non_indexed_paths.len()); - - let removed_count = remove_non_existing_file_paths(to_remove, db, sync).await?; - - let Some(Metadata { - indexed_count, - updated_count, - }) = save_and_update( - &location, - to_create, - to_update, - Arc::clone(db), - sync.clone(), - device_id, - dispatcher, - ) - .await? - else { - return Ok(errors); - }; - - if indexed_count > 0 || removed_count > 0 || updated_count > 0 { - update_directory_sizes( - HashMap::from([(directory_iso_file_path, total_size)]), - db, - sync, - ) - .await?; - - if to_walk_path != location_path { - reverse_update_directories_sizes( - &*to_walk_path, - location.id, - &*location_path, - db, - sync, - &mut errors, - ) - .await?; - } - - update_location_size(location.id, location.pub_id, ctx).await?; - } - - if indexed_count > 0 || removed_count > 0 { - ctx.invalidate_query("search.paths"); - } - - Ok(errors) -} - -#[instrument( - skip_all, - fields(to_walk_path = %to_walk_path.display()) -)] -async fn walk( - location: &location_with_indexer_rules::Data, - location_path: Arc, - to_walk_path: Arc, - db: Arc, - dispatcher: &BaseTaskDispatcher, -) -> Result>, Error> { - let Ok(task_handle) = dispatcher - .dispatch(tasks::Walker::new_shallow( - ToWalkEntry::from(&*to_walk_path), - to_walk_path, - location - .indexer_rules - .iter() - .map(|rule| IndexerRule::try_from(&rule.indexer_rule)) - .collect::, _>>() - .map(IndexerRuler::new) - .map_err(indexer::Error::from)?, - IsoFilePathFactory { - location_id: location.id, - location_path, - }, - WalkerDBProxy { - location_id: location.id, - db, - }, - )?) - .await - else { - debug!("Task system is shutting down while a shallow indexer was in progress"); - return Ok(None); - }; - - match task_handle.await? { - sd_task_system::TaskStatus::Done((_, TaskOutput::Out(data))) => Ok(Some( - *data - .downcast::>() - .expect("we just dispatched this task"), - )), - sd_task_system::TaskStatus::Done((_, TaskOutput::Empty)) => { - warn!("Shallow indexer's walker task finished without any output"); - Ok(None) - } - sd_task_system::TaskStatus::Error(e) => Err(e), - - sd_task_system::TaskStatus::Shutdown(_) => { - debug!("Spacedrive is shuting down while a shallow indexer was in progress"); - Ok(None) - } - sd_task_system::TaskStatus::Canceled | sd_task_system::TaskStatus::ForcedAbortion => { - unreachable!("WalkDirTask on shallow indexer can never be canceled or aborted") - } - } -} - -struct Metadata { - indexed_count: u64, - updated_count: u64, -} - -async fn save_and_update( - location: &location_with_indexer_rules::Data, - to_create: Vec, - to_update: Vec, - db: Arc, - sync: SyncManager, - device_id: device::id::Type, - dispatcher: &BaseTaskDispatcher, -) -> Result, Error> { - let save_and_update_tasks = to_create - .into_iter() - .chunks(BATCH_SIZE) - .into_iter() - .map(|chunk| { - tasks::Saver::new_shallow( - location.id, - location.pub_id.clone(), - chunk.collect::>(), - Arc::clone(&db), - sync.clone(), - device_id, - ) - }) - .map(IntoTask::into_task) - .chain( - to_update - .into_iter() - .chunks(BATCH_SIZE) - .into_iter() - .map(|chunk| { - tasks::Updater::new_shallow( - chunk.collect::>(), - Arc::clone(&db), - sync.clone(), - ) - }) - .map(IntoTask::into_task), - ) - .collect::>(); - - let mut metadata = Metadata { - indexed_count: 0, - updated_count: 0, - }; - - let Ok(tasks_handles) = dispatcher.dispatch_many_boxed(save_and_update_tasks).await else { - debug!("Task system is shutting down while a shallow indexer was in progress"); - return Ok(None); - }; - - for task_status in tasks_handles - .into_iter() - .map(CancelTaskOnDrop::new) - .collect::>() - .try_join() - .await? - { - match task_status { - sd_task_system::TaskStatus::Done((_, TaskOutput::Out(data))) => { - if data.is::() { - metadata.indexed_count += data - .downcast::() - .expect("just checked") - .saved_count; - } else { - metadata.updated_count += data - .downcast::() - .expect("just checked") - .updated_count; - } - } - sd_task_system::TaskStatus::Done((_, TaskOutput::Empty)) => { - warn!("Shallow indexer's saver or updater task finished without any output"); - return Ok(None); - } - sd_task_system::TaskStatus::Error(e) => return Err(e), - - sd_task_system::TaskStatus::Shutdown(_) => { - debug!("Spacedrive is shuting down while a shallow indexer was in progress"); - return Ok(None); - } - sd_task_system::TaskStatus::Canceled | sd_task_system::TaskStatus::ForcedAbortion => { - unreachable!( - "Save or Updater tasks on shallow indexer can never be canceled or aborted" - ); - } - } - } - - Ok(Some(metadata)) -} diff --git a/core/crates/heavy-lifting/src/indexer/tasks/mod.rs b/core/crates/heavy-lifting/src/indexer/tasks/mod.rs deleted file mode 100644 index 0d2bf5f18..000000000 --- a/core/crates/heavy-lifting/src/indexer/tasks/mod.rs +++ /dev/null @@ -1,7 +0,0 @@ -pub mod saver; -pub mod updater; -pub mod walker; - -pub use saver::Saver; -pub use updater::Updater; -pub use walker::Walker; diff --git a/core/crates/heavy-lifting/src/indexer/tasks/saver.rs b/core/crates/heavy-lifting/src/indexer/tasks/saver.rs deleted file mode 100644 index c5d0951d0..000000000 --- a/core/crates/heavy-lifting/src/indexer/tasks/saver.rs +++ /dev/null @@ -1,309 +0,0 @@ -use crate::{indexer, Error}; - -use sd_core_file_path_helper::{FilePathMetadata, IsolatedFilePathDataParts}; -use sd_core_sync::SyncManager; - -use sd_prisma::{ - prisma::{device, file_path, location, PrismaClient}, - prisma_sync, -}; -use sd_sync::{sync_db_entry, sync_entry, OperationFactory}; -use sd_task_system::{ExecStatus, Interrupter, IntoAnyTaskOutput, SerializableTask, Task, TaskId}; -use sd_utils::db::{inode_to_db, size_in_bytes_to_db}; - -use std::{sync::Arc, time::Duration}; - -use chrono::Utc; -use serde::{Deserialize, Serialize}; -use tokio::time::Instant; -use tracing::{instrument, trace, Level}; - -use super::walker::WalkedEntry; - -#[derive(Debug)] -pub struct Saver { - // Task control - id: TaskId, - is_shallow: bool, - - // Received input args - location_id: location::id::Type, - location_pub_id: location::pub_id::Type, - device_id: device::id::Type, - walked_entries: Vec, - - // Dependencies - db: Arc, - sync: SyncManager, -} - -/// [`Save`] Task output -#[derive(Debug)] -pub struct Output { - /// Number of records inserted on database - pub saved_count: u64, - /// Time spent saving records - pub save_duration: Duration, -} - -#[async_trait::async_trait] -impl Task for Saver { - fn id(&self) -> TaskId { - self.id - } - - fn with_priority(&self) -> bool { - // If we're running in shallow mode, then we want priority - self.is_shallow - } - - #[instrument( - skip_all, - fields( - task_id = %self.id, - location_id = %self.location_id, - to_save_count = %self.walked_entries.len(), - is_shallow = self.is_shallow, - ), - ret(level = Level::TRACE), - err, - )] - #[allow(clippy::blocks_in_conditions)] // Due to `err` on `instrument` macro above - async fn run(&mut self, _: &Interrupter) -> Result { - use file_path::{ - create_unchecked, date_created, date_indexed, date_modified, device, device_id, - extension, hidden, inode, is_dir, location, location_id, materialized_path, name, - size_in_bytes_bytes, - }; - - let start_time = Instant::now(); - - let Self { - location_id, - location_pub_id, - device_id, - walked_entries, - db, - sync, - .. - } = self; - - let (create_crdt_ops, paths): (Vec<_>, Vec<_>) = walked_entries - .drain(..) - .map( - |WalkedEntry { - pub_id, - maybe_object_id, - iso_file_path, - metadata: - FilePathMetadata { - inode, - size_in_bytes, - created_at, - modified_at, - hidden, - }, - }| { - let IsolatedFilePathDataParts { - materialized_path, - is_dir, - name, - extension, - .. - } = iso_file_path.to_parts(); - - assert!( - maybe_object_id.is_none(), - "Object ID must be None as this tasks only created \ - new file_paths and they were not identified yet" - ); - - let (sync_params, db_params) = [ - ( - sync_entry!( - prisma_sync::location::SyncId { - pub_id: location_pub_id.clone() - }, - location - ), - location_id::set(Some(*location_id)), - ), - sync_db_entry!(materialized_path, materialized_path), - sync_db_entry!(name, name), - sync_db_entry!(is_dir, is_dir), - sync_db_entry!(extension, extension), - sync_db_entry!(size_in_bytes_to_db(size_in_bytes), size_in_bytes_bytes), - sync_db_entry!(inode_to_db(inode), inode), - sync_db_entry!(created_at, date_created), - sync_db_entry!(modified_at, date_modified), - sync_db_entry!(Utc::now(), date_indexed), - sync_db_entry!(hidden, hidden), - ( - sync_entry!( - prisma_sync::device::SyncId { - pub_id: sync.device_pub_id.to_db(), - }, - device - ), - device_id::set(Some(*device_id)), - ), - ] - .into_iter() - .unzip::<_, _, Vec<_>, Vec<_>>(); - - ( - sync.shared_create( - prisma_sync::file_path::SyncId { - pub_id: pub_id.to_db(), - }, - sync_params, - ), - create_unchecked(pub_id.into(), db_params), - ) - }, - ) - .unzip(); - - if create_crdt_ops.is_empty() && paths.is_empty() { - return Ok(ExecStatus::Done( - Output { - saved_count: 0, - save_duration: Duration::ZERO, - } - .into_output(), - )); - } - - #[allow(clippy::cast_sign_loss)] - let saved_count = sync - .write_ops( - db, - ( - create_crdt_ops, - db.file_path().create_many(paths).skip_duplicates(), - ), - ) - .await - .map_err(indexer::Error::from)? as u64; - - let save_duration = start_time.elapsed(); - - trace!(saved_count, "Inserted records;"); - - Ok(ExecStatus::Done( - Output { - saved_count, - save_duration, - } - .into_output(), - )) - } -} - -impl Saver { - #[must_use] - pub fn new_deep( - location_id: location::id::Type, - location_pub_id: location::pub_id::Type, - walked_entries: Vec, - db: Arc, - sync: SyncManager, - device_id: device::id::Type, - ) -> Self { - Self { - id: TaskId::new_v4(), - location_id, - location_pub_id, - device_id, - walked_entries, - db, - sync, - is_shallow: false, - } - } - - #[must_use] - pub fn new_shallow( - location_id: location::id::Type, - location_pub_id: location::pub_id::Type, - walked_entries: Vec, - db: Arc, - sync: SyncManager, - device_id: device::id::Type, - ) -> Self { - Self { - id: TaskId::new_v4(), - location_id, - location_pub_id, - device_id, - walked_entries, - db, - sync, - is_shallow: true, - } - } -} - -#[derive(Debug, Serialize, Deserialize)] -struct SaveState { - id: TaskId, - is_shallow: bool, - - location_id: location::id::Type, - location_pub_id: location::pub_id::Type, - device_id: device::id::Type, - walked_entries: Vec, -} - -impl SerializableTask for Saver { - type SerializeError = rmp_serde::encode::Error; - - type DeserializeError = rmp_serde::decode::Error; - - type DeserializeCtx = (Arc, SyncManager); - - async fn serialize(self) -> Result, Self::SerializeError> { - let Self { - id, - is_shallow, - location_id, - location_pub_id, - device_id, - walked_entries, - .. - } = self; - rmp_serde::to_vec_named(&SaveState { - id, - is_shallow, - location_id, - location_pub_id, - device_id, - walked_entries, - }) - } - - async fn deserialize( - data: &[u8], - (db, sync): Self::DeserializeCtx, - ) -> Result { - rmp_serde::from_slice(data).map( - |SaveState { - id, - is_shallow, - location_id, - location_pub_id, - device_id, - walked_entries, - }| Self { - id, - is_shallow, - location_id, - location_pub_id, - device_id, - walked_entries, - db, - sync, - }, - ) - } -} diff --git a/core/crates/heavy-lifting/src/indexer/tasks/updater.rs b/core/crates/heavy-lifting/src/indexer/tasks/updater.rs deleted file mode 100644 index 80cf3d6f4..000000000 --- a/core/crates/heavy-lifting/src/indexer/tasks/updater.rs +++ /dev/null @@ -1,307 +0,0 @@ -use crate::{indexer, Error}; - -use sd_core_file_path_helper::{FilePathMetadata, IsolatedFilePathDataParts}; -use sd_core_sync::SyncManager; - -use sd_prisma::{ - prisma::{file_path, object, PrismaClient}, - prisma_sync, -}; -use sd_sync::{sync_db_entry, OperationFactory}; -use sd_task_system::{ - check_interruption, ExecStatus, Interrupter, IntoAnyTaskOutput, SerializableTask, Task, TaskId, -}; -use sd_utils::{ - chain_optional_iter, - db::{inode_to_db, size_in_bytes_to_db}, - msgpack, -}; - -use std::{collections::HashSet, sync::Arc, time::Duration}; - -use serde::{Deserialize, Serialize}; -use tokio::time::Instant; -use tracing::{instrument, trace, Level}; - -use super::walker::WalkedEntry; - -#[derive(Debug)] -pub struct Updater { - // Task control - id: TaskId, - is_shallow: bool, - - // Received input args - walked_entries: Vec, - - // Inner state - object_ids_that_should_be_unlinked: HashSet, - - // Dependencies - db: Arc, - sync: SyncManager, -} - -/// [`Update`] Task output -#[derive(Debug)] -pub struct Output { - /// Number of records updated on database - pub updated_count: u64, - /// Time spent updating records - pub update_duration: Duration, -} - -#[async_trait::async_trait] -impl Task for Updater { - fn id(&self) -> TaskId { - self.id - } - - fn with_priority(&self) -> bool { - // If we're running in shallow mode, then we want priority - self.is_shallow - } - - #[instrument( - skip_all, - fields( - task_id = %self.id, - to_update_count = %self.walked_entries.len(), - is_shallow = self.is_shallow, - ), - ret(level = Level::TRACE), - err, - )] - #[allow(clippy::blocks_in_conditions)] // Due to `err` on `instrument` macro above - async fn run(&mut self, interrupter: &Interrupter) -> Result { - use file_path::{ - cas_id, date_created, date_modified, hidden, inode, is_dir, object, object_id, - size_in_bytes_bytes, - }; - - let start_time = Instant::now(); - - let Self { - walked_entries, - db, - sync, - object_ids_that_should_be_unlinked, - .. - } = self; - - fetch_objects_ids_to_unlink(walked_entries, object_ids_that_should_be_unlinked, db).await?; - - check_interruption!(interrupter); - - let (crdt_ops, paths_to_update) = walked_entries - .drain(..) - .map( - |WalkedEntry { - pub_id, - maybe_object_id, - iso_file_path, - metadata: - FilePathMetadata { - inode, - size_in_bytes, - created_at, - modified_at, - hidden, - }, - }| { - let IsolatedFilePathDataParts { is_dir, .. } = &iso_file_path.to_parts(); - - let should_unlink_object = maybe_object_id.map_or(false, |object_id| { - object_ids_that_should_be_unlinked.contains(&object_id) - }); - - let (sync_params, db_params) = chain_optional_iter( - [ - ((cas_id::NAME, msgpack!(nil)), cas_id::set(None)), - sync_db_entry!(*is_dir, is_dir), - sync_db_entry!(size_in_bytes_to_db(size_in_bytes), size_in_bytes_bytes), - sync_db_entry!(inode_to_db(inode), inode), - sync_db_entry!(created_at, date_created), - sync_db_entry!(modified_at, date_modified), - sync_db_entry!(hidden, hidden), - ], - [ - // As this file was updated while Spacedrive was offline, we mark the object_id and cas_id as null - // So this file_path will be updated at file identifier job - should_unlink_object.then_some(( - (object_id::NAME, msgpack!(nil)), - object::disconnect(), - )), - ], - ) - .into_iter() - .unzip::<_, _, Vec<_>, Vec<_>>(); - - ( - sync.shared_update( - prisma_sync::file_path::SyncId { - pub_id: pub_id.to_db(), - }, - sync_params, - ), - db.file_path() - .update(file_path::pub_id::equals(pub_id.into()), db_params) - // selecting id to avoid fetching whole object from database - .select(file_path::select!({ id })), - ) - }, - ) - .unzip::<_, _, Vec<_>, Vec<_>>(); - - if crdt_ops.is_empty() && paths_to_update.is_empty() { - return Ok(ExecStatus::Done( - Output { - updated_count: 0, - update_duration: Duration::ZERO, - } - .into_output(), - )); - } - - let updated = sync - .write_ops(db, (crdt_ops, paths_to_update)) - .await - .map_err(indexer::Error::from)?; - - let update_duration = start_time.elapsed(); - - trace!(?updated, "Updated records;"); - - Ok(ExecStatus::Done( - Output { - updated_count: updated.len() as u64, - update_duration, - } - .into_output(), - )) - } -} - -impl Updater { - #[must_use] - pub fn new_deep( - walked_entries: Vec, - db: Arc, - sync: SyncManager, - ) -> Self { - Self { - id: TaskId::new_v4(), - walked_entries, - db, - sync, - object_ids_that_should_be_unlinked: HashSet::new(), - is_shallow: false, - } - } - - #[must_use] - pub fn new_shallow( - walked_entries: Vec, - db: Arc, - sync: SyncManager, - ) -> Self { - Self { - id: TaskId::new_v4(), - walked_entries, - db, - sync, - object_ids_that_should_be_unlinked: HashSet::new(), - is_shallow: true, - } - } -} - -async fn fetch_objects_ids_to_unlink( - walked_entries: &[WalkedEntry], - object_ids_that_should_be_unlinked: &mut HashSet, - db: &PrismaClient, -) -> Result<(), indexer::Error> { - if object_ids_that_should_be_unlinked.is_empty() { - // First we consult which file paths we should unlink - let object_ids = walked_entries - .iter() - .filter_map(|entry| entry.maybe_object_id) - .collect::>() // Removing possible duplicates - .into_iter() - .collect::>(); - - *object_ids_that_should_be_unlinked = db - ._batch( - object_ids - .iter() - .map(|object_id| { - db.file_path() - .count(vec![file_path::object_id::equals(Some(*object_id))]) - }) - .collect::>(), - ) - .await? - .into_iter() - .zip(object_ids) - .filter_map(|(count, object_id)| (count > 1).then_some(object_id)) - .collect::>(); - } - - Ok(()) -} - -#[derive(Debug, Serialize, Deserialize)] -struct SaveState { - id: TaskId, - is_shallow: bool, - - walked_entries: Vec, - - object_ids_that_should_be_unlinked: HashSet, -} - -impl SerializableTask for Updater { - type SerializeError = rmp_serde::encode::Error; - - type DeserializeError = rmp_serde::decode::Error; - - type DeserializeCtx = (Arc, SyncManager); - - async fn serialize(self) -> Result, Self::SerializeError> { - let Self { - id, - walked_entries, - object_ids_that_should_be_unlinked, - is_shallow, - .. - } = self; - - rmp_serde::to_vec_named(&SaveState { - id, - is_shallow, - walked_entries, - object_ids_that_should_be_unlinked, - }) - } - - async fn deserialize( - data: &[u8], - (db, sync): Self::DeserializeCtx, - ) -> Result { - rmp_serde::from_slice(data).map( - |SaveState { - id, - is_shallow, - walked_entries, - object_ids_that_should_be_unlinked, - }| Self { - id, - is_shallow, - walked_entries, - object_ids_that_should_be_unlinked, - db, - sync, - }, - ) - } -} diff --git a/core/crates/heavy-lifting/src/indexer/tasks/walker/entry.rs b/core/crates/heavy-lifting/src/indexer/tasks/walker/entry.rs deleted file mode 100644 index 4e460f972..000000000 --- a/core/crates/heavy-lifting/src/indexer/tasks/walker/entry.rs +++ /dev/null @@ -1,93 +0,0 @@ -use sd_core_file_path_helper::{FilePathMetadata, IsolatedFilePathData}; - -use sd_core_prisma_helpers::FilePathPubId; -use sd_prisma::prisma::file_path; - -use std::{ - hash::{Hash, Hasher}, - path::{Path, PathBuf}, -}; - -use serde::{Deserialize, Serialize}; - -/// `WalkedEntry` represents a single path in the filesystem -#[derive(Debug, Serialize, Deserialize)] -pub struct WalkedEntry { - pub pub_id: FilePathPubId, - pub maybe_object_id: file_path::object_id::Type, - pub iso_file_path: IsolatedFilePathData<'static>, - pub metadata: FilePathMetadata, -} - -impl PartialEq for WalkedEntry { - fn eq(&self, other: &Self) -> bool { - self.iso_file_path == other.iso_file_path - } -} - -impl Eq for WalkedEntry {} - -impl Hash for WalkedEntry { - fn hash(&self, state: &mut H) { - self.iso_file_path.hash(state); - } -} - -#[derive(Debug, Serialize, Deserialize)] -pub(super) struct WalkingEntry { - pub(super) iso_file_path: IsolatedFilePathData<'static>, - pub(super) metadata: FilePathMetadata, -} - -impl From for WalkedEntry { - fn from( - WalkingEntry { - iso_file_path, - metadata, - }: WalkingEntry, - ) -> Self { - Self { - pub_id: FilePathPubId::new(), - maybe_object_id: None, - iso_file_path, - metadata, - } - } -} - -impl> From<(PubId, file_path::object_id::Type, WalkingEntry)> - for WalkedEntry -{ - fn from( - ( - pub_id, - maybe_object_id, - WalkingEntry { - iso_file_path, - metadata, - }, - ): (PubId, file_path::object_id::Type, WalkingEntry), - ) -> Self { - Self { - pub_id: pub_id.into(), - maybe_object_id, - iso_file_path, - metadata, - } - } -} - -#[derive(Debug, Serialize, Deserialize)] -pub struct ToWalkEntry { - pub(super) path: PathBuf, - pub(super) parent_dir_accepted_by_its_children: Option, -} - -impl> From

for ToWalkEntry { - fn from(path: P) -> Self { - Self { - path: path.as_ref().into(), - parent_dir_accepted_by_its_children: None, - } - } -} diff --git a/core/crates/heavy-lifting/src/indexer/tasks/walker/metadata.rs b/core/crates/heavy-lifting/src/indexer/tasks/walker/metadata.rs deleted file mode 100644 index 33e5c8935..000000000 --- a/core/crates/heavy-lifting/src/indexer/tasks/walker/metadata.rs +++ /dev/null @@ -1,64 +0,0 @@ -use crate::indexer; - -use sd_core_file_path_helper::FilePathMetadata; -use sd_core_indexer_rules::MetadataForIndexerRules; - -use std::{fs::Metadata, path::Path}; - -use chrono::{DateTime, Utc}; -use serde::{Deserialize, Serialize}; - -#[derive(Debug, Serialize, Deserialize)] -pub(super) struct InnerMetadata { - pub is_dir: bool, - pub is_symlink: bool, - pub inode: u64, - pub size_in_bytes: u64, - pub hidden: bool, - pub created_at: DateTime, - pub modified_at: DateTime, -} - -impl InnerMetadata { - pub fn new( - path: impl AsRef + Copy, - metadata: &Metadata, - ) -> Result { - let FilePathMetadata { - inode, - size_in_bytes, - created_at, - modified_at, - hidden, - } = FilePathMetadata::from_path(path, metadata) - .map_err(|e| indexer::NonCriticalIndexerError::FilePathMetadata(e.to_string()))?; - - Ok(Self { - is_dir: metadata.is_dir(), - is_symlink: metadata.is_symlink(), - inode, - size_in_bytes, - hidden, - created_at, - modified_at, - }) - } -} - -impl MetadataForIndexerRules for InnerMetadata { - fn is_dir(&self) -> bool { - self.is_dir - } -} - -impl From for FilePathMetadata { - fn from(metadata: InnerMetadata) -> Self { - Self { - inode: metadata.inode, - size_in_bytes: metadata.size_in_bytes, - hidden: metadata.hidden, - created_at: metadata.created_at, - modified_at: metadata.modified_at, - } - } -} diff --git a/core/crates/heavy-lifting/src/indexer/tasks/walker/mod.rs b/core/crates/heavy-lifting/src/indexer/tasks/walker/mod.rs deleted file mode 100644 index 5fef9e689..000000000 --- a/core/crates/heavy-lifting/src/indexer/tasks/walker/mod.rs +++ /dev/null @@ -1,1176 +0,0 @@ -use crate::{ - indexer::{ - self, - tasks::walker::rules::{apply_indexer_rules, process_rules_results}, - }, - Error, NonCriticalError, -}; - -use sd_core_file_path_helper::{FilePathError, FilePathMetadata, IsolatedFilePathData}; -use sd_core_indexer_rules::{ - seed::{GitIgnoreRules, GITIGNORE}, - IndexerRuler, MetadataForIndexerRules, RuleKind, -}; -use sd_core_prisma_helpers::{file_path_pub_and_cas_ids, file_path_walker}; - -use sd_prisma::prisma::file_path; -use sd_task_system::{ - check_interruption, ExecStatus, Interrupter, IntoAnyTaskOutput, Task, TaskId, -}; -use sd_utils::{ - db::{inode_from_db, inode_to_db}, - error::FileIOError, -}; - -use std::{ - collections::{HashMap, HashSet}, - fmt, - future::Future, - mem, - path::{Path, PathBuf}, - sync::Arc, - time::Duration, -}; - -use chrono::{DateTime, Duration as ChronoDuration, FixedOffset}; -use futures_concurrency::future::Join; -use tokio::{fs, time::Instant}; -use tokio_stream::{wrappers::ReadDirStream, StreamExt}; -use tracing::{instrument, trace, Level}; - -mod entry; -mod metadata; -mod rules; -mod save_state; - -pub use entry::{ToWalkEntry, WalkedEntry}; - -use entry::WalkingEntry; -use metadata::InnerMetadata; - -pub trait IsoFilePathFactory: Clone + Send + Sync + fmt::Debug + 'static { - fn build( - &self, - path: impl AsRef, - is_dir: bool, - ) -> Result, FilePathError>; -} - -pub trait WalkerDBProxy: Clone + Send + Sync + fmt::Debug + 'static { - fn fetch_file_paths( - &self, - found_paths: Vec, - ) -> impl Future, indexer::Error>> + Send; - - fn fetch_file_paths_to_remove( - &self, - parent_iso_file_path: &IsolatedFilePathData<'_>, - existing_inodes: HashSet>, - unique_location_id_materialized_path_name_extension_params: Vec, - ) -> impl Future< - Output = Result, indexer::NonCriticalIndexerError>, - > + Send; -} - -#[derive(Debug)] -pub struct Walker -where - DBProxy: WalkerDBProxy, - IsoPathFactory: IsoFilePathFactory, -{ - // Task control - id: TaskId, - is_shallow: bool, - - // Received input args - entry: ToWalkEntry, - root: Arc, - entry_iso_file_path: IsolatedFilePathData<'static>, - indexer_ruler: IndexerRuler, - - // Inner state - stage: WalkerStage, - - // Dependencies - iso_file_path_factory: IsoPathFactory, - db_proxy: DBProxy, - - // Non critical errors that happened during the task execution - errors: Vec, - - // Time spent walking through the received directory - scan_time: Duration, -} - -/// [`Walker`] Task output -#[derive(Debug)] -pub struct Output -where - DBProxy: WalkerDBProxy, - IsoPathFactory: IsoFilePathFactory, -{ - /// Entries found in the file system that need to be created in database - pub to_create: Vec, - /// Entries found in the file system that need to be updated in database - pub to_update: Vec, - /// Entries found in the file system that need to be removed from database - pub to_remove: Vec, - /// Entries found in the file system that will not be indexed - pub non_indexed_paths: Vec, - /// Ancestors of entries that were indexed - pub accepted_ancestors: HashSet, - /// Errors that happened during the task execution - pub errors: Vec, - /// Directory that was indexed - pub directory_iso_file_path: IsolatedFilePathData<'static>, - /// Total size of the directory that was indexed - pub total_size: u64, - /// Task handles that were dispatched to run `WalkDir` tasks for inner directories - pub keep_walking_tasks: Vec>, - /// Time spent walking through the received directory - pub scan_time: Duration, -} - -#[async_trait::async_trait] -impl Task for Walker -where - DBProxy: WalkerDBProxy, - IsoPathFactory: IsoFilePathFactory, -{ - fn id(&self) -> TaskId { - self.id - } - - fn with_priority(&self) -> bool { - // If we're running in shallow mode, then we want priority - self.is_shallow - } - - #[instrument( - skip_all, - fields( - task_id = %self.id, - walked_entry = %self.entry.path.display(), - is_shallow = self.is_shallow, - ), - ret(level = Level::TRACE), - err, - )] - #[allow(clippy::blocks_in_conditions)] // Due to `err` on `instrument` macro above - async fn run(&mut self, interrupter: &Interrupter) -> Result { - let is_shallow = self.is_shallow; - let Self { - root, - entry: ToWalkEntry { - path, - parent_dir_accepted_by_its_children, - }, - entry_iso_file_path, - iso_file_path_factory, - indexer_ruler, - db_proxy, - stage, - errors, - scan_time, - .. - } = self; - - let start_time = Instant::now(); - - let ( - to_create, - to_update, - to_remove, - non_indexed_paths, - accepted_ancestors, - total_size, - keep_walking_tasks, - ) = loop { - match stage { - WalkerStage::Start => { - trace!("Preparing git indexer rules for walking root"); - if indexer_ruler.has_system(&GITIGNORE) { - if let Some(rules) = - GitIgnoreRules::get_rules_if_in_git_repo(root.as_ref(), path).await - { - trace!("Found gitignore rules to follow"); - indexer_ruler.extend(rules.map(Into::into)); - } - } - - *stage = WalkerStage::Walking { - read_dir_stream: ReadDirStream::new(fs::read_dir(&path).await.map_err( - |e| { - indexer::Error::FileIO( - (&path, e, "Failed to open directory to read its entries") - .into(), - ) - }, - )?), - found_paths: Vec::new(), - }; - trace!("Starting to walk!"); - } - - WalkerStage::Walking { - read_dir_stream, - found_paths, - } => { - trace!("Walking..."); - while let Some(res) = read_dir_stream.next().await { - match res { - Ok(dir_entry) => { - found_paths.push(dir_entry.path()); - trace!( - new_path = %dir_entry.path().display(), - total_paths = found_paths.len(), - "Found path;" - ); - } - Err(e) => { - errors.push(NonCriticalError::Indexer( - indexer::NonCriticalIndexerError::FailedDirectoryEntry( - FileIOError::from((&path, e)).to_string(), - ), - )); - } - } - - check_interruption!(interrupter, start_time, scan_time); - } - - trace!(total_paths = found_paths.len(), "Finished walking!;"); - - *stage = WalkerStage::CollectingMetadata { - found_paths: mem::take(found_paths), - }; - - check_interruption!(interrupter, start_time, scan_time); - } - - WalkerStage::CollectingMetadata { found_paths } => { - trace!("Collecting metadata for found paths"); - *stage = WalkerStage::CheckingIndexerRules { - paths_and_metadatas: collect_metadata(found_paths, errors).await, - }; - trace!("Finished collecting metadata!"); - - check_interruption!(interrupter, start_time, scan_time); - } - - WalkerStage::CheckingIndexerRules { - paths_and_metadatas, - } => { - trace!("Checking indexer rules for found paths"); - *stage = WalkerStage::ProcessingRulesResults { - paths_metadatas_and_acceptance: apply_indexer_rules( - paths_and_metadatas, - indexer_ruler, - errors, - ) - .await, - }; - trace!("Finished checking indexer rules!"); - - check_interruption!(interrupter, start_time, scan_time); - } - - WalkerStage::ProcessingRulesResults { - paths_metadatas_and_acceptance, - } => { - trace!("Processing rules results"); - let mut maybe_to_keep_walking = (!is_shallow).then(Vec::new); - let (accepted_paths, accepted_ancestors, rejected_paths) = - process_rules_results( - root, - iso_file_path_factory, - *parent_dir_accepted_by_its_children, - paths_metadatas_and_acceptance, - &mut maybe_to_keep_walking, - is_shallow, - errors, - ) - .await; - - trace!( - total_accepted_paths = accepted_paths.len(), - total_accepted_ancestors = accepted_ancestors.len(), - collect_rejected_paths = self.is_shallow, - total_rejected_paths = rejected_paths.len(), - "Finished processing rules results!;" - ); - - *stage = WalkerStage::GatheringFilePathsToRemove { - accepted_paths, - maybe_to_keep_walking, - accepted_ancestors, - non_indexed_paths: rejected_paths, - }; - - check_interruption!(interrupter, start_time, scan_time); - } - - WalkerStage::GatheringFilePathsToRemove { - accepted_paths, - maybe_to_keep_walking, - accepted_ancestors, - non_indexed_paths, - } => { - trace!("Gathering file paths to remove"); - let (walking_entries, to_remove_entries) = gather_file_paths_to_remove( - accepted_paths, - entry_iso_file_path, - iso_file_path_factory, - db_proxy, - errors, - ) - .await; - trace!("Finished gathering file paths to remove!"); - - *stage = WalkerStage::Finalize { - walking_entries, - to_remove_entries, - maybe_to_keep_walking: mem::take(maybe_to_keep_walking), - accepted_ancestors: mem::take(accepted_ancestors), - non_indexed_paths: mem::take(non_indexed_paths), - }; - - check_interruption!(interrupter, start_time, scan_time); - } - - // From this points onwards, we will not allow to be interrupted anymore - WalkerStage::Finalize { - walking_entries, - to_remove_entries, - maybe_to_keep_walking, - accepted_ancestors, - non_indexed_paths, - } => { - trace!("Segregating creates and updates"); - let (to_create, to_update, total_size) = - segregate_creates_and_updates(walking_entries, db_proxy).await?; - trace!( - total_to_create = to_create.len(), - total_to_update = to_update.len(), - total_to_remove = to_remove_entries.len(), - total_non_indexed_paths = non_indexed_paths.len(), - total_size, - "Finished segregating creates and updates!;" - ); - - let keep_walking_tasks = keep_walking( - root, - indexer_ruler, - iso_file_path_factory, - db_proxy, - maybe_to_keep_walking.as_mut(), - errors, - ); - - break ( - to_create, - to_update, - mem::take(to_remove_entries), - mem::take(non_indexed_paths), - mem::take(accepted_ancestors), - total_size, - keep_walking_tasks, - ); - } - } - }; - - *scan_time += start_time.elapsed(); - - // Taking out some data as the task is finally complete - Ok(ExecStatus::Done( - Output { - to_create, - to_update, - to_remove, - non_indexed_paths, - accepted_ancestors, - errors: mem::take(errors), - directory_iso_file_path: mem::take(entry_iso_file_path), - total_size, - keep_walking_tasks, - scan_time: *scan_time, - } - .into_output(), - )) - } -} - -#[derive(Debug)] -enum WalkerStage { - Start, - Walking { - read_dir_stream: ReadDirStream, - found_paths: Vec, - }, - CollectingMetadata { - found_paths: Vec, - }, - CheckingIndexerRules { - paths_and_metadatas: HashMap, - }, - ProcessingRulesResults { - paths_metadatas_and_acceptance: - HashMap>)>, - }, - GatheringFilePathsToRemove { - accepted_paths: HashMap, - maybe_to_keep_walking: Option>, - accepted_ancestors: HashSet, - non_indexed_paths: Vec, - }, - Finalize { - walking_entries: Vec, - accepted_ancestors: HashSet, - to_remove_entries: Vec, - maybe_to_keep_walking: Option>, - non_indexed_paths: Vec, - }, -} - -impl Walker -where - DBProxy: WalkerDBProxy, - IsoPathFactory: IsoFilePathFactory, -{ - pub fn new_deep( - entry: impl Into + Send, - root: Arc, - indexer_ruler: IndexerRuler, - iso_file_path_factory: IsoPathFactory, - db_proxy: DBProxy, - ) -> Result { - let entry = entry.into(); - Ok(Self { - id: TaskId::new_v4(), - root, - indexer_ruler, - entry_iso_file_path: iso_file_path_factory.build(&entry.path, true)?, - iso_file_path_factory, - db_proxy, - stage: WalkerStage::Start, - entry, - is_shallow: false, - errors: Vec::new(), - scan_time: Duration::ZERO, - }) - } -} - -impl Walker -where - DBProxy: WalkerDBProxy, - IsoPathFactory: IsoFilePathFactory, -{ - pub fn new_shallow( - entry: impl Into + Send, - root: Arc, - indexer_ruler: IndexerRuler, - iso_file_path_factory: IsoPathFactory, - db_proxy: DBProxy, - ) -> Result { - let entry = entry.into(); - Ok(Self { - id: TaskId::new_v4(), - root, - indexer_ruler, - entry_iso_file_path: iso_file_path_factory.build(&entry.path, true)?, - iso_file_path_factory, - db_proxy, - stage: WalkerStage::Start, - entry, - is_shallow: true, - errors: Vec::new(), - scan_time: Duration::ZERO, - }) - } -} - -#[instrument( - skip_all, - fields(entries_count = walking_entries.len()), - err, -)] -async fn segregate_creates_and_updates( - walking_entries: &mut Vec, - db_proxy: &impl WalkerDBProxy, -) -> Result<(Vec, Vec, u64), Error> { - if walking_entries.is_empty() { - Ok((vec![], vec![], 0)) - } else { - let iso_paths_already_in_db = db_proxy - .fetch_file_paths( - walking_entries - .iter() - .map(|entry| file_path::WhereParam::from(&entry.iso_file_path)) - .collect(), - ) - .await? - .into_iter() - .flat_map(|file_path| { - IsolatedFilePathData::try_from(file_path.clone()) - .map(|iso_file_path| (iso_file_path, file_path)) - }) - .collect::>(); - - Ok(walking_entries.drain(..).fold( - (Vec::new(), Vec::new(), 0), - |(mut to_create, mut to_update, mut total_size), entry| { - let WalkingEntry { - iso_file_path, - metadata, - } = &entry; - - total_size += metadata.size_in_bytes; - - if let Some(file_path) = iso_paths_already_in_db.get(iso_file_path) { - if let (Some(inode), Some(date_modified)) = - (&file_path.inode, &file_path.date_modified) - { - if ( - inode_from_db(&inode[0..8]) != metadata.inode - // Datetimes stored in DB loses a bit of precision, - // so we need to check against a delta - // instead of using != operator - || ( - DateTime::::from(metadata.modified_at) - *date_modified - > ChronoDuration::milliseconds(1) - ) - || file_path.hidden.is_none() - || metadata.hidden != file_path.hidden.unwrap_or_default() - ) - // We ignore the size of directories because it is not reliable, we need to - // calculate it ourselves later - && !( - iso_file_path.to_parts().is_dir - && metadata.size_in_bytes - != file_path - .size_in_bytes_bytes - .as_ref() - .map(|size_in_bytes_bytes| { - u64::from_be_bytes([ - size_in_bytes_bytes[0], - size_in_bytes_bytes[1], - size_in_bytes_bytes[2], - size_in_bytes_bytes[3], - size_in_bytes_bytes[4], - size_in_bytes_bytes[5], - size_in_bytes_bytes[6], - size_in_bytes_bytes[7], - ]) - }) - .unwrap_or_default() - ) { - to_update.push(WalkedEntry::from(( - &file_path.pub_id, - file_path.object_id, - entry, - ))); - } - } - } else { - to_create.push(WalkedEntry::from(entry)); - } - - (to_create, to_update, total_size) - }, - )) - } -} - -fn keep_walking( - root: &Arc, - indexer_ruler: &IndexerRuler, - iso_file_path_factory: &IsoPathFactory, - db_proxy: &DBProxy, - maybe_to_keep_walking: Option<&mut Vec>, - errors: &mut Vec, -) -> Vec> -where - DBProxy: WalkerDBProxy, - IsoPathFactory: IsoFilePathFactory, -{ - maybe_to_keep_walking - .map(|to_keep_walking| { - to_keep_walking - .drain(..) - .map(|entry| { - Walker::new_deep( - entry, - Arc::clone(root), - indexer_ruler.clone(), - iso_file_path_factory.clone(), - db_proxy.clone(), - ) - .map_err(|e| { - indexer::NonCriticalIndexerError::DispatchKeepWalking(e.to_string()) - }) - }) - .filter_map(|res| res.map_err(|e| errors.push(e.into())).ok()) - .collect() - }) - .unwrap_or_default() -} - -async fn collect_metadata( - found_paths: &mut Vec, - errors: &mut Vec, -) -> HashMap { - found_paths - .drain(..) - .map(|current_path| async move { - fs::metadata(¤t_path) - .await - .map_err(|e| { - indexer::NonCriticalIndexerError::Metadata( - FileIOError::from((¤t_path, e)).to_string(), - ) - }) - .and_then(|metadata| { - InnerMetadata::new(¤t_path, &metadata) - .map(|metadata| (current_path, metadata)) - }) - }) - .collect::>() - .join() - .await - .into_iter() - .filter_map(|res| res.map_err(|e| errors.push(e.into())).ok()) - .collect() -} - -async fn gather_file_paths_to_remove( - accepted_paths: &mut HashMap, - entry_iso_file_path: &IsolatedFilePathData<'_>, - iso_file_path_factory: &impl IsoFilePathFactory, - db_proxy: &impl WalkerDBProxy, - errors: &mut Vec, -) -> (Vec, Vec) { - let mut existing_inodes = HashSet::new(); - - let (walking, to_delete_params) = accepted_paths - .drain() - .filter_map(|(path, metadata)| { - iso_file_path_factory - .build(&path, metadata.is_dir()) - .map(|iso_file_path| { - let params = file_path::WhereParam::from(&iso_file_path); - existing_inodes.insert(inode_to_db(metadata.inode)); - - ( - WalkingEntry { - iso_file_path, - metadata: FilePathMetadata::from(metadata), - }, - params, - ) - }) - .map_err(|e| { - errors - .push(indexer::NonCriticalIndexerError::IsoFilePath(e.to_string()).into()); - }) - .ok() - }) - .unzip::<_, _, Vec<_>, Vec<_>>(); - - // We continue the function even if we fail to fetch `file_path`s to remove, - // the DB will have old `file_path`s but at least this is better than - // don't adding the newly indexed paths - let to_remove_entries = db_proxy - .fetch_file_paths_to_remove(entry_iso_file_path, existing_inodes, to_delete_params) - .await - .map_err(|e| errors.push(e.into())) - .unwrap_or_default(); - - (walking, to_remove_entries) -} - -#[cfg(test)] -mod tests { - use super::*; - - use sd_core_indexer_rules::{IndexerRule, RulePerKind}; - use sd_core_prisma_helpers::FilePathPubId; - use sd_task_system::{TaskOutput, TaskStatus, TaskSystem}; - - use chrono::Utc; - use futures::stream::FuturesUnordered; - use globset::{Glob, GlobSetBuilder}; - use lending_stream::{LendingStream, StreamExt}; - use tempfile::{tempdir, TempDir}; - use tokio::{fs, io::AsyncWriteExt}; - use tracing::debug; - use tracing_test::traced_test; - - #[derive(Debug, Clone)] - struct DummyIsoPathFactory { - root_path: Arc, - } - - impl IsoFilePathFactory for DummyIsoPathFactory { - fn build( - &self, - path: impl AsRef, - is_dir: bool, - ) -> Result, FilePathError> { - IsolatedFilePathData::new(0, self.root_path.as_ref(), path, is_dir).map_err(Into::into) - } - } - - #[derive(Debug, Clone)] - struct DummyDBProxy; - - impl WalkerDBProxy for DummyDBProxy { - async fn fetch_file_paths( - &self, - _: Vec, - ) -> Result, indexer::Error> { - Ok(vec![]) - } - - async fn fetch_file_paths_to_remove( - &self, - _: &IsolatedFilePathData<'_>, - _: HashSet>, - _: Vec, - ) -> Result, indexer::NonCriticalIndexerError> { - Ok(vec![]) - } - } - - fn new_indexer_rule( - name: impl Into, - default: bool, - rules: Vec, - ) -> IndexerRule { - IndexerRule { - id: None, - name: name.into(), - default, - rules, - date_created: Utc::now(), - date_modified: Utc::now(), - } - } - - #[allow(clippy::cognitive_complexity)] - async fn prepare_location() -> TempDir { - // root - // |__ rust_project - // | |__ .git - // | | |__ - // | |__ .gitignore - // | |__ ignorable.file - // | |__ Cargo.toml - // | |__ src - // | | |__ main.rs - // | |__ target - // | | |__ debug - // | | |__ main - // | |__ partial - // | | |__ ignoreme - // | | |__ readme - // | |__ inner - // | |__ node_project - // | |__ .git - // | | |__ - // | |__ .gitignore - // | |__ ignorable.file - // | |__ package.json - // | |__ src - // | | |__ App.tsx - // | |__ node_modules - // | |__ react - // | |__ package.json - // |__ photos - // |__ photo1.png - // |__ photo2.jpg - // |__ photo3.jpeg - // |__ text.txt - - let root = tempdir().unwrap(); - let root_path = root.path(); - let rust_project = root_path.join("rust_project"); - let inner_project = root_path.join("inner"); - let node_project = inner_project.join("node_project"); - let photos = root_path.join("photos"); - - fs::create_dir(&rust_project).await.unwrap(); - fs::create_dir(&inner_project).await.unwrap(); - fs::create_dir(&node_project).await.unwrap(); - fs::create_dir(&photos).await.unwrap(); - - // Inner directory partially ignored by git - let partial_dir = rust_project.join("partial"); - fs::create_dir(&partial_dir).await.unwrap(); - fs::File::create(partial_dir.join("ignoreme")) - .await - .unwrap(); - fs::File::create(partial_dir.join("readme")).await.unwrap(); - - // Making rust and node projects a git repository - fs::create_dir(rust_project.join(".git")).await.unwrap(); - let gitignore = rust_project.join(".gitignore"); - let mut file = fs::File::create(gitignore).await.unwrap(); - file.write_all(b"*.file\n/target\npartial/ignoreme") - .await - .unwrap(); - fs::create_dir(node_project.join(".git")).await.unwrap(); - let gitignore = node_project.join(".gitignore"); - let mut file = fs::File::create(gitignore).await.unwrap(); - file.write_all(b"ignorable.file").await.unwrap(); - - // Populating rust project - fs::File::create(rust_project.join("Cargo.toml")) - .await - .unwrap(); - fs::File::create(rust_project.join("ignorable.file")) - .await - .unwrap(); - let rust_src_dir = rust_project.join("src"); - fs::create_dir(&rust_src_dir).await.unwrap(); - fs::File::create(rust_src_dir.join("main.rs")) - .await - .unwrap(); - let rust_target_dir = rust_project.join("target"); - fs::create_dir(&rust_target_dir).await.unwrap(); - let rust_build_dir = rust_target_dir.join("debug"); - fs::create_dir(&rust_build_dir).await.unwrap(); - fs::File::create(rust_build_dir.join("main")).await.unwrap(); - - // Populating node project - fs::File::create(node_project.join("package.json")) - .await - .unwrap(); - fs::File::create(node_project.join("ignorable.file")) - .await - .unwrap(); - let node_src_dir = node_project.join("src"); - fs::create_dir(&node_src_dir).await.unwrap(); - fs::File::create(node_src_dir.join("App.tsx")) - .await - .unwrap(); - let node_modules = node_project.join("node_modules"); - fs::create_dir(&node_modules).await.unwrap(); - let node_modules_dep = node_modules.join("react"); - fs::create_dir(&node_modules_dep).await.unwrap(); - fs::File::create(node_modules_dep.join("package.json")) - .await - .unwrap(); - - // Photos directory - for photo in ["photo1.png", "photo2.jpg", "photo3.jpeg", "text.txt"] { - fs::File::create(photos.join(photo)).await.unwrap(); - } - - root - } - - async fn run_test( - root_path: &Path, - indexer_ruler: IndexerRuler, - expected: HashSet, - ) { - let system = TaskSystem::new(); - - let handle = system - .dispatch( - Walker::new_deep( - root_path.to_path_buf(), - Arc::new(root_path.to_path_buf()), - indexer_ruler, - DummyIsoPathFactory { - root_path: Arc::new(root_path.to_path_buf()), - }, - DummyDBProxy, - ) - .unwrap(), - ) - .await - .unwrap(); - - let group = FuturesUnordered::new(); - - group.push(handle); - - let mut group = group.lend_mut(); - - let mut actual_set = HashSet::new(); - - let mut ancestors = HashSet::new(); - - while let Some((group, task_result)) = group.next().await { - let TaskStatus::Done((_task_id, TaskOutput::Out(output))) = task_result.unwrap() else { - panic!("unexpected task output") - }; - - let Output { - to_create, - accepted_ancestors, - errors, - keep_walking_tasks, - .. - } = *output - .downcast::>() - .unwrap(); - - assert!(errors.is_empty(), "errors: {errors:#?}"); - - actual_set.extend(to_create); - ancestors.extend(accepted_ancestors); - - group.extend(system.dispatch_many(keep_walking_tasks).await.unwrap()); - } - - for actual in &actual_set { - ancestors.remove(actual); - } - - if !ancestors.is_empty() { - debug!(?ancestors, "Adding ancestors to actual"); - actual_set.extend(ancestors); - } - - assert_eq!( - actual_set, - expected, - "Expected \\ Actual: {:#?};\n Actual \\ Expected: {:#?}", - expected.difference(&actual_set), - actual_set.difference(&expected) - ); - } - - #[tokio::test] - #[traced_test] - async fn test_walk_without_rules() { - let root = prepare_location().await; - let root_path = root.path(); - - let metadata = FilePathMetadata { - inode: 0, - size_in_bytes: 0, - created_at: Utc::now(), - modified_at: Utc::now(), - hidden: false, - }; - - let f = |path, is_dir| IsolatedFilePathData::new(0, root_path, path, is_dir).unwrap(); - let pub_id = FilePathPubId::new(); - let maybe_object_id = None; - - #[rustfmt::skip] - let expected = [ - WalkedEntry { pub_id: pub_id.clone(), maybe_object_id, iso_file_path: f(root_path.join("rust_project"), true), metadata }, - WalkedEntry { pub_id: pub_id.clone(), maybe_object_id, iso_file_path: f(root_path.join("rust_project/.git"), true), metadata }, - WalkedEntry { pub_id: pub_id.clone(), maybe_object_id, iso_file_path: f(root_path.join("rust_project/.gitignore"), false), metadata }, - WalkedEntry { pub_id: pub_id.clone(), maybe_object_id, iso_file_path: f(root_path.join("rust_project/Cargo.toml"), false), metadata }, - WalkedEntry { pub_id: pub_id.clone(), maybe_object_id, iso_file_path: f(root_path.join("rust_project/partial"), true), metadata }, - WalkedEntry { pub_id: pub_id.clone(), maybe_object_id, iso_file_path: f(root_path.join("rust_project/partial/readme"), false), metadata }, - WalkedEntry { pub_id: pub_id.clone(), maybe_object_id, iso_file_path: f(root_path.join("rust_project/src"), true), metadata }, - WalkedEntry { pub_id: pub_id.clone(), maybe_object_id, iso_file_path: f(root_path.join("rust_project/src/main.rs"), false), metadata }, - WalkedEntry { pub_id: pub_id.clone(), maybe_object_id, iso_file_path: f(root_path.join("inner"), true), metadata }, - WalkedEntry { pub_id: pub_id.clone(), maybe_object_id, iso_file_path: f(root_path.join("inner/node_project"), true), metadata }, - WalkedEntry { pub_id: pub_id.clone(), maybe_object_id, iso_file_path: f(root_path.join("inner/node_project/.git"), true), metadata }, - WalkedEntry { pub_id: pub_id.clone(), maybe_object_id, iso_file_path: f(root_path.join("inner/node_project/.gitignore"), false), metadata }, - WalkedEntry { pub_id: pub_id.clone(), maybe_object_id, iso_file_path: f(root_path.join("inner/node_project/package.json"), false), metadata }, - WalkedEntry { pub_id: pub_id.clone(), maybe_object_id, iso_file_path: f(root_path.join("inner/node_project/src"), true), metadata }, - WalkedEntry { pub_id: pub_id.clone(), maybe_object_id, iso_file_path: f(root_path.join("inner/node_project/src/App.tsx"), false), metadata }, - WalkedEntry { pub_id: pub_id.clone(), maybe_object_id, iso_file_path: f(root_path.join("inner/node_project/node_modules"), true), metadata }, - WalkedEntry { pub_id: pub_id.clone(), maybe_object_id, iso_file_path: f(root_path.join("inner/node_project/node_modules/react"), true), metadata }, - WalkedEntry { pub_id: pub_id.clone(), maybe_object_id, iso_file_path: f(root_path.join("inner/node_project/node_modules/react/package.json"), false), metadata }, - WalkedEntry { pub_id: pub_id.clone(), maybe_object_id, iso_file_path: f(root_path.join("photos"), true), metadata }, - WalkedEntry { pub_id: pub_id.clone(), maybe_object_id, iso_file_path: f(root_path.join("photos/photo1.png"), false), metadata }, - WalkedEntry { pub_id: pub_id.clone(), maybe_object_id, iso_file_path: f(root_path.join("photos/photo2.jpg"), false), metadata }, - WalkedEntry { pub_id: pub_id.clone(), maybe_object_id, iso_file_path: f(root_path.join("photos/photo3.jpeg"), false), metadata }, - WalkedEntry { pub_id, maybe_object_id, iso_file_path: f(root_path.join("photos/text.txt"), false), metadata }, - ] - .into_iter() - .collect::>(); - - run_test(root_path, IndexerRuler::default(), expected).await; - } - - #[tokio::test] - #[traced_test] - async fn test_only_photos() { - let root = prepare_location().await; - let root_path = root.path(); - - let metadata = FilePathMetadata { - inode: 0, - size_in_bytes: 0, - created_at: Utc::now(), - modified_at: Utc::now(), - hidden: false, - }; - - let f = |path, is_dir| IsolatedFilePathData::new(0, root_path, path, is_dir).unwrap(); - let pub_id = FilePathPubId::new(); - let maybe_object_id = None; - - #[rustfmt::skip] - let expected = [ - WalkedEntry { pub_id: pub_id.clone(), maybe_object_id, iso_file_path: f(root_path.join("photos"), true), metadata }, - WalkedEntry { pub_id: pub_id.clone(), maybe_object_id, iso_file_path: f(root_path.join("photos/photo1.png"), false), metadata }, - WalkedEntry { pub_id: pub_id.clone(), maybe_object_id, iso_file_path: f(root_path.join("photos/photo2.jpg"), false), metadata }, - WalkedEntry { pub_id, maybe_object_id, iso_file_path: f(root_path.join("photos/photo3.jpeg"), false), metadata }, - ] - .into_iter() - .collect::>(); - - run_test( - root_path, - IndexerRuler::new(vec![new_indexer_rule( - "only photos", - false, - vec![RulePerKind::AcceptFilesByGlob( - vec![], - GlobSetBuilder::new() - .add(Glob::new("{*.png,*.jpg,*.jpeg}").unwrap()) - .build() - .unwrap(), - )], - )]), - expected, - ) - .await; - } - - #[tokio::test] - #[traced_test] - async fn test_git_repos() { - let root = prepare_location().await; - let root_path = root.path(); - - let metadata = FilePathMetadata { - inode: 0, - size_in_bytes: 0, - created_at: Utc::now(), - modified_at: Utc::now(), - hidden: false, - }; - - let f = |path, is_dir| IsolatedFilePathData::new(0, root_path, path, is_dir).unwrap(); - let pub_id = FilePathPubId::new(); - let maybe_object_id = None; - - #[rustfmt::skip] - let expected = [ - WalkedEntry { pub_id: pub_id.clone(), maybe_object_id, iso_file_path: f(root_path.join("rust_project"), true), metadata }, - WalkedEntry { pub_id: pub_id.clone(), maybe_object_id, iso_file_path: f(root_path.join("rust_project/.git"), true), metadata }, - WalkedEntry { pub_id: pub_id.clone(), maybe_object_id, iso_file_path: f(root_path.join("rust_project/.gitignore"), false), metadata }, - WalkedEntry { pub_id: pub_id.clone(), maybe_object_id, iso_file_path: f(root_path.join("rust_project/Cargo.toml"), false), metadata }, - WalkedEntry { pub_id: pub_id.clone(), maybe_object_id, iso_file_path: f(root_path.join("rust_project/src"), true), metadata }, - WalkedEntry { pub_id: pub_id.clone(), maybe_object_id, iso_file_path: f(root_path.join("rust_project/src/main.rs"), false), metadata }, - WalkedEntry { pub_id: pub_id.clone(), maybe_object_id, iso_file_path: f(root_path.join("rust_project/partial"), true), metadata }, - WalkedEntry { pub_id: pub_id.clone(), maybe_object_id, iso_file_path: f(root_path.join("rust_project/partial/readme"), false), metadata }, - WalkedEntry { pub_id: pub_id.clone(), maybe_object_id, iso_file_path: f(root_path.join("inner"), true), metadata }, - WalkedEntry { pub_id: pub_id.clone(), maybe_object_id, iso_file_path: f(root_path.join("inner/node_project"), true), metadata }, - WalkedEntry { pub_id: pub_id.clone(), maybe_object_id, iso_file_path: f(root_path.join("inner/node_project/.git"), true), metadata }, - WalkedEntry { pub_id: pub_id.clone(), maybe_object_id, iso_file_path: f(root_path.join("inner/node_project/package.json"), false), metadata }, - WalkedEntry { pub_id: pub_id.clone(), maybe_object_id, iso_file_path: f(root_path.join("inner/node_project/.gitignore"), false), metadata }, - WalkedEntry { pub_id: pub_id.clone(), maybe_object_id, iso_file_path: f(root_path.join("inner/node_project/src"), true), metadata }, - WalkedEntry { pub_id: pub_id.clone(), maybe_object_id, iso_file_path: f(root_path.join("inner/node_project/src/App.tsx"), false), metadata }, - WalkedEntry { pub_id: pub_id.clone(), maybe_object_id, iso_file_path: f(root_path.join("inner/node_project/node_modules"), true), metadata }, - WalkedEntry { pub_id: pub_id.clone(), maybe_object_id, iso_file_path: f(root_path.join("inner/node_project/node_modules/react"), true), metadata }, - WalkedEntry { pub_id, maybe_object_id, iso_file_path: f(root_path.join("inner/node_project/node_modules/react/package.json"), false), metadata }, - ] - .into_iter() - .collect::>(); - - run_test( - root_path, - IndexerRuler::new(vec![new_indexer_rule( - "git repos", - false, - vec![RulePerKind::AcceptIfChildrenDirectoriesArePresent( - HashSet::from([".git".to_string()]), - )], - )]), - expected, - ) - .await; - } - - #[tokio::test] - #[traced_test] - async fn git_repos_without_deps_or_build_dirs() { - let root = prepare_location().await; - let root_path = root.path(); - - let metadata = FilePathMetadata { - inode: 0, - size_in_bytes: 0, - created_at: Utc::now(), - modified_at: Utc::now(), - hidden: false, - }; - - let f = |path, is_dir| IsolatedFilePathData::new(0, root_path, path, is_dir).unwrap(); - let pub_id = FilePathPubId::new(); - let maybe_object_id = None; - - #[rustfmt::skip] - let expected = [ - WalkedEntry { pub_id: pub_id.clone(), maybe_object_id, iso_file_path: f(root_path.join("rust_project"), true), metadata }, - WalkedEntry { pub_id: pub_id.clone(), maybe_object_id, iso_file_path: f(root_path.join("rust_project/.git"), true), metadata }, - WalkedEntry { pub_id: pub_id.clone(), maybe_object_id, iso_file_path: f(root_path.join("rust_project/.gitignore"), false), metadata }, - WalkedEntry { pub_id: pub_id.clone(), maybe_object_id, iso_file_path: f(root_path.join("rust_project/Cargo.toml"), false), metadata }, - WalkedEntry { pub_id: pub_id.clone(), maybe_object_id, iso_file_path: f(root_path.join("rust_project/partial"), true), metadata }, - WalkedEntry { pub_id: pub_id.clone(), maybe_object_id, iso_file_path: f(root_path.join("rust_project/partial/readme"), false), metadata }, - WalkedEntry { pub_id: pub_id.clone(), maybe_object_id, iso_file_path: f(root_path.join("rust_project/src"), true), metadata }, - WalkedEntry { pub_id: pub_id.clone(), maybe_object_id, iso_file_path: f(root_path.join("rust_project/src/main.rs"), false), metadata }, - WalkedEntry { pub_id: pub_id.clone(), maybe_object_id, iso_file_path: f(root_path.join("inner"), true), metadata }, - WalkedEntry { pub_id: pub_id.clone(), maybe_object_id, iso_file_path: f(root_path.join("inner/node_project"), true), metadata }, - WalkedEntry { pub_id: pub_id.clone(), maybe_object_id, iso_file_path: f(root_path.join("inner/node_project/.git"), true), metadata }, - WalkedEntry { pub_id: pub_id.clone(), maybe_object_id, iso_file_path: f(root_path.join("inner/node_project/.gitignore"), false), metadata }, - WalkedEntry { pub_id: pub_id.clone(), maybe_object_id, iso_file_path: f(root_path.join("inner/node_project/package.json"), false), metadata }, - WalkedEntry { pub_id: pub_id.clone(), maybe_object_id, iso_file_path: f(root_path.join("inner/node_project/src"), true), metadata }, - WalkedEntry { pub_id, maybe_object_id, iso_file_path: f(root_path.join("inner/node_project/src/App.tsx"), false), metadata }, - ] - .into_iter() - .collect::>(); - - run_test( - root_path, - IndexerRuler::new(vec![ - new_indexer_rule( - "git repos", - false, - vec![RulePerKind::AcceptIfChildrenDirectoriesArePresent( - HashSet::from([".git".into()]), - )], - ), - new_indexer_rule( - "reject node_modules", - false, - vec![RulePerKind::RejectFilesByGlob( - vec![], - GlobSetBuilder::new() - .add(Glob::new("{**/node_modules/*,**/node_modules}").unwrap()) - .build() - .unwrap(), - )], - ), - new_indexer_rule( - "reject rust build dir", - false, - vec![RulePerKind::RejectFilesByGlob( - vec![], - GlobSetBuilder::new() - .add(Glob::new("{**/target/*,**/target}").unwrap()) - .build() - .unwrap(), - )], - ), - ]), - expected, - ) - .await; - } -} diff --git a/core/crates/heavy-lifting/src/indexer/tasks/walker/rules.rs b/core/crates/heavy-lifting/src/indexer/tasks/walker/rules.rs deleted file mode 100644 index f8f3fd0e1..000000000 --- a/core/crates/heavy-lifting/src/indexer/tasks/walker/rules.rs +++ /dev/null @@ -1,261 +0,0 @@ -use crate::{indexer, NonCriticalError}; - -use sd_core_file_path_helper::{FilePathMetadata, IsolatedFilePathData}; -use sd_core_indexer_rules::{IndexerRuler, MetadataForIndexerRules, RuleKind}; - -use sd_utils::error::FileIOError; - -use std::{ - collections::{hash_map::Entry, HashMap, HashSet}, - path::{Path, PathBuf}, - sync::Arc, -}; - -use futures_concurrency::future::Join; -use tokio::fs; -use tracing::{instrument, trace}; - -use super::{ - entry::{ToWalkEntry, WalkingEntry}, - InnerMetadata, IsoFilePathFactory, WalkedEntry, -}; - -pub(super) async fn apply_indexer_rules( - paths_and_metadatas: &mut HashMap, - indexer_ruler: &IndexerRuler, - errors: &mut Vec, -) -> HashMap>)> { - paths_and_metadatas - .drain() - // TODO: Hard ignoring symlinks for now, but this should be configurable - .filter(|(_, metadata)| !metadata.is_symlink) - .map(|(current_path, metadata)| async { - indexer_ruler - .apply_all(¤t_path, &metadata) - .await - .map(|acceptance_per_rule_kind| { - (current_path, (metadata, acceptance_per_rule_kind)) - }) - .map_err(|e| indexer::NonCriticalIndexerError::IndexerRule(e.to_string())) - }) - .collect::>() - .join() - .await - .into_iter() - .filter_map(|res| res.map_err(|e| errors.push(e.into())).ok()) - .collect() -} - -pub(super) async fn process_rules_results( - root: &Arc, - iso_file_path_factory: &impl IsoFilePathFactory, - parent_dir_accepted_by_its_children: Option, - paths_metadatas_and_acceptance: &mut HashMap< - PathBuf, - (InnerMetadata, HashMap>), - >, - maybe_to_keep_walking: &mut Option>, - collect_rejected_paths: bool, - errors: &mut Vec, -) -> ( - HashMap, - HashSet, - Vec, -) { - let (accepted, accepted_ancestors, rejected) = segregate_paths( - root, - iso_file_path_factory, - paths_metadatas_and_acceptance.drain(), - parent_dir_accepted_by_its_children, - maybe_to_keep_walking, - collect_rejected_paths, - errors, - ); - - ( - accepted, - accepted_ancestors - .into_iter() - .map(|(ancestor_iso_file_path, ancestor_path)| async move { - fs::metadata(&ancestor_path) - .await - .map_err(|e| { - indexer::NonCriticalIndexerError::Metadata( - FileIOError::from((&ancestor_path, e)).to_string(), - ) - }) - .and_then(|metadata| { - FilePathMetadata::from_path(&ancestor_path, &metadata) - .map(|metadata| { - WalkingEntry { - iso_file_path: ancestor_iso_file_path, - metadata, - } - .into() - }) - .map_err(|e| { - indexer::NonCriticalIndexerError::FilePathMetadata(e.to_string()) - }) - }) - }) - .collect::>() - .join() - .await - .into_iter() - .filter_map(|res| res.map_err(|e| errors.push(e.into())).ok()) - .collect(), - rejected, - ) -} - -fn segregate_paths( - root: &Arc, - iso_file_path_factory: &impl IsoFilePathFactory, - paths_metadatas_and_acceptance: impl IntoIterator< - Item = (PathBuf, (InnerMetadata, HashMap>)), - >, - parent_dir_accepted_by_its_children: Option, - maybe_to_keep_walking: &mut Option>, - collect_rejected_paths: bool, - errors: &mut Vec, -) -> ( - HashMap, - HashMap, PathBuf>, - Vec, -) { - let root = root.as_ref(); - - let mut accepted = HashMap::new(); - let mut accepted_ancestors = HashMap::new(); - let mut rejected = Vec::new(); - - for (current_path, (metadata, acceptance_per_rule_kind)) in paths_metadatas_and_acceptance { - // Accept by children has three states, - // None if we don't now yet or if this check doesn't apply - // Some(true) if this check applies and it passes - // Some(false) if this check applies and it was rejected - // and we pass the current parent state to its children - let mut accept_by_children_dir = parent_dir_accepted_by_its_children; - - if !reject_path( - ¤t_path, - &metadata, - &acceptance_per_rule_kind, - &mut accept_by_children_dir, - maybe_to_keep_walking, - ) && accept_by_children_dir.unwrap_or(true) - { - accept_path_and_ancestors( - current_path, - metadata, - root, - &mut accepted, - iso_file_path_factory, - &mut accepted_ancestors, - errors, - ); - - continue; - } - - if collect_rejected_paths { - rejected.push(current_path); - } - } - - (accepted, accepted_ancestors, rejected) -} - -#[instrument(skip_all, fields(current_path = %current_path.display()))] -fn reject_path( - current_path: &Path, - metadata: &InnerMetadata, - acceptance_per_rule_kind: &HashMap>, - accept_by_children_dir: &mut Option, - maybe_to_keep_walking: &mut Option>, -) -> bool { - IndexerRuler::rejected_by_reject_glob(acceptance_per_rule_kind) - || IndexerRuler::rejected_by_git_ignore(acceptance_per_rule_kind) - || (metadata.is_dir() - && process_and_maybe_reject_by_directory_rules( - current_path, - acceptance_per_rule_kind, - accept_by_children_dir, - maybe_to_keep_walking, - )) || IndexerRuler::rejected_by_accept_glob(acceptance_per_rule_kind) -} - -fn process_and_maybe_reject_by_directory_rules( - current_path: &Path, - acceptance_per_rule_kind: &HashMap>, - accept_by_children_dir: &mut Option, - maybe_to_keep_walking: &mut Option>, -) -> bool { - // If it is a directory, first we check if we must reject it and its children entirely - if IndexerRuler::rejected_by_children_directories(acceptance_per_rule_kind) { - return true; - } - - // Then we check if we must accept it and its children - if let Some(accepted_by_children_rules) = - acceptance_per_rule_kind.get(&RuleKind::AcceptIfChildrenDirectoriesArePresent) - { - if accepted_by_children_rules.iter().any(|accept| *accept) { - *accept_by_children_dir = Some(true); - } - - // If it wasn't accepted then we mark as rejected - if accept_by_children_dir.is_none() { - trace!( - "Rejected because it didn't passed in any \ - `RuleKind::AcceptIfChildrenDirectoriesArePresent` rule", - ); - *accept_by_children_dir = Some(false); - } - } - - // Then we mark this directory to maybe be walked in too - if let Some(ref mut to_keep_walking) = maybe_to_keep_walking { - to_keep_walking.push(ToWalkEntry { - path: current_path.to_path_buf(), - parent_dir_accepted_by_its_children: *accept_by_children_dir, - }); - } - - false -} - -fn accept_path_and_ancestors( - current_path: PathBuf, - metadata: InnerMetadata, - root: &Path, - accepted: &mut HashMap, - iso_file_path_factory: &impl IsoFilePathFactory, - accepted_ancestors: &mut HashMap, PathBuf>, - errors: &mut Vec, -) { - // If the ancestors directories wasn't indexed before, now we do - for ancestor in current_path - .ancestors() - .skip(1) // Skip the current directory as it was already indexed - .take_while(|&ancestor| ancestor != root) - { - if let Ok(iso_file_path) = iso_file_path_factory.build(ancestor, true).map_err(|e| { - errors.push(indexer::NonCriticalIndexerError::IsoFilePath(e.to_string()).into()); - }) { - match accepted_ancestors.entry(iso_file_path) { - Entry::Occupied(_) => { - // If we already accepted this ancestor, then it will contain - // also all if its ancestors too, so we can stop here - break; - } - Entry::Vacant(entry) => { - trace!(ancestor = %ancestor.display(), "Accepted ancestor"); - entry.insert(ancestor.to_path_buf()); - } - } - } - } - - accepted.insert(current_path, metadata); -} diff --git a/core/crates/heavy-lifting/src/indexer/tasks/walker/save_state.rs b/core/crates/heavy-lifting/src/indexer/tasks/walker/save_state.rs deleted file mode 100644 index 2dd66fc50..000000000 --- a/core/crates/heavy-lifting/src/indexer/tasks/walker/save_state.rs +++ /dev/null @@ -1,219 +0,0 @@ -use crate::{Error, NonCriticalError}; - -use sd_core_file_path_helper::IsolatedFilePathData; -use sd_core_indexer_rules::{IndexerRuler, RuleKind}; -use sd_core_prisma_helpers::file_path_pub_and_cas_ids; - -use std::{ - collections::{HashMap, HashSet}, - path::PathBuf, - sync::Arc, - time::Duration, -}; - -use sd_task_system::{SerializableTask, TaskId}; -use serde::{Deserialize, Serialize}; - -use super::{ - entry::{ToWalkEntry, WalkingEntry}, - metadata::InnerMetadata, - IsoFilePathFactory, WalkedEntry, Walker, WalkerDBProxy, WalkerStage, -}; - -#[derive(Debug, Serialize, Deserialize)] -pub(super) struct WalkDirSaveState { - id: TaskId, - is_shallow: bool, - - entry: ToWalkEntry, - root: Arc, - entry_iso_file_path: IsolatedFilePathData<'static>, - - stage: WalkerStageSaveState, - - errors: Vec, - scan_time: Duration, -} - -#[derive(Debug, Serialize, Deserialize)] -pub(super) enum WalkerStageSaveState { - Start, - CollectingMetadata { - found_paths: Vec, - }, - CheckingIndexerRules { - paths_and_metadatas: HashMap, - }, - ProcessingRulesResults { - paths_metadatas_and_acceptance: - HashMap>)>, - }, - GatheringFilePathsToRemove { - accepted_paths: HashMap, - maybe_to_keep_walking: Option>, - accepted_ancestors: HashSet, - non_indexed_paths: Vec, - }, - Finalize { - walking_entries: Vec, - accepted_ancestors: HashSet, - to_remove_entries: Vec, - maybe_to_keep_walking: Option>, - non_indexed_paths: Vec, - }, -} - -impl From for WalkerStageSaveState { - fn from(stage: WalkerStage) -> Self { - match stage { - // We can't store the current state of `ReadDirStream` so we start again from the beginning - WalkerStage::Start | WalkerStage::Walking { .. } => Self::Start, - WalkerStage::CollectingMetadata { found_paths } => { - Self::CollectingMetadata { found_paths } - } - WalkerStage::CheckingIndexerRules { - paths_and_metadatas, - } => Self::CheckingIndexerRules { - paths_and_metadatas, - }, - WalkerStage::ProcessingRulesResults { - paths_metadatas_and_acceptance, - } => Self::ProcessingRulesResults { - paths_metadatas_and_acceptance, - }, - WalkerStage::GatheringFilePathsToRemove { - accepted_paths, - maybe_to_keep_walking, - accepted_ancestors, - non_indexed_paths, - } => Self::GatheringFilePathsToRemove { - accepted_paths, - maybe_to_keep_walking, - accepted_ancestors, - non_indexed_paths, - }, - WalkerStage::Finalize { - walking_entries, - accepted_ancestors, - to_remove_entries, - maybe_to_keep_walking, - non_indexed_paths, - } => Self::Finalize { - walking_entries, - accepted_ancestors, - to_remove_entries, - maybe_to_keep_walking, - non_indexed_paths, - }, - } - } -} - -impl From for WalkerStage { - fn from(value: WalkerStageSaveState) -> Self { - match value { - WalkerStageSaveState::Start => Self::Start, - WalkerStageSaveState::CollectingMetadata { found_paths } => { - Self::CollectingMetadata { found_paths } - } - WalkerStageSaveState::CheckingIndexerRules { - paths_and_metadatas, - } => Self::CheckingIndexerRules { - paths_and_metadatas, - }, - WalkerStageSaveState::ProcessingRulesResults { - paths_metadatas_and_acceptance, - } => Self::ProcessingRulesResults { - paths_metadatas_and_acceptance, - }, - WalkerStageSaveState::GatheringFilePathsToRemove { - accepted_paths, - maybe_to_keep_walking, - accepted_ancestors, - non_indexed_paths, - } => Self::GatheringFilePathsToRemove { - accepted_paths, - maybe_to_keep_walking, - accepted_ancestors, - non_indexed_paths, - }, - WalkerStageSaveState::Finalize { - walking_entries, - accepted_ancestors, - to_remove_entries, - maybe_to_keep_walking, - non_indexed_paths, - } => Self::Finalize { - walking_entries, - accepted_ancestors, - to_remove_entries, - maybe_to_keep_walking, - non_indexed_paths, - }, - } - } -} - -impl SerializableTask for Walker -where - DBProxy: WalkerDBProxy, - IsoPathFactory: IsoFilePathFactory, -{ - type SerializeError = rmp_serde::encode::Error; - type DeserializeError = rmp_serde::decode::Error; - type DeserializeCtx = (IndexerRuler, DBProxy, IsoPathFactory); - - async fn serialize(self) -> Result, Self::SerializeError> { - let Self { - id, - entry, - root, - entry_iso_file_path, - stage, - errors, - scan_time, - is_shallow, - .. - } = self; - rmp_serde::to_vec_named(&WalkDirSaveState { - id, - is_shallow, - entry, - root, - entry_iso_file_path, - stage: stage.into(), - errors, - scan_time, - }) - } - - async fn deserialize( - data: &[u8], - (indexer_ruler, db_proxy, iso_file_path_factory): Self::DeserializeCtx, - ) -> Result { - rmp_serde::from_slice(data).map( - |WalkDirSaveState { - id, - entry, - root, - entry_iso_file_path, - stage, - errors, - scan_time, - is_shallow, - }| Self { - id, - entry, - root, - entry_iso_file_path, - indexer_ruler, - iso_file_path_factory, - db_proxy, - stage: stage.into(), - errors, - scan_time, - is_shallow, - }, - ) - } -} diff --git a/core/crates/heavy-lifting/src/job_system/error.rs b/core/crates/heavy-lifting/src/job_system/error.rs deleted file mode 100644 index 4ae944b2e..000000000 --- a/core/crates/heavy-lifting/src/job_system/error.rs +++ /dev/null @@ -1,76 +0,0 @@ -use sd_task_system::{DispatcherShutdownError, Task}; -use sd_utils::error::FileIOError; - -use prisma_client_rust::QueryError; - -use super::{job::JobName, report::ReportError, JobId}; - -#[derive(thiserror::Error, Debug)] -pub enum JobSystemError { - #[error("job not found: ")] - NotFound(JobId), - #[error("job already running: ")] - AlreadyRunning { - new_id: JobId, - job_name: JobName, - already_running_id: JobId, - }, - - #[error("failed to load job reports from database to resume jobs: {0}")] - LoadReportsForResume(#[from] QueryError), - - #[error("failed to serialize job to be saved and resumed later: {0}")] - Serialize(#[from] rmp_serde::encode::Error), - - #[error("failed to deserialize job to be resumed: {0}")] - Deserialize(#[from] rmp_serde::decode::Error), - - #[error("failed to save or load jobs on disk: {0}")] - StoredJobs(FileIOError), - - #[error(transparent)] - Report(#[from] ReportError), - - #[error("internal job panic! ")] - Panic(JobId), -} - -impl From for rspc::Error { - fn from(e: JobSystemError) -> Self { - match e { - JobSystemError::NotFound(_) => { - Self::with_cause(rspc::ErrorCode::NotFound, e.to_string(), e) - } - - JobSystemError::AlreadyRunning { .. } => { - Self::with_cause(rspc::ErrorCode::Conflict, e.to_string(), e) - } - - JobSystemError::Report(e) => e.into(), - - _ => Self::with_cause(rspc::ErrorCode::InternalServerError, e.to_string(), e), - } - } -} - -#[derive(thiserror::Error, Debug)] -pub enum DispatcherError { - #[error("job canceled: ")] - JobCanceled(JobId), - #[error("system entered on shutdown mode ", .0.len())] - Shutdown(Vec>>), -} - -#[derive(Debug, thiserror::Error)] -pub enum JobErrorOrDispatcherError> { - #[error(transparent)] - JobError(#[from] JobError), - #[error(transparent)] - Dispatcher(#[from] DispatcherError), -} - -impl From> for DispatcherError { - fn from(DispatcherShutdownError(tasks): DispatcherShutdownError) -> Self { - Self::Shutdown(tasks) - } -} diff --git a/core/crates/heavy-lifting/src/job_system/job.rs b/core/crates/heavy-lifting/src/job_system/job.rs deleted file mode 100644 index ec664c327..000000000 --- a/core/crates/heavy-lifting/src/job_system/job.rs +++ /dev/null @@ -1,1189 +0,0 @@ -use crate::{Error, NonCriticalError, UpdateEvent}; - -use sd_core_sync::SyncManager; - -use sd_prisma::prisma::PrismaClient; -use sd_task_system::{ - BaseTaskDispatcher, Task, TaskDispatcher, TaskHandle, TaskRemoteController, TaskSystemError, -}; - -use std::{ - collections::{hash_map::DefaultHasher, VecDeque}, - fmt, - hash::{Hash, Hasher}, - marker::PhantomData, - ops::{Deref, DerefMut}, - panic::AssertUnwindSafe, - path::Path, - pin::pin, - sync::Arc, - time::Duration, -}; - -use async_channel as chan; -use chrono::{DateTime, Utc}; -use futures::{stream, Future, FutureExt, StreamExt}; -use futures_concurrency::{ - future::{Join, TryJoin}, - stream::Merge, -}; -use serde::{Deserialize, Serialize}; -use specta::Type; -use strum::{Display, EnumString}; -use tokio::{ - spawn, - sync::{oneshot, watch, Mutex}, - time::Instant, -}; -use tracing::{debug, error, instrument, trace, warn, Instrument, Level}; -use uuid::Uuid; - -use super::{ - error::DispatcherError, - report::{ - Report, ReportBuilder, ReportInputMetadata, ReportMetadata, ReportOutputMetadata, Status, - }, - Command, JobId, JobSystemError, SerializableJob, SerializedTasks, -}; - -#[derive( - Debug, Serialize, Deserialize, EnumString, Display, Clone, Copy, Type, Hash, PartialEq, Eq, -)] -#[strum(use_phf, serialize_all = "snake_case")] -pub enum JobName { - Indexer, - FileIdentifier, - MediaProcessor, - // TODO: Add more job names as needed - Copy, - Move, - Delete, - Erase, - FileValidator, -} - -pub enum ReturnStatus { - Completed(JobReturn), - Shutdown(Result>, rmp_serde::encode::Error>), - Canceled(JobReturn), -} - -impl fmt::Debug for ReturnStatus { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::Completed(job_return) => f.debug_tuple("Completed").field(job_return).finish(), - Self::Shutdown(_) => f.write_str("Shutdown()"), - Self::Canceled(job_return) => f.debug_tuple("Canceled").field(job_return).finish(), - } - } -} - -pub enum ProgressUpdate { - TaskCount(u64), - CompletedTaskCount(u64), - Message(String), - Phase(String), -} - -impl ProgressUpdate { - pub fn message(message: impl Into) -> Self { - Self::Message(message.into()) - } - - pub fn phase(phase: impl Into) -> Self { - Self::Phase(phase.into()) - } -} - -pub trait OuterContext: Send + Sync + Clone + 'static { - fn id(&self) -> Uuid; - fn db(&self) -> &Arc; - fn sync(&self) -> &SyncManager; - fn invalidate_query(&self, query: &'static str); - fn query_invalidator(&self) -> impl Fn(&'static str) + Send + Sync; - fn report_update(&self, update: UpdateEvent); - fn get_data_directory(&self) -> &Path; -} - -pub trait JobContext: OuterContext { - fn new(report: Report, ctx: OuterCtx) -> Self; - fn progress( - &self, - updates: impl IntoIterator + Send, - ) -> impl Future + Send; - fn progress_msg(&self, msg: impl Into) -> impl Future + Send { - let msg = msg.into(); - async move { - self.progress([ProgressUpdate::Message(msg)]).await; - } - } - fn report(&self) -> impl Future + Send> + Send; - fn report_mut(&self) -> impl Future + Send> + Send; - fn get_outer_ctx(&self) -> OuterCtx; -} - -pub trait Job: Send + Sync + Hash + 'static { - const NAME: JobName; - - #[allow(unused_variables)] - fn resume_tasks( - &mut self, - dispatcher: &JobTaskDispatcher, - ctx: &impl JobContext, - serialized_tasks: SerializedTasks, - ) -> impl Future> + Send { - async move { Ok(()) } - } - - fn run( - self, - dispatcher: JobTaskDispatcher, - ctx: impl JobContext, - ) -> impl Future> + Send; -} - -pub trait IntoJob -where - J: Job + SerializableJob, - OuterCtx: OuterContext, - JobCtx: JobContext, -{ - fn into_job(self) -> Box>; -} - -impl IntoJob for J -where - J: Job + SerializableJob, - OuterCtx: OuterContext, - JobCtx: JobContext, -{ - fn into_job(self) -> Box> { - let id = JobId::now_v7(); - - Box::new(JobHolder { - id, - job: self, - run_time: Duration::ZERO, - report: ReportBuilder::new(id, J::NAME).build(), - next_jobs: VecDeque::new(), - _ctx: PhantomData, - }) - } -} - -impl IntoJob for JobEnqueuer -where - J: Job + SerializableJob, - OuterCtx: OuterContext, - JobCtx: JobContext, -{ - fn into_job(self) -> Box> { - self.build() - } -} - -#[derive(Debug)] -pub struct JobReturn { - data: JobOutputData, - metadata: Vec, - non_critical_errors: Vec, -} - -impl JobReturn { - #[must_use] - pub fn builder() -> JobReturnBuilder { - JobReturnBuilder { - job_return: Self::default(), - } - } -} - -impl Default for JobReturn { - fn default() -> Self { - Self { - data: JobOutputData::Empty, - metadata: vec![], - non_critical_errors: vec![], - } - } -} - -#[derive(Debug, Default)] -pub struct JobReturnBuilder { - job_return: JobReturn, -} - -impl JobReturnBuilder { - #[must_use] - pub const fn with_data(mut self, data: JobOutputData) -> Self { - self.job_return.data = data; - self - } - - #[must_use] - pub fn with_metadata(mut self, metadata: impl Into>) -> Self { - self.job_return.metadata = metadata.into(); - self - } - - #[must_use] - pub fn with_non_critical_errors(mut self, errors: Vec) -> Self { - if self.job_return.non_critical_errors.is_empty() { - self.job_return.non_critical_errors = errors; - } else { - self.job_return.non_critical_errors.extend(errors); - } - self - } - - #[must_use] - pub fn build(self) -> JobReturn { - self.job_return - } -} - -#[derive(Serialize, Type)] -pub struct JobOutput { - id: JobId, - status: Status, - job_name: JobName, - data: JobOutputData, - metadata: Vec, - non_critical_errors: Vec, -} - -impl JobOutput { - #[instrument( - skip_all, - fields( - name = %report.name, - non_critical_errors_count = non_critical_errors.len(), - ) - )] - pub fn prepare_output_and_report( - JobReturn { - data, - metadata, - non_critical_errors, - }: JobReturn, - report: &mut Report, - ) -> Self { - if non_critical_errors.is_empty() { - report.status = Status::Completed; - debug!("Job completed"); - } else { - report.status = Status::CompletedWithErrors; - report.non_critical_errors.extend(non_critical_errors); - - warn!( - non_critical_errors = ?report.non_critical_errors, - "Job completed with errors;", - ); - } - - report.metadata.extend(metadata.into_iter().map(Into::into)); - - report.completed_at = Some(Utc::now()); - - Self { - id: report.id, - status: report.status, - job_name: report.name, - data, - metadata: report.metadata.clone(), - non_critical_errors: report.non_critical_errors.clone(), - } - } -} - -#[derive(Debug, Serialize, Type)] -pub enum JobOutputData { - Empty, - // TODO: Add more types as needed -} - -pub struct JobEnqueuer -where - J: Job + SerializableJob, - OuterCtx: OuterContext, - JobCtx: JobContext, -{ - id: JobId, - job: J, - report_builder: ReportBuilder, - next_jobs: VecDeque>>, - _ctx: PhantomData, -} - -impl JobEnqueuer -where - J: Job + SerializableJob, - OuterCtx: OuterContext, - JobCtx: JobContext, -{ - fn build(self) -> Box> { - Box::new(JobHolder { - id: self.id, - job: self.job, - run_time: Duration::ZERO, - report: self.report_builder.build(), - next_jobs: self.next_jobs, - _ctx: self._ctx, - }) - } - - pub fn new(job: J) -> Self { - let id = JobId::now_v7(); - Self { - id, - job, - report_builder: ReportBuilder::new(id, J::NAME), - next_jobs: VecDeque::new(), - _ctx: PhantomData, - } - } - - #[must_use] - pub fn with_action(mut self, action: impl Into) -> Self { - self.report_builder = self.report_builder.with_action(action); - self - } - - #[must_use] - pub fn with_parent_id(mut self, parent_id: JobId) -> Self { - self.report_builder = self.report_builder.with_parent_id(parent_id); - self - } - - #[must_use] - pub fn with_metadata(mut self, metadata: ReportInputMetadata) -> Self { - self.report_builder = self.report_builder.with_metadata(metadata); - self - } - - #[must_use] - pub fn enqueue_next(mut self, next: impl Job + SerializableJob) -> Self { - let next_job_order = self.next_jobs.len() + 1; - - let mut child_job_builder = JobEnqueuer::new(next).with_parent_id(self.id); - - if let Some(parent_action) = &self.report_builder.action { - child_job_builder = - child_job_builder.with_action(format!("{parent_action}-{next_job_order}")); - } - - self.next_jobs.push_back(child_job_builder.build()); - - self - } -} - -pub struct JobHolder -where - J: Job + SerializableJob, - OuterCtx: OuterContext, - JobCtx: JobContext, -{ - pub(super) id: JobId, - pub(super) job: J, - pub(super) report: Report, - pub(super) run_time: Duration, - pub(super) next_jobs: VecDeque>>, - pub(super) _ctx: PhantomData, -} - -pub struct JobHandle> { - pub(crate) id: JobId, - pub(crate) start_time: Instant, - pub(crate) run_time: Duration, - pub(crate) is_running: bool, - pub(crate) next_jobs: VecDeque>>, - pub(crate) ctx: JobCtx, - pub(crate) commands_tx: chan::Sender<(Command, oneshot::Sender<()>)>, -} - -impl> JobHandle { - #[instrument(skip(self, outer_ack_tx), fields(job_id = %self.id))] - pub async fn send_command( - &mut self, - command: Command, - outer_ack_tx: oneshot::Sender>, - ) { - trace!("JobHandle sending command"); - - let (ack_tx, ack_rx) = oneshot::channel(); - - let res = if self.commands_tx.send((command, ack_tx)).await.is_err() { - warn!("Tried to send command to a job that was already completed"); - - Ok(()) - } else { - ack_rx - .await - .expect("inner ack channel closed before sending response to handle a job command"); - - match self.execute_command(command).await { - Ok(()) => self.command_children(command).await, - Err(e) => Err(e), - } - }; - - if res.is_ok() { - match command { - Command::Pause | Command::Cancel | Command::Shutdown => self.is_running = false, - Command::Resume => self.is_running = true, - } - } - - outer_ack_tx - .send(res) - .unwrap_or_else(|_| panic!("ack channel closed before sending {command:?} response")); - } - - #[instrument(skip_all, err)] - async fn execute_command(&mut self, command: Command) -> Result<(), JobSystemError> { - let (new_status, completed_at) = match command { - Command::Pause => (Status::Paused, None), - Command::Resume => (Status::Running, None), - Command::Cancel => (Status::Canceled, Some(Utc::now())), - Command::Shutdown => { - // We don't need to do anything here, we will handle when the job returns its output - return Ok(()); - } - }; - - { - let mut report = self.ctx.report_mut().await; - - report.status = new_status; - report.completed_at = completed_at; - - report.update(self.ctx.db()).await?; - } - - Ok(()) - } - - #[instrument(skip_all, err)] - async fn command_children(&mut self, command: Command) -> Result<(), JobSystemError> { - let (new_status, completed_at) = match command { - Command::Pause | Command::Shutdown => (Status::Paused, None), - Command::Resume => (Status::Queued, None), - Command::Cancel => (Status::Canceled, Some(Utc::now())), - }; - - self.next_jobs - .iter_mut() - .map(|dyn_job| dyn_job.report_mut()) - .map(|next_job_report| async { - next_job_report.status = new_status; - next_job_report.completed_at = completed_at; - - trace!( - %next_job_report.id, - "Parent job sent command to children job;", - ); - - next_job_report.update(self.ctx.db()).await - }) - .collect::>() - .try_join() - .await - .map(|_| ()) - .map_err(Into::into) - } - - #[instrument( - skip(self), - fields(job_id = %self.id), - ret(level = Level::TRACE), - err, - )] - pub async fn register_start( - &mut self, - start_time: DateTime, - ) -> Result<(), JobSystemError> { - trace!("JobHandle registering start of job"); - - let Self { next_jobs, ctx, .. } = self; - let db = ctx.db(); - - let now = Utc::now(); - - { - let mut report = ctx.report_mut().await; - - report.status = Status::Running; - if report.started_at.is_none() { - report.started_at = Some(start_time); - } - - // If the report doesn't have a created_at date, it's a new report - if report.created_at.is_none() { - report.create(db, now).await?; - } else { - // Otherwise it can be a job being resumed or a children job that was already been created - report.update(db).await?; - } - } - - // Registering children jobs - let res = next_jobs - .iter_mut() - .enumerate() - .map(|(idx, dyn_job)| (idx, dyn_job.report_mut())) - .map(|(idx, next_job_report)| async move { - trace!( - %next_job_report.id, - "Parent job registering children;", - ); - if next_job_report.created_at.is_none() { - next_job_report - .create(db, now + Duration::from_secs((idx + 1) as u64)) - .await - } else { - Ok(()) - } - }) - .collect::>() - .try_join() - .await - .map(|_| ()) - .map_err(Into::into); - - ctx.invalidate_query("jobs.isActive"); - ctx.invalidate_query("jobs.reports"); - - res - } - - #[instrument( - skip_all, - fields( - id = %self.id, - - ), - err - )] - pub async fn complete_job( - &mut self, - job_return: JobReturn, - ) -> Result { - let Self { ctx, .. } = self; - - let mut report = ctx.report_mut().await; - - trace!("JobHandle completing"); - - let output = JobOutput::prepare_output_and_report(job_return, &mut report); - - report.update(ctx.db()).await?; - - trace!("JobHandle completed"); - - Ok(output) - } - - #[instrument( - skip(self), - fields( - id = %self.id, - ), - err - )] - pub async fn failed_job(&mut self, e: &Error) -> Result<(), JobSystemError> { - trace!("JobHandle registering failed job"); - - let db = self.ctx.db(); - { - let mut report = self.ctx.report_mut().await; - - error!( - job_name = %report.name, - "Job failed with a critical error;", - ); - - report.status = Status::Failed; - report.critical_error = Some(e.to_string()); - report.completed_at = Some(Utc::now()); - - report.update(db).await?; - } - - trace!("JobHandle sending cancel command to children due to failure"); - - self.command_children(Command::Cancel).await - } - - #[instrument( - skip(self), - fields( - id = %self.id, - ), - err - )] - pub async fn cancel_job( - &mut self, - JobReturn { - data, - metadata, - non_critical_errors, - }: JobReturn, - ) -> Result { - trace!("JobHandle canceling job"); - let db = self.ctx.db(); - - let output = { - let mut report = self.ctx.report_mut().await; - - debug!( - job_name = %report.name, - "Job canceled, we will cancel all children jobs;", - ); - - report.status = Status::Canceled; - report.non_critical_errors.extend(non_critical_errors); - report.metadata.extend(metadata.into_iter().map(Into::into)); - report.completed_at = Some(Utc::now()); - - report.update(db).await?; - - JobOutput { - id: report.id, - status: report.status, - job_name: report.name, - data, - metadata: report.metadata.clone(), - non_critical_errors: report.non_critical_errors.clone(), - } - }; - - trace!("JobHandle sending cancel command to children"); - - self.command_children(Command::Cancel).await?; - - Ok(output) - } -} - -#[async_trait::async_trait] -pub trait DynJob>: - Send + Sync + 'static -{ - fn id(&self) -> JobId; - - fn job_name(&self) -> JobName; - - fn hash(&self) -> u64; - - fn report_mut(&mut self) -> &mut Report; - - fn set_next_jobs(&mut self, next_jobs: VecDeque>>); - - fn next_jobs(&self) -> &VecDeque>>; - - async fn serialize(self: Box) -> Result>, rmp_serde::encode::Error>; - - fn dispatch( - self: Box, - base_dispatcher: BaseTaskDispatcher, - ctx: OuterCtx, - done_tx: chan::Sender<(JobId, Result)>, - ) -> JobHandle; - - fn resume( - self: Box, - base_dispatcher: BaseTaskDispatcher, - ctx: OuterCtx, - serialized_tasks: Option, - done_tx: chan::Sender<(JobId, Result)>, - ) -> JobHandle; -} - -#[async_trait::async_trait] -impl DynJob for JobHolder -where - J: Job + SerializableJob, - OuterCtx: OuterContext, - JobCtx: JobContext, -{ - fn id(&self) -> JobId { - self.id - } - - fn job_name(&self) -> JobName { - J::NAME - } - - fn hash(&self) -> u64 { - let mut hasher = DefaultHasher::new(); - J::NAME.hash(&mut hasher); - self.job.hash(&mut hasher); - hasher.finish() - } - - fn report_mut(&mut self) -> &mut Report { - &mut self.report - } - - fn set_next_jobs(&mut self, next_jobs: VecDeque>>) { - self.next_jobs = next_jobs; - } - - fn next_jobs(&self) -> &VecDeque>> { - &self.next_jobs - } - - async fn serialize(self: Box) -> Result>, rmp_serde::encode::Error> { - self.job.serialize().await - } - - #[instrument(skip_all, fields(id = %self.id))] - fn dispatch( - self: Box, - base_dispatcher: BaseTaskDispatcher, - ctx: OuterCtx, - done_tx: chan::Sender<(JobId, Result)>, - ) -> JobHandle { - let (commands_tx, commands_rx) = chan::bounded(8); - - let ctx = JobCtx::new(self.report, ctx); - - trace!("Dispatching job"); - - spawn({ - let id = self.id; - let job = self.job; - let ctx = ctx.clone(); - - async move { - if AssertUnwindSafe(to_spawn_job::( - id, - job, - ctx, - None, - base_dispatcher, - commands_rx, - done_tx, - )) - .catch_unwind() - .await - .is_err() - { - error!("job panicked"); - } - } - }); - - JobHandle { - id: self.id, - start_time: Instant::now(), - is_running: true, - run_time: Duration::ZERO, - next_jobs: self.next_jobs, - ctx, - commands_tx, - } - } - - #[instrument( - skip_all, - fields( - id = %self.id, - has_serialized_tasks = %serialized_tasks.is_some(), - ) - )] - fn resume( - self: Box, - base_dispatcher: BaseTaskDispatcher, - ctx: OuterCtx, - serialized_tasks: Option, - done_tx: chan::Sender<(JobId, Result)>, - ) -> JobHandle { - let (commands_tx, commands_rx) = chan::bounded(8); - - let ctx = JobCtx::new(self.report, ctx); - - trace!("Resuming job"); - - spawn({ - let id = self.id; - let job = self.job; - let ctx = ctx.clone(); - - async move { - if AssertUnwindSafe(to_spawn_job::( - id, - job, - ctx, - serialized_tasks, - base_dispatcher, - commands_rx, - done_tx, - )) - .catch_unwind() - .await - .is_err() - { - error!("job panicked"); - } - } - }); - - JobHandle { - id: self.id, - start_time: Instant::now(), - is_running: true, - run_time: self.run_time, - next_jobs: self.next_jobs, - ctx, - commands_tx, - } - } -} - -#[instrument(name = "job_executor", skip_all, fields(%job_id, name = %J::NAME))] -async fn to_spawn_job( - job_id: JobId, - mut job: J, - ctx: JobCtx, - existing_tasks: Option, - base_dispatcher: BaseTaskDispatcher, - commands_rx: chan::Receiver<(Command, oneshot::Sender<()>)>, - done_tx: chan::Sender<(JobId, Result)>, -) where - OuterCtx: OuterContext, - JobCtx: JobContext, - J: Job, -{ - enum StreamMessage { - Commands((Command, oneshot::Sender<()>)), - NewRemoteController(TaskRemoteController), - Done(Result), - } - - let mut remote_controllers = vec![]; - - let (running_state_tx, running_state_rx) = watch::channel(JobRunningState::Running); - - let (dispatcher, remote_controllers_rx) = - JobTaskDispatcher::new(job_id, base_dispatcher, running_state_rx); - - if let Some(existing_tasks) = existing_tasks { - if let Err(e) = job.resume_tasks(&dispatcher, &ctx, existing_tasks).await { - done_tx - .send((job_id, Err(e))) - .await - .expect("jobs done tx closed on error at resume_tasks"); - - return; - } - } - - let (tx, rx) = chan::bounded(1); - - spawn( - async move { - tx.send( - AssertUnwindSafe(job.run::(dispatcher, ctx)) - .catch_unwind() - .await - .unwrap_or(Err(Error::JobSystem(JobSystemError::Panic(job_id)))), - ) - .await - .expect("job run channel closed"); - } - .in_current_span(), - ); - - let commands_rx_to_close = commands_rx.clone(); - - let mut msgs_stream = pin!(( - commands_rx.map(StreamMessage::Commands), - remote_controllers_rx - .clone() - .map(StreamMessage::NewRemoteController), - stream::once({ - let rx = rx.clone(); - async move { rx.recv().await.expect("job run rx closed") } - }) - .map(StreamMessage::Done), - ) - .merge()); - - while let Some(msg) = msgs_stream.next().await { - match msg { - StreamMessage::NewRemoteController(remote_controller) => { - trace!("new remote controller received"); - remote_controllers.push(remote_controller); - trace!("added new remote controller"); - } - StreamMessage::Commands((command, ack_tx)) => { - // Add any possible pending remote controllers to the list - while let Ok(remote_controller) = remote_controllers_rx.try_recv() { - remote_controllers.push(remote_controller); - } - - remote_controllers.retain(|controller| !controller.is_done()); - - match command { - Command::Pause => { - trace!("Pausing job"); - running_state_tx.send_modify(|state| *state = JobRunningState::Paused); - trace!(tasks_count = remote_controllers.len(), "pausing tasks;"); - - remote_controllers - .iter() - .map(TaskRemoteController::pause) - .collect::>() - .join() - .await - .into_iter() - .for_each(|res| { - if let Err(e) = res { - assert!(matches!(e, TaskSystemError::TaskNotFound(_))); - - trace!("Tried to pause a task that was already completed"); - } - }); - - ack_tx.send(()).expect("ack channel closed"); - trace!("paused job"); - } - - Command::Resume => { - trace!("Resuming job"); - running_state_tx.send_modify(|state| *state = JobRunningState::Running); - trace!(tasks_count = remote_controllers.len(), "resuming tasks"); - - remote_controllers - .iter() - .map(TaskRemoteController::resume) - .collect::>() - .join() - .await - .into_iter() - .for_each(|res| { - if let Err(e) = res { - assert!(matches!(e, TaskSystemError::TaskNotFound(_))); - - trace!("Tried to resume a task that was already completed"); - } - }); - - ack_tx.send(()).expect("ack channel closed"); - trace!("resumed job"); - } - - Command::Cancel => { - trace!("Canceling job"); - running_state_tx.send_modify(|state| *state = JobRunningState::Canceled); - trace!(tasks_count = remote_controllers.len(), "canceling tasks;"); - - remote_controllers - .iter() - .map(TaskRemoteController::cancel) - .collect::>() - .join() - .await - .into_iter() - .for_each(|res| { - if let Err(e) = res { - assert!(matches!(e, TaskSystemError::TaskNotFound(_))); - - trace!("Tried to cancel a task that was already completed"); - } - }); - - trace!("canceled job"); - - commands_rx_to_close.close(); - let res = rx.recv().await.expect("job run rx closed"); - ack_tx.send(()).expect("ack channel closed"); - trace!("Job cancellation done"); - - return finish_job(job_id, res, remote_controllers, done_tx).await; - } - - Command::Shutdown => { - trace!("Shutting down job"); - running_state_tx.send_modify(|state| *state = JobRunningState::Shutdown); - debug!( - tasks_count = remote_controllers.len(), - "shutting down tasks;" - ); - - commands_rx_to_close.close(); - // Just need to wait for the job to finish with the shutdown status - let res = rx.recv().await.expect("job run rx closed"); - ack_tx.send(()).expect("ack channel closed"); - trace!("Job shutdown done"); - - return finish_job(job_id, res, remote_controllers, done_tx).await; - } - } - } - - StreamMessage::Done(res) => { - trace!("Job done"); - commands_rx_to_close.close(); - return finish_job(job_id, res, remote_controllers, done_tx).await; - } - } - } -} - -#[instrument(skip(remote_controllers, done_tx))] -async fn finish_job( - job_id: JobId, - job_result: Result, - mut remote_controllers: Vec, - done_tx: chan::Sender<(JobId, Result)>, -) { - trace!("Checking remove controllers"); - #[cfg(debug_assertions)] - { - // Just a sanity check to make sure we don't have any pending tasks left - remote_controllers.retain(|controller| !controller.is_done()); - assert!(remote_controllers.is_empty()); - // Using #[cfg(debug_assertions)] to don't pay this retain cost in release builds - } - - trace!("Sending job done message"); - - done_tx - .send((job_id, job_result)) - .await - .expect("jobs done tx closed"); -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum JobRunningState { - Running, - Paused, - Canceled, - Shutdown, -} - -impl Default for JobRunningState { - fn default() -> Self { - Self::Running - } -} - -#[derive(Debug, Clone)] -pub struct JobTaskDispatcher { - job_id: JobId, - dispatcher: BaseTaskDispatcher, - remote_controllers_tx: chan::Sender, - running_state: Arc>>, -} - -impl TaskDispatcher for JobTaskDispatcher { - type DispatchError = DispatcherError; - - async fn dispatch_boxed( - &self, - boxed_task: Box>, - ) -> Result, Self::DispatchError> { - match self.wait_for_dispatch_approval().await { - DispatchApproval::Canceled => Err(DispatcherError::JobCanceled(self.job_id)), - DispatchApproval::Shutdown => Err(DispatcherError::Shutdown(vec![boxed_task])), - DispatchApproval::Approved => { - let handle = self.dispatcher.dispatch_boxed(boxed_task).await?; - - self.remote_controllers_tx - .send(handle.remote_controller()) - .await - .expect("remote controllers tx closed"); - - Ok(handle) - } - } - } - - async fn dispatch_many_boxed( - &self, - boxed_tasks: impl IntoIterator>> + Send, - ) -> Result>, Self::DispatchError> { - match self.wait_for_dispatch_approval().await { - DispatchApproval::Canceled => Err(DispatcherError::JobCanceled(self.job_id)), - DispatchApproval::Shutdown => { - Err(DispatcherError::Shutdown(boxed_tasks.into_iter().collect())) - } - DispatchApproval::Approved => { - let handles = self.dispatcher.dispatch_many_boxed(boxed_tasks).await?; - - handles - .iter() - .map(|handle| self.remote_controllers_tx.send(handle.remote_controller())) - .collect::>() - .try_join() - .await - .expect("remote controllers tx closed"); - - Ok(handles) - } - } - } -} - -enum DispatchApproval { - Approved, - Canceled, - Shutdown, -} - -impl JobTaskDispatcher { - fn new( - job_id: JobId, - dispatcher: BaseTaskDispatcher, - running_state_rx: watch::Receiver, - ) -> (Self, chan::Receiver) { - let (remote_controllers_tx, remote_controllers_rx) = chan::unbounded(); - - ( - Self { - job_id, - dispatcher, - remote_controllers_tx, - running_state: Arc::new(Mutex::new(running_state_rx)), - }, - remote_controllers_rx, - ) - } - - async fn wait_for_dispatch_approval(&self) -> DispatchApproval { - { - let mut running_state_rx = self.running_state.lock().await; - - if running_state_rx - .has_changed() - .expect("job running state watch channel unexpectedly closed") - { - trace!("waiting for job running state to change"); - running_state_rx - .wait_for(|state| { - matches!( - *state, - JobRunningState::Running - | JobRunningState::Canceled - | JobRunningState::Shutdown - ) - }) - .await - .expect("job running state watch channel unexpectedly closed"); - - let state = { *running_state_rx.borrow() }; - - match state { - JobRunningState::Shutdown => return DispatchApproval::Shutdown, - JobRunningState::Canceled => return DispatchApproval::Canceled, - _ => {} - } - } - } - - DispatchApproval::Approved - } -} diff --git a/core/crates/heavy-lifting/src/job_system/mod.rs b/core/crates/heavy-lifting/src/job_system/mod.rs deleted file mode 100644 index 64a315a02..000000000 --- a/core/crates/heavy-lifting/src/job_system/mod.rs +++ /dev/null @@ -1,378 +0,0 @@ -use crate::{Error, JobContext}; - -use sd_prisma::prisma::location; -use sd_task_system::BaseTaskDispatcher; -use sd_utils::error::FileIOError; - -use std::{ - cell::RefCell, - collections::hash_map::HashMap, - panic, - path::{Path, PathBuf}, - sync::Arc, -}; - -use async_channel as chan; -use futures::Stream; -use futures_concurrency::future::{Join, TryJoin}; -use tokio::{fs, spawn, sync::oneshot, task::JoinHandle}; -use tracing::{debug, error, info, instrument, trace, warn}; -use uuid::Uuid; - -mod error; -pub mod job; -pub mod report; -mod runner; -mod store; -pub mod utils; - -pub use error::{DispatcherError, JobErrorOrDispatcherError, JobSystemError}; -use job::{IntoJob, Job, JobName, JobOutput, OuterContext}; -use report::Report; -use runner::{run, JobSystemRunner, RunnerMessage}; -use store::{load_jobs, StoredJobEntry}; - -pub use store::{SerializableJob, SerializedTasks}; - -const PENDING_JOBS_FILE: &str = "pending_jobs.bin"; - -pub type JobId = Uuid; - -#[derive(Debug, Clone, Copy)] -pub enum Command { - Pause, - Resume, - Cancel, - Shutdown, -} - -pub struct JobSystem> { - msgs_tx: chan::Sender>, - job_outputs_rx: chan::Receiver<(JobId, Result)>, - store_jobs_file: Arc, - runner_handle: RefCell>>, -} - -impl> JobSystem { - pub fn new( - base_dispatcher: BaseTaskDispatcher, - data_directory: impl AsRef, - ) -> Self { - let (job_outputs_tx, job_outputs_rx) = chan::unbounded(); - let (job_done_tx, job_done_rx) = chan::bounded(16); - let (msgs_tx, msgs_rx) = chan::bounded(8); - - let store_jobs_file = Arc::new(data_directory.as_ref().join(PENDING_JOBS_FILE)); - - let runner_handle = RefCell::new(Some(spawn({ - let store_jobs_file = Arc::clone(&store_jobs_file); - async move { - trace!("Job System Runner starting..."); - while let Err(e) = spawn({ - let store_jobs_file = Arc::clone(&store_jobs_file); - let base_dispatcher = base_dispatcher.clone(); - let job_return_status_tx = job_done_tx.clone(); - let job_done_rx = job_done_rx.clone(); - let job_outputs_tx = job_outputs_tx.clone(); - let msgs_rx = msgs_rx.clone(); - - async move { - run( - JobSystemRunner::new( - base_dispatcher, - job_return_status_tx, - job_outputs_tx, - ), - store_jobs_file.as_ref(), - msgs_rx, - job_done_rx, - ) - .await; - } - }) - .await - { - if e.is_panic() { - error!(?e, "Job system panicked;"); - } else { - trace!("JobSystemRunner received shutdown signal and will exit..."); - break; - } - trace!("Restarting JobSystemRunner processing task..."); - } - - info!("JobSystemRunner gracefully shutdown"); - } - }))); - - Self { - msgs_tx, - job_outputs_rx, - store_jobs_file, - runner_handle, - } - } - - pub async fn init( - &self, - previously_existing_contexts: &HashMap, - ) -> Result<(), JobSystemError> { - load_stored_job_entries( - &*self.store_jobs_file, - previously_existing_contexts, - &self.msgs_tx, - ) - .await - } - - /// Get a map of all active reports with their respective job ids - /// - /// # Panics - /// - /// Panics only happen if internal channels are unexpectedly closed - pub async fn get_active_reports(&self) -> HashMap { - let (ack_tx, ack_rx) = oneshot::channel(); - self.msgs_tx - .send(RunnerMessage::GetActiveReports { ack_tx }) - .await - .expect("runner msgs channel unexpectedly closed on get active reports request"); - - ack_rx - .await - .expect("ack channel closed before receiving get active reports response") - } - - /// Checks if *any* of the desired jobs is running for the desired location - /// - /// # Panics - /// - /// Panics only happen if internal channels are unexpectedly closed - pub async fn check_running_jobs( - &self, - job_names: Vec, - location_id: location::id::Type, - ) -> bool { - let (ack_tx, ack_rx) = oneshot::channel(); - - self.msgs_tx - .send(RunnerMessage::CheckIfJobsAreRunning { - job_names, - location_id, - ack_tx, - }) - .await - .expect("runner msgs channel unexpectedly closed on check running job request"); - - ack_rx - .await - .expect("ack channel closed before receiving check running job response") - } - - /// Shutdown the job system - /// - /// # Panics - /// - /// Panics only happen if internal channels are unexpectedly closed - pub async fn shutdown(&self) { - if let Some(handle) = self - .runner_handle - .try_borrow_mut() - .ok() - .and_then(|mut maybe_handle| maybe_handle.take()) - { - self.msgs_tx - .send(RunnerMessage::Shutdown) - .await - .expect("runner msgs channel unexpectedly closed on shutdown request"); - - if let Err(e) = handle.await { - if e.is_panic() { - error!(?e, "JobSystem panicked;"); - } - } - info!("JobSystem gracefully shutdown"); - } else { - warn!("JobSystem already shutdown"); - } - } - - /// Dispatch a new job to the system - /// - /// # Panics - /// - /// Panics only happen if internal channels are unexpectedly closed - pub async fn dispatch>( - &self, - job: impl IntoJob + Send, - location_id: location::id::Type, - ctx: OuterCtx, - ) -> Result { - let dyn_job = job.into_job(); - let id = dyn_job.id(); - - let (ack_tx, ack_rx) = oneshot::channel(); - self.msgs_tx - .send(RunnerMessage::NewJob { - job_id: id, - location_id, - dyn_job, - ctx, - ack_tx, - }) - .await - .expect("runner msgs channel unexpectedly closed on new job request"); - - ack_rx - .await - .expect("ack channel closed before receiving new job request") - .map(|()| id) - } - - /// Check if there are any active jobs for the desired [`OuterContext`] - /// - /// # Panics - /// - /// Panics only happen if internal channels are unexpectedly closed - pub async fn has_active_jobs(&self, ctx: OuterCtx) -> bool { - let ctx_id = ctx.id(); - - let (ack_tx, ack_rx) = oneshot::channel(); - self.msgs_tx - .send(RunnerMessage::HasActiveJobs { ctx_id, ack_tx }) - .await - .expect("runner msgs channel unexpectedly closed on has active jobs request"); - - ack_rx - .await - .expect("ack channel closed before receiving has active jobs response") - } - - pub fn receive_job_outputs(&self) -> impl Stream)> { - self.job_outputs_rx.clone() - } - - #[instrument(skip(self), err)] - async fn send_command(&self, job_id: JobId, command: Command) -> Result<(), JobSystemError> { - let (ack_tx, ack_rx) = oneshot::channel(); - self.msgs_tx - .send(RunnerMessage::Command { - job_id, - command, - ack_tx, - }) - .await - .unwrap_or_else(|_| { - panic!("runner msgs channel unexpectedly closed on {command:?} request") - }); - - ack_rx - .await - .unwrap_or_else(|_| panic!("ack channel closed before receiving {command:?} response")) - } - - pub async fn pause(&self, job_id: JobId) -> Result<(), JobSystemError> { - self.send_command(job_id, Command::Pause).await - } - - pub async fn resume(&self, job_id: JobId) -> Result<(), JobSystemError> { - self.send_command(job_id, Command::Resume).await - } - - pub async fn cancel(&self, job_id: JobId) -> Result<(), JobSystemError> { - self.send_command(job_id, Command::Cancel).await - } -} - -/// SAFETY: Due to usage of refcell we lost `Sync` impl, but we only use it to have a shutdown method -/// receiving `&self` which is called once, and we also use `try_borrow_mut` so we never panic -unsafe impl> Sync - for JobSystem -{ -} - -async fn load_stored_job_entries>( - store_jobs_file: impl AsRef + Send, - previously_existing_job_contexts: &HashMap, - msgs_tx: &chan::Sender>, -) -> Result<(), JobSystemError> { - let store_jobs_file = store_jobs_file.as_ref(); - - let stores_jobs_by_db = rmp_serde::from_slice::>>( - &match fs::read(store_jobs_file).await { - Ok(bytes) => bytes, - Err(e) if e.kind() == std::io::ErrorKind::NotFound => { - debug!("No pending jobs found on disk"); - return Ok(()); - } - Err(e) => { - return Err(JobSystemError::StoredJobs(FileIOError::from(( - store_jobs_file, - e, - "Failed to load jobs from disk", - )))) - } - }, - )?; - - stores_jobs_by_db - .into_iter() - .filter_map(|(ctx_id, entries)| { - previously_existing_job_contexts.get(&ctx_id).map_or_else( - || { - warn!(%ctx_id, "Found stored jobs for a database that doesn't exist anymore;"); - None - }, - |ctx| Some((entries, ctx.clone())), - ) - }) - .map(|(entries, ctx)| async move { - load_jobs(entries, &ctx) - .await - .map(|stored_jobs| (stored_jobs, ctx)) - }) - .collect::>() - .join() - .await - .into_iter() - .filter_map(|res| { - res.map_err(|e| error!(?e, "Failed to load stored jobs;")) - .ok() - }) - .flat_map(|(stored_jobs, ctx)| { - stored_jobs - .into_iter() - .map(move |(location_id, dyn_job, serialized_tasks)| { - let ctx = ctx.clone(); - async move { - let (ack_tx, ack_rx) = oneshot::channel(); - - msgs_tx - .send(RunnerMessage::ResumeStoredJob { - job_id: dyn_job.id(), - location_id, - dyn_job, - ctx, - serialized_tasks, - ack_tx, - }) - .await - .expect("runner msgs channel unexpectedly closed on stored job resume"); - - ack_rx.await.expect( - "ack channel closed before receiving stored job resume response", - ) - } - }) - }) - .collect::>() - .try_join() - .await?; - - fs::remove_file(store_jobs_file).await.map_err(|e| { - JobSystemError::StoredJobs(FileIOError::from(( - store_jobs_file, - e, - "Failed to clean stored jobs file", - ))) - }) -} diff --git a/core/crates/heavy-lifting/src/job_system/report.rs b/core/crates/heavy-lifting/src/job_system/report.rs deleted file mode 100644 index b747b8195..000000000 --- a/core/crates/heavy-lifting/src/job_system/report.rs +++ /dev/null @@ -1,436 +0,0 @@ -use crate::NonCriticalError; - -use sd_prisma::prisma::{file_path, job, location, PrismaClient}; -use sd_utils::db::{maybe_missing, MissingFieldError}; - -use std::{collections::HashMap, fmt, path::PathBuf, str::FromStr}; - -use chrono::{DateTime, Utc}; -use prisma_client_rust::QueryError; -use serde::{Deserialize, Serialize}; -use specta::Type; -use strum::ParseError; - -use super::{job::JobName, JobId}; - -#[derive(thiserror::Error, Debug)] -pub enum ReportError { - #[error("failed to create job report in database: {0}")] - Create(QueryError), - #[error("failed to update job report in database: {0}")] - Update(QueryError), - #[error("invalid job status integer: {0}")] - InvalidJobStatusInt(i32), - #[error("job not found in database: ")] - MissingReport(JobId), - #[error("json error: {0}")] - Json(#[from] serde_json::Error), - #[error(transparent)] - MissingField(#[from] MissingFieldError), - #[error("failed to parse job name from database: {0}")] - JobNameParse(#[from] ParseError), -} - -impl From for rspc::Error { - fn from(e: ReportError) -> Self { - match e { - ReportError::Create(_) - | ReportError::Update(_) - | ReportError::InvalidJobStatusInt(_) => { - Self::with_cause(rspc::ErrorCode::BadRequest, e.to_string(), e) - } - - ReportError::MissingReport(_) => { - Self::with_cause(rspc::ErrorCode::NotFound, e.to_string(), e) - } - ReportError::Json(_) | ReportError::MissingField(_) | ReportError::JobNameParse(_) => { - Self::with_cause(rspc::ErrorCode::InternalServerError, e.to_string(), e) - } - } - } -} - -#[derive(Debug, Serialize, Deserialize, Type, Clone)] -#[serde(rename_all = "snake_case")] -#[serde(tag = "type", content = "metadata")] -pub enum ReportMetadata { - Input(ReportInputMetadata), - Output(ReportOutputMetadata), -} - -#[derive(Debug, Serialize, Deserialize, Type, Clone)] -#[serde(rename_all = "snake_case")] -#[serde(tag = "type", content = "data")] -pub enum ReportInputMetadata { - // TODO: Add more variants as needed - Location(location::Data), - SubPath(PathBuf), -} - -#[derive(Debug, Serialize, Deserialize, Type, Clone)] -#[serde(rename_all = "snake_case")] -#[serde(tag = "type", content = "data")] -pub enum ReportOutputMetadata { - Metrics(HashMap), - Indexer { - total_paths: (u32, u32), - }, - FileIdentifier { - total_orphan_paths: (u32, u32), - total_objects_created: (u32, u32), - total_objects_linked: (u32, u32), - }, - MediaProcessor { - media_data_extracted: (u32, u32), - media_data_skipped: (u32, u32), - thumbnails_generated: (u32, u32), - thumbnails_skipped: (u32, u32), - }, - Copier { - source_location_id: location::id::Type, - target_location_id: location::id::Type, - sources_file_path_ids: Vec, - target_location_relative_directory_path: PathBuf, - }, - Mover { - source_location_id: location::id::Type, - target_location_id: location::id::Type, - sources_file_path_ids: Vec, - target_location_relative_directory_path: PathBuf, - }, - Deleter { - location_id: location::id::Type, - file_path_ids: Vec, - }, - Eraser { - location_id: location::id::Type, - file_path_ids: Vec, - passes: u32, - }, - FileValidator { - location_id: location::id::Type, - sub_path: Option, - }, -} - -impl From for ReportMetadata { - fn from(value: ReportInputMetadata) -> Self { - Self::Input(value) - } -} - -impl From for ReportMetadata { - fn from(value: ReportOutputMetadata) -> Self { - Self::Output(value) - } -} - -#[derive(Debug, Serialize, Type, Clone)] -pub struct Report { - pub id: JobId, - pub name: JobName, - pub action: Option, - - pub metadata: Vec, - pub critical_error: Option, - pub non_critical_errors: Vec, - - pub created_at: Option>, - pub started_at: Option>, - pub completed_at: Option>, - - pub parent_id: Option, - - pub status: Status, - pub task_count: i32, - pub completed_task_count: i32, - pub info: String, - - pub phase: String, - pub message: String, - pub estimated_completion: DateTime, -} - -impl fmt::Display for Report { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!( - f, - "Job {:#?}", - self.name, self.id, self.status - ) - } -} - -// convert database struct into a resource struct -impl TryFrom for Report { - type Error = ReportError; - - fn try_from( - job::Data { - id, - name, - action, - status, - errors_text: _, // Deprecated - critical_error, - non_critical_errors, - data: _, // Deprecated - metadata, - parent_id, - task_count, - info, - completed_task_count, - date_estimated_completion, - date_created, - date_started, - date_completed, - .. - }: job::Data, - ) -> Result { - Ok(Self { - id: JobId::from_slice(&id).expect("corrupted database"), - name: JobName::from_str(&maybe_missing(name, "job.name")?)?, - action, - metadata: if let Some(metadata) = metadata { - serde_json::from_slice(&metadata)? - } else { - vec![] - }, - critical_error, - non_critical_errors: if let Some(non_critical_errors) = non_critical_errors { - serde_json::from_slice(&non_critical_errors)? - } else { - vec![] - }, - created_at: date_created.map(DateTime::into), - started_at: date_started.map(DateTime::into), - completed_at: date_completed.map(DateTime::into), - parent_id: parent_id.map(|id| JobId::from_slice(&id).expect("corrupted database")), - status: Status::try_from(maybe_missing(status, "job.status")?) - .expect("corrupted database"), - task_count: task_count.unwrap_or(0), - completed_task_count: completed_task_count.unwrap_or(0), - info: info.unwrap_or_default(), - phase: String::new(), - message: String::new(), - estimated_completion: date_estimated_completion.map_or_else(Utc::now, DateTime::into), - }) - } -} - -impl Report { - #[must_use] - pub fn new(uuid: JobId, name: JobName) -> Self { - Self { - id: uuid, - name, - action: None, - created_at: None, - started_at: None, - completed_at: None, - status: Status::Queued, - critical_error: None, - non_critical_errors: vec![], - task_count: 0, - metadata: vec![], - parent_id: None, - completed_task_count: 0, - info: String::new(), - phase: String::new(), - message: String::new(), - estimated_completion: Utc::now(), - } - } - - pub fn push_metadata(&mut self, metadata: ReportOutputMetadata) { - self.metadata.push(metadata.into()); - } - - #[must_use] - pub fn get_action_name_and_group_key(&self) -> (String, Option) { - // actions are formatted like "added_location" or "added_location-1" - let Some(action_name) = self - .action - .as_ref() - .and_then(|action| action.split('-').next().map(str::to_string)) - else { - return (self.id.to_string(), None); - }; - // create a unique group_key, EG: "added_location-" - let group_key = self.parent_id.map_or_else( - || format!("{action_name}-{}", self.id), - |parent_id| format!("{action_name}-{parent_id}"), - ); - - (action_name, Some(group_key)) - } - - pub async fn create( - &mut self, - db: &PrismaClient, - created_at: DateTime, - ) -> Result<(), ReportError> { - db.job() - .create( - self.id.as_bytes().to_vec(), - sd_utils::chain_optional_iter( - [ - job::name::set(Some(self.name.to_string())), - job::action::set(self.action.clone()), - job::date_created::set(Some(created_at.into())), - job::metadata::set(Some(serde_json::to_vec(&self.metadata)?)), - job::status::set(Some(self.status as i32)), - job::date_started::set(self.started_at.map(Into::into)), - job::task_count::set(Some(0)), - job::info::set(Some(self.info.clone())), - job::completed_task_count::set(Some(0)), - ], - [self - .parent_id - .map(|id| job::parent::connect(job::id::equals(id.as_bytes().to_vec())))], - ), - ) - .select(job::select!({ id })) - .exec() - .await - .map_err(ReportError::Create)?; - - // Only setting created_at after we successfully created the job in DB - self.created_at = Some(created_at); - - Ok(()) - } - - pub async fn update(&self, db: &PrismaClient) -> Result<(), ReportError> { - db.job() - .update( - job::id::equals(self.id.as_bytes().to_vec()), - vec![ - job::status::set(Some(self.status as i32)), - job::critical_error::set(self.critical_error.clone()), - job::non_critical_errors::set(Some(serde_json::to_vec( - &self.non_critical_errors, - )?)), - job::metadata::set(Some(serde_json::to_vec(&self.metadata)?)), - job::task_count::set(Some(self.task_count)), - job::info::set(Some(self.info.clone())), - job::completed_task_count::set(Some(self.completed_task_count)), - job::date_started::set(self.started_at.map(Into::into)), - job::date_completed::set(self.completed_at.map(Into::into)), - ], - ) - .select(job::select!({ id })) - .exec() - .await - .map_err(ReportError::Update)?; - - Ok(()) - } -} - -#[repr(i32)] -#[derive(Debug, Clone, Copy, Serialize, Deserialize, Type, Eq, PartialEq)] -pub enum Status { - Queued = 0, - Running = 1, - Completed = 2, - Canceled = 3, - Failed = 4, - Paused = 5, - CompletedWithErrors = 6, -} - -impl Status { - #[must_use] - pub const fn is_finished(self) -> bool { - matches!( - self, - Self::Completed - | Self::Canceled - | Self::Paused - | Self::Failed - | Self::CompletedWithErrors - ) - } -} - -impl TryFrom for Status { - type Error = ReportError; - - fn try_from(value: i32) -> Result { - let s = match value { - 0 => Self::Queued, - 1 => Self::Running, - 2 => Self::Completed, - 3 => Self::Canceled, - 4 => Self::Failed, - 5 => Self::Paused, - 6 => Self::CompletedWithErrors, - _ => return Err(Self::Error::InvalidJobStatusInt(value)), - }; - - Ok(s) - } -} - -pub struct ReportBuilder { - pub id: JobId, - pub name: JobName, - pub action: Option, - pub metadata: Vec, - pub parent_id: Option, -} - -impl ReportBuilder { - #[must_use] - pub fn build(self) -> Report { - Report { - id: self.id, - name: self.name, - action: self.action, - created_at: None, - started_at: None, - completed_at: None, - status: Status::Queued, - critical_error: None, - task_count: 0, - non_critical_errors: vec![], - metadata: self.metadata, - parent_id: self.parent_id, - completed_task_count: 0, - info: String::new(), - phase: String::new(), - message: String::new(), - estimated_completion: Utc::now(), - } - } - - #[must_use] - pub const fn new(id: JobId, name: JobName) -> Self { - Self { - id, - name, - action: None, - metadata: vec![], - parent_id: None, - } - } - - #[must_use] - pub fn with_action(mut self, action: impl Into) -> Self { - self.action = Some(action.into()); - self - } - - #[must_use] - pub fn with_metadata(mut self, metadata: ReportInputMetadata) -> Self { - self.metadata.push(metadata.into()); - self - } - - #[must_use] - pub const fn with_parent_id(mut self, parent_id: JobId) -> Self { - self.parent_id = Some(parent_id); - self - } -} diff --git a/core/crates/heavy-lifting/src/job_system/runner.rs b/core/crates/heavy-lifting/src/job_system/runner.rs deleted file mode 100644 index 57c237ead..000000000 --- a/core/crates/heavy-lifting/src/job_system/runner.rs +++ /dev/null @@ -1,713 +0,0 @@ -use crate::{Error, JobContext}; - -use sd_prisma::prisma::location; -use sd_task_system::BaseTaskDispatcher; -use sd_utils::error::FileIOError; - -use std::{ - collections::{hash_map::Entry, HashMap, HashSet}, - mem, - path::Path, - pin::pin, - time::Duration, -}; - -use async_channel as chan; -use chrono::Utc; -use futures::StreamExt; -use futures_concurrency::{ - future::{Join, TryJoin}, - stream::Merge, -}; -use serde_json::json; -use tokio::{ - fs, - sync::oneshot, - time::{interval_at, Instant}, -}; -use tokio_stream::wrappers::IntervalStream; -use tracing::{debug, error, info, instrument, trace, warn}; -use uuid::Uuid; - -use super::{ - job::{DynJob, JobHandle, JobName, JobOutput, OuterContext, ReturnStatus}, - report::{self, ReportOutputMetadata}, - store::{StoredJob, StoredJobEntry}, - Command, JobId, JobSystemError, SerializedTasks, -}; - -const JOBS_INITIAL_CAPACITY: usize = 32; -const FIVE_MINUTES: Duration = Duration::from_secs(5 * 60); - -pub(super) enum RunnerMessage> { - NewJob { - job_id: JobId, - location_id: location::id::Type, - dyn_job: Box>, - ctx: OuterCtx, - ack_tx: oneshot::Sender>, - }, - ResumeStoredJob { - job_id: JobId, - location_id: location::id::Type, - dyn_job: Box>, - ctx: OuterCtx, - serialized_tasks: Option, - ack_tx: oneshot::Sender>, - }, - Command { - job_id: JobId, - command: Command, - ack_tx: oneshot::Sender>, - }, - GetActiveReports { - ack_tx: oneshot::Sender>, - }, - CheckIfJobsAreRunning { - job_names: Vec, - location_id: location::id::Type, - ack_tx: oneshot::Sender, - }, - Shutdown, - HasActiveJobs { - ctx_id: Uuid, - ack_tx: oneshot::Sender, - }, -} - -struct JobsWorktables { - job_hashes: HashMap, - job_hashes_by_id: HashMap, - running_jobs_by_job_id: HashMap, - running_jobs_set: HashSet<(JobName, location::id::Type)>, - jobs_to_store_by_ctx_id: HashMap>, -} - -pub(super) struct JobSystemRunner> { - on_shutdown_mode: bool, - base_dispatcher: BaseTaskDispatcher, - handles: HashMap>, - worktables: JobsWorktables, - job_return_status_tx: chan::Sender<(JobId, Result)>, - job_outputs_tx: chan::Sender<(JobId, Result)>, -} - -impl> JobSystemRunner { - pub(super) fn new( - base_dispatcher: BaseTaskDispatcher, - job_return_status_tx: chan::Sender<(JobId, Result)>, - job_outputs_tx: chan::Sender<(JobId, Result)>, - ) -> Self { - Self { - on_shutdown_mode: false, - base_dispatcher, - handles: HashMap::with_capacity(JOBS_INITIAL_CAPACITY), - worktables: JobsWorktables { - job_hashes: HashMap::with_capacity(JOBS_INITIAL_CAPACITY), - job_hashes_by_id: HashMap::with_capacity(JOBS_INITIAL_CAPACITY), - running_jobs_by_job_id: HashMap::with_capacity(JOBS_INITIAL_CAPACITY), - running_jobs_set: HashSet::with_capacity(JOBS_INITIAL_CAPACITY), - jobs_to_store_by_ctx_id: HashMap::new(), - }, - job_return_status_tx, - job_outputs_tx, - } - } - - async fn new_job( - &mut self, - job_id: JobId, - location_id: location::id::Type, - dyn_job: Box>, - ctx: OuterCtx, - maybe_existing_tasks: Option, - ) -> Result<(), JobSystemError> { - let Self { - base_dispatcher, - handles, - worktables: - JobsWorktables { - job_hashes, - job_hashes_by_id, - running_jobs_by_job_id, - running_jobs_set, - .. - }, - job_return_status_tx, - .. - } = self; - - let job_name = dyn_job.job_name(); - - let job_hash = dyn_job.hash(); - if let Some(&already_running_id) = job_hashes.get(&job_hash) { - return Err(JobSystemError::AlreadyRunning { - new_id: job_id, - already_running_id, - job_name, - }); - } - - running_jobs_by_job_id.insert(job_id, (job_name, location_id)); - running_jobs_set.insert((job_name, location_id)); - - job_hashes.insert(job_hash, job_id); - job_hashes_by_id.insert(job_id, job_hash); - - let mut handle = if maybe_existing_tasks.is_some() { - dyn_job.resume( - base_dispatcher.clone(), - ctx.clone(), - maybe_existing_tasks, - job_return_status_tx.clone(), - ) - } else { - dyn_job.dispatch( - base_dispatcher.clone(), - ctx.clone(), - job_return_status_tx.clone(), - ) - }; - - handle.register_start(Utc::now()).await?; - - handles.insert(job_id, handle); - - Ok(()) - } - - async fn get_active_reports(&self) -> HashMap { - self.handles - .iter() - .map(|(job_id, handle)| async { (*job_id, handle.ctx.report().await.clone()) }) - .collect::>() - .join() - .await - .into_iter() - .collect() - } - - async fn process_command( - &mut self, - job_id: JobId, - command: Command, - ack_tx: oneshot::Sender>, - ) { - if let Some(handle) = self.handles.get_mut(&job_id) { - match (command, handle.is_running) { - (Command::Pause, false) => { - warn!("Tried to pause a job already paused"); - return ack_tx.send(Ok(())).expect( - "ack channel closed before sending response to already paused job", - ); - } - (Command::Resume, true) => { - warn!("Tried to resume a job already running"); - return ack_tx.send(Ok(())).expect( - "ack channel closed before sending response to already running job", - ); - } - _ => {} - } - match command { - Command::Pause | Command::Cancel | Command::Shutdown => { - handle.is_running = false; - } - Command::Resume => { - handle.is_running = true; - } - } - handle.send_command(command, ack_tx).await; - handle.ctx.invalidate_query("jobs.isActive"); - handle.ctx.invalidate_query("jobs.reports"); - } else { - error!("Job not found"); - ack_tx - .send(Err(JobSystemError::NotFound(job_id))) - .unwrap_or_else(|_| { - panic!("ack channel closed before sending {command:?} response") - }); - } - } - - fn is_empty(&self) -> bool { - self.handles.is_empty() - && self.worktables.job_hashes.is_empty() - && self.worktables.job_hashes_by_id.is_empty() - } - - fn total_jobs(&self) -> usize { - self.handles.len() - } - - fn check_if_jobs_are_running( - &self, - job_names: Vec, - location_id: location::id::Type, - ) -> bool { - job_names.into_iter().any(|job_name| { - self.worktables - .running_jobs_set - .contains(&(job_name, location_id)) - }) - } - - #[instrument(skip_all, fields(%job_id))] - async fn process_return_status( - &mut self, - job_id: JobId, - status: Result, - ) -> Result<(), JobSystemError> { - let Self { - on_shutdown_mode, - handles, - worktables, - job_outputs_tx, - job_return_status_tx, - base_dispatcher, - .. - } = self; - - let job_hash = worktables - .job_hashes_by_id - .remove(&job_id) - .expect("it must be here"); - - let (job_name, location_id) = worktables - .running_jobs_by_job_id - .remove(&job_id) - .expect("a JobName and location_id must've been inserted in the map with the job id"); - - assert!(worktables.running_jobs_set.remove(&(job_name, location_id))); - assert!(worktables.job_hashes.remove(&job_hash).is_some()); - - let mut handle = handles.remove(&job_id).expect("it must be here"); - handle.run_time += handle.start_time.elapsed(); - - handle - .ctx - .report_mut() - .await - .push_metadata(ReportOutputMetadata::Metrics(HashMap::from([( - "job_run_time".into(), - json!(handle.run_time), - )]))); - - let res = match status { - Ok(ReturnStatus::Completed(job_return)) => { - try_dispatch_next_job( - &mut handle, - location_id, - base_dispatcher.clone(), - worktables, - handles, - job_return_status_tx.clone(), - ) - .await?; - - handle.complete_job(job_return).await.map_err(Into::into) - } - - Ok(ReturnStatus::Shutdown(res)) => { - match res { - Ok(Some(serialized_job)) => { - let name = { - let db = handle.ctx.db(); - let report = handle.ctx.report().await; - if let Err(e) = report.update(db).await { - error!(?e, "Failed to update report on job shutdown;"); - } - report.name - }; - - worktables - .jobs_to_store_by_ctx_id - .entry(handle.ctx.id()) - .or_default() - .push(StoredJobEntry { - location_id, - root_job: StoredJob { - id: job_id, - run_time: handle.start_time.elapsed(), - name, - serialized_job, - }, - next_jobs: serialize_next_jobs_to_shutdown( - job_id, - job_name, - handle.next_jobs, - ) - .await - .unwrap_or_default(), - }); - - debug!(%name, "Job was shutdown and serialized;"); - } - - Ok(None) => { - debug!( - "Job was shutdown but didn't returned any serialized data, \ - probably it isn't resumable job" - ); - } - - Err(e) => { - error!(?e, "Failed to serialize job;"); - } - } - - if *on_shutdown_mode && handles.is_empty() { - // Job system is empty and in shutdown mode so we close this channel to finish the shutdown process - job_return_status_tx.close(); - } - - return Ok(()); - } - - Ok(ReturnStatus::Canceled(job_return)) => { - handle.cancel_job(job_return).await.map_err(Into::into) - } - Err(e) => handle - .failed_job(&e) - .await - .map_err(Into::into) - .and_then(|()| Err(e)), - }; - - job_outputs_tx - .send((job_id, res)) - .await - .expect("job outputs channel unexpectedly closed on job completion"); - - handle.ctx.invalidate_query("jobs.isActive"); - handle.ctx.invalidate_query("jobs.reports"); - - Ok(()) - } - - fn clean_memory(&mut self) { - if self.handles.capacity() > JOBS_INITIAL_CAPACITY - && self.handles.len() < JOBS_INITIAL_CAPACITY - { - self.handles.shrink_to(JOBS_INITIAL_CAPACITY); - } - - if self.worktables.job_hashes.capacity() > JOBS_INITIAL_CAPACITY - && self.worktables.job_hashes.len() < JOBS_INITIAL_CAPACITY - { - self.worktables.job_hashes.shrink_to(JOBS_INITIAL_CAPACITY); - } - - if self.worktables.job_hashes_by_id.capacity() > JOBS_INITIAL_CAPACITY - && self.worktables.job_hashes_by_id.len() < JOBS_INITIAL_CAPACITY - { - self.worktables - .job_hashes_by_id - .shrink_to(JOBS_INITIAL_CAPACITY); - } - - if self.worktables.running_jobs_by_job_id.capacity() > JOBS_INITIAL_CAPACITY - && self.worktables.running_jobs_by_job_id.len() < JOBS_INITIAL_CAPACITY - { - self.worktables - .running_jobs_by_job_id - .shrink_to(JOBS_INITIAL_CAPACITY); - } - - if self.worktables.running_jobs_set.capacity() > JOBS_INITIAL_CAPACITY - && self.worktables.running_jobs_set.len() < JOBS_INITIAL_CAPACITY - { - self.worktables - .running_jobs_set - .shrink_to(JOBS_INITIAL_CAPACITY); - } - } - - async fn save_jobs( - self, - store_jobs_file: impl AsRef + Send, - ) -> Result<(), JobSystemError> { - let store_jobs_file = store_jobs_file.as_ref(); - - let Self { - handles, - worktables: - JobsWorktables { - job_hashes, - job_hashes_by_id, - jobs_to_store_by_ctx_id, - .. - }, - .. - } = self; - - assert!( - handles.is_empty() && job_hashes.is_empty() && job_hashes_by_id.is_empty(), - "All jobs must be completed before saving" - ); - - if jobs_to_store_by_ctx_id.is_empty() { - info!("No jobs to store in disk for job system shutdown!"); - return Ok(()); - } - - fs::write( - store_jobs_file, - rmp_serde::to_vec_named(&jobs_to_store_by_ctx_id)?, - ) - .await - .map_err(|e| JobSystemError::StoredJobs(FileIOError::from((store_jobs_file, e)))) - } - - fn has_active_jobs(&self, ctx_id: Uuid) -> bool { - self.handles - .values() - .any(|handle| handle.ctx.id() == ctx_id && handle.is_running) - } - - async fn dispatch_shutdown_command_to_jobs(&mut self) { - self.handles - .values_mut() - .map(|handle| async move { - let (tx, rx) = oneshot::channel(); - - handle.send_command(Command::Shutdown, tx).await; - - rx.await.expect("Worker failed to ack shutdown request") - }) - .collect::>() - .join() - .await - .into_iter() - .for_each(|res| { - if let Err(e) = res { - error!(?e, "Failed to shutdown job;"); - } - }); - } -} - -#[instrument(skip(next_jobs))] -async fn serialize_next_jobs_to_shutdown>( - parent_job_id: JobId, - parent_job_name: JobName, - next_jobs: impl IntoIterator>> + Send, -) -> Option> { - next_jobs - .into_iter() - .map(|next_job| async move { - let next_id = next_job.id(); - let next_name = next_job.job_name(); - next_job - .serialize() - .await - .map(|maybe_serialized_job| { - maybe_serialized_job.map(|serialized_job| StoredJob { - id: next_id, - run_time: Duration::ZERO, - name: next_name, - serialized_job, - }) - }) - .map_err(|e| { - error!(%next_id, %next_name, ?e, "Failed to serialize next job;"); - }) - }) - .collect::>() - .try_join() - .await - .map(|maybe_serialized_next_jobs| { - maybe_serialized_next_jobs.into_iter().flatten().collect() - }) - .ok() -} - -#[instrument( - skip_all, - fields( - job_id = %handle.id, - next_jobs_count = handle.next_jobs.len(), - location_id = %location_id, - total_running_jobs = handles.len(), - ) -)] -async fn try_dispatch_next_job>( - handle: &mut JobHandle, - location_id: location::id::Type, - base_dispatcher: BaseTaskDispatcher, - JobsWorktables { - job_hashes, - job_hashes_by_id, - running_jobs_by_job_id, - running_jobs_set, - .. - }: &mut JobsWorktables, - handles: &mut HashMap>, - job_return_status_tx: chan::Sender<(JobId, Result)>, -) -> Result<(), JobSystemError> { - if let Some(next) = handle.next_jobs.pop_front() { - let next_id = next.id(); - let next_hash = next.hash(); - let next_name = next.job_name(); - - if let Entry::Vacant(e) = job_hashes.entry(next_hash) { - e.insert(next_id); - trace!(%next_id, %next_name, "Dispatching next job;"); - - job_hashes_by_id.insert(next_id, next_hash); - running_jobs_by_job_id.insert(next_id, (next_name, location_id)); - running_jobs_set.insert((next_name, location_id)); - - let mut next_handle = next.dispatch( - base_dispatcher, - handle.ctx.get_outer_ctx(), - job_return_status_tx, - ); - - next_handle.register_start(Utc::now()).await?; - - assert!( - next_handle.next_jobs.is_empty(), - "Only the root job will have next jobs, the rest will be empty and \ - we will swap with remaining ones from the previous job" - ); - - next_handle.next_jobs = mem::take(&mut handle.next_jobs); - - handles.insert(next_id, next_handle); - } else { - warn!(%next_id, %next_name, "Unexpectedly found a job with the same hash as the next job;"); - } - } else { - trace!("No next jobs to dispatch"); - } - - Ok(()) -} - -pub(super) async fn run>( - mut runner: JobSystemRunner, - store_jobs_file: impl AsRef + Send, - msgs_rx: chan::Receiver>, - job_done_rx: chan::Receiver<(JobId, Result)>, -) { - enum StreamMessage> { - ReturnStatus((JobId, Result)), - RunnerMessage(RunnerMessage), - CleanMemoryTick, - } - - let memory_cleanup_interval = interval_at(Instant::now() + FIVE_MINUTES, FIVE_MINUTES); - - let job_return_status_rx_to_shutdown = job_done_rx.clone(); - - let mut msg_stream = pin!(( - msgs_rx.map(StreamMessage::RunnerMessage), - job_done_rx.map(StreamMessage::ReturnStatus), - IntervalStream::new(memory_cleanup_interval).map(|_| StreamMessage::CleanMemoryTick), - ) - .merge()); - - while let Some(msg) = msg_stream.next().await { - match msg { - // Job return status messages - StreamMessage::ReturnStatus((job_id, status)) => { - if let Err(e) = runner.process_return_status(job_id, status).await { - error!(?e, "Failed to process return status;"); - } - } - - // Runner messages - StreamMessage::RunnerMessage(RunnerMessage::NewJob { - job_id, - location_id, - dyn_job, - ctx, - ack_tx, - }) => { - ack_tx - .send( - runner - .new_job(job_id, location_id, dyn_job, ctx, None) - .await, - ) - .expect("ack channel closed before sending new job response"); - } - - StreamMessage::RunnerMessage(RunnerMessage::HasActiveJobs { ctx_id, ack_tx }) => { - ack_tx - .send(runner.has_active_jobs(ctx_id)) - .expect("ack channel closed before sending has active jobs response"); - } - - StreamMessage::RunnerMessage(RunnerMessage::GetActiveReports { ack_tx }) => { - ack_tx - .send(runner.get_active_reports().await) - .expect("ack channel closed before sending active reports response"); - } - StreamMessage::RunnerMessage(RunnerMessage::ResumeStoredJob { - job_id, - location_id, - dyn_job, - ctx, - serialized_tasks, - ack_tx, - }) => { - ack_tx - .send( - runner - .new_job(job_id, location_id, dyn_job, ctx, serialized_tasks) - .await, - ) - .expect("ack channel closed before sending resume job response"); - } - - StreamMessage::RunnerMessage(RunnerMessage::Command { - job_id: id, - command, - ack_tx, - }) => runner.process_command(id, command, ack_tx).await, - - StreamMessage::RunnerMessage(RunnerMessage::Shutdown) => { - runner.on_shutdown_mode = true; - // Consuming all pending return status messages - if !runner.is_empty() { - let mut job_return_status_stream = pin!(job_return_status_rx_to_shutdown); - - runner.dispatch_shutdown_command_to_jobs().await; - - debug!( - total_jobs = runner.total_jobs(), - "Waiting for jobs to shutdown before shutting down the job system...;", - ); - - while let Some((job_id, status)) = job_return_status_stream.next().await { - if let Err(e) = runner.process_return_status(job_id, status).await { - error!(?e, "Failed to process return status before shutting down;"); - } - } - - // Now the runner can shutdown - if let Err(e) = runner.save_jobs(store_jobs_file).await { - error!(?e, "Failed to save jobs before shutting down;"); - } - } - - return; - } - - StreamMessage::RunnerMessage(RunnerMessage::CheckIfJobsAreRunning { - job_names, - location_id, - ack_tx, - }) => { - ack_tx - .send(runner.check_if_jobs_are_running(job_names, location_id)) - .expect("ack channel closed before sending resume job response"); - } - - // Memory cleanup tick - StreamMessage::CleanMemoryTick => runner.clean_memory(), - } - } -} diff --git a/core/crates/heavy-lifting/src/job_system/store.rs b/core/crates/heavy-lifting/src/job_system/store.rs deleted file mode 100644 index 82847dadc..000000000 --- a/core/crates/heavy-lifting/src/job_system/store.rs +++ /dev/null @@ -1,234 +0,0 @@ -use crate::{file_identifier, indexer, media_processor, JobContext}; - -use sd_prisma::prisma::{job, location}; -use sd_utils::uuid_to_bytes; - -use std::{ - collections::{HashMap, VecDeque}, - future::Future, - iter, - marker::PhantomData, - time::Duration, -}; - -use futures_concurrency::future::TryJoin; -use serde::{Deserialize, Serialize}; - -use super::{ - job::{DynJob, Job, JobHolder, JobName, OuterContext}, - report::{Report, ReportError}, - JobId, JobSystemError, -}; - -#[derive(Debug, Serialize, Deserialize)] -#[repr(transparent)] -#[serde(transparent)] -pub struct SerializedTasks(pub Vec); - -pub trait SerializableJob: 'static -where - Self: Sized, -{ - fn serialize( - self, - ) -> impl Future>, rmp_serde::encode::Error>> + Send { - async move { Ok(None) } - } - - #[allow(unused_variables)] - fn deserialize( - serialized_job: &[u8], - ctx: &OuterCtx, - ) -> impl Future< - Output = Result)>, rmp_serde::decode::Error>, - > + Send { - async move { Ok(None) } - } -} - -#[derive(Debug, Serialize, Deserialize)] -pub struct StoredJob { - pub(super) id: JobId, - pub(super) name: JobName, - pub(super) run_time: Duration, - pub(super) serialized_job: Vec, -} - -#[derive(Debug, Serialize, Deserialize)] -pub struct StoredJobEntry { - pub(super) location_id: location::id::Type, - pub(super) root_job: StoredJob, - pub(super) next_jobs: Vec, -} - -pub async fn load_jobs>( - entries: Vec, - ctx: &OuterCtx, -) -> Result< - Vec<( - location::id::Type, - Box>, - Option, - )>, - JobSystemError, -> { - let mut reports = ctx - .db() - .job() - .find_many(vec![job::id::in_vec( - entries - .iter() - .flat_map( - |StoredJobEntry { - root_job: StoredJob { id, .. }, - next_jobs, - .. - }| { - iter::once(*id).chain(next_jobs.iter().map(|StoredJob { id, .. }| *id)) - }, - ) - .map(|job_id| uuid_to_bytes(&job_id)) - .collect::>(), - )]) - .exec() - .await - .map_err(JobSystemError::LoadReportsForResume)? - .into_iter() - .map(Report::try_from) - .map(|report_res| report_res.map(|report| (report.id, report))) - .collect::, _>>()?; - - entries - .into_iter() - .map( - |StoredJobEntry { - location_id, - root_job, - next_jobs, - }| { - let report = reports - .remove(&root_job.id) - .ok_or(ReportError::MissingReport(root_job.id))?; - - Ok(async move { - load_job(root_job, report, ctx) - .await - .map(|maybe_loaded_job| { - maybe_loaded_job - .map(|(dyn_job, tasks)| (location_id, dyn_job, tasks, next_jobs)) - }) - }) - }, - ) - .collect::, JobSystemError>>()? - .try_join() - .await? - .into_iter() - .flatten() - .map(|(location_id, mut dyn_job, tasks, next_jobs)| { - let next_jobs_and_reports = next_jobs - .into_iter() - .map(|next_job| { - let next_job_id = next_job.id; - reports - .remove(&next_job.id) - .map(|report| (next_job, report)) - .ok_or(ReportError::MissingReport(next_job_id)) - }) - .collect::, _>>()?; - - Ok(async move { - next_jobs_and_reports - .into_iter() - .map(|(next_job, report)| async move { - load_job(next_job, report, ctx) - .await - .map(|maybe_loaded_next_job| { - maybe_loaded_next_job.map(|(next_dyn_job, next_tasks)| { - assert!( - next_tasks.is_none(), - "Next jobs must not have tasks as they haven't run yet" - ); - assert!( - next_dyn_job.next_jobs().is_empty(), - "Next jobs must not have next jobs" - ); - next_dyn_job - }) - }) - }) - .collect::>() - .try_join() - .await - .map(|maybe_next_dyn_jobs| { - dyn_job.set_next_jobs(maybe_next_dyn_jobs.into_iter().flatten().collect()); - (location_id, dyn_job, tasks) - }) - }) - }) - .collect::, JobSystemError>>()? - .try_join() - .await -} - -macro_rules! match_deserialize_job { - ($stored_job:ident, $report:ident, $outer_ctx:ident, $outer_ctx_type:ty, $job_ctx_type:ty, [$($job_type:ty),+ $(,)?]) => {{ - let StoredJob { - id, - name, - run_time, - serialized_job, - } = $stored_job; - - - match name { - $(<$job_type as Job>::NAME => <$job_type as SerializableJob<$outer_ctx_type>>::deserialize( - &serialized_job, - $outer_ctx, - ).await - .map(|maybe_job| maybe_job.map(|(job, maybe_tasks)| -> ( - Box>, - Option - ) { - ( - Box::new(JobHolder { - id, - job, - run_time, - report: $report, - next_jobs: VecDeque::new(), - _ctx: PhantomData, - }), - maybe_tasks.and_then( - |tasks| (!tasks.0.is_empty()).then_some(tasks) - ), - ) - } - )) - .map_err(Into::into),)+ - - // TODO(fogodev): this is temporary until we can get rid of the old job system - _ => unimplemented!("Job not implemented"), - } - }}; -} - -async fn load_job>( - stored_job: StoredJob, - report: Report, - ctx: &OuterCtx, -) -> Result>, Option)>, JobSystemError> { - match_deserialize_job!( - stored_job, - report, - ctx, - OuterCtx, - JobCtx, - [ - indexer::job::Indexer, - file_identifier::job::FileIdentifier, - media_processor::job::MediaProcessor, - // TODO: Add more jobs here - ] - ) -} diff --git a/core/crates/heavy-lifting/src/job_system/utils.rs b/core/crates/heavy-lifting/src/job_system/utils.rs deleted file mode 100644 index 8ac0070c4..000000000 --- a/core/crates/heavy-lifting/src/job_system/utils.rs +++ /dev/null @@ -1,35 +0,0 @@ -use crate::Error; - -use sd_task_system::{TaskHandle, TaskStatus}; - -use futures::{stream::FuturesUnordered, StreamExt}; -use futures_concurrency::future::Join; -use tracing::{error, trace}; - -pub async fn cancel_pending_tasks(pending_tasks: &mut FuturesUnordered>) { - pending_tasks - .iter() - .map(TaskHandle::cancel) - .collect::>() - .join() - .await; - - trace!(total_tasks = %pending_tasks.len(), "canceled all pending tasks, now waiting completion"); - - while let Some(task_result) = pending_tasks.next().await { - match task_result { - Ok(TaskStatus::Done((task_id, _))) => trace!( - %task_id, - "tasks cancellation received a completed task;", - ), - - Ok(TaskStatus::Canceled | TaskStatus::ForcedAbortion | TaskStatus::Shutdown(_)) => { - // Job canceled task - } - - Ok(TaskStatus::Error(e)) => error!(%e, "job canceled an errored task;"), - - Err(e) => error!(%e, "task system failed to cancel a task;"), - } - } -} diff --git a/core/crates/heavy-lifting/src/lib.rs b/core/crates/heavy-lifting/src/lib.rs deleted file mode 100644 index c137584e7..000000000 --- a/core/crates/heavy-lifting/src/lib.rs +++ /dev/null @@ -1,114 +0,0 @@ -#![recursion_limit = "256"] -#![warn( - clippy::all, - clippy::pedantic, - clippy::correctness, - clippy::perf, - clippy::style, - clippy::suspicious, - clippy::complexity, - clippy::nursery, - clippy::unwrap_used, - unused_qualifications, - rust_2018_idioms, - trivial_casts, - trivial_numeric_casts, - unused_allocation, - clippy::unnecessary_cast, - clippy::cast_lossless, - clippy::cast_possible_truncation, - clippy::cast_possible_wrap, - clippy::cast_precision_loss, - clippy::cast_sign_loss, - clippy::dbg_macro, - clippy::deprecated_cfg_attr, - clippy::separated_literal_suffix, - deprecated -)] -#![forbid(deprecated_in_future)] -#![allow(clippy::missing_errors_doc, clippy::module_name_repetitions)] - -use sd_prisma::prisma::file_path; -use sd_task_system::TaskSystemError; - -use serde::{Deserialize, Serialize}; -use specta::Type; -use thiserror::Error; - -pub mod file_identifier; -pub mod indexer; -pub mod job_system; -pub mod media_processor; -pub mod utils; - -use media_processor::ThumbKey; - -pub use job_system::{ - job::{ - IntoJob, JobContext, JobEnqueuer, JobName, JobOutput, JobOutputData, OuterContext, - ProgressUpdate, - }, - report::Report, - JobId, JobSystem, JobSystemError, -}; - -#[derive(Error, Debug)] -pub enum Error { - #[error(transparent)] - Indexer(#[from] indexer::Error), - #[error(transparent)] - FileIdentifier(#[from] file_identifier::Error), - #[error(transparent)] - MediaProcessor(#[from] media_processor::Error), - - #[error(transparent)] - TaskSystem(#[from] TaskSystemError), - - #[error(transparent)] - JobSystem(#[from] JobSystemError), -} - -impl From for rspc::Error { - fn from(e: Error) -> Self { - match e { - Error::Indexer(e) => e.into(), - Error::FileIdentifier(e) => e.into(), - Error::MediaProcessor(e) => e.into(), - Error::TaskSystem(e) => { - Self::with_cause(rspc::ErrorCode::InternalServerError, e.to_string(), e) - } - Error::JobSystem(e) => e.into(), - } - } -} - -#[derive(thiserror::Error, Debug, Serialize, Deserialize, Type, Clone)] -#[serde(rename_all = "snake_case")] -pub enum NonCriticalError { - // TODO: Add variants as needed - #[error(transparent)] - Indexer(#[from] indexer::NonCriticalIndexerError), - #[error(transparent)] - FileIdentifier(#[from] file_identifier::NonCriticalFileIdentifierError), - #[error(transparent)] - MediaProcessor(#[from] media_processor::NonCriticalMediaProcessorError), -} - -#[repr(i32)] -#[derive(Debug, Clone, Copy, Serialize, Deserialize, Type, Eq, PartialEq)] -pub enum LocationScanState { - Pending = 0, - Indexed = 1, - FilesIdentified = 2, - Completed = 3, -} - -#[derive(Debug, Serialize, Type)] -pub enum UpdateEvent { - NewThumbnail { - thumb_key: ThumbKey, - }, - NewIdentifiedObjects { - file_path_ids: Vec, - }, -} diff --git a/core/crates/heavy-lifting/src/media_processor/helpers/exif_media_data.rs b/core/crates/heavy-lifting/src/media_processor/helpers/exif_media_data.rs deleted file mode 100644 index 854ba314c..000000000 --- a/core/crates/heavy-lifting/src/media_processor/helpers/exif_media_data.rs +++ /dev/null @@ -1,172 +0,0 @@ -use crate::media_processor::{self, media_data_extractor}; - -use sd_core_prisma_helpers::ObjectPubId; -use sd_core_sync::{DevicePubId, SyncManager}; - -use sd_file_ext::extensions::{Extension, ImageExtension, ALL_IMAGE_EXTENSIONS}; -use sd_media_metadata::ExifMetadata; -use sd_prisma::{ - prisma::{device, exif_data, object, PrismaClient}, - prisma_sync, -}; -use sd_sync::{option_sync_db_entry, sync_entry, OperationFactory}; -use sd_utils::chain_optional_iter; - -use std::{path::Path, sync::LazyLock}; - -use futures_concurrency::future::TryJoin; - -use super::from_slice_option_to_option; - -pub static AVAILABLE_EXTENSIONS: LazyLock> = LazyLock::new(|| { - ALL_IMAGE_EXTENSIONS - .iter() - .copied() - .filter(|&ext| can_extract(ext)) - .map(Extension::Image) - .collect() -}); - -#[must_use] -pub const fn can_extract(image_extension: ImageExtension) -> bool { - use ImageExtension::{ - Avci, Avcs, Avif, Dng, Heic, Heif, Heifs, Hif, Jpeg, Jpg, Png, Tiff, Webp, - }; - matches!( - image_extension, - Tiff | Dng | Jpeg | Jpg | Heif | Heifs | Heic | Avif | Avcs | Avci | Hif | Png | Webp - ) -} - -#[must_use] -fn to_query( - ExifMetadata { - resolution, - date_taken, - location, - camera_data, - artist, - description, - copyright, - exif_version, - }: ExifMetadata, - object_id: exif_data::object_id::Type, - device_pub_id: &DevicePubId, -) -> (Vec<(&'static str, rmpv::Value)>, exif_data::Create) { - let device_pub_id = device_pub_id.to_db(); - - let (sync_params, db_params) = chain_optional_iter( - [( - sync_entry!( - prisma_sync::device::SyncId { - pub_id: device_pub_id.clone() - }, - exif_data::device - ), - exif_data::device::connect(device::pub_id::equals(device_pub_id)), - )], - [ - option_sync_db_entry!( - serde_json::to_vec(&camera_data).ok(), - exif_data::camera_data - ), - option_sync_db_entry!(serde_json::to_vec(&date_taken).ok(), exif_data::media_date), - option_sync_db_entry!(serde_json::to_vec(&resolution).ok(), exif_data::resolution), - option_sync_db_entry!( - serde_json::to_vec(&location).ok(), - exif_data::media_location - ), - option_sync_db_entry!(artist, exif_data::artist), - option_sync_db_entry!(description, exif_data::description), - option_sync_db_entry!(copyright, exif_data::copyright), - option_sync_db_entry!(exif_version, exif_data::exif_version), - option_sync_db_entry!( - date_taken.map(|x| x.unix_timestamp()), - exif_data::epoch_time - ), - ], - ) - .into_iter() - .unzip(); - - ( - sync_params, - exif_data::Create { - object: object::id::equals(object_id), - _params: db_params, - }, - ) -} - -pub async fn extract( - path: impl AsRef + Send, -) -> Result, media_processor::NonCriticalMediaProcessorError> { - let path = path.as_ref(); - - ExifMetadata::from_path(&path).await.map_err(|e| { - media_data_extractor::NonCriticalMediaDataExtractorError::FailedToExtractImageMediaData( - path.to_path_buf(), - e.to_string(), - ) - .into() - }) -} - -pub async fn save( - exif_datas: impl IntoIterator + Send, - db: &PrismaClient, - sync: &SyncManager, -) -> Result { - exif_datas - .into_iter() - .map(|(exif_data, object_id, object_pub_id)| async move { - let (sync_params, create) = to_query(exif_data, object_id, &sync.device_pub_id); - let db_params = create._params.clone(); - - sync.write_op( - db, - sync.shared_create( - prisma_sync::exif_data::SyncId { - object: prisma_sync::object::SyncId { - pub_id: object_pub_id.into(), - }, - }, - sync_params, - ), - db.exif_data() - .upsert(exif_data::object_id::equals(object_id), create, db_params) - .select(exif_data::select!({ id })), - ) - .await - }) - .collect::>() - .try_join() - .await - .map(|created_vec| created_vec.len() as u64) -} - -#[must_use] -pub fn from_prisma_data( - exif_data::Data { - resolution, - media_date, - media_location, - camera_data, - artist, - description, - copyright, - exif_version, - .. - }: exif_data::Data, -) -> ExifMetadata { - ExifMetadata { - camera_data: from_slice_option_to_option(camera_data).unwrap_or_default(), - date_taken: from_slice_option_to_option(media_date).unwrap_or_default(), - resolution: from_slice_option_to_option(resolution).unwrap_or_default(), - location: from_slice_option_to_option(media_location), - artist, - description, - copyright, - exif_version, - } -} diff --git a/core/crates/heavy-lifting/src/media_processor/helpers/ffmpeg_media_data.rs b/core/crates/heavy-lifting/src/media_processor/helpers/ffmpeg_media_data.rs deleted file mode 100644 index 30b63718d..000000000 --- a/core/crates/heavy-lifting/src/media_processor/helpers/ffmpeg_media_data.rs +++ /dev/null @@ -1,789 +0,0 @@ -use crate::media_processor::{self, media_data_extractor}; - -use sd_core_prisma_helpers::object_with_media_data; - -use sd_file_ext::extensions::{ - AudioExtension, Extension, VideoExtension, ALL_AUDIO_EXTENSIONS, ALL_VIDEO_EXTENSIONS, -}; -use sd_media_metadata::{ - ffmpeg::{ - audio_props::AudioProps, - chapter::Chapter, - codec::{Codec, Props}, - metadata::Metadata, - program::Program, - stream::Stream, - video_props::VideoProps, - }, - FFmpegMetadata, -}; -use sd_prisma::prisma::{ - ffmpeg_data, ffmpeg_media_audio_props, ffmpeg_media_chapter, ffmpeg_media_codec, - ffmpeg_media_program, ffmpeg_media_stream, ffmpeg_media_video_props, object, PrismaClient, -}; -use sd_utils::{ - db::{ffmpeg_data_field_from_db, ffmpeg_data_field_to_db}, - i64_to_frontend, -}; - -use std::{collections::HashMap, path::Path, sync::LazyLock}; - -use futures_concurrency::future::TryJoin; -use prisma_client_rust::QueryError; -use tracing::error; - -use super::from_slice_option_to_option; - -pub static AVAILABLE_EXTENSIONS: LazyLock> = LazyLock::new(|| { - ALL_AUDIO_EXTENSIONS - .iter() - .copied() - .filter(|&ext| can_extract_for_audio(ext)) - .map(Extension::Audio) - .chain( - ALL_VIDEO_EXTENSIONS - .iter() - .copied() - .filter(|&ext| can_extract_for_video(ext)) - .map(Extension::Video), - ) - .collect() -}); - -#[must_use] -pub const fn can_extract_for_audio(audio_extension: AudioExtension) -> bool { - use AudioExtension::{ - Aac, Adts, Aif, Aiff, Amr, Aptx, Ast, Caf, Flac, Loas, M4a, Mid, Mp2, Mp3, Oga, Ogg, Opus, - Tta, Voc, Wav, Wma, Wv, - }; - - matches!( - audio_extension, - Mp3 | Mp2 - | M4a | Wav - | Aiff | Aif - | Flac | Ogg - | Oga | Opus - | Wma | Amr - | Aac | Wv - | Voc | Tta - | Loas | Caf - | Aptx | Adts - | Ast | Mid - ) -} - -#[must_use] -pub const fn can_extract_for_video(video_extension: VideoExtension) -> bool { - use VideoExtension::{ - Asf, Avi, Avifs, F4v, Flv, Hevc, M2ts, M2v, M4v, Mjpeg, Mkv, Mov, Mp4, Mpe, Mpeg, Mpg, Mxf, - Ogv, Qt, Swf, Vob, Webm, Wm, Wmv, Wtv, _3gp, - }; - - matches!( - video_extension, - Avi | Avifs - | Qt | Mov - | Swf | Mjpeg - | Mpeg | Mxf - | M2v | Mpg - | Mpe | M2ts - | Flv | Wm - | _3gp | M4v - | Wmv | Asf - | Mp4 | Webm - | Mkv | Vob - | Ogv | Wtv - | Hevc | F4v // | Ts | Mts TODO: Uncomment when we start using magic instead of extension - ) -} - -pub async fn extract( - path: impl AsRef + Send, -) -> Result { - let path = path.as_ref(); - - FFmpegMetadata::from_path(&path).await.map_err(|e| { - media_data_extractor::NonCriticalMediaDataExtractorError::FailedToExtractImageMediaData( - path.to_path_buf(), - e.to_string(), - ) - .into() - }) -} - -pub async fn save( - ffmpeg_datas: impl IntoIterator + Send, - db: &PrismaClient, -) -> Result { - ffmpeg_datas - .into_iter() - .map( - move |( - FFmpegMetadata { - formats, - duration, - start_time, - bit_rate, - chapters, - programs, - metadata, - }, - object_id, - )| { - db._transaction() - .with_timeout(30 * 1000) - .run(move |db| async move { - let data_id = create_ffmpeg_data( - formats, bit_rate, duration, start_time, metadata, object_id, &db, - ) - .await?; - - create_ffmpeg_chapters(data_id, chapters, &db).await?; - - let streams = create_ffmpeg_programs(data_id, programs, &db).await?; - - let codecs = create_ffmpeg_streams(data_id, streams, &db).await?; - - let (audio_props, video_props) = - create_ffmpeg_codecs(data_id, codecs, &db).await?; - - ( - create_ffmpeg_audio_props(audio_props, &db), - create_ffmpeg_video_props(video_props, &db), - ) - .try_join() - .await - .map(|_| ()) - }) - }, - ) - .collect::>() - .try_join() - .await - .map(|created| created.len() as u64) - .map_err(Into::into) -} - -async fn create_ffmpeg_data( - formats: Vec, - (bit_rate_high, bit_rate_low): (i32, u32), - maybe_duration: Option<(i32, u32)>, - maybe_start_time: Option<(i32, u32)>, - metadata: Metadata, - object_id: i32, - db: &PrismaClient, -) -> Result { - db.ffmpeg_data() - .create( - formats.join(","), - ffmpeg_data_field_to_db(i64::from(bit_rate_high) << 32 | i64::from(bit_rate_low)), - object::id::equals(object_id), - vec![ - ffmpeg_data::duration::set(maybe_duration.map(|(duration_high, duration_low)| { - ffmpeg_data_field_to_db( - i64::from(duration_high) << 32 | i64::from(duration_low), - ) - })), - ffmpeg_data::start_time::set(maybe_start_time.map( - |(start_time_high, start_time_low)| { - ffmpeg_data_field_to_db( - i64::from(start_time_high) << 32 | i64::from(start_time_low), - ) - }, - )), - ffmpeg_data::metadata::set( - serde_json::to_vec(&metadata) - .map_err(|e| { - error!(?e, "Error reading FFmpegData metadata;"); - e - }) - .ok(), - ), - ], - ) - .select(ffmpeg_data::select!({ id })) - .exec() - .await - .map(|data| data.id) -} - -async fn create_ffmpeg_chapters( - ffmpeg_data_id: ffmpeg_data::id::Type, - chapters: Vec, - db: &PrismaClient, -) -> Result<(), QueryError> { - db.ffmpeg_media_chapter() - .create_many( - chapters - .into_iter() - .map( - |Chapter { - id: chapter_id, - start: (start_high, start_low), - end: (end_high, end_low), - time_base_den, - time_base_num, - metadata, - }| ffmpeg_media_chapter::CreateUnchecked { - chapter_id, - start: ffmpeg_data_field_to_db( - i64::from(start_high) << 32 | i64::from(start_low), - ), - end: ffmpeg_data_field_to_db( - i64::from(end_high) << 32 | i64::from(end_low), - ), - time_base_den, - time_base_num, - ffmpeg_data_id, - _params: vec![ffmpeg_media_chapter::metadata::set( - serde_json::to_vec(&metadata) - .map_err(|e| { - error!(?e, "Error reading FFmpegMediaChapter metadata;"); - e - }) - .ok(), - )], - }, - ) - .collect(), - ) - .exec() - .await - .map(|_| ()) -} - -async fn create_ffmpeg_programs( - data_id: i32, - programs: Vec, - db: &PrismaClient, -) -> Result)>, QueryError> { - let (creates, streams_by_program_id) = programs - .into_iter() - .map( - |Program { - id: program_id, - name, - metadata, - streams, - }| { - ( - ffmpeg_media_program::CreateUnchecked { - program_id, - ffmpeg_data_id: data_id, - _params: vec![ - ffmpeg_media_program::name::set(name), - ffmpeg_media_program::metadata::set( - serde_json::to_vec(&metadata) - .map_err(|e| { - error!(?e, "Error reading FFmpegMediaProgram metadata;"); - e - }) - .ok(), - ), - ], - }, - (program_id, streams), - ) - }, - ) - .unzip::<_, _, Vec<_>, Vec<_>>(); - - db.ffmpeg_media_program() - .create_many(creates) - .exec() - .await - .map(|_| streams_by_program_id) -} - -async fn create_ffmpeg_streams( - ffmpeg_data_id: ffmpeg_data::id::Type, - streams: Vec<(ffmpeg_media_program::program_id::Type, Vec)>, - db: &PrismaClient, -) -> Result< - Vec<( - ffmpeg_media_program::program_id::Type, - ffmpeg_media_stream::stream_id::Type, - Codec, - )>, - QueryError, -> { - let (creates, maybe_codecs) = streams - .into_iter() - .flat_map(|(program_id, streams)| { - streams.into_iter().map( - move |Stream { - id: stream_id, - name, - codec: maybe_codec, - aspect_ratio_num, - aspect_ratio_den, - frames_per_second_num, - frames_per_second_den, - time_base_real_den, - time_base_real_num, - dispositions, - metadata, - }| { - ( - ffmpeg_media_stream::CreateUnchecked { - stream_id, - aspect_ratio_num, - aspect_ratio_den, - frames_per_second_num, - frames_per_second_den, - time_base_real_den, - time_base_real_num, - program_id, - ffmpeg_data_id, - _params: vec![ - ffmpeg_media_stream::name::set(name), - ffmpeg_media_stream::dispositions::set( - (!dispositions.is_empty()).then_some(dispositions.join(",")), - ), - ffmpeg_media_stream::title::set(metadata.title.clone()), - ffmpeg_media_stream::encoder::set(metadata.encoder.clone()), - ffmpeg_media_stream::language::set(metadata.language.clone()), - ffmpeg_media_stream::metadata::set( - serde_json::to_vec(&metadata) - .map_err(|e| { - error!(?e, "Error reading FFmpegMediaStream metadata;"); - e - }) - .ok(), - ), - ], - }, - maybe_codec.map(|codec| (program_id, stream_id, codec)), - ) - }, - ) - }) - .unzip::<_, _, Vec<_>, Vec<_>>(); - - db.ffmpeg_media_stream() - .create_many(creates) - .exec() - .await - .map(|_| maybe_codecs.into_iter().flatten().collect()) -} - -async fn create_ffmpeg_codecs( - ffmpeg_data_id: ffmpeg_data::id::Type, - codecs: Vec<( - ffmpeg_media_program::program_id::Type, - ffmpeg_media_stream::stream_id::Type, - Codec, - )>, - db: &PrismaClient, -) -> Result< - ( - Vec<(ffmpeg_media_codec::id::Type, AudioProps)>, - Vec<(ffmpeg_media_codec::id::Type, VideoProps)>, - ), - QueryError, -> { - let expected_creates = codecs.len(); - - let (creates, mut audio_props, mut video_props) = codecs.into_iter().enumerate().fold( - ( - Vec::with_capacity(expected_creates), - HashMap::with_capacity(expected_creates), - HashMap::with_capacity(expected_creates), - ), - |(mut creates, mut audio_props, mut video_props), - ( - idx, - ( - program_id, - stream_id, - Codec { - kind, - sub_kind, - tag, - name, - profile, - bit_rate, - props: maybe_props, - }, - ), - )| { - creates.push(ffmpeg_media_codec::CreateUnchecked { - bit_rate, - stream_id, - program_id, - ffmpeg_data_id, - _params: vec![ - ffmpeg_media_codec::kind::set(kind), - ffmpeg_media_codec::sub_kind::set(sub_kind), - ffmpeg_media_codec::tag::set(tag), - ffmpeg_media_codec::name::set(name), - ffmpeg_media_codec::profile::set(profile), - ], - }); - - if let Some(props) = maybe_props { - match props { - Props::Audio(props) => { - audio_props.insert(idx, props); - } - Props::Video(props) => { - video_props.insert(idx, props); - } - Props::Subtitle(_) => { - // We don't care about subtitles props for now :D - } - } - } - - (creates, audio_props, video_props) - }, - ); - - let created_ids = creates - .into_iter() - .map( - |ffmpeg_media_codec::CreateUnchecked { - bit_rate, - stream_id, - program_id, - ffmpeg_data_id, - _params: params, - }| { - db.ffmpeg_media_codec() - .create_unchecked(bit_rate, stream_id, program_id, ffmpeg_data_id, params) - .select(ffmpeg_media_codec::select!({ id })) - .exec() - }, - ) - .collect::>() - .try_join() - .await?; - - assert_eq!( - created_ids.len(), - expected_creates, - "Not all codecs were created and our invariant is broken!" - ); - - debug_assert!( - created_ids - .windows(2) - .all(|window| window[0].id < window[1].id), - "Codecs were created in a different order than we expected, our invariant is broken!" - ); - - Ok(created_ids.into_iter().enumerate().fold( - ( - Vec::with_capacity(audio_props.len()), - Vec::with_capacity(video_props.len()), - ), - |(mut a_props, mut v_props), (idx, codec_data)| { - if let Some(audio_props) = audio_props.remove(&idx) { - a_props.push((codec_data.id, audio_props)); - } else if let Some(video_props) = video_props.remove(&idx) { - v_props.push((codec_data.id, video_props)); - } - - (a_props, v_props) - }, - )) -} - -async fn create_ffmpeg_audio_props( - audio_props: Vec<(ffmpeg_media_codec::id::Type, AudioProps)>, - db: &PrismaClient, -) -> Result<(), QueryError> { - db.ffmpeg_media_audio_props() - .create_many( - audio_props - .into_iter() - .map( - |( - codec_id, - AudioProps { - delay, - padding, - sample_rate, - sample_format, - bit_per_sample, - channel_layout, - }, - )| ffmpeg_media_audio_props::CreateUnchecked { - delay, - padding, - codec_id, - _params: vec![ - ffmpeg_media_audio_props::sample_rate::set(sample_rate), - ffmpeg_media_audio_props::sample_format::set(sample_format), - ffmpeg_media_audio_props::bit_per_sample::set(bit_per_sample), - ffmpeg_media_audio_props::channel_layout::set(channel_layout), - ], - }, - ) - .collect(), - ) - .exec() - .await - .map(|_| ()) -} - -async fn create_ffmpeg_video_props( - video_props: Vec<(ffmpeg_media_codec::id::Type, VideoProps)>, - db: &PrismaClient, -) -> Result<(), QueryError> { - db.ffmpeg_media_video_props() - .create_many( - video_props - .into_iter() - .map( - |( - codec_id, - VideoProps { - pixel_format, - color_range, - bits_per_channel, - color_space, - color_primaries, - color_transfer, - field_order, - chroma_location, - width, - height, - aspect_ratio_num, - aspect_ratio_den, - properties, - }, - )| { - ffmpeg_media_video_props::CreateUnchecked { - width, - height, - codec_id, - _params: vec![ - ffmpeg_media_video_props::pixel_format::set(pixel_format), - ffmpeg_media_video_props::color_range::set(color_range), - ffmpeg_media_video_props::bits_per_channel::set(bits_per_channel), - ffmpeg_media_video_props::color_space::set(color_space), - ffmpeg_media_video_props::color_primaries::set(color_primaries), - ffmpeg_media_video_props::color_transfer::set(color_transfer), - ffmpeg_media_video_props::field_order::set(field_order), - ffmpeg_media_video_props::chroma_location::set(chroma_location), - ffmpeg_media_video_props::aspect_ratio_num::set(aspect_ratio_num), - ffmpeg_media_video_props::aspect_ratio_den::set(aspect_ratio_den), - ffmpeg_media_video_props::properties::set(Some( - properties.join(","), - )), - ], - } - }, - ) - .collect(), - ) - .exec() - .await - .map(|_| ()) -} - -pub fn from_prisma_data( - object_with_media_data::ffmpeg_data::Data { - formats, - duration, - start_time, - bit_rate, - metadata, - chapters, - programs, - .. - }: object_with_media_data::ffmpeg_data::Data, -) -> FFmpegMetadata { - FFmpegMetadata { - formats: formats.split(',').map(String::from).collect::>(), - duration: duration.map(|duration| i64_to_frontend(ffmpeg_data_field_from_db(&duration))), - start_time: start_time - .map(|start_time| i64_to_frontend(ffmpeg_data_field_from_db(&start_time))), - bit_rate: i64_to_frontend(ffmpeg_data_field_from_db(&bit_rate)), - chapters: chapters_from_prisma_data(chapters), - programs: programs_from_prisma_data(programs), - metadata: from_slice_option_to_option(metadata).unwrap_or_default(), - } -} - -#[inline] -fn chapters_from_prisma_data(chapters: Vec) -> Vec { - chapters - .into_iter() - .map( - |ffmpeg_media_chapter::Data { - chapter_id, - start, - end, - time_base_den, - time_base_num, - metadata, - .. - }| Chapter { - id: chapter_id, - start: i64_to_frontend(ffmpeg_data_field_from_db(&start)), - end: i64_to_frontend(ffmpeg_data_field_from_db(&end)), - time_base_den, - time_base_num, - metadata: from_slice_option_to_option(metadata).unwrap_or_default(), - }, - ) - .collect() -} - -#[inline] -fn programs_from_prisma_data( - programs: Vec, -) -> Vec { - programs - .into_iter() - .map( - |object_with_media_data::ffmpeg_data::programs::Data { - program_id, - name, - metadata, - streams, - .. - }| Program { - id: program_id, - name, - streams: streams_from_prisma_data(streams), - metadata: from_slice_option_to_option(metadata).unwrap_or_default(), - }, - ) - .collect() -} - -fn streams_from_prisma_data( - streams: Vec, -) -> Vec { - streams - .into_iter() - .map( - |object_with_media_data::ffmpeg_data::programs::streams::Data { - stream_id, - name, - aspect_ratio_num, - aspect_ratio_den, - frames_per_second_num, - frames_per_second_den, - time_base_real_den, - time_base_real_num, - dispositions, - metadata, - codec, - .. - }| { - Stream { - id: stream_id, - name, - codec: codec_from_prisma_data(codec), - aspect_ratio_num, - aspect_ratio_den, - frames_per_second_num, - frames_per_second_den, - time_base_real_den, - time_base_real_num, - dispositions: dispositions - .map(|dispositions| { - dispositions - .split(',') - .map(String::from) - .collect::>() - }) - .unwrap_or_default(), - metadata: from_slice_option_to_option(metadata).unwrap_or_default(), - } - }, - ) - .collect() -} - -fn codec_from_prisma_data( - codec: Option, -) -> Option { - codec.map( - |object_with_media_data::ffmpeg_data::programs::streams::codec::Data { - kind, - sub_kind, - tag, - name, - profile, - bit_rate, - audio_props, - video_props, - .. - }| Codec { - kind, - sub_kind, - tag, - name, - profile, - bit_rate, - props: match (audio_props, video_props) { - ( - Some(ffmpeg_media_audio_props::Data { - delay, - padding, - sample_rate, - sample_format, - bit_per_sample, - channel_layout, - .. - }), - None, - ) => Some(Props::Audio(AudioProps { - delay, - padding, - sample_rate, - sample_format, - bit_per_sample, - channel_layout, - })), - ( - None, - Some(ffmpeg_media_video_props::Data { - pixel_format, - color_range, - bits_per_channel, - color_space, - color_primaries, - color_transfer, - field_order, - chroma_location, - width, - height, - aspect_ratio_num, - aspect_ratio_den, - properties, - .. - }), - ) => Some(Props::Video(VideoProps { - pixel_format, - color_range, - bits_per_channel, - color_space, - color_primaries, - color_transfer, - field_order, - chroma_location, - width, - height, - aspect_ratio_num, - aspect_ratio_den, - properties: properties - .map(|dispositions| { - dispositions - .split(',') - .map(String::from) - .collect::>() - }) - .unwrap_or_default(), - })), - _ => None, - }, - }, - ) -} diff --git a/core/crates/heavy-lifting/src/media_processor/helpers/mod.rs b/core/crates/heavy-lifting/src/media_processor/helpers/mod.rs deleted file mode 100644 index 702981355..000000000 --- a/core/crates/heavy-lifting/src/media_processor/helpers/mod.rs +++ /dev/null @@ -1,12 +0,0 @@ -pub mod exif_media_data; -pub mod ffmpeg_media_data; -pub mod thumbnailer; - -#[must_use] -fn from_slice_option_to_option( - value: Option>, -) -> Option { - value - .map(|x| serde_json::from_slice(&x).ok()) - .unwrap_or_default() -} diff --git a/core/crates/heavy-lifting/src/media_processor/helpers/thumbnailer.rs b/core/crates/heavy-lifting/src/media_processor/helpers/thumbnailer.rs deleted file mode 100644 index 6d919fbae..000000000 --- a/core/crates/heavy-lifting/src/media_processor/helpers/thumbnailer.rs +++ /dev/null @@ -1,547 +0,0 @@ -use crate::media_processor::thumbnailer; - -use sd_core_prisma_helpers::CasId; - -use sd_file_ext::extensions::{ - DocumentExtension, Extension, ImageExtension, ALL_DOCUMENT_EXTENSIONS, ALL_IMAGE_EXTENSIONS, -}; -use sd_images::{format_image, scale_dimensions, ConvertibleExtension}; -use sd_media_metadata::exif::Orientation; -use sd_utils::error::FileIOError; - -#[cfg(feature = "ffmpeg")] -use sd_file_ext::extensions::{VideoExtension, ALL_VIDEO_EXTENSIONS}; - -use std::{ - ops::Deref, - panic, - path::{Path, PathBuf}, - str::FromStr, - sync::LazyLock, - time::Duration, -}; - -use image::{imageops, DynamicImage, GenericImageView}; -use serde::{Deserialize, Serialize}; -use specta::Type; -use tokio::{ - fs::{self, File}, - io::{self, AsyncWriteExt}, - sync::{oneshot, Mutex}, - task::spawn_blocking, - time::{sleep, Instant}, -}; -use tracing::{error, instrument, trace}; -use uuid::Uuid; -use webp::{Encoder, WebPConfig}; - -// Files names constants -pub const THUMBNAIL_CACHE_DIR_NAME: &str = "thumbnails"; -pub const WEBP_EXTENSION: &str = "webp"; -pub const EPHEMERAL_DIR: &str = "ephemeral"; - -/// This is the target pixel count for all thumbnails to be resized to, and it is eventually downscaled -/// to [`TARGET_QUALITY`]. -pub const TARGET_PX: f32 = 1_048_576.0; // 1024x1024 - -/// This is the target quality that we render thumbnails at, it is a float between 0-100 -/// and is treated as a percentage (so 60% in this case, or it's the same as multiplying by `0.6`). -pub const TARGET_QUALITY: f32 = 60.0; - -/// How much time we allow for the thumbnailer task to complete before we give up. -pub const THUMBNAILER_TASK_TIMEOUT: Duration = Duration::from_secs(60 * 5); - -pub fn get_thumbnails_directory(data_directory: impl AsRef) -> PathBuf { - data_directory.as_ref().join(THUMBNAIL_CACHE_DIR_NAME) -} - -#[cfg(feature = "ffmpeg")] -pub static THUMBNAILABLE_VIDEO_EXTENSIONS: LazyLock> = LazyLock::new(|| { - ALL_VIDEO_EXTENSIONS - .iter() - .copied() - .filter(|&ext| can_generate_thumbnail_for_video(ext)) - .map(Extension::Video) - .collect() -}); - -pub static THUMBNAILABLE_EXTENSIONS: LazyLock> = LazyLock::new(|| { - ALL_IMAGE_EXTENSIONS - .iter() - .copied() - .filter(|&ext| can_generate_thumbnail_for_image(ext)) - .map(Extension::Image) - .chain( - ALL_DOCUMENT_EXTENSIONS - .iter() - .copied() - .filter(|&ext| can_generate_thumbnail_for_document(ext)) - .map(Extension::Document), - ) - .collect() -}); - -pub static ALL_THUMBNAILABLE_EXTENSIONS: LazyLock> = LazyLock::new(|| { - #[cfg(feature = "ffmpeg")] - return THUMBNAILABLE_EXTENSIONS - .iter() - .cloned() - .chain(THUMBNAILABLE_VIDEO_EXTENSIONS.iter().cloned()) - .collect(); - - #[cfg(not(feature = "ffmpeg"))] - THUMBNAILABLE_EXTENSIONS.clone() -}); - -static WEBP_CONFIG: LazyLock = LazyLock::new(|| { - let mut config = WebPConfig::new().expect("failed to instantiate global webp config"); - config.lossless = 0; - config.alpha_compression = 1; - config.quality = TARGET_QUALITY; - - config -}); - -const HALF_SEC: Duration = Duration::from_millis(500); - -static LAST_SINGLE_THUMB_GENERATED_LOCK: LazyLock> = - LazyLock::new(|| Mutex::new(Instant::now())); - -/// This type is used to pass the relevant data to the frontend so it can request the thumbnail. -/// Tt supports extending the shard hex to support deeper directory structures in the future -#[derive(Debug, Serialize, Deserialize, Type, Clone)] -pub struct ThumbKey { - pub shard_hex: String, - pub cas_id: CasId<'static>, - pub base_directory_str: String, -} - -impl ThumbKey { - #[must_use] - pub fn new(cas_id: CasId<'static>, kind: &ThumbnailKind) -> Self { - Self { - shard_hex: get_shard_hex(&cas_id).to_string(), - cas_id, - base_directory_str: match kind { - ThumbnailKind::Ephemeral => String::from(EPHEMERAL_DIR), - ThumbnailKind::Indexed(library_id) => library_id.to_string(), - }, - } - } - - #[must_use] - pub fn new_indexed(cas_id: CasId<'static>, library_id: Uuid) -> Self { - Self { - shard_hex: get_shard_hex(&cas_id).to_string(), - cas_id, - base_directory_str: library_id.to_string(), - } - } - - #[must_use] - pub fn new_ephemeral(cas_id: CasId<'static>) -> Self { - Self { - shard_hex: get_shard_hex(&cas_id).to_string(), - cas_id, - base_directory_str: String::from(EPHEMERAL_DIR), - } - } -} - -#[derive(Debug, Serialize, Deserialize, Type, Clone, Copy)] -pub enum ThumbnailKind { - Ephemeral, - Indexed(Uuid), -} - -impl ThumbnailKind { - pub fn compute_path(&self, data_directory: impl AsRef, cas_id: &CasId<'_>) -> PathBuf { - let mut thumb_path = get_thumbnails_directory(data_directory); - match self { - Self::Ephemeral => thumb_path.push(EPHEMERAL_DIR), - Self::Indexed(library_id) => { - thumb_path.push(library_id.to_string()); - } - } - thumb_path.push(get_shard_hex(cas_id)); - thumb_path.push(cas_id.as_str()); - thumb_path.set_extension(WEBP_EXTENSION); - - thumb_path - } -} - -#[derive(Debug, Serialize, Deserialize)] -pub struct GenerateThumbnailArgs<'cas_id> { - pub extension: String, - pub cas_id: CasId<'cas_id>, - pub path: PathBuf, -} - -impl<'cas_id> GenerateThumbnailArgs<'cas_id> { - #[must_use] - pub const fn new(extension: String, cas_id: CasId<'cas_id>, path: PathBuf) -> Self { - Self { - extension, - cas_id, - path, - } - } -} - -/// The practice of dividing files into hex coded folders, often called "sharding," -/// is mainly used to optimize file system performance. File systems can start to slow down -/// as the number of files in a directory increases. Thus, it's often beneficial to split -/// files into multiple directories to avoid this performance degradation. -/// -/// `get_shard_hex` takes a `cas_id` (a hexadecimal hash) as input and returns the first -/// three characters of the hash as the directory name. Because we're using these first -/// three characters of a the hash, this will give us 4096 (16^3) possible directories, -/// named 000 to fff. -#[inline] -#[must_use] -pub fn get_shard_hex<'cas_id>(cas_id: &'cas_id CasId<'cas_id>) -> &'cas_id str { - // Use the first three characters of the hash as the directory name - &cas_id.as_str()[0..3] -} - -#[cfg(feature = "ffmpeg")] -#[must_use] -pub const fn can_generate_thumbnail_for_video(video_extension: VideoExtension) -> bool { - use VideoExtension::{Hevc, M2ts, M2v, Mpg, Mts, Swf, Ts}; - // File extensions that are specifically not supported by the thumbnailer - !matches!(video_extension, Mpg | Swf | M2v | Hevc | M2ts | Mts | Ts) -} - -#[must_use] -pub const fn can_generate_thumbnail_for_image(image_extension: ImageExtension) -> bool { - use ImageExtension::{ - Avif, Bmp, Gif, Heic, Heics, Heif, Heifs, Ico, Jpeg, Jpg, Png, Svg, Webp, - }; - - matches!( - image_extension, - Jpg | Jpeg | Png | Webp | Gif | Svg | Heic | Heics | Heif | Heifs | Avif | Bmp | Ico - ) -} - -#[must_use] -pub const fn can_generate_thumbnail_for_document(document_extension: DocumentExtension) -> bool { - use DocumentExtension::Pdf; - - matches!(document_extension, Pdf) -} - -#[derive(Debug)] -pub enum GenerationStatus { - Generated, - Skipped, -} - -#[instrument(skip(thumbnails_directory, cas_id, should_regenerate, kind))] -pub async fn generate_thumbnail( - thumbnails_directory: &Path, - GenerateThumbnailArgs { - extension, - cas_id, - path, - }: &GenerateThumbnailArgs<'_>, - kind: &ThumbnailKind, - should_regenerate: bool, -) -> ( - Duration, - Result<(ThumbKey, GenerationStatus), thumbnailer::NonCriticalThumbnailerError>, -) { - trace!("Generating thumbnail"); - let start = Instant::now(); - - let mut output_path = match kind { - ThumbnailKind::Ephemeral => thumbnails_directory.join(EPHEMERAL_DIR), - ThumbnailKind::Indexed(library_id) => thumbnails_directory.join(library_id.to_string()), - }; - - output_path.push(get_shard_hex(cas_id)); - output_path.push(cas_id.as_str()); - output_path.set_extension(WEBP_EXTENSION); - - if let Err(e) = fs::metadata(&*output_path).await { - if e.kind() != io::ErrorKind::NotFound { - error!( - ?e, - "Failed to check if thumbnail exists, but we will try to generate it anyway;" - ); - } - // Otherwise we good, thumbnail doesn't exist so we can generate it - } else if !should_regenerate { - trace!("Skipping thumbnail generation because it already exists"); - return ( - start.elapsed(), - Ok(( - ThumbKey::new(cas_id.to_owned(), kind), - GenerationStatus::Skipped, - )), - ); - } - - if let Ok(extension) = ImageExtension::from_str(extension) { - if can_generate_thumbnail_for_image(extension) { - trace!("Generating image thumbnail"); - if let Err(e) = generate_image_thumbnail(&path, &output_path).await { - return (start.elapsed(), Err(e)); - } - trace!("Generated image thumbnail"); - } - } else if let Ok(extension) = DocumentExtension::from_str(extension) { - if can_generate_thumbnail_for_document(extension) { - trace!("Generating document thumbnail"); - if let Err(e) = generate_image_thumbnail(&path, &output_path).await { - return (start.elapsed(), Err(e)); - } - trace!("Generating document thumbnail"); - } - } - - #[cfg(feature = "ffmpeg")] - { - use crate::media_processor::helpers::thumbnailer::can_generate_thumbnail_for_video; - use sd_file_ext::extensions::VideoExtension; - - if let Ok(extension) = VideoExtension::from_str(extension) { - if can_generate_thumbnail_for_video(extension) { - trace!("Generating video thumbnail"); - if let Err(e) = generate_video_thumbnail(&path, &output_path).await { - return (start.elapsed(), Err(e)); - } - trace!("Generated video thumbnail"); - } - } - } - - trace!("Generated thumbnail"); - - ( - start.elapsed(), - Ok(( - ThumbKey::new(cas_id.to_owned(), kind), - GenerationStatus::Generated, - )), - ) -} - -fn inner_generate_image_thumbnail( - file_path: &PathBuf, -) -> Result, thumbnailer::NonCriticalThumbnailerError> { - let mut img = format_image(file_path).map_err(|e| { - thumbnailer::NonCriticalThumbnailerError::FormatImage(file_path.clone(), e.to_string()) - })?; - - let (w, h) = img.dimensions(); - - #[allow(clippy::cast_precision_loss)] - let (w_scaled, h_scaled) = scale_dimensions(w as f32, h as f32, TARGET_PX); - - // Optionally, resize the existing photo and convert back into DynamicImage - if w != w_scaled && h != h_scaled { - img = DynamicImage::ImageRgba8(imageops::resize( - &img, - w_scaled, - h_scaled, - imageops::FilterType::Triangle, - )); - } - - // this corrects the rotation/flip of the image based on the *available* exif data - // not all images have exif data, so we don't error. we also don't rotate HEIF as that's against the spec - if let Some(orientation) = Orientation::from_path(file_path) { - if ConvertibleExtension::try_from(file_path.as_ref()) - .expect("we already checked if the image was convertible") - .should_rotate() - { - img = orientation.correct_thumbnail(img); - } - } - - // Create the WebP encoder for the above image - let encoder = Encoder::from_image(&img).map_err(|reason| { - thumbnailer::NonCriticalThumbnailerError::WebPEncoding( - file_path.clone(), - reason.to_string(), - ) - })?; - - let thumb = encoder.encode_advanced(&WEBP_CONFIG).map_err(|reason| { - thumbnailer::NonCriticalThumbnailerError::WebPEncoding( - file_path.clone(), - format!("{reason:?}"), - ) - })?; - - // Type `WebPMemory` is !Send, which makes the `Future` in this function `!Send`, - // this make us `deref` to have a `&[u8]` and then `to_owned` to make a `Vec` - // which implies on a unwanted clone... - Ok(thumb.deref().to_owned()) -} - -#[instrument( - skip_all, - fields( - input_path = %file_path.as_ref().display(), - output_path = %output_path.as_ref().display() - ) -)] -async fn generate_image_thumbnail( - file_path: impl AsRef + Send, - output_path: impl AsRef + Send, -) -> Result<(), thumbnailer::NonCriticalThumbnailerError> { - let file_path = file_path.as_ref().to_path_buf(); - - let (tx, rx) = oneshot::channel(); - - // Using channel instead of waiting the JoinHandle as for some reason - // the JoinHandle can take some extra time to complete - let handle = spawn_blocking({ - let file_path = file_path.clone(); - - move || { - // Handling error on receiver side - - let _ = tx.send( - panic::catch_unwind(|| inner_generate_image_thumbnail(&file_path)).unwrap_or_else( - move |_| { - Err( - thumbnailer::NonCriticalThumbnailerError::PanicWhileGeneratingThumbnail( - file_path, - "Internal panic on third party crate".to_string(), - ), - ) - }, - ), - ); - } - }); - - let webp = if let Ok(res) = rx.await { - res? - } else { - error!("Failed to generate thumbnail"); - return Err( - thumbnailer::NonCriticalThumbnailerError::PanicWhileGeneratingThumbnail( - file_path, - handle - .await - .expect_err("as the channel was closed, then the spawned task panicked") - .to_string(), - ), - ); - }; - - trace!("Generated thumbnail bytes"); - - let output_path = output_path.as_ref(); - - if let Some(shard_dir) = output_path.parent() { - fs::create_dir_all(shard_dir).await.map_err(|e| { - thumbnailer::NonCriticalThumbnailerError::CreateShardDirectory( - FileIOError::from((shard_dir, e)).to_string(), - ) - })?; - } else { - error!("Failed to get parent directory for sharding parent directory"); - } - - trace!("Created shard directory and writing it to disk"); - - let mut file = File::create(output_path).await.map_err(|e| { - thumbnailer::NonCriticalThumbnailerError::SaveThumbnail( - file_path.clone(), - FileIOError::from((output_path, e)).to_string(), - ) - })?; - - file.write_all(&webp).await.map_err(|e| { - thumbnailer::NonCriticalThumbnailerError::SaveThumbnail( - file_path.clone(), - FileIOError::from((output_path, e)).to_string(), - ) - })?; - - file.sync_all().await.map_err(|e| { - thumbnailer::NonCriticalThumbnailerError::SaveThumbnail( - file_path, - FileIOError::from((output_path, e)).to_string(), - ) - })?; - - trace!("Wrote thumbnail to disk"); - return Ok(()); -} - -#[instrument( - skip_all, - fields( - input_path = %file_path.as_ref().display(), - output_path = %output_path.as_ref().display() - ) -)] -#[cfg(feature = "ffmpeg")] -async fn generate_video_thumbnail( - file_path: impl AsRef + Send, - output_path: impl AsRef + Send, -) -> Result<(), thumbnailer::NonCriticalThumbnailerError> { - use sd_ffmpeg::{to_thumbnail, ThumbnailSize}; - - let file_path = file_path.as_ref(); - - to_thumbnail( - file_path, - output_path, - ThumbnailSize::Scale(1024), - TARGET_QUALITY, - ) - .await - .map_err(|e| { - thumbnailer::NonCriticalThumbnailerError::VideoThumbnailGenerationFailed( - file_path.to_path_buf(), - e.to_string(), - ) - }) -} - -/// WARNING!!!! DON'T USE THIS FUNCTION IN A LOOP!!!!!!!!!!!!! It will be pretty slow on purpose! -pub async fn generate_single_thumbnail( - thumbnails_directory: impl AsRef + Send, - extension: String, - cas_id: CasId<'static>, - path: impl AsRef + Send, - kind: ThumbnailKind, -) -> Result<(), thumbnailer::NonCriticalThumbnailerError> { - let mut last_single_thumb_generated_guard = LAST_SINGLE_THUMB_GENERATED_LOCK.lock().await; - - let elapsed = Instant::now() - *last_single_thumb_generated_guard; - if elapsed < HALF_SEC { - // This will choke up in case someone try to use this method in a loop, otherwise - // it will consume all the machine resources like a gluton monster from hell - sleep(HALF_SEC - elapsed).await; - } - - let (_duration, res) = generate_thumbnail( - thumbnails_directory.as_ref(), - &GenerateThumbnailArgs { - extension, - cas_id, - path: path.as_ref().to_path_buf(), - }, - &kind, - false, - ) - .await; - - let (_thumb_key, status) = res?; - - if matches!(status, GenerationStatus::Generated) { - *last_single_thumb_generated_guard = Instant::now(); - drop(last_single_thumb_generated_guard); // Clippy was weirdly complaining about not doing an "early" drop here - } - - Ok(()) -} diff --git a/core/crates/heavy-lifting/src/media_processor/job.rs b/core/crates/heavy-lifting/src/media_processor/job.rs deleted file mode 100644 index fb622e162..000000000 --- a/core/crates/heavy-lifting/src/media_processor/job.rs +++ /dev/null @@ -1,1096 +0,0 @@ -use crate::{ - job_system::{ - job::{Job, JobReturn, JobTaskDispatcher, ReturnStatus}, - report::ReportOutputMetadata, - utils::cancel_pending_tasks, - DispatcherError, JobErrorOrDispatcherError, SerializableJob, SerializedTasks, - }, - media_processor::{self, helpers::thumbnailer::THUMBNAIL_CACHE_DIR_NAME}, - utils::sub_path::maybe_get_iso_file_path_from_sub_path, - Error, JobContext, JobName, LocationScanState, OuterContext, ProgressUpdate, -}; - -use sd_core_file_path_helper::IsolatedFilePathData; -use sd_core_prisma_helpers::file_path_for_media_processor; - -use sd_file_ext::extensions::Extension; -use sd_prisma::{ - prisma::{location, PrismaClient}, - prisma_sync, -}; -use sd_sync::{sync_db_not_null_entry, OperationFactory}; -use sd_task_system::{ - AnyTaskOutput, IntoTask, SerializableTask, Task, TaskDispatcher, TaskHandle, TaskId, - TaskOutput, TaskStatus, TaskSystemError, -}; -use sd_utils::{db::maybe_missing, u64_to_frontend}; - -use std::{ - collections::{HashMap, HashSet}, - fmt, - hash::{Hash, Hasher}, - mem, - path::PathBuf, - sync::Arc, - time::Duration, -}; - -use futures::{stream::FuturesUnordered, StreamExt}; -use futures_concurrency::future::TryJoin; -use itertools::Itertools; -use prisma_client_rust::{raw, PrismaValue}; -use serde::{Deserialize, Serialize}; -use serde_json::json; -use tracing::{debug, error, instrument, trace, warn, Level}; - -use super::{ - get_direct_children_files_by_extensions, helpers, - tasks::{ - self, media_data_extractor, - thumbnailer::{self, NewThumbnailReporter}, - }, - NewThumbnailsReporter, RawFilePathForMediaProcessor, BATCH_SIZE, -}; - -#[derive(Debug, Clone, Copy, Serialize, Deserialize)] -enum TaskKind { - MediaDataExtractor, - Thumbnailer, -} - -#[derive(Debug, Serialize, Deserialize, Clone, Copy)] -enum Phase { - MediaDataExtraction, - ThumbnailGeneration, - // LabelsGeneration, // TODO: Implement labels generation -} - -impl Default for Phase { - fn default() -> Self { - Self::MediaDataExtraction - } -} - -impl fmt::Display for Phase { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::MediaDataExtraction => write!(f, "media_data"), - Self::ThumbnailGeneration => write!(f, "thumbnails"), - // Self::LabelsGeneration => write!(f, "labels"), // TODO: Implement labels generation - } - } -} - -#[derive(Debug)] -pub struct MediaProcessor { - // Received arguments - location: Arc, - location_path: Arc, - sub_path: Option, - regenerate_thumbnails: bool, - - // Job control - total_media_data_extraction_files: u64, - total_media_data_extraction_tasks: u64, - total_thumbnailer_tasks: u64, - total_thumbnailer_files: u64, - phase: Phase, - - // Run data - metadata: Metadata, - errors: Vec, - - // On shutdown data - pending_tasks_on_resume: Vec>, - tasks_for_shutdown: Vec>>, -} - -impl Job for MediaProcessor { - const NAME: JobName = JobName::MediaProcessor; - - async fn resume_tasks( - &mut self, - dispatcher: &JobTaskDispatcher, - ctx: &impl JobContext, - SerializedTasks(serialized_tasks): SerializedTasks, - ) -> Result<(), Error> { - let reporter: Arc = - Arc::new(NewThumbnailsReporter { ctx: ctx.clone() }); - - if let Ok(tasks) = dispatcher - .dispatch_many_boxed( - rmp_serde::from_slice::)>>(&serialized_tasks) - .map_err(media_processor::Error::from)? - .into_iter() - .map(|(task_kind, task_bytes)| { - let reporter = Arc::clone(&reporter); - async move { - match task_kind { - TaskKind::MediaDataExtractor => { - tasks::MediaDataExtractor::deserialize( - &task_bytes, - (Arc::clone(ctx.db()), ctx.sync().clone()), - ) - .await - .map(IntoTask::into_task) - } - - TaskKind::Thumbnailer => { - tasks::Thumbnailer::deserialize(&task_bytes, reporter) - .await - .map(IntoTask::into_task) - } - } - } - }) - .collect::>() - .try_join() - .await - .map_err(media_processor::Error::from)?, - ) - .await - { - self.pending_tasks_on_resume = tasks; - } else { - warn!("Failed to dispatch tasks to resume as job was already canceled"); - } - - Ok(()) - } - - #[instrument( - skip_all, - fields( - location_id = self.location.id, - location_path = ?self.location.path, - sub_path = ?self.sub_path.as_ref().map(|path| path.display()), - regenerate_thumbnails = self.regenerate_thumbnails, - ), - ret(level = Level::TRACE), - err, - )] - async fn run( - mut self, - dispatcher: JobTaskDispatcher, - ctx: impl JobContext, - ) -> Result { - let mut pending_running_tasks = FuturesUnordered::new(); - - match self - .init_or_resume(&mut pending_running_tasks, &ctx, &dispatcher) - .await - { - Ok(()) => { /* Everything is awesome! */ } - Err(JobErrorOrDispatcherError::JobError(e)) => { - return Err(e.into()); - } - Err(JobErrorOrDispatcherError::Dispatcher(DispatcherError::JobCanceled(_))) => { - return Ok(self.cancel_job(&mut pending_running_tasks).await); - } - Err(JobErrorOrDispatcherError::Dispatcher(DispatcherError::Shutdown(tasks))) => { - self.tasks_for_shutdown.extend(tasks); - - if pending_running_tasks.is_empty() { - // If no task managed to be dispatched, we can just shutdown - // otherwise we have to process handles below and wait for them to be shutdown too - return Ok(ReturnStatus::Shutdown( - SerializableJob::::serialize(self).await, - )); - } - } - } - - if let Some(res) = self.process_handles(&mut pending_running_tasks, &ctx).await { - return res; - } - - if !self.tasks_for_shutdown.is_empty() { - return Ok(ReturnStatus::Shutdown( - SerializableJob::::serialize(self).await, - )); - } - - // From this point onward, we are done with the job and it can't be interrupted anymore - let Self { - location, - metadata, - errors, - .. - } = self; - - let (sync_param, db_param) = - sync_db_not_null_entry!(LocationScanState::Completed as i32, location::scan_state); - - ctx.sync() - .write_op( - ctx.db(), - ctx.sync().shared_update( - prisma_sync::location::SyncId { - pub_id: location.pub_id.clone(), - }, - [sync_param], - ), - ctx.db() - .location() - .update(location::id::equals(location.id), vec![db_param]) - .select(location::select!({ id })), - ) - .await - .map_err(media_processor::Error::from)?; - - Ok(ReturnStatus::Completed( - JobReturn::builder() - .with_metadata(metadata) - .with_non_critical_errors(errors) - .build(), - )) - } -} - -impl MediaProcessor { - pub fn new( - location: location::Data, - sub_path: Option, - regenerate_thumbnails: bool, - ) -> Result { - Ok(Self { - location_path: maybe_missing(&location.path, "location.path") - .map(PathBuf::from) - .map(Arc::new)?, - location: Arc::new(location), - sub_path, - regenerate_thumbnails, - total_media_data_extraction_files: 0, - total_media_data_extraction_tasks: 0, - total_thumbnailer_tasks: 0, - total_thumbnailer_files: 0, - phase: Phase::default(), - metadata: Metadata::default(), - errors: Vec::new(), - pending_tasks_on_resume: Vec::new(), - tasks_for_shutdown: Vec::new(), - }) - } - - #[allow(clippy::too_many_lines)] - async fn init_or_resume( - &mut self, - pending_running_tasks: &mut FuturesUnordered>, - job_ctx: &impl JobContext, - dispatcher: &JobTaskDispatcher, - ) -> Result<(), JobErrorOrDispatcherError> { - // if we don't have any pending task, then this is a fresh job - if self.pending_tasks_on_resume.is_empty() { - let location_id = self.location.id; - let location_path = &*self.location_path; - - let iso_file_path = maybe_get_iso_file_path_from_sub_path::( - location_id, - self.sub_path.as_ref(), - &*self.location_path, - job_ctx.db(), - ) - .await? - .map_or_else( - || { - IsolatedFilePathData::new(location_id, location_path, location_path, true) - .map_err(media_processor::Error::from) - }, - Ok, - )?; - - // First we will dispatch all tasks for media data extraction so we have a nice reporting - let media_data_extraction_tasks_res = self - .dispatch_media_data_extractor_tasks(&iso_file_path, dispatcher, job_ctx) - .await; - - // Now we dispatch thumbnailer tasks - let thumbnailer_tasks_res = self - .dispatch_thumbnailer_tasks( - &iso_file_path, - self.regenerate_thumbnails, - dispatcher, - job_ctx, - ) - .await; - - match (media_data_extraction_tasks_res, thumbnailer_tasks_res) { - (Ok(media_data_extraction_task_handles), Ok(thumbnailer_task_handles)) => { - pending_running_tasks.extend( - media_data_extraction_task_handles - .into_iter() - .chain(thumbnailer_task_handles), - ); - } - - ( - Ok(task_handles), - Err(JobErrorOrDispatcherError::Dispatcher(DispatcherError::JobCanceled(e))), - ) - | ( - Err(JobErrorOrDispatcherError::Dispatcher(DispatcherError::JobCanceled(e))), - Ok(task_handles), - ) => { - pending_running_tasks.extend(task_handles); - return Err(JobErrorOrDispatcherError::Dispatcher( - DispatcherError::JobCanceled(e), - )); - } - - ( - Ok(task_handles), - Err(JobErrorOrDispatcherError::Dispatcher(DispatcherError::Shutdown(tasks))), - ) - | ( - Err(JobErrorOrDispatcherError::Dispatcher(DispatcherError::Shutdown(tasks))), - Ok(task_handles), - ) => { - self.tasks_for_shutdown.extend(tasks); - pending_running_tasks.extend(task_handles); - } - - ( - Err(JobErrorOrDispatcherError::Dispatcher(DispatcherError::Shutdown( - media_data_extraction_tasks, - ))), - Err(JobErrorOrDispatcherError::Dispatcher(DispatcherError::Shutdown( - thumbnailer_tasks, - ))), - ) => { - self.tasks_for_shutdown.extend( - media_data_extraction_tasks - .into_iter() - .chain(thumbnailer_tasks), - ); - } - - ( - Err(JobErrorOrDispatcherError::Dispatcher(DispatcherError::JobCanceled(e))), - _, - ) - | ( - _, - Err(JobErrorOrDispatcherError::Dispatcher(DispatcherError::JobCanceled(e))), - ) => { - return Err(JobErrorOrDispatcherError::Dispatcher( - DispatcherError::JobCanceled(e), - )); - } - - (Err(JobErrorOrDispatcherError::JobError(e)), _) - | (_, Err(JobErrorOrDispatcherError::JobError(e))) => { - return Err(e.into()); - } - } - } else { - let updates = match self.phase { - Phase::MediaDataExtraction => vec![ - ProgressUpdate::TaskCount(self.total_media_data_extraction_files), - ProgressUpdate::CompletedTaskCount( - self.metadata.media_data_metrics.extracted - + self.metadata.media_data_metrics.skipped, - ), - ProgressUpdate::Phase(self.phase.to_string()), - ProgressUpdate::Message(format!( - "Preparing to process {} files in {} chunks", - self.total_media_data_extraction_files, - self.total_media_data_extraction_tasks - )), - ], - Phase::ThumbnailGeneration => vec![ - ProgressUpdate::TaskCount(self.total_thumbnailer_files), - ProgressUpdate::CompletedTaskCount( - self.metadata.thumbnailer_metrics_acc.generated - + self.metadata.thumbnailer_metrics_acc.skipped, - ), - ProgressUpdate::Phase(self.phase.to_string()), - ProgressUpdate::Message(format!( - "Preparing to process {} files in {} chunks", - self.total_thumbnailer_files, self.total_thumbnailer_tasks - )), - ], - }; - - job_ctx.progress(updates).await; - - pending_running_tasks.extend(mem::take(&mut self.pending_tasks_on_resume)); - } - - Ok(()) - } - - async fn process_handles( - &mut self, - pending_running_tasks: &mut FuturesUnordered>, - job_ctx: &impl JobContext, - ) -> Option> { - while let Some(task) = pending_running_tasks.next().await { - match task { - Ok(TaskStatus::Done((task_id, TaskOutput::Out(out)))) => { - self.process_task_output(task_id, out, job_ctx).await; - } - - Ok(TaskStatus::Done((task_id, TaskOutput::Empty))) => { - warn!(%task_id, "Task returned an empty output;"); - } - - Ok(TaskStatus::Shutdown(task)) => { - self.tasks_for_shutdown.push(task); - } - - Ok(TaskStatus::Error(e)) => { - cancel_pending_tasks(pending_running_tasks).await; - - return Some(Err(e)); - } - - Ok(TaskStatus::Canceled | TaskStatus::ForcedAbortion) => { - return Some(Ok(self.cancel_job(pending_running_tasks).await)); - } - - Err(TaskSystemError::TaskTimeout(task_id)) => { - warn!( - %task_id, - "Thumbnailer task timed out, we will keep processing the rest of the tasks;", - ); - self.errors.push( - media_processor::NonCriticalMediaProcessorError::Thumbnailer( - media_processor::NonCriticalThumbnailerError::TaskTimeout(task_id), - ) - .into(), - ); - } - - Err(e) => { - error!(?e, "Task System error;"); - cancel_pending_tasks(pending_running_tasks).await; - - return Some(Err(e.into())); - } - } - } - - None - } - - async fn process_task_output( - &mut self, - task_id: TaskId, - any_task_output: Box, - job_ctx: &impl JobContext, - ) { - if any_task_output.is::() { - let media_data_extractor::Output { - extracted, - skipped, - db_read_time, - filtering_time, - extraction_time, - db_write_time, - errors, - } = *any_task_output.downcast().expect("just checked"); - - self.metadata.media_data_metrics.extracted += extracted; - self.metadata.media_data_metrics.skipped += skipped; - self.metadata.media_data_metrics.mean_db_read_time += db_read_time; - self.metadata.media_data_metrics.mean_filtering_time += filtering_time; - self.metadata.media_data_metrics.mean_extraction_time += extraction_time; - self.metadata.media_data_metrics.mean_db_write_time += db_write_time; - self.metadata.media_data_metrics.total_successful_tasks += 1; - - if !errors.is_empty() { - warn!(?errors, "Non critical errors while extracting media data;"); - self.errors.extend(errors); - } - - debug!( - "Processed ({}/{}) media data extraction tasks, took: {:?};", - self.metadata.media_data_metrics.total_successful_tasks, - self.total_media_data_extraction_tasks, - db_read_time + filtering_time + extraction_time + db_write_time, - ); - job_ctx - .progress(vec![ProgressUpdate::CompletedTaskCount( - self.metadata.media_data_metrics.extracted - + self.metadata.media_data_metrics.skipped, - )]) - .await; - - if self.total_media_data_extraction_tasks - == self.metadata.media_data_metrics.total_successful_tasks - { - debug!("All media data extraction tasks have been processed"); - - self.phase = Phase::ThumbnailGeneration; - - job_ctx - .progress(vec![ - ProgressUpdate::TaskCount(self.total_thumbnailer_files), - ProgressUpdate::Phase(self.phase.to_string()), - ProgressUpdate::Message(format!( - "Waiting for processing of {} thumbnails in {} tasks", - self.total_thumbnailer_files, self.total_thumbnailer_tasks - )), - ]) - .await; - } - } else if any_task_output.is::() { - let thumbnailer::Output { - generated, - skipped, - errors, - total_time, - mean_time_acc, - std_dev_acc, - } = *any_task_output.downcast().expect("just checked"); - - self.metadata.thumbnailer_metrics_acc.generated += generated; - self.metadata.thumbnailer_metrics_acc.skipped += skipped; - self.metadata.thumbnailer_metrics_acc.mean_total_time += total_time; - self.metadata.thumbnailer_metrics_acc.mean_time_acc += mean_time_acc; - self.metadata.thumbnailer_metrics_acc.std_dev_acc += std_dev_acc; - self.metadata.thumbnailer_metrics_acc.total_successful_tasks += 1; - - if !errors.is_empty() { - warn!(?errors, "Non critical errors while generating thumbnails;"); - self.errors.extend(errors); - } - - debug!( - "Processed ({}/{}) thumbnailer tasks, took: {total_time:?}", - self.metadata.thumbnailer_metrics_acc.total_successful_tasks, - self.total_thumbnailer_tasks - ); - - if matches!(self.phase, Phase::ThumbnailGeneration) { - job_ctx - .progress(vec![ProgressUpdate::CompletedTaskCount( - self.metadata.thumbnailer_metrics_acc.generated - + self.metadata.thumbnailer_metrics_acc.skipped, - )]) - .await; - } - - // if self.total_thumbnailer_tasks - // == self.metadata.thumbnailer_metrics_acc.total_successful_tasks - // { - // debug!("All thumbnailer tasks have been processed"); - - // self.phase = Phase::LabelsGeneration; - - // ctx.progress(vec![ - // ProgressUpdate::TaskCount(self.total_thumbnailer_files), - // ProgressUpdate::Phase(self.phase.to_string()), - // ProgressUpdate::Message(format!( - // "Waiting for processing of {} labels in {} tasks", - // self.total_labeller_files, self.total_labeller_tasks - // )), - // ]).await; - // } - } else { - unreachable!("Unexpected task output type: "); - } - } - - async fn cancel_job( - &mut self, - pending_running_tasks: &mut FuturesUnordered>, - ) -> ReturnStatus { - cancel_pending_tasks(pending_running_tasks).await; - - ReturnStatus::Canceled( - JobReturn::builder() - .with_metadata(mem::take(&mut self.metadata)) - .with_non_critical_errors(mem::take(&mut self.errors)) - .build(), - ) - } - - #[instrument(skip_all, fields(parent_iso_file_path = %parent_iso_file_path.as_ref().display()))] - async fn dispatch_media_data_extractor_tasks( - &mut self, - parent_iso_file_path: &IsolatedFilePathData<'_>, - dispatcher: &JobTaskDispatcher, - job_ctx: &impl JobContext, - ) -> Result>, JobErrorOrDispatcherError> { - let db = job_ctx.db(); - let sync = job_ctx.sync(); - - let (extract_exif_file_paths, extract_ffmpeg_file_paths) = ( - get_all_children_files_by_extensions( - parent_iso_file_path, - &helpers::exif_media_data::AVAILABLE_EXTENSIONS, - db, - ), - get_all_children_files_by_extensions( - parent_iso_file_path, - &helpers::ffmpeg_media_data::AVAILABLE_EXTENSIONS, - db, - ), - ) - .try_join() - .await?; - - let files_count = (extract_exif_file_paths.len() + extract_ffmpeg_file_paths.len()) as u64; - - let tasks = extract_exif_file_paths - .into_iter() - .chunks(BATCH_SIZE) - .into_iter() - .map(Iterator::collect::>) - .map(|chunked_file_paths| { - tasks::MediaDataExtractor::new_exif( - &chunked_file_paths, - parent_iso_file_path.location_id(), - Arc::clone(&self.location_path), - Arc::clone(db), - sync.clone(), - ) - }) - .map(IntoTask::into_task) - .chain( - extract_ffmpeg_file_paths - .into_iter() - .chunks(BATCH_SIZE) - .into_iter() - .map(Iterator::collect::>) - .map(|chunked_file_paths| { - tasks::MediaDataExtractor::new_ffmpeg( - &chunked_file_paths, - parent_iso_file_path.location_id(), - Arc::clone(&self.location_path), - Arc::clone(db), - sync.clone(), - ) - }) - .map(IntoTask::into_task), - ) - .collect::>(); - - trace!( - tasks_count = tasks.len(), - %files_count, - "Dispatching media data extraction tasks;", - ); - - self.total_media_data_extraction_files = files_count; - self.total_media_data_extraction_tasks = tasks.len() as u64; - - job_ctx - .progress(vec![ - ProgressUpdate::TaskCount(self.total_media_data_extraction_files), - ProgressUpdate::Phase(self.phase.to_string()), - ProgressUpdate::Message(format!( - "Preparing to process {} files in {} chunks", - self.total_media_data_extraction_files, self.total_media_data_extraction_tasks - )), - ]) - .await; - - dispatcher - .dispatch_many_boxed(tasks) - .await - .map_err(Into::into) - } - - async fn dispatch_thumbnailer_tasks( - &mut self, - parent_iso_file_path: &IsolatedFilePathData<'_>, - should_regenerate: bool, - dispatcher: &JobTaskDispatcher, - ctx: &impl OuterContext, - ) -> Result>, JobErrorOrDispatcherError> { - let thumbnails_directory_path = - Arc::new(ctx.get_data_directory().join(THUMBNAIL_CACHE_DIR_NAME)); - let location_id = parent_iso_file_path.location_id(); - let library_id = ctx.id(); - let db = ctx.db(); - let reporter: Arc = - Arc::new(NewThumbnailsReporter { ctx: ctx.clone() }); - - let priority_file_paths = get_direct_children_files_by_extensions( - parent_iso_file_path, - &helpers::thumbnailer::ALL_THUMBNAILABLE_EXTENSIONS, - db, - ) - .await?; - - let priority_file_path_ids = priority_file_paths - .iter() - .map(|file_path| file_path.id) - .collect::>(); - - let mut file_paths = get_all_children_files_by_extensions( - parent_iso_file_path, - &helpers::thumbnailer::ALL_THUMBNAILABLE_EXTENSIONS, - db, - ) - .await?; - - file_paths.retain(|file_path| !priority_file_path_ids.contains(&file_path.id)); - - if priority_file_path_ids.is_empty() && file_paths.is_empty() { - return Ok(Vec::new()); - } - - let thumbs_count = (priority_file_paths.len() + file_paths.len()) as u64; - - let priority_tasks = priority_file_paths - .into_iter() - .chunks(BATCH_SIZE) - .into_iter() - .map(|chunk| { - tasks::Thumbnailer::new_indexed( - Arc::clone(&thumbnails_directory_path), - &chunk.collect::>(), - (location_id, &self.location_path), - library_id, - should_regenerate, - true, - Arc::clone(&reporter), - ) - }) - .map(IntoTask::into_task) - .collect::>(); - - let non_priority_tasks = file_paths - .into_iter() - .chunks(BATCH_SIZE) - .into_iter() - .map(|chunk| { - tasks::Thumbnailer::new_indexed( - Arc::clone(&thumbnails_directory_path), - &chunk.collect::>(), - (location_id, &self.location_path), - library_id, - should_regenerate, - false, - Arc::clone(&reporter), - ) - }) - .map(IntoTask::into_task) - .collect::>(); - - debug!( - %thumbs_count, - priority_tasks_count = priority_tasks.len(), - non_priority_tasks_count = non_priority_tasks.len(), - "Dispatching thumbnails to be processed;", - ); - - self.total_thumbnailer_tasks = (priority_tasks.len() + non_priority_tasks.len()) as u64; - self.total_thumbnailer_files = thumbs_count; - - dispatcher - .dispatch_many_boxed(priority_tasks.into_iter().chain(non_priority_tasks)) - .await - .map_err(Into::into) - } -} - -#[derive(Debug, Serialize, Deserialize, Default)] -struct Metadata { - media_data_metrics: MediaExtractorMetrics, - thumbnailer_metrics_acc: ThumbnailerMetricsAccumulator, -} - -impl From for Vec { - fn from( - Metadata { - media_data_metrics, - thumbnailer_metrics_acc: thumbnailer_metrics_accumulator, - }: Metadata, - ) -> Self { - let thumbnailer_metrics = ThumbnailerMetrics::from(thumbnailer_metrics_accumulator); - - vec![ - ReportOutputMetadata::MediaProcessor { - media_data_extracted: u64_to_frontend(media_data_metrics.extracted), - media_data_skipped: u64_to_frontend(media_data_metrics.skipped), - thumbnails_generated: u64_to_frontend(thumbnailer_metrics.generated), - thumbnails_skipped: u64_to_frontend(thumbnailer_metrics.skipped), - }, - ReportOutputMetadata::Metrics(HashMap::from([ - // - // Media data extractor - // - ( - "media_data_extraction_metrics".into(), - json!(media_data_metrics), - ), - // - // Thumbnailer - // - ("thumbnailer_metrics".into(), json!(thumbnailer_metrics)), - ])), - ] - } -} - -#[derive(Debug, Serialize, Deserialize, Default)] -struct MediaExtractorMetrics { - extracted: u64, - skipped: u64, - mean_db_read_time: Duration, - mean_filtering_time: Duration, - mean_extraction_time: Duration, - mean_db_write_time: Duration, - total_successful_tasks: u64, -} - -#[derive(Debug, Serialize, Deserialize, Default)] -struct ThumbnailerMetricsAccumulator { - generated: u64, - skipped: u64, - mean_total_time: Duration, - mean_time_acc: f64, - std_dev_acc: f64, - total_successful_tasks: u64, -} - -#[derive(Debug, Serialize, Deserialize, Default)] -struct ThumbnailerMetrics { - generated: u64, - skipped: u64, - mean_total_time: Duration, - mean_generation_time: Duration, - std_dev: Duration, - total_successful_tasks: u64, -} - -impl From for ThumbnailerMetrics { - fn from( - ThumbnailerMetricsAccumulator { - generated, - skipped, - mean_total_time, - mean_time_acc: mean_generation_time_acc, - std_dev_acc, - total_successful_tasks, - }: ThumbnailerMetricsAccumulator, - ) -> Self { - if generated + skipped == 0 { - return Self { - generated, - skipped, - mean_total_time, - mean_generation_time: Duration::ZERO, - std_dev: Duration::ZERO, - total_successful_tasks, - }; - } - - #[allow(clippy::cast_precision_loss)] - // SAFETY: we're probably won't have 2^52 thumbnails being generated on a single job for this cast to have - // a precision loss issue - let total = (generated + skipped) as f64; - let mean_generation_time = mean_generation_time_acc / total; - - let std_dev = if generated > 1 { - Duration::from_secs_f64( - (mean_generation_time.mul_add(-mean_generation_time, std_dev_acc / total)).sqrt(), - ) - } else { - Duration::ZERO - }; - - Self { - generated, - skipped, - mean_total_time, - mean_generation_time: Duration::from_secs_f64(if generated > 1 { - mean_generation_time - } else { - mean_generation_time_acc - }), - std_dev, - total_successful_tasks, - } - } -} - -async fn get_all_children_files_by_extensions( - parent_iso_file_path: &IsolatedFilePathData<'_>, - extensions: &[Extension], - db: &PrismaClient, -) -> Result, media_processor::Error> { - // FIXME: Had to use format! macro because PCR doesn't support IN with Vec for SQLite - // We have no data coming from the user, so this is sql injection safe - let unique_by_object_id = db - ._query_raw::(raw!( - &format!( - "SELECT - file_path.id, - file_path.materialized_path, - file_path.is_dir, - file_path.name, - file_path.extension, - file_path.cas_id, - object.id as 'object_id', - object.pub_id as 'object_pub_id' - FROM file_path - INNER JOIN object ON object.id = file_path.object_id - WHERE - file_path.location_id={{}} - AND file_path.cas_id IS NOT NULL - AND LOWER(file_path.extension) IN ({}) - AND file_path.materialized_path LIKE {{}} - ORDER BY materialized_path ASC, name ASC", - // Ordering by materialized_path so we can prioritize processing the first files - // in the above part of the directories tree - extensions - .iter() - .map(|ext| format!("LOWER('{ext}')")) - .collect::>() - .join(",") - ), - PrismaValue::Int(parent_iso_file_path.location_id()), - PrismaValue::String(format!( - "{}%", - parent_iso_file_path - .materialized_path_for_children() - .expect("sub path iso_file_path must be a directory") - )) - )) - .exec() - .await? - .into_iter() - .map(|raw_file_path| (raw_file_path.object_id, raw_file_path)) - .collect::>(); - - Ok(unique_by_object_id.into_values().map(Into::into).collect()) -} - -#[derive(Serialize, Deserialize)] -struct SaveState { - location: Arc, - location_path: Arc, - sub_path: Option, - regenerate_thumbnails: bool, - - total_media_data_extraction_files: u64, - total_media_data_extraction_tasks: u64, - total_thumbnailer_tasks: u64, - total_thumbnailer_files: u64, - - phase: Phase, - - metadata: Metadata, - - errors: Vec, - - tasks_for_shutdown_bytes: Option, -} - -impl SerializableJob for MediaProcessor { - async fn serialize(self) -> Result>, rmp_serde::encode::Error> { - let Self { - location, - location_path, - sub_path, - regenerate_thumbnails, - total_media_data_extraction_files, - total_media_data_extraction_tasks, - total_thumbnailer_tasks, - total_thumbnailer_files, - phase, - metadata, - errors, - tasks_for_shutdown, - .. - } = self; - - let serialized_tasks = tasks_for_shutdown - .into_iter() - .map(|task| async move { - if task.is::() { - task.downcast::() - .expect("just checked") - .serialize() - .await - .map(|bytes| (TaskKind::MediaDataExtractor, bytes)) - } else if task.is::() { - task.downcast::() - .expect("just checked") - .serialize() - .await - .map(|bytes| (TaskKind::Thumbnailer, bytes)) - } else { - unreachable!("Unexpected task type: ") - } - }) - .collect::>() - .try_join() - .await?; - - let tasks_for_shutdown_bytes = if serialized_tasks.is_empty() { - None - } else { - Some(SerializedTasks(rmp_serde::to_vec_named(&serialized_tasks)?)) - }; - - rmp_serde::to_vec_named(&SaveState { - location, - location_path, - sub_path, - regenerate_thumbnails, - total_media_data_extraction_files, - total_media_data_extraction_tasks, - total_thumbnailer_tasks, - total_thumbnailer_files, - phase, - metadata, - errors, - tasks_for_shutdown_bytes, - }) - .map(Some) - } - - async fn deserialize( - serialized_job: &[u8], - _: &OuterCtx, - ) -> Result)>, rmp_serde::decode::Error> { - let SaveState { - location, - location_path, - sub_path, - regenerate_thumbnails, - total_media_data_extraction_files, - total_media_data_extraction_tasks, - total_thumbnailer_tasks, - total_thumbnailer_files, - phase, - metadata, - errors, - tasks_for_shutdown_bytes, - } = rmp_serde::from_slice::(serialized_job)?; - - Ok(Some(( - Self { - location, - location_path, - sub_path, - regenerate_thumbnails, - total_media_data_extraction_files, - total_media_data_extraction_tasks, - total_thumbnailer_tasks, - total_thumbnailer_files, - phase, - metadata, - errors, - pending_tasks_on_resume: Vec::new(), - tasks_for_shutdown: Vec::new(), - }, - tasks_for_shutdown_bytes, - ))) - } -} - -impl Hash for MediaProcessor { - fn hash(&self, state: &mut H) { - self.location.id.hash(state); - if let Some(ref sub_path) = self.sub_path { - sub_path.hash(state); - } - } -} diff --git a/core/crates/heavy-lifting/src/media_processor/mod.rs b/core/crates/heavy-lifting/src/media_processor/mod.rs deleted file mode 100644 index 4ac817cb8..000000000 --- a/core/crates/heavy-lifting/src/media_processor/mod.rs +++ /dev/null @@ -1,193 +0,0 @@ -use crate::{utils::sub_path, OuterContext, UpdateEvent}; - -use sd_core_file_path_helper::{FilePathError, IsolatedFilePathData}; -use sd_core_prisma_helpers::file_path_for_media_processor; - -use sd_file_ext::extensions::Extension; -use sd_prisma::prisma::{file_path, object, PrismaClient}; -use sd_utils::db::MissingFieldError; - -use std::{collections::HashMap, fmt}; - -use prisma_client_rust::{raw, PrismaValue}; -use serde::{Deserialize, Serialize}; -use specta::Type; - -mod helpers; -pub mod job; -mod shallow; -mod tasks; - -pub use tasks::{ - media_data_extractor::{self, MediaDataExtractor}, - thumbnailer::{self, Thumbnailer}, -}; - -pub use helpers::{ - exif_media_data, ffmpeg_media_data, - thumbnailer::{ - can_generate_thumbnail_for_document, can_generate_thumbnail_for_image, - generate_single_thumbnail, get_shard_hex, get_thumbnails_directory, GenerateThumbnailArgs, - ThumbKey, ThumbnailKind, WEBP_EXTENSION, - }, -}; - -#[cfg(feature = "ffmpeg")] -pub use helpers::thumbnailer::can_generate_thumbnail_for_video; - -pub use shallow::shallow; - -use media_data_extractor::NonCriticalMediaDataExtractorError; -use thumbnailer::{NewThumbnailReporter, NonCriticalThumbnailerError}; - -#[cfg(target_os = "ios")] -const BATCH_SIZE: usize = 2; // Much smaller batch size for iOS - -#[cfg(target_os = "android")] -const BATCH_SIZE: usize = 2; // Much smaller batch size for Android - -#[cfg(not(any(target_os = "ios", target_os = "android")))] -const BATCH_SIZE: usize = 10; // Keep original batch size for other platforms - -#[derive(thiserror::Error, Debug)] -pub enum Error { - #[error("missing field on database: {0}")] - MissingField(#[from] MissingFieldError), - #[error("database error: {0}")] - Database(#[from] prisma_client_rust::QueryError), - #[error("failed to deserialized stored tasks for job resume: {0}")] - DeserializeTasks(#[from] rmp_serde::decode::Error), - - #[error(transparent)] - FilePathError(#[from] FilePathError), - #[error(transparent)] - SubPath(#[from] sub_path::Error), - #[error(transparent)] - Sync(#[from] sd_core_sync::Error), -} - -impl From for rspc::Error { - fn from(e: Error) -> Self { - match e { - Error::SubPath(sub_path_err) => sub_path_err.into(), - - _ => Self::with_cause(rspc::ErrorCode::InternalServerError, e.to_string(), e), - } - } -} - -#[derive(thiserror::Error, Debug, Serialize, Deserialize, Type, Clone)] -#[serde(rename_all = "snake_case")] -pub enum NonCriticalMediaProcessorError { - #[error(transparent)] - MediaDataExtractor(#[from] NonCriticalMediaDataExtractorError), - #[error(transparent)] - Thumbnailer(#[from] NonCriticalThumbnailerError), -} - -#[derive(Clone)] -pub struct NewThumbnailsReporter { - pub ctx: OuterCtx, -} - -impl fmt::Debug for NewThumbnailsReporter { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("NewThumbnailsReporter").finish() - } -} - -impl NewThumbnailReporter for NewThumbnailsReporter { - fn new_thumbnail(&self, thumb_key: ThumbKey) { - self.ctx - .report_update(UpdateEvent::NewThumbnail { thumb_key }); - } -} - -#[derive(Deserialize)] -struct RawFilePathForMediaProcessor { - id: file_path::id::Type, - materialized_path: file_path::materialized_path::Type, - is_dir: file_path::is_dir::Type, - name: file_path::name::Type, - extension: file_path::extension::Type, - cas_id: file_path::cas_id::Type, - object_id: object::id::Type, - object_pub_id: object::pub_id::Type, -} - -impl From for file_path_for_media_processor::Data { - fn from( - RawFilePathForMediaProcessor { - id, - materialized_path, - is_dir, - name, - extension, - cas_id, - object_id, - object_pub_id, - }: RawFilePathForMediaProcessor, - ) -> Self { - Self { - id, - materialized_path, - is_dir, - name, - extension, - cas_id, - object: Some(file_path_for_media_processor::object::Data { - id: object_id, - pub_id: object_pub_id, - }), - } - } -} - -async fn get_direct_children_files_by_extensions( - parent_iso_file_path: &IsolatedFilePathData<'_>, - extensions: &[Extension], - db: &PrismaClient, -) -> Result, Error> { - // FIXME: Had to use format! macro because PCR doesn't support IN with Vec for SQLite - // We have no data coming from the user, so this is sql injection safe - let unique_by_object_id = db - ._query_raw::(raw!( - &format!( - "SELECT - file_path.id, - file_path.materialized_path, - file_path.is_dir, - file_path.name, - file_path.extension, - file_path.cas_id, - object.id as 'object_id', - object.pub_id as 'object_pub_id' - FROM file_path - INNER JOIN object ON object.id = file_path.object_id - WHERE - location_id={{}} - AND cas_id IS NOT NULL - AND LOWER(extension) IN ({}) - AND materialized_path = {{}} - ORDER BY name ASC", - extensions - .iter() - .map(|ext| format!("LOWER('{ext}')")) - .collect::>() - .join(",") - ), - PrismaValue::Int(parent_iso_file_path.location_id()), - PrismaValue::String( - parent_iso_file_path - .materialized_path_for_children() - .expect("sub path iso_file_path must be a directory") - ) - )) - .exec() - .await? - .into_iter() - .map(|raw_file_path| (raw_file_path.object_id, raw_file_path)) - .collect::>(); - - Ok(unique_by_object_id.into_values().map(Into::into).collect()) -} diff --git a/core/crates/heavy-lifting/src/media_processor/shallow.rs b/core/crates/heavy-lifting/src/media_processor/shallow.rs deleted file mode 100644 index 675dcd791..000000000 --- a/core/crates/heavy-lifting/src/media_processor/shallow.rs +++ /dev/null @@ -1,271 +0,0 @@ -use crate::{ - media_processor, utils::sub_path::maybe_get_iso_file_path_from_sub_path, Error, - NonCriticalError, OuterContext, -}; - -use sd_core_file_path_helper::IsolatedFilePathData; -use sd_core_sync::SyncManager; - -use sd_prisma::prisma::{location, PrismaClient}; -use sd_task_system::{ - BaseTaskDispatcher, CancelTaskOnDrop, IntoTask, TaskDispatcher, TaskHandle, TaskOutput, - TaskStatus, -}; -use sd_utils::db::maybe_missing; - -use std::{ - path::{Path, PathBuf}, - sync::Arc, -}; - -use futures::{stream::FuturesUnordered, StreamExt}; -use futures_concurrency::future::TryJoin; -use itertools::Itertools; -use tracing::{debug, warn}; - -use super::{ - get_direct_children_files_by_extensions, - helpers::{self, exif_media_data, ffmpeg_media_data, thumbnailer::THUMBNAIL_CACHE_DIR_NAME}, - tasks::{ - self, media_data_extractor, - thumbnailer::{self, NewThumbnailReporter}, - }, - NewThumbnailsReporter, BATCH_SIZE, -}; - -#[allow(clippy::missing_panics_doc)] // SAFETY: It doesn't actually panics -pub async fn shallow( - location: location::Data, - sub_path: impl AsRef + Send, - dispatcher: &BaseTaskDispatcher, - ctx: &impl OuterContext, -) -> Result, Error> { - let sub_path = sub_path.as_ref(); - - let location_path = maybe_missing(&location.path, "location.path") - .map(PathBuf::from) - .map(Arc::new) - .map_err(media_processor::Error::from)?; - - let location = Arc::new(location); - - let sub_iso_file_path = maybe_get_iso_file_path_from_sub_path::( - location.id, - Some(sub_path), - &*location_path, - ctx.db(), - ) - .await? - .map_or_else( - || { - IsolatedFilePathData::new(location.id, &*location_path, &*location_path, true) - .map_err(media_processor::Error::from) - }, - Ok, - )?; - - let mut errors = vec![]; - - let media_data_extraction_tasks = dispatch_media_data_extractor_tasks( - ctx.db(), - ctx.sync(), - &sub_iso_file_path, - &location_path, - dispatcher, - ) - .await?; - - let total_media_data_extraction_tasks = media_data_extraction_tasks.len(); - - let thumbnailer_tasks = - dispatch_thumbnailer_tasks(&sub_iso_file_path, false, &location_path, dispatcher, ctx) - .await?; - - let total_thumbnailer_tasks = thumbnailer_tasks.len(); - - let mut futures = media_data_extraction_tasks - .into_iter() - .chain(thumbnailer_tasks.into_iter()) - .map(CancelTaskOnDrop::new) - .collect::>(); - - let mut completed_media_data_extraction_tasks = 0; - let mut completed_thumbnailer_tasks = 0; - - while let Some(res) = futures.next().await { - match res { - Ok(TaskStatus::Done((_, TaskOutput::Out(out)))) => { - if out.is::() { - let media_data_extractor::Output { - db_read_time, - filtering_time, - extraction_time, - db_write_time, - errors: new_errors, - .. - } = *out - .downcast::() - .expect("just checked"); - - errors.extend(new_errors); - - completed_media_data_extraction_tasks += 1; - - debug!( - "Media data extraction task ({completed_media_data_extraction_tasks}/\ - {total_media_data_extraction_tasks}) completed in {:?};", - db_read_time + filtering_time + extraction_time + db_write_time - ); - } else if out.is::() { - let thumbnailer::Output { - total_time, - errors: new_errors, - .. - } = *out.downcast::().expect("just checked"); - - errors.extend(new_errors); - - completed_thumbnailer_tasks += 1; - - debug!( - "Thumbnailer task ({completed_thumbnailer_tasks}/{total_thumbnailer_tasks}) \ - completed in {total_time:?};", - ); - } else { - unreachable!( - "Task returned unexpected output type on media processor shallow job" - ); - } - } - Ok(TaskStatus::Done((_, TaskOutput::Empty))) => { - warn!("Task returned empty output on media processor shallow job"); - } - Ok(TaskStatus::Canceled | TaskStatus::ForcedAbortion | TaskStatus::Shutdown(_)) => { - return Ok(errors); - } - Ok(TaskStatus::Error(e)) => return Err(e), - - Err(e) => return Err(e.into()), - } - } - - Ok(errors) -} - -async fn dispatch_media_data_extractor_tasks( - db: &Arc, - sync: &SyncManager, - parent_iso_file_path: &IsolatedFilePathData<'_>, - location_path: &Arc, - dispatcher: &BaseTaskDispatcher, -) -> Result>, Error> { - let (extract_exif_file_paths, extract_ffmpeg_file_paths) = ( - get_direct_children_files_by_extensions( - parent_iso_file_path, - &exif_media_data::AVAILABLE_EXTENSIONS, - db, - ), - get_direct_children_files_by_extensions( - parent_iso_file_path, - &ffmpeg_media_data::AVAILABLE_EXTENSIONS, - db, - ), - ) - .try_join() - .await?; - - let tasks = extract_exif_file_paths - .into_iter() - .chunks(BATCH_SIZE) - .into_iter() - .map(Iterator::collect::>) - .map(|chunked_file_paths| { - tasks::MediaDataExtractor::new_exif( - &chunked_file_paths, - parent_iso_file_path.location_id(), - Arc::clone(location_path), - Arc::clone(db), - sync.clone(), - ) - }) - .map(IntoTask::into_task) - .chain( - extract_ffmpeg_file_paths - .into_iter() - .chunks(BATCH_SIZE) - .into_iter() - .map(Iterator::collect::>) - .map(|chunked_file_paths| { - tasks::MediaDataExtractor::new_ffmpeg( - &chunked_file_paths, - parent_iso_file_path.location_id(), - Arc::clone(location_path), - Arc::clone(db), - sync.clone(), - ) - }) - .map(IntoTask::into_task), - ) - .collect::>(); - - dispatcher.dispatch_many_boxed(tasks).await.map_or_else( - |_| { - debug!("Task system is shutting down while a shallow media processor was in progress"); - Ok(vec![]) - }, - Ok, - ) -} - -async fn dispatch_thumbnailer_tasks( - parent_iso_file_path: &IsolatedFilePathData<'_>, - should_regenerate: bool, - location_path: &Path, - dispatcher: &BaseTaskDispatcher, - ctx: &impl OuterContext, -) -> Result>, Error> { - let thumbnails_directory_path = - Arc::new(ctx.get_data_directory().join(THUMBNAIL_CACHE_DIR_NAME)); - let location_id = parent_iso_file_path.location_id(); - let library_id = ctx.id(); - let db = ctx.db(); - let reporter: Arc = - Arc::new(NewThumbnailsReporter { ctx: ctx.clone() }); - - let file_paths = get_direct_children_files_by_extensions( - parent_iso_file_path, - &helpers::thumbnailer::ALL_THUMBNAILABLE_EXTENSIONS, - db, - ) - .await?; - - let thumbs_count = file_paths.len() as u64; - - let tasks = file_paths - .into_iter() - .chunks(BATCH_SIZE) - .into_iter() - .map(|chunk| { - tasks::Thumbnailer::new_indexed( - Arc::clone(&thumbnails_directory_path), - &chunk.collect::>(), - (location_id, location_path), - library_id, - should_regenerate, - true, - Arc::clone(&reporter), - ) - }) - .map(IntoTask::into_task) - .collect::>(); - - debug!(%thumbs_count, priority_tasks_count = tasks.len(), "Dispatching thumbnails to be processed;"); - - dispatcher.dispatch_many_boxed(tasks).await.map_or_else( - |_| { - debug!("Task system is shutting down while a shallow media processor was in progress"); - Ok(vec![]) - }, - Ok, - ) -} diff --git a/core/crates/heavy-lifting/src/media_processor/tasks/media_data_extractor.rs b/core/crates/heavy-lifting/src/media_processor/tasks/media_data_extractor.rs deleted file mode 100644 index 3d72845e1..000000000 --- a/core/crates/heavy-lifting/src/media_processor/tasks/media_data_extractor.rs +++ /dev/null @@ -1,604 +0,0 @@ -use crate::{ - media_processor::{ - self, - helpers::{exif_media_data, ffmpeg_media_data}, - }, - Error, -}; - -use sd_core_file_path_helper::IsolatedFilePathData; -use sd_core_prisma_helpers::{file_path_for_media_processor, ObjectPubId}; -use sd_core_sync::SyncManager; - -use sd_media_metadata::{ExifMetadata, FFmpegMetadata}; -use sd_prisma::prisma::{exif_data, ffmpeg_data, file_path, location, object, PrismaClient}; -use sd_task_system::{ - check_interruption, ExecStatus, Interrupter, InterruptionKind, IntoAnyTaskOutput, - SerializableTask, Task, TaskId, -}; - -use std::{ - collections::{HashMap, HashSet}, - future::{Future, IntoFuture}, - mem, - path::{Path, PathBuf}, - pin::pin, - sync::Arc, - time::Duration, -}; - -use futures::{stream::FuturesUnordered, FutureExt, StreamExt}; -use futures_concurrency::future::Race; -use serde::{Deserialize, Serialize}; -use specta::Type; -use tokio::time::Instant; -use tracing::{debug, instrument, trace, Level}; - -#[derive(thiserror::Error, Debug, Serialize, Deserialize, Type, Clone)] -pub enum NonCriticalMediaDataExtractorError { - #[error("failed to extract media data from : {1}", path = .0.display())] - FailedToExtractImageMediaData(PathBuf, String), - #[error("file path missing object id: ")] - FilePathMissingObjectId(file_path::id::Type), - #[error("failed to construct isolated file path data: : {1}")] - FailedToConstructIsolatedFilePathData(file_path::id::Type, String), -} - -#[derive(Debug, Serialize, Deserialize, Clone, Copy, PartialEq, Eq)] -enum Kind { - Exif, - FFmpeg, -} - -#[derive(Debug)] -pub struct MediaDataExtractor { - // Task control - id: TaskId, - kind: Kind, - - // Received input args - file_paths: Vec, - location_id: location::id::Type, - location_path: Arc, - - // Inner state - stage: Stage, - - // Out collector - output: Output, - - // Dependencies - db: Arc, - sync: SyncManager, -} - -#[derive(Debug, Serialize, Deserialize)] -enum Stage { - Starting, - FetchedObjectsAlreadyWithMediaData(Vec), - ExtractingMediaData { - paths_by_id: HashMap, - exif_media_datas: Vec<(ExifMetadata, object::id::Type, ObjectPubId)>, - ffmpeg_media_datas: Vec<(FFmpegMetadata, object::id::Type)>, - extract_ids_to_remove_from_map: Vec, - }, - SaveMediaData { - exif_media_datas: Vec<(ExifMetadata, object::id::Type, ObjectPubId)>, - ffmpeg_media_datas: Vec<(FFmpegMetadata, object::id::Type)>, - }, -} - -/// [`MediaDataExtractor`] task output -#[derive(Serialize, Deserialize, Default, Debug)] -pub struct Output { - /// How many files were successfully processed - pub extracted: u64, - /// How many files were skipped - pub skipped: u64, - /// Time spent reading data from database - pub db_read_time: Duration, - /// Time spent filtering files to extract media data and files to skip - pub filtering_time: Duration, - /// Time spent extracting media data - pub extraction_time: Duration, - /// Time spent writing media data to database - pub db_write_time: Duration, - /// Errors encountered during the task - pub errors: Vec, -} - -#[async_trait::async_trait] -impl Task for MediaDataExtractor { - fn id(&self) -> TaskId { - self.id - } - - /// MediaDataExtractor never needs priority, as the data it generates are only accessed through - /// the media inspector, so it isn't latency sensitive like other tasks, like FileIdentifier or - /// the Thumbnailer - fn with_priority(&self) -> bool { - false - } - - #[instrument( - skip_all, - fields( - task_id = %self.id, - kind = ?self.kind, - location_id = %self.location_id, - location_path = %self.location_path.display(), - file_paths_count = %self.file_paths.len(), - ), - ret(level = Level::TRACE), - err, - )] - #[allow(clippy::blocks_in_conditions)] // Due to `err` on `instrument` macro above - #[allow(clippy::too_many_lines)] - async fn run(&mut self, interrupter: &Interrupter) -> Result { - loop { - match &mut self.stage { - Stage::Starting => { - let db_read_start = Instant::now(); - let object_ids = fetch_objects_already_with_media_data( - self.kind, - &self.file_paths, - &self.db, - ) - .await?; - self.output.db_read_time = db_read_start.elapsed(); - trace!( - object_ids_count = object_ids.len(), - "Fetched objects already with media data;", - ); - - self.stage = Stage::FetchedObjectsAlreadyWithMediaData(object_ids); - } - - Stage::FetchedObjectsAlreadyWithMediaData(objects_already_with_media_data) => { - if self.file_paths.len() == objects_already_with_media_data.len() { - self.output.skipped = self.file_paths.len() as u64; // Files already have media data, skipping - debug!("Skipped all files as they already have media data"); - break; - } - - let filtering_start = Instant::now(); - let paths_by_id = filter_files_to_extract_media_data( - mem::take(objects_already_with_media_data), - self.location_id, - &self.location_path, - &mut self.file_paths, - &mut self.output, - ); - self.output.filtering_time = filtering_start.elapsed(); - - trace!( - paths_needing_media_data_extraction_count = paths_by_id.len(), - "Filtered files to extract media data;", - ); - - self.stage = Stage::ExtractingMediaData { - extract_ids_to_remove_from_map: Vec::with_capacity(paths_by_id.len()), - exif_media_datas: if self.kind == Kind::Exif { - Vec::with_capacity(paths_by_id.len()) - } else { - Vec::new() - }, - ffmpeg_media_datas: if self.kind == Kind::FFmpeg { - Vec::with_capacity(paths_by_id.len()) - } else { - Vec::new() - }, - paths_by_id, - }; - } - - Stage::ExtractingMediaData { - paths_by_id, - exif_media_datas, - ffmpeg_media_datas, - extract_ids_to_remove_from_map, - } => { - { - // This inner scope is necessary to appease the mighty borrowck - let extraction_start = Instant::now(); - for id in extract_ids_to_remove_from_map.drain(..) { - paths_by_id.remove(&id); - } - - let mut futures = pin!(prepare_extraction_futures( - self.kind, - paths_by_id, - interrupter - )); - - while let Some(race_output) = futures.next().await { - match race_output { - InterruptRace::Processed(out) => { - process_output( - out, - exif_media_datas, - ffmpeg_media_datas, - extract_ids_to_remove_from_map, - &mut self.output, - ); - } - - InterruptRace::Interrupted(kind) => { - self.output.extraction_time += extraction_start.elapsed(); - return Ok(match kind { - InterruptionKind::Pause => ExecStatus::Paused, - InterruptionKind::Cancel => ExecStatus::Canceled, - }); - } - } - } - } - - self.stage = Stage::SaveMediaData { - exif_media_datas: mem::take(exif_media_datas), - ffmpeg_media_datas: mem::take(ffmpeg_media_datas), - }; - } - - Stage::SaveMediaData { - exif_media_datas, - ffmpeg_media_datas, - } => { - let db_write_start = Instant::now(); - self.output.extracted = save( - self.kind, - exif_media_datas, - ffmpeg_media_datas, - &self.db, - &self.sync, - ) - .await?; - self.output.db_write_time = db_write_start.elapsed(); - - self.output.skipped += self.output.errors.len() as u64; - - break; - } - } - - check_interruption!(interrupter); - } - - Ok(ExecStatus::Done(mem::take(&mut self.output).into_output())) - } -} - -impl MediaDataExtractor { - fn new( - kind: Kind, - file_paths: &[file_path_for_media_processor::Data], - location_id: location::id::Type, - location_path: Arc, - db: Arc, - sync: SyncManager, - ) -> Self { - let mut output = Output::default(); - - Self { - id: TaskId::new_v4(), - kind, - file_paths: file_paths - .iter() - .filter(|file_path| { - if file_path.object.is_some() { - true - } else { - output.errors.push( - media_processor::NonCriticalMediaProcessorError::from( - NonCriticalMediaDataExtractorError::FilePathMissingObjectId( - file_path.id, - ), - ) - .into(), - ); - false - } - }) - .cloned() - .collect(), - location_id, - location_path, - stage: Stage::Starting, - db, - sync, - output, - } - } - - #[must_use] - pub fn new_exif( - file_paths: &[file_path_for_media_processor::Data], - location_id: location::id::Type, - location_path: Arc, - db: Arc, - sync: SyncManager, - ) -> Self { - Self::new(Kind::Exif, file_paths, location_id, location_path, db, sync) - } - - #[must_use] - pub fn new_ffmpeg( - file_paths: &[file_path_for_media_processor::Data], - location_id: location::id::Type, - location_path: Arc, - db: Arc, - sync: SyncManager, - ) -> Self { - Self::new( - Kind::FFmpeg, - file_paths, - location_id, - location_path, - db, - sync, - ) - } -} - -#[inline] -async fn fetch_objects_already_with_media_data( - kind: Kind, - file_paths: &[file_path_for_media_processor::Data], - db: &PrismaClient, -) -> Result, media_processor::Error> { - let object_ids = file_paths - .iter() - .filter_map(|file_path| file_path.object.as_ref().map(|object| object.id)) - .collect(); - - match kind { - Kind::Exif => db - .exif_data() - .find_many(vec![exif_data::object_id::in_vec(object_ids)]) - .select(exif_data::select!({ object_id })) - .exec() - .await - .map(|object_ids| object_ids.into_iter().map(|data| data.object_id).collect()) - .map_err(Into::into), - - Kind::FFmpeg => db - .ffmpeg_data() - .find_many(vec![ffmpeg_data::object_id::in_vec(object_ids)]) - .select(ffmpeg_data::select!({ object_id })) - .exec() - .await - .map(|object_ids| object_ids.into_iter().map(|data| data.object_id).collect()) - .map_err(Into::into), - } -} - -#[inline] -fn filter_files_to_extract_media_data( - objects_already_with_media_data: Vec, - location_id: location::id::Type, - location_path: &Path, - file_paths: &mut Vec, - Output { - skipped, errors, .. - }: &mut Output, -) -> HashMap { - let unique_objects_already_with_media_data = objects_already_with_media_data - .into_iter() - .collect::>(); - - *skipped = unique_objects_already_with_media_data.len() as u64; - - file_paths.retain(|file_path| { - !unique_objects_already_with_media_data - .contains(&file_path.object.as_ref().expect("already checked").id) - }); - - file_paths - .iter() - .filter_map(|file_path| { - IsolatedFilePathData::try_from((location_id, file_path)) - .map_err(|e| { - errors.push( - media_processor::NonCriticalMediaProcessorError::from( - NonCriticalMediaDataExtractorError::FailedToConstructIsolatedFilePathData( - file_path.id, - e.to_string(), - ), - ) - .into(), - ); - }) - .map(|iso_file_path| { - let object = file_path.object.as_ref().expect("already checked"); - - ( - file_path.id, - ( - location_path.join(iso_file_path), - object.id, - object.pub_id.as_slice().into(), - ), - ) - }) - .ok() - }) - .collect() -} - -enum ExtractionOutputKind { - Exif(Result, media_processor::NonCriticalMediaProcessorError>), - FFmpeg(Result), -} - -struct ExtractionOutput { - file_path_id: file_path::id::Type, - object_id: object::id::Type, - object_pub_id: ObjectPubId, - kind: ExtractionOutputKind, -} - -#[allow(clippy::large_enum_variant)] -/* - * NOTE(fogodev): Interrupts will be pretty rare, so paying the boxing price for - * the Processed variant isn't worth it to avoid the enum size disparity between variants - */ -enum InterruptRace { - Interrupted(InterruptionKind), - Processed(ExtractionOutput), -} - -#[inline] -fn prepare_extraction_futures<'a>( - kind: Kind, - paths_by_id: &'a HashMap, - interrupter: &'a Interrupter, -) -> FuturesUnordered + 'a> { - paths_by_id - .iter() - .map( - |(file_path_id, (path, object_id, object_pub_id))| async move { - InterruptRace::Processed(ExtractionOutput { - file_path_id: *file_path_id, - object_id: *object_id, - object_pub_id: object_pub_id.clone(), - kind: match kind { - Kind::Exif => { - ExtractionOutputKind::Exif(exif_media_data::extract(path).await) - } - Kind::FFmpeg => { - ExtractionOutputKind::FFmpeg(ffmpeg_media_data::extract(path).await) - } - }, - }) - }, - ) - .map(|fut| { - ( - fut, - interrupter.into_future().map(InterruptRace::Interrupted), - ) - .race() - }) - .collect::>() -} - -#[instrument(skip_all, fields(%file_path_id, %object_id))] -#[inline] -fn process_output( - ExtractionOutput { - file_path_id, - object_id, - object_pub_id, - kind, - }: ExtractionOutput, - exif_media_datas: &mut Vec<(ExifMetadata, object::id::Type, ObjectPubId)>, - ffmpeg_media_datas: &mut Vec<(FFmpegMetadata, object::id::Type)>, - extract_ids_to_remove_from_map: &mut Vec, - output: &mut Output, -) { - trace!("Processing extracted media data"); - - match kind { - ExtractionOutputKind::Exif(Ok(Some(exif_data))) => { - exif_media_datas.push((exif_data, object_id, object_pub_id)); - } - ExtractionOutputKind::Exif(Ok(None)) => { - // No exif media data found - output.skipped += 1; - } - ExtractionOutputKind::FFmpeg(Ok(ffmpeg_data)) => { - ffmpeg_media_datas.push((ffmpeg_data, object_id)); - } - ExtractionOutputKind::Exif(Err(e)) | ExtractionOutputKind::FFmpeg(Err(e)) => { - output.errors.push(e.into()); - } - } - - extract_ids_to_remove_from_map.push(file_path_id); -} - -#[inline] -async fn save( - kind: Kind, - exif_media_datas: &mut Vec<(ExifMetadata, object::id::Type, ObjectPubId)>, - ffmpeg_media_datas: &mut Vec<(FFmpegMetadata, object::id::Type)>, - db: &PrismaClient, - sync: &SyncManager, -) -> Result { - trace!("Saving media data on database"); - - match kind { - Kind::Exif => exif_media_data::save(mem::take(exif_media_datas), db, sync).await, - Kind::FFmpeg => ffmpeg_media_data::save(mem::take(ffmpeg_media_datas), db).await, - } - .map_err(Into::into) -} - -#[derive(Debug, Serialize, Deserialize)] -struct SaveState { - id: TaskId, - kind: Kind, - file_paths: Vec, - location_id: location::id::Type, - location_path: Arc, - stage: Stage, - output: Output, -} - -impl SerializableTask for MediaDataExtractor { - type SerializeError = rmp_serde::encode::Error; - - type DeserializeError = rmp_serde::decode::Error; - - type DeserializeCtx = (Arc, SyncManager); - - async fn serialize(self) -> Result, Self::SerializeError> { - let Self { - id, - kind, - file_paths, - location_id, - location_path, - stage, - output, - .. - } = self; - - rmp_serde::to_vec_named(&SaveState { - id, - kind, - file_paths, - location_id, - location_path, - stage, - output, - }) - } - - async fn deserialize( - data: &[u8], - (db, sync): Self::DeserializeCtx, - ) -> Result { - rmp_serde::from_slice(data).map( - |SaveState { - id, - kind, - file_paths, - location_id, - location_path, - stage, - output, - }| Self { - id, - kind, - file_paths, - location_id, - location_path, - stage, - output, - db, - sync, - }, - ) - } -} diff --git a/core/crates/heavy-lifting/src/media_processor/tasks/mod.rs b/core/crates/heavy-lifting/src/media_processor/tasks/mod.rs deleted file mode 100644 index cb88d09d0..000000000 --- a/core/crates/heavy-lifting/src/media_processor/tasks/mod.rs +++ /dev/null @@ -1,5 +0,0 @@ -pub mod media_data_extractor; -pub mod thumbnailer; - -pub use media_data_extractor::MediaDataExtractor; -pub use thumbnailer::Thumbnailer; diff --git a/core/crates/heavy-lifting/src/media_processor/tasks/thumbnailer.rs b/core/crates/heavy-lifting/src/media_processor/tasks/thumbnailer.rs deleted file mode 100644 index 79687f8f6..000000000 --- a/core/crates/heavy-lifting/src/media_processor/tasks/thumbnailer.rs +++ /dev/null @@ -1,488 +0,0 @@ -//! Thumbnails directory have the following structure: -//! thumbnails/ -//! ├── version.txt -//! ├── ephemeral/ # ephemeral ones have it's own directory -//! │ └── <`cas_id`>[0..3]/ # sharding -//! │ └── <`cas_id`>.webp -//! └── <`library_id`>/ # we segregate thumbnails by library -//! └── <`cas_id`>[0..3]/ # sharding -//! └── <`cas_id`>.webp - -use crate::{ - media_processor::{ - self, - helpers::thumbnailer::{ - generate_thumbnail, GenerateThumbnailArgs, GenerationStatus, THUMBNAILER_TASK_TIMEOUT, - }, - ThumbKey, ThumbnailKind, - }, - Error, -}; - -use sd_core_file_path_helper::IsolatedFilePathData; -use sd_core_prisma_helpers::{file_path_for_media_processor, CasId}; - -use sd_prisma::prisma::{file_path, location}; -use sd_task_system::{ - ExecStatus, Interrupter, InterruptionKind, IntoAnyTaskOutput, SerializableTask, Task, TaskId, -}; - -use std::{ - collections::HashMap, - fmt, - future::IntoFuture, - mem, - path::{Path, PathBuf}, - pin::pin, - sync::Arc, - time::Duration, -}; - -use futures::{stream::FuturesUnordered, FutureExt, StreamExt}; -use futures_concurrency::future::Race; -use serde::{Deserialize, Serialize}; -use specta::Type; -use tokio::time::Instant; -use tracing::{error, instrument, trace, Level}; -use uuid::Uuid; - -pub type ThumbnailId = u32; - -pub trait NewThumbnailReporter: Send + Sync + fmt::Debug + 'static { - fn new_thumbnail(&self, thumb_key: ThumbKey); -} - -#[derive(Debug)] -pub struct Thumbnailer { - // Task control - id: TaskId, - with_priority: bool, - - // Received input args - thumbs_kind: ThumbnailKind, - thumbnails_directory_path: Arc, - thumbnails_to_generate: HashMap>, - should_regenerate: bool, - - // Inner state - already_processed_ids: Vec, - - // Out collector - output: Output, - - // Dependencies - reporter: Arc, -} - -#[async_trait::async_trait] -impl Task for Thumbnailer { - fn id(&self) -> TaskId { - self.id - } - - fn with_priority(&self) -> bool { - self.with_priority - } - - fn with_timeout(&self) -> Option { - Some(THUMBNAILER_TASK_TIMEOUT) // The entire task must not take more than this constant - } - - #[instrument( - skip_all, - fields( - task_id = %self.id, - thumbs_kind = ?self.thumbs_kind, - should_regenerate = self.should_regenerate, - thumbnails_to_generate_count = self.thumbnails_to_generate.len(), - already_processed_ids_count = self.already_processed_ids.len(), - with_priority = self.with_priority, - ), - ret(level = Level::TRACE), - err, - )] - #[allow(clippy::blocks_in_conditions)] // Due to `err` on `instrument` macro above - async fn run(&mut self, interrupter: &Interrupter) -> Result { - enum InterruptRace { - Interrupted(InterruptionKind), - Processed(ThumbnailGenerationOutput), - } - - let Self { - thumbs_kind, - thumbnails_directory_path, - thumbnails_to_generate, - already_processed_ids, - should_regenerate, - with_priority, - reporter, - output, - .. - } = self; - - // Removing already processed thumbnails from a possible previous run - already_processed_ids.drain(..).for_each(|id| { - thumbnails_to_generate.remove(&id); - }); - - let start = Instant::now(); - - let futures = thumbnails_to_generate - .iter() - .map(|(id, generate_args)| { - generate_thumbnail( - thumbnails_directory_path, - generate_args, - thumbs_kind, - *should_regenerate, - ) - .map(|res| InterruptRace::Processed((*id, res))) - }) - .map(|fut| { - ( - fut, - interrupter.into_future().map(InterruptRace::Interrupted), - ) - .race() - }) - .collect::>(); - - let mut futures = pin!(futures); - - while let Some(race_output) = futures.next().await { - match race_output { - InterruptRace::Processed(out) => process_thumbnail_generation_output( - out, - *with_priority, - reporter.as_ref(), - already_processed_ids, - output, - ), - - InterruptRace::Interrupted(kind) => { - output.total_time += start.elapsed(); - return Ok(match kind { - InterruptionKind::Pause => ExecStatus::Paused, - InterruptionKind::Cancel => ExecStatus::Canceled, - }); - } - } - } - - output.total_time += start.elapsed(); - - if output.generated > 1 { - #[allow(clippy::cast_precision_loss)] - // SAFETY: we're probably won't have 2^52 thumbnails being generated on a single task for this cast to have - // a precision loss issue - let total = (output.generated + output.skipped) as f64; - let mean_generation_time_f64 = output.mean_time_acc / total; - - trace!( - generated = output.generated, - skipped = output.skipped, - "mean generation time: {mean_generation_time:?} ± {generation_time_std_dev:?};", - mean_generation_time = Duration::from_secs_f64(mean_generation_time_f64), - generation_time_std_dev = Duration::from_secs_f64( - (mean_generation_time_f64 - .mul_add(-mean_generation_time_f64, output.std_dev_acc / total)) - .sqrt(), - ) - ); - } - - Ok(ExecStatus::Done(mem::take(output).into_output())) - } -} - -#[derive(Serialize, Deserialize, Default, Debug)] -pub struct Output { - pub generated: u64, - pub skipped: u64, - pub errors: Vec, - pub total_time: Duration, - pub mean_time_acc: f64, - pub std_dev_acc: f64, -} - -#[derive(thiserror::Error, Debug, Serialize, Deserialize, Type, Clone)] -pub enum NonCriticalThumbnailerError { - #[error("file path has no cas_id")] - MissingCasId(file_path::id::Type), - #[error("failed to extract isolated file path data from file path : {1}")] - FailedToExtractIsolatedFilePathData(file_path::id::Type, String), - #[error("failed to generate video file thumbnail : {1}", path = .0.display())] - VideoThumbnailGenerationFailed(PathBuf, String), - #[error("failed to format image : {1}", path = .0.display())] - FormatImage(PathBuf, String), - #[error("failed to encode webp image : {1}", path = .0.display())] - WebPEncoding(PathBuf, String), - #[error("processing thread panicked while generating thumbnail from : {1}", path = .0.display())] - PanicWhileGeneratingThumbnail(PathBuf, String), - #[error("failed to create shard directory for thumbnail: {0}")] - CreateShardDirectory(String), - #[error("failed to save thumbnail : {1}", path = .0.display())] - SaveThumbnail(PathBuf, String), - #[error("task timed out: {0}")] - TaskTimeout(TaskId), -} - -impl Thumbnailer { - fn new( - thumbs_kind: ThumbnailKind, - thumbnails_directory_path: Arc, - thumbnails_to_generate: HashMap>, - errors: Vec, - should_regenerate: bool, - with_priority: bool, - reporter: Arc, - ) -> Self { - Self { - id: TaskId::new_v4(), - thumbs_kind, - thumbnails_directory_path, - already_processed_ids: Vec::with_capacity(thumbnails_to_generate.len()), - thumbnails_to_generate, - should_regenerate, - with_priority, - output: Output { - errors, - ..Default::default() - }, - reporter, - } - } - - #[must_use] - pub fn new_ephemeral( - thumbnails_directory_path: Arc, - thumbnails_to_generate: Vec>, - reporter: Arc, - ) -> Self { - Self::new( - ThumbnailKind::Ephemeral, - thumbnails_directory_path, - thumbnails_to_generate - .into_iter() - .enumerate() - .map(|(i, args)| { - #[allow(clippy::cast_possible_truncation)] - { - // SAFETY: it's fine, we will never process more than 4 billion thumbnails - // on a single task LMAO - (i as ThumbnailId, args) - } - }) - .collect(), - Vec::new(), - false, - true, - reporter, - ) - } - - #[must_use] - pub fn new_indexed( - thumbnails_directory_path: Arc, - file_paths: &[file_path_for_media_processor::Data], - (location_id, location_path): (location::id::Type, &Path), - library_id: Uuid, - should_regenerate: bool, - with_priority: bool, - reporter: Arc, - ) -> Self { - let mut errors = Vec::new(); - - Self::new( - ThumbnailKind::Indexed(library_id), - thumbnails_directory_path, - file_paths - .iter() - .filter_map(|file_path| { - if let Some(cas_id) = file_path - .cas_id - .as_ref() - .map(CasId::from) - .map(CasId::into_owned) - { - let file_path_id = file_path.id; - IsolatedFilePathData::try_from((location_id, file_path)) - .map_err(|e| { - errors.push( - media_processor::NonCriticalMediaProcessorError::from( - NonCriticalThumbnailerError::FailedToExtractIsolatedFilePathData( - file_path_id, - e.to_string(), - ), - ) - .into(), - ); - }) - .ok() - .map(|iso_file_path| (file_path_id, cas_id, iso_file_path)) - } else { - errors.push( - media_processor::NonCriticalMediaProcessorError::from( - NonCriticalThumbnailerError::MissingCasId(file_path.id), - ) - .into(), - ); - None - } - }) - .map(|(file_path_id, cas_id, iso_file_path)| { - let full_path = location_path.join(&iso_file_path); - - #[allow(clippy::cast_sign_loss)] - { - ( - // SAFETY: db doesn't have negative indexes - file_path_id as u32, - GenerateThumbnailArgs::new( - iso_file_path.extension().to_string(), - cas_id, - full_path, - ), - ) - } - }) - .collect::>(), - errors, - should_regenerate, - with_priority, - reporter, - ) - } -} - -#[instrument(skip_all, fields(thumb_id = id, %generated, %skipped, ?elapsed_time, ?res))] -fn process_thumbnail_generation_output( - (id, (elapsed_time, res)): ThumbnailGenerationOutput, - with_priority: bool, - reporter: &dyn NewThumbnailReporter, - already_processed_ids: &mut Vec, - Output { - generated, - skipped, - errors, - mean_time_acc: mean_generation_time_accumulator, - std_dev_acc: std_dev_accumulator, - .. - }: &mut Output, -) { - let elapsed_time = elapsed_time.as_secs_f64(); - *mean_generation_time_accumulator += elapsed_time; - *std_dev_accumulator += elapsed_time * elapsed_time; - - match res { - Ok((thumb_key, status)) => { - match status { - GenerationStatus::Generated => { - *generated += 1; - // This if is REALLY needed, due to the sheer performance of the thumbnailer, - // I restricted to only send events notifying for thumbnails in the current - // opened directory, sending events for the entire location turns into a - // humongous bottleneck in the frontend lol, since it doesn't even knows - // what to do with thumbnails for inner directories lol - // - fogodev - if with_priority { - reporter.new_thumbnail(thumb_key); - } - } - GenerationStatus::Skipped => { - *skipped += 1; - } - } - } - Err(e) => { - errors.push(media_processor::NonCriticalMediaProcessorError::from(e).into()); - *skipped += 1; - } - } - - already_processed_ids.push(id); - - trace!("Thumbnail processed"); -} - -#[derive(Debug, Serialize, Deserialize)] -struct SaveState { - id: TaskId, - thumbs_kind: ThumbnailKind, - thumbnails_directory_path: Arc, - thumbnails_to_generate: HashMap>, - should_regenerate: bool, - with_priority: bool, - output: Output, -} - -impl SerializableTask for Thumbnailer { - type SerializeError = rmp_serde::encode::Error; - - type DeserializeError = rmp_serde::decode::Error; - - type DeserializeCtx = Arc; - - async fn serialize(self) -> Result, Self::SerializeError> { - let Self { - id, - thumbs_kind, - thumbnails_directory_path, - mut thumbnails_to_generate, - already_processed_ids, - should_regenerate, - with_priority, - output, - .. - } = self; - - for id in already_processed_ids { - thumbnails_to_generate.remove(&id); - } - - rmp_serde::to_vec_named(&SaveState { - id, - thumbs_kind, - thumbnails_directory_path, - thumbnails_to_generate, - should_regenerate, - with_priority, - output, - }) - } - - async fn deserialize( - data: &[u8], - reporter: Self::DeserializeCtx, - ) -> Result { - rmp_serde::from_slice(data).map( - |SaveState { - id, - thumbs_kind, - thumbnails_to_generate, - thumbnails_directory_path, - should_regenerate, - with_priority, - output, - }| Self { - id, - reporter, - thumbs_kind, - thumbnails_to_generate, - thumbnails_directory_path, - already_processed_ids: Vec::new(), - should_regenerate, - with_priority, - output, - }, - ) - } -} - -type ThumbnailGenerationOutput = ( - ThumbnailId, - ( - Duration, - Result<(ThumbKey, GenerationStatus), NonCriticalThumbnailerError>, - ), -); diff --git a/core/crates/heavy-lifting/src/utils/mod.rs b/core/crates/heavy-lifting/src/utils/mod.rs deleted file mode 100644 index 538257e2c..000000000 --- a/core/crates/heavy-lifting/src/utils/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub mod sub_path; diff --git a/core/crates/heavy-lifting/src/utils/sub_path.rs b/core/crates/heavy-lifting/src/utils/sub_path.rs deleted file mode 100644 index ea188e133..000000000 --- a/core/crates/heavy-lifting/src/utils/sub_path.rs +++ /dev/null @@ -1,113 +0,0 @@ -use sd_core_file_path_helper::{ - ensure_file_path_exists, ensure_sub_path_is_directory, ensure_sub_path_is_in_location, - FilePathError, IsolatedFilePathData, -}; - -use sd_prisma::prisma::{location, PrismaClient}; - -use std::path::{Path, PathBuf}; - -use prisma_client_rust::QueryError; -use rspc::ErrorCode; - -#[derive(thiserror::Error, Debug)] -pub enum Error { - #[error("received sub path not in database: ", .0.display())] - SubPathNotFound(Box), - - #[error("database error: {0}")] - Database(#[from] QueryError), - - #[error(transparent)] - IsoFilePath(#[from] FilePathError), -} - -impl From for rspc::Error { - fn from(e: Error) -> Self { - match e { - Error::SubPathNotFound(_) => Self::with_cause(ErrorCode::NotFound, e.to_string(), e), - - _ => Self::with_cause(ErrorCode::InternalServerError, e.to_string(), e), - } - } -} - -pub async fn get_full_path_from_sub_path>( - location_id: location::id::Type, - sub_path: Option + Send + Sync>, - location_path: impl AsRef + Send, - db: &PrismaClient, -) -> Result { - async fn inner( - location_id: location::id::Type, - sub_path: Option<&Path>, - location_path: &Path, - db: &PrismaClient, - ) -> Result { - match sub_path { - Some(sub_path) if sub_path != Path::new("") => { - let full_path = ensure_sub_path_is_in_location(location_path, sub_path).await?; - - ensure_sub_path_is_directory(location_path, sub_path).await?; - - ensure_file_path_exists( - sub_path, - &IsolatedFilePathData::new(location_id, location_path, &full_path, true)?, - db, - Error::SubPathNotFound, - ) - .await?; - - Ok(full_path) - } - _ => Ok(location_path.to_path_buf()), - } - } - - inner( - location_id, - sub_path.as_ref().map(AsRef::as_ref), - location_path.as_ref(), - db, - ) - .await - .map_err(E::from) -} - -pub async fn maybe_get_iso_file_path_from_sub_path>( - location_id: location::id::Type, - sub_path: Option + Send + Sync>, - location_path: impl AsRef + Send, - db: &PrismaClient, -) -> Result>, E> { - async fn inner( - location_id: location::id::Type, - sub_path: Option<&Path>, - location_path: &Path, - db: &PrismaClient, - ) -> Result>, Error> { - match sub_path { - Some(sub_path) if sub_path != Path::new("") => { - let full_path = ensure_sub_path_is_in_location(location_path, sub_path).await?; - ensure_sub_path_is_directory(location_path, sub_path).await?; - - let sub_iso_file_path = - IsolatedFilePathData::new(location_id, location_path, &full_path, true)?; - - ensure_file_path_exists(sub_path, &sub_iso_file_path, db, Error::SubPathNotFound) - .await - .map(|()| Some(sub_iso_file_path)) - } - _ => Ok(None), - } - } - - inner( - location_id, - sub_path.as_ref().map(AsRef::as_ref), - location_path.as_ref(), - db, - ) - .await - .map_err(E::from) -} diff --git a/core/crates/indexer-rules/Cargo.toml b/core/crates/indexer-rules/Cargo.toml deleted file mode 100644 index 472bd2442..000000000 --- a/core/crates/indexer-rules/Cargo.toml +++ /dev/null @@ -1,33 +0,0 @@ -[package] -name = "sd-core-indexer-rules" -version = "0.1.0" - -authors = ["Ericson Soares ", "Vítor Vasconcellos "] -edition.workspace = true -license.workspace = true -repository.workspace = true - -[dependencies] -# Spacedrive Sub-crates -sd-prisma = { path = "../../../crates/prisma" } -sd-utils = { path = "../../../crates/utils" } - -# Workspace dependencies -chrono = { workspace = true } -futures-concurrency = { workspace = true } -globset = { workspace = true, features = ["serde1"] } -prisma-client-rust = { workspace = true } -rmp-serde = { workspace = true } -rspc = { workspace = true } -serde = { workspace = true, features = ["derive", "rc"] } -specta = { workspace = true } -thiserror = { workspace = true } -tokio = { workspace = true, features = ["fs"] } -tracing = { workspace = true } -uuid = { workspace = true, features = ["serde", "v4"] } - -# Specific Indexer Rules dependencies -gix-ignore = { version = "0.11.2", features = ["serde"] } - -[dev-dependencies] -tempfile = { workspace = true } diff --git a/core/crates/indexer-rules/src/lib.rs b/core/crates/indexer-rules/src/lib.rs deleted file mode 100644 index f239e6a27..000000000 --- a/core/crates/indexer-rules/src/lib.rs +++ /dev/null @@ -1,947 +0,0 @@ -#![warn( - clippy::all, - clippy::pedantic, - clippy::correctness, - clippy::perf, - clippy::style, - clippy::suspicious, - clippy::complexity, - clippy::nursery, - clippy::unwrap_used, - unused_qualifications, - rust_2018_idioms, - trivial_casts, - trivial_numeric_casts, - unused_allocation, - clippy::unnecessary_cast, - clippy::cast_lossless, - clippy::cast_possible_truncation, - clippy::cast_possible_wrap, - clippy::cast_precision_loss, - clippy::cast_sign_loss, - clippy::dbg_macro, - clippy::deprecated_cfg_attr, - clippy::separated_literal_suffix, - deprecated -)] -#![forbid(deprecated_in_future)] -#![allow(clippy::missing_errors_doc)] - -use sd_prisma::prisma::{indexer_rule, PrismaClient}; -use sd_utils::{ - db::{maybe_missing, MissingFieldError}, - error::{FileIOError, NonUtf8PathError}, -}; -use seed::SystemIndexerRule; -use serde::{Deserialize, Serialize}; - -use std::{ - collections::{HashMap, HashSet}, - fs::Metadata, - path::{Path, PathBuf}, - sync::Arc, -}; - -use chrono::{DateTime, Utc}; -use futures_concurrency::future::TryJoin; -use gix_ignore::{glob::pattern::Case, Search}; -use globset::{Glob, GlobSet, GlobSetBuilder}; -use rmp_serde::{decode, encode}; -use rspc::ErrorCode; - -use specta::Type; -use thiserror::Error; -use tokio::fs; -use tracing::{debug, instrument, trace}; -use uuid::Uuid; - -pub mod seed; -mod serde_impl; - -#[derive(Error, Debug)] -pub enum Error { - // User errors - #[error("invalid indexer rule kind integer: {0}")] - InvalidRuleKindInt(i32), - #[error("glob builder error: {0}")] - Glob(#[from] globset::Error), - #[error(transparent)] - NonUtf8Path(#[from] NonUtf8PathError), - - // Internal Errors - #[error("indexer rule parameters encode error: {0}")] - RuleParametersRMPEncode(#[from] encode::Error), - #[error("indexer rule parameters decode error: {0}")] - RuleParametersRMPDecode(#[from] decode::Error), - #[error("accept by its children file I/O error: {0}")] - AcceptByItsChildrenFileIO(FileIOError), - #[error("reject by its children file I/O error: {0}")] - RejectByItsChildrenFileIO(FileIOError), - #[error("database error: {0}")] - Database(#[from] prisma_client_rust::QueryError), - #[error("missing-field: {0}")] - MissingField(#[from] MissingFieldError), -} - -impl From for rspc::Error { - fn from(e: Error) -> Self { - match e { - Error::InvalidRuleKindInt(_) | Error::Glob(_) | Error::NonUtf8Path(_) => { - Self::with_cause(ErrorCode::BadRequest, e.to_string(), e) - } - - _ => Self::with_cause(ErrorCode::InternalServerError, e.to_string(), e), - } - } -} - -/// `IndexerRuleCreateArgs` is the argument received from the client using rspc to create a new indexer rule. -/// Note that `rules` field is a vector of tuples of `RuleKind` and `parameters`. -/// -/// In case of `RuleKind::AcceptFilesByGlob` or `RuleKind::RejectFilesByGlob`, it will be a -/// vector of strings containing a glob patterns. -/// -/// In case of `RuleKind::AcceptIfChildrenDirectoriesArePresent` or `RuleKind::RejectIfChildrenDirectoriesArePresent` the -/// `parameters` field must be a vector of strings containing the names of the directories. -#[derive(Type, Deserialize)] -pub struct IndexerRuleCreateArgs { - pub name: String, - pub dry_run: bool, - pub rules: Vec<(RuleKind, Vec)>, -} - -impl IndexerRuleCreateArgs { - #[instrument(skip_all, fields(name = %self.name, rules = ?self.rules), err)] - pub async fn create(self, db: &PrismaClient) -> Result, Error> { - use indexer_rule::{date_created, date_modified, name, rules_per_kind}; - - debug!( - "{} a new indexer rule", - if self.dry_run { - "Dry run: Would create" - } else { - "Trying to create" - }, - ); - - let rules_data = rmp_serde::to_vec_named( - &self - .rules - .into_iter() - .map(|(kind, parameters)| match kind { - RuleKind::AcceptFilesByGlob => { - RulePerKind::new_accept_files_by_globs_str(parameters) - } - RuleKind::RejectFilesByGlob => { - RulePerKind::new_reject_files_by_globs_str(parameters) - } - RuleKind::AcceptIfChildrenDirectoriesArePresent => { - Ok(RulePerKind::AcceptIfChildrenDirectoriesArePresent( - parameters.into_iter().collect(), - )) - } - RuleKind::RejectIfChildrenDirectoriesArePresent => { - Ok(RulePerKind::RejectIfChildrenDirectoriesArePresent( - parameters.into_iter().collect(), - )) - } - RuleKind::IgnoredByGit => { - Ok(RulePerKind::IgnoredByGit(PathBuf::new(), Search::default())) - } - }) - .collect::, _>>()?, - )?; - - if self.dry_run { - return Ok(None); - } - - let date_created = Utc::now(); - - Ok(Some( - db.indexer_rule() - .create( - sd_utils::uuid_to_bytes(&generate_pub_id()), - vec![ - name::set(Some(self.name)), - rules_per_kind::set(Some(rules_data)), - date_created::set(Some(date_created.into())), - date_modified::set(Some(date_created.into())), - ], - ) - .exec() - .await?, - )) - } -} - -#[repr(i32)] -#[non_exhaustive] -#[derive(Debug, Clone, Copy, Serialize, Deserialize, Type, Eq, PartialEq, Hash)] -pub enum RuleKind { - AcceptFilesByGlob = 0, - RejectFilesByGlob = 1, - AcceptIfChildrenDirectoriesArePresent = 2, - RejectIfChildrenDirectoriesArePresent = 3, - IgnoredByGit = 4, -} - -impl RuleKind { - #[must_use] - pub const fn variant_count() -> usize { - // TODO: Use https://doc.rust-lang.org/std/mem/fn.variant_count.html if it ever gets stabilized - 5 - } -} - -/// `ParametersPerKind` is a mapping from `RuleKind` to the parameters required for each kind of rule. -/// In case of doubt about globs, consult -/// -/// We store directly globs in the database, serialized using [rmp_serde](https://docs.rs/rmp-serde). -/// -/// In case of `ParametersPerKind::AcceptIfChildrenDirectoriesArePresent` or -/// `ParametersPerKind::RejectIfChildrenDirectoriesArePresent` -/// first we change the data structure to a vector, then we serialize it. -#[derive(Debug, Clone)] -pub enum RulePerKind { - // TODO: Add an indexer rule that filter files based on their extended attributes - // https://learn.microsoft.com/en-us/windows/win32/fileio/file-attribute-constants - // https://en.wikipedia.org/wiki/Extended_file_attributes - AcceptFilesByGlob(Vec, GlobSet), - RejectFilesByGlob(Vec, GlobSet), - AcceptIfChildrenDirectoriesArePresent(HashSet), - RejectIfChildrenDirectoriesArePresent(HashSet), - IgnoredByGit(PathBuf, Search), -} - -impl RulePerKind { - fn new_files_by_globs_str_and_kind( - globs_str: impl IntoIterator>, - kind_fn: impl Fn(Vec, GlobSet) -> Self, - ) -> Result { - globs_str - .into_iter() - .map(|s| s.as_ref().parse::()) - .collect::, _>>() - .and_then(|globs| { - globs - .iter() - .cloned() - .fold(&mut GlobSetBuilder::new(), |builder, glob| { - builder.add(glob) - }) - .build() - .map(move |glob_set| kind_fn(globs, glob_set)) - .map_err(Into::into) - }) - .map_err(Into::into) - } - - pub fn new_accept_files_by_globs_str( - globs_str: impl IntoIterator>, - ) -> Result { - Self::new_files_by_globs_str_and_kind(globs_str, Self::AcceptFilesByGlob) - } - - pub fn new_reject_files_by_globs_str( - globs_str: impl IntoIterator>, - ) -> Result { - Self::new_files_by_globs_str_and_kind(globs_str, Self::RejectFilesByGlob) - } -} - -pub trait MetadataForIndexerRules: Send + Sync + 'static { - fn is_dir(&self) -> bool; -} - -impl MetadataForIndexerRules for Metadata { - fn is_dir(&self) -> bool { - self.is_dir() - } -} - -impl RulePerKind { - async fn apply( - &self, - source: impl AsRef + Send, - metadata: &impl MetadataForIndexerRules, - ) -> Result<(RuleKind, bool), Error> { - match self { - Self::AcceptIfChildrenDirectoriesArePresent(children) => { - accept_dir_for_its_children(source, metadata, children) - .await - .map(|accepted| (RuleKind::AcceptIfChildrenDirectoriesArePresent, accepted)) - } - Self::RejectIfChildrenDirectoriesArePresent(children) => { - reject_dir_for_its_children(source, metadata, children) - .await - .map(|rejected| (RuleKind::RejectIfChildrenDirectoriesArePresent, rejected)) - } - - Self::AcceptFilesByGlob(_globs, accept_glob_set) => Ok(( - RuleKind::AcceptFilesByGlob, - accept_by_glob(source, accept_glob_set), - )), - Self::RejectFilesByGlob(_globs, reject_glob_set) => Ok(( - RuleKind::RejectFilesByGlob, - reject_by_glob(source, reject_glob_set), - )), - Self::IgnoredByGit(base_dir, patterns) => Ok(( - RuleKind::IgnoredByGit, - accept_by_git_pattern(source, base_dir, patterns), - )), - } - } -} - -fn accept_by_git_pattern( - source: impl AsRef, - base_dir: impl AsRef, - search: &Search, -) -> bool { - fn inner(source: &Path, base_dir: &Path, search: &Search) -> bool { - let relative = source - .strip_prefix(base_dir) - .expect("`base_dir` should be our git repo, and `source` should be inside of it"); - - let Some(src) = relative.to_str().map(|s| s.as_bytes().into()) else { - return false; - }; - - search - .pattern_matching_relative_path(src, Some(source.is_dir()), Case::Fold) - .map_or(true, |rule| rule.pattern.is_negative()) - } - - inner(source.as_ref(), base_dir.as_ref(), search) -} - -#[derive(Debug, Serialize, Deserialize, Clone)] -pub struct IndexerRule { - pub id: Option, - pub name: String, - pub default: bool, - pub rules: Vec, - pub date_created: DateTime, - pub date_modified: DateTime, -} - -impl IndexerRule { - pub async fn apply( - &self, - source: impl AsRef + Send, - metadata: &impl MetadataForIndexerRules, - ) -> Result, Error> { - async fn inner( - rules: &[RulePerKind], - source: &Path, - metadata: &impl MetadataForIndexerRules, - ) -> Result, Error> { - rules - .iter() - .map(|rule| rule.apply(source, metadata)) - .collect::>() - .try_join() - .await - } - - inner(&self.rules, source.as_ref(), metadata).await - } -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum RulerDecision { - Accept, - Reject, -} - -#[derive(Debug, Default, Serialize, Deserialize)] -pub struct IndexerRuler { - base: Arc>, - extra: Vec, -} - -impl Clone for IndexerRuler { - fn clone(&self) -> Self { - Self { - base: Arc::clone(&self.base), - // Each instance of IndexerRules MUST have its own extra rules no clones allowed! - extra: Vec::new(), - } - } -} - -impl IndexerRuler { - #[must_use] - pub fn new(rules: Vec) -> Self { - Self { - base: Arc::new(rules), - extra: Vec::new(), - } - } - - pub async fn evaluate_path( - &self, - source: impl AsRef + Send, - metadata: &impl MetadataForIndexerRules, - ) -> Result { - async fn inner( - this: &IndexerRuler, - source: &Path, - metadata: &impl MetadataForIndexerRules, - ) -> Result { - Ok( - if IndexerRuler::reject_path( - source, - metadata.is_dir(), - &this.apply_all(source, metadata).await?, - ) { - RulerDecision::Reject - } else { - RulerDecision::Accept - }, - ) - } - - inner(self, source.as_ref(), metadata).await - } - - pub async fn apply_all( - &self, - source: impl AsRef + Send, - metadata: &impl MetadataForIndexerRules, - ) -> Result>, Error> { - async fn inner( - base: &[IndexerRule], - extra: &[IndexerRule], - source: &Path, - metadata: &impl MetadataForIndexerRules, - ) -> Result>, Error> { - base.iter() - .chain(extra.iter()) - .map(|rule| rule.apply(source, metadata)) - .collect::>() - .try_join() - .await - .map(|results| { - results.into_iter().flatten().fold( - HashMap::<_, Vec<_>>::with_capacity(RuleKind::variant_count()), - |mut map, (kind, result)| { - map.entry(kind).or_default().push(result); - map - }, - ) - }) - } - - inner(&self.base, &self.extra, source.as_ref(), metadata).await - } - - /// Extend the indexer rules with the contents from an iterator of rules - pub fn extend(&mut self, iter: impl IntoIterator + Send) { - self.extra.extend(iter); - } - - #[must_use] - pub fn has_system(&self, rule: &SystemIndexerRule) -> bool { - self.base - .iter() - .chain(self.extra.iter()) - .any(|inner_rule| rule == inner_rule) - } - - #[instrument(skip_all, fields(current_path = %current_path.display()))] - fn reject_path( - current_path: &Path, - is_dir: bool, - acceptance_per_rule_kind: &HashMap>, - ) -> bool { - Self::rejected_by_reject_glob(acceptance_per_rule_kind) - || Self::rejected_by_git_ignore(acceptance_per_rule_kind) - || (is_dir && Self::rejected_by_children_directories(acceptance_per_rule_kind)) - || Self::rejected_by_accept_glob(acceptance_per_rule_kind) - } - - pub fn rejected_by_accept_glob( - acceptance_per_rule_kind: &HashMap>, - ) -> bool { - let res = acceptance_per_rule_kind - .get(&RuleKind::AcceptFilesByGlob) - .map_or(false, |accept_rules| { - accept_rules.iter().all(|accept| !accept) - }); - - if res { - trace!("Reject because it didn't passed in any `RuleKind::AcceptFilesByGlob` rules"); - } - - res - } - - pub fn rejected_by_children_directories( - acceptance_per_rule_kind: &HashMap>, - ) -> bool { - let res = acceptance_per_rule_kind - .get(&RuleKind::RejectIfChildrenDirectoriesArePresent) - .map_or(false, |reject_results| { - reject_results.iter().any(|reject| !reject) - }); - - if res { - trace!("Rejected by rule `RuleKind::RejectIfChildrenDirectoriesArePresent`"); - } - - res - } - - pub fn rejected_by_reject_glob( - acceptance_per_rule_kind: &HashMap>, - ) -> bool { - let res = acceptance_per_rule_kind - .get(&RuleKind::RejectFilesByGlob) - .map_or(false, |reject_results| { - reject_results.iter().any(|reject| !reject) - }); - - if res { - trace!("Rejected by `RuleKind::RejectFilesByGlob`"); - } - - res - } - - pub fn rejected_by_git_ignore(acceptance_per_rule_kind: &HashMap>) -> bool { - let res = acceptance_per_rule_kind - .get(&RuleKind::IgnoredByGit) - .map_or(false, |reject_results| { - reject_results.iter().any(|reject| !reject) - }); - - if res { - trace!("Rejected by `RuleKind::IgnoredByGit`"); - } - - res - } -} - -impl TryFrom<&indexer_rule::Data> for IndexerRule { - type Error = Error; - - fn try_from(data: &indexer_rule::Data) -> Result { - Ok(Self { - id: Some(data.id), - name: maybe_missing(data.name.clone(), "indexer_rule.name")?, - default: data.default.unwrap_or_default(), - rules: rmp_serde::from_slice(maybe_missing( - &data.rules_per_kind, - "indexer_rule.rules_per_kind", - )?)?, - date_created: maybe_missing(data.date_created, "indexer_rule.date_created")?.into(), - date_modified: maybe_missing(data.date_modified, "indexer_rule.date_modified")?.into(), - }) - } -} - -impl TryFrom for IndexerRule { - type Error = Error; - - fn try_from(data: indexer_rule::Data) -> Result { - Self::try_from(&data) - } -} - -fn accept_by_glob(source: impl AsRef, accept_glob_set: &GlobSet) -> bool { - accept_glob_set.is_match(source.as_ref()) -} - -fn reject_by_glob(source: impl AsRef, reject_glob_set: &GlobSet) -> bool { - !accept_by_glob(source.as_ref(), reject_glob_set) -} - -async fn accept_dir_for_its_children( - source: impl AsRef + Send, - metadata: &impl MetadataForIndexerRules, - children: &HashSet, -) -> Result { - async fn inner( - source: &Path, - metadata: &impl MetadataForIndexerRules, - children: &HashSet, - ) -> Result { - // FIXME(fogodev): Just check for io::ErrorKind::NotADirectory error instead (feature = "io_error_more", issue = "86442") - if !metadata.is_dir() { - return Ok(false); - } - - let mut read_dir = fs::read_dir(source) - .await // TODO: Check NotADirectory error here when available - .map_err(|e| Error::AcceptByItsChildrenFileIO(FileIOError::from((source, e))))?; - while let Some(entry) = read_dir - .next_entry() - .await - .map_err(|e| Error::AcceptByItsChildrenFileIO(FileIOError::from((source, e))))? - { - let entry_name = entry - .file_name() - .to_str() - .ok_or_else(|| NonUtf8PathError(entry.path().into()))? - .to_string(); - - if entry - .metadata() - .await - .map_err(|e| Error::AcceptByItsChildrenFileIO(FileIOError::from((source, e))))? - .is_dir() && children.contains(&entry_name) - { - return Ok(true); - } - } - - Ok(false) - } - - inner(source.as_ref(), metadata, children).await -} - -async fn reject_dir_for_its_children( - source: impl AsRef + Send, - metadata: &impl MetadataForIndexerRules, - children: &HashSet, -) -> Result { - let source = source.as_ref(); - - // FIXME(fogodev): Just check for io::ErrorKind::NotADirectory error instead (feature = "io_error_more", issue = "86442") - if !metadata.is_dir() { - return Ok(true); - } - - let mut read_dir = fs::read_dir(source) - .await // TODO: Check NotADirectory error here when available - .map_err(|e| Error::RejectByItsChildrenFileIO(FileIOError::from((source, e))))?; - while let Some(entry) = read_dir - .next_entry() - .await - .map_err(|e| Error::RejectByItsChildrenFileIO(FileIOError::from((source, e))))? - { - if entry - .metadata() - .await - .map_err(|e| Error::RejectByItsChildrenFileIO(FileIOError::from((source, e))))? - .is_dir() && children.contains( - entry - .file_name() - .to_str() - .ok_or_else(|| NonUtf8PathError(entry.path().into()))?, - ) { - return Ok(false); - } - } - - Ok(true) -} - -#[must_use] -pub fn generate_pub_id() -> Uuid { - loop { - let pub_id = Uuid::new_v4(); - if pub_id.as_u128() >= 0xFFF { - return pub_id; - } - } -} - -#[cfg(test)] -#[allow(clippy::unwrap_used)] -mod tests { - use super::*; - use tempfile::tempdir; - - impl IndexerRule { - #[must_use] - pub fn new(name: String, default: bool, rules: Vec) -> Self { - Self { - id: None, - name, - default, - rules, - date_created: Utc::now(), - date_modified: Utc::now(), - } - } - } - - fn check_rule(indexer_rule: &IndexerRule, path: impl AsRef) -> bool { - let path = path.as_ref(); - indexer_rule - .rules - .iter() - .map(|rule| match rule { - RulePerKind::AcceptFilesByGlob(_globs, accept_glob_set) => ( - RuleKind::AcceptFilesByGlob, - accept_by_glob(path, accept_glob_set), - ), - RulePerKind::RejectFilesByGlob(_globs, reject_glob_set) => ( - RuleKind::RejectFilesByGlob, - reject_by_glob(path, reject_glob_set), - ), - RulePerKind::IgnoredByGit(git_repo, patterns) => ( - RuleKind::IgnoredByGit, - accept_by_git_pattern(path, git_repo, patterns), - ), - - _ => unimplemented!("can't use simple `apply` for this rule: {:?}", rule), - }) - .all(|(_kind, res)| res) - } - - async fn check_rule_with_metadata( - indexer_rule: &IndexerRule, - path: impl AsRef + Send, - metadata: &impl MetadataForIndexerRules, - ) -> bool { - indexer_rule - .apply(path.as_ref(), metadata) - .await - .unwrap() - .into_iter() - .all(|(_kind, res)| res) - } - - #[tokio::test] - async fn test_reject_hidden_file() { - let hidden = Path::new(".hidden.txt"); - let normal = Path::new("normal.txt"); - let hidden_inner_dir = Path::new("/test/.hidden/"); - let hidden_inner_file = Path::new("/test/.hidden/file.txt"); - let normal_inner_dir = Path::new("/test/normal/"); - let normal_inner_file = Path::new("/test/normal/inner.txt"); - let rule = IndexerRule::new( - "ignore hidden files".to_string(), - false, - vec![RulePerKind::RejectFilesByGlob( - vec![], - GlobSetBuilder::new() - .add(Glob::new("**/.*").unwrap()) - .build() - .unwrap(), - )], - ); - - assert!(!check_rule(&rule, hidden)); - assert!(check_rule(&rule, normal)); - assert!(!check_rule(&rule, hidden_inner_dir)); - assert!(!check_rule(&rule, hidden_inner_file)); - assert!(check_rule(&rule, normal_inner_dir)); - assert!(check_rule(&rule, normal_inner_file)); - } - - #[tokio::test] - async fn test_reject_specific_dir() { - let project_file = Path::new("/test/project/src/main.rs"); - let project_build_dir = Path::new("/test/project/target"); - let project_build_dir_inner = Path::new("/test/project/target/debug/"); - - let rule = IndexerRule::new( - "ignore build directory".to_string(), - false, - vec![RulePerKind::RejectFilesByGlob( - vec![], - GlobSetBuilder::new() - .add(Glob::new("{**/target/*,**/target}").unwrap()) - .build() - .unwrap(), - )], - ); - - assert!(check_rule(&rule, project_file)); - assert!(!check_rule(&rule, project_build_dir)); - assert!(!check_rule(&rule, project_build_dir_inner)); - } - - #[tokio::test] - #[allow(clippy::similar_names)] - async fn test_only_photos() { - let text = Path::new("file.txt"); - let png = Path::new("photo1.png"); - let jpg = Path::new("photo1.png"); - let jpeg = Path::new("photo3.jpeg"); - let inner_text = Path::new("/test/file.txt"); - let inner_png = Path::new("/test/photo1.png"); - let inner_jpg = Path::new("/test/photo2.jpg"); - let inner_jpeg = Path::new("/test/photo3.jpeg"); - let many_inner_dirs_text = Path::new("/test/1/2/3/4/4/5/6/file.txt"); - let many_inner_dirs_png = Path::new("/test/1/2/3/4/4/5/6/photo1.png"); - let rule = IndexerRule::new( - "only photos".to_string(), - false, - vec![RulePerKind::AcceptFilesByGlob( - vec![], - GlobSetBuilder::new() - .add(Glob::new("*.{jpg,png,jpeg}").unwrap()) - .build() - .unwrap(), - )], - ); - - assert!(!check_rule(&rule, text)); - assert!(check_rule(&rule, png)); - assert!(check_rule(&rule, jpg)); - assert!(check_rule(&rule, jpeg)); - assert!(!check_rule(&rule, inner_text)); - assert!(check_rule(&rule, inner_png)); - assert!(check_rule(&rule, inner_jpg)); - assert!(check_rule(&rule, inner_jpeg)); - assert!(!check_rule(&rule, many_inner_dirs_text)); - assert!(check_rule(&rule, many_inner_dirs_png)); - } - - #[tokio::test] - async fn test_directory_has_children() { - let root = tempdir().unwrap(); - - let project1 = root.path().join("project1"); - let project2 = root.path().join("project2"); - let not_project = root.path().join("not_project"); - - fs::create_dir(&project1).await.unwrap(); - fs::create_dir(&project2).await.unwrap(); - fs::create_dir(¬_project).await.unwrap(); - - fs::create_dir(project1.join(".git")).await.unwrap(); - fs::create_dir(project2.join(".git")).await.unwrap(); - fs::create_dir(project2.join("books")).await.unwrap(); - - let childrens = HashSet::from([".git".to_string()]); - - let rule = IndexerRule::new( - "git projects".to_string(), - false, - vec![RulePerKind::AcceptIfChildrenDirectoriesArePresent( - childrens, - )], - ); - - assert!( - !check_rule_with_metadata(&rule, &project1, &fs::metadata(&project1).await.unwrap()) - .await - ); - assert!( - !check_rule_with_metadata(&rule, &project2, &fs::metadata(&project2).await.unwrap()) - .await - ); - assert!( - check_rule_with_metadata( - &rule, - ¬_project, - &fs::metadata(¬_project).await.unwrap() - ) - .await - ); - } - - #[tokio::test] - async fn test_reject_directory_by_its_children() { - let root = tempdir().unwrap(); - - let project1 = root.path().join("project1"); - let project2 = root.path().join("project2"); - let not_project = root.path().join("not_project"); - - fs::create_dir(&project1).await.unwrap(); - fs::create_dir(&project2).await.unwrap(); - fs::create_dir(¬_project).await.unwrap(); - - fs::create_dir(project1.join(".git")).await.unwrap(); - fs::create_dir(project2.join(".git")).await.unwrap(); - fs::create_dir(project2.join("books")).await.unwrap(); - - let childrens = HashSet::from([".git".to_string()]); - - let rule = IndexerRule::new( - "git projects".to_string(), - false, - vec![RulePerKind::RejectIfChildrenDirectoriesArePresent( - childrens, - )], - ); - - assert!( - !check_rule_with_metadata(&rule, &project1, &fs::metadata(&project1).await.unwrap()) - .await - ); - assert!( - !check_rule_with_metadata(&rule, &project2, &fs::metadata(&project2).await.unwrap()) - .await - ); - assert!( - check_rule_with_metadata( - &rule, - ¬_project, - &fs::metadata(¬_project).await.unwrap() - ) - .await - ); - } - - impl PartialEq for RulePerKind { - fn eq(&self, other: &Self) -> bool { - match (self, other) { - ( - Self::AcceptFilesByGlob(self_globs, _), - Self::AcceptFilesByGlob(other_globs, _), - ) - | ( - Self::RejectFilesByGlob(self_globs, _), - Self::RejectFilesByGlob(other_globs, _), - ) => self_globs == other_globs, - - ( - Self::AcceptIfChildrenDirectoriesArePresent(self_childrens), - Self::AcceptIfChildrenDirectoriesArePresent(other_childrens), - ) - | ( - Self::RejectIfChildrenDirectoriesArePresent(self_childrens), - Self::RejectIfChildrenDirectoriesArePresent(other_childrens), - ) => self_childrens == other_childrens, - - _ => false, - } - } - } - - impl Eq for RulePerKind {} - - impl PartialEq for IndexerRule { - fn eq(&self, other: &Self) -> bool { - self.id == other.id - && self.name == other.name - && self.default == other.default - && self.rules == other.rules - && self.date_created == other.date_created - && self.date_modified == other.date_modified - } - } - - impl Eq for IndexerRule {} - - #[test] - fn serde_smoke_test() { - let actual = IndexerRule::new( - "No Hidden".to_string(), - true, - vec![RulePerKind::RejectFilesByGlob( - vec![Glob::new("**/.*").unwrap()], - Glob::new("**/.*") - .and_then(|glob| GlobSetBuilder::new().add(glob).build()) - .unwrap(), - )], - ); - - let expected = - rmp_serde::from_slice::(&rmp_serde::to_vec_named(&actual).unwrap()) - .unwrap(); - - assert_eq!(actual, expected); - } -} diff --git a/core/crates/indexer-rules/src/seed.rs b/core/crates/indexer-rules/src/seed.rs deleted file mode 100644 index a9dc2fdf0..000000000 --- a/core/crates/indexer-rules/src/seed.rs +++ /dev/null @@ -1,346 +0,0 @@ -use sd_prisma::prisma::{indexer_rule, PrismaClient}; - -use std::{ - path::{Path, PathBuf}, - sync::LazyLock, -}; - -use chrono::Utc; -use futures_concurrency::future::Join; -use gix_ignore::{glob::search::pattern::List, search::Ignore, Search}; -use tokio::fs; -use uuid::Uuid; - -use super::{Error, IndexerRule, RulePerKind}; - -#[derive(thiserror::Error, Debug)] -pub enum SeederError { - #[error("Failed to run indexer rules seeder: {0}")] - IndexerRules(#[from] Error), - #[error("An error occurred with the database while applying migrations: {0}")] - DatabaseError(#[from] prisma_client_rust::QueryError), - #[error("Failed to parse indexer rules based on external system")] - InheritedExternalRules, -} - -#[derive(Debug)] -pub struct GitIgnoreRules { - rules: RulePerKind, -} - -impl GitIgnoreRules { - pub async fn get_rules_if_in_git_repo( - location_root: &Path, - current: &Path, - ) -> Option> { - let mut git_repo = None; - - let mut ignores = Vec::new(); - - for ancestor in current - .ancestors() - .take_while(|&path| path.starts_with(location_root)) - { - let git_ignore = ancestor.join(".gitignore"); - - // consider any `.gitignore` files that are inside a git repo - if matches!(fs::try_exists(&git_ignore).await, Ok(true)) { - ignores.push(git_ignore); - } - - if Self::is_git_repo(ancestor).await { - git_repo.replace(ancestor); - break; - } - } - - let git_repo = git_repo?; - Some(Self::parse_git_repo(git_repo, ignores).await) - } - - async fn parse_git_repo( - git_repo: &Path, - git_ignores: Vec, - ) -> Result { - let mut search = Search::default(); - - let git_ignores = git_ignores - .into_iter() - .map(Self::parse_git_ignore) - .collect::>() - .join() - .await; - search - .patterns - .extend(git_ignores.into_iter().filter_map(Result::ok)); - - let git_exclude_rules = Self::parse_git_exclude(git_repo.join(".git")).await; - if let Ok(rules) = git_exclude_rules { - search.patterns.extend(rules); - } - - Ok(Self { - rules: RulePerKind::IgnoredByGit(git_repo.to_owned(), search), - }) - } - - async fn parse_git_ignore(gitignore: PathBuf) -> Result, SeederError> { - tokio::task::spawn_blocking(move || { - let mut buf = Vec::with_capacity(30); - if let Ok(Some(patterns)) = List::from_file(gitignore, None, true, &mut buf) { - Ok(patterns) - } else { - Err(SeederError::InheritedExternalRules) - } - }) - .await - .map_err(|_| SeederError::InheritedExternalRules)? - } - - async fn parse_git_exclude(dot_git: PathBuf) -> Result>, SeederError> { - tokio::task::spawn_blocking(move || { - let mut buf = Vec::new(); - Search::from_git_dir(dot_git.as_ref(), None, &mut buf) - .map(|search| search.patterns) - .map_err(|_| SeederError::InheritedExternalRules) - }) - .await - .map_err(|_| SeederError::InheritedExternalRules)? - } - - async fn is_git_repo(path: &Path) -> bool { - let path = path.join(".git"); - tokio::task::spawn_blocking(move || path.is_dir()) - .await - .unwrap_or_default() - } -} - -impl From for IndexerRule { - fn from(git: GitIgnoreRules) -> Self { - Self { - id: None, - name: ".gitignore'd".to_owned(), - default: true, - date_created: Utc::now(), - date_modified: Utc::now(), - rules: vec![git.rules], - } - } -} - -#[derive(Debug)] -pub struct SystemIndexerRule { - name: &'static str, - rules: Vec, - default: bool, -} - -impl PartialEq for SystemIndexerRule { - fn eq(&self, other: &IndexerRule) -> bool { - self.name == other.name - } -} - -impl From for IndexerRule { - fn from(rule: SystemIndexerRule) -> Self { - Self { - id: None, - name: rule.name.to_string(), - default: rule.default, - rules: rule.rules, - date_created: Utc::now(), - date_modified: Utc::now(), - } - } -} - -impl From<&SystemIndexerRule> for IndexerRule { - fn from(rule: &SystemIndexerRule) -> Self { - Self { - id: None, - name: rule.name.to_string(), - default: rule.default, - rules: rule.rules.clone(), - date_created: Utc::now(), - date_modified: Utc::now(), - } - } -} - -/// Seeds system indexer rules into a new or existing library, -pub async fn new_or_existing_library(db: &PrismaClient) -> Result<(), SeederError> { - use indexer_rule::{date_created, date_modified, default, name, rules_per_kind}; - - // DO NOT REORDER THIS ARRAY! - for (i, rule) in [ - &NO_SYSTEM_FILES, - &NO_HIDDEN, - &NO_GIT, - &GITIGNORE, - &ONLY_IMAGES, - ] - .into_iter() - .enumerate() - { - let pub_id = sd_utils::uuid_to_bytes(&Uuid::from_u128(i as u128)); - let rules = rmp_serde::to_vec_named(&rule.rules).map_err(Error::from)?; - - let data = vec![ - name::set(Some(rule.name.to_string())), - rules_per_kind::set(Some(rules.clone())), - default::set(Some(rule.default)), - date_created::set(Some(Utc::now().into())), - date_modified::set(Some(Utc::now().into())), - ]; - - db.indexer_rule() - .upsert( - indexer_rule::pub_id::equals(pub_id.clone()), - indexer_rule::create(pub_id.clone(), data.clone()), - data, - ) - .exec() - .await?; - } - - Ok(()) -} - -pub static NO_SYSTEM_FILES: LazyLock = LazyLock::new(|| { - SystemIndexerRule { - // TODO: On windows, beside the listed files, any file with the FILE_ATTRIBUTE_SYSTEM should be considered a system file - // https://learn.microsoft.com/en-us/windows/win32/fileio/file-attribute-constants#FILE_ATTRIBUTE_SYSTEM - name: "No System files", - default: true, - rules: vec![ - RulePerKind::new_reject_files_by_globs_str( - [ - vec![ - "**/.spacedrive", - ], - // Globset, even on Windows, requires the use of / as a separator - // https://github.com/github/gitignore/blob/main/Global/Windows.gitignore - // https://learn.microsoft.com/en-us/windows/win32/fileio/naming-a-file - #[cfg(target_os = "windows")] - vec![ - // Windows thumbnail cache files - "**/{Thumbs.db,Thumbs.db:encryptable,ehthumbs.db,ehthumbs_vista.db}", - // Dump file - "**/*.stackdump", - // Folder config file - "**/[Dd]esktop.ini", - // Recycle Bin used on file shares - "**/$RECYCLE.BIN", - // Chkdsk recovery directory - "**/FOUND.[0-9][0-9][0-9]", - // Reserved names - "**/{CON,PRN,AUX,NUL,COM0,COM1,COM2,COM3,COM4,COM5,COM6,COM7,COM8,COM9,LPT0,LPT1,LPT2,LPT3,LPT4,LPT5,LPT6,LPT7,LPT8,LPT9}", - "**/{CON,PRN,AUX,NUL,COM0,COM1,COM2,COM3,COM4,COM5,COM6,COM7,COM8,COM9,LPT0,LPT1,LPT2,LPT3,LPT4,LPT5,LPT6,LPT7,LPT8,LPT9}.*", - // User special files - "C:/Users/*/NTUSER.DAT*", - "C:/Users/*/ntuser.dat*", - "C:/Users/*/{ntuser.ini,ntuser.dat,NTUSER.DAT}", - // User special folders (most of these the user don't even have permission to access) - "C:/Users/*/{Cookies,AppData,NetHood,Recent,PrintHood,SendTo,Templates,Start Menu,Application Data,Local Settings,My Documents}", - // System special folders - "C:/{$Recycle.Bin,$WinREAgent,Documents and Settings,Program Files,Program Files (x86),ProgramData,Recovery,PerfLogs,Windows,Windows.old}", - // NTFS internal dir, can exists on any drive - "[A-Z]:/System Volume Information", - // System special files - "C:/{config,pagefile,hiberfil}.sys", - // Windows can create a swapfile on any drive - "[A-Z]:/swapfile.sys", - "C:/DumpStack.log.tmp", - ], - // https://github.com/github/gitignore/blob/main/Global/macOS.gitignore - // https://developer.apple.com/library/archive/documentation/FileManagement/Conceptual/FileSystemProgrammingGuide/FileSystemOverview/FileSystemOverview.html#//apple_ref/doc/uid/TP40010672-CH2-SW14 - #[cfg(any(target_os = "ios", target_os = "macos"))] - vec![ - "**/.{DS_Store,AppleDouble,LSOverride}", - // Icon must end with two \r - "**/Icon\r\r", - // Thumbnails - "**/._*", - ], - #[cfg(target_os = "macos")] - vec![ - "/{System,Network,Library,Applications,.PreviousSystemInformation,.com.apple.templatemigration.boot-install}", - "/System/Volumes/Data/{System,Network,Library,Applications,.PreviousSystemInformation,.com.apple.templatemigration.boot-install}", - "/Users/*/{Library,Applications}", - "/System/Volumes/Data/Users/*/{Library,Applications}", - "**/*.photoslibrary/{database,external,private,resources,scope}", - // Files that might appear in the root of a volume - "**/.{DocumentRevisions-V100,fseventsd,Spotlight-V100,TemporaryItems,Trashes,VolumeIcon.icns,com.apple.timemachine.donotpresent}", - // Directories potentially created on remote AFP share - "**/.{AppleDB,AppleDesktop,apdisk}", - "**/{Network Trash Folder,Temporary Items}", - ], - // https://github.com/github/gitignore/blob/main/Global/Linux.gitignore - #[cfg(target_os = "linux")] - vec![ - "**/*~", - // temporary files which can be created if a process still has a handle open of a deleted file - "**/.fuse_hidden*", - // KDE directory preferences - "**/.directory", - // Linux trash folder which might appear on any partition or disk - "**/.Trash-*", - // .nfs files are created when an open file is removed but is still being accessed - "**/.nfs*", - ], - #[cfg(target_os = "android")] - vec![ - "**/.nomedia", - "**/.thumbnails", - ], - // https://en.wikipedia.org/wiki/Unix_filesystem#Conventional_directory_layout - // https://en.wikipedia.org/wiki/Filesystem_Hierarchy_Standard - #[cfg(target_family = "unix")] - vec![ - // Directories containing unix memory/device mapped files/dirs - "/{dev,sys,proc}", - // Directories containing special files for current running programs - "/{run,var,boot}", - // ext2-4 recovery directory - "**/lost+found", - ], - ] - .into_iter() - .flatten() - ).expect("this is hardcoded and should always work"), - ], -} -}); - -pub static NO_HIDDEN: LazyLock = LazyLock::new(|| SystemIndexerRule { - name: "No Hidden files", - default: false, - rules: vec![RulePerKind::new_reject_files_by_globs_str(["**/.*"]) - .expect("this is hardcoded and should always work")], -}); - -pub static NO_GIT: LazyLock = LazyLock::new(|| SystemIndexerRule { - name: "No Git files", - default: true, - rules: vec![RulePerKind::new_reject_files_by_globs_str([ - "**/{.git,.gitignore,.gitattributes,.gitkeep,.gitconfig,.gitmodules}", - ]) - .expect("this is hardcoded and should always work")], -}); - -pub static GITIGNORE: LazyLock = LazyLock::new(|| SystemIndexerRule { - name: "Gitignore", - default: true, - // Empty rules because this rule is only used to allow frontend to toggle GitIgnoreRules - rules: vec![], -}); - -pub static ONLY_IMAGES: LazyLock = LazyLock::new(|| SystemIndexerRule { - name: "Only Images", - default: false, - rules: vec![RulePerKind::new_accept_files_by_globs_str([ - "*.{avif,bmp,gif,ico,jpeg,jpg,png,svg,tif,tiff,webp}", - ]) - .expect("this is hardcoded and should always work")], -}); diff --git a/core/crates/indexer-rules/src/serde_impl.rs b/core/crates/indexer-rules/src/serde_impl.rs deleted file mode 100644 index 461630669..000000000 --- a/core/crates/indexer-rules/src/serde_impl.rs +++ /dev/null @@ -1,217 +0,0 @@ -use std::{collections::HashSet, marker::PhantomData}; - -use globset::{Glob, GlobSetBuilder}; -use serde::{de, ser, Deserialize, Serialize}; - -use super::RulePerKind; - -/// We're implementing `Serialize` by hand as `GlobSet`s aren't serializable, so we ignore them on -/// serialization -impl Serialize for RulePerKind { - fn serialize(&self, serializer: S) -> Result - where - S: ser::Serializer, - { - match *self { - Self::AcceptFilesByGlob(ref globs, ref _glob_set) => serializer - .serialize_newtype_variant("ParametersPerKind", 0, "AcceptFilesByGlob", globs), - Self::RejectFilesByGlob(ref globs, ref _glob_set) => serializer - .serialize_newtype_variant("ParametersPerKind", 1, "RejectFilesByGlob", globs), - Self::AcceptIfChildrenDirectoriesArePresent(ref children) => serializer - .serialize_newtype_variant( - "ParametersPerKind", - 2, - "AcceptIfChildrenDirectoriesArePresent", - children, - ), - Self::RejectIfChildrenDirectoriesArePresent(ref children) => serializer - .serialize_newtype_variant( - "ParametersPerKind", - 3, - "RejectIfChildrenDirectoriesArePresent", - children, - ), - Self::IgnoredByGit(_, _) => { - unreachable!("git ignore rules are dynamic and not serialized") - } - } - } -} - -impl<'de> Deserialize<'de> for RulePerKind { - #[allow(clippy::too_many_lines)] - fn deserialize(deserializer: D) -> Result - where - D: de::Deserializer<'de>, - { - const VARIANTS: &[&str] = &[ - "AcceptFilesByGlob", - "RejectFilesByGlob", - "AcceptIfChildrenDirectoriesArePresent", - "RejectIfChildrenDirectoriesArePresent", - ]; - - enum Fields { - AcceptFilesByGlob, - RejectFilesByGlob, - AcceptIfChildrenDirectoriesArePresent, - RejectIfChildrenDirectoriesArePresent, - } - - struct FieldsVisitor; - - impl de::Visitor<'_> for FieldsVisitor { - type Value = Fields; - - fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - formatter.write_str( - "`AcceptFilesByGlob` \ - or `RejectFilesByGlob` \ - or `AcceptIfChildrenDirectoriesArePresent` \ - or `RejectIfChildrenDirectoriesArePresent`", - ) - } - - fn visit_u64(self, value: u64) -> Result - where - E: de::Error, - { - match value { - 0 => Ok(Fields::AcceptFilesByGlob), - 1 => Ok(Fields::RejectFilesByGlob), - 2 => Ok(Fields::AcceptIfChildrenDirectoriesArePresent), - 3 => Ok(Fields::RejectIfChildrenDirectoriesArePresent), - _ => Err(de::Error::invalid_value( - de::Unexpected::Unsigned(value), - &"variant index 0 <= i < 3", - )), - } - } - fn visit_str(self, value: &str) -> Result - where - E: de::Error, - { - match value { - "AcceptFilesByGlob" => Ok(Fields::AcceptFilesByGlob), - "RejectFilesByGlob" => Ok(Fields::RejectFilesByGlob), - "AcceptIfChildrenDirectoriesArePresent" => { - Ok(Fields::AcceptIfChildrenDirectoriesArePresent) - } - "RejectIfChildrenDirectoriesArePresent" => { - Ok(Fields::RejectIfChildrenDirectoriesArePresent) - } - _ => Err(de::Error::unknown_variant(value, VARIANTS)), - } - } - fn visit_bytes(self, bytes: &[u8]) -> Result - where - E: de::Error, - { - match bytes { - b"AcceptFilesByGlob" => Ok(Fields::AcceptFilesByGlob), - b"RejectFilesByGlob" => Ok(Fields::RejectFilesByGlob), - b"AcceptIfChildrenDirectoriesArePresent" => { - Ok(Fields::AcceptIfChildrenDirectoriesArePresent) - } - b"RejectIfChildrenDirectoriesArePresent" => { - Ok(Fields::RejectIfChildrenDirectoriesArePresent) - } - _ => Err(de::Error::unknown_variant( - &String::from_utf8_lossy(bytes), - VARIANTS, - )), - } - } - } - - impl<'de> Deserialize<'de> for Fields { - #[inline] - fn deserialize(deserializer: D) -> Result - where - D: de::Deserializer<'de>, - { - deserializer.deserialize_identifier(FieldsVisitor) - } - } - - struct ParametersPerKindVisitor<'de> { - marker: PhantomData, - lifetime: PhantomData<&'de ()>, - } - - impl<'de> de::Visitor<'de> for ParametersPerKindVisitor<'de> { - type Value = RulePerKind; - - fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - formatter.write_str("enum ParametersPerKind") - } - - fn visit_enum(self, data: PPK) -> Result - where - PPK: de::EnumAccess<'de>, - { - use de::Error; - - de::EnumAccess::variant(data).and_then(|value| match value { - (Fields::AcceptFilesByGlob, accept_files_by_glob) => { - de::VariantAccess::newtype_variant::>(accept_files_by_glob) - .and_then(|globs| { - globs - .iter() - .fold(&mut GlobSetBuilder::new(), |builder, glob| { - builder.add(glob.to_owned()) - }) - .build() - .map_or_else( - |e| Err(PPK::Error::custom(e)), - |glob_set| { - Ok(Self::Value::AcceptFilesByGlob(globs, glob_set)) - }, - ) - }) - } - (Fields::RejectFilesByGlob, reject_files_by_glob) => { - de::VariantAccess::newtype_variant::>(reject_files_by_glob) - .and_then(|globs| { - globs - .iter() - .fold(&mut GlobSetBuilder::new(), |builder, glob| { - builder.add(glob.to_owned()) - }) - .build() - .map_or_else( - |e| Err(PPK::Error::custom(e)), - |glob_set| { - Ok(Self::Value::RejectFilesByGlob(globs, glob_set)) - }, - ) - }) - } - ( - Fields::AcceptIfChildrenDirectoriesArePresent, - accept_if_children_directories_are_present, - ) => de::VariantAccess::newtype_variant::>( - accept_if_children_directories_are_present, - ) - .map(Self::Value::AcceptIfChildrenDirectoriesArePresent), - ( - Fields::RejectIfChildrenDirectoriesArePresent, - reject_if_children_directories_are_present, - ) => de::VariantAccess::newtype_variant::>( - reject_if_children_directories_are_present, - ) - .map(Self::Value::RejectIfChildrenDirectoriesArePresent), - }) - } - } - - deserializer.deserialize_enum( - "ParametersPerKind", - VARIANTS, - ParametersPerKindVisitor { - marker: PhantomData::, - lifetime: PhantomData, - }, - ) - } -} diff --git a/core/crates/p2p/Cargo.toml b/core/crates/p2p/Cargo.toml deleted file mode 100644 index 9427dd581..000000000 --- a/core/crates/p2p/Cargo.toml +++ /dev/null @@ -1,70 +0,0 @@ -[package] -edition = "2021" -name = "sd-core-p2p" -version = "0.1.0" - -[lints.rust] -# Warns -deprecated = "warn" -rust_2018_idioms = { level = "warn", priority = -1 } -trivial_casts = "warn" -trivial_numeric_casts = "warn" -unused_allocation = "warn" -unused_qualifications = "warn" -# Forbids -deprecated_in_future = "forbid" - -[lints.clippy] -# Warns -all = { level = "warn", priority = -1 } -cast_lossless = "warn" -cast_possible_truncation = "warn" -cast_possible_wrap = "warn" -cast_precision_loss = "warn" -cast_sign_loss = "warn" -complexity = { level = "warn", priority = -1 } -correctness = { level = "warn", priority = -1 } -dbg_macro = "warn" -deprecated_cfg_attr = "warn" -nursery = { level = "warn", priority = -1 } -pedantic = { level = "warn", priority = -1 } -perf = { level = "warn", priority = -1 } -separated_literal_suffix = "warn" -style = { level = "warn", priority = -1 } -suspicious = { level = "warn", priority = -1 } -unnecessary_cast = "warn" -unwrap_used = "warn" -# Allows -missing_errors_doc = "allow" -module_name_repetitions = "allow" - -[dependencies] -# Core Spacedrive Sub-crates -sd-core-cloud-services = { path = "../cloud-services" } - -# Spacedrive Sub-crates -sd-cloud-schema = { workspace = true } -sd-crypto = { path = "../../../crates/crypto" } - - -# Workspace dependencies -anyhow = { workspace = true } -async-stream = { workspace = true } -flume = { workspace = true } -futures = { workspace = true } -futures-concurrency = { workspace = true } -iroh = { workspace = true, features = ["discovery-local-network"] } -quic-rpc = { workspace = true, features = ["iroh-transport"] } -serde = { workspace = true, features = ["derive"] } -specta = { workspace = true, features = ["chrono", "uuid"] } -thiserror = { workspace = true } -tokio = { workspace = true, features = ["sync"] } -tokio-util = { workspace = true } -tracing = { workspace = true } - -# External dependencies -derive_more = { version = "1.0", features = ["display", "from", "try_into"] } -nested_enum_utils = "0.1" -postcard = { version = "1.1", features = ["use-std"] } -quic-rpc-derive = "0.17" -url = "2.5" diff --git a/core/crates/p2p/src/error.rs b/core/crates/p2p/src/error.rs deleted file mode 100644 index d004227f5..000000000 --- a/core/crates/p2p/src/error.rs +++ /dev/null @@ -1,24 +0,0 @@ -use std::io; - -#[derive(Debug, thiserror::Error)] -pub enum Error { - // Network setup errors - #[error("Setup iroh endpoint: {0}")] - SetupEndpoint(anyhow::Error), - #[error("Setup iroh listener: {0}")] - SetupListener(io::Error), - #[error("Initialize LocalSwarmDiscovery: {0}")] - LocalSwarmDiscoveryInit(anyhow::Error), - #[error("Initialize DhtDiscovery: {0}")] - DhtDiscoveryInit(anyhow::Error), - - // Known hosts loading errors - #[error("Serialize known devices: {0}")] - SerializeKnownDevices(postcard::Error), - #[error("Deserialize known devices: {0}")] - DeserializeKnownDevices(postcard::Error), - #[error("Load known devices from file: {0}")] - LoadKnownDevices(io::Error), - #[error("Save known devices to file: {0}")] - SaveKnownDevices(io::Error), -} diff --git a/core/crates/p2p/src/lib.rs b/core/crates/p2p/src/lib.rs deleted file mode 100644 index d1f864639..000000000 --- a/core/crates/p2p/src/lib.rs +++ /dev/null @@ -1,162 +0,0 @@ -use sd_core_cloud_services::CloudServices; - -use sd_cloud_schema::devices; -use sd_crypto::CryptoRng; - -use std::{path::Path, sync::Arc}; - -use iroh::{ - discovery::{ - dns::DnsDiscovery, local_swarm_discovery::LocalSwarmDiscovery, pkarr::dht::DhtDiscovery, - ConcurrentDiscovery, - }, - key::SecretKey, - Endpoint, NodeId, RelayMap, RelayMode, RelayUrl, -}; -use quic_rpc::{server::IrohListener, RpcServer}; -use tokio::{ - fs, io, - sync::{oneshot, RwLock}, -}; -use url::Url; - -mod error; -mod schema; -mod server; - -use server::Server; - -pub use error::Error; - -const KNOWN_DEVICES_FILE_NAME: &str = "known_devices.bin"; - -#[derive(Debug, Clone)] -pub struct P2P { - current_device_pub_id: devices::PubId, - known_devices_file_path: Arc>, - endpoint: Endpoint, - cloud_services: Arc>>, - known_devices: Arc>>, - cancel_tx: flume::Sender>, -} - -impl P2P { - pub async fn new( - data_directory: impl AsRef + Send, - current_device_pub_id: devices::PubId, - rng: CryptoRng, - iroh_secret_key: SecretKey, - dns_origin_domain: String, - dns_pkarr_url: Url, - relay_url: RelayUrl, - ) -> Result { - async fn inner( - data_directory: &Path, - current_device_pub_id: devices::PubId, - rng: CryptoRng, - iroh_secret_key: SecretKey, - dns_origin_domain: String, - dns_pkarr_url: Url, - relay_url: RelayUrl, - ) -> Result { - let endpoint = Endpoint::builder() - .alpns(vec![schema::ALPN::LATEST.to_vec()]) - .discovery(Box::new(ConcurrentDiscovery::from_services(vec![ - Box::new(DnsDiscovery::new(dns_origin_domain)), - Box::new( - LocalSwarmDiscovery::new(iroh_secret_key.public()) - .map_err(Error::LocalSwarmDiscoveryInit)?, - ), - Box::new( - DhtDiscovery::builder() - .secret_key(iroh_secret_key.clone()) - .pkarr_relay(dns_pkarr_url) - .build() - .map_err(Error::DhtDiscoveryInit)?, - ), - ]))) - .secret_key(iroh_secret_key) - .relay_mode(RelayMode::Custom(RelayMap::from_url(relay_url))) - .bind() - .await - .map_err(Error::SetupEndpoint)?; - - let (cancel_tx, cancel_rx) = flume::bounded(1); - - let known_devices_file_path = data_directory - .join(KNOWN_DEVICES_FILE_NAME) - .into_boxed_path(); - - let known_devices = Arc::new(RwLock::new( - P2P::load_known_devices(&known_devices_file_path).await?, - )); - - let cloud_services = Arc::default(); - - Server::new( - current_device_pub_id, - Arc::clone(&cloud_services), - Arc::clone(&known_devices), - ) - .dispatch( - RpcServer::new( - IrohListener::::new(endpoint.clone()) - .map_err(Error::SetupListener)?, - ), - cancel_rx, - ); - - Ok(P2P { - current_device_pub_id, - endpoint, - cloud_services, - known_devices, - known_devices_file_path: Arc::new(known_devices_file_path), - cancel_tx, - }) - } - - inner( - data_directory.as_ref(), - current_device_pub_id, - rng, - iroh_secret_key, - dns_origin_domain, - dns_pkarr_url, - relay_url, - ) - .await - } - - async fn load_known_devices( - known_devices_file_path: impl AsRef + Send, - ) -> Result, Error> { - async fn inner(known_devices_file_path: &Path) -> Result, Error> { - match fs::read(known_devices_file_path).await { - Ok(data) => postcard::from_bytes(&data).map_err(Error::DeserializeKnownDevices), - Err(e) if e.kind() == io::ErrorKind::NotFound => Ok(Vec::new()), - Err(e) => Err(Error::LoadKnownDevices(e)), - } - } - - inner(known_devices_file_path.as_ref()).await - } - - pub async fn set_cloud_services(&self, cloud_services: CloudServices) { - self.cloud_services.write().await.replace(cloud_services); - } - - pub async fn shutdown(&self) -> Result<(), Error> { - let (tx, rx) = oneshot::channel(); - self.cancel_tx.send_async(tx).await.unwrap(); - rx.await.unwrap(); - - fs::write( - self.known_devices_file_path.as_ref(), - &postcard::to_stdvec(&*self.known_devices.read().await) - .map_err(Error::SerializeKnownDevices)?, - ) - .await - .map_err(Error::SaveKnownDevices) - } -} diff --git a/core/crates/p2p/src/schema/cloud_services/authorize_new_device_in_sync_group.rs b/core/crates/p2p/src/schema/cloud_services/authorize_new_device_in_sync_group.rs deleted file mode 100644 index fbea51866..000000000 --- a/core/crates/p2p/src/schema/cloud_services/authorize_new_device_in_sync_group.rs +++ /dev/null @@ -1,22 +0,0 @@ -use sd_cloud_schema::{devices, libraries, sync::groups}; - -use serde::{Deserialize, Serialize}; - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub struct Request { - pub sync_group: groups::GroupWithDevices, - pub asking_device: devices::Device, -} - -#[derive(Debug, PartialEq, Eq, Serialize, Deserialize)] -pub enum Response { - Accepted { - authorizor_device: devices::Device, - keys: Vec>, - library_pub_id: libraries::PubId, - library_name: String, - library_description: Option, - }, - Rejected, - TimedOut, -} diff --git a/core/crates/p2p/src/schema/cloud_services/mod.rs b/core/crates/p2p/src/schema/cloud_services/mod.rs deleted file mode 100644 index 39d1ff7ec..000000000 --- a/core/crates/p2p/src/schema/cloud_services/mod.rs +++ /dev/null @@ -1,25 +0,0 @@ -use crate::schema::Service; - -use nested_enum_utils::enum_conversions; -use serde::{Deserialize, Serialize}; - -pub mod authorize_new_device_in_sync_group; -pub mod notify_new_sync_messages; - -#[allow(clippy::large_enum_variant)] -#[nested_enum_utils::enum_conversions(super::Request)] -#[derive(Debug, Serialize, Deserialize)] -#[quic_rpc_derive::rpc_requests(Service)] -pub enum Request { - #[rpc(response = authorize_new_device_in_sync_group::Response)] - AuthorizeNewDeviceInSyncGroup(authorize_new_device_in_sync_group::Request), - #[rpc(response = notify_new_sync_messages::Response)] - NotifyNewSyncMessages(notify_new_sync_messages::Request), -} - -#[derive(Debug, Serialize, Deserialize)] -#[enum_conversions(super::Response)] -pub enum Response { - AuthorizeNewDeviceInSyncGroup(authorize_new_device_in_sync_group::Response), - NotifyNewSyncMessages(notify_new_sync_messages::Response), -} diff --git a/core/crates/p2p/src/schema/cloud_services/notify_new_sync_messages.rs b/core/crates/p2p/src/schema/cloud_services/notify_new_sync_messages.rs deleted file mode 100644 index 415e0dd6b..000000000 --- a/core/crates/p2p/src/schema/cloud_services/notify_new_sync_messages.rs +++ /dev/null @@ -1,12 +0,0 @@ -use sd_cloud_schema::{devices, sync::groups}; - -use serde::{Deserialize, Serialize}; - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub struct Request { - pub sync_group_pub_id: groups::PubId, - pub device_pub_id: devices::PubId, -} - -#[derive(Debug, Serialize, Deserialize)] -pub struct Response; diff --git a/core/crates/p2p/src/schema/mod.rs b/core/crates/p2p/src/schema/mod.rs deleted file mode 100644 index 8e40a360a..000000000 --- a/core/crates/p2p/src/schema/mod.rs +++ /dev/null @@ -1,36 +0,0 @@ -use serde::{Deserialize, Serialize}; - -pub mod cloud_services; - -/// ALPN for the Spacedrive P2P protocol -/// -/// P2P with associated constants for each existing version and an alias for the latest version. -/// This application layer protocol is used when a cloud service needs to devices communicating -/// with each other, like for sending sync keys, or other strictly P2P features, like Spacedrop. -pub struct ALPN; - -impl ALPN { - pub const LATEST: &'static [u8] = Self::V1; - pub const V1: &'static [u8] = b"sd-p2p/v1"; -} - -#[derive(Debug, Clone)] -pub struct Service; - -impl quic_rpc::Service for Service { - type Req = Request; - - type Res = Response; -} - -#[nested_enum_utils::enum_conversions] -#[derive(Debug, Serialize, Deserialize)] -pub enum Request { - CloudServices(cloud_services::Request), -} - -#[nested_enum_utils::enum_conversions] -#[derive(Debug, Serialize, Deserialize)] -pub enum Response { - CloudServices(cloud_services::Response), -} diff --git a/core/crates/p2p/src/server/mod.rs b/core/crates/p2p/src/server/mod.rs deleted file mode 100644 index 14d874092..000000000 --- a/core/crates/p2p/src/server/mod.rs +++ /dev/null @@ -1,191 +0,0 @@ -use sd_core_cloud_services::CloudServices; - -use sd_cloud_schema::devices; - -use std::{pin::pin, sync::Arc, time::Duration}; - -use anyhow::Context as _; -use futures::StreamExt as _; -use futures_concurrency::stream::Merge as _; -use iroh::NodeId; -use quic_rpc::{ - server::{Accepting, RpcServerError}, - Listener, RpcServer, -}; -use tokio::{ - spawn, - sync::{oneshot, RwLock}, - task::JoinError, - time::timeout, -}; -use tracing::{error, info, warn}; - -use super::schema; - -mod router; - -const SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(20); - -#[derive(Debug, Clone)] -pub struct Server { - current_device_pub_id: devices::PubId, - cloud_services: Arc>>, - known_devices: Arc>>, -} - -impl Server { - pub fn new( - current_device_pub_id: devices::PubId, - cloud_services: Arc>>, - known_devices: Arc>>, - ) -> Self { - Self { - current_device_pub_id, - cloud_services, - known_devices, - } - } - - async fn handle_single_request( - self, - accepting: Accepting>, - out_tx: flume::Sender, JoinError>>, - ) { - async fn inner( - server: Server, - accepting: Accepting>, - ) -> anyhow::Result<()> { - let (req, chan) = accepting - .read_first() - .await - .context("Failed to receive request")?; - - router::handle(server, req, chan).await - } - - // Running on a detached task to avoid panicking the main task - let res = spawn(inner(self, accepting)).await; - out_tx.send_async(res).await.expect("channel never closes"); - } - - pub fn dispatch( - self, - rpc_server: RpcServer>, - cancel_rx: flume::Receiver>, - ) { - spawn({ - async move { - loop { - info!("Starting P2P Server"); - if let Err(e) = - spawn(self.clone().run_loop(rpc_server.clone(), cancel_rx.clone())).await - { - if e.is_panic() { - error!(?e, "P2P Server crashed, restarting..."); - } else { - break; - } - } - } - } - }); - } - - async fn run_loop( - self, - rpc_server: RpcServer>, - cancel_rx: flume::Receiver>, - ) { - enum StreamMessage> { - AcceptResult(Result, RpcServerError>), - RequestOutcome(Result, JoinError>), - Shutdown(oneshot::Sender<()>), - } - - let (out_tx, out_rx) = flume::bounded(32); - - let mut msg_stream = pin!(( - async_stream::stream! { - loop { - yield StreamMessage::AcceptResult(rpc_server.accept().await); - } - }, - cancel_rx.stream().map(StreamMessage::Shutdown), - out_rx.stream().map(StreamMessage::RequestOutcome) - ) - .merge()); - - let mut inflight_count = 0u32; - - info!("P2P listening for connections..."); - - while let Some(msg) = msg_stream.next().await { - match msg { - StreamMessage::AcceptResult(Ok(accepting)) => { - spawn( - self.clone() - .handle_single_request(accepting, out_tx.clone()), - ); - inflight_count += 1; - } - StreamMessage::AcceptResult(Err(e)) => { - error!(?e, "Failed to accept request;"); - } - - StreamMessage::RequestOutcome(out) => { - process_request_outcome(out); - inflight_count -= 1; - } - - StreamMessage::Shutdown(tx) => { - // Received an Interrupt signal, which means the user wants to stop the server, - // so we wait for all inflight requests to finish before exiting - // this way we're doing a graceful shutdown - - let wait_all_to_finish = async { - while inflight_count > 0 { - process_request_outcome( - // SAFETY: channel never closes - out_rx.recv_async().await.expect("channel never closes"), - ); - inflight_count -= 1; - } - }; - - if let Err(elapsed) = timeout(SHUTDOWN_TIMEOUT, wait_all_to_finish).await { - warn!(?elapsed, %inflight_count, "Server graceful shutdown timed out"); - } else { - info!("Server graceful shutdown complete!"); - } - - if tx.send(()).is_err() { - warn!("Failed to send P2P shutdown completion response;"); - } - - break; - } - } - } - } -} - -fn process_request_outcome(out: Result, JoinError>) { - match out { - Ok(Err(e)) => { - error!(?e, "Failed to handle request;"); - } - Err(e) if e.is_panic() => { - if let Some(msg) = e.into_panic().downcast_ref::<&str>() { - error!(?msg, "Panic in request handler!"); - } else { - error!("Some unknown panic in request handler!"); - } - } - Ok(Ok(())) | Err(_) => { - // The request was handled successfully, or the JoinHandle was aborted, - // which can't happen because we don't even have the handle, so... - // ... - // Everything is Awesome! - } - } -} diff --git a/core/crates/p2p/src/server/router/cloud_services.rs b/core/crates/p2p/src/server/router/cloud_services.rs deleted file mode 100644 index 0378c4c7a..000000000 --- a/core/crates/p2p/src/server/router/cloud_services.rs +++ /dev/null @@ -1,47 +0,0 @@ -use crate::{ - schema::{ - self, - cloud_services::{self, authorize_new_device_in_sync_group, notify_new_sync_messages}, - }, - server::Server, -}; - -use anyhow::Context as _; -use quic_rpc::{server::RpcChannel, Listener}; - -pub async fn router( - server: Server, - request: cloud_services::Request, - chan: RpcChannel>, -) -> anyhow::Result<()> { - match request { - cloud_services::Request::AuthorizeNewDeviceInSyncGroup(req) => { - chan.rpc(req, server, authorize_new_device_in_sync_group) - .await - } - cloud_services::Request::NotifyNewSyncMessages(req) => { - chan.rpc(req, server, notify_new_sync_messages).await - } - } - .context("Failed to handle cloud services request") -} - -async fn authorize_new_device_in_sync_group( - server: Server, - authorize_new_device_in_sync_group::Request { - sync_group, - asking_device, - }: authorize_new_device_in_sync_group::Request, -) -> authorize_new_device_in_sync_group::Response { - todo!() -} - -async fn notify_new_sync_messages( - server: Server, - notify_new_sync_messages::Request { - sync_group_pub_id, - device_pub_id, - }: notify_new_sync_messages::Request, -) -> notify_new_sync_messages::Response { - todo!() -} diff --git a/core/crates/p2p/src/server/router/mod.rs b/core/crates/p2p/src/server/router/mod.rs deleted file mode 100644 index 8710b9017..000000000 --- a/core/crates/p2p/src/server/router/mod.rs +++ /dev/null @@ -1,19 +0,0 @@ -use anyhow::Context; -use quic_rpc::{server::RpcChannel, Listener}; - -use crate::schema; - -use super::Server; - -mod cloud_services; - -pub async fn handle( - server: Server, - request: schema::Request, - chan: RpcChannel>, -) -> anyhow::Result<()> { - match request { - schema::Request::CloudServices(req) => cloud_services::router(server, req, chan).await, - } - .context("Failed to handle p2p request") -} diff --git a/core/crates/prisma-helpers/Cargo.toml b/core/crates/prisma-helpers/Cargo.toml deleted file mode 100644 index 8a4b490ea..000000000 --- a/core/crates/prisma-helpers/Cargo.toml +++ /dev/null @@ -1,20 +0,0 @@ -[package] -name = "sd-core-prisma-helpers" -version = "0.1.0" - -authors = ["Ericson Soares "] -edition.workspace = true -license.workspace = true -repository.workspace = true - -[dependencies] -# Spacedrive Sub-crates -sd-cloud-schema = { workspace = true } -sd-prisma = { path = "../../../crates/prisma" } -sd-utils = { path = "../../../crates/utils" } - -# Workspace dependencies -prisma-client-rust = { workspace = true } -serde = { workspace = true, features = ["derive"] } -specta = { workspace = true } -uuid = { workspace = true, features = ["serde", "v4"] } diff --git a/core/crates/prisma-helpers/src/lib.rs b/core/crates/prisma-helpers/src/lib.rs deleted file mode 100644 index 311d81947..000000000 --- a/core/crates/prisma-helpers/src/lib.rs +++ /dev/null @@ -1,576 +0,0 @@ -#![recursion_limit = "256"] -#![warn( - clippy::all, - clippy::pedantic, - clippy::correctness, - clippy::perf, - clippy::style, - clippy::suspicious, - clippy::complexity, - clippy::nursery, - clippy::unwrap_used, - unused_qualifications, - rust_2018_idioms, - trivial_casts, - trivial_numeric_casts, - unused_allocation, - clippy::unnecessary_cast, - clippy::cast_lossless, - clippy::cast_possible_truncation, - clippy::cast_possible_wrap, - clippy::cast_precision_loss, - clippy::cast_sign_loss, - clippy::dbg_macro, - clippy::deprecated_cfg_attr, - clippy::separated_literal_suffix, - deprecated -)] -#![forbid(deprecated_in_future)] -#![allow(clippy::missing_errors_doc, clippy::module_name_repetitions)] - -use sd_prisma::prisma::{file_path, job, label, location, object}; -use sd_utils::{from_bytes_to_uuid, uuid_to_bytes}; - -use std::{borrow::Cow, fmt}; - -use serde::{Deserialize, Serialize}; -use uuid::Uuid; - -// File Path selectables! -file_path::select!(file_path_id { id }); -file_path::select!(file_path_pub_id { pub_id }); -file_path::select!(file_path_pub_and_cas_ids { id pub_id cas_id }); -file_path::select!(file_path_just_pub_id_materialized_path { - pub_id - materialized_path -}); -file_path::select!(file_path_for_file_identifier { - id - pub_id - materialized_path - date_created - is_dir - name - extension - object_id -}); -file_path::select!(file_path_for_object_validator { - pub_id - materialized_path - is_dir - name - extension - integrity_checksum -}); -file_path::select!(file_path_for_media_processor { - id - materialized_path - is_dir - name - extension - cas_id - object: select { - id - pub_id - } -}); -file_path::select!(file_path_watcher_remove { - id - pub_id - location_id - materialized_path - is_dir - name - extension - object: select { - id - pub_id - } - -}); -file_path::select!(file_path_to_isolate { - location_id - materialized_path - is_dir - name - extension -}); -file_path::select!(file_path_to_isolate_with_pub_id { - pub_id - location_id - materialized_path - is_dir - name - extension -}); -file_path::select!(file_path_to_isolate_with_id { - id - location_id - materialized_path - is_dir - name - extension -}); -file_path::select!(file_path_walker { - pub_id - location_id - object_id - materialized_path - is_dir - name - extension - date_modified - inode - size_in_bytes_bytes - hidden -}); -file_path::select!(file_path_to_handle_custom_uri { - pub_id - materialized_path - is_dir - name - extension - location: select { - id - path - instance: select { - identity - remote_identity - node_remote_identity - } - } -}); -file_path::select!(file_path_to_handle_p2p_serve_file { - materialized_path - name - extension - is_dir // For isolated file path - location: select { - id - path - } -}); -file_path::select!(file_path_to_full_path { - id - materialized_path - is_dir - name - extension - location: select { - id - path - } -}); -file_path::select!(file_path_to_create_object { - id - pub_id - date_created -}); - -// File Path includes! -file_path::include!(file_path_with_object { object }); -file_path::include!(file_path_for_frontend { - object: include { - tags: include { tag } - exif_data: select { - resolution - media_date - media_location - camera_data - artist - description - copyright - exif_version - } - } -}); - -// Object selectables! -object::select!(object_ids { id pub_id }); -object::select!(object_for_file_identifier { - pub_id - file_paths: select { pub_id cas_id extension is_dir materialized_path name } -}); - -// Object includes! -object::include!(object_with_file_paths { - file_paths: include { - object: include { - exif_data: select { - resolution - media_date - media_location - camera_data - artist - description - copyright - exif_version - } - ffmpeg_data: include { - chapters - programs: include { - streams: include { - codec: include { - audio_props - video_props - } - } - } - } - } - } -}); -object::include!(object_with_media_data { - exif_data - ffmpeg_data: include { - chapters - programs: include { - streams: include { - codec: include { - audio_props - video_props - } - } - } - } -}); - -// Job selectables! -job::select!(job_without_data { - id - name - action - status - parent_id - errors_text - metadata - date_created - date_started - date_completed - task_count - info - completed_task_count - date_estimated_completion -}); - -// Location selectables! -location::select!(location_ids_and_path { - id - pub_id - device: select { pub_id } - path -}); - -// Location includes! -location::include!(location_with_indexer_rules { - indexer_rules: select { indexer_rule } -}); - -impl From for location::Data { - fn from(data: location_with_indexer_rules::Data) -> Self { - Self { - id: data.id, - pub_id: data.pub_id, - path: data.path, - device_id: data.device_id, - instance_id: data.instance_id, - name: data.name, - total_capacity: data.total_capacity, - available_capacity: data.available_capacity, - is_archived: data.is_archived, - size_in_bytes: data.size_in_bytes, - generate_preview_media: data.generate_preview_media, - sync_preview_media: data.sync_preview_media, - hidden: data.hidden, - date_created: data.date_created, - scan_state: data.scan_state, - file_paths: None, - indexer_rules: None, - device: None, - instance: None, - } - } -} - -impl From<&location_with_indexer_rules::Data> for location::Data { - fn from(data: &location_with_indexer_rules::Data) -> Self { - Self { - id: data.id, - pub_id: data.pub_id.clone(), - path: data.path.clone(), - device_id: data.device_id, - instance_id: data.instance_id, - name: data.name.clone(), - total_capacity: data.total_capacity, - available_capacity: data.available_capacity, - size_in_bytes: data.size_in_bytes.clone(), - is_archived: data.is_archived, - generate_preview_media: data.generate_preview_media, - sync_preview_media: data.sync_preview_media, - hidden: data.hidden, - date_created: data.date_created, - scan_state: data.scan_state, - file_paths: None, - indexer_rules: None, - device: None, - instance: None, - } - } -} - -// Label includes! -label::include!((take: i64) => label_with_objects { - label_objects(vec![]).take(take): select { - object: select { - id - file_paths(vec![]).take(1) - } - } -}); - -#[derive(Debug, Serialize, Deserialize, Hash, PartialEq, Eq, specta::Type)] -#[serde(transparent)] -pub struct CasId<'cas_id>(Cow<'cas_id, str>); - -impl Clone for CasId<'_> { - fn clone(&self) -> CasId<'static> { - CasId(Cow::Owned(self.0.clone().into_owned())) - } -} - -impl CasId<'_> { - #[must_use] - pub fn as_str(&self) -> &str { - self.0.as_ref() - } - - #[must_use] - pub fn to_owned(&self) -> CasId<'static> { - CasId(Cow::Owned(self.0.clone().into_owned())) - } - - #[must_use] - pub fn into_owned(self) -> CasId<'static> { - CasId(Cow::Owned(self.0.clone().into_owned())) - } -} - -impl From<&CasId<'_>> for file_path::cas_id::Type { - fn from(CasId(cas_id): &CasId<'_>) -> Self { - Some(cas_id.clone().into_owned()) - } -} - -impl<'cas_id> From<&'cas_id str> for CasId<'cas_id> { - fn from(cas_id: &'cas_id str) -> Self { - Self(Cow::Borrowed(cas_id)) - } -} - -impl<'cas_id> From<&'cas_id String> for CasId<'cas_id> { - fn from(cas_id: &'cas_id String) -> Self { - Self(Cow::Borrowed(cas_id)) - } -} - -impl From for CasId<'static> { - fn from(cas_id: String) -> Self { - Self(cas_id.into()) - } -} - -impl From> for String { - fn from(CasId(cas_id): CasId<'_>) -> Self { - cas_id.into_owned() - } -} - -impl From<&CasId<'_>> for String { - fn from(CasId(cas_id): &CasId<'_>) -> Self { - cas_id.clone().into_owned() - } -} - -#[derive(Debug, Serialize, Deserialize, Hash, PartialEq, Eq, Clone, specta::Type)] -#[serde(transparent)] -#[repr(transparent)] -#[specta(rename = "CoreDevicePubId")] -pub struct DevicePubId(PubId); - -impl From for sd_cloud_schema::devices::PubId { - fn from(DevicePubId(pub_id): DevicePubId) -> Self { - Self(pub_id.into()) - } -} - -#[derive(Debug, Serialize, Deserialize, Hash, PartialEq, Eq, Clone, specta::Type)] -#[serde(transparent)] -#[repr(transparent)] -#[specta(rename = "CoreFilePathPubId")] -pub struct FilePathPubId(PubId); - -#[derive(Debug, Serialize, Deserialize, Hash, PartialEq, Eq, Clone, specta::Type)] -#[serde(transparent)] -#[repr(transparent)] -#[specta(rename = "CoreObjectPubId")] -pub struct ObjectPubId(PubId); - -#[derive(Debug, Serialize, Deserialize, Hash, PartialEq, Eq, Clone, specta::Type)] -#[specta(rename = "CorePubId")] -enum PubId { - Uuid(Uuid), - Vec(Vec), -} - -impl PubId { - fn new() -> Self { - Self::Uuid(Uuid::now_v7()) - } - - fn to_db(&self) -> Vec { - match self { - Self::Uuid(uuid) => uuid_to_bytes(uuid), - Self::Vec(bytes) => bytes.clone(), - } - } -} - -impl Default for PubId { - fn default() -> Self { - Self::new() - } -} - -impl From for PubId { - fn from(uuid: Uuid) -> Self { - Self::Uuid(uuid) - } -} - -impl From> for PubId { - fn from(bytes: Vec) -> Self { - Self::Vec(bytes) - } -} - -impl From<&Vec> for PubId { - fn from(bytes: &Vec) -> Self { - Self::Vec(bytes.clone()) - } -} - -impl From<&[u8]> for PubId { - fn from(bytes: &[u8]) -> Self { - Self::Vec(bytes.to_vec()) - } -} - -impl From for Vec { - fn from(pub_id: PubId) -> Self { - match pub_id { - PubId::Uuid(uuid) => uuid_to_bytes(&uuid), - PubId::Vec(bytes) => bytes, - } - } -} - -impl From for Uuid { - fn from(pub_id: PubId) -> Self { - match pub_id { - PubId::Uuid(uuid) => uuid, - PubId::Vec(bytes) => from_bytes_to_uuid(&bytes), - } - } -} - -impl From<&PubId> for Uuid { - fn from(pub_id: &PubId) -> Self { - match pub_id { - PubId::Uuid(uuid) => *uuid, - PubId::Vec(bytes) => from_bytes_to_uuid(bytes), - } - } -} - -impl fmt::Display for PubId { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::Uuid(uuid) => write!(f, "{uuid}"), - Self::Vec(bytes) => write!(f, "{}", from_bytes_to_uuid(bytes)), - } - } -} - -macro_rules! delegate_pub_id { - ($($type_name:ty),+ $(,)?) => { - $( - impl From<::uuid::Uuid> for $type_name { - fn from(uuid: ::uuid::Uuid) -> Self { - Self(uuid.into()) - } - } - - impl From> for $type_name { - fn from(bytes: Vec) -> Self { - Self(bytes.into()) - } - } - - impl From<&Vec> for $type_name { - fn from(bytes: &Vec) -> Self { - Self(bytes.into()) - } - } - - impl From<&[u8]> for $type_name { - fn from(bytes: &[u8]) -> Self { - Self(bytes.into()) - } - } - - impl From<$type_name> for Vec { - fn from(pub_id: $type_name) -> Self { - pub_id.0.into() - } - } - - impl From<$type_name> for ::uuid::Uuid { - fn from(pub_id: $type_name) -> Self { - pub_id.0.into() - } - } - - impl From<&$type_name> for ::uuid::Uuid { - fn from(pub_id: &$type_name) -> Self { - (&pub_id.0).into() - } - } - - impl ::std::fmt::Display for $type_name { - fn fmt(&self, f: &mut ::std::fmt::Formatter<'_>) -> ::std::fmt::Result { - write!(f, "{}", self.0) - } - } - - impl $type_name { - #[must_use] - pub fn new() -> Self { - Self(PubId::new()) - } - - #[must_use] - pub fn to_db(&self) -> Vec { - self.0.to_db() - } - } - - impl Default for $type_name { - fn default() -> Self { - Self::new() - } - } - )+ - }; -} - -delegate_pub_id!(FilePathPubId, ObjectPubId, DevicePubId); diff --git a/core-new/spacedrive-jobs-derive/Cargo.toml b/core/crates/spacedrive-jobs-derive/Cargo.toml similarity index 100% rename from core-new/spacedrive-jobs-derive/Cargo.toml rename to core/crates/spacedrive-jobs-derive/Cargo.toml diff --git a/core-new/spacedrive-jobs-derive/src/lib.rs b/core/crates/spacedrive-jobs-derive/src/lib.rs similarity index 100% rename from core-new/spacedrive-jobs-derive/src/lib.rs rename to core/crates/spacedrive-jobs-derive/src/lib.rs diff --git a/core/crates/sync/Cargo.toml b/core/crates/sync/Cargo.toml deleted file mode 100644 index d2a7dfc1e..000000000 --- a/core/crates/sync/Cargo.toml +++ /dev/null @@ -1,39 +0,0 @@ -[package] -name = "sd-core-sync" -version = "0.0.0" - -edition = "2021" - -[features] -default = [] - -[dependencies] -# Spacedrive Sub-crates -sd-core-prisma-helpers = { path = "../prisma-helpers" } - -sd-actors = { path = "../../../crates/actors" } -sd-prisma = { path = "../../../crates/prisma" } -sd-sync = { path = "../../../crates/sync" } -sd-utils = { path = "../../../crates/utils" } - -# Workspace dependencies -async-channel = { workspace = true } -async-stream = { workspace = true } -chrono = { workspace = true } -futures = { workspace = true } -futures-concurrency = { workspace = true } -itertools = { workspace = true } -prisma-client-rust = { workspace = true, features = ["rspc"] } -rmp-serde = { workspace = true } -rmpv = { workspace = true } -rspc = { workspace = true } -serde = { workspace = true } -thiserror = { workspace = true } -tokio = { workspace = true } -tracing = { workspace = true } -uhlc = { workspace = true } -uuid = { workspace = true } - -[dev-dependencies] -tracing-subscriber = { workspace = true, features = ["env-filter"] } -tracing-test = { workspace = true } diff --git a/core/crates/sync/README.md b/core/crates/sync/README.md deleted file mode 100644 index 11d8f53da..000000000 --- a/core/crates/sync/README.md +++ /dev/null @@ -1,132 +0,0 @@ -# `sd-core-sync` - -Spacedrive's sync system. Consumes types and helpers from `sd-sync`. - -### Creating Records - -Prepare a sync id by creating or obtaining its value, -and then wrapping it in the model's `SyncId` struct, -available at `prisma_sync::{model}::SyncId`. - -Next, prepare the sync operations using some varaints of the `sync_entry` macros. -`sync_entry` and `option_sync_entry` take the value first, and then the path to the field's prisma module. -`sync_db_entry` and `option_sync_db_entry` take the same inputs, but additionally produce a prisma operation in a tuple with the sync operation, intended to be put into a `Vec` and unzipped. - -Finally, use `sync.shared/relation_create` depending on if you're creating a standalone record or a relation between two records, and then write it to the database with `write_ops`. - -```rs -let (sync_params, db_params): (Vec<_>, Vec<_>) = [ - sync_db_entry!(self.name, tag::name), - sync_db_entry!(self.color, tag::color), - sync_db_entry!(false, tag::is_hidden), - sync_db_entry!(date_created, tag::date_created), -] -.into_iter() -.unzip(); - -sync.write_ops( - db, - ( - sync.shared_create( - prisma_sync::tag::SyncId { pub_id }, - sync_params, - ), - db.tag().create(pub_id, db_params), - ), -) -``` - -### Updating Records - -This follows a similar process to creation, but with `sync.shared/relation_create`. - -```rs -let (sync_params, db_params): (Vec<_>, Vec<_>) = [ - sync_db_entry!(name, tag::name), - sync_db_entry!(color, tag::color), -] -.into_iter() -.unzip(); - -sync.write_ops( - db, - ( - sync.shared_update(prisma_sync::tag::SyncId { pub_id }, k, v), - db.tag().update(tag::id::equals(id), db_params); - ) -) -``` - -### Deleting Records - -This only requires a sync ID. - -```rs -sync.write_op( - db, - sync.shared_delete(prisma_sync::tag::SyncId { pub_id }), - db.tag().delete(tag::id::equals(id)); -) -``` - -### Relation Records - -Relations require sync IDs for both the item and the group being related together. -Apart from that they're basically the same as shared operations. - -```rs -let (sync_params, db_params): (Vec<_>, Vec<_>) = [ - sync_db_entry!(date_created, tag_on_object::date_created) -] -.into_iter() -.unzip(); - -sync.write_ops( - db, - ( - sync.relation_create( - prisma_sync::tag_on_object::SyncId { - tag: prisma_sync::tag::SyncId { pub_id: tag_pub_id }, - object: prisma_sync::object::SyncId { pub_id: object_pub_id }, - }, - sync_params - ), - db.tag_on_object().create( - object::id::equals(object_id), - tag::id::equals(tag_id), - db_params - ) - ) -) -``` - -### Setting Relation Fields - -Setting relation fields requires providing the Sync ID of the relation. -Setting the relation field's scalar fields instead will not properly sync then relation, -usually because the scalar fields are local and disconnected from the Sync ID. - -```rs -let (sync_params, db_params): (Vec<_>, Vec<_>) = [ - sync_db_entry!( - prisma_sync::object::SyncId { pub_id: object_pub_id }, - file_path::object - ) -].into_iter().unzip(); - -sync.write_ops( - db, - ( - sync.shared_update( - prisma_sync::file_path::SyncId { - pub_id: file_path_pub_id - }, - sync_params - ), - db.file_path().update( - file_path::id::equals(file_path_id), - db_params - ) - ) -) -``` diff --git a/core/crates/sync/src/backfill.rs b/core/crates/sync/src/backfill.rs deleted file mode 100644 index 2032eab3f..000000000 --- a/core/crates/sync/src/backfill.rs +++ /dev/null @@ -1,669 +0,0 @@ -use sd_prisma::{ - prisma::{ - crdt_operation, device, exif_data, file_path, label, label_on_object, location, object, - tag, tag_on_object, volume, PrismaClient, SortOrder, - }, - prisma_sync, -}; -use sd_sync::{option_sync_entry, sync_entry, OperationFactory}; -use sd_utils::chain_optional_iter; - -use std::future::Future; - -use futures_concurrency::future::TryJoin; -use tokio::time::Instant; -use tracing::{debug, instrument}; - -use super::{crdt_op_unchecked_db, Error, SyncManager}; - -/// Takes all the syncable data in the database and generates [`CRDTOperations`] for it. -/// This is a requirement before the library can sync. -pub async fn backfill_operations(sync: &SyncManager) -> Result<(), Error> { - let _lock_guard = sync.sync_lock.lock().await; - - let db = &sync.db; - - let local_device = db - .device() - .find_unique(device::pub_id::equals(sync.device_pub_id.to_db())) - .exec() - .await? - .ok_or(Error::DeviceNotFound(sync.device_pub_id.clone()))?; - - let local_device_id = local_device.id; - - db._transaction() - .with_timeout(9_999_999_999) - .run(|db| async move { - debug!("backfill started"); - let start = Instant::now(); - db.crdt_operation() - .delete_many(vec![crdt_operation::device_pub_id::equals( - sync.device_pub_id.to_db(), - )]) - .exec() - .await?; - - backfill_device(&db, sync, local_device).await?; - - ( - backfill_volumes(&db, sync, local_device_id), - paginate_tags(&db, sync), - paginate_locations(&db, sync, local_device_id), - paginate_objects(&db, sync, local_device_id), - paginate_labels(&db, sync), - ) - .try_join() - .await?; - - ( - paginate_exif_datas(&db, sync, local_device_id), - paginate_file_paths(&db, sync, local_device_id), - paginate_tags_on_objects(&db, sync, local_device_id), - paginate_labels_on_objects(&db, sync, local_device_id), - ) - .try_join() - .await?; - - debug!(elapsed = ?start.elapsed(), "backfill ended"); - - Ok(()) - }) - .await -} - -#[instrument(skip(db, sync), err)] -async fn backfill_device( - db: &PrismaClient, - sync: &SyncManager, - local_device: device::Data, -) -> Result<(), Error> { - db.crdt_operation() - .create_many(vec![crdt_op_unchecked_db(&sync.shared_create( - prisma_sync::device::SyncId { - pub_id: local_device.pub_id, - }, - chain_optional_iter( - [], - [ - option_sync_entry!(local_device.name, device::name), - option_sync_entry!(local_device.os, device::os), - option_sync_entry!(local_device.hardware_model, device::hardware_model), - option_sync_entry!(local_device.timestamp, device::timestamp), - option_sync_entry!(local_device.date_created, device::date_created), - option_sync_entry!(local_device.date_deleted, device::date_deleted), - ], - ), - ))?]) - .exec() - .await?; - - Ok(()) -} - -#[instrument(skip(db, sync), err)] -async fn backfill_volumes( - db: &PrismaClient, - sync: &SyncManager, - device_id: device::id::Type, -) -> Result<(), Error> { - let Some(volume) = db - .volume() - .find_first(vec![volume::device_id::equals(Some(device_id))]) - .include(volume::include!({device: select { pub_id }})) - .exec() - .await? - else { - // Nothing to do - return Ok(()); - }; - - db.crdt_operation() - .create_many(vec![crdt_op_unchecked_db(&sync.shared_create( - prisma_sync::volume::SyncId { - pub_id: volume.pub_id, - }, - chain_optional_iter( - [ - sync_entry!(volume.name, volume::name), - sync_entry!(volume.mount_type, volume::mount_type), - sync_entry!(volume.mount_point, volume::mount_point), - sync_entry!(volume.is_mounted, volume::is_mounted), - sync_entry!(volume.disk_type, volume::disk_type), - sync_entry!(volume.file_system, volume::file_system), - sync_entry!(volume.read_only, volume::read_only), - sync_entry!(volume.error_status, volume::error_status), - sync_entry!(volume.total_bytes_capacity, volume::total_bytes_capacity), - sync_entry!(volume.total_bytes_available, volume::total_bytes_available), - ], - [option_sync_entry!( - volume.device.map(|device| { - prisma_sync::device::SyncId { - pub_id: device.pub_id, - } - }), - volume::device - )], - ), - ))?]) - .exec() - .await?; - - Ok(()) -} - -async fn paginate( - getter: impl Fn(i32) -> GetterFut + Send, - id: impl Fn(&T) -> i32 + Send, - operations: impl Fn(Vec) -> Result + Send, -) -> Result<(), Error> -where - T: Send, - E1: Send, - E2: Send, - E3: Send, - Error: From + From + From + Send, - GetterFut: Future, E1>> + Send, - OperationsFut: Future> + Send, -{ - let mut next_cursor = Some(-1); - loop { - let Some(cursor) = next_cursor else { - break; - }; - - let items = getter(cursor).await?; - next_cursor = items.last().map(&id); - operations(items)?.await?; - } - - Ok(()) -} - -async fn paginate_relation( - getter: impl Fn(i32, i32) -> GetterFut + Send, - id: impl Fn(&T) -> (i32, i32) + Send, - operations: impl Fn(Vec) -> Result + Send, -) -> Result<(), Error> -where - T: Send, - E1: Send, - E2: Send, - E3: Send, - Error: From + From + From + Send, - GetterFut: Future, E1>> + Send, - OperationsFut: Future> + Send, -{ - let mut next_cursor = Some((-1, -1)); - loop { - let Some(cursor) = next_cursor else { - break; - }; - - let items = getter(cursor.0, cursor.1).await?; - next_cursor = items.last().map(&id); - operations(items)?.await?; - } - - Ok(()) -} - -#[instrument(skip(db, sync), err)] -async fn paginate_tags(db: &PrismaClient, sync: &SyncManager) -> Result<(), Error> { - paginate( - |cursor| { - db.tag() - .find_many(vec![tag::id::gt(cursor)]) - .order_by(tag::id::order(SortOrder::Asc)) - .exec() - }, - |tag| tag.id, - |tags| { - tags.into_iter() - .map(|t| { - sync.shared_create( - prisma_sync::tag::SyncId { pub_id: t.pub_id }, - chain_optional_iter( - [], - [ - option_sync_entry!(t.name, tag::name), - option_sync_entry!(t.color, tag::color), - option_sync_entry!(t.date_created, tag::date_created), - option_sync_entry!(t.date_modified, tag::date_modified), - ], - ), - ) - }) - .map(|o| crdt_op_unchecked_db(&o)) - .collect::, _>>() - .map(|creates| db.crdt_operation().create_many(creates).exec()) - }, - ) - .await -} - -#[instrument(skip(db, sync), err)] -async fn paginate_locations( - db: &PrismaClient, - sync: &SyncManager, - device_id: device::id::Type, -) -> Result<(), Error> { - paginate( - |cursor| { - db.location() - .find_many(vec![ - location::id::gt(cursor), - location::device_id::equals(Some(device_id)), - ]) - .order_by(location::id::order(SortOrder::Asc)) - .take(1000) - .include(location::include!({ - device: select { pub_id } - })) - .exec() - }, - |location| location.id, - |locations| { - locations - .into_iter() - .map(|l| { - sync.shared_create( - prisma_sync::location::SyncId { pub_id: l.pub_id }, - chain_optional_iter( - [], - [ - option_sync_entry!(l.name, location::name), - option_sync_entry!(l.path, location::path), - option_sync_entry!(l.total_capacity, location::total_capacity), - option_sync_entry!( - l.available_capacity, - location::available_capacity - ), - option_sync_entry!(l.size_in_bytes, location::size_in_bytes), - option_sync_entry!(l.is_archived, location::is_archived), - option_sync_entry!( - l.generate_preview_media, - location::generate_preview_media - ), - option_sync_entry!( - l.sync_preview_media, - location::sync_preview_media - ), - option_sync_entry!(l.hidden, location::hidden), - option_sync_entry!(l.date_created, location::date_created), - option_sync_entry!( - l.device.map(|device| { - prisma_sync::device::SyncId { - pub_id: device.pub_id, - } - }), - location::device - ), - ], - ), - ) - }) - .map(|o| crdt_op_unchecked_db(&o)) - .collect::, _>>() - .map(|creates| db.crdt_operation().create_many(creates).exec()) - }, - ) - .await -} - -#[instrument(skip(db, sync), err)] -async fn paginate_objects( - db: &PrismaClient, - sync: &SyncManager, - device_id: device::id::Type, -) -> Result<(), Error> { - paginate( - |cursor| { - db.object() - .find_many(vec![ - object::id::gt(cursor), - object::device_id::equals(Some(device_id)), - ]) - .order_by(object::id::order(SortOrder::Asc)) - .take(1000) - .include(object::include!({ - device: select { pub_id } - })) - .exec() - }, - |object| object.id, - |objects| { - objects - .into_iter() - .map(|o| { - sync.shared_create( - prisma_sync::object::SyncId { pub_id: o.pub_id }, - chain_optional_iter( - [], - [ - option_sync_entry!(o.kind, object::kind), - option_sync_entry!(o.hidden, object::hidden), - option_sync_entry!(o.favorite, object::favorite), - option_sync_entry!(o.important, object::important), - option_sync_entry!(o.note, object::note), - option_sync_entry!(o.date_created, object::date_created), - option_sync_entry!(o.date_accessed, object::date_accessed), - option_sync_entry!( - o.device.map(|device| { - prisma_sync::device::SyncId { - pub_id: device.pub_id, - } - }), - object::device - ), - ], - ), - ) - }) - .map(|o| crdt_op_unchecked_db(&o)) - .collect::, _>>() - .map(|creates| db.crdt_operation().create_many(creates).exec()) - }, - ) - .await -} - -#[instrument(skip(db, sync), err)] -async fn paginate_exif_datas( - db: &PrismaClient, - sync: &SyncManager, - device_id: device::id::Type, -) -> Result<(), Error> { - paginate( - |cursor| { - db.exif_data() - .find_many(vec![ - exif_data::id::gt(cursor), - exif_data::device_id::equals(Some(device_id)), - ]) - .order_by(exif_data::id::order(SortOrder::Asc)) - .take(1000) - .include(exif_data::include!({ - object: select { pub_id } - device: select { pub_id } - })) - .exec() - }, - |ed| ed.id, - |exif_datas| { - exif_datas - .into_iter() - .map(|ed| { - sync.shared_create( - prisma_sync::exif_data::SyncId { - object: prisma_sync::object::SyncId { - pub_id: ed.object.pub_id, - }, - }, - chain_optional_iter( - [], - [ - option_sync_entry!(ed.resolution, exif_data::resolution), - option_sync_entry!(ed.media_date, exif_data::media_date), - option_sync_entry!(ed.media_location, exif_data::media_location), - option_sync_entry!(ed.camera_data, exif_data::camera_data), - option_sync_entry!(ed.artist, exif_data::artist), - option_sync_entry!(ed.description, exif_data::description), - option_sync_entry!(ed.copyright, exif_data::copyright), - option_sync_entry!(ed.exif_version, exif_data::exif_version), - option_sync_entry!(ed.epoch_time, exif_data::epoch_time), - option_sync_entry!( - ed.device.map(|device| { - prisma_sync::device::SyncId { - pub_id: device.pub_id, - } - }), - exif_data::device - ), - ], - ), - ) - }) - .map(|o| crdt_op_unchecked_db(&o)) - .collect::, _>>() - .map(|creates| db.crdt_operation().create_many(creates).exec()) - }, - ) - .await -} - -#[instrument(skip(db, sync), err)] -async fn paginate_file_paths( - db: &PrismaClient, - sync: &SyncManager, - device_id: device::id::Type, -) -> Result<(), Error> { - paginate( - |cursor| { - db.file_path() - .find_many(vec![ - file_path::id::gt(cursor), - file_path::device_id::equals(Some(device_id)), - ]) - .order_by(file_path::id::order(SortOrder::Asc)) - .include(file_path::include!({ - location: select { pub_id } - object: select { pub_id } - device: select { pub_id } - })) - .exec() - }, - |o| o.id, - |file_paths| { - file_paths - .into_iter() - .map(|fp| { - sync.shared_create( - prisma_sync::file_path::SyncId { pub_id: fp.pub_id }, - chain_optional_iter( - [], - [ - option_sync_entry!(fp.is_dir, file_path::is_dir), - option_sync_entry!(fp.cas_id, file_path::cas_id), - option_sync_entry!( - fp.integrity_checksum, - file_path::integrity_checksum - ), - option_sync_entry!( - fp.location.map(|l| { - prisma_sync::location::SyncId { pub_id: l.pub_id } - }), - file_path::location - ), - option_sync_entry!( - fp.object.map(|o| { - prisma_sync::object::SyncId { pub_id: o.pub_id } - }), - file_path::object - ), - option_sync_entry!( - fp.materialized_path, - file_path::materialized_path - ), - option_sync_entry!(fp.name, file_path::name), - option_sync_entry!(fp.extension, file_path::extension), - option_sync_entry!(fp.hidden, file_path::hidden), - option_sync_entry!( - fp.size_in_bytes_bytes, - file_path::size_in_bytes_bytes - ), - option_sync_entry!(fp.inode, file_path::inode), - option_sync_entry!(fp.date_created, file_path::date_created), - option_sync_entry!(fp.date_modified, file_path::date_modified), - option_sync_entry!(fp.date_indexed, file_path::date_indexed), - option_sync_entry!( - fp.device.map(|device| { - prisma_sync::device::SyncId { - pub_id: device.pub_id, - } - }), - file_path::device - ), - ], - ), - ) - }) - .map(|o| crdt_op_unchecked_db(&o)) - .collect::, _>>() - .map(|creates| db.crdt_operation().create_many(creates).exec()) - }, - ) - .await -} - -#[instrument(skip(db, sync), err)] -async fn paginate_tags_on_objects( - db: &PrismaClient, - sync: &SyncManager, - device_id: device::id::Type, -) -> Result<(), Error> { - paginate_relation( - |group_id, item_id| { - db.tag_on_object() - .find_many(vec![ - tag_on_object::tag_id::gt(group_id), - tag_on_object::object_id::gt(item_id), - tag_on_object::device_id::equals(Some(device_id)), - ]) - .order_by(tag_on_object::tag_id::order(SortOrder::Asc)) - .order_by(tag_on_object::object_id::order(SortOrder::Asc)) - .include(tag_on_object::include!({ - tag: select { pub_id } - object: select { pub_id } - device: select { pub_id } - })) - .exec() - }, - |t_o| (t_o.tag_id, t_o.object_id), - |tag_on_objects| { - tag_on_objects - .into_iter() - .map(|t_o| { - sync.relation_create( - prisma_sync::tag_on_object::SyncId { - tag: prisma_sync::tag::SyncId { - pub_id: t_o.tag.pub_id, - }, - object: prisma_sync::object::SyncId { - pub_id: t_o.object.pub_id, - }, - }, - chain_optional_iter( - [], - [ - option_sync_entry!(t_o.date_created, tag_on_object::date_created), - option_sync_entry!( - t_o.device.map(|device| { - prisma_sync::device::SyncId { - pub_id: device.pub_id, - } - }), - tag_on_object::device - ), - ], - ), - ) - }) - .map(|o| crdt_op_unchecked_db(&o)) - .collect::, _>>() - .map(|creates| db.crdt_operation().create_many(creates).exec()) - }, - ) - .await -} - -#[instrument(skip(db, sync), err)] -async fn paginate_labels(db: &PrismaClient, sync: &SyncManager) -> Result<(), Error> { - paginate( - |cursor| { - db.label() - .find_many(vec![label::id::gt(cursor)]) - .order_by(label::id::order(SortOrder::Asc)) - .exec() - }, - |label| label.id, - |labels| { - labels - .into_iter() - .map(|l| { - sync.shared_create( - prisma_sync::label::SyncId { name: l.name }, - chain_optional_iter( - [], - [ - option_sync_entry!(l.date_created, label::date_created), - option_sync_entry!(l.date_modified, label::date_modified), - ], - ), - ) - }) - .map(|o| crdt_op_unchecked_db(&o)) - .collect::, _>>() - .map(|creates| db.crdt_operation().create_many(creates).exec()) - }, - ) - .await -} - -#[instrument(skip(db, sync), err)] -async fn paginate_labels_on_objects( - db: &PrismaClient, - sync: &SyncManager, - device_id: device::id::Type, -) -> Result<(), Error> { - paginate_relation( - |group_id, item_id| { - db.label_on_object() - .find_many(vec![ - label_on_object::label_id::gt(group_id), - label_on_object::object_id::gt(item_id), - label_on_object::device_id::equals(Some(device_id)), - ]) - .order_by(label_on_object::label_id::order(SortOrder::Asc)) - .order_by(label_on_object::object_id::order(SortOrder::Asc)) - .include(label_on_object::include!({ - object: select { pub_id } - label: select { name } - device: select { pub_id } - })) - .exec() - }, - |l_o| (l_o.label_id, l_o.object_id), - |label_on_objects| { - label_on_objects - .into_iter() - .map(|l_o| { - sync.relation_create( - prisma_sync::label_on_object::SyncId { - label: prisma_sync::label::SyncId { - name: l_o.label.name, - }, - object: prisma_sync::object::SyncId { - pub_id: l_o.object.pub_id, - }, - }, - chain_optional_iter( - [sync_entry!(l_o.date_created, label_on_object::date_created)], - [option_sync_entry!( - l_o.device.map(|device| { - prisma_sync::device::SyncId { - pub_id: device.pub_id, - } - }), - label_on_object::device - )], - ), - ) - }) - .map(|o| crdt_op_unchecked_db(&o)) - .collect::, _>>() - .map(|creates| db.crdt_operation().create_many(creates).exec()) - }, - ) - .await -} diff --git a/core/crates/sync/src/db_operation.rs b/core/crates/sync/src/db_operation.rs deleted file mode 100644 index 1bdb2422c..000000000 --- a/core/crates/sync/src/db_operation.rs +++ /dev/null @@ -1,103 +0,0 @@ -use sd_core_prisma_helpers::DevicePubId; - -use sd_prisma::prisma::{cloud_crdt_operation, crdt_operation, PrismaClient}; -use sd_sync::CRDTOperation; -use sd_utils::uuid_to_bytes; - -use tracing::instrument; -use uhlc::NTP64; - -use super::Error; - -#[instrument(skip(op, db), err)] -pub async fn write_crdt_op_to_db(op: &CRDTOperation, db: &PrismaClient) -> Result<(), Error> { - crdt_operation::Create { - timestamp: { - #[allow(clippy::cast_possible_wrap)] - // SAFETY: we have to store using i64 due to SQLite limitations - { - op.timestamp.0 as i64 - } - }, - device_pub_id: uuid_to_bytes(&op.device_pub_id), - kind: op.kind().to_string(), - data: rmp_serde::to_vec(&op.data)?, - model: i32::from(op.model_id), - record_id: rmp_serde::to_vec(&op.record_id)?, - _params: vec![], - } - .to_query(db) - .select(crdt_operation::select!({ id })) // To don't fetch the whole object for nothing - .exec() - .await?; - - Ok(()) -} - -pub fn from_crdt_ops( - crdt_operation::Data { - timestamp, - model, - record_id, - data, - device_pub_id, - .. - }: crdt_operation::Data, -) -> Result { - Ok(CRDTOperation { - device_pub_id: DevicePubId::from(device_pub_id).into(), - timestamp: { - #[allow(clippy::cast_sign_loss)] - { - // SAFETY: we had to store using i64 due to SQLite limitations - NTP64(timestamp as u64) - } - }, - model_id: { - #[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)] - { - // SAFETY: we will not have more than 2^16 models and we had to store using signed - // integers due to SQLite limitations - model as u16 - } - }, - record_id: rmp_serde::from_slice(&record_id)?, - data: rmp_serde::from_slice(&data)?, - }) -} - -pub fn from_cloud_crdt_ops( - cloud_crdt_operation::Data { - id, - timestamp, - model, - record_id, - data, - device_pub_id, - .. - }: cloud_crdt_operation::Data, -) -> Result<(cloud_crdt_operation::id::Type, CRDTOperation), Error> { - Ok(( - id, - CRDTOperation { - device_pub_id: DevicePubId::from(device_pub_id).into(), - timestamp: { - #[allow(clippy::cast_sign_loss)] - { - // SAFETY: we had to store using i64 due to SQLite limitations - NTP64(timestamp as u64) - } - }, - model_id: { - #[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)] - { - // SAFETY: we will not have more than 2^16 models and we had to store using signed - // integers due to SQLite limitations - model as u16 - } - }, - record_id: rmp_serde::from_slice(&record_id)?, - data: rmp_serde::from_slice(&data)?, - }, - )) -} diff --git a/core/crates/sync/src/ingest_utils.rs b/core/crates/sync/src/ingest_utils.rs deleted file mode 100644 index 5f60ccfdf..000000000 --- a/core/crates/sync/src/ingest_utils.rs +++ /dev/null @@ -1,517 +0,0 @@ -use sd_core_prisma_helpers::DevicePubId; - -use sd_prisma::{ - prisma::{crdt_operation, PrismaClient}, - prisma_sync::ModelSyncData, -}; -use sd_sync::{ - CRDTOperation, CRDTOperationData, CompressedCRDTOperation, ModelId, OperationKind, RecordId, -}; - -use std::{collections::BTreeMap, num::NonZeroU128, sync::Arc}; - -use futures_concurrency::future::TryJoin; -use tokio::sync::Mutex; -use tracing::{debug, instrument, trace, warn}; -use uhlc::{Timestamp, HLC, NTP64}; -use uuid::Uuid; - -use super::{db_operation::write_crdt_op_to_db, Error, TimestampPerDevice}; - -crdt_operation::select!(crdt_operation_id { id }); - -// where the magic happens -#[instrument(skip(clock, ops), fields(operations_count = %ops.len()), err)] -pub async fn process_crdt_operations( - clock: &HLC, - timestamp_per_device: &TimestampPerDevice, - sync_lock: Arc>, - db: &PrismaClient, - device_pub_id: DevicePubId, - model_id: ModelId, - (record_id, mut ops): (RecordId, Vec), -) -> Result<(), Error> { - ops.sort_by_key(|op| op.timestamp); - - let new_timestamp = ops.last().expect("Empty ops array").timestamp; - - update_clock(clock, new_timestamp, &device_pub_id); - - // Delete - ignores all other messages - if let Some(delete_op) = ops - .iter() - .rev() - .find(|op| matches!(op.data, CRDTOperationData::Delete)) - { - trace!("Deleting operation"); - handle_crdt_deletion( - db, - &sync_lock, - &device_pub_id, - model_id, - record_id, - delete_op, - ) - .await?; - } - // Create + > 0 Update - overwrites the create's data with the updates - else if let Some(timestamp) = ops - .iter() - .rev() - .find_map(|op| matches!(&op.data, CRDTOperationData::Create(_)).then_some(op.timestamp)) - { - trace!("Create + Updates operations"); - - // conflict resolution - let delete_count = db - .crdt_operation() - .count(vec![ - crdt_operation::model::equals(i32::from(model_id)), - crdt_operation::record_id::equals(rmp_serde::to_vec(&record_id)?), - crdt_operation::kind::equals(OperationKind::Delete.to_string()), - ]) - .exec() - .await?; - - if delete_count > 0 { - debug!("Found a previous delete operation with the same SyncId, will ignore these operations"); - return Ok(()); - } - - handle_crdt_create_and_updates( - db, - &sync_lock, - &device_pub_id, - model_id, - record_id, - ops, - timestamp, - ) - .await?; - } - // > 0 Update - batches updates with a fake Create op - else { - trace!("Updates operation"); - - let mut data = BTreeMap::new(); - - for op in ops.into_iter().rev() { - let CRDTOperationData::Update(fields_and_values) = op.data else { - unreachable!("Create + Delete should be filtered out!"); - }; - - for (field, value) in fields_and_values { - data.insert(field, (value, op.timestamp)); - } - } - - let earlier_time = data.values().fold( - NTP64(u64::from(u32::MAX)), - |earlier_time, (_, timestamp)| { - if timestamp.0 < earlier_time.0 { - *timestamp - } else { - earlier_time - } - }, - ); - - // conflict resolution - let (create, possible_newer_updates_count) = db - ._batch(( - db.crdt_operation().count(vec![ - crdt_operation::model::equals(i32::from(model_id)), - crdt_operation::record_id::equals(rmp_serde::to_vec(&record_id)?), - crdt_operation::kind::equals(OperationKind::Create.to_string()), - ]), - // Fetching all update operations newer than our current earlier timestamp - db.crdt_operation() - .find_many(vec![ - crdt_operation::timestamp::gt({ - #[allow(clippy::cast_possible_wrap)] - // SAFETY: we had to store using i64 due to SQLite limitations - { - earlier_time.as_u64() as i64 - } - }), - crdt_operation::model::equals(i32::from(model_id)), - crdt_operation::record_id::equals(rmp_serde::to_vec(&record_id)?), - crdt_operation::kind::starts_with("u".to_string()), - ]) - .select(crdt_operation::select!({ kind timestamp })), - )) - .await?; - - if create == 0 { - warn!("Failed to find a previous create operation with the same SyncId"); - return Ok(()); - } - - for candidate in possible_newer_updates_count { - // The first element is "u" meaning that this is an update, so we skip it - for key in candidate - .kind - .split(':') - .filter(|field| !field.is_empty()) - .skip(1) - { - // remove entries if we possess locally more recent updates for this field - if data.get(key).is_some_and(|(_, new_timestamp)| { - #[allow(clippy::cast_sign_loss)] - { - // we need to store as i64 due to SQLite limitations - *new_timestamp < NTP64(candidate.timestamp as u64) - } - }) { - data.remove(key); - } - } - - if data.is_empty() { - break; - } - } - - handle_crdt_updates(db, &sync_lock, &device_pub_id, model_id, record_id, data).await?; - } - - update_timestamp_per_device(timestamp_per_device, device_pub_id, new_timestamp).await; - - Ok(()) -} - -pub async fn bulk_ingest_create_only_ops( - clock: &HLC, - timestamp_per_device: &TimestampPerDevice, - db: &PrismaClient, - device_pub_id: DevicePubId, - model_id: ModelId, - ops: Vec<(RecordId, CompressedCRDTOperation)>, - sync_lock: Arc>, -) -> Result<(), Error> { - let latest_timestamp = ops.iter().fold(NTP64(0), |latest, (_, op)| { - if latest < op.timestamp { - op.timestamp - } else { - latest - } - }); - - update_clock(clock, latest_timestamp, &device_pub_id); - - let ops = ops - .into_iter() - .map(|(record_id, op)| { - rmp_serde::to_vec(&record_id) - .map(|serialized_record_id| (record_id, serialized_record_id, op)) - }) - .collect::, _>>()?; - - // conflict resolution - let delete_counts = db - ._batch( - ops.iter() - .map(|(_, serialized_record_id, _)| { - db.crdt_operation().count(vec![ - crdt_operation::model::equals(i32::from(model_id)), - crdt_operation::record_id::equals(serialized_record_id.clone()), - crdt_operation::kind::equals(OperationKind::Delete.to_string()), - ]) - }) - .collect::>(), - ) - .await?; - - let lock_guard = sync_lock.lock().await; - - db._transaction() - .with_timeout(30 * 10000) - .with_max_wait(30 * 10000) - .run(|db| { - let device_pub_id = device_pub_id.clone(); - - async move { - // complying with borrowck - let device_pub_id = &device_pub_id; - - let (crdt_creates, model_sync_data) = ops - .into_iter() - .zip(delete_counts) - .filter_map(|(data, delete_count)| (delete_count == 0).then_some(data)) - .map( - |( - record_id, - serialized_record_id, - CompressedCRDTOperation { timestamp, data }, - )| { - let crdt_create = crdt_operation::CreateUnchecked { - timestamp: { - #[allow(clippy::cast_possible_wrap)] - // SAFETY: we have to store using i64 due to SQLite limitations - { - timestamp.0 as i64 - } - }, - model: i32::from(model_id), - record_id: serialized_record_id, - kind: "c".to_string(), - data: rmp_serde::to_vec(&data)?, - device_pub_id: device_pub_id.to_db(), - _params: vec![], - }; - - // NOTE(@fogodev): I wish I could do a create many here instead of creating separately each - // entry, but it's not supported by PCR - let model_sync_data = ModelSyncData::from_op(CRDTOperation { - device_pub_id: Uuid::from(device_pub_id), - model_id, - record_id, - timestamp, - data, - })? - .exec(&db); - - Ok::<_, Error>((crdt_create, model_sync_data)) - }, - ) - .collect::, _>>()? - .into_iter() - .unzip::<_, _, Vec<_>, Vec<_>>(); - - model_sync_data.try_join().await?; - - db.crdt_operation().create_many(crdt_creates).exec().await?; - - Ok::<_, Error>(()) - } - }) - .await?; - - drop(lock_guard); - - update_timestamp_per_device(timestamp_per_device, device_pub_id, latest_timestamp).await; - - Ok(()) -} - -#[instrument(skip_all, err)] -async fn handle_crdt_updates( - db: &PrismaClient, - sync_lock: &Mutex<()>, - device_pub_id: &DevicePubId, - model_id: ModelId, - record_id: rmpv::Value, - data: BTreeMap, -) -> Result<(), Error> { - let device_pub_id = sd_sync::DevicePubId::from(device_pub_id); - - let _lock_guard = sync_lock.lock().await; - - db._transaction() - .with_timeout(30 * 10000) - .with_max_wait(30 * 10000) - .run(|db| async move { - // fake operation to batch them all at once, inserting the latest data on appropriate table - ModelSyncData::from_op(CRDTOperation { - device_pub_id, - model_id, - record_id: record_id.clone(), - timestamp: NTP64(0), - data: CRDTOperationData::Create( - data.iter() - .map(|(k, (data, _))| (k.clone(), data.clone())) - .collect(), - ), - })? - .exec(&db) - .await?; - - let (fields_and_values, latest_timestamp) = data.into_iter().fold( - (BTreeMap::new(), NTP64::default()), - |(mut fields_and_values, mut latest_time_stamp), (field, (value, timestamp))| { - fields_and_values.insert(field, value); - if timestamp > latest_time_stamp { - latest_time_stamp = timestamp; - } - (fields_and_values, latest_time_stamp) - }, - ); - - write_crdt_op_to_db( - &CRDTOperation { - device_pub_id, - model_id, - record_id, - timestamp: latest_timestamp, - data: CRDTOperationData::Update(fields_and_values), - }, - &db, - ) - .await - }) - .await -} - -#[instrument(skip_all, err)] -async fn handle_crdt_create_and_updates( - db: &PrismaClient, - sync_lock: &Mutex<()>, - device_pub_id: &DevicePubId, - model_id: ModelId, - record_id: rmpv::Value, - ops: Vec, - timestamp: NTP64, -) -> Result<(), Error> { - let mut data = BTreeMap::new(); - let device_pub_id = sd_sync::DevicePubId::from(device_pub_id); - - let mut applied_ops = vec![]; - - // search for all Updates until a Create is found - for op in ops.into_iter().rev() { - match &op.data { - CRDTOperationData::Delete => unreachable!("Delete can't exist here!"), - CRDTOperationData::Create(create_data) => { - for (k, v) in create_data { - data.entry(k.clone()).or_insert_with(|| v.clone()); - } - - applied_ops.push(op); - - break; - } - CRDTOperationData::Update(fields_and_values) => { - for (field, value) in fields_and_values { - data.insert(field.clone(), value.clone()); - } - - applied_ops.push(op); - } - } - } - - let _lock_guard = sync_lock.lock().await; - - db._transaction() - .with_timeout(30 * 10000) - .with_max_wait(30 * 10000) - .run(|db| async move { - // fake a create with a bunch of data rather than individual insert - ModelSyncData::from_op(CRDTOperation { - device_pub_id, - model_id, - record_id: record_id.clone(), - timestamp, - data: CRDTOperationData::Create(data), - })? - .exec(&db) - .await?; - - applied_ops - .into_iter() - .map(|CompressedCRDTOperation { timestamp, data }| { - let record_id = record_id.clone(); - let db = &db; - async move { - write_crdt_op_to_db( - &CRDTOperation { - device_pub_id, - timestamp, - model_id, - record_id, - data, - }, - db, - ) - .await - } - }) - .collect::>() - .try_join() - .await - .map(|_| ()) - }) - .await -} - -#[instrument(skip_all, err)] -async fn handle_crdt_deletion( - db: &PrismaClient, - sync_lock: &Mutex<()>, - device_pub_id: &DevicePubId, - model: u16, - record_id: rmpv::Value, - delete_op: &CompressedCRDTOperation, -) -> Result<(), Error> { - // deletes are the be all and end all, except if we never created the object to begin with - // in this case we don't need to delete anything - - if db - .crdt_operation() - .count(vec![ - crdt_operation::model::equals(i32::from(model)), - crdt_operation::record_id::equals(rmp_serde::to_vec(&record_id)?), - ]) - .exec() - .await? - == 0 - { - // This means that in the other device this entry was created and deleted, before this - // device here could even take notice of it. So we don't need to do anything here. - return Ok(()); - } - - let op = CRDTOperation { - device_pub_id: device_pub_id.into(), - model_id: model, - record_id, - timestamp: delete_op.timestamp, - data: CRDTOperationData::Delete, - }; - - let _lock_guard = sync_lock.lock().await; - - db._transaction() - .with_timeout(30 * 10000) - .with_max_wait(30 * 10000) - .run(|db| async move { - ModelSyncData::from_op(op.clone())?.exec(&db).await?; - - write_crdt_op_to_db(&op, &db).await - }) - .await -} - -fn update_clock(clock: &HLC, latest_timestamp: NTP64, device_pub_id: &DevicePubId) { - // first, we update the HLC's timestamp with the incoming one. - // this involves a drift check + sets the last time of the clock - clock - .update_with_timestamp(&Timestamp::new( - latest_timestamp, - uhlc::ID::from( - NonZeroU128::new(Uuid::from(device_pub_id).to_u128_le()).expect("Non zero id"), - ), - )) - .expect("timestamp has too much drift!"); -} - -async fn update_timestamp_per_device( - timestamp_per_device: &TimestampPerDevice, - device_pub_id: DevicePubId, - latest_timestamp: NTP64, -) { - // read the timestamp for the operation's device, or insert one if it doesn't exist - let current_last_timestamp = timestamp_per_device - .read() - .await - .get(&device_pub_id) - .copied(); - - // update the stored timestamp for this device - will be derived from the crdt operations table on restart - let new_ts = NTP64::max(current_last_timestamp.unwrap_or_default(), latest_timestamp); - - timestamp_per_device - .write() - .await - .insert(device_pub_id, new_ts); -} diff --git a/core/crates/sync/src/lib.rs b/core/crates/sync/src/lib.rs deleted file mode 100644 index 5b8d90efe..000000000 --- a/core/crates/sync/src/lib.rs +++ /dev/null @@ -1,153 +0,0 @@ -#![warn( - clippy::all, - clippy::pedantic, - clippy::correctness, - clippy::perf, - clippy::style, - clippy::suspicious, - clippy::complexity, - clippy::nursery, - clippy::unwrap_used, - unused_qualifications, - rust_2018_idioms, - trivial_casts, - trivial_numeric_casts, - unused_allocation, - clippy::unnecessary_cast, - clippy::cast_lossless, - clippy::cast_possible_truncation, - clippy::cast_possible_wrap, - clippy::cast_precision_loss, - clippy::cast_sign_loss, - clippy::dbg_macro, - clippy::deprecated_cfg_attr, - clippy::separated_literal_suffix, - deprecated -)] -#![forbid(deprecated_in_future)] -#![allow(clippy::missing_errors_doc, clippy::module_name_repetitions)] - -use sd_prisma::{ - prisma::{cloud_crdt_operation, crdt_operation}, - prisma_sync, -}; -use sd_utils::uuid_to_bytes; - -use std::{collections::HashMap, sync::Arc}; - -use tokio::{sync::RwLock, task::JoinError}; - -pub mod backfill; -mod db_operation; -mod ingest_utils; -mod manager; - -pub use db_operation::{from_cloud_crdt_ops, from_crdt_ops, write_crdt_op_to_db}; -pub use manager::Manager as SyncManager; -pub use uhlc::NTP64; - -#[derive(Clone, Debug)] -pub enum SyncEvent { - Ingested, - Created, -} - -pub use sd_core_prisma_helpers::DevicePubId; -pub use sd_sync::{ - CRDTOperation, CompressedCRDTOperation, CompressedCRDTOperationsPerModel, - CompressedCRDTOperationsPerModelPerDevice, ModelId, OperationFactory, RecordId, RelationSyncId, - RelationSyncModel, SharedSyncModel, SyncId, SyncModel, -}; - -pub type TimestampPerDevice = Arc>>; - -#[derive(thiserror::Error, Debug)] -pub enum Error { - #[error("serialization error: {0}")] - Serialization(#[from] rmp_serde::encode::Error), - #[error("deserialization error: {0}")] - Deserialization(#[from] rmp_serde::decode::Error), - #[error("database error: {0}")] - Database(#[from] prisma_client_rust::QueryError), - #[error("PrismaSync error: {0}")] - PrismaSync(#[from] prisma_sync::Error), - #[error("invalid model id: {0}")] - InvalidModelId(ModelId), - #[error("tried to write an empty operations list")] - EmptyOperations, - #[error("device not found: {0}")] - DeviceNotFound(DevicePubId), - #[error("processes crdt task panicked")] - ProcessCrdtPanic(JoinError), -} - -impl From for rspc::Error { - fn from(e: Error) -> Self { - match e { - Error::Database(e) => e.into(), - Error::InvalidModelId(id) => Self::new( - rspc::ErrorCode::BadRequest, - format!("Invalid model id "), - ), - _ => Self::with_cause( - rspc::ErrorCode::InternalServerError, - "Internal sync error".to_string(), - e, - ), - } - } -} - -pub fn crdt_op_db(op: &CRDTOperation) -> Result { - Ok(crdt_operation::Create { - timestamp: { - #[allow(clippy::cast_possible_wrap)] - // SAFETY: we had to store using i64 due to SQLite limitations - { - op.timestamp.as_u64() as i64 - } - }, - device_pub_id: uuid_to_bytes(&op.device_pub_id), - kind: op.kind().to_string(), - data: rmp_serde::to_vec(&op.data)?, - model: i32::from(op.model_id), - record_id: rmp_serde::to_vec(&op.record_id)?, - _params: vec![], - }) -} - -pub fn crdt_op_unchecked_db(op: &CRDTOperation) -> Result { - Ok(crdt_operation::CreateUnchecked { - timestamp: { - #[allow(clippy::cast_possible_wrap)] - // SAFETY: we had to store using i64 due to SQLite limitations - { - op.timestamp.as_u64() as i64 - } - }, - device_pub_id: uuid_to_bytes(&op.device_pub_id), - kind: op.kind().to_string(), - data: rmp_serde::to_vec(&op.data)?, - model: i32::from(op.model_id), - record_id: rmp_serde::to_vec(&op.record_id)?, - _params: vec![], - }) -} - -pub fn cloud_crdt_op_db(op: &CRDTOperation) -> Result { - Ok(cloud_crdt_operation::Create { - timestamp: { - #[allow(clippy::cast_possible_wrap)] - // SAFETY: we had to store using i64 due to SQLite limitations - { - op.timestamp.as_u64() as i64 - } - }, - device_pub_id: uuid_to_bytes(&op.device_pub_id), - kind: op.data.as_kind().to_string(), - data: rmp_serde::to_vec(&op.data)?, - model: i32::from(op.model_id), - record_id: rmp_serde::to_vec(&op.record_id)?, - _params: vec![], - }) -} diff --git a/core/crates/sync/src/manager.rs b/core/crates/sync/src/manager.rs deleted file mode 100644 index e2f27335e..000000000 --- a/core/crates/sync/src/manager.rs +++ /dev/null @@ -1,750 +0,0 @@ -use sd_core_prisma_helpers::DevicePubId; - -use sd_prisma::{ - prisma::{cloud_crdt_operation, crdt_operation, device, PrismaClient, SortOrder}, - prisma_sync, -}; -use sd_sync::{ - CRDTOperation, CRDTOperationData, CompressedCRDTOperation, ModelId, OperationFactory, RecordId, -}; -use sd_utils::timestamp_to_datetime; - -use std::{ - collections::{hash_map::Entry, BTreeMap, HashMap}, - fmt, mem, - num::NonZeroU128, - sync::{ - atomic::{self, AtomicBool}, - Arc, - }, - time::{Duration, SystemTime}, -}; - -use async_stream::stream; -use futures::{stream::FuturesUnordered, Stream, TryStreamExt}; -use futures_concurrency::future::TryJoin; -use itertools::Itertools; -use tokio::{ - spawn, - sync::{broadcast, Mutex, Notify, RwLock}, - time::Instant, -}; -use tracing::{debug, instrument, warn}; -use uhlc::{HLCBuilder, HLC}; -use uuid::Uuid; - -use super::{ - crdt_op_db, - db_operation::{from_cloud_crdt_ops, from_crdt_ops}, - ingest_utils::{bulk_ingest_create_only_ops, process_crdt_operations}, - Error, SyncEvent, TimestampPerDevice, NTP64, -}; - -const INGESTION_BATCH_SIZE: i64 = 10_000; - -/// Wrapper that spawns the ingest actor and provides utilities for reading and writing sync operations. -#[derive(Clone)] -pub struct Manager { - pub tx: broadcast::Sender, - pub db: Arc, - pub emit_messages_flag: Arc, - pub device_pub_id: DevicePubId, - pub timestamp_per_device: TimestampPerDevice, - pub clock: Arc, - pub active: Arc, - pub active_notify: Arc, - pub(crate) sync_lock: Arc>, - pub(crate) available_parallelism: usize, -} - -impl fmt::Debug for Manager { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("SyncManager").finish() - } -} - -impl Manager { - /// Creates a new manager that can be used to read and write CRDT operations. - /// Sync messages are received on the returned [`broadcast::Receiver`]. - pub async fn new( - db: Arc, - current_device_pub_id: &DevicePubId, - emit_messages_flag: Arc, - ) -> Result<(Self, broadcast::Receiver), Error> { - let existing_devices = db.device().find_many(vec![]).exec().await?; - - Self::with_existing_devices( - db, - current_device_pub_id, - emit_messages_flag, - &existing_devices, - ) - .await - } - - /// Creates a new manager that can be used to read and write CRDT operations from a list of existing instances. - /// Sync messages are received on the returned [`broadcast::Receiver`]. - /// - /// # Panics - /// Panics if the `current_device_pub_id` UUID is zeroed, which will never happen as we use `UUIDv7` for the - /// device pub id. As this version have a timestamp part, instead of being totally random. So the only - /// possible way to get zero from a `UUIDv7` is to go back in time to 1970 - pub async fn with_existing_devices( - db: Arc, - current_device_pub_id: &DevicePubId, - emit_messages_flag: Arc, - existing_devices: &[device::Data], - ) -> Result<(Self, broadcast::Receiver), Error> { - let latest_timestamp_per_device = db - ._batch( - existing_devices - .iter() - .map(|device| { - db.crdt_operation() - .find_first(vec![crdt_operation::device_pub_id::equals( - device.pub_id.clone(), - )]) - .order_by(crdt_operation::timestamp::order(SortOrder::Desc)) - }) - .collect::>(), - ) - .await? - .into_iter() - .zip(existing_devices) - .map(|(op, device)| { - ( - DevicePubId::from(&device.pub_id), - #[allow(clippy::cast_sign_loss)] - // SAFETY: we had to store using i64 due to SQLite limitations - NTP64(op.map(|o| o.timestamp).unwrap_or_default() as u64), - ) - }) - .collect(); - - let (tx, rx) = broadcast::channel(64); - - Ok(( - Self { - tx, - db, - device_pub_id: current_device_pub_id.clone(), - clock: Arc::new( - HLCBuilder::new() - .with_id(uhlc::ID::from( - NonZeroU128::new(Uuid::from(current_device_pub_id).to_u128_le()) - .expect("Non zero id"), - )) - .build(), - ), - timestamp_per_device: Arc::new(RwLock::new(latest_timestamp_per_device)), - emit_messages_flag, - active: Arc::default(), - active_notify: Arc::default(), - sync_lock: Arc::new(Mutex::default()), - available_parallelism: std::thread::available_parallelism() - .map_or(1, std::num::NonZero::get), - }, - rx, - )) - } - - async fn fetch_cloud_crdt_ops( - &self, - model_id: ModelId, - batch_size: i64, - ) -> Result<(Vec, Vec), Error> { - self.db - .cloud_crdt_operation() - .find_many(vec![cloud_crdt_operation::model::equals(i32::from( - model_id, - ))]) - .take(batch_size) - .order_by(cloud_crdt_operation::timestamp::order(SortOrder::Asc)) - .exec() - .await? - .into_iter() - .map(from_cloud_crdt_ops) - .collect::, Vec<_>), _>>() - } - - #[instrument(skip(self))] - async fn ingest_by_model(&self, model_id: ModelId) -> Result { - let mut total_count = 0; - - let mut buckets = (0..self.available_parallelism) - .map(|_| FuturesUnordered::new()) - .collect::>(); - - let mut total_fetch_time = Duration::ZERO; - let mut total_compression_time = Duration::ZERO; - let mut total_work_distribution_time = Duration::ZERO; - let mut total_process_time = Duration::ZERO; - - loop { - let fetching_start = Instant::now(); - - let (ops_ids, ops) = self - .fetch_cloud_crdt_ops(model_id, INGESTION_BATCH_SIZE) - .await?; - if ops_ids.is_empty() { - break; - } - - total_fetch_time += fetching_start.elapsed(); - - let messages_count = ops.len(); - - debug!( - messages_count, - first_message = ?ops - .first() - .map_or_else(|| SystemTime::UNIX_EPOCH.into(), |op| timestamp_to_datetime(op.timestamp)), - last_message = ?ops - .last() - .map_or_else(|| SystemTime::UNIX_EPOCH.into(), |op| timestamp_to_datetime(op.timestamp)), - "Messages by model to ingest", - ); - - let compression_start = Instant::now(); - - let mut compressed_map = - BTreeMap::, (RecordId, Vec)>>::new(); - - for CRDTOperation { - device_pub_id, - timestamp, - model_id: _, // Ignoring model_id as we know it already - record_id, - data, - } in ops - { - let records = compressed_map.entry(device_pub_id).or_default(); - - // Can't use RecordId as a key because rmpv::Value doesn't implement Hash + Eq. - // So we use it's serialized bytes as a key. - let record_id_bytes = - rmp_serde::to_vec_named(&record_id).expect("already serialized to Value"); - - match records.entry(record_id_bytes) { - Entry::Occupied(mut entry) => { - entry - .get_mut() - .1 - .push(CompressedCRDTOperation { timestamp, data }); - } - Entry::Vacant(entry) => { - entry - .insert((record_id, vec![CompressedCRDTOperation { timestamp, data }])); - } - } - } - - // Now that we separated all operations by their record_ids, we can do an optimization - // to process all records that only posses a single create operation, batching them together - let mut create_only_ops: BTreeMap> = - BTreeMap::new(); - for (device_pub_id, records) in &mut compressed_map { - for (record_id, ops) in records.values_mut() { - if ops.len() == 1 && matches!(ops[0].data, CRDTOperationData::Create(_)) { - create_only_ops - .entry(*device_pub_id) - .or_default() - .push((mem::replace(record_id, rmpv::Value::Nil), ops.remove(0))); - } - } - } - - total_count += bulk_process_of_create_only_ops( - self.available_parallelism, - Arc::clone(&self.clock), - Arc::clone(&self.timestamp_per_device), - Arc::clone(&self.db), - Arc::clone(&self.sync_lock), - model_id, - create_only_ops, - ) - .await?; - - total_compression_time += compression_start.elapsed(); - - let work_distribution_start = Instant::now(); - - compressed_map - .into_iter() - .flat_map(|(device_pub_id, records)| { - records.into_values().filter_map(move |(record_id, ops)| { - if record_id.is_nil() { - return None; - } - - // We can process each record in parallel as they are independent - - let clock = Arc::clone(&self.clock); - let timestamp_per_device = Arc::clone(&self.timestamp_per_device); - let db = Arc::clone(&self.db); - let device_pub_id = device_pub_id.into(); - let sync_lock = Arc::clone(&self.sync_lock); - - Some(async move { - let count = ops.len(); - - process_crdt_operations( - &clock, - ×tamp_per_device, - sync_lock, - &db, - device_pub_id, - model_id, - (record_id, ops), - ) - .await - .map(|()| count) - }) - }) - }) - .enumerate() - .for_each(|(idx, fut)| buckets[idx % self.available_parallelism].push(fut)); - - total_work_distribution_time += work_distribution_start.elapsed(); - - let processing_start = Instant::now(); - - let handles = buckets - .iter_mut() - .enumerate() - .filter(|(_idx, bucket)| !bucket.is_empty()) - .map(|(idx, bucket)| { - let mut bucket = mem::take(bucket); - - spawn(async move { - let mut ops_count = 0; - let processing_start = Instant::now(); - while let Some(count) = bucket.try_next().await? { - ops_count += count; - } - - debug!( - "Ingested {ops_count} operations in {:?}", - processing_start.elapsed() - ); - - Ok::<_, Error>((ops_count, idx, bucket)) - }) - }) - .collect::>(); - - let results = handles.try_join().await.map_err(Error::ProcessCrdtPanic)?; - - total_process_time += processing_start.elapsed(); - - for res in results { - let (count, idx, bucket) = res?; - - buckets[idx] = bucket; - - total_count += count; - } - - self.db - .cloud_crdt_operation() - .delete_many(vec![cloud_crdt_operation::id::in_vec(ops_ids)]) - .exec() - .await?; - } - - debug!( - total_count, - ?total_fetch_time, - ?total_compression_time, - ?total_work_distribution_time, - ?total_process_time, - "Ingested all operations of this model" - ); - - Ok(total_count) - } - - pub async fn ingest_ops(&self) -> Result { - let mut total_count = 0; - - // WARN: this order here exists because sync messages MUST be processed in this exact order - // due to relationship dependencies between these tables. - total_count += self.ingest_by_model(prisma_sync::device::MODEL_ID).await?; - - total_count += [ - self.ingest_by_model(prisma_sync::volume::MODEL_ID), - self.ingest_by_model(prisma_sync::tag::MODEL_ID), - self.ingest_by_model(prisma_sync::location::MODEL_ID), - self.ingest_by_model(prisma_sync::object::MODEL_ID), - self.ingest_by_model(prisma_sync::label::MODEL_ID), - ] - .try_join() - .await? - .into_iter() - .sum::(); - - total_count += [ - self.ingest_by_model(prisma_sync::exif_data::MODEL_ID), - self.ingest_by_model(prisma_sync::file_path::MODEL_ID), - self.ingest_by_model(prisma_sync::tag_on_object::MODEL_ID), - self.ingest_by_model(prisma_sync::label_on_object::MODEL_ID), - ] - .try_join() - .await? - .into_iter() - .sum::(); - - if self.tx.send(SyncEvent::Ingested).is_err() { - warn!("failed to send ingested message on `ingest_ops`"); - } - - Ok(total_count) - } - - #[must_use] - pub fn subscribe(&self) -> broadcast::Receiver { - self.tx.subscribe() - } - - pub async fn write_ops<'item, Q>( - &self, - tx: &PrismaClient, - (ops, queries): (Vec, Q), - ) -> Result - where - Q: prisma_client_rust::BatchItem<'item, ReturnValue: Send> + Send, - { - if ops.is_empty() { - return Err(Error::EmptyOperations); - } - - let ret = if self.emit_messages_flag.load(atomic::Ordering::Relaxed) { - let lock_guard = self.sync_lock.lock().await; - - let (res, _) = tx - ._batch(( - queries, - ops.iter() - .map(|op| crdt_op_db(op).map(|q| q.to_query(tx))) - .collect::, _>>()?, - )) - .await?; - - if let Some(last) = ops.last() { - self.timestamp_per_device - .write() - .await - .insert(self.device_pub_id.clone(), last.timestamp); - } - - if self.tx.send(SyncEvent::Created).is_err() { - warn!("failed to send created message on `write_ops`"); - } - - drop(lock_guard); - - res - } else { - tx._batch([queries]).await?.remove(0) - }; - - Ok(ret) - } - - pub async fn write_op<'item, Q>( - &self, - tx: &PrismaClient, - op: CRDTOperation, - query: Q, - ) -> Result - where - Q: prisma_client_rust::BatchItem<'item, ReturnValue: Send> + Send, - { - let ret = if self.emit_messages_flag.load(atomic::Ordering::Relaxed) { - let lock_guard = self.sync_lock.lock().await; - - let ret = tx._batch((crdt_op_db(&op)?.to_query(tx), query)).await?.1; - - if self.tx.send(SyncEvent::Created).is_err() { - warn!("failed to send created message on `write_op`"); - } - - drop(lock_guard); - - ret - } else { - tx._batch(vec![query]).await?.remove(0) - }; - - self.timestamp_per_device - .write() - .await - .insert(self.device_pub_id.clone(), op.timestamp); - - Ok(ret) - } - - // pub async fn get_device_ops( - // &self, - // count: u32, - // device_pub_id: DevicePubId, - // timestamp: NTP64, - // ) -> Result, Error> { - // self.db - // .crdt_operation() - // .find_many(vec![ - // crdt_operation::device_pub_id::equals(device_pub_id.into()), - // #[allow(clippy::cast_possible_wrap)] - // crdt_operation::timestamp::gt(timestamp.as_u64() as i64), - // ]) - // .take(i64::from(count)) - // .order_by(crdt_operation::timestamp::order(SortOrder::Asc)) - // .exec() - // .await? - // .into_iter() - // .map(from_crdt_ops) - // .collect() - // } - - pub fn stream_device_ops<'a>( - &'a self, - device_pub_id: &'a DevicePubId, - chunk_size: u32, - initial_timestamp: NTP64, - ) -> impl Stream, Error>> + Send + 'a { - stream! { - let mut current_initial_timestamp = initial_timestamp; - - loop { - match self.db.crdt_operation() - .find_many(vec![ - crdt_operation::device_pub_id::equals(device_pub_id.to_db()), - #[allow(clippy::cast_possible_wrap)] - crdt_operation::timestamp::gt(current_initial_timestamp.as_u64() as i64), - ]) - .take(i64::from(chunk_size)) - .order_by(crdt_operation::timestamp::order(SortOrder::Asc)) - .exec() - .await - { - Ok(ops) if ops.is_empty() => break, - - Ok(ops) => match ops - .into_iter() - .map(from_crdt_ops) - .collect::, _>>() - { - Ok(ops) => { - debug!( - start_datetime = ?ops - .first() - .map(|op| timestamp_to_datetime(op.timestamp)), - end_datetime = ?ops - .last() - .map(|op| timestamp_to_datetime(op.timestamp)), - count = ops.len(), - "Streaming crdt ops", - ); - - if let Some(last_op) = ops.last() { - current_initial_timestamp = last_op.timestamp; - } - - yield Ok(ops); - } - - Err(e) => return yield Err(e), - } - - Err(e) => return yield Err(e.into()) - } - } - } - } - - // pub async fn get_ops( - // &self, - // count: u32, - // timestamp_per_device: Vec<(DevicePubId, NTP64)>, - // ) -> Result, Error> { - // let mut ops = self - // .db - // .crdt_operation() - // .find_many(vec![or(timestamp_per_device - // .iter() - // .map(|(device_pub_id, timestamp)| { - // and![ - // crdt_operation::device_pub_id::equals(device_pub_id.to_db()), - // crdt_operation::timestamp::gt({ - // #[allow(clippy::cast_possible_wrap)] - // // SAFETY: we had to store using i64 due to SQLite limitations - // { - // timestamp.as_u64() as i64 - // } - // }) - // ] - // }) - // .chain([crdt_operation::device_pub_id::not_in_vec( - // timestamp_per_device - // .iter() - // .map(|(device_pub_id, _)| device_pub_id.to_db()) - // .collect(), - // )]) - // .collect())]) - // .take(i64::from(count)) - // .order_by(crdt_operation::timestamp::order(SortOrder::Asc)) - // .exec() - // .await?; - - // ops.sort_by(|a, b| match a.timestamp.cmp(&b.timestamp) { - // cmp::Ordering::Equal => { - // from_bytes_to_uuid(&a.device_pub_id).cmp(&from_bytes_to_uuid(&b.device_pub_id)) - // } - // o => o, - // }); - - // ops.into_iter() - // .take(count as usize) - // .map(from_crdt_ops) - // .collect() - // } - - // pub async fn get_cloud_ops( - // &self, - // count: u32, - // timestamp_per_device: Vec<(DevicePubId, NTP64)>, - // ) -> Result, Error> { - // let mut ops = self - // .db - // .cloud_crdt_operation() - // .find_many(vec![or(timestamp_per_device - // .iter() - // .map(|(device_pub_id, timestamp)| { - // and![ - // cloud_crdt_operation::device_pub_id::equals(device_pub_id.to_db()), - // cloud_crdt_operation::timestamp::gt({ - // #[allow(clippy::cast_possible_wrap)] - // // SAFETY: we had to store using i64 due to SQLite limitations - // { - // timestamp.as_u64() as i64 - // } - // }) - // ] - // }) - // .chain([cloud_crdt_operation::device_pub_id::not_in_vec( - // timestamp_per_device - // .iter() - // .map(|(device_pub_id, _)| device_pub_id.to_db()) - // .collect(), - // )]) - // .collect())]) - // .take(i64::from(count)) - // .order_by(cloud_crdt_operation::timestamp::order(SortOrder::Asc)) - // .exec() - // .await?; - - // ops.sort_by(|a, b| match a.timestamp.cmp(&b.timestamp) { - // cmp::Ordering::Equal => { - // from_bytes_to_uuid(&a.device_pub_id).cmp(&from_bytes_to_uuid(&b.device_pub_id)) - // } - // o => o, - // }); - - // ops.into_iter() - // .take(count as usize) - // .map(from_cloud_crdt_ops) - // .collect() - // } -} - -async fn bulk_process_of_create_only_ops( - available_parallelism: usize, - clock: Arc, - timestamp_per_device: TimestampPerDevice, - db: Arc, - sync_lock: Arc>, - model_id: ModelId, - create_only_ops: BTreeMap>, -) -> Result { - let buckets = (0..available_parallelism) - .map(|_| FuturesUnordered::new()) - .collect::>(); - - let mut bucket_idx = 0; - - for (device_pub_id, records) in create_only_ops { - records - .into_iter() - .chunks(100) - .into_iter() - .for_each(|chunk| { - let ops = chunk.collect::>(); - - buckets[bucket_idx % available_parallelism].push({ - let clock = Arc::clone(&clock); - let timestamp_per_device = Arc::clone(×tamp_per_device); - let db = Arc::clone(&db); - let device_pub_id = device_pub_id.into(); - let sync_lock = Arc::clone(&sync_lock); - - async move { - let count = ops.len(); - bulk_ingest_create_only_ops( - &clock, - ×tamp_per_device, - &db, - device_pub_id, - model_id, - ops, - sync_lock, - ) - .await - .map(|()| count) - } - }); - - bucket_idx += 1; - }); - } - - let handles = buckets - .into_iter() - .map(|mut bucket| { - spawn(async move { - let mut total_count = 0; - - let process_creates_batch_start = Instant::now(); - - while let Some(count) = bucket.try_next().await? { - total_count += count; - } - - debug!( - "Processed {total_count} creates in {:?}", - process_creates_batch_start.elapsed() - ); - - Ok::<_, Error>(total_count) - }) - }) - .collect::>(); - - Ok(handles - .try_join() - .await - .map_err(Error::ProcessCrdtPanic)? - .into_iter() - .collect::, _>>()? - .into_iter() - .sum()) -} - -impl OperationFactory for Manager { - fn get_clock(&self) -> &HLC { - &self.clock - } - - fn get_device_pub_id(&self) -> sd_sync::DevicePubId { - sd_sync::DevicePubId::from(&self.device_pub_id) - } -} diff --git a/core-new/crush.json b/core/crush.json similarity index 100% rename from core-new/crush.json rename to core/crush.json diff --git a/core-new/examples/file_type_demo.rs b/core/examples/file_type_demo.rs similarity index 100% rename from core-new/examples/file_type_demo.rs rename to core/examples/file_type_demo.rs diff --git a/core-new/examples/indexing_demo.rs b/core/examples/indexing_demo.rs similarity index 100% rename from core-new/examples/indexing_demo.rs rename to core/examples/indexing_demo.rs diff --git a/core/examples/indexing_showcase.rs b/core/examples/indexing_showcase.rs new file mode 100644 index 000000000..73a45cde1 --- /dev/null +++ b/core/examples/indexing_showcase.rs @@ -0,0 +1,136 @@ +//! Showcase of the production-ready indexer implementation +//! +//! This example demonstrates the sophisticated features of our new indexer: +//! - Multi-phase processing (Discovery → Processing → Content) +//! - Hardcoded filtering with should_skip_path +//! - Incremental indexing with inode tracking +//! - Performance metrics and reporting +//! - Full resumability with checkpoints + +use std::path::Path; + +fn main() { + println!("🚀 Spacedrive Production Indexer Showcase\n"); + + // Demonstrate the filtering system + showcase_filtering(); + + // Show the modular architecture + showcase_architecture(); + + // Display sample metrics output + showcase_metrics(); +} + +fn showcase_filtering() { + println!("📁 Smart Filtering System"); + println!("========================\n"); + + // Import the actual function from our implementation + use sd_core_new::operations::indexing::filters::should_skip_path; + + let test_paths = vec![ + // Files that should be skipped + (".DS_Store", true, "macOS system file"), + ("Thumbs.db", true, "Windows thumbnail cache"), + ("node_modules", true, "npm packages directory"), + (".git", true, "Git repository data"), + ("target", true, "Rust build directory"), + ("__pycache__", true, "Python cache"), + (".mypy_cache", true, "Python type checker cache"), + // Files that should NOT be skipped + ("document.pdf", false, "Regular document"), + ("photo.jpg", false, "Image file"), + ("src", false, "Source code directory"), + (".config", false, "User config directory (allowed)"), + ("project.rs", false, "Rust source file"), + ]; + + println!("Testing path filtering:"); + for (path_str, should_skip, description) in test_paths { + let path = Path::new(path_str); + let skipped = should_skip_path(path); + let result = if skipped == should_skip { "✅" } else { "❌" }; + println!( + " {} {:20} -> {:8} ({})", + result, + path_str, + if skipped { "SKIP" } else { "INDEX" }, + description + ); + } + + println!("\n💡 Note: This is where the future IndexerRuleEngine will integrate!"); + println!(" The should_skip_path function has a clear TODO marker for rules system.\n"); +} + +fn showcase_architecture() { + println!("🏗️ Modular Architecture"); + println!("=======================\n"); + + println!("core/src/operations/indexing/"); + println!("├── mod.rs # Module exports and documentation"); + println!("├── job.rs # Main IndexerJob with state machine"); + println!("├── state.rs # Resumable state management"); + println!("├── entry.rs # Entry processing with inode support"); + println!("├── filters.rs # Hardcoded filtering (→ future rules)"); + println!("├── metrics.rs # Performance tracking"); + println!("├── change_detection/ # Incremental indexing"); + println!("│ └── mod.rs # Inode-based change detection"); + println!("└── phases/ # Multi-phase processing"); + println!(" ├── discovery.rs # Directory walking"); + println!(" ├── processing.rs # Database operations"); + println!(" └── content.rs # CAS ID generation\n"); + + println!("Key Features:"); + println!("✅ Full resumability with checkpoint system"); + println!("✅ Inode tracking for move/rename detection"); + println!("✅ Batch processing (1000 items per batch)"); + println!("✅ Non-critical error collection"); + println!("✅ Path prefix optimization"); + println!("✅ Content deduplication ready\n"); +} + +fn showcase_metrics() { + println!("📊 Performance Metrics"); + println!("=====================\n"); + + // Show what metrics output looks like + let sample_output = r#"Indexing completed in 12.5s: +- Files: 10,234 (818.7/s) +- Directories: 1,523 (121.8/s) +- Total size: 2.34 GB (191.23 MB/s) +- Database writes: 10,234 in 11 batches (avg 930.4 items/batch) +- Errors: 5 (skipped 1,523 paths) +- Phase timing: discovery 5.2s, processing 6.1s, content 1.2s"#; + + println!("Sample metrics output:"); + println!("{}\n", sample_output); + + // Show the indexer progress phases + println!("Progress Tracking Phases:"); + println!("1️⃣ Discovery: 'Found 245 entries in /Users/demo/Documents'"); + println!("2️⃣ Processing: 'Batch 3/11' (database operations)"); + println!("3️⃣ Content: 'Generating content identities (456/1234)'"); + println!("4️⃣ Finalizing: 'Cleaning up and saving final state'\n"); + + // Show change detection in action + println!("🔄 Incremental Indexing Example:"); + println!("First run: Indexed 5,000 files"); + println!("Second run: Detected 3 new, 5 modified, 2 moved files"); + println!(" Only processed 10 files instead of 5,000!"); + println!(" Used inode tracking to detect moves efficiently\n"); +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_showcase_runs() { + // Just verify our showcase compiles and runs + showcase_filtering(); + showcase_architecture(); + showcase_metrics(); + } +} diff --git a/core-new/examples/job_logging_test.rs b/core/examples/job_logging_test.rs similarity index 100% rename from core-new/examples/job_logging_test.rs rename to core/examples/job_logging_test.rs diff --git a/core-new/examples/library_demo.rs b/core/examples/library_demo.rs similarity index 100% rename from core-new/examples/library_demo.rs rename to core/examples/library_demo.rs diff --git a/core-new/examples/location_watcher_demo.rs b/core/examples/location_watcher_demo.rs similarity index 100% rename from core-new/examples/location_watcher_demo.rs rename to core/examples/location_watcher_demo.rs diff --git a/core-new/examples/pause_resume_demo.rs b/core/examples/pause_resume_demo.rs similarity index 100% rename from core-new/examples/pause_resume_demo.rs rename to core/examples/pause_resume_demo.rs diff --git a/core-new/examples/shutdown_demo.rs b/core/examples/shutdown_demo.rs similarity index 100% rename from core-new/examples/shutdown_demo.rs rename to core/examples/shutdown_demo.rs diff --git a/core-new/examples/simple_pause_resume.rs b/core/examples/simple_pause_resume.rs similarity index 100% rename from core-new/examples/simple_pause_resume.rs rename to core/examples/simple_pause_resume.rs diff --git a/core-new/examples/test_migration.rs b/core/examples/test_migration.rs similarity index 100% rename from core-new/examples/test_migration.rs rename to core/examples/test_migration.rs diff --git a/core-new/examples/volume_demo.rs b/core/examples/volume_demo.rs similarity index 100% rename from core-new/examples/volume_demo.rs rename to core/examples/volume_demo.rs diff --git a/core/prisma/migrations/20230616064440_init/migration.sql b/core/prisma/migrations/20230616064440_init/migration.sql deleted file mode 100644 index 4b92fca35..000000000 --- a/core/prisma/migrations/20230616064440_init/migration.sql +++ /dev/null @@ -1,271 +0,0 @@ --- CreateTable -CREATE TABLE "owned_operation" ( - "id" BLOB NOT NULL PRIMARY KEY, - "timestamp" BIGINT NOT NULL, - "data" BLOB NOT NULL, - "model" TEXT NOT NULL, - "node_id" INTEGER NOT NULL, - CONSTRAINT "owned_operation_node_id_fkey" FOREIGN KEY ("node_id") REFERENCES "node" ("id") ON DELETE RESTRICT ON UPDATE CASCADE -); - --- CreateTable -CREATE TABLE "shared_operation" ( - "id" BLOB NOT NULL PRIMARY KEY, - "timestamp" BIGINT NOT NULL, - "model" TEXT NOT NULL, - "record_id" BLOB NOT NULL, - "kind" TEXT NOT NULL, - "data" BLOB NOT NULL, - "node_id" INTEGER NOT NULL, - CONSTRAINT "shared_operation_node_id_fkey" FOREIGN KEY ("node_id") REFERENCES "node" ("id") ON DELETE RESTRICT ON UPDATE CASCADE -); - --- CreateTable -CREATE TABLE "statistics" ( - "id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, - "date_captured" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, - "total_object_count" INTEGER NOT NULL DEFAULT 0, - "library_db_size" TEXT NOT NULL DEFAULT '0', - "total_bytes_used" TEXT NOT NULL DEFAULT '0', - "total_bytes_capacity" TEXT NOT NULL DEFAULT '0', - "total_unique_bytes" TEXT NOT NULL DEFAULT '0', - "total_bytes_free" TEXT NOT NULL DEFAULT '0', - "preview_media_bytes" TEXT NOT NULL DEFAULT '0' -); - --- CreateTable -CREATE TABLE "node" ( - "id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, - "pub_id" BLOB NOT NULL, - "name" TEXT NOT NULL, - "platform" INTEGER NOT NULL DEFAULT 0, - "date_created" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP -); - --- CreateTable -CREATE TABLE "volume" ( - "id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, - "node_id" INTEGER NOT NULL, - "name" TEXT NOT NULL, - "mount_point" TEXT NOT NULL, - "total_bytes_capacity" TEXT NOT NULL DEFAULT '0', - "total_bytes_available" TEXT NOT NULL DEFAULT '0', - "disk_type" TEXT, - "filesystem" TEXT, - "is_system" BOOLEAN NOT NULL DEFAULT false, - "date_modified" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP -); - --- CreateTable -CREATE TABLE "location" ( - "id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, - "pub_id" BLOB NOT NULL, - "node_id" INTEGER, - "name" TEXT, - "path" TEXT, - "total_capacity" INTEGER, - "available_capacity" INTEGER, - "is_archived" BOOLEAN, - "generate_preview_media" BOOLEAN, - "sync_preview_media" BOOLEAN, - "hidden" BOOLEAN, - "date_created" DATETIME, - CONSTRAINT "location_node_id_fkey" FOREIGN KEY ("node_id") REFERENCES "node" ("id") ON DELETE SET NULL ON UPDATE CASCADE -); - --- CreateTable -CREATE TABLE "file_path" ( - "id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, - "pub_id" BLOB NOT NULL, - "is_dir" BOOLEAN, - "cas_id" TEXT, - "integrity_checksum" TEXT, - "location_id" INTEGER, - "materialized_path" TEXT, - "name" TEXT, - "extension" TEXT, - "size_in_bytes" TEXT, - "inode" BLOB, - "device" BLOB, - "object_id" INTEGER, - "key_id" INTEGER, - "date_created" DATETIME, - "date_modified" DATETIME, - "date_indexed" DATETIME, - CONSTRAINT "file_path_location_id_fkey" FOREIGN KEY ("location_id") REFERENCES "location" ("id") ON DELETE CASCADE ON UPDATE CASCADE, - CONSTRAINT "file_path_object_id_fkey" FOREIGN KEY ("object_id") REFERENCES "object" ("id") ON DELETE RESTRICT ON UPDATE CASCADE -); - --- CreateTable -CREATE TABLE "object" ( - "id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, - "pub_id" BLOB NOT NULL, - "kind" INTEGER, - "key_id" INTEGER, - "hidden" BOOLEAN, - "favorite" BOOLEAN, - "important" BOOLEAN, - "note" TEXT, - "date_created" DATETIME, - "date_accessed" DATETIME -); - --- CreateTable -CREATE TABLE "media_data" ( - "id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, - "pixel_width" INTEGER, - "pixel_height" INTEGER, - "longitude" REAL, - "latitude" REAL, - "fps" INTEGER, - "capture_device_make" TEXT, - "capture_device_model" TEXT, - "capture_device_software" TEXT, - "duration_seconds" INTEGER, - "codecs" TEXT, - "streams" INTEGER, - CONSTRAINT "media_data_id_fkey" FOREIGN KEY ("id") REFERENCES "object" ("id") ON DELETE CASCADE ON UPDATE CASCADE -); - --- CreateTable -CREATE TABLE "tag" ( - "id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, - "pub_id" BLOB NOT NULL, - "name" TEXT, - "color" TEXT, - "redundancy_goal" INTEGER, - "date_created" DATETIME, - "date_modified" DATETIME -); - --- CreateTable -CREATE TABLE "tag_on_object" ( - "tag_id" INTEGER NOT NULL, - "object_id" INTEGER NOT NULL, - - PRIMARY KEY ("tag_id", "object_id"), - CONSTRAINT "tag_on_object_tag_id_fkey" FOREIGN KEY ("tag_id") REFERENCES "tag" ("id") ON DELETE NO ACTION ON UPDATE NO ACTION, - CONSTRAINT "tag_on_object_object_id_fkey" FOREIGN KEY ("object_id") REFERENCES "object" ("id") ON DELETE NO ACTION ON UPDATE NO ACTION -); - --- CreateTable -CREATE TABLE "label" ( - "id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, - "pub_id" BLOB NOT NULL, - "name" TEXT, - "date_created" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, - "date_modified" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP -); - --- CreateTable -CREATE TABLE "label_on_object" ( - "date_created" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, - "label_id" INTEGER NOT NULL, - "object_id" INTEGER NOT NULL, - - PRIMARY KEY ("label_id", "object_id"), - CONSTRAINT "label_on_object_label_id_fkey" FOREIGN KEY ("label_id") REFERENCES "label" ("id") ON DELETE NO ACTION ON UPDATE NO ACTION, - CONSTRAINT "label_on_object_object_id_fkey" FOREIGN KEY ("object_id") REFERENCES "object" ("id") ON DELETE NO ACTION ON UPDATE NO ACTION -); - --- CreateTable -CREATE TABLE "space" ( - "id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, - "pub_id" BLOB NOT NULL, - "name" TEXT, - "description" TEXT, - "date_created" DATETIME, - "date_modified" DATETIME -); - --- CreateTable -CREATE TABLE "object_in_space" ( - "space_id" INTEGER NOT NULL, - "object_id" INTEGER NOT NULL, - - PRIMARY KEY ("space_id", "object_id"), - CONSTRAINT "object_in_space_space_id_fkey" FOREIGN KEY ("space_id") REFERENCES "space" ("id") ON DELETE NO ACTION ON UPDATE NO ACTION, - CONSTRAINT "object_in_space_object_id_fkey" FOREIGN KEY ("object_id") REFERENCES "object" ("id") ON DELETE NO ACTION ON UPDATE NO ACTION -); - --- CreateTable -CREATE TABLE "job" ( - "id" BLOB NOT NULL PRIMARY KEY, - "name" TEXT NOT NULL, - "node_id" INTEGER NOT NULL, - "action" TEXT, - "status" INTEGER NOT NULL DEFAULT 0, - "errors_text" TEXT, - "data" BLOB, - "metadata" BLOB, - "parent_id" BLOB, - "task_count" INTEGER NOT NULL DEFAULT 1, - "completed_task_count" INTEGER NOT NULL DEFAULT 0, - "date_estimated_completion" DATETIME, - "date_created" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, - "date_started" DATETIME DEFAULT CURRENT_TIMESTAMP, - "date_completed" DATETIME, - CONSTRAINT "job_node_id_fkey" FOREIGN KEY ("node_id") REFERENCES "node" ("id") ON DELETE CASCADE ON UPDATE CASCADE, - CONSTRAINT "job_parent_id_fkey" FOREIGN KEY ("parent_id") REFERENCES "job" ("id") ON DELETE CASCADE ON UPDATE CASCADE -); - --- CreateTable -CREATE TABLE "indexer_rule" ( - "id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, - "pub_id" BLOB, - "name" TEXT NOT NULL, - "default" BOOLEAN NOT NULL DEFAULT false, - "rules_per_kind" BLOB NOT NULL, - "date_created" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, - "date_modified" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP -); - --- CreateTable -CREATE TABLE "indexer_rule_in_location" ( - "date_created" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, - "location_id" INTEGER NOT NULL, - "indexer_rule_id" INTEGER NOT NULL, - - PRIMARY KEY ("location_id", "indexer_rule_id"), - CONSTRAINT "indexer_rule_in_location_location_id_fkey" FOREIGN KEY ("location_id") REFERENCES "location" ("id") ON DELETE NO ACTION ON UPDATE NO ACTION, - CONSTRAINT "indexer_rule_in_location_indexer_rule_id_fkey" FOREIGN KEY ("indexer_rule_id") REFERENCES "indexer_rule" ("id") ON DELETE NO ACTION ON UPDATE NO ACTION -); - --- CreateIndex -CREATE UNIQUE INDEX "node_pub_id_key" ON "node"("pub_id"); - --- CreateIndex -CREATE UNIQUE INDEX "volume_node_id_mount_point_name_key" ON "volume"("node_id", "mount_point", "name"); - --- CreateIndex -CREATE UNIQUE INDEX "location_pub_id_key" ON "location"("pub_id"); - --- CreateIndex -CREATE UNIQUE INDEX "file_path_pub_id_key" ON "file_path"("pub_id"); - --- CreateIndex -CREATE INDEX "file_path_location_id_idx" ON "file_path"("location_id"); - --- CreateIndex -CREATE INDEX "file_path_location_id_materialized_path_idx" ON "file_path"("location_id", "materialized_path"); - --- CreateIndex -CREATE UNIQUE INDEX "file_path_location_id_materialized_path_name_extension_key" ON "file_path"("location_id", "materialized_path", "name", "extension"); - --- CreateIndex -CREATE UNIQUE INDEX "file_path_location_id_inode_device_key" ON "file_path"("location_id", "inode", "device"); - --- CreateIndex -CREATE UNIQUE INDEX "object_pub_id_key" ON "object"("pub_id"); - --- CreateIndex -CREATE UNIQUE INDEX "tag_pub_id_key" ON "tag"("pub_id"); - --- CreateIndex -CREATE UNIQUE INDEX "label_pub_id_key" ON "label"("pub_id"); - --- CreateIndex -CREATE UNIQUE INDEX "space_pub_id_key" ON "space"("pub_id"); - --- CreateIndex -CREATE UNIQUE INDEX "indexer_rule_pub_id_key" ON "indexer_rule"("pub_id"); diff --git a/core/prisma/migrations/20230618234117_remove_owned_operations/migration.sql b/core/prisma/migrations/20230618234117_remove_owned_operations/migration.sql deleted file mode 100644 index 7d56c1ab4..000000000 --- a/core/prisma/migrations/20230618234117_remove_owned_operations/migration.sql +++ /dev/null @@ -1,63 +0,0 @@ -/* - Warnings: - - - You are about to drop the `owned_operation` table. If the table is not empty, all the data it contains will be lost. - - You are about to drop the column `date_created` on the `indexer_rule_in_location` table. All the data in the column will be lost. - - Made the column `pub_id` on table `indexer_rule` required. This step will fail if there are existing NULL values in that column. - -*/ --- DropTable -PRAGMA foreign_keys=off; -DROP TABLE "owned_operation"; -PRAGMA foreign_keys=on; - --- RedefineTables -PRAGMA foreign_keys=OFF; -CREATE TABLE "new_indexer_rule" ( - "id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, - "pub_id" BLOB NOT NULL, - "name" TEXT, - "default" BOOLEAN, - "rules_per_kind" BLOB, - "date_created" DATETIME, - "date_modified" DATETIME -); -INSERT INTO "new_indexer_rule" ("date_created", "date_modified", "default", "id", "name", "pub_id", "rules_per_kind") SELECT "date_created", "date_modified", "default", "id", "name", "pub_id", "rules_per_kind" FROM "indexer_rule"; -DROP TABLE "indexer_rule"; -ALTER TABLE "new_indexer_rule" RENAME TO "indexer_rule"; -CREATE UNIQUE INDEX "indexer_rule_pub_id_key" ON "indexer_rule"("pub_id"); -CREATE TABLE "new_job" ( - "id" BLOB NOT NULL PRIMARY KEY, - "name" TEXT, - "node_id" INTEGER, - "action" TEXT, - "status" INTEGER, - "errors_text" TEXT, - "data" BLOB, - "metadata" BLOB, - "parent_id" BLOB, - "task_count" INTEGER, - "completed_task_count" INTEGER, - "date_estimated_completion" DATETIME, - "date_created" DATETIME, - "date_started" DATETIME, - "date_completed" DATETIME, - CONSTRAINT "job_node_id_fkey" FOREIGN KEY ("node_id") REFERENCES "node" ("id") ON DELETE CASCADE ON UPDATE CASCADE, - CONSTRAINT "job_parent_id_fkey" FOREIGN KEY ("parent_id") REFERENCES "job" ("id") ON DELETE CASCADE ON UPDATE CASCADE -); -INSERT INTO "new_job" ("action", "completed_task_count", "data", "date_completed", "date_created", "date_estimated_completion", "date_started", "errors_text", "id", "metadata", "name", "node_id", "parent_id", "status", "task_count") SELECT "action", "completed_task_count", "data", "date_completed", "date_created", "date_estimated_completion", "date_started", "errors_text", "id", "metadata", "name", "node_id", "parent_id", "status", "task_count" FROM "job"; -DROP TABLE "job"; -ALTER TABLE "new_job" RENAME TO "job"; -CREATE TABLE "new_indexer_rule_in_location" ( - "location_id" INTEGER NOT NULL, - "indexer_rule_id" INTEGER NOT NULL, - - PRIMARY KEY ("location_id", "indexer_rule_id"), - CONSTRAINT "indexer_rule_in_location_location_id_fkey" FOREIGN KEY ("location_id") REFERENCES "location" ("id") ON DELETE NO ACTION ON UPDATE NO ACTION, - CONSTRAINT "indexer_rule_in_location_indexer_rule_id_fkey" FOREIGN KEY ("indexer_rule_id") REFERENCES "indexer_rule" ("id") ON DELETE NO ACTION ON UPDATE NO ACTION -); -INSERT INTO "new_indexer_rule_in_location" ("indexer_rule_id", "location_id") SELECT "indexer_rule_id", "location_id" FROM "indexer_rule_in_location"; -DROP TABLE "indexer_rule_in_location"; -ALTER TABLE "new_indexer_rule_in_location" RENAME TO "indexer_rule_in_location"; -PRAGMA foreign_key_check; -PRAGMA foreign_keys=ON; diff --git a/core/prisma/migrations/20230619032753_p2p/migration.sql b/core/prisma/migrations/20230619032753_p2p/migration.sql deleted file mode 100644 index a6bb4ebaa..000000000 --- a/core/prisma/migrations/20230619032753_p2p/migration.sql +++ /dev/null @@ -1,17 +0,0 @@ --- RedefineTables -PRAGMA foreign_keys=OFF; -CREATE TABLE "new_node" ( - "id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, - "pub_id" BLOB NOT NULL, - "name" TEXT NOT NULL, - "platform" INTEGER NOT NULL, - "date_created" DATETIME NOT NULL, - "identity" BLOB, - "node_peer_id" TEXT -); -INSERT INTO "new_node" ("date_created", "id", "name", "platform", "pub_id") SELECT "date_created", "id", "name", "platform", "pub_id" FROM "node"; -DROP TABLE "node"; -ALTER TABLE "new_node" RENAME TO "node"; -CREATE UNIQUE INDEX "node_pub_id_key" ON "node"("pub_id"); -PRAGMA foreign_key_check; -PRAGMA foreign_keys=ON; diff --git a/core/prisma/migrations/20230619144125_collate_nocase/migration.sql b/core/prisma/migrations/20230619144125_collate_nocase/migration.sql deleted file mode 100644 index ceb264ff1..000000000 --- a/core/prisma/migrations/20230619144125_collate_nocase/migration.sql +++ /dev/null @@ -1,33 +0,0 @@ --- RedefineTables -PRAGMA foreign_keys=OFF; -CREATE TABLE "new_file_path" ( - "id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, - "pub_id" BLOB NOT NULL, - "is_dir" BOOLEAN, - "cas_id" TEXT, - "integrity_checksum" TEXT, - "location_id" INTEGER, - "materialized_path" TEXT, - "name" TEXT COLLATE NOCASE, - "extension" TEXT COLLATE NOCASE, - "size_in_bytes" TEXT, - "inode" BLOB, - "device" BLOB, - "object_id" INTEGER, - "key_id" INTEGER, - "date_created" DATETIME, - "date_modified" DATETIME, - "date_indexed" DATETIME, - CONSTRAINT "file_path_location_id_fkey" FOREIGN KEY ("location_id") REFERENCES "location" ("id") ON DELETE CASCADE ON UPDATE CASCADE, - CONSTRAINT "file_path_object_id_fkey" FOREIGN KEY ("object_id") REFERENCES "object" ("id") ON DELETE RESTRICT ON UPDATE CASCADE -); -INSERT INTO "new_file_path" ("cas_id", "date_created", "date_indexed", "date_modified", "device", "extension", "id", "inode", "integrity_checksum", "is_dir", "key_id", "location_id", "materialized_path", "name", "object_id", "pub_id", "size_in_bytes") SELECT "cas_id", "date_created", "date_indexed", "date_modified", "device", "extension", "id", "inode", "integrity_checksum", "is_dir", "key_id", "location_id", "materialized_path", "name", "object_id", "pub_id", "size_in_bytes" FROM "file_path"; -DROP TABLE "file_path"; -ALTER TABLE "new_file_path" RENAME TO "file_path"; -CREATE UNIQUE INDEX "file_path_pub_id_key" ON "file_path"("pub_id"); -CREATE INDEX "file_path_location_id_idx" ON "file_path"("location_id"); -CREATE INDEX "file_path_location_id_materialized_path_idx" ON "file_path"("location_id", "materialized_path"); -CREATE UNIQUE INDEX "file_path_location_id_materialized_path_name_extension_key" ON "file_path"("location_id", "materialized_path", "name", "extension"); -CREATE UNIQUE INDEX "file_path_location_id_inode_device_key" ON "file_path"("location_id", "inode", "device"); -PRAGMA foreign_key_check; -PRAGMA foreign_keys=ON; diff --git a/core/prisma/migrations/20230621173906_size_in_bytes_bytes/migration.sql b/core/prisma/migrations/20230621173906_size_in_bytes_bytes/migration.sql deleted file mode 100644 index f8c123b3e..000000000 --- a/core/prisma/migrations/20230621173906_size_in_bytes_bytes/migration.sql +++ /dev/null @@ -1,2 +0,0 @@ --- AlterTable -ALTER TABLE "file_path" ADD COLUMN "size_in_bytes_bytes" BLOB; diff --git a/core/prisma/migrations/20230711114013_preferences/migration.sql b/core/prisma/migrations/20230711114013_preferences/migration.sql deleted file mode 100644 index 2552b1fe3..000000000 --- a/core/prisma/migrations/20230711114013_preferences/migration.sql +++ /dev/null @@ -1,5 +0,0 @@ --- CreateTable -CREATE TABLE "preference" ( - "key" TEXT NOT NULL PRIMARY KEY, - "value" BLOB -); diff --git a/core/prisma/migrations/20230712050046_library_instance/migration.sql b/core/prisma/migrations/20230712050046_library_instance/migration.sql deleted file mode 100644 index 743381e62..000000000 --- a/core/prisma/migrations/20230712050046_library_instance/migration.sql +++ /dev/null @@ -1,95 +0,0 @@ -/* - Warnings: - - - You are about to drop the column `node_id` on the `job` table. All the data in the column will be lost. - - You are about to drop the column `node_id` on the `shared_operation` table. All the data in the column will be lost. - - You are about to drop the column `node_id` on the `location` table. All the data in the column will be lost. - - You are about to drop the column `node_id` on the `volume` table. All the data in the column will be lost. - - Added the required column `instance_id` to the `shared_operation` table without a default value. This is not possible if the table is not empty. - -*/ --- CreateTable -CREATE TABLE "instance" ( - "id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, - "pub_id" BLOB NOT NULL, - "identity" BLOB NOT NULL, - "node_id" BLOB NOT NULL, - "node_name" TEXT NOT NULL, - "node_platform" INTEGER NOT NULL, - "last_seen" DATETIME NOT NULL, - "date_created" DATETIME NOT NULL -); - --- RedefineTables -PRAGMA foreign_keys=OFF; -CREATE TABLE "new_job" ( - "id" BLOB NOT NULL PRIMARY KEY, - "name" TEXT, - "action" TEXT, - "status" INTEGER, - "errors_text" TEXT, - "data" BLOB, - "metadata" BLOB, - "parent_id" BLOB, - "task_count" INTEGER, - "completed_task_count" INTEGER, - "date_estimated_completion" DATETIME, - "date_created" DATETIME, - "date_started" DATETIME, - "date_completed" DATETIME, - CONSTRAINT "job_parent_id_fkey" FOREIGN KEY ("parent_id") REFERENCES "job" ("id") ON DELETE CASCADE ON UPDATE CASCADE -); -INSERT INTO "new_job" ("action", "completed_task_count", "data", "date_completed", "date_created", "date_estimated_completion", "date_started", "errors_text", "id", "metadata", "name", "parent_id", "status", "task_count") SELECT "action", "completed_task_count", "data", "date_completed", "date_created", "date_estimated_completion", "date_started", "errors_text", "id", "metadata", "name", "parent_id", "status", "task_count" FROM "job"; -DROP TABLE "job"; -ALTER TABLE "new_job" RENAME TO "job"; -CREATE TABLE "new_shared_operation" ( - "id" BLOB NOT NULL PRIMARY KEY, - "timestamp" BIGINT NOT NULL, - "model" TEXT NOT NULL, - "record_id" BLOB NOT NULL, - "kind" TEXT NOT NULL, - "data" BLOB NOT NULL, - "instance_id" INTEGER NOT NULL, - CONSTRAINT "shared_operation_instance_id_fkey" FOREIGN KEY ("instance_id") REFERENCES "instance" ("id") ON DELETE RESTRICT ON UPDATE CASCADE -); --- INSERT INTO "new_shared_operation" ("data", "id", "kind", "model", "record_id", "timestamp") SELECT "data", "id", "kind", "model", "record_id", "timestamp" FROM "shared_operation"; -DROP TABLE "shared_operation"; -ALTER TABLE "new_shared_operation" RENAME TO "shared_operation"; -CREATE TABLE "new_location" ( - "id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, - "pub_id" BLOB NOT NULL, - "name" TEXT, - "path" TEXT, - "total_capacity" INTEGER, - "available_capacity" INTEGER, - "is_archived" BOOLEAN, - "generate_preview_media" BOOLEAN, - "sync_preview_media" BOOLEAN, - "hidden" BOOLEAN, - "date_created" DATETIME, - "instance_id" INTEGER -); -INSERT INTO "new_location" ("available_capacity", "date_created", "generate_preview_media", "hidden", "id", "is_archived", "name", "path", "pub_id", "sync_preview_media", "total_capacity") SELECT "available_capacity", "date_created", "generate_preview_media", "hidden", "id", "is_archived", "name", "path", "pub_id", "sync_preview_media", "total_capacity" FROM "location"; -DROP TABLE "location"; -ALTER TABLE "new_location" RENAME TO "location"; -CREATE UNIQUE INDEX "location_pub_id_key" ON "location"("pub_id"); -CREATE TABLE "new_volume" ( - "id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, - "name" TEXT NOT NULL, - "mount_point" TEXT NOT NULL, - "total_bytes_capacity" TEXT NOT NULL DEFAULT '0', - "total_bytes_available" TEXT NOT NULL DEFAULT '0', - "disk_type" TEXT, - "filesystem" TEXT, - "is_system" BOOLEAN NOT NULL DEFAULT false, - "date_modified" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP -); -INSERT INTO "new_volume" ("date_modified", "disk_type", "filesystem", "id", "is_system", "mount_point", "name", "total_bytes_available", "total_bytes_capacity") SELECT "date_modified", "disk_type", "filesystem", "id", "is_system", "mount_point", "name", "total_bytes_available", "total_bytes_capacity" FROM "volume"; -DROP TABLE "volume"; -ALTER TABLE "new_volume" RENAME TO "volume"; -CREATE UNIQUE INDEX "volume_mount_point_name_key" ON "volume"("mount_point", "name"); -PRAGMA foreign_key_check; -PRAGMA foreign_keys=ON; - --- CreateIndex -CREATE UNIQUE INDEX "instance_pub_id_key" ON "instance"("pub_id"); diff --git a/core/prisma/migrations/20230712063345_notifications/migration.sql b/core/prisma/migrations/20230712063345_notifications/migration.sql deleted file mode 100644 index 3d250b5d3..000000000 --- a/core/prisma/migrations/20230712063345_notifications/migration.sql +++ /dev/null @@ -1,7 +0,0 @@ --- CreateTable -CREATE TABLE "notification" ( - "id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, - "read" BOOLEAN NOT NULL DEFAULT false, - "data" BLOB NOT NULL, - "expires_at" DATETIME -); diff --git a/core/prisma/migrations/20230724131659_relation_operation/migration.sql b/core/prisma/migrations/20230724131659_relation_operation/migration.sql deleted file mode 100644 index 4acfb50ff..000000000 --- a/core/prisma/migrations/20230724131659_relation_operation/migration.sql +++ /dev/null @@ -1,15 +0,0 @@ --- AlterTable -ALTER TABLE "instance" ADD COLUMN "timestamp" BIGINT; - --- CreateTable -CREATE TABLE "relation_operation" ( - "id" BLOB NOT NULL PRIMARY KEY, - "timestamp" BIGINT NOT NULL, - "relation" TEXT NOT NULL, - "item_id" BLOB NOT NULL, - "group_id" BLOB NOT NULL, - "kind" TEXT NOT NULL, - "data" BLOB NOT NULL, - "instance_id" INTEGER NOT NULL, - CONSTRAINT "relation_operation_instance_id_fkey" FOREIGN KEY ("instance_id") REFERENCES "instance" ("id") ON DELETE RESTRICT ON UPDATE CASCADE -); diff --git a/core/prisma/migrations/20230812141757_added_albums/migration.sql b/core/prisma/migrations/20230812141757_added_albums/migration.sql deleted file mode 100644 index 07b269941..000000000 --- a/core/prisma/migrations/20230812141757_added_albums/migration.sql +++ /dev/null @@ -1,23 +0,0 @@ --- CreateTable -CREATE TABLE "album" ( - "id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, - "pub_id" BLOB NOT NULL, - "name" TEXT, - "is_hidden" BOOLEAN, - "date_created" DATETIME, - "date_modified" DATETIME -); - --- CreateTable -CREATE TABLE "object_in_album" ( - "date_created" DATETIME, - "album_id" INTEGER NOT NULL, - "object_id" INTEGER NOT NULL, - - PRIMARY KEY ("album_id", "object_id"), - CONSTRAINT "object_in_album_album_id_fkey" FOREIGN KEY ("album_id") REFERENCES "album" ("id") ON DELETE NO ACTION ON UPDATE CASCADE, - CONSTRAINT "object_in_album_object_id_fkey" FOREIGN KEY ("object_id") REFERENCES "object" ("id") ON DELETE NO ACTION ON UPDATE CASCADE -); - --- CreateIndex -CREATE UNIQUE INDEX "album_pub_id_key" ON "album"("pub_id"); diff --git a/core/prisma/migrations/20230828195811_media_data/migration.sql b/core/prisma/migrations/20230828195811_media_data/migration.sql deleted file mode 100644 index 4a362f217..000000000 --- a/core/prisma/migrations/20230828195811_media_data/migration.sql +++ /dev/null @@ -1,152 +0,0 @@ -/* - Warnings: - - - You are about to drop the column `capture_device_make` on the `media_data` table. All the data in the column will be lost. - - You are about to drop the column `capture_device_model` on the `media_data` table. All the data in the column will be lost. - - You are about to drop the column `capture_device_software` on the `media_data` table. All the data in the column will be lost. - - You are about to drop the column `codecs` on the `media_data` table. All the data in the column will be lost. - - You are about to drop the column `duration_seconds` on the `media_data` table. All the data in the column will be lost. - - You are about to drop the column `fps` on the `media_data` table. All the data in the column will be lost. - - You are about to drop the column `latitude` on the `media_data` table. All the data in the column will be lost. - - You are about to drop the column `longitude` on the `media_data` table. All the data in the column will be lost. - - You are about to drop the column `pixel_height` on the `media_data` table. All the data in the column will be lost. - - You are about to drop the column `pixel_width` on the `media_data` table. All the data in the column will be lost. - - You are about to drop the column `streams` on the `media_data` table. All the data in the column will be lost. - - Added the required column `object_id` to the `media_data` table without a default value. This is not possible if the table is not empty. - -*/ --- RedefineTables -PRAGMA foreign_keys=OFF; -CREATE TABLE "new_label_on_object" ( - "date_created" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, - "label_id" INTEGER NOT NULL, - "object_id" INTEGER NOT NULL, - - PRIMARY KEY ("label_id", "object_id"), - CONSTRAINT "label_on_object_label_id_fkey" FOREIGN KEY ("label_id") REFERENCES "label" ("id") ON DELETE RESTRICT ON UPDATE CASCADE, - CONSTRAINT "label_on_object_object_id_fkey" FOREIGN KEY ("object_id") REFERENCES "object" ("id") ON DELETE RESTRICT ON UPDATE CASCADE -); -INSERT INTO "new_label_on_object" ("date_created", "label_id", "object_id") SELECT "date_created", "label_id", "object_id" FROM "label_on_object"; -DROP TABLE "label_on_object"; -ALTER TABLE "new_label_on_object" RENAME TO "label_on_object"; -CREATE TABLE "new_tag_on_object" ( - "tag_id" INTEGER NOT NULL, - "object_id" INTEGER NOT NULL, - - PRIMARY KEY ("tag_id", "object_id"), - CONSTRAINT "tag_on_object_tag_id_fkey" FOREIGN KEY ("tag_id") REFERENCES "tag" ("id") ON DELETE RESTRICT ON UPDATE CASCADE, - CONSTRAINT "tag_on_object_object_id_fkey" FOREIGN KEY ("object_id") REFERENCES "object" ("id") ON DELETE RESTRICT ON UPDATE CASCADE -); -INSERT INTO "new_tag_on_object" ("object_id", "tag_id") SELECT "object_id", "tag_id" FROM "tag_on_object"; -DROP TABLE "tag_on_object"; -ALTER TABLE "new_tag_on_object" RENAME TO "tag_on_object"; -CREATE TABLE "new_file_path" ( - "id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, - "pub_id" BLOB NOT NULL, - "is_dir" BOOLEAN, - "cas_id" TEXT, - "integrity_checksum" TEXT, - "location_id" INTEGER, - "materialized_path" TEXT, - "name" TEXT, - "extension" TEXT, - "size_in_bytes" TEXT, - "size_in_bytes_bytes" BLOB, - "inode" BLOB, - "device" BLOB, - "object_id" INTEGER, - "key_id" INTEGER, - "date_created" DATETIME, - "date_modified" DATETIME, - "date_indexed" DATETIME, - CONSTRAINT "file_path_location_id_fkey" FOREIGN KEY ("location_id") REFERENCES "location" ("id") ON DELETE SET NULL ON UPDATE CASCADE, - CONSTRAINT "file_path_object_id_fkey" FOREIGN KEY ("object_id") REFERENCES "object" ("id") ON DELETE SET NULL ON UPDATE CASCADE -); -INSERT INTO "new_file_path" ("cas_id", "date_created", "date_indexed", "date_modified", "device", "extension", "id", "inode", "integrity_checksum", "is_dir", "key_id", "location_id", "materialized_path", "name", "object_id", "pub_id", "size_in_bytes", "size_in_bytes_bytes") SELECT "cas_id", "date_created", "date_indexed", "date_modified", "device", "extension", "id", "inode", "integrity_checksum", "is_dir", "key_id", "location_id", "materialized_path", "name", "object_id", "pub_id", "size_in_bytes", "size_in_bytes_bytes" FROM "file_path"; -DROP TABLE "file_path"; -ALTER TABLE "new_file_path" RENAME TO "file_path"; -CREATE UNIQUE INDEX "file_path_pub_id_key" ON "file_path"("pub_id"); -CREATE INDEX "file_path_location_id_idx" ON "file_path"("location_id"); -CREATE INDEX "file_path_location_id_materialized_path_idx" ON "file_path"("location_id", "materialized_path"); -CREATE UNIQUE INDEX "file_path_location_id_materialized_path_name_extension_key" ON "file_path"("location_id", "materialized_path", "name", "extension"); -CREATE UNIQUE INDEX "file_path_location_id_inode_device_key" ON "file_path"("location_id", "inode", "device"); -CREATE TABLE "new_location" ( - "id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, - "pub_id" BLOB NOT NULL, - "name" TEXT, - "path" TEXT, - "total_capacity" INTEGER, - "available_capacity" INTEGER, - "is_archived" BOOLEAN, - "generate_preview_media" BOOLEAN, - "sync_preview_media" BOOLEAN, - "hidden" BOOLEAN, - "date_created" DATETIME, - "instance_id" INTEGER, - CONSTRAINT "location_instance_id_fkey" FOREIGN KEY ("instance_id") REFERENCES "instance" ("id") ON DELETE SET NULL ON UPDATE CASCADE -); -INSERT INTO "new_location" ("available_capacity", "date_created", "generate_preview_media", "hidden", "id", "instance_id", "is_archived", "name", "path", "pub_id", "sync_preview_media", "total_capacity") SELECT "available_capacity", "date_created", "generate_preview_media", "hidden", "id", "instance_id", "is_archived", "name", "path", "pub_id", "sync_preview_media", "total_capacity" FROM "location"; -DROP TABLE "location"; -ALTER TABLE "new_location" RENAME TO "location"; -CREATE UNIQUE INDEX "location_pub_id_key" ON "location"("pub_id"); -CREATE TABLE "new_job" ( - "id" BLOB NOT NULL PRIMARY KEY, - "name" TEXT, - "action" TEXT, - "status" INTEGER, - "errors_text" TEXT, - "data" BLOB, - "metadata" BLOB, - "parent_id" BLOB, - "task_count" INTEGER, - "completed_task_count" INTEGER, - "date_estimated_completion" DATETIME, - "date_created" DATETIME, - "date_started" DATETIME, - "date_completed" DATETIME, - CONSTRAINT "job_parent_id_fkey" FOREIGN KEY ("parent_id") REFERENCES "job" ("id") ON DELETE SET NULL ON UPDATE CASCADE -); -INSERT INTO "new_job" ("action", "completed_task_count", "data", "date_completed", "date_created", "date_estimated_completion", "date_started", "errors_text", "id", "metadata", "name", "parent_id", "status", "task_count") SELECT "action", "completed_task_count", "data", "date_completed", "date_created", "date_estimated_completion", "date_started", "errors_text", "id", "metadata", "name", "parent_id", "status", "task_count" FROM "job"; -DROP TABLE "job"; -ALTER TABLE "new_job" RENAME TO "job"; -CREATE TABLE "new_indexer_rule_in_location" ( - "location_id" INTEGER NOT NULL, - "indexer_rule_id" INTEGER NOT NULL, - - PRIMARY KEY ("location_id", "indexer_rule_id"), - CONSTRAINT "indexer_rule_in_location_location_id_fkey" FOREIGN KEY ("location_id") REFERENCES "location" ("id") ON DELETE RESTRICT ON UPDATE CASCADE, - CONSTRAINT "indexer_rule_in_location_indexer_rule_id_fkey" FOREIGN KEY ("indexer_rule_id") REFERENCES "indexer_rule" ("id") ON DELETE RESTRICT ON UPDATE CASCADE -); -INSERT INTO "new_indexer_rule_in_location" ("indexer_rule_id", "location_id") SELECT "indexer_rule_id", "location_id" FROM "indexer_rule_in_location"; -DROP TABLE "indexer_rule_in_location"; -ALTER TABLE "new_indexer_rule_in_location" RENAME TO "indexer_rule_in_location"; -CREATE TABLE "new_media_data" ( - "id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, - "dimensions" BLOB, - "media_date" BLOB, - "media_location" BLOB, - "camera_data" BLOB, - "artist" TEXT, - "description" TEXT, - "copyright" TEXT, - "exif_version" TEXT, - "object_id" INTEGER NOT NULL, - CONSTRAINT "media_data_object_id_fkey" FOREIGN KEY ("object_id") REFERENCES "object" ("id") ON DELETE CASCADE ON UPDATE CASCADE -); -INSERT INTO "new_media_data" ("id") SELECT "id" FROM "media_data"; -DROP TABLE "media_data"; -ALTER TABLE "new_media_data" RENAME TO "media_data"; -CREATE UNIQUE INDEX "media_data_object_id_key" ON "media_data"("object_id"); -CREATE TABLE "new_object_in_space" ( - "space_id" INTEGER NOT NULL, - "object_id" INTEGER NOT NULL, - - PRIMARY KEY ("space_id", "object_id"), - CONSTRAINT "object_in_space_space_id_fkey" FOREIGN KEY ("space_id") REFERENCES "space" ("id") ON DELETE RESTRICT ON UPDATE CASCADE, - CONSTRAINT "object_in_space_object_id_fkey" FOREIGN KEY ("object_id") REFERENCES "object" ("id") ON DELETE RESTRICT ON UPDATE CASCADE -); -INSERT INTO "new_object_in_space" ("object_id", "space_id") SELECT "object_id", "space_id" FROM "object_in_space"; -DROP TABLE "object_in_space"; -ALTER TABLE "new_object_in_space" RENAME TO "object_in_space"; -PRAGMA foreign_key_check; -PRAGMA foreign_keys=ON; diff --git a/core/prisma/migrations/20230912065026_file_path_hidden/migration.sql b/core/prisma/migrations/20230912065026_file_path_hidden/migration.sql deleted file mode 100644 index 40acd26cd..000000000 --- a/core/prisma/migrations/20230912065026_file_path_hidden/migration.sql +++ /dev/null @@ -1,2 +0,0 @@ --- AlterTable -ALTER TABLE "file_path" ADD COLUMN "hidden" BOOLEAN; diff --git a/core/prisma/migrations/20230921203938_add_size_bytes_in_location_table/migration.sql b/core/prisma/migrations/20230921203938_add_size_bytes_in_location_table/migration.sql deleted file mode 100644 index 7af11991e..000000000 --- a/core/prisma/migrations/20230921203938_add_size_bytes_in_location_table/migration.sql +++ /dev/null @@ -1,2 +0,0 @@ --- AlterTable -ALTER TABLE "location" ADD COLUMN "size_in_bytes" BLOB; diff --git a/core/prisma/migrations/20230926180932_remove_device_from_file_path_table/migration.sql b/core/prisma/migrations/20230926180932_remove_device_from_file_path_table/migration.sql deleted file mode 100644 index e3bbf18be..000000000 --- a/core/prisma/migrations/20230926180932_remove_device_from_file_path_table/migration.sql +++ /dev/null @@ -1,40 +0,0 @@ -/* - Warnings: - - - You are about to drop the column `device` on the `file_path` table. All the data in the column will be lost. - -*/ --- RedefineTables -PRAGMA foreign_keys=OFF; -CREATE TABLE "new_file_path" ( - "id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, - "pub_id" BLOB NOT NULL, - "is_dir" BOOLEAN, - "cas_id" TEXT, - "integrity_checksum" TEXT, - "location_id" INTEGER, - "materialized_path" TEXT, - "name" TEXT, - "extension" TEXT, - "hidden" BOOLEAN, - "size_in_bytes" TEXT, - "size_in_bytes_bytes" BLOB, - "inode" BLOB, - "object_id" INTEGER, - "key_id" INTEGER, - "date_created" DATETIME, - "date_modified" DATETIME, - "date_indexed" DATETIME, - CONSTRAINT "file_path_location_id_fkey" FOREIGN KEY ("location_id") REFERENCES "location" ("id") ON DELETE SET NULL ON UPDATE CASCADE, - CONSTRAINT "file_path_object_id_fkey" FOREIGN KEY ("object_id") REFERENCES "object" ("id") ON DELETE SET NULL ON UPDATE CASCADE -); -INSERT INTO "new_file_path" ("cas_id", "date_created", "date_indexed", "date_modified", "extension", "hidden", "id", "inode", "integrity_checksum", "is_dir", "key_id", "location_id", "materialized_path", "name", "object_id", "pub_id", "size_in_bytes", "size_in_bytes_bytes") SELECT "cas_id", "date_created", "date_indexed", "date_modified", "extension", "hidden", "id", "inode", "integrity_checksum", "is_dir", "key_id", "location_id", "materialized_path", "name", "object_id", "pub_id", "size_in_bytes", "size_in_bytes_bytes" FROM "file_path"; -DROP TABLE "file_path"; -ALTER TABLE "new_file_path" RENAME TO "file_path"; -CREATE UNIQUE INDEX "file_path_pub_id_key" ON "file_path"("pub_id"); -CREATE INDEX "file_path_location_id_idx" ON "file_path"("location_id"); -CREATE INDEX "file_path_location_id_materialized_path_idx" ON "file_path"("location_id", "materialized_path"); -CREATE UNIQUE INDEX "file_path_location_id_materialized_path_name_extension_key" ON "file_path"("location_id", "materialized_path", "name", "extension"); -CREATE UNIQUE INDEX "file_path_location_id_inode_key" ON "file_path"("location_id", "inode"); -PRAGMA foreign_key_check; -PRAGMA foreign_keys=ON; diff --git a/core/prisma/migrations/20231005202254_sort_by_image_taken/migration.sql b/core/prisma/migrations/20231005202254_sort_by_image_taken/migration.sql deleted file mode 100644 index ab0ead21b..000000000 --- a/core/prisma/migrations/20231005202254_sort_by_image_taken/migration.sql +++ /dev/null @@ -1,28 +0,0 @@ -/* - Warnings: - - - You are about to drop the column `dimensions` on the `media_data` table. All the data in the column will be lost. - -*/ --- RedefineTables -PRAGMA foreign_keys=OFF; -CREATE TABLE "new_media_data" ( - "id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, - "resolution" BLOB, - "media_date" BLOB, - "media_location" BLOB, - "camera_data" BLOB, - "artist" TEXT, - "description" TEXT, - "copyright" TEXT, - "exif_version" TEXT, - "epoch_time" BIGINT, - "object_id" INTEGER NOT NULL, - CONSTRAINT "media_data_object_id_fkey" FOREIGN KEY ("object_id") REFERENCES "object" ("id") ON DELETE CASCADE ON UPDATE CASCADE -); -INSERT INTO "new_media_data" ("artist", "camera_data", "copyright", "description", "exif_version", "id", "media_date", "media_location", "object_id") SELECT "artist", "camera_data", "copyright", "description", "exif_version", "id", "media_date", "media_location", "object_id" FROM "media_data"; -DROP TABLE "media_data"; -ALTER TABLE "new_media_data" RENAME TO "media_data"; -CREATE UNIQUE INDEX "media_data_object_id_key" ON "media_data"("object_id"); -PRAGMA foreign_key_check; -PRAGMA foreign_keys=ON; diff --git a/core/prisma/migrations/20231022013532_saved_searches/migration.sql b/core/prisma/migrations/20231022013532_saved_searches/migration.sql deleted file mode 100644 index a93924aed..000000000 --- a/core/prisma/migrations/20231022013532_saved_searches/migration.sql +++ /dev/null @@ -1,15 +0,0 @@ --- CreateTable -CREATE TABLE "saved_search" ( - "id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, - "pub_id" BLOB NOT NULL, - "filters" BLOB, - "name" TEXT, - "icon" TEXT, - "description" TEXT, - "order" INTEGER, - "date_created" DATETIME, - "date_modified" DATETIME -); - --- CreateIndex -CREATE UNIQUE INDEX "saved_search_pub_id_key" ON "saved_search"("pub_id"); diff --git a/core/prisma/migrations/20231113140411_tag_date_created/migration.sql b/core/prisma/migrations/20231113140411_tag_date_created/migration.sql deleted file mode 100644 index c4ba0be1c..000000000 --- a/core/prisma/migrations/20231113140411_tag_date_created/migration.sql +++ /dev/null @@ -1,2 +0,0 @@ --- AlterTable -ALTER TABLE "tag_on_object" ADD COLUMN "date_created" DATETIME; diff --git a/core/prisma/migrations/20231121173834_filters_string/migration.sql b/core/prisma/migrations/20231121173834_filters_string/migration.sql deleted file mode 100644 index 65e52bafa..000000000 --- a/core/prisma/migrations/20231121173834_filters_string/migration.sql +++ /dev/null @@ -1,24 +0,0 @@ -/* - Warnings: - - - You are about to drop the column `order` on the `saved_search` table. All the data in the column will be lost. - -*/ --- RedefineTables -PRAGMA foreign_keys=OFF; -CREATE TABLE "new_saved_search" ( - "id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, - "pub_id" BLOB NOT NULL, - "filters" TEXT, - "name" TEXT, - "icon" TEXT, - "description" TEXT, - "date_created" DATETIME, - "date_modified" DATETIME -); -INSERT INTO "new_saved_search" ("date_created", "date_modified", "description", "filters", "icon", "id", "name", "pub_id") SELECT "date_created", "date_modified", "description", "filters", "icon", "id", "name", "pub_id" FROM "saved_search"; -DROP TABLE "saved_search"; -ALTER TABLE "new_saved_search" RENAME TO "saved_search"; -CREATE UNIQUE INDEX "saved_search_pub_id_key" ON "saved_search"("pub_id"); -PRAGMA foreign_key_check; -PRAGMA foreign_keys=ON; diff --git a/core/prisma/migrations/20231204174640_update_tags_and_labels/migration.sql b/core/prisma/migrations/20231204174640_update_tags_and_labels/migration.sql deleted file mode 100644 index 545148522..000000000 --- a/core/prisma/migrations/20231204174640_update_tags_and_labels/migration.sql +++ /dev/null @@ -1,39 +0,0 @@ -/* - Warnings: - - - You are about to drop the column `redundancy_goal` on the `tag` table. All the data in the column will be lost. - - Made the column `name` on table `label` required. This step will fail if there are existing NULL values in that column. - -*/ --- AlterTable -ALTER TABLE "saved_search" ADD COLUMN "search" TEXT; - --- RedefineTables -PRAGMA foreign_keys=OFF; -CREATE TABLE "new_label" ( - "id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, - "pub_id" BLOB NOT NULL, - "name" TEXT NOT NULL, - "date_created" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, - "date_modified" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP -); -INSERT INTO "new_label" ("date_created", "date_modified", "id", "name", "pub_id") SELECT "date_created", "date_modified", "id", "name", "pub_id" FROM "label"; -DROP TABLE "label"; -ALTER TABLE "new_label" RENAME TO "label"; -CREATE UNIQUE INDEX "label_pub_id_key" ON "label"("pub_id"); -CREATE UNIQUE INDEX "label_name_key" ON "label"("name"); -CREATE TABLE "new_tag" ( - "id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, - "pub_id" BLOB NOT NULL, - "name" TEXT, - "color" TEXT, - "is_hidden" BOOLEAN, - "date_created" DATETIME, - "date_modified" DATETIME -); -INSERT INTO "new_tag" ("color", "date_created", "date_modified", "id", "name", "pub_id") SELECT "color", "date_created", "date_modified", "id", "name", "pub_id" FROM "tag"; -DROP TABLE "tag"; -ALTER TABLE "new_tag" RENAME TO "tag"; -CREATE UNIQUE INDEX "tag_pub_id_key" ON "tag"("pub_id"); -PRAGMA foreign_key_check; -PRAGMA foreign_keys=ON; diff --git a/core/prisma/migrations/20231219105608_crdt_operation/migration.sql b/core/prisma/migrations/20231219105608_crdt_operation/migration.sql deleted file mode 100644 index e99f63345..000000000 --- a/core/prisma/migrations/20231219105608_crdt_operation/migration.sql +++ /dev/null @@ -1,40 +0,0 @@ -/* - Warnings: - - - You are about to drop the `relation_operation` table. If the table is not empty, all the data it contains will be lost. - - You are about to drop the `shared_operation` table. If the table is not empty, all the data it contains will be lost. - -*/ --- DropTable -PRAGMA foreign_keys=off; -DROP TABLE "relation_operation"; -PRAGMA foreign_keys=on; - --- DropTable -PRAGMA foreign_keys=off; -DROP TABLE "shared_operation"; -PRAGMA foreign_keys=on; - --- CreateTable -CREATE TABLE "crdt_operation" ( - "id" BLOB NOT NULL PRIMARY KEY, - "timestamp" BIGINT NOT NULL, - "model" TEXT NOT NULL, - "record_id" BLOB NOT NULL, - "kind" TEXT NOT NULL, - "data" BLOB NOT NULL, - "instance_id" INTEGER NOT NULL, - CONSTRAINT "crdt_operation_instance_id_fkey" FOREIGN KEY ("instance_id") REFERENCES "instance" ("id") ON DELETE RESTRICT ON UPDATE CASCADE -); - --- CreateTable -CREATE TABLE "cloud_crdt_operation" ( - "id" BLOB NOT NULL PRIMARY KEY, - "timestamp" BIGINT NOT NULL, - "model" TEXT NOT NULL, - "record_id" BLOB NOT NULL, - "kind" TEXT NOT NULL, - "data" BLOB NOT NULL, - "instance_id" INTEGER NOT NULL, - CONSTRAINT "cloud_crdt_operation_instance_id_fkey" FOREIGN KEY ("instance_id") REFERENCES "instance" ("id") ON DELETE RESTRICT ON UPDATE CASCADE -); diff --git a/core/prisma/migrations/20240221044741_drop_node_peer_id/migration.sql b/core/prisma/migrations/20240221044741_drop_node_peer_id/migration.sql deleted file mode 100644 index deda288c5..000000000 --- a/core/prisma/migrations/20240221044741_drop_node_peer_id/migration.sql +++ /dev/null @@ -1,33 +0,0 @@ -/* - Warnings: - - - You are about to drop the column `pub_id` on the `label` table. All the data in the column will be lost. - - You are about to drop the column `node_peer_id` on the `node` table. All the data in the column will be lost. - -*/ --- RedefineTables -PRAGMA foreign_keys=OFF; -CREATE TABLE "new_label" ( - "id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, - "name" TEXT NOT NULL, - "date_created" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, - "date_modified" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP -); -INSERT INTO "new_label" ("date_created", "date_modified", "id", "name") SELECT "date_created", "date_modified", "id", "name" FROM "label"; -DROP TABLE "label"; -ALTER TABLE "new_label" RENAME TO "label"; -CREATE UNIQUE INDEX "label_name_key" ON "label"("name"); -CREATE TABLE "new_node" ( - "id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, - "pub_id" BLOB NOT NULL, - "name" TEXT NOT NULL, - "platform" INTEGER NOT NULL, - "date_created" DATETIME NOT NULL, - "identity" BLOB -); -INSERT INTO "new_node" ("date_created", "id", "identity", "name", "platform", "pub_id") SELECT "date_created", "id", "identity", "name", "platform", "pub_id" FROM "node"; -DROP TABLE "node"; -ALTER TABLE "new_node" RENAME TO "node"; -CREATE UNIQUE INDEX "node_pub_id_key" ON "node"("pub_id"); -PRAGMA foreign_key_check; -PRAGMA foreign_keys=ON; diff --git a/core/prisma/migrations/20240314064009_cache_instance_metadata/migration.sql b/core/prisma/migrations/20240314064009_cache_instance_metadata/migration.sql deleted file mode 100644 index 7d4160e13..000000000 --- a/core/prisma/migrations/20240314064009_cache_instance_metadata/migration.sql +++ /dev/null @@ -1,35 +0,0 @@ -/* - Warnings: - - - You are about to drop the column `node_name` on the `instance` table. All the data in the column will be lost. - - You are about to drop the column `node_platform` on the `instance` table. All the data in the column will be lost. - -*/ --- RedefineTables -PRAGMA foreign_keys=OFF; -CREATE TABLE "new_instance" ( - "id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, - "pub_id" BLOB NOT NULL, - "identity" BLOB NOT NULL, - "node_id" BLOB NOT NULL, - "metadata" BLOB, - "last_seen" DATETIME NOT NULL, - "date_created" DATETIME NOT NULL, - "timestamp" BIGINT -); -INSERT INTO "new_instance" ("date_created", "id", "identity", "last_seen", "node_id", "pub_id", "timestamp") SELECT "date_created", "id", "identity", "last_seen", "node_id", "pub_id", "timestamp" FROM "instance"; -DROP TABLE "instance"; -ALTER TABLE "new_instance" RENAME TO "instance"; -CREATE UNIQUE INDEX "instance_pub_id_key" ON "instance"("pub_id"); -CREATE TABLE "new_label" ( - "id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, - "name" TEXT NOT NULL, - "date_created" DATETIME, - "date_modified" DATETIME -); -INSERT INTO "new_label" ("date_created", "date_modified", "id", "name") SELECT "date_created", "date_modified", "id", "name" FROM "label"; -DROP TABLE "label"; -ALTER TABLE "new_label" RENAME TO "label"; -CREATE UNIQUE INDEX "label_name_key" ON "label"("name"); -PRAGMA foreign_key_check; -PRAGMA foreign_keys=ON; diff --git a/core/prisma/migrations/20240319065147_remove_remoteidentity/migration.sql b/core/prisma/migrations/20240319065147_remove_remoteidentity/migration.sql deleted file mode 100644 index a09ffa66f..000000000 --- a/core/prisma/migrations/20240319065147_remove_remoteidentity/migration.sql +++ /dev/null @@ -1,57 +0,0 @@ -/* - Warnings: - - - The primary key for the `cloud_crdt_operation` table will be changed. If it partially fails, the table could be left without primary key constraint. - - You are about to alter the column `id` on the `cloud_crdt_operation` table. The data in that column could be lost. The data in that column will be cast from `Binary` to `Int`. - - The primary key for the `crdt_operation` table will be changed. If it partially fails, the table could be left without primary key constraint. - - You are about to alter the column `id` on the `crdt_operation` table. The data in that column could be lost. The data in that column will be cast from `Binary` to `Int`. - - Added the required column `remote_identity` to the `instance` table without a default value. This is not possible if the table is not empty. - - - @oscartbeaumont modified the migration Prisma generated to fill the `NOT NULL` `remote_identity` field with the existing IdentityOrRemoteIdentity value so we can handle it in the app migrations. - -*/ --- RedefineTables -PRAGMA foreign_keys=OFF; -CREATE TABLE "new_cloud_crdt_operation" ( - "id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, - "timestamp" BIGINT NOT NULL, - "model" TEXT NOT NULL, - "record_id" BLOB NOT NULL, - "kind" TEXT NOT NULL, - "data" BLOB NOT NULL, - "instance_id" INTEGER NOT NULL, - CONSTRAINT "cloud_crdt_operation_instance_id_fkey" FOREIGN KEY ("instance_id") REFERENCES "instance" ("id") ON DELETE RESTRICT ON UPDATE CASCADE -); -INSERT INTO "new_cloud_crdt_operation" ("data", "id", "instance_id", "kind", "model", "record_id", "timestamp") SELECT "data", "id", "instance_id", "kind", "model", "record_id", "timestamp" FROM "cloud_crdt_operation"; -DROP TABLE "cloud_crdt_operation"; -ALTER TABLE "new_cloud_crdt_operation" RENAME TO "cloud_crdt_operation"; -CREATE TABLE "new_instance" ( - "id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, - "pub_id" BLOB NOT NULL, - "identity" BLOB, - "remote_identity" BLOB NOT NULL, - "node_id" BLOB NOT NULL, - "metadata" BLOB, - "last_seen" DATETIME NOT NULL, - "date_created" DATETIME NOT NULL, - "timestamp" BIGINT -); -INSERT INTO "new_instance" ("date_created", "id", "identity", "remote_identity", "last_seen", "metadata", "node_id", "pub_id", "timestamp") SELECT "date_created", "id", "identity", "identity", "last_seen", "metadata", "node_id", "pub_id", "timestamp" FROM "instance"; -DROP TABLE "instance"; -ALTER TABLE "new_instance" RENAME TO "instance"; -CREATE UNIQUE INDEX "instance_pub_id_key" ON "instance"("pub_id"); -CREATE TABLE "new_crdt_operation" ( - "id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, - "timestamp" BIGINT NOT NULL, - "model" TEXT NOT NULL, - "record_id" BLOB NOT NULL, - "kind" TEXT NOT NULL, - "data" BLOB NOT NULL, - "instance_id" INTEGER NOT NULL, - CONSTRAINT "crdt_operation_instance_id_fkey" FOREIGN KEY ("instance_id") REFERENCES "instance" ("id") ON DELETE RESTRICT ON UPDATE CASCADE -); -INSERT INTO "new_crdt_operation" ("data", "id", "instance_id", "kind", "model", "record_id", "timestamp") SELECT "data", "id", "instance_id", "kind", "model", "record_id", "timestamp" FROM "crdt_operation"; -DROP TABLE "crdt_operation"; -ALTER TABLE "new_crdt_operation" RENAME TO "crdt_operation"; -PRAGMA foreign_key_check; -PRAGMA foreign_keys=ON; diff --git a/core/prisma/migrations/20240405185748_add_target_to_saved_search/migration.sql b/core/prisma/migrations/20240405185748_add_target_to_saved_search/migration.sql deleted file mode 100644 index 725f72a96..000000000 --- a/core/prisma/migrations/20240405185748_add_target_to_saved_search/migration.sql +++ /dev/null @@ -1,2 +0,0 @@ --- AlterTable -ALTER TABLE "saved_search" ADD COLUMN "target" TEXT; diff --git a/core/prisma/migrations/20240408081222_numeric_sync_model_id/migration.sql b/core/prisma/migrations/20240408081222_numeric_sync_model_id/migration.sql deleted file mode 100644 index 4d18f839d..000000000 --- a/core/prisma/migrations/20240408081222_numeric_sync_model_id/migration.sql +++ /dev/null @@ -1,37 +0,0 @@ -/* - Warnings: - - - You are about to alter the column `model` on the `cloud_crdt_operation` table. The data in that column could be lost. The data in that column will be cast from `String` to `Int`. - - You are about to alter the column `model` on the `crdt_operation` table. The data in that column could be lost. The data in that column will be cast from `String` to `Int`. - -*/ --- RedefineTables -PRAGMA foreign_keys=OFF; -CREATE TABLE "new_cloud_crdt_operation" ( - "id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, - "timestamp" BIGINT NOT NULL, - "model" INTEGER NOT NULL, - "record_id" BLOB NOT NULL, - "kind" TEXT NOT NULL, - "data" BLOB NOT NULL, - "instance_id" INTEGER NOT NULL, - CONSTRAINT "cloud_crdt_operation_instance_id_fkey" FOREIGN KEY ("instance_id") REFERENCES "instance" ("id") ON DELETE RESTRICT ON UPDATE CASCADE -); -INSERT INTO "new_cloud_crdt_operation" ("data", "id", "instance_id", "kind", "model", "record_id", "timestamp") SELECT "data", "id", "instance_id", "kind", "model", "record_id", "timestamp" FROM "cloud_crdt_operation"; -DROP TABLE "cloud_crdt_operation"; -ALTER TABLE "new_cloud_crdt_operation" RENAME TO "cloud_crdt_operation"; -CREATE TABLE "new_crdt_operation" ( - "id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, - "timestamp" BIGINT NOT NULL, - "model" INTEGER NOT NULL, - "record_id" BLOB NOT NULL, - "kind" TEXT NOT NULL, - "data" BLOB NOT NULL, - "instance_id" INTEGER NOT NULL, - CONSTRAINT "crdt_operation_instance_id_fkey" FOREIGN KEY ("instance_id") REFERENCES "instance" ("id") ON DELETE RESTRICT ON UPDATE CASCADE -); -INSERT INTO "new_crdt_operation" ("data", "id", "instance_id", "kind", "model", "record_id", "timestamp") SELECT "data", "id", "instance_id", "kind", "model", "record_id", "timestamp" FROM "crdt_operation"; -DROP TABLE "crdt_operation"; -ALTER TABLE "new_crdt_operation" RENAME TO "crdt_operation"; -PRAGMA foreign_key_check; -PRAGMA foreign_keys=ON; diff --git a/core/prisma/migrations/20240408215355_add_location_scan_state/migration.sql b/core/prisma/migrations/20240408215355_add_location_scan_state/migration.sql deleted file mode 100644 index 1bcf4117c..000000000 --- a/core/prisma/migrations/20240408215355_add_location_scan_state/migration.sql +++ /dev/null @@ -1,29 +0,0 @@ --- AlterTable -ALTER TABLE "job" ADD COLUMN "critical_error" TEXT; -ALTER TABLE "job" ADD COLUMN "non_critical_errors" BLOB; - --- RedefineTables -PRAGMA foreign_keys=OFF; -CREATE TABLE "new_location" ( - "id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, - "pub_id" BLOB NOT NULL, - "name" TEXT, - "path" TEXT, - "total_capacity" INTEGER, - "available_capacity" INTEGER, - "size_in_bytes" BLOB, - "is_archived" BOOLEAN, - "generate_preview_media" BOOLEAN, - "sync_preview_media" BOOLEAN, - "hidden" BOOLEAN, - "date_created" DATETIME, - "scan_state" INTEGER NOT NULL DEFAULT 0, - "instance_id" INTEGER, - CONSTRAINT "location_instance_id_fkey" FOREIGN KEY ("instance_id") REFERENCES "instance" ("id") ON DELETE SET NULL ON UPDATE CASCADE -); -INSERT INTO "new_location" ("available_capacity", "date_created", "generate_preview_media", "hidden", "id", "instance_id", "is_archived", "name", "path", "pub_id", "size_in_bytes", "sync_preview_media", "total_capacity") SELECT "available_capacity", "date_created", "generate_preview_media", "hidden", "id", "instance_id", "is_archived", "name", "path", "pub_id", "size_in_bytes", "sync_preview_media", "total_capacity" FROM "location"; -DROP TABLE "location"; -ALTER TABLE "new_location" RENAME TO "location"; -CREATE UNIQUE INDEX "location_pub_id_key" ON "location"("pub_id"); -PRAGMA foreign_key_check; -PRAGMA foreign_keys=ON; diff --git a/core/prisma/migrations/20240409202941_rename_media_data_to_exif_data/migration.sql b/core/prisma/migrations/20240409202941_rename_media_data_to_exif_data/migration.sql deleted file mode 100644 index c9069a8de..000000000 --- a/core/prisma/migrations/20240409202941_rename_media_data_to_exif_data/migration.sql +++ /dev/null @@ -1,52 +0,0 @@ --- CreateTable -CREATE TABLE "exif_data" ( - "id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, - "resolution" BLOB, - "media_date" BLOB, - "media_location" BLOB, - "camera_data" BLOB, - "artist" TEXT, - "description" TEXT, - "copyright" TEXT, - "exif_version" TEXT, - "epoch_time" BIGINT, - "object_id" INTEGER NOT NULL, - CONSTRAINT "exif_data_object_id_fkey" FOREIGN KEY ("object_id") REFERENCES "object" ("id") ON DELETE CASCADE ON UPDATE CASCADE -); - --- CopyData -INSERT INTO "exif_data" ( - "id", - "resolution", - "media_date", - "media_location", - "camera_data", - "artist", - "description", - "copyright", - "exif_version", - "epoch_time", - "object_id" -) -SELECT - "id", - "resolution", - "media_date", - "media_location", - "camera_data", - "artist", - "description", - "copyright", - "exif_version", - "epoch_time", - "object_id" -FROM - "media_data"; - --- DropTable -PRAGMA foreign_keys=off; -DROP TABLE "media_data"; -PRAGMA foreign_keys=on; - --- CreateIndex -CREATE UNIQUE INDEX "exif_data_object_id_key" ON "exif_data"("object_id"); diff --git a/core/prisma/migrations/20240508074615_ffmpeg_media_data/migration.sql b/core/prisma/migrations/20240508074615_ffmpeg_media_data/migration.sql deleted file mode 100644 index 2bc689a7e..000000000 --- a/core/prisma/migrations/20240508074615_ffmpeg_media_data/migration.sql +++ /dev/null @@ -1,128 +0,0 @@ --- CreateTable -CREATE TABLE "ffmpeg_data" ( - "id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, - "formats" TEXT NOT NULL, - "bit_rate" BLOB NOT NULL, - "duration" BLOB, - "start_time" BLOB, - "title" TEXT, - "creation_time" DATETIME, - "date" DATETIME, - "album_artist" TEXT, - "disc" TEXT, - "track" TEXT, - "album" TEXT, - "artist" TEXT, - "metadata" BLOB, - "object_id" INTEGER NOT NULL, - CONSTRAINT "ffmpeg_data_object_id_fkey" FOREIGN KEY ("object_id") REFERENCES "object" ("id") ON DELETE CASCADE ON UPDATE CASCADE -); - --- CreateTable -CREATE TABLE "ffmpeg_media_chapter" ( - "chapter_id" INTEGER NOT NULL, - "start" BLOB NOT NULL, - "end" BLOB NOT NULL, - "time_base_den" INTEGER NOT NULL, - "time_base_num" INTEGER NOT NULL, - "title" TEXT, - "metadata" BLOB, - "ffmpeg_data_id" INTEGER NOT NULL, - - PRIMARY KEY ("ffmpeg_data_id", "chapter_id"), - CONSTRAINT "ffmpeg_media_chapter_ffmpeg_data_id_fkey" FOREIGN KEY ("ffmpeg_data_id") REFERENCES "ffmpeg_data" ("id") ON DELETE CASCADE ON UPDATE CASCADE -); - --- CreateTable -CREATE TABLE "ffmpeg_media_program" ( - "program_id" INTEGER NOT NULL, - "name" TEXT, - "metadata" BLOB, - "ffmpeg_data_id" INTEGER NOT NULL, - - PRIMARY KEY ("ffmpeg_data_id", "program_id"), - CONSTRAINT "ffmpeg_media_program_ffmpeg_data_id_fkey" FOREIGN KEY ("ffmpeg_data_id") REFERENCES "ffmpeg_data" ("id") ON DELETE CASCADE ON UPDATE CASCADE -); - --- CreateTable -CREATE TABLE "ffmpeg_media_stream" ( - "stream_id" INTEGER NOT NULL, - "name" TEXT, - "aspect_ratio_num" INTEGER NOT NULL, - "aspect_ratio_den" INTEGER NOT NULL, - "frames_per_second_num" INTEGER NOT NULL, - "frames_per_second_den" INTEGER NOT NULL, - "time_base_real_den" INTEGER NOT NULL, - "time_base_real_num" INTEGER NOT NULL, - "dispositions" TEXT, - "title" TEXT, - "encoder" TEXT, - "language" TEXT, - "duration" BLOB, - "metadata" BLOB, - "program_id" INTEGER NOT NULL, - "ffmpeg_data_id" INTEGER NOT NULL, - - PRIMARY KEY ("ffmpeg_data_id", "program_id", "stream_id"), - CONSTRAINT "ffmpeg_media_stream_ffmpeg_data_id_program_id_fkey" FOREIGN KEY ("ffmpeg_data_id", "program_id") REFERENCES "ffmpeg_media_program" ("ffmpeg_data_id", "program_id") ON DELETE CASCADE ON UPDATE CASCADE -); - --- CreateTable -CREATE TABLE "ffmpeg_media_codec" ( - "id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, - "kind" TEXT, - "sub_kind" TEXT, - "tag" TEXT, - "name" TEXT, - "profile" TEXT, - "bit_rate" INTEGER NOT NULL, - "stream_id" INTEGER NOT NULL, - "program_id" INTEGER NOT NULL, - "ffmpeg_data_id" INTEGER NOT NULL, - CONSTRAINT "ffmpeg_media_codec_ffmpeg_data_id_program_id_stream_id_fkey" FOREIGN KEY ("ffmpeg_data_id", "program_id", "stream_id") REFERENCES "ffmpeg_media_stream" ("ffmpeg_data_id", "program_id", "stream_id") ON DELETE CASCADE ON UPDATE CASCADE -); - --- CreateTable -CREATE TABLE "ffmpeg_media_video_props" ( - "id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, - "pixel_format" TEXT, - "color_range" TEXT, - "bits_per_channel" INTEGER, - "color_space" TEXT, - "color_primaries" TEXT, - "color_transfer" TEXT, - "field_order" TEXT, - "chroma_location" TEXT, - "width" INTEGER NOT NULL, - "height" INTEGER NOT NULL, - "aspect_ratio_num" INTEGER, - "aspect_ratio_Den" INTEGER, - "properties" TEXT, - "codec_id" INTEGER NOT NULL, - CONSTRAINT "ffmpeg_media_video_props_codec_id_fkey" FOREIGN KEY ("codec_id") REFERENCES "ffmpeg_media_codec" ("id") ON DELETE CASCADE ON UPDATE CASCADE -); - --- CreateTable -CREATE TABLE "ffmpeg_media_audio_props" ( - "id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, - "delay" INTEGER NOT NULL, - "padding" INTEGER NOT NULL, - "sample_rate" INTEGER, - "sample_format" TEXT, - "bit_per_sample" INTEGER, - "channel_layout" TEXT, - "codec_id" INTEGER NOT NULL, - CONSTRAINT "ffmpeg_media_audio_props_codec_id_fkey" FOREIGN KEY ("codec_id") REFERENCES "ffmpeg_media_codec" ("id") ON DELETE CASCADE ON UPDATE CASCADE -); - --- CreateIndex -CREATE UNIQUE INDEX "ffmpeg_data_object_id_key" ON "ffmpeg_data"("object_id"); - --- CreateIndex -CREATE UNIQUE INDEX "ffmpeg_media_codec_ffmpeg_data_id_program_id_stream_id_key" ON "ffmpeg_media_codec"("ffmpeg_data_id", "program_id", "stream_id"); - --- CreateIndex -CREATE UNIQUE INDEX "ffmpeg_media_video_props_codec_id_key" ON "ffmpeg_media_video_props"("codec_id"); - --- CreateIndex -CREATE UNIQUE INDEX "ffmpeg_media_audio_props_codec_id_key" ON "ffmpeg_media_audio_props"("codec_id"); diff --git a/core/prisma/migrations/20240512011221_better_statistics/migration.sql b/core/prisma/migrations/20240512011221_better_statistics/migration.sql deleted file mode 100644 index 437f9d913..000000000 --- a/core/prisma/migrations/20240512011221_better_statistics/migration.sql +++ /dev/null @@ -1,29 +0,0 @@ -/* - Warnings: - - - You are about to drop the column `preview_media_bytes` on the `statistics` table. All the data in the column will be lost. - - You are about to drop the column `total_bytes_capacity` on the `statistics` table. All the data in the column will be lost. - - You are about to drop the column `total_bytes_free` on the `statistics` table. All the data in the column will be lost. - - You are about to drop the column `total_bytes_used` on the `statistics` table. All the data in the column will be lost. - - You are about to drop the column `total_unique_bytes` on the `statistics` table. All the data in the column will be lost. - -*/ --- RedefineTables -PRAGMA foreign_keys=OFF; -CREATE TABLE "new_statistics" ( - "id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, - "date_captured" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, - "total_object_count" INTEGER NOT NULL DEFAULT 0, - "library_db_size" TEXT NOT NULL DEFAULT '0', - "total_local_bytes_used" TEXT NOT NULL DEFAULT '0', - "total_local_bytes_capacity" TEXT NOT NULL DEFAULT '0', - "total_local_bytes_free" TEXT NOT NULL DEFAULT '0', - "total_library_bytes" TEXT NOT NULL DEFAULT '0', - "total_library_unique_bytes" TEXT NOT NULL DEFAULT '0', - "total_library_preview_media_bytes" TEXT NOT NULL DEFAULT '0' -); -INSERT INTO "new_statistics" ("date_captured", "id", "library_db_size", "total_object_count") SELECT "date_captured", "id", "library_db_size", "total_object_count" FROM "statistics"; -DROP TABLE "statistics"; -ALTER TABLE "new_statistics" RENAME TO "statistics"; -PRAGMA foreign_key_check; -PRAGMA foreign_keys=ON; diff --git a/core/prisma/migrations/20240517045456_change_indexer_rule_name_to_unique/migration.sql b/core/prisma/migrations/20240517045456_change_indexer_rule_name_to_unique/migration.sql deleted file mode 100644 index c938aff04..000000000 --- a/core/prisma/migrations/20240517045456_change_indexer_rule_name_to_unique/migration.sql +++ /dev/null @@ -1,12 +0,0 @@ --- Update duplicate names to make them unique -UPDATE "indexer_rule" -SET "name" = "name" || '_' || "id" -WHERE "name" IN ( - SELECT "name" - FROM "indexer_rule" - GROUP BY "name" - HAVING COUNT(*) > 1 -); - --- CreateIndex -CREATE UNIQUE INDEX "indexer_rule_name_key" ON "indexer_rule"("name"); diff --git a/core/prisma/migrations/20240531034707_add_node_remote_identity_to_instance/migration.sql b/core/prisma/migrations/20240531034707_add_node_remote_identity_to_instance/migration.sql deleted file mode 100644 index 8b2917a32..000000000 --- a/core/prisma/migrations/20240531034707_add_node_remote_identity_to_instance/migration.sql +++ /dev/null @@ -1,2 +0,0 @@ --- AlterTable -ALTER TABLE "instance" ADD COLUMN "node_remote_identity" BLOB; diff --git a/core/prisma/migrations/20240705205916_create_object_kind_statistics_table/migration.sql b/core/prisma/migrations/20240705205916_create_object_kind_statistics_table/migration.sql deleted file mode 100644 index b1e5c6340..000000000 --- a/core/prisma/migrations/20240705205916_create_object_kind_statistics_table/migration.sql +++ /dev/null @@ -1,6 +0,0 @@ --- CreateTable -CREATE TABLE "object_kind_statistics" ( - "kind" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, - "total_bytes" BIGINT NOT NULL DEFAULT 0, - "files_count" BIGINT NOT NULL DEFAULT 0 -); diff --git a/core/prisma/migrations/20240710015044_add_info_colum_to_job/migration.sql b/core/prisma/migrations/20240710015044_add_info_colum_to_job/migration.sql deleted file mode 100644 index 2b5f53d6b..000000000 --- a/core/prisma/migrations/20240710015044_add_info_colum_to_job/migration.sql +++ /dev/null @@ -1,2 +0,0 @@ --- AlterTable -ALTER TABLE "job" ADD COLUMN "info" TEXT; diff --git a/core/prisma/migrations/20240712155557_create_storage_statistics_table/migration.sql b/core/prisma/migrations/20240712155557_create_storage_statistics_table/migration.sql deleted file mode 100644 index bf465a4db..000000000 --- a/core/prisma/migrations/20240712155557_create_storage_statistics_table/migration.sql +++ /dev/null @@ -1,15 +0,0 @@ --- CreateTable -CREATE TABLE "storage_statistics" ( - "id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, - "pub_id" BLOB NOT NULL, - "total_capacity" BIGINT NOT NULL DEFAULT 0, - "available_capacity" BIGINT NOT NULL DEFAULT 0, - "instance_pub_id" BLOB, - CONSTRAINT "storage_statistics_instance_pub_id_fkey" FOREIGN KEY ("instance_pub_id") REFERENCES "instance" ("pub_id") ON DELETE CASCADE ON UPDATE CASCADE -); - --- CreateIndex -CREATE UNIQUE INDEX "storage_statistics_pub_id_key" ON "storage_statistics"("pub_id"); - --- CreateIndex -CREATE UNIQUE INDEX "storage_statistics_instance_pub_id_key" ON "storage_statistics"("instance_pub_id"); diff --git a/core/prisma/migrations/20250130041311_adding_devices/migration.sql b/core/prisma/migrations/20250130041311_adding_devices/migration.sql deleted file mode 100644 index ef8d6fa05..000000000 --- a/core/prisma/migrations/20250130041311_adding_devices/migration.sql +++ /dev/null @@ -1,226 +0,0 @@ -/* - Warnings: - - - You are about to drop the `node` table. If the table is not empty, all the data it contains will be lost. - - You are about to drop the `storage_statistics` table. If the table is not empty, all the data it contains will be lost. - - You are about to drop the column `instance_id` on the `cloud_crdt_operation` table. All the data in the column will be lost. - - You are about to drop the column `instance_id` on the `crdt_operation` table. All the data in the column will be lost. - - You are about to drop the column `filesystem` on the `volume` table. All the data in the column will be lost. - - You are about to drop the column `is_system` on the `volume` table. All the data in the column will be lost. - - Added the required column `device_pub_id` to the `cloud_crdt_operation` table without a default value. This is not possible if the table is not empty. - - Added the required column `device_pub_id` to the `crdt_operation` table without a default value. This is not possible if the table is not empty. - - Added the required column `pub_id` to the `volume` table without a default value. This is not possible if the table is not empty. - -*/ --- DropIndex -DROP INDEX "node_pub_id_key"; - --- DropIndex -DROP INDEX "storage_statistics_instance_pub_id_key"; - --- DropIndex -DROP INDEX "storage_statistics_pub_id_key"; - --- DropTable -PRAGMA foreign_keys=off; -DROP TABLE "node"; -PRAGMA foreign_keys=on; - --- DropTable -PRAGMA foreign_keys=off; -DROP TABLE "storage_statistics"; -PRAGMA foreign_keys=on; - --- CreateTable -CREATE TABLE "device" ( - "id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, - "pub_id" BLOB NOT NULL, - "name" TEXT, - "os" INTEGER, - "hardware_model" INTEGER, - "timestamp" BIGINT, - "date_created" DATETIME, - "date_deleted" DATETIME -); - --- RedefineTables -PRAGMA defer_foreign_keys=ON; -PRAGMA foreign_keys=OFF; -CREATE TABLE "new_cloud_crdt_operation" ( - "id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, - "timestamp" BIGINT NOT NULL, - "model" INTEGER NOT NULL, - "record_id" BLOB NOT NULL, - "kind" TEXT NOT NULL, - "data" BLOB NOT NULL, - "device_pub_id" BLOB NOT NULL -); -INSERT INTO "new_cloud_crdt_operation" ("data", "id", "kind", "model", "record_id", "timestamp") SELECT "data", "id", "kind", "model", "record_id", "timestamp" FROM "cloud_crdt_operation"; -DROP TABLE "cloud_crdt_operation"; -ALTER TABLE "new_cloud_crdt_operation" RENAME TO "cloud_crdt_operation"; -CREATE INDEX "cloud_crdt_operation_timestamp_idx" ON "cloud_crdt_operation"("timestamp"); -CREATE TABLE "new_crdt_operation" ( - "id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, - "timestamp" BIGINT NOT NULL, - "model" INTEGER NOT NULL, - "record_id" BLOB NOT NULL, - "kind" TEXT NOT NULL, - "data" BLOB NOT NULL, - "device_pub_id" BLOB NOT NULL -); -INSERT INTO "new_crdt_operation" ("data", "id", "kind", "model", "record_id", "timestamp") SELECT "data", "id", "kind", "model", "record_id", "timestamp" FROM "crdt_operation"; -DROP TABLE "crdt_operation"; -ALTER TABLE "new_crdt_operation" RENAME TO "crdt_operation"; -CREATE INDEX "crdt_operation_timestamp_idx" ON "crdt_operation"("timestamp"); -CREATE TABLE "new_exif_data" ( - "id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, - "resolution" BLOB, - "media_date" BLOB, - "media_location" BLOB, - "camera_data" BLOB, - "artist" TEXT, - "description" TEXT, - "copyright" TEXT, - "exif_version" TEXT, - "epoch_time" BIGINT, - "object_id" INTEGER NOT NULL, - "device_id" INTEGER, - CONSTRAINT "exif_data_object_id_fkey" FOREIGN KEY ("object_id") REFERENCES "object" ("id") ON DELETE CASCADE ON UPDATE CASCADE, - CONSTRAINT "exif_data_device_id_fkey" FOREIGN KEY ("device_id") REFERENCES "device" ("id") ON DELETE CASCADE ON UPDATE CASCADE -); -INSERT INTO "new_exif_data" ("artist", "camera_data", "copyright", "description", "epoch_time", "exif_version", "id", "media_date", "media_location", "object_id", "resolution") SELECT "artist", "camera_data", "copyright", "description", "epoch_time", "exif_version", "id", "media_date", "media_location", "object_id", "resolution" FROM "exif_data"; -DROP TABLE "exif_data"; -ALTER TABLE "new_exif_data" RENAME TO "exif_data"; -CREATE UNIQUE INDEX "exif_data_object_id_key" ON "exif_data"("object_id"); -CREATE TABLE "new_file_path" ( - "id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, - "pub_id" BLOB NOT NULL, - "is_dir" BOOLEAN, - "cas_id" TEXT, - "integrity_checksum" TEXT, - "location_id" INTEGER, - "materialized_path" TEXT, - "name" TEXT, - "extension" TEXT, - "hidden" BOOLEAN, - "size_in_bytes" TEXT, - "size_in_bytes_bytes" BLOB, - "inode" BLOB, - "object_id" INTEGER, - "key_id" INTEGER, - "date_created" DATETIME, - "date_modified" DATETIME, - "date_indexed" DATETIME, - "device_id" INTEGER, - CONSTRAINT "file_path_location_id_fkey" FOREIGN KEY ("location_id") REFERENCES "location" ("id") ON DELETE SET NULL ON UPDATE CASCADE, - CONSTRAINT "file_path_object_id_fkey" FOREIGN KEY ("object_id") REFERENCES "object" ("id") ON DELETE SET NULL ON UPDATE CASCADE, - CONSTRAINT "file_path_device_id_fkey" FOREIGN KEY ("device_id") REFERENCES "device" ("id") ON DELETE CASCADE ON UPDATE CASCADE -); -INSERT INTO "new_file_path" ("cas_id", "date_created", "date_indexed", "date_modified", "extension", "hidden", "id", "inode", "integrity_checksum", "is_dir", "key_id", "location_id", "materialized_path", "name", "object_id", "pub_id", "size_in_bytes", "size_in_bytes_bytes") SELECT "cas_id", "date_created", "date_indexed", "date_modified", "extension", "hidden", "id", "inode", "integrity_checksum", "is_dir", "key_id", "location_id", "materialized_path", "name", "object_id", "pub_id", "size_in_bytes", "size_in_bytes_bytes" FROM "file_path"; -DROP TABLE "file_path"; -ALTER TABLE "new_file_path" RENAME TO "file_path"; -CREATE UNIQUE INDEX "file_path_pub_id_key" ON "file_path"("pub_id"); -CREATE INDEX "file_path_location_id_idx" ON "file_path"("location_id"); -CREATE INDEX "file_path_location_id_materialized_path_idx" ON "file_path"("location_id", "materialized_path"); -CREATE UNIQUE INDEX "file_path_location_id_materialized_path_name_extension_key" ON "file_path"("location_id", "materialized_path", "name", "extension"); -CREATE UNIQUE INDEX "file_path_location_id_inode_key" ON "file_path"("location_id", "inode"); -CREATE TABLE "new_label_on_object" ( - "date_created" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, - "object_id" INTEGER NOT NULL, - "label_id" INTEGER NOT NULL, - "device_id" INTEGER, - - PRIMARY KEY ("label_id", "object_id"), - CONSTRAINT "label_on_object_object_id_fkey" FOREIGN KEY ("object_id") REFERENCES "object" ("id") ON DELETE RESTRICT ON UPDATE CASCADE, - CONSTRAINT "label_on_object_label_id_fkey" FOREIGN KEY ("label_id") REFERENCES "label" ("id") ON DELETE RESTRICT ON UPDATE CASCADE, - CONSTRAINT "label_on_object_device_id_fkey" FOREIGN KEY ("device_id") REFERENCES "device" ("id") ON DELETE CASCADE ON UPDATE CASCADE -); -INSERT INTO "new_label_on_object" ("date_created", "label_id", "object_id") SELECT "date_created", "label_id", "object_id" FROM "label_on_object"; -DROP TABLE "label_on_object"; -ALTER TABLE "new_label_on_object" RENAME TO "label_on_object"; -CREATE TABLE "new_location" ( - "id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, - "pub_id" BLOB NOT NULL, - "name" TEXT, - "path" TEXT, - "total_capacity" INTEGER, - "available_capacity" INTEGER, - "size_in_bytes" BLOB, - "is_archived" BOOLEAN, - "generate_preview_media" BOOLEAN, - "sync_preview_media" BOOLEAN, - "hidden" BOOLEAN, - "date_created" DATETIME, - "scan_state" INTEGER NOT NULL DEFAULT 0, - "device_id" INTEGER, - "instance_id" INTEGER, - CONSTRAINT "location_device_id_fkey" FOREIGN KEY ("device_id") REFERENCES "device" ("id") ON DELETE CASCADE ON UPDATE CASCADE, - CONSTRAINT "location_instance_id_fkey" FOREIGN KEY ("instance_id") REFERENCES "instance" ("id") ON DELETE SET NULL ON UPDATE CASCADE -); --- INSERT INTO "new_location" ("available_capacity", "date_created", "generate_preview_media", "hidden", "id", "instance_id", "is_archived", "name", "path", "pub_id", "scan_state", "size_in_bytes", "sync_preview_media", "total_capacity") SELECT "available_capacity", "date_created", "generate_preview_media", "hidden", "id", "instance_id", "is_archived", "name", "path", "pub_id", "scan_state", "size_in_bytes", "sync_preview_media", "total_capacity" FROM "location"; -INSERT INTO "new_location" ("available_capacity", "date_created", "generate_preview_media", "hidden", "id", "instance_id", "is_archived", "name", "path", "pub_id", "scan_state", "size_in_bytes", "sync_preview_media", "total_capacity", "device_id") -SELECT "available_capacity", "date_created", "generate_preview_media", "hidden", "id", "instance_id", "is_archived", "name", "path", "pub_id", "scan_state", "size_in_bytes", "sync_preview_media", "total_capacity", 1 FROM "location"; -DROP TABLE "location"; -ALTER TABLE "new_location" RENAME TO "location"; -CREATE UNIQUE INDEX "location_pub_id_key" ON "location"("pub_id"); -CREATE TABLE "new_object" ( - "id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, - "pub_id" BLOB NOT NULL, - "kind" INTEGER, - "key_id" INTEGER, - "hidden" BOOLEAN, - "favorite" BOOLEAN, - "important" BOOLEAN, - "note" TEXT, - "date_created" DATETIME, - "date_accessed" DATETIME, - "device_id" INTEGER, - CONSTRAINT "object_device_id_fkey" FOREIGN KEY ("device_id") REFERENCES "device" ("id") ON DELETE CASCADE ON UPDATE CASCADE -); -INSERT INTO "new_object" ("date_accessed", "date_created", "favorite", "hidden", "id", "important", "key_id", "kind", "note", "pub_id") SELECT "date_accessed", "date_created", "favorite", "hidden", "id", "important", "key_id", "kind", "note", "pub_id" FROM "object"; -DROP TABLE "object"; -ALTER TABLE "new_object" RENAME TO "object"; -CREATE UNIQUE INDEX "object_pub_id_key" ON "object"("pub_id"); -CREATE TABLE "new_tag_on_object" ( - "object_id" INTEGER NOT NULL, - "tag_id" INTEGER NOT NULL, - "date_created" DATETIME, - "device_id" INTEGER, - - PRIMARY KEY ("tag_id", "object_id"), - CONSTRAINT "tag_on_object_object_id_fkey" FOREIGN KEY ("object_id") REFERENCES "object" ("id") ON DELETE RESTRICT ON UPDATE CASCADE, - CONSTRAINT "tag_on_object_tag_id_fkey" FOREIGN KEY ("tag_id") REFERENCES "tag" ("id") ON DELETE RESTRICT ON UPDATE CASCADE, - CONSTRAINT "tag_on_object_device_id_fkey" FOREIGN KEY ("device_id") REFERENCES "device" ("id") ON DELETE CASCADE ON UPDATE CASCADE -); -INSERT INTO "new_tag_on_object" ("date_created", "object_id", "tag_id") SELECT "date_created", "object_id", "tag_id" FROM "tag_on_object"; -DROP TABLE "tag_on_object"; -ALTER TABLE "new_tag_on_object" RENAME TO "tag_on_object"; -CREATE TABLE "new_volume" ( - "id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, - "pub_id" BLOB NOT NULL, - "name" TEXT, - "mount_point" TEXT, - "mount_type" TEXT, - "total_bytes_capacity" TEXT, - "total_bytes_available" TEXT, - "disk_type" TEXT, - "file_system" TEXT, - "date_modified" DATETIME, - "is_mounted" BOOLEAN, - "read_speed_mbps" BIGINT, - "write_speed_mbps" BIGINT, - "read_only" BOOLEAN, - "error_status" TEXT, - "device_id" INTEGER, - CONSTRAINT "volume_device_id_fkey" FOREIGN KEY ("device_id") REFERENCES "device" ("id") ON DELETE CASCADE ON UPDATE CASCADE -); -INSERT INTO "new_volume" ("date_modified", "disk_type", "id", "mount_point", "name", "total_bytes_available", "total_bytes_capacity") SELECT "date_modified", "disk_type", "id", "mount_point", "name", "total_bytes_available", "total_bytes_capacity" FROM "volume"; -DROP TABLE "volume"; -ALTER TABLE "new_volume" RENAME TO "volume"; -CREATE UNIQUE INDEX "volume_pub_id_key" ON "volume"("pub_id"); -CREATE UNIQUE INDEX "volume_device_id_mount_point_name_total_bytes_capacity_file_system_key" ON "volume"("device_id", "mount_point", "name", "total_bytes_capacity", "file_system"); -PRAGMA foreign_keys=ON; -PRAGMA defer_foreign_keys=OFF; - --- CreateIndex -CREATE UNIQUE INDEX "device_pub_id_key" ON "device"("pub_id"); diff --git a/core/prisma/migrations/migration_lock.toml b/core/prisma/migrations/migration_lock.toml deleted file mode 100644 index e5e5c4705..000000000 --- a/core/prisma/migrations/migration_lock.toml +++ /dev/null @@ -1,3 +0,0 @@ -# Please do not edit this file manually -# It should be added in your version-control system (i.e. Git) -provider = "sqlite" \ No newline at end of file diff --git a/core/prisma/schema.prisma b/core/prisma/schema.prisma deleted file mode 100644 index a32a20863..000000000 --- a/core/prisma/schema.prisma +++ /dev/null @@ -1,741 +0,0 @@ -datasource db { - provider = "sqlite" - url = "file:dev.db" -} - -generator client { - provider = "cargo prisma" - output = "../../crates/prisma/src/prisma" - module_path = "prisma" - client_format = "folder" -} - -generator sync { - provider = "cargo prisma-sync" - output = "../../crates/prisma/src/prisma_sync" - client_format = "folder" -} - -/// @local -model CRDTOperation { - id Int @id @default(autoincrement()) - - timestamp BigInt - model Int - - record_id Bytes - // Enum: ?? - kind String - data Bytes - - // We just need the actual device_pub_id here, but we don't need as an actual relation - device_pub_id Bytes - - @@index([timestamp]) - @@map("crdt_operation") -} - -/// @local -model CloudCRDTOperation { - id Int @id @default(autoincrement()) - - timestamp BigInt - model Int - - record_id Bytes - // Enum: ?? - kind String - data Bytes - - // We just need the actual device_pub_id here, but we don't need as an actual relation - device_pub_id Bytes - - @@index([timestamp]) - @@map("cloud_crdt_operation") -} - -/// Devices are the owner machines connected to this library -/// @shared(id: pub_id, modelId: 12) -model Device { - id Int @id @default(autoincrement()) - // uuid v7 - pub_id Bytes @unique - name String? // Not actually NULLABLE, but we have to comply with current sync implementation BS - - // Enum: sd_cloud_schema::device::DeviceOS - os Int? // Not actually NULLABLE, but we have to comply with current sync implementation BS - // Enum: sd_cloud_schema::device::HardwareModel - hardware_model Int? // Not actually NULLABLE, but we have to comply with current sync implementation BS - - // clock timestamp for sync - timestamp BigInt? - - date_created DateTime? // Not actually NULLABLE, but we have to comply with current sync implementation BS - date_deleted DateTime? - - Location Location[] - FilePath FilePath[] - Object Object[] - ExifData ExifData[] - TagOnObject TagOnObject[] - LabelOnObject LabelOnObject[] - Volume Volume[] - - @@map("device") -} - -// represents a single `.db` file (SQLite DB) that is paired to the current library. -// A `LibraryInstance` is always owned by a single `Node` but it's possible for that node to change (or two to be owned by a single node). -/// @local(id: pub_id) -model Instance { - id Int @id @default(autoincrement()) // This is is NOT globally unique - pub_id Bytes @unique // This UUID is meaningless and exists soley cause the `uhlc::ID` must be 16-bit. Really this should be derived from the `identity` field. - // Enum: sd_p2p::Identity (or sd_core::p2p::IdentityOrRemoteIdentity in early versions) - identity Bytes? - // Enum: sd_core::node::RemoteIdentity - remote_identity Bytes - - // Enum: uuid::Uuid - node_id Bytes - // Enum: sd_core::node::RemoteIdentity - node_remote_identity Bytes? // TODO: This should not be optional - metadata Bytes? // TODO: This should not be optional - - last_seen DateTime // Time core started for owner, last P2P message for P2P node - date_created DateTime - - // clock timestamp for sync - timestamp BigInt? - Location Location[] - - @@map("instance") -} - -/// @local -model Statistics { - id Int @id @default(autoincrement()) - date_captured DateTime @default(now()) - total_object_count Int @default(0) - library_db_size String @default("0") - // local calulations - total_local_bytes_used String @default("0") - total_local_bytes_capacity String @default("0") - total_local_bytes_free String @default("0") - // library calculations - total_library_bytes String @default("0") - total_library_unique_bytes String @default("0") - total_library_preview_media_bytes String @default("0") - - @@map("statistics") -} - -/// @local -model ObjectKindStatistics { - kind Int @id - total_bytes BigInt @default(0) - files_count BigInt @default(0) - - @@map("object_kind_statistics") -} - -/// @shared(id: pub_id, modelId: 13) -model Volume { - id Int @id @default(autoincrement()) - pub_id Bytes @unique - - name String? - mount_point String? - mount_type String? - total_bytes_capacity String? - total_bytes_available String? - disk_type String? - file_system String? - date_modified DateTime? - is_mounted Boolean? - read_speed_mbps BigInt? - write_speed_mbps BigInt? - read_only Boolean? - error_status String? - - device_id Int? - device Device? @relation(fields: [device_id], references: [id], onDelete: Cascade) - - @@unique([device_id, mount_point, name, total_bytes_capacity, file_system]) - @@map("volume") -} - -/// @shared(id: pub_id, modelId: 1) -model Location { - id Int @id @default(autoincrement()) - pub_id Bytes @unique - - name String? - path String? - total_capacity Int? - available_capacity Int? - size_in_bytes Bytes? - is_archived Boolean? - generate_preview_media Boolean? - sync_preview_media Boolean? - hidden Boolean? - date_created DateTime? - - scan_state Int @default(0) // Enum: sd_core::location::ScanState - - device_id Int? - device Device? @relation(fields: [device_id], references: [id], onDelete: Cascade) - - // this should just be a local-only cache but it's too much effort to broadcast online locations rn (@brendan) - instance_id Int? - instance Instance? @relation(fields: [instance_id], references: [id], onDelete: SetNull) - - file_paths FilePath[] - indexer_rules IndexerRulesInLocation[] - - @@map("location") -} - -/// @shared(id: pub_id, modelId: 2) -model FilePath { - id Int @id @default(autoincrement()) - pub_id Bytes @unique - - is_dir Boolean? - - // content addressable storage id - blake3 sampled checksum - cas_id String? - // full byte contents digested into blake3 checksum - integrity_checksum String? - - // location that owns this path - location_id Int? - location Location? @relation(fields: [location_id], references: [id], onDelete: SetNull) - - // the path of the file relative to its location - materialized_path String? - - // the name and extension, MUST have 'COLLATE NOCASE' in migration - name String? - extension String? - hidden Boolean? - - size_in_bytes String? // deprecated - size_in_bytes_bytes Bytes? - - inode Bytes? // This is actually an unsigned 64 bit integer, but we don't have this type in SQLite - - // the unique Object for this file path - object_id Int? - object Object? @relation(fields: [object_id], references: [id], onDelete: SetNull) - - key_id Int? // replacement for encryption - // permissions String? - - date_created DateTime? - date_modified DateTime? - date_indexed DateTime? - - device_id Int? - device Device? @relation(fields: [device_id], references: [id], onDelete: Cascade) - - // key Key? @relation(fields: [key_id], references: [id]) - - @@unique([location_id, materialized_path, name, extension]) - @@unique([location_id, inode]) - @@index([location_id]) - @@index([location_id, materialized_path]) - @@map("file_path") -} - -/// @shared(id: pub_id, modelId: 3) -model Object { - id Int @id @default(autoincrement()) - pub_id Bytes @unique - // Enum: sd_file_ext::kind::ObjectKind - kind Int? - - key_id Int? - // handy ways to mark an object - hidden Boolean? - favorite Boolean? - important Boolean? - // if we have generated preview media for this object on at least one Node - // commented out for now by @brendonovich since they they're irrelevant to the sync system - // has_thumbnail Boolean? - // has_thumbstrip Boolean? - // has_video_preview Boolean? - // TODO: change above to: - // has_generated_thumbnail Boolean @default(false) - // has_generated_thumbstrip Boolean @default(false) - // has_generated_video_preview Boolean @default(false) - // integration with ipfs - // ipfs_id String? - // plain text note - note String? - // the original known creation date of this object - date_created DateTime? - date_accessed DateTime? - - tags TagOnObject[] - labels LabelOnObject[] - albums ObjectInAlbum[] - spaces ObjectInSpace[] - file_paths FilePath[] - // comments Comment[] - exif_data ExifData? - ffmpeg_data FfmpegData? - - device_id Int? - device Device? @relation(fields: [device_id], references: [id], onDelete: Cascade) - - // key Key? @relation(fields: [key_id], references: [id]) - - @@map("object") -} - -// // keys allow us to know exactly which files can be decrypted with a given key -// // they can be "mounted" to a client, and then used to decrypt files automatically -// /// @shared(id: uuid) -// model Key { -// id Int @id @default(autoincrement()) -// // uuid to identify the key -// uuid String @unique -// version String -// key_type String -// // the name that the user sets -// name String? -// // is this key the default for encryption? -// // was not tagged as unique as i'm not too sure if PCR will handle it -// // can always be tagged as unique, the keys API will need updating to use `find_unique()` -// default Boolean @default(false) -// // nullable if concealed for security -// date_created DateTime? @default(now()) -// // encryption algorithm used to encrypt the key -// algorithm String -// // hashing algorithm used for hashing the key with the content salt -// hashing_algorithm String -// // salt used for encrypting data with this key -// content_salt Bytes -// // the *encrypted* master key (48 bytes) -// master_key Bytes -// // the nonce used for encrypting the master key -// master_key_nonce Bytes -// // the nonce used for encrypting the key -// key_nonce Bytes -// // the *encrypted* key -// key Bytes -// // the salt used for deriving the KEK (used for encrypting the master key) from the root key -// salt Bytes - -// automount Boolean @default(false) - -// objects Object[] -// file_paths FilePath[] - -// @@map("key") -// } - -/// @shared(id: object, modelId: 4) -model ExifData { - id Int @id @default(autoincrement()) - - resolution Bytes? - media_date Bytes? - media_location Bytes? - camera_data Bytes? - artist String? - description String? - copyright String? - exif_version String? - - // purely for sorting/ordering, never sent to the frontend as they'd be useless - // these are also usually one-way, and not reversible - // (e.g. we can't get `MediaDate::Utc(2023-09-26T22:04:37+01:00)` from `1695758677` as we don't store the TZ) - epoch_time BigInt? // time since unix epoch - - object_id Int @unique - object Object @relation(fields: [object_id], references: [id], onDelete: Cascade) - - device_id Int? - device Device? @relation(fields: [device_id], references: [id], onDelete: Cascade) - - @@map("exif_data") -} - -model FfmpegData { - id Int @id @default(autoincrement()) - - // Internal FFmpeg properties - formats String - bit_rate Bytes // Actually a i64 in the backend - duration Bytes? // Actually a i64 in the backend - start_time Bytes? // Actually a i64 in the backend - - chapters FfmpegMediaChapter[] - programs FfmpegMediaProgram[] - - // Metadata for search - title String? - creation_time DateTime? - date DateTime? - album_artist String? - disc String? - track String? - album String? - artist String? - metadata Bytes? - - object Object @relation(fields: [object_id], references: [id], onDelete: Cascade) - object_id Int @unique - - @@map("ffmpeg_data") -} - -model FfmpegMediaChapter { - chapter_id Int - - start Bytes // Actually a i64 in the backend - end Bytes // Actually a i64 in the backend - - time_base_den Int - time_base_num Int - - // Metadata for search - title String? - metadata Bytes? - - ffmpeg_data FfmpegData @relation(fields: [ffmpeg_data_id], references: [id], onDelete: Cascade) - ffmpeg_data_id Int - - @@id(name: "likeId", [ffmpeg_data_id, chapter_id]) - @@map("ffmpeg_media_chapter") -} - -model FfmpegMediaProgram { - program_id Int - - streams FfmpegMediaStream[] - - // Metadata for search - name String? - metadata Bytes? - - ffmpeg_data FfmpegData @relation(fields: [ffmpeg_data_id], references: [id], onDelete: Cascade) - ffmpeg_data_id Int - - @@id(name: "likeId", [ffmpeg_data_id, program_id]) - @@map("ffmpeg_media_program") -} - -model FfmpegMediaStream { - stream_id Int - - name String? - codec FfmpegMediaCodec? - aspect_ratio_num Int - aspect_ratio_den Int - frames_per_second_num Int - frames_per_second_den Int - time_base_real_den Int - time_base_real_num Int - dispositions String? - - // Metadata for search - title String? - encoder String? - language String? - duration Bytes? // Actually a i64 in the backend - metadata Bytes? - - program FfmpegMediaProgram @relation(fields: [ffmpeg_data_id, program_id], references: [ffmpeg_data_id, program_id], onDelete: Cascade) - program_id Int - ffmpeg_data_id Int - - @@id(name: "likeId", [ffmpeg_data_id, program_id, stream_id]) - @@map("ffmpeg_media_stream") -} - -model FfmpegMediaCodec { - id Int @id @default(autoincrement()) - - kind String? - sub_kind String? - tag String? - name String? - profile String? - bit_rate Int - - video_props FfmpegMediaVideoProps? - audio_props FfmpegMediaAudioProps? - - stream FfmpegMediaStream @relation(fields: [ffmpeg_data_id, program_id, stream_id], references: [ffmpeg_data_id, program_id, stream_id], onDelete: Cascade) - stream_id Int - program_id Int - ffmpeg_data_id Int - - @@unique([ffmpeg_data_id, program_id, stream_id]) - @@map("ffmpeg_media_codec") -} - -model FfmpegMediaVideoProps { - id Int @id @default(autoincrement()) - - pixel_format String? - color_range String? - bits_per_channel Int? - color_space String? - color_primaries String? - color_transfer String? - field_order String? - chroma_location String? - width Int - height Int - aspect_ratio_num Int? - aspect_ratio_Den Int? - properties String? - - codec FfmpegMediaCodec @relation(fields: [codec_id], references: [id], onDelete: Cascade) - codec_id Int @unique - - @@map("ffmpeg_media_video_props") -} - -model FfmpegMediaAudioProps { - id Int @id @default(autoincrement()) - - delay Int - padding Int - sample_rate Int? - sample_format String? - bit_per_sample Int? - channel_layout String? - - codec FfmpegMediaCodec @relation(fields: [codec_id], references: [id], onDelete: Cascade) - codec_id Int @unique - - @@map("ffmpeg_media_audio_props") -} - -//// Tag //// - -/// @shared(id: pub_id, modelId: 5) -model Tag { - id Int @id @default(autoincrement()) - pub_id Bytes @unique - name String? - color String? - - is_hidden Boolean? // user hidden entire tag - - date_created DateTime? - date_modified DateTime? - - tag_objects TagOnObject[] - - @@map("tag") -} - -/// @relation(item: object, group: tag, modelId: 6) -model TagOnObject { - object_id Int - object Object @relation(fields: [object_id], references: [id], onDelete: Restrict) - - tag_id Int - tag Tag @relation(fields: [tag_id], references: [id], onDelete: Restrict) - - date_created DateTime? - - device_id Int? - device Device? @relation(fields: [device_id], references: [id], onDelete: Cascade) - - @@id([tag_id, object_id]) - @@map("tag_on_object") -} - -//// Label //// - -/// @shared(id: name, modelId: 7) -model Label { - id Int @id @default(autoincrement()) - name String @unique - date_created DateTime? - date_modified DateTime? - - label_objects LabelOnObject[] - - @@map("label") -} - -/// @relation(item: object, group: label, modelId: 8) -model LabelOnObject { - date_created DateTime @default(now()) - - object_id Int - object Object @relation(fields: [object_id], references: [id], onDelete: Restrict) - - label_id Int - label Label @relation(fields: [label_id], references: [id], onDelete: Restrict) - - device_id Int? - device Device? @relation(fields: [device_id], references: [id], onDelete: Cascade) - - @@id([label_id, object_id]) - @@map("label_on_object") -} - -//// Space //// - -model Space { - id Int @id @default(autoincrement()) - pub_id Bytes @unique - name String? - description String? - date_created DateTime? - date_modified DateTime? - - objects ObjectInSpace[] - - @@map("space") -} - -model ObjectInSpace { - space_id Int - space Space @relation(fields: [space_id], references: [id], onDelete: Restrict) - - object_id Int - object Object @relation(fields: [object_id], references: [id], onDelete: Restrict) - - @@id([space_id, object_id]) - @@map("object_in_space") -} - -//// Job //// - -model Job { - id Bytes @id - - name String? - action String? // Will be composed of "{action_description}(-{children_order})*" - - // Enum: sd_core::job::job_manager:JobStatus - status Int? // 0 = Queued - - // List of errors, separated by "\n\n" in case of failed jobs or completed with errors - errors_text String? // Deprecated, use `critical_error` or `non_critical_errors` instead - critical_error String? // Serialized error field with info about the failed job after completion - non_critical_errors Bytes? // Serialized non-critical errors field with info about the completed job with errors after completion - - data Bytes? // Deprecated - metadata Bytes? // Serialized metadata field with info about the job after completion - - parent_id Bytes? - - task_count Int? - completed_task_count Int? - info String? - date_estimated_completion DateTime? // Estimated timestamp that the job will be complete at - - date_created DateTime? - date_started DateTime? // Started execution - date_completed DateTime? // Finished execution - - parent Job? @relation("jobs_dependency", fields: [parent_id], references: [id], onDelete: SetNull) - children Job[] @relation("jobs_dependency") - - @@map("job") -} - -//// Album //// - -model Album { - id Int @id - pub_id Bytes @unique - name String? - is_hidden Boolean? - - date_created DateTime? - date_modified DateTime? - - objects ObjectInAlbum[] - - @@map("album") -} - -model ObjectInAlbum { - date_created DateTime? - album_id Int - album Album @relation(fields: [album_id], references: [id], onDelete: NoAction) - - object_id Int - object Object @relation(fields: [object_id], references: [id], onDelete: NoAction) - - @@id([album_id, object_id]) - @@map("object_in_album") -} - -//// Indexer Rules //// - -model IndexerRule { - id Int @id @default(autoincrement()) - pub_id Bytes @unique - - name String? @unique - default Boolean? - rules_per_kind Bytes? - date_created DateTime? - date_modified DateTime? - - locations IndexerRulesInLocation[] - - @@map("indexer_rule") -} - -model IndexerRulesInLocation { - location_id Int - location Location @relation(fields: [location_id], references: [id], onDelete: Restrict) - - indexer_rule_id Int - indexer_rule IndexerRule @relation(fields: [indexer_rule_id], references: [id], onDelete: Restrict) - - @@id([location_id, indexer_rule_id]) - @@map("indexer_rule_in_location") -} - -/// @shared(id: key, modelId: 9) -model Preference { - key String @id - value Bytes? - - @@map("preference") -} - -model Notification { - id Int @id @default(autoincrement()) - read Boolean @default(false) - // Enum: crate::api::notifications::NotificationData - data Bytes - expires_at DateTime? - - @@map("notification") -} - -/// @shared(id: pub_id, modelId: 10) -model SavedSearch { - id Int @id @default(autoincrement()) - pub_id Bytes @unique - - // enum: crate::api::search::saved::SearchTarget - target String? - search String? - filters String? - - name String? - icon String? - description String? - // order Int? // Add this line to include ordering - - date_created DateTime? - date_modified DateTime? - - @@map("saved_search") -} diff --git a/core-new/rust-toolchain.toml b/core/rust-toolchain.toml similarity index 100% rename from core-new/rust-toolchain.toml rename to core/rust-toolchain.toml diff --git a/core/src/api/backups.rs b/core/src/api/backups.rs deleted file mode 100644 index 01e49ced0..000000000 --- a/core/src/api/backups.rs +++ /dev/null @@ -1,499 +0,0 @@ -use crate::{ - invalidate_query, - library::{Library, LibraryManagerError}, - Node, -}; - -use sd_utils::error::FileIOError; - -use std::{ - cmp, - path::{Path, PathBuf}, - sync::Arc, - time::{SystemTime, UNIX_EPOCH}, -}; - -use flate2::{bufread::GzDecoder, write::GzEncoder, Compression}; -use futures::executor::block_on; -use futures_concurrency::future::TryJoin; -use rspc::{alpha::AlphaRouter, ErrorCode}; -use serde::{Serialize, Serializer}; -use specta::Type; -use tar::Archive; -use tempfile::tempdir; -use thiserror::Error; -use tokio::{ - fs::{self, File}, - io::{ - self, AsyncBufReadExt, AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt, BufReader, - BufWriter, - }, - spawn, -}; -use tracing::{error, info}; -use uuid::Uuid; - -use super::{utils::library, Ctx, R}; - -pub(crate) fn mount() -> AlphaRouter { - R.router() - .procedure("getAll", { - #[derive(Serialize, Type)] - pub struct Backup { - #[serde(flatten)] - header: Header, - path: PathBuf, - } - - #[derive(Serialize, Type)] - pub struct GetAll { - backups: Vec, - directory: PathBuf, - } - - async fn process_backups(path: impl AsRef) -> Result, BackupError> { - let path = path.as_ref(); - - let mut read_dir = fs::read_dir(path).await.map_err(|e| { - FileIOError::from((&path, e, "Failed to read backups directory")) - })?; - - let mut backups = vec![]; - - while let Some(entry) = read_dir.next_entry().await.map_err(|e| { - FileIOError::from((path, e, "Failed to read next entry to backup")) - })? { - let entry_path = entry.path(); - - let metadata = entry.metadata().await.map_err(|e| { - FileIOError::from(( - &entry_path, - e, - "Failed to read metadata from backup entry", - )) - })?; - - if metadata.is_file() { - backups.push(async move { - let mut file = File::open(&entry_path).await.map_err(|e| { - FileIOError::from((&entry_path, e, "Failed to open backup entry")) - })?; - - Header::read(&mut file, &entry_path) - .await - .map(|header| Backup { - header, - path: entry_path, - }) - }); - } - } - - backups.try_join().await - } - - R.query(|node, _: ()| async move { - let directory = node.data_dir.join("backups"); - - let backups = match fs::metadata(&directory).await { - Ok(_) => process_backups(directory.clone()).await.map_err(|e| { - rspc::Error::with_cause( - ErrorCode::InternalServerError, - "Failed to fetch backups".to_string(), - e, - ) - })?, - Err(e) if e.kind() == io::ErrorKind::NotFound => vec![], - Err(e) => { - return Err( - FileIOError::from((&directory, e, "Failed to fetch backups")).into(), - ) - } - }; - - Ok(GetAll { backups, directory }) - }) - }) - .procedure("backup", { - R.with2(library()) - .mutation( - |(node, library), _: ()| async move { Ok(start_backup(node, library).await) }, - ) - }) - .procedure("restore", { - R.mutation(|node, path: PathBuf| async move { - start_restore(node, path).await; - Ok(()) - }) - }) - .procedure("delete", { - R.mutation(|node, path: PathBuf| async move { - fs::remove_file(path) - .await - .map(|_| { - invalidate_query!(node; node, "backups.getAll"); - }) - .map_err(|_| { - rspc::Error::new( - ErrorCode::InternalServerError, - "Error deleting backup!".to_string(), - ) - }) - }) - }) -} - -async fn start_backup(node: Arc, library: Arc) -> Uuid { - let bkp_id = Uuid::new_v4(); - - spawn(async move { - match do_backup(bkp_id, &node, &library).await { - Ok(path) => { - info!( - backup_id = %bkp_id, - library_id = %library.id, - path = %path.display(), - "Backup created!;", - ); - invalidate_query!(library, "backups.getAll"); - } - Err(e) => { - error!( - backup_id = %bkp_id, - library_id = %library.id, - ?e, - "Error with backup for library;", - ); - - // TODO: Alert user something went wrong - } - } - }); - - bkp_id -} - -#[derive(Error, Debug)] -enum BackupError { - #[error("library manager error: {0}")] - LibraryManager(#[from] LibraryManagerError), - #[error("malformed header")] - MalformedHeader, - #[error("Library already exists, please remove it and try again!")] - LibraryAlreadyExists, - - #[error(transparent)] - FileIO(#[from] FileIOError), -} - -async fn do_backup(id: Uuid, node: &Node, library: &Library) -> Result { - let backups_dir = node.data_dir.join("backups"); - fs::create_dir_all(&backups_dir) - .await - .map_err(|e| FileIOError::from((&backups_dir, e)))?; - - let timestamp = SystemTime::now() - .duration_since(UNIX_EPOCH) - .expect("Time went backwards") - .as_millis(); - - let bkp_path = backups_dir.join(format!("{id}.bkp")); - let mut bkp_file = BufWriter::new( - File::create(&bkp_path) - .await - .map_err(|e| FileIOError::from((&bkp_path, e, "Failed to create backup file")))?, - ); - - // Header. We do this so the file is self-sufficient. - Header { - id, - timestamp, - library_id: library.id, - library_name: library.config().await.name.to_string(), - } - .write(&mut bkp_file) - .await - .map_err(|e| FileIOError::from((&bkp_path, e, "Failed to create backup file")))?; - - // Introducing this adapter here to bridge tokio stuff to std::io stuff - struct WriterAdapter(BufWriter); - - impl std::io::Write for WriterAdapter { - fn write(&mut self, buf: &[u8]) -> io::Result { - block_on(self.0.write(buf)) - } - - fn flush(&mut self) -> io::Result<()> { - block_on(self.0.flush()) - } - } - - // Regular tar.gz encoded data - let mut tar = tar::Builder::new(GzEncoder::new( - WriterAdapter(bkp_file), - Compression::default(), - )); - - let library_config_path = node - .libraries - .libraries_dir - .join(format!("{}.sdlibrary", library.id)); - - tar.append_file( - "library.sdlibrary", - &mut std::fs::File::open(&library_config_path).map_err(|e| { - FileIOError::from(( - library_config_path, - e, - "Failed to open library config file to do a backup", - )) - })?, - ) - .map_err(|e| { - FileIOError::from(( - &bkp_path, - e, - "Failed to append library config file to out backup tar.gz file", - )) - })?; - - let library_db_path = node - .libraries - .libraries_dir - .join(format!("{}.db", library.id)); - - tar.append_file( - "library.db", - &mut std::fs::File::open(&library_db_path).map_err(|e| { - FileIOError::from(( - library_db_path, - e, - "Failed to open library database file to do a backup", - )) - })?, - ) - .map_err(|e| { - FileIOError::from(( - &bkp_path, - e, - "Failed to append library database file to out backup tar.gz file", - )) - })?; - - Ok(bkp_path) -} - -async fn start_restore(node: Arc, path: PathBuf) { - match restore_backup(&node, &path).await { - Ok(Header { id, library_id, .. }) => { - info!(%id, %library_id, "Restored backup for library!"); - } - Err(e) => { - error!(path = %path.display(), ?e, "Error restoring backup;"); - - // TODO: Alert user something went wrong - } - } -} - -async fn restore_backup(node: &Arc, path: impl AsRef) -> Result { - let path = path.as_ref(); - - let mut file = BufReader::new(fs::File::open(path).await.map_err(|e| { - FileIOError::from((path, e, "Failed trying to open backup file to be restored")) - })?); - - let header = Header::read(&mut file, path).await?; - - // TODO: Actually handle restoring into a library that exists. For now it's easier to error out. - let None = node.libraries.get_library(&header.library_id).await else { - return Err(BackupError::LibraryAlreadyExists); - }; - - let temp_dir = tempdir().map_err(|e| { - FileIOError::from(( - "/tmp", - e, - "Failed to get a temporary directory to restore backup", - )) - })?; - - // Introducing this adapter here to bridge tokio stuff to std::io stuff - struct ReaderAdapter(BufReader); - - impl std::io::Read for ReaderAdapter { - fn read(&mut self, buf: &mut [u8]) -> io::Result { - block_on(self.0.read(buf)) - } - } - - impl std::io::BufRead for ReaderAdapter { - fn fill_buf(&mut self) -> io::Result<&[u8]> { - block_on(self.0.fill_buf()) - } - - fn consume(&mut self, amt: usize) { - self.0.consume(amt) - } - } - - let temp_dir_path = temp_dir.path(); - - let mut archive = Archive::new(GzDecoder::new(ReaderAdapter(file))); - archive.unpack(&temp_dir).map_err(|e| { - FileIOError::from((temp_dir_path, e, "Failed to unpack backup compressed data")) - })?; - - let library_config_path = temp_dir_path.join("library.sdlibrary"); - - let library_config_restored_path = node - .libraries - .libraries_dir - .join(format!("{}.sdlibrary", header.library_id)); - - fs::copy(library_config_path, &library_config_restored_path) - .await - .map_err(|e| { - FileIOError::from(( - &library_config_restored_path, - e, - "Failed to restore library config file from backup", - )) - })?; - - let db_path = temp_dir_path.join("library.db"); - let db_restored_path = node - .libraries - .libraries_dir - .join(format!("{}.db", header.library_id)); - - fs::copy(db_path, &db_restored_path).await.map_err(|e| { - FileIOError::from(( - &db_restored_path, - e, - "Failed to restore library database file from backup", - )) - })?; - - node.libraries - .load( - header.library_id, - db_restored_path, - library_config_restored_path, - None, - None, - true, - node, - ) - .await?; - - Ok(header) -} - -#[derive(Debug, PartialEq, Eq, Serialize, Type)] -struct Header { - // Backup unique id - id: Uuid, - // Time since epoch the backup was created at - #[specta(type = String)] - #[serde(serialize_with = "as_string")] - timestamp: u128, - // Library id - library_id: Uuid, - // Library display name - library_name: String, -} - -fn as_string(x: &T, s: S) -> Result -where - S: Serializer, -{ - s.serialize_str(&x.to_string()) -} - -impl Header { - async fn write(&self, file: &mut (impl AsyncWrite + Unpin)) -> Result<(), io::Error> { - // For future versioning we can bump `1` to `2` and match on it in the decoder. - file.write_all(b"sdbkp1").await?; - file.write_all(&self.id.to_bytes_le()).await?; - file.write_all(&self.timestamp.to_le_bytes()).await?; - file.write_all(&self.library_id.to_bytes_le()).await?; - { - let bytes = &self.library_name.as_bytes() - [..cmp::min(u32::MAX as usize, self.library_name.len())]; - file.write_all(&(bytes.len() as u32).to_le_bytes()).await?; - file.write_all(bytes).await?; - } - - Ok(()) - } - - async fn read( - file: &mut (impl AsyncRead + Unpin), - path: impl AsRef, - ) -> Result { - let mut buf = vec![0u8; 6 + 16 + 16 + 16 + 4]; - let path = path.as_ref(); - file.read_exact(&mut buf) - .await - .map_err(|e| FileIOError::from((path, e)))?; - - if &buf[..6] != b"sdbkp1" { - return Err(BackupError::MalformedHeader); - } - - Ok(Self { - id: Uuid::from_bytes_le( - buf[6..22] - .try_into() - .map_err(|_| BackupError::MalformedHeader)?, - ), - timestamp: u128::from_le_bytes( - buf[22..38] - .try_into() - .map_err(|_| BackupError::MalformedHeader)?, - ), - library_id: Uuid::from_bytes_le( - buf[38..54] - .try_into() - .map_err(|_| BackupError::MalformedHeader)?, - ), - - library_name: { - let len = u32::from_le_bytes( - buf[54..58] - .try_into() - .map_err(|_| BackupError::MalformedHeader)?, - ); - - let mut name = vec![0; len as usize]; - file.read_exact(&mut name) - .await - .map_err(|e| FileIOError::from((path, e)))?; - - String::from_utf8(name).map_err(|_| BackupError::MalformedHeader)? - }, - }) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[tokio::test] - async fn test_backup_header() { - let original = Header { - id: Uuid::new_v4(), - timestamp: 1234567890, - library_id: Uuid::new_v4(), - library_name: "Test Library".to_string(), - }; - - let mut buf = Vec::new(); - original.write(&mut buf).await.unwrap(); - - let decoded = Header::read(&mut buf.as_slice(), "").await.unwrap(); - assert_eq!(original, decoded); - } -} diff --git a/core/src/api/cloud/devices.rs b/core/src/api/cloud/devices.rs deleted file mode 100644 index ead1db58e..000000000 --- a/core/src/api/cloud/devices.rs +++ /dev/null @@ -1,397 +0,0 @@ -use crate::api::{Ctx, R}; - -use sd_core_cloud_services::QuinnConnector; - -use sd_cloud_schema::{ - auth::AccessToken, - devices::{self, DeviceOS, HardwareModel, PubId}, - opaque_ke::{ - ClientLogin, ClientLoginFinishParameters, ClientLoginFinishResult, ClientLoginStartResult, - ClientRegistration, ClientRegistrationFinishParameters, ClientRegistrationFinishResult, - ClientRegistrationStartResult, - }, - Client, NodeId, Request, Response, SpacedriveCipherSuite, -}; -use sd_crypto::{cloud::secret_key::SecretKey, CryptoRng}; - -use blake3::Hash; -use futures::{FutureExt, SinkExt, StreamExt}; -use futures_concurrency::future::TryJoin; -use rspc::alpha::AlphaRouter; -use serde::Deserialize; -use tracing::{debug, error}; - -pub fn mount() -> AlphaRouter { - R.router() - .procedure("get", { - R.query(|node, pub_id: devices::PubId| async move { - use devices::get::{Request, Response}; - - let (client, access_token) = super::get_client_and_access_token(&node).await?; - - let Response(device) = super::handle_comm_error( - client - .devices() - .get(Request { - pub_id, - access_token, - }) - .await, - "Failed to get device;", - )??; - - debug!(?device, "Got device"); - - Ok(device) - }) - }) - .procedure("list", { - R.query(|node, _: ()| async move { - use devices::list::{Request, Response}; - - let ((client, access_token), pub_id) = ( - super::get_client_and_access_token(&node), - node.config.get().map(|config| Ok(config.id.into())), - ) - .try_join() - .await?; - - let Response(mut devices) = super::handle_comm_error( - client.devices().list(Request { access_token }).await, - "Failed to list devices;", - )??; - - // Filter out the local device by matching pub_id - devices.retain(|device| device.pub_id != pub_id); - - debug!(?devices, "Listed devices"); - - Ok(devices) - }) - }) - .procedure("get_current_device", { - R.query(|node, _: ()| async move { - use devices::get::{Request, Response}; - - let ((client, access_token), pub_id) = ( - super::get_client_and_access_token(&node), - node.config.get().map(|config| Ok(config.id.into())), - ) - .try_join() - .await?; - - let Response(device) = super::handle_comm_error( - client - .devices() - .get(Request { - pub_id, - access_token, - }) - .await, - "Failed to get current device;", - )??; - Ok(device) - }) - }) - .procedure("delete", { - R.mutation(|node, pub_id: devices::PubId| async move { - use devices::delete::Request; - - let (client, access_token) = super::get_client_and_access_token(&node).await?; - - super::handle_comm_error( - client - .devices() - .delete(Request { - pub_id, - access_token, - }) - .await, - "Failed to delete device;", - )??; - - debug!("Deleted device"); - - Ok(()) - }) - }) - .procedure("update", { - #[derive(Deserialize, specta::Type)] - struct CloudUpdateDeviceArgs { - pub_id: devices::PubId, - name: String, - } - - R.mutation( - |node, CloudUpdateDeviceArgs { pub_id, name }: CloudUpdateDeviceArgs| async move { - use devices::update::Request; - - let (client, access_token) = super::get_client_and_access_token(&node).await?; - - super::handle_comm_error( - client - .devices() - .update(Request { - access_token, - pub_id, - name, - }) - .await, - "Failed to update device;", - )??; - - debug!("Updated device"); - - Ok(()) - }, - ) - }) -} - -pub async fn hello( - client: &Client>, - access_token: AccessToken, - device_pub_id: PubId, - hashed_pub_id: Hash, - rng: &mut CryptoRng, -) -> Result { - use devices::hello::{Request, RequestUpdate, Response, State}; - - let ClientLoginStartResult { message, state } = - ClientLogin::::start(rng, hashed_pub_id.as_bytes().as_slice()) - .map_err(|e| { - error!(?e, "OPAQUE error initializing device hello request;"); - rspc::Error::new( - rspc::ErrorCode::InternalServerError, - "Failed to initialize device login".into(), - ) - })?; - - let (mut hello_continuation, mut res_stream) = super::handle_comm_error( - client - .devices() - .hello(Request { - access_token, - pub_id: device_pub_id, - opaque_login_message: Box::new(message), - }) - .await, - "Failed to send device hello request;", - )?; - - let Some(res) = res_stream.next().await else { - let message = "Server did not send a device hello response;"; - error!("{message}"); - return Err(rspc::Error::new( - rspc::ErrorCode::InternalServerError, - message.to_string(), - )); - }; - - let credential_response = match super::handle_comm_error( - res, - "Communication error on device hello response;", - )? { - Ok(Response(State::LoginResponse(credential_response))) => credential_response, - - Ok(Response(State::End)) => { - unreachable!("Device hello response MUST not be End here, this is a serious bug and should crash;"); - } - - Err(e) => { - error!(?e, "Device hello response error;"); - return Err(e.into()); - } - }; - - let ClientLoginFinishResult { - message, - export_key, - .. - } = state - .finish( - hashed_pub_id.as_bytes().as_slice(), - *credential_response, - ClientLoginFinishParameters::default(), - ) - .map_err(|e| { - error!(?e, "Device hello finish error;"); - rspc::Error::new( - rspc::ErrorCode::InternalServerError, - "Failed to finish device login".into(), - ) - })?; - - hello_continuation - .send(RequestUpdate { - opaque_login_finish: Box::new(message), - }) - .await - .map_err(|e| { - error!(?e, "Failed to send device hello request continuation;"); - rspc::Error::new( - rspc::ErrorCode::InternalServerError, - "Failed to finish device login procedure;".into(), - ) - })?; - - let Some(res) = res_stream.next().await else { - let message = "Server did not send a device hello END response;"; - error!("{message}"); - return Err(rspc::Error::new( - rspc::ErrorCode::InternalServerError, - message.to_string(), - )); - }; - - match super::handle_comm_error(res, "Communication error on device hello response;")? { - Ok(Response(State::LoginResponse(_))) => { - unreachable!("Device hello final response MUST be End here, this is a serious bug and should crash;"); - } - - Ok(Response(State::End)) => { - // Protocol completed successfully - Ok(SecretKey::from(export_key)) - } - - Err(e) => { - error!(?e, "Device hello final response error;"); - Err(e.into()) - } - } -} - -pub struct DeviceRegisterData { - pub pub_id: PubId, - pub name: String, - pub os: DeviceOS, - pub hardware_model: HardwareModel, - pub connection_id: NodeId, -} - -pub async fn register( - client: &Client>, - access_token: AccessToken, - DeviceRegisterData { - pub_id, - name, - os, - hardware_model, - connection_id, - }: DeviceRegisterData, - hashed_pub_id: Hash, - rng: &mut CryptoRng, -) -> Result { - use devices::register::{Request, RequestUpdate, Response, State}; - - let ClientRegistrationStartResult { message, state } = - ClientRegistration::::start( - rng, - hashed_pub_id.as_bytes().as_slice(), - ) - .map_err(|e| { - error!(?e, "OPAQUE error initializing device register request;"); - rspc::Error::new( - rspc::ErrorCode::InternalServerError, - "Failed to initialize device register".into(), - ) - })?; - - let (mut register_continuation, mut res_stream) = super::handle_comm_error( - client - .devices() - .register(Request { - access_token, - pub_id, - name, - os, - hardware_model, - connection_id, - opaque_register_message: Box::new(message), - }) - .await, - "Failed to send device register request;", - )?; - - let Some(res) = res_stream.next().await else { - let message = "Server did not send a device register response;"; - error!("{message}"); - return Err(rspc::Error::new( - rspc::ErrorCode::InternalServerError, - message.to_string(), - )); - }; - - let registration_response = match super::handle_comm_error( - res, - "Communication error on device register response;", - )? { - Ok(Response(State::RegistrationResponse(res))) => res, - - Ok(Response(State::End)) => { - unreachable!("Device hello response MUST not be End here, this is a serious bug and should crash;"); - } - - Err(e) => { - error!(?e, "Device hello response error;"); - return Err(e.into()); - } - }; - - let ClientRegistrationFinishResult { - message, - export_key, - .. - } = state - .finish( - rng, - hashed_pub_id.as_bytes().as_slice(), - *registration_response, - ClientRegistrationFinishParameters::default(), - ) - .map_err(|e| { - error!(?e, "Device register finish error;"); - rspc::Error::new( - rspc::ErrorCode::InternalServerError, - "Failed to finish device register".into(), - ) - })?; - - register_continuation - .send(RequestUpdate { - opaque_registration_finish: Box::new(message), - }) - .await - .map_err(|e| { - error!(?e, "Failed to send device register request continuation;"); - rspc::Error::new( - rspc::ErrorCode::InternalServerError, - "Failed to finish device register procedure;".into(), - ) - })?; - - let Some(res) = res_stream.next().await else { - let message = "Server did not send a device register END response;"; - error!("{message}"); - return Err(rspc::Error::new( - rspc::ErrorCode::InternalServerError, - message.to_string(), - )); - }; - - match super::handle_comm_error(res, "Communication error on device register response;")? { - Ok(Response(State::RegistrationResponse(_))) => { - unreachable!("Device register final response MUST be End here, this is a serious bug and should crash;"); - } - - Ok(Response(State::End)) => { - // Protocol completed successfully - Ok(SecretKey::from(export_key)) - } - - Err(e) => { - error!(?e, "Device register final response error;"); - Err(e.into()) - } - } -} diff --git a/core/src/api/cloud/libraries.rs b/core/src/api/cloud/libraries.rs deleted file mode 100644 index 884e5e21b..000000000 --- a/core/src/api/cloud/libraries.rs +++ /dev/null @@ -1,140 +0,0 @@ -use crate::api::{utils::library, Ctx, R}; - -use sd_cloud_schema::libraries; - -use futures::FutureExt; -use futures_concurrency::future::TryJoin; -use rspc::alpha::AlphaRouter; -use serde::Deserialize; -use tracing::debug; - -pub fn mount() -> AlphaRouter { - R.router() - .procedure("get", { - #[derive(Deserialize, specta::Type)] - struct CloudGetLibraryArgs { - pub_id: libraries::PubId, - with_device: bool, - } - - R.query( - |node, - CloudGetLibraryArgs { - pub_id, - with_device, - }: CloudGetLibraryArgs| async move { - use libraries::get::{Request, Response}; - - let (client, access_token) = super::get_client_and_access_token(&node).await?; - - let Response(library) = super::handle_comm_error( - client - .libraries() - .get(Request { - access_token, - pub_id, - with_device, - }) - .await, - "Failed to get library;", - )??; - - debug!(?library, "Got library"); - - Ok(library) - }, - ) - }) - .procedure("list", { - R.query(|node, with_device: bool| async move { - use libraries::list::{Request, Response}; - - let (client, access_token) = super::get_client_and_access_token(&node).await?; - - let Response(libraries) = super::handle_comm_error( - client - .libraries() - .list(Request { - access_token, - with_device, - }) - .await, - "Failed to list libraries;", - )??; - - debug!(?libraries, "Listed libraries"); - - Ok(libraries) - }) - }) - .procedure("create", { - R.with2(library()) - .mutation(|(node, library), _: ()| async move { - let ((client, access_token), name, device_pub_id) = ( - super::get_client_and_access_token(&node), - library.config().map(|config| Ok(config.name.to_string())), - node.config.get().map(|config| Ok(config.id.into())), - ) - .try_join() - .await?; - - super::handle_comm_error( - client - .libraries() - .create(libraries::create::Request { - name, - access_token, - pub_id: libraries::PubId(library.id), - device_pub_id, - }) - .await, - "Failed to create library;", - )??; - - Ok(()) - }) - }) - .procedure("delete", { - R.with2(library()) - .mutation(|(node, library), _: ()| async move { - let (client, access_token) = super::get_client_and_access_token(&node).await?; - - super::handle_comm_error( - client - .libraries() - .delete(libraries::delete::Request { - access_token, - pub_id: libraries::PubId(library.id), - }) - .await, - "Failed to delete library;", - )??; - - debug!("Deleted library"); - - Ok(()) - }) - }) - .procedure("update", { - R.with2(library()) - .mutation(|(node, library), name: String| async move { - let (client, access_token) = super::get_client_and_access_token(&node).await?; - - super::handle_comm_error( - client - .libraries() - .update(libraries::update::Request { - access_token, - pub_id: libraries::PubId(library.id), - name, - }) - .await, - "Failed to update library;", - )??; - - debug!("Updated library"); - - Ok(()) - }) - }) -} diff --git a/core/src/api/cloud/locations.rs b/core/src/api/cloud/locations.rs deleted file mode 100644 index e41e3a865..000000000 --- a/core/src/api/cloud/locations.rs +++ /dev/null @@ -1,112 +0,0 @@ -use crate::api::{Ctx, R}; - -use sd_cloud_schema::{devices, libraries, locations}; - -use rspc::alpha::AlphaRouter; -use serde::Deserialize; -use tracing::debug; - -pub fn mount() -> AlphaRouter { - R.router() - .procedure("list", { - #[derive(Deserialize, specta::Type)] - struct CloudListLocationsArgs { - pub library_pub_id: libraries::PubId, - pub with_library: bool, - pub with_device: bool, - } - - R.query( - |node, - CloudListLocationsArgs { - library_pub_id, - with_library, - with_device, - }: CloudListLocationsArgs| async move { - use locations::list::{Request, Response}; - - let (client, access_token) = super::get_client_and_access_token(&node).await?; - - let Response(locations) = super::handle_comm_error( - client - .locations() - .list(Request { - access_token, - library_pub_id, - with_library, - with_device, - }) - .await, - "Failed to list locations;", - )??; - - debug!(?locations, "Got locations"); - - Ok(locations) - }, - ) - }) - .procedure("create", { - #[derive(Deserialize, specta::Type)] - struct CloudCreateLocationArgs { - pub pub_id: locations::PubId, - pub name: String, - pub library_pub_id: libraries::PubId, - pub device_pub_id: devices::PubId, - } - - R.mutation( - |node, - CloudCreateLocationArgs { - pub_id, - name, - library_pub_id, - device_pub_id, - }: CloudCreateLocationArgs| async move { - use locations::create::Request; - - let (client, access_token) = super::get_client_and_access_token(&node).await?; - - super::handle_comm_error( - client - .locations() - .create(Request { - access_token, - pub_id, - name, - library_pub_id, - device_pub_id, - }) - .await, - "Failed to list locations;", - )??; - - debug!("Created cloud location"); - - Ok(()) - }, - ) - }) - .procedure("delete", { - R.mutation(|node, pub_id: locations::PubId| async move { - use locations::delete::Request; - - let (client, access_token) = super::get_client_and_access_token(&node).await?; - - super::handle_comm_error( - client - .locations() - .delete(Request { - access_token, - pub_id, - }) - .await, - "Failed to list locations;", - )??; - - debug!("Created cloud location"); - - Ok(()) - }) - }) -} diff --git a/core/src/api/cloud/mod.rs b/core/src/api/cloud/mod.rs deleted file mode 100644 index 606fca1ed..000000000 --- a/core/src/api/cloud/mod.rs +++ /dev/null @@ -1,391 +0,0 @@ -use crate::{ - library::LibraryManagerError, - node::{config::NodeConfig, HardwareModel}, - Node, -}; - -use sd_core_cloud_services::{CloudP2P, KeyManager, QuinnConnector, UserResponse}; - -use sd_cloud_schema::{ - auth, - error::{ClientSideError, Error}, - sync::groups, - users, Client, Request, Response, SecretKey as IrohSecretKey, -}; -use sd_crypto::{CryptoRng, SeedableRng}; -use sd_prisma::prisma::{location, SortOrder}; -use sd_utils::error::report_error; - -use std::pin::pin; - -use async_stream::stream; -use futures::{FutureExt, StreamExt}; -use futures_concurrency::future::TryJoin; -use rspc::alpha::AlphaRouter; -use tracing::{debug, error, instrument}; - -use super::{utils::library, Ctx, R}; - -mod devices; -mod libraries; -mod locations; -mod sync_groups; -mod thumbnails; - -async fn try_get_cloud_services_client( - node: &Node, -) -> Result>, sd_core_cloud_services::Error> { - node.cloud_services - .client() - .await - .map_err(report_error("Failed to get cloud services client")) -} - -pub(crate) fn mount() -> AlphaRouter { - R.router() - .merge("libraries.", libraries::mount()) - .merge("locations.", locations::mount()) - .merge("devices.", devices::mount()) - .merge("syncGroups.", sync_groups::mount()) - .merge("thumbnails.", thumbnails::mount()) - .procedure("bootstrap", { - R.with2(library()).mutation( - |(node, library), - (access_token, refresh_token): (auth::AccessToken, auth::RefreshToken)| async move { - use sd_cloud_schema::devices; - - // Only allow a single bootstrap request in flight at a time - let mut has_bootstrapped_lock = node - .cloud_services - .has_bootstrapped - .try_lock() - .map_err(|_| { - rspc::Error::new( - rspc::ErrorCode::Conflict, - String::from("Bootstrap in progress"), - ) - })?; - - if *has_bootstrapped_lock { - return Err(rspc::Error::new( - rspc::ErrorCode::Conflict, - String::from("Already bootstrapped"), - )); - } - - node.cloud_services - .token_refresher - .init(access_token, refresh_token) - .await?; - - let client = try_get_cloud_services_client(&node).await?; - let data_directory = node.config.data_directory(); - - let mut rng = - CryptoRng::from_seed(node.master_rng.lock().await.generate_fixed()); - - // create user route is idempotent, so we can safely keep creating the same user over and over - handle_comm_error( - client - .users() - .create(users::create::Request { - access_token: node - .cloud_services - .token_refresher - .get_access_token() - .await?, - }) - .await, - "Failed to create user;", - )??; - - let (device_pub_id, name, os) = { - let NodeConfig { id, name, os, .. } = node.config.get().await; - (devices::PubId(id.into()), name, os) - }; - - let hashed_pub_id = blake3::hash(device_pub_id.0.as_bytes().as_slice()); - - let key_manager = match handle_comm_error( - client - .devices() - .get(devices::get::Request { - access_token: node - .cloud_services - .token_refresher - .get_access_token() - .await?, - pub_id: device_pub_id, - }) - .await, - "Failed to get device on cloud bootstrap;", - )? { - Ok(_) => { - // Device registered, we execute a device hello flow - let master_key = self::devices::hello( - &client, - node.cloud_services - .token_refresher - .get_access_token() - .await?, - device_pub_id, - hashed_pub_id, - &mut rng, - ) - .await?; - - debug!("Device hello successful"); - - KeyManager::load(master_key, data_directory.clone()).await? - } - Err(Error::Client(ClientSideError::NotFound(_))) => { - // Device not registered, we execute a device register flow - let iroh_secret_key = IrohSecretKey::generate_with_rng(&mut rng); - let hardware_model = Into::into( - HardwareModel::try_get().unwrap_or(HardwareModel::Other), - ); - - let master_key = self::devices::register( - &client, - node.cloud_services - .token_refresher - .get_access_token() - .await?, - self::devices::DeviceRegisterData { - pub_id: device_pub_id, - name, - os, - hardware_model, - connection_id: iroh_secret_key.public(), - }, - hashed_pub_id, - &mut rng, - ) - .await?; - - debug!("Device registered successfully"); - - KeyManager::new(master_key, iroh_secret_key, data_directory.clone(), &mut rng) - .await? - } - Err(e) => return Err(e.into()), - }; - - let iroh_secret_key = key_manager.iroh_secret_key().await; - - node.cloud_services.set_key_manager(key_manager).await; - - node.cloud_services - .set_cloud_p2p( - CloudP2P::new( - device_pub_id, - &node.cloud_services, - rng, - iroh_secret_key, - node.cloud_services.cloud_p2p_dns_origin_name.clone(), - node.cloud_services.cloud_p2p_dns_pkarr_url.clone(), - node.cloud_services.cloud_p2p_relay_url.clone(), - data_directory.clone(), - ) - .await?, - ) - .await; - - let groups::list::Response(groups) = handle_comm_error( - client - .sync() - .groups() - .list(groups::list::Request { - access_token: node - .cloud_services - .token_refresher - .get_access_token() - .await?, - }) - .await, - "Failed to list sync groups on bootstrap", - )??; - - groups - .into_iter() - .map( - |groups::GroupBaseData { - pub_id, - library, - // TODO(@fogodev): We can use this latest key hash to check if we - // already have the latest key hash for this group locally - // issuing a ask for key hash request for other devices if we don't - latest_key_hash: _latest_key_hash, - .. - }| { - let node = &node; - - async move { - match initialize_cloud_sync(pub_id, library, node).await { - // If we don't have this library locally, we didn't joined this group yet - Ok(()) | Err(LibraryManagerError::LibraryNotFound) => { - Ok(()) - } - Err(e) => Err(e), - } - } - }, - ) - .collect::>() - .try_join() - .await?; - - // If locations are not saved in the cloud, we need to save them - // Get locations from library db - let locations = library - .db - .location() - .find_many(vec![]) - .order_by(location::date_created::order(SortOrder::Desc)) - .exec() - .await?; - - let library_pub_id = sd_cloud_schema::libraries::PubId(library.id); - - // Fetch locations from cloud - let sd_cloud_schema::locations::list::Response(cloud_locations) = handle_comm_error( - client - .locations() - .list(sd_cloud_schema::locations::list::Request { - access_token: node - .cloud_services - .token_refresher - .get_access_token() - .await?, - library_pub_id, - with_library: true, - with_device: true, - }) - .await, - "Failed to list locations on bootstrap", - )??; - - // Save locations that are not in the cloud - for location in locations { - let location_uuid = uuid::Uuid::from_slice(&location.pub_id).unwrap(); - debug!( - location_id = %location_uuid, - "Processing location during bootstrap" - ); - - if !cloud_locations.iter().any(|l| l.pub_id.0 == location_uuid) { - debug!( - location_id = %location_uuid, - location_name = %location.name.clone().unwrap_or_else(|| format!("Location {}", location.id)), - "Creating location in cloud during bootstrap" - ); - - handle_comm_error( - client - .locations() - .create(sd_cloud_schema::locations::create::Request { - access_token: node - .cloud_services - .token_refresher - .get_access_token() - .await?, - pub_id: sd_cloud_schema::locations::PubId(location_uuid), - name: location.name.clone().unwrap_or_else(|| format!("Location {}", location.id)), - library_pub_id, - device_pub_id: node.config.get().await.id.into(), - }) - .await, - "Failed to create location on bootstrap", - )?; - } else { - debug!( - location_id = %location_uuid, - "Location already exists in cloud, skipping creation" - ); - } - } - - *has_bootstrapped_lock = true; - - Ok(()) - }, - ) - }) - .procedure( - "listenCloudServicesNotifications", - R.subscription(|node, _: ()| async move { - stream! { - let mut notifications_stream = - pin!(node.cloud_services.stream_user_notifications()); - - while let Some(notification) = notifications_stream.next().await { - yield notification; - } - } - }), - ) - .procedure( - "userResponse", - R.mutation(|node, response: UserResponse| async move { - node.cloud_services.send_user_response(response).await; - - Ok(()) - }), - ) - .procedure( - "hasBootstrapped", - R.query(|node, _: ()| async move { - // If we can't lock immediately, it means that there is a bootstrap in progress - // so we didn't bootstrapped yet - Ok(node - .cloud_services - .has_bootstrapped - .try_lock() - .map(|lock| *lock) - .unwrap_or(false)) - }), - ) -} - -fn handle_comm_error( - res: Result, - message: &'static str, -) -> Result { - res.map_err(|e| { - error!(?e, "Communication with cloud services error: {message}"); - rspc::Error::with_cause(rspc::ErrorCode::InternalServerError, message.into(), e) - }) -} - -#[instrument(skip_all, fields(%group_pub_id, %library_pub_id), err)] -async fn initialize_cloud_sync( - group_pub_id: groups::PubId, - sd_cloud_schema::libraries::Library { - pub_id: sd_cloud_schema::libraries::PubId(library_pub_id), - .. - }: sd_cloud_schema::libraries::Library, - node: &Node, -) -> Result<(), LibraryManagerError> { - let library = node - .libraries - .get_library(&library_pub_id) - .await - .ok_or(LibraryManagerError::LibraryNotFound)?; - - library.init_cloud_sync(node, group_pub_id).await -} - -async fn get_client_and_access_token( - node: &Node, -) -> Result<(Client>, auth::AccessToken), rspc::Error> { - ( - try_get_cloud_services_client(node), - node.cloud_services - .token_refresher - .get_access_token() - .map(|res| res.map_err(Into::into)), - ) - .try_join() - .await - .map_err(Into::into) -} diff --git a/core/src/api/cloud/sync_groups.rs b/core/src/api/cloud/sync_groups.rs deleted file mode 100644 index 6095b01bb..000000000 --- a/core/src/api/cloud/sync_groups.rs +++ /dev/null @@ -1,404 +0,0 @@ -use crate::{ - api::{utils::library, Ctx, R}, - library::LibraryName, - Node, -}; - -use sd_core_cloud_services::JoinedLibraryCreateArgs; - -use sd_cloud_schema::{ - cloud_p2p, devices, libraries, - sync::{groups, KeyHash}, -}; -use sd_crypto::{cloud::secret_key::SecretKey, CryptoRng, SeedableRng}; - -use std::sync::Arc; - -use futures::FutureExt; -use futures_concurrency::future::TryJoin; -use rspc::alpha::AlphaRouter; -use serde::{Deserialize, Serialize}; -use tokio::{spawn, sync::oneshot}; -use tracing::{debug, error}; - -pub fn mount() -> AlphaRouter { - R.router() - .procedure("create", { - R.with2(library()) - .mutation(|(node, library), _: ()| async move { - use groups::create::{Request, Response}; - - let ((client, access_token), device_pub_id, mut rng, key_manager) = ( - super::get_client_and_access_token(&node), - node.config.get().map(|config| Ok(config.id.into())), - node.master_rng - .lock() - .map(|mut rng| Ok(CryptoRng::from_seed(rng.generate_fixed()))), - node.cloud_services - .key_manager() - .map(|res| res.map_err(Into::into)), - ) - .try_join() - .await?; - - let new_key = SecretKey::generate(&mut rng); - let key_hash = KeyHash(blake3::hash(new_key.as_ref()).to_hex().to_string()); - - let Response(group_pub_id) = super::handle_comm_error( - client - .sync() - .groups() - .create(Request { - access_token: access_token.clone(), - key_hash: key_hash.clone(), - library_pub_id: libraries::PubId(library.id), - device_pub_id, - }) - .await, - "Failed to create sync group;", - )??; - - if let Err(e) = key_manager - .add_key_with_hash(group_pub_id, new_key, key_hash.clone(), &mut rng) - .await - { - super::handle_comm_error( - client - .sync() - .groups() - .delete(groups::delete::Request { - access_token, - pub_id: group_pub_id, - }) - .await, - "Failed to delete sync group after we failed to store secret key in key manager;", - )??; - - return Err(e.into()); - } - - library.init_cloud_sync(&node, group_pub_id).await?; - - debug!(%group_pub_id, ?key_hash, "Created sync group"); - - Ok(()) - }) - }) - .procedure("delete", { - R.mutation(|node, pub_id: groups::PubId| async move { - use groups::delete::Request; - - let (client, access_token) = super::get_client_and_access_token(&node).await?; - - super::handle_comm_error( - client - .sync() - .groups() - .delete(Request { - access_token, - pub_id, - }) - .await, - "Failed to delete sync group;", - )??; - - debug!(%pub_id, "Deleted sync group"); - - Ok(()) - }) - }) - .procedure("get", { - #[derive(Deserialize, specta::Type)] - struct CloudGetSyncGroupArgs { - pub pub_id: groups::PubId, - pub kind: groups::get::RequestKind, - } - - // This is a compatibility layer because quic-rpc uses bincode for serialization - // and bincode doesn't support serde's tagged enums, and we need them for serializing - // to frontend - #[derive(Debug, Serialize, specta::Type)] - #[serde(tag = "kind", content = "data")] - pub enum CloudSyncGroupGetResponseKind { - WithDevices(groups::GroupWithDevices), - FullData(groups::Group), - } - - impl From for CloudSyncGroupGetResponseKind { - fn from(kind: groups::get::ResponseKind) -> Self { - match kind { - groups::get::ResponseKind::WithDevices(data) => { - CloudSyncGroupGetResponseKind::WithDevices(data) - } - - groups::get::ResponseKind::FullData(data) => { - CloudSyncGroupGetResponseKind::FullData(data) - } - groups::get::ResponseKind::DevicesConnectionIds(_) => { - unreachable!( - "DevicesConnectionIds response is not expected, as we requested it" - ); - } - } - } - } - - R.query( - |node, CloudGetSyncGroupArgs { pub_id, kind }: CloudGetSyncGroupArgs| async move { - use groups::get::{Request, Response}; - - let (client, access_token) = super::get_client_and_access_token(&node).await?; - - if matches!(kind, groups::get::RequestKind::DevicesConnectionIds) { - return Err(rspc::Error::new( - rspc::ErrorCode::PreconditionFailed, - "This request isn't allowed here".into(), - )); - } - - let Response(response_kind) = super::handle_comm_error( - client - .sync() - .groups() - .get(Request { - access_token, - pub_id, - kind, - }) - .await, - "Failed to get sync group;", - )??; - - debug!(?response_kind, "Got sync group"); - - Ok(CloudSyncGroupGetResponseKind::from(response_kind)) - }, - ) - }) - .procedure("leave", { - R.query(|node, pub_id: groups::PubId| async move { - let ((client, access_token), current_device_pub_id, mut rng, key_manager) = ( - super::get_client_and_access_token(&node), - node.config.get().map(|config| Ok(config.id.into())), - node.master_rng - .lock() - .map(|mut rng| Ok(CryptoRng::from_seed(rng.generate_fixed()))), - node.cloud_services - .key_manager() - .map(|res| res.map_err(Into::into)), - ) - .try_join() - .await?; - - super::handle_comm_error( - client - .sync() - .groups() - .leave(groups::leave::Request { - access_token, - pub_id, - current_device_pub_id, - }) - .await, - "Failed to leave sync group;", - )??; - - key_manager.remove_group(pub_id, &mut rng).await?; - - debug!(%pub_id, "Left sync group"); - - Ok(()) - }) - }) - .procedure("list", { - R.query(|node, _: ()| async move { - use groups::list::{Request, Response}; - - let (client, access_token) = super::get_client_and_access_token(&node).await?; - - let Response(groups) = super::handle_comm_error( - client.sync().groups().list(Request { access_token }).await, - "Failed to list groups;", - )??; - - debug!(?groups, "Listed sync groups"); - - Ok(groups) - }) - }) - .procedure("remove_device", { - #[derive(Deserialize, specta::Type)] - struct CloudSyncGroupsRemoveDeviceArgs { - group_pub_id: groups::PubId, - to_remove_device_pub_id: devices::PubId, - } - R.query( - |node, - CloudSyncGroupsRemoveDeviceArgs { - group_pub_id, - to_remove_device_pub_id, - }: CloudSyncGroupsRemoveDeviceArgs| async move { - use groups::remove_device::Request; - - let ((client, access_token), current_device_pub_id, mut rng, key_manager) = ( - super::get_client_and_access_token(&node), - node.config.get().map(|config| Ok(config.id.into())), - node.master_rng - .lock() - .map(|mut rng| Ok(CryptoRng::from_seed(rng.generate_fixed()))), - node.cloud_services - .key_manager() - .map(|res| res.map_err(Into::into)), - ) - .try_join() - .await?; - - let new_key = SecretKey::generate(&mut rng); - let new_key_hash = KeyHash(blake3::hash(new_key.as_ref()).to_hex().to_string()); - - key_manager - .add_key_with_hash(group_pub_id, new_key, new_key_hash.clone(), &mut rng) - .await?; - - super::handle_comm_error( - client - .sync() - .groups() - .remove_device(Request { - access_token, - group_pub_id, - new_key_hash, - current_device_pub_id, - to_remove_device_pub_id, - }) - .await, - "Failed to remove device from sync group;", - )??; - - debug!(%to_remove_device_pub_id, %group_pub_id, "Removed device"); - - Ok(()) - }, - ) - }) - .procedure("request_join", { - #[derive(Deserialize, specta::Type)] - struct SyncGroupsRequestJoinArgs { - sync_group: groups::GroupWithDevices, - asking_device: devices::Device, - } - - R.mutation( - |node, - SyncGroupsRequestJoinArgs { - sync_group, - asking_device, - }: SyncGroupsRequestJoinArgs| async move { - let ((client, access_token), current_device_pub_id, cloud_p2p) = ( - super::get_client_and_access_token(&node), - node.config.get().map(|config| Ok(config.id.into())), - node.cloud_services - .cloud_p2p() - .map(|res| res.map_err(Into::into)), - ) - .try_join() - .await?; - - let group_pub_id = sync_group.pub_id; - - debug!("My pub id: {:?}", current_device_pub_id); - debug!("Asking device pub id: {:?}", asking_device.pub_id); - if asking_device.pub_id != current_device_pub_id { - return Err(rspc::Error::new( - rspc::ErrorCode::BadRequest, - String::from("Asking device must be the current device"), - )); - } - - let groups::request_join::Response(existing_devices) = - super::handle_comm_error( - client - .sync() - .groups() - .request_join(groups::request_join::Request { - access_token, - group_pub_id, - current_device_pub_id, - }) - .await, - "Failed to update library;", - )??; - - let (tx, rx) = oneshot::channel(); - - cloud_p2p - .request_join_sync_group( - existing_devices, - cloud_p2p::authorize_new_device_in_sync_group::Request { - sync_group, - asking_device, - }, - tx, - ) - .await; - - JoinedSyncGroupReceiver { - node, - group_pub_id, - rx, - } - .dispatch(); - - debug!(%group_pub_id, "Requested to join sync group"); - - Ok(()) - }, - ) - }) -} - -struct JoinedSyncGroupReceiver { - node: Arc, - group_pub_id: groups::PubId, - rx: oneshot::Receiver, -} - -impl JoinedSyncGroupReceiver { - fn dispatch(self) { - spawn(async move { - let Self { - node, - group_pub_id, - rx, - } = self; - - if let Ok(JoinedLibraryCreateArgs { - pub_id: libraries::PubId(pub_id), - name, - description, - }) = rx.await - { - let Ok(name) = - LibraryName::new(name).map_err(|e| error!(?e, "Invalid library name")) - else { - return; - }; - - let Ok(library) = node - .libraries - .create_with_uuid(pub_id, name, description, true, None, &node) - .await - .map_err(|e| { - error!(?e, "Failed to create library from sync group join response") - }) - else { - return; - }; - - if let Err(e) = library.init_cloud_sync(&node, group_pub_id).await { - error!(?e, "Failed to initialize cloud sync for library"); - } - } - }); - } -} diff --git a/core/src/api/cloud/thumbnails.rs b/core/src/api/cloud/thumbnails.rs deleted file mode 100644 index 3b10b3626..000000000 --- a/core/src/api/cloud/thumbnails.rs +++ /dev/null @@ -1,61 +0,0 @@ -use crate::api::{Ctx, R}; - -use sd_cloud_schema::{devices, libraries}; -use sd_prisma::prisma::file_path::cas_id; - -use futures::FutureExt; -use futures_concurrency::future::TryJoin; -use rspc::alpha::AlphaRouter; -use serde::Deserialize; -use tokio::sync::oneshot; -use tracing::{debug, error}; - -pub fn mount() -> AlphaRouter { - R.router().procedure("get", { - #[derive(Deserialize, specta::Type)] - struct CloudThumbnailRequestArgs { - device_pub_id: devices::PubId, - library_pub_id: libraries::PubId, - cas_id: cas_id::Type, - } - - R.mutation( - |node, - CloudThumbnailRequestArgs { - device_pub_id, - library_pub_id, - cas_id, - }: CloudThumbnailRequestArgs| async move { - let ((client, access_token), cloud_p2p) = ( - super::get_client_and_access_token(&node), - node.cloud_services - .cloud_p2p() - .map(|res| res.map_err(Into::into)), - ) - .try_join() - .await?; - - let (tx, rx) = oneshot::channel(); - - cloud_p2p - .request_thumbnail_data(device_pub_id, cas_id, library_pub_id, tx) - .await; - - // Log rx output - let out = rx.await; - - let out = out.map_err(|e| { - error!(?e, "Failed to receive thumbnail data"); - rspc::Error::new( - rspc::ErrorCode::InternalServerError, - String::from("Failed to receive thumbnail data"), - ) - })?; - - debug!(?out, "Received thumbnail data"); - - Ok(()) - }, - ) - }) -} diff --git a/core/src/api/devices.rs b/core/src/api/devices.rs deleted file mode 100644 index fe9be2456..000000000 --- a/core/src/api/devices.rs +++ /dev/null @@ -1,64 +0,0 @@ -use crate::node::HardwareModel; -use rspc::alpha::AlphaRouter; -use sd_cloud_schema::devices::DeviceOS; -use sd_core_prisma_helpers::DevicePubId; -use sd_prisma::prisma::device; -use serde::Serialize; -use specta::Type; - -use super::{utils::library, Ctx, R}; - -#[derive(Type, Serialize, Clone, Debug)] -pub struct Device { - pub id: i32, - pub pub_id: DevicePubId, - pub name: String, - pub os: DeviceOS, - pub hardware_model: HardwareModel, - pub date_created: chrono::DateTime, - - pub is_current_device: bool, -} - -impl From<(device::Data, &DevicePubId)> for Device { - fn from((d, current_device_pub_id): (device::Data, &DevicePubId)) -> Self { - let pub_id = DevicePubId::from(d.pub_id); - - Self { - id: d.id, - is_current_device: pub_id == *current_device_pub_id, - pub_id, - name: d.name.unwrap_or_default(), - os: d - .os - .expect("is not actually optional") - .try_into() - .expect("is not actually optional"), - hardware_model: d - .hardware_model - .expect("is not actually optional") - .try_into() - .expect("is not actually optional"), - date_created: d.date_created.expect("is not actually optional"), - } - } -} - -pub(crate) fn mount() -> AlphaRouter { - R.router().procedure( - "list", - R.with2(library()) - .query(|(node, library), _: ()| async move { - let current_device_pub_id = node.config.get().await.id; - Ok(library - .db - .device() - .find_many(vec![]) - .exec() - .await? - .into_iter() - .map(|d| Device::from((d, ¤t_device_pub_id))) - .collect::>()) - }), - ) -} diff --git a/core/src/api/ephemeral_files.rs b/core/src/api/ephemeral_files.rs deleted file mode 100644 index 3acd7c573..000000000 --- a/core/src/api/ephemeral_files.rs +++ /dev/null @@ -1,661 +0,0 @@ -use crate::{ - api::{ - files::{create_file, MediaData}, - utils::library, - }, - invalidate_query, - library::Library, - object::{ - fs::{error::FileSystemJobsError, find_available_filename_for_duplicate}, - // media::exif_metadata_extractor::{can_extract_exif_data_for_image, extract_exif_data}, - }, -}; - -use sd_core_file_path_helper::IsolatedFilePathData; -use sd_core_heavy_lifting::media_processor::exif_media_data; - -use sd_file_ext::{ - extensions::{Extension, ImageExtension}, - kind::ObjectKind, -}; -use sd_media_metadata::FFmpegMetadata; -use sd_utils::error::FileIOError; - -use std::{ffi::OsStr, path::PathBuf, str::FromStr}; - -use futures_concurrency::future::TryJoin; -use regex::Regex; -use rspc::{alpha::AlphaRouter, ErrorCode}; -use serde::Deserialize; -use specta::Type; -use tokio::{fs, io}; -use tokio_stream::{wrappers::ReadDirStream, StreamExt}; -use tracing::{error, warn}; -#[cfg(not(any(target_os = "ios", target_os = "android")))] -use trash; - -use super::{ - files::{create_directory, FromPattern}, - Ctx, R, -}; - -const UNTITLED_FOLDER_STR: &str = "Untitled Folder"; -const UNTITLED_FILE_STR: &str = "Untitled"; -const UNTITLED_TEXT_FILE_STR: &str = "Untitled.txt"; - -#[derive(Type, Deserialize)] -#[serde(rename_all = "camelCase")] -enum EphemeralFileCreateContextTypes { - Empty, - Text, -} - -pub(crate) fn mount() -> AlphaRouter { - R.router() - .procedure("getMediaData", { - R.query(|_, full_path: PathBuf| async move { - let kind: Option = Extension::resolve_conflicting(&full_path, false) - .await - .map(Into::into); - match kind { - Some(ObjectKind::Image) => { - let Some(extension) = full_path.extension().and_then(|ext| ext.to_str()) - else { - return Ok(None); - }; - - let image_extension = ImageExtension::from_str(extension).map_err(|e| { - error!(?e, "Failed to parse image extension;"); - rspc::Error::new( - ErrorCode::BadRequest, - "Invalid image extension".to_string(), - ) - })?; - - if !exif_media_data::can_extract(image_extension) { - return Ok(None); - } - - let exif_data = exif_media_data::extract(full_path) - .await - .map_err(|e| { - rspc::Error::with_cause( - ErrorCode::InternalServerError, - "Failed to extract media data".to_string(), - e, - ) - })? - .map(MediaData::Exif); - - Ok(exif_data) - } - Some(v) if v == ObjectKind::Audio || v == ObjectKind::Video => { - let ffmpeg_data = MediaData::FFmpeg( - FFmpegMetadata::from_path(full_path).await.map_err(|e| { - error!(?e, "Failed to extract ffmpeg metadata;"); - rspc::Error::with_cause( - ErrorCode::InternalServerError, - e.to_string(), - e, - ) - })?, - ); - - Ok(Some(ffmpeg_data)) - } - _ => Ok(None), // No media data - } - }) - }) - .procedure("createFolder", { - #[derive(Type, Deserialize)] - pub struct CreateEphemeralFolderArgs { - pub path: PathBuf, - pub name: Option, - } - R.with2(library()).mutation( - |(_, library), - CreateEphemeralFolderArgs { mut path, name }: CreateEphemeralFolderArgs| async move { - path.push(name.as_deref().unwrap_or(UNTITLED_FOLDER_STR)); - - create_directory(path, &library).await - }, - ) - }) - .procedure("createFile", { - #[derive(Type, Deserialize)] - pub struct CreateEphemeralFileArgs { - pub path: PathBuf, - pub context: EphemeralFileCreateContextTypes, - pub name: Option, - } - R.with2(library()).mutation( - |(_, library), - CreateEphemeralFileArgs { - mut path, - name, - context, - }: CreateEphemeralFileArgs| async move { - match context { - EphemeralFileCreateContextTypes::Empty => { - path.push(name.as_deref().unwrap_or(UNTITLED_FILE_STR)); - } - EphemeralFileCreateContextTypes::Text => { - path.push(name.as_deref().unwrap_or(UNTITLED_TEXT_FILE_STR)); - } - } - - create_file(path, &library).await - }, - ) - }) - .procedure("deleteFiles", { - R.with2(library()) - .mutation(|(_, library), paths: Vec| async move { - paths - .into_iter() - .map(|path| async move { - match fs::metadata(&path).await { - Ok(metadata) => if metadata.is_dir() { - fs::remove_dir_all(&path).await - } else { - fs::remove_file(&path).await - } - .map_err(|e| FileIOError::from((path, e, "Failed to delete file"))), - Err(e) if e.kind() == io::ErrorKind::NotFound => Ok(()), - Err(e) => Err(FileIOError::from(( - path, - e, - "Failed to get file metadata for deletion", - ))), - } - }) - .collect::>() - .try_join() - .await?; - - invalidate_query!(library, "search.ephemeralPaths"); - - Ok(()) - }) - }) - .procedure("moveToTrash", { - R.with2(library()) - .mutation(|(_, library), paths: Vec| async move { - if cfg!(target_os = "ios") || cfg!(target_os = "android") { - return Err(rspc::Error::new( - ErrorCode::MethodNotSupported, - "Moving to trash is not supported on this platform".to_string(), - )); - } - - paths - .into_iter() - .map(|path| async move { - match fs::metadata(&path).await { - Ok(_) => { - #[cfg(not(any(target_os = "ios", target_os = "android")))] - trash::delete(&path).map_err(|e| { - FileIOError::from(( - path, - match e { - #[cfg(all(unix, not(target_os = "macos")))] - trash::Error::FileSystem { path: _, source: e } => e, - _ => io::Error::other(e), - }, - "Failed to delete file", - )) - })?; - - Ok::<_, rspc::Error>(()) - } - Err(e) if e.kind() == io::ErrorKind::NotFound => Ok(()), - Err(e) => Err(FileIOError::from(( - path, - e, - "Failed to get file metadata for deletion", - )) - .into()), - } - }) - .collect::>() - .try_join() - .await?; - - invalidate_query!(library, "search.ephemeralPaths"); - - Ok(()) - }) - }) - .procedure("copyFiles", { - R.with2(library()) - .mutation(|(_, library), args: EphemeralFileSystemOps| async move { - args.copy(&library).await - }) - }) - .procedure("cutFiles", { - R.with2(library()) - .mutation(|(_, library), args: EphemeralFileSystemOps| async move { - args.cut(&library).await - }) - }) - .procedure("renameFile", { - #[derive(Type, Deserialize)] - pub struct EphemeralRenameOne { - pub from_path: PathBuf, - pub to: String, - } - - #[derive(Type, Deserialize)] - pub struct EphemeralRenameMany { - pub from_pattern: FromPattern, - pub to_pattern: String, - pub from_paths: Vec, - } - - #[derive(Type, Deserialize)] - pub enum EphemeralRenameKind { - One(EphemeralRenameOne), - Many(EphemeralRenameMany), - } - - #[derive(Type, Deserialize)] - pub struct EphemeralRenameFileArgs { - pub kind: EphemeralRenameKind, - } - - impl EphemeralRenameFileArgs { - pub async fn rename_one( - EphemeralRenameOne { from_path, to }: EphemeralRenameOne, - ) -> Result<(), rspc::Error> { - let Some(old_name) = from_path.file_name() else { - return Err(rspc::Error::new( - ErrorCode::BadRequest, - "Missing file name on file to be renamed".to_string(), - )); - }; - - if old_name == OsStr::new(&to) { - return Ok(()); - } - - let (new_file_name, new_extension) = - IsolatedFilePathData::separate_name_and_extension_from_str(&to).map_err( - |e| rspc::Error::with_cause(ErrorCode::BadRequest, e.to_string(), e), - )?; - - if !IsolatedFilePathData::accept_file_name(new_file_name) { - return Err(rspc::Error::new( - ErrorCode::BadRequest, - "Invalid file name".to_string(), - )); - } - - let Some(parent) = from_path.parent() else { - return Err(rspc::Error::new( - ErrorCode::BadRequest, - "Missing parent path on file to be renamed".to_string(), - )); - }; - - let new_file_full_path = parent.join(if !new_extension.is_empty() { - &to - } else { - new_file_name - }); - - match fs::metadata(&new_file_full_path).await { - Ok(_) => Err(rspc::Error::new( - ErrorCode::Conflict, - "Renaming would overwrite a file".to_string(), - )), - - Err(e) => { - if e.kind() != std::io::ErrorKind::NotFound { - return Err(rspc::Error::with_cause( - ErrorCode::InternalServerError, - "Failed to check if file exists".to_string(), - e, - )); - } - - fs::rename(&from_path, new_file_full_path) - .await - .map_err(|e| { - FileIOError::from((from_path, e, "Failed to rename file")) - .into() - }) - } - } - } - - pub async fn rename_many( - EphemeralRenameMany { - ref from_pattern, - ref to_pattern, - from_paths, - }: EphemeralRenameMany, - ) -> Result<(), rspc::Error> { - let from_regex = &Regex::new(&from_pattern.pattern).map_err(|e| { - rspc::Error::with_cause( - rspc::ErrorCode::BadRequest, - "Invalid `from` regex pattern".to_string(), - e, - ) - })?; - - from_paths - .into_iter() - .map(|old_path| async move { - let Some(old_name) = old_path.file_name() else { - return Err(rspc::Error::new( - ErrorCode::BadRequest, - "Missing file name on file to be renamed".to_string(), - )); - }; - - let Some(old_name_str) = old_name.to_str() else { - return Err(rspc::Error::new( - ErrorCode::BadRequest, - "File with non UTF-8 name".to_string(), - )); - }; - - let replaced_full_name = if from_pattern.replace_all { - from_regex.replace_all(old_name_str, to_pattern) - } else { - from_regex.replace(old_name_str, to_pattern) - }; - - if !IsolatedFilePathData::accept_file_name(replaced_full_name.as_ref()) - { - return Err(rspc::Error::new( - ErrorCode::BadRequest, - "Invalid file name".to_string(), - )); - } - - let Some(parent) = old_path.parent() else { - return Err(rspc::Error::new( - ErrorCode::BadRequest, - "Missing parent path on file to be renamed".to_string(), - )); - }; - - let new_path = parent.join(replaced_full_name.as_ref()); - - fs::rename(&old_path, &new_path).await.map_err(|e| { - error!( - old_path = %old_path.display(), - new_path = %new_path.display(), - ?e, - "Failed to rename file;", - ); - let e = FileIOError::from((old_path, e, "Failed to rename file")); - rspc::Error::with_cause(ErrorCode::Conflict, e.to_string(), e) - }) - }) - .collect::>() - .try_join() - .await?; - - Ok(()) - } - } - - R.with2(library()).mutation( - |(_, library), EphemeralRenameFileArgs { kind }: EphemeralRenameFileArgs| async move { - let res = match kind { - EphemeralRenameKind::One(one) => { - EphemeralRenameFileArgs::rename_one(one).await - } - EphemeralRenameKind::Many(many) => { - EphemeralRenameFileArgs::rename_many(many).await - } - }; - - if res.is_ok() { - invalidate_query!(library, "search.ephemeralPaths"); - } - - res - }, - ) - }) -} - -#[derive(Type, Deserialize)] -struct EphemeralFileSystemOps { - sources: Vec, - target_dir: PathBuf, -} - -impl EphemeralFileSystemOps { - async fn check_target_directory(&self) -> Result<(), rspc::Error> { - match fs::metadata(&self.target_dir).await { - Ok(metadata) => { - if !metadata.is_dir() { - return Err(rspc::Error::new( - ErrorCode::BadRequest, - "Target is not a directory".to_string(), - )); - } - } - Err(e) if e.kind() == io::ErrorKind::NotFound => { - let e = FileIOError::from((&self.target_dir, e, "Target directory not found")); - return Err(rspc::Error::with_cause( - ErrorCode::BadRequest, - e.to_string(), - e, - )); - } - Err(e) => { - return Err(FileIOError::from(( - &self.target_dir, - e, - "Failed to get target metadata", - )) - .into()); - } - } - - Ok(()) - } - - fn check_sources(&self) -> Result<(), rspc::Error> { - if self.sources.is_empty() { - return Err(rspc::Error::new( - ErrorCode::BadRequest, - "Sources cannot be empty".to_string(), - )); - } - - Ok(()) - } - - async fn check(&self) -> Result<(), rspc::Error> { - self.check_sources()?; - self.check_target_directory().await?; - - Ok(()) - } - - async fn copy(self, library: &Library) -> Result<(), rspc::Error> { - self.check().await?; - - let EphemeralFileSystemOps { - sources, - target_dir, - } = self; - - let (directories_to_create, files_to_copy) = sources - .into_iter() - .filter_map(|source| { - if let Some(name) = source.file_name() { - let target = target_dir.join(name); - Some((source, target)) - } else { - warn!(source = %source.display(), "Skipping file with no name;"); - None - } - }) - .map(|(source, target)| async move { - match fs::metadata(&source).await { - Ok(metadata) => Ok((source, target, metadata.is_dir())), - Err(e) => Err(FileIOError::from(( - source, - e, - "Failed to get source file metadata", - ))), - } - }) - .collect::>() - .try_join() - .await? - .into_iter() - .partition::, _>(|(_, _, is_dir)| *is_dir); - - files_to_copy - .into_iter() - .map(|(source, mut target, _)| async move { - match fs::metadata(&target).await { - Ok(_) => target = find_available_filename_for_duplicate(&target).await?, - Err(e) if e.kind() == io::ErrorKind::NotFound => { - // Everything is awesome! - } - Err(e) => { - return Err(FileSystemJobsError::FileIO(FileIOError::from(( - target, - e, - "Failed to get target file metadata", - )))); - } - } - - fs::copy(&source, target).await.map_err(|e| { - FileSystemJobsError::FileIO(FileIOError::from(( - source, - e, - "Failed to copy file", - ))) - }) - }) - .collect::>() - .try_join() - .await?; - - if !directories_to_create.is_empty() { - directories_to_create - .into_iter() - .map(|(source, mut target, _)| async move { - match fs::metadata(&target).await { - Ok(_) => target = find_available_filename_for_duplicate(&target).await?, - Err(e) if e.kind() == io::ErrorKind::NotFound => { - // Everything is awesome! - } - Err(e) => { - return Err(rspc::Error::from(FileIOError::from(( - target, - e, - "Failed to get target file metadata", - )))); - } - } - - fs::create_dir_all(&target).await.map_err(|e| { - FileIOError::from((&target, e, "Failed to create directory")) - })?; - - let more_files = - ReadDirStream::new(fs::read_dir(&source).await.map_err(|e| { - FileIOError::from((&source, e, "Failed to read directory to be copied")) - })?) - .map(|read_dir| match read_dir { - Ok(dir_entry) => Ok(dir_entry.path()), - Err(e) => Err(FileIOError::from(( - &source, - e, - "Failed to read directory to be copied", - ))), - }) - .collect::, _>>() - .await?; - - if !more_files.is_empty() { - Box::pin( - Self { - sources: more_files, - target_dir: target, - } - .copy(library), - ) - .await - } else { - Ok(()) - } - }) - .collect::>() - .try_join() - .await?; - } - - invalidate_query!(library, "search.ephemeralPaths"); - - Ok(()) - } - - async fn cut(self, library: &Library) -> Result<(), rspc::Error> { - self.check().await?; - - let EphemeralFileSystemOps { - sources, - target_dir, - } = self; - - sources - .into_iter() - .filter_map(|source| { - if let Some(name) = source.file_name() { - let target = target_dir.join(name); - Some((source, target)) - } else { - warn!(source = %source.display(), "Skipping file with no name;"); - None - } - }) - .map(|(source, target)| async move { - match fs::metadata(&target).await { - Ok(_) => { - return Err(FileSystemJobsError::WouldOverwrite( - target.into_boxed_path(), - )); - } - Err(e) if e.kind() == io::ErrorKind::NotFound => { - // Everything is awesome! - } - Err(e) => { - return Err(FileSystemJobsError::FileIO(FileIOError::from(( - source, - e, - "Failed to get target file metadata", - )))); - } - } - - fs::rename(&source, target).await.map_err(|e| { - FileSystemJobsError::FileIO(FileIOError::from(( - source, - e, - "Failed to move file", - ))) - }) - }) - .collect::>() - .try_join() - .await?; - - invalidate_query!(library, "search.ephemeralPaths"); - - Ok(()) - } -} diff --git a/core/src/api/files.rs b/core/src/api/files.rs deleted file mode 100644 index 155fd2884..000000000 --- a/core/src/api/files.rs +++ /dev/null @@ -1,1053 +0,0 @@ -use crate::{ - api::utils::library, - invalidate_query, - library::Library, - location::{get_location_path_from_location_id, LocationError}, - object::{ - fs::{ - error::FileSystemJobsError, find_available_filename_for_duplicate, - old_copy::OldFileCopierJobInit, old_cut::OldFileCutterJobInit, - old_delete::OldFileDeleterJobInit, old_erase::OldFileEraserJobInit, - }, - // media::{exif_media_data_from_prisma_data, ffmpeg_data_from_prisma_data}, - }, - old_job::OldJob, -}; - -use sd_core_file_path_helper::{FilePathError, IsolatedFilePathData}; -use sd_core_heavy_lifting::media_processor::{exif_media_data, ffmpeg_media_data}; -use sd_core_prisma_helpers::{ - file_path_to_isolate, file_path_to_isolate_with_id, object_with_file_paths, - object_with_media_data, -}; - -use sd_file_ext::kind::ObjectKind; -use sd_images::ConvertibleExtension; -use sd_media_metadata::{ExifMetadata, FFmpegMetadata}; -use sd_prisma::{ - prisma::{file_path, location, object}, - prisma_sync, -}; -use sd_sync::{sync_db_entry, sync_db_nullable_entry, sync_entry, OperationFactory}; -use sd_utils::{db::maybe_missing, error::FileIOError}; - -use std::{ - ffi::OsString, - path::{Path, PathBuf}, - sync::Arc, -}; - -use chrono::{DateTime, FixedOffset, Utc}; -use futures::future::join_all; -use regex::Regex; -use rspc::{alpha::AlphaRouter, ErrorCode}; -use serde::{Deserialize, Serialize}; -use specta::Type; -use tokio::{fs, io, task::spawn_blocking}; -use tracing::{error, warn}; -#[cfg(not(any(target_os = "ios", target_os = "android")))] -use trash; - -use super::{Ctx, R}; - -const UNTITLED_FOLDER_STR: &str = "Untitled Folder"; -const UNTITLED_FILE_STR: &str = "Untitled"; -const UNTITLED_TEXT_FILE_STR: &str = "Untitled.txt"; - -#[derive(Type, Deserialize)] -#[serde(rename_all = "camelCase")] -enum FileCreateContextTypes { - Empty, - Text, -} - -#[derive(Serialize, Type)] -pub(crate) enum MediaData { - Exif(ExifMetadata), - FFmpeg(FFmpegMetadata), -} - -pub(crate) fn mount() -> AlphaRouter { - R.router() - .procedure("get", { - #[derive(Type, Serialize)] - pub struct ObjectWithFilePaths2 { - pub id: i32, - pub pub_id: Vec, - pub kind: Option, - pub key_id: Option, - pub hidden: Option, - pub favorite: Option, - pub important: Option, - pub note: Option, - pub date_created: Option>, - pub date_accessed: Option>, - pub file_paths: Vec, - } - - impl ObjectWithFilePaths2 { - pub fn from_db(item: object_with_file_paths::Data) -> Self { - Self { - id: item.id, - pub_id: item.pub_id, - kind: item.kind, - key_id: item.key_id, - hidden: item.hidden, - favorite: item.favorite, - important: item.important, - note: item.note, - date_created: item.date_created, - date_accessed: item.date_accessed, - file_paths: item.file_paths, - } - } - } - - R.with2(library()) - .query(|(_, library), object_id: i32| async move { - Ok(library - .db - .object() - .find_unique(object::id::equals(object_id)) - .include(object_with_file_paths::include()) - .exec() - .await? - .map(ObjectWithFilePaths2::from_db)) - }) - }) - .procedure("getMediaData", { - R.with2(library()) - .query(|(_, library), args: object::id::Type| async move { - library - .db - .object() - .find_unique(object::id::equals(args)) - .include(object_with_media_data::include()) - .exec() - .await? - .and_then(|obj| { - Some(match obj.kind { - Some(v) if v == ObjectKind::Image as i32 => MediaData::Exif( - exif_media_data::from_prisma_data(obj.exif_data?), - ), - Some(v) - if v == ObjectKind::Audio as i32 - || v == ObjectKind::Video as i32 => - { - MediaData::FFmpeg(ffmpeg_media_data::from_prisma_data( - obj.ffmpeg_data?, - )) - } - _ => return None, // No media data - }) - }) - .ok_or_else(|| { - rspc::Error::new(ErrorCode::NotFound, "Object not found".to_string()) - }) - }) - }) - .procedure("getPath", { - R.with2(library()) - .query(|(_, library), id: i32| async move { - let isolated_path = IsolatedFilePathData::try_from( - library - .db - .file_path() - .find_unique(file_path::id::equals(id)) - .select(file_path_to_isolate::select()) - .exec() - .await? - .ok_or(LocationError::FilePath(FilePathError::IdNotFound(id)))?, - ) - .map_err(LocationError::MissingField)?; - - let location_id = isolated_path.location_id(); - let location_path = - get_location_path_from_location_id(&library.db, location_id).await?; - - Ok(Path::new(&location_path) - .join(&isolated_path) - .to_str() - .map(|str| str.to_string())) - }) - }) - .procedure("setNote", { - #[derive(Type, Deserialize)] - pub struct SetNoteArgs { - pub id: i32, - pub note: Option, - } - - R.with2(library()) - .mutation(|(_, library), args: SetNoteArgs| async move { - let Library { db, sync, .. } = library.as_ref(); - - let object = db - .object() - .find_unique(object::id::equals(args.id)) - .select(object::select!({ pub_id })) - .exec() - .await? - .ok_or_else(|| { - rspc::Error::new( - rspc::ErrorCode::NotFound, - "Object not found".to_string(), - ) - })?; - - let (sync_param, db_param) = sync_db_nullable_entry!(args.note, object::note); - - sync.write_op( - db, - sync.shared_update( - prisma_sync::object::SyncId { - pub_id: object.pub_id, - }, - [sync_param], - ), - db.object() - .update(object::id::equals(args.id), vec![db_param]) - .select(object::select!({ id })), - ) - .await?; - - invalidate_query!(library, "search.paths"); - invalidate_query!(library, "search.objects"); - - Ok(()) - }) - }) - .procedure("setFavorite", { - #[derive(Type, Deserialize)] - pub struct SetFavoriteArgs { - pub id: i32, - pub favorite: bool, - } - - R.with2(library()) - .mutation(|(_, library), args: SetFavoriteArgs| async move { - let Library { sync, db, .. } = library.as_ref(); - - let object = db - .object() - .find_unique(object::id::equals(args.id)) - .select(object::select!({ pub_id })) - .exec() - .await? - .ok_or_else(|| { - rspc::Error::new( - rspc::ErrorCode::NotFound, - "Object not found".to_string(), - ) - })?; - - let (sync_param, db_param) = sync_db_entry!(args.favorite, object::favorite); - - sync.write_op( - db, - sync.shared_update( - prisma_sync::object::SyncId { - pub_id: object.pub_id, - }, - [sync_param], - ), - db.object() - .update(object::id::equals(args.id), vec![db_param]) - .select(object::select!({ id })), - ) - .await?; - - invalidate_query!(library, "search.paths"); - invalidate_query!(library, "search.objects"); - - Ok(()) - }) - }) - .procedure("createFolder", { - #[derive(Type, Deserialize)] - pub struct CreateFolderArgs { - pub location_id: location::id::Type, - pub sub_path: Option, - pub name: Option, - } - R.with2(library()).mutation( - |(_, library), - CreateFolderArgs { - location_id, - sub_path, - name, - }: CreateFolderArgs| async move { - let mut path = - get_location_path_from_location_id(&library.db, location_id).await?; - - if let Some(sub_path) = sub_path - .as_ref() - .and_then(|sub_path| sub_path.strip_prefix("/").ok()) - { - path.push(sub_path); - } - - path.push(name.as_deref().unwrap_or(UNTITLED_FOLDER_STR)); - - create_directory(path, &library).await - }, - ) - }) - .procedure("createFile", { - #[derive(Type, Deserialize)] - pub struct CreateFileArgs { - pub location_id: location::id::Type, - pub sub_path: Option, - pub name: Option, - pub context: FileCreateContextTypes, - } - R.with2(library()).mutation( - |(_, library), - CreateFileArgs { - location_id, - sub_path, - context, - name, - }: CreateFileArgs| async move { - let mut path = - get_location_path_from_location_id(&library.db, location_id).await?; - - if let Some(sub_path) = sub_path - .as_ref() - .and_then(|sub_path| sub_path.strip_prefix("/").ok()) - { - path.push(sub_path); - } - - match context { - FileCreateContextTypes::Empty => { - path.push(name.as_deref().unwrap_or(UNTITLED_FILE_STR)) - } - FileCreateContextTypes::Text => { - path.push(name.as_deref().unwrap_or(UNTITLED_TEXT_FILE_STR)) - } - } - - create_file(path, &library).await - }, - ) - }) - .procedure("updateAccessTime", { - R.with2(library()) - .mutation(|(_, library), ids: Vec| async move { - let Library { sync, db, .. } = library.as_ref(); - - let objects = db - .object() - .find_many(vec![object::id::in_vec(ids)]) - .select(object::select!({ id pub_id })) - .exec() - .await?; - - let date_accessed = Utc::now().into(); - - let (ops, object_ids) = objects - .into_iter() - .map(|object| { - ( - sync.shared_update( - prisma_sync::object::SyncId { - pub_id: object.pub_id, - }, - [sync_entry!(date_accessed, object::date_accessed)], - ), - object.id, - ) - }) - .unzip::<_, _, Vec<_>, Vec<_>>(); - - if !ops.is_empty() && !object_ids.is_empty() { - sync.write_ops( - db, - ( - ops, - db.object().update_many( - vec![object::id::in_vec(object_ids)], - vec![object::date_accessed::set(Some(date_accessed))], - ), - ), - ) - .await?; - - invalidate_query!(library, "search.paths"); - invalidate_query!(library, "search.objects"); - } - - Ok(()) - }) - }) - .procedure("removeAccessTime", { - R.with2(library()) - .mutation(|(_, library), object_ids: Vec| async move { - let Library { db, sync, .. } = library.as_ref(); - - let objects = db - .object() - .find_many(vec![object::id::in_vec(object_ids)]) - .select(object::select!({ id pub_id })) - .exec() - .await?; - - let (ops, object_ids) = objects - .into_iter() - .map(|object| { - ( - sync.shared_update( - prisma_sync::object::SyncId { - pub_id: object.pub_id, - }, - [sync_entry!(nil, object::date_accessed)], - ), - object.id, - ) - }) - .unzip::<_, _, Vec<_>, Vec<_>>(); - - if !ops.is_empty() && !object_ids.is_empty() { - sync.write_ops( - db, - ( - ops, - db.object().update_many( - vec![object::id::in_vec(object_ids)], - vec![object::date_accessed::set(None)], - ), - ), - ) - .await?; - - invalidate_query!(library, "search.objects"); - invalidate_query!(library, "search.paths"); - } - - Ok(()) - }) - }) - // .procedure("encryptFiles", { - // R.with2(library()) - // .mutation(|(node, library), args: FileEncryptorJobInit| async move { - // Job::new(args).spawn(&node, &library).await.map_err(Into::into) - // }) - // }) - // .procedure("decryptFiles", { - // R.with2(library()) - // .mutation(|(node, library), args: FileDecryptorJobInit| async move { - // Job::new(args).spawn(&node, &library).await.map_err(Into::into) - // }) - // }) - .procedure("deleteFiles", { - R.with2(library()) - .mutation(|(node, library), args: OldFileDeleterJobInit| async move { - match args.file_path_ids.len() { - 0 => Ok(()), - 1 => { - let (maybe_location, maybe_file_path) = library - .db - ._batch(( - library - .db - .location() - .find_unique(location::id::equals(args.location_id)) - .select(location::select!({ path })), - library - .db - .file_path() - .find_unique(file_path::id::equals(args.file_path_ids[0])) - .select(file_path_to_isolate::select()), - )) - .await?; - - let location_path = maybe_location - .ok_or(LocationError::IdNotFound(args.location_id))? - .path - .ok_or(LocationError::MissingPath(args.location_id))?; - - let file_path = maybe_file_path.ok_or(LocationError::FilePath( - FilePathError::IdNotFound(args.file_path_ids[0]), - ))?; - - let full_path = Path::new(&location_path).join( - IsolatedFilePathData::try_from(&file_path) - .map_err(LocationError::MissingField)?, - ); - - match if maybe_missing(file_path.is_dir, "file_path.is_dir") - .map_err(LocationError::MissingField)? - { - fs::remove_dir_all(&full_path).await - } else { - fs::remove_file(&full_path).await - } { - Ok(()) => Ok(()), - Err(e) if e.kind() == io::ErrorKind::NotFound => { - warn!( - path = %full_path.display(), - "File not found in the file system, will remove from database;", - ); - - let file_path_pub_id = library - .db - .file_path() - .find_unique(file_path::id::equals(args.file_path_ids[0])) - .select(file_path::select!({ pub_id })) - .exec() - .await? - .ok_or(LocationError::FilePath(FilePathError::IdNotFound( - args.file_path_ids[0], - )))? - .pub_id; - - library - .sync - .write_op( - &library.db, - library.sync.shared_delete( - prisma_sync::file_path::SyncId { - pub_id: file_path_pub_id, - }, - ), - library.db.file_path().delete(file_path::id::equals( - args.file_path_ids[0], - )), - ) - .await - .map_err(LocationError::from)?; - - Ok(()) - } - Err(e) => { - Err(LocationError::from(FileIOError::from((full_path, e))) - .into()) - } - } - } - _ => OldJob::new(args) - .spawn(&node, &library) - .await - .map_err(Into::into), - } - }) - }) - .procedure("moveToTrash", { - R.with2(library()) - .mutation(|(node, library), args: OldFileDeleterJobInit| async move { - if cfg!(target_os = "ios") || cfg!(target_os = "android") { - return Err(rspc::Error::new( - ErrorCode::MethodNotSupported, - "Moving to trash is not supported on this platform".to_string(), - )); - } - - match args.file_path_ids.len() { - 0 => Ok(()), - 1 => { - let (maybe_location, maybe_file_path) = library - .db - ._batch(( - library - .db - .location() - .find_unique(location::id::equals(args.location_id)) - .select(location::select!({ path })), - library - .db - .file_path() - .find_unique(file_path::id::equals(args.file_path_ids[0])) - .select(file_path_to_isolate::select()), - )) - .await?; - - let location_path = maybe_location - .ok_or(LocationError::IdNotFound(args.location_id))? - .path - .ok_or(LocationError::MissingPath(args.location_id))?; - - let file_path = maybe_file_path.ok_or(LocationError::FilePath( - FilePathError::IdNotFound(args.file_path_ids[0]), - ))?; - - let full_path = Path::new(&location_path).join( - IsolatedFilePathData::try_from(&file_path) - .map_err(LocationError::MissingField)?, - ); - - #[cfg(not(any(target_os = "ios", target_os = "android")))] - trash::delete(&full_path).map_err(|e| { - FileIOError::from(( - full_path, - match e { - #[cfg(all(unix, not(target_os = "macos")))] - trash::Error::FileSystem { path: _, source: e } => e, - _ => io::Error::other(e), - }, - "Failed to delete file", - )) - })?; - - Ok(()) - } - _ => OldJob::new(args) - .spawn(&node, &library) - .await - .map_err(Into::into), - } - }) - }) - .procedure("convertImage", { - #[derive(Type, Deserialize)] - struct ConvertImageArgs { - location_id: location::id::Type, - file_path_id: file_path::id::Type, - delete_src: bool, // if set, we delete the src image after - desired_extension: ConvertibleExtension, - quality_percentage: Option, // 1% - 125% - } - R.with2(library()) - .mutation(|(_, library), args: ConvertImageArgs| async move { - // TODO:(fogodev) I think this will have to be a Job due to possibly being too much CPU Bound for rspc - - let location_path = - get_location_path_from_location_id(&library.db, args.location_id).await?; - - let isolated_path = IsolatedFilePathData::try_from( - library - .db - .file_path() - .find_unique(file_path::id::equals(args.file_path_id)) - .select(file_path_to_isolate::select()) - .exec() - .await? - .ok_or(LocationError::FilePath(FilePathError::IdNotFound( - args.file_path_id, - )))?, - )?; - - let path = Path::new(&location_path).join(&isolated_path); - - if let Err(e) = fs::metadata(&path).await { - if e.kind() == io::ErrorKind::NotFound { - return Err(LocationError::FilePath(FilePathError::NotFound( - path.into_boxed_path(), - )) - .into()); - } else { - return Err(FileIOError::from(( - path, - e, - "Got an error trying to read metadata from image to convert", - )) - .into()); - } - } - - args.quality_percentage.map(|x| x.clamp(1, 125)); - - let path = Arc::new(path); - - let output_extension = - Arc::new(OsString::from(args.desired_extension.to_string())); - - // TODO(fogodev): Refactor this if Rust get async scoped spawns someday - let inner_path = Arc::clone(&path); - let inner_output_extension = Arc::clone(&output_extension); - let image = spawn_blocking(move || { - sd_images::convert_image(inner_path.as_ref(), &inner_output_extension).map( - |mut image| { - if let Some(quality_percentage) = args.quality_percentage { - image = image.resize( - image.width() - * (quality_percentage as f32 / 100_f32) as u32, - image.height() - * (quality_percentage as f32 / 100_f32) as u32, - image::imageops::FilterType::Triangle, - ); - } - image - }, - ) - }) - .await - .map_err(|e| { - error!(?e, "Failed to convert image;"); - rspc::Error::with_cause( - ErrorCode::InternalServerError, - "Had an internal problem converting image".to_string(), - e, - ) - })??; - - let output_path = path.with_extension(output_extension.as_ref()); - - if fs::metadata(&output_path) - .await - .map(|_| true) - .map_err(|e| { - FileIOError::from( - ( - &output_path, - e, - "Got an error trying to check if the desired converted file already exists" - ) - ) - })? { - return Err(rspc::Error::new( - ErrorCode::Conflict, - "There is already a file with same name and extension in this directory" - .to_string(), - )); - } else { - fs::write(&output_path, image.as_bytes()) - .await - .map_err(|e| { - FileIOError::from(( - output_path, - e, - "There was an error while writing the image to the output path", - )) - })?; - } - - if args.delete_src { - fs::remove_file(path.as_ref()).await.map_err(|e| { - // Let's also invalidate the query here, because we succeeded in converting the file - invalidate_query!(library, "search.paths"); - invalidate_query!(library, "search.objects"); - - FileIOError::from(( - path.as_ref(), - e, - "There was an error while deleting the source image", - )) - })?; - } - - invalidate_query!(library, "search.paths"); - invalidate_query!(library, "search.objects"); - - Ok(()) - }) - }) - .procedure("getConvertibleImageExtensions", { - R.query(|_, _: ()| async move { Ok(sd_images::all_compatible_extensions()) }) - }) - .procedure("eraseFiles", { - R.with2(library()) - .mutation(|(node, library), args: OldFileEraserJobInit| async move { - OldJob::new(args) - .spawn(&node, &library) - .await - .map_err(Into::into) - }) - }) - .procedure("copyFiles", { - R.with2(library()) - .mutation(|(node, library), args: OldFileCopierJobInit| async move { - OldJob::new(args) - .spawn(&node, &library) - .await - .map_err(Into::into) - }) - }) - .procedure("cutFiles", { - R.with2(library()) - .mutation(|(node, library), args: OldFileCutterJobInit| async move { - OldJob::new(args) - .spawn(&node, &library) - .await - .map_err(Into::into) - }) - }) - .procedure("renameFile", { - #[derive(Type, Deserialize)] - pub struct RenameOne { - pub from_file_path_id: file_path::id::Type, - pub to: String, - } - - #[derive(Type, Deserialize)] - pub struct RenameMany { - pub from_pattern: FromPattern, - pub to_pattern: String, - pub from_file_path_ids: Vec, - } - - #[derive(Type, Deserialize)] - pub enum RenameKind { - One(RenameOne), - Many(RenameMany), - } - - #[derive(Type, Deserialize)] - pub struct RenameFileArgs { - pub location_id: location::id::Type, - pub kind: RenameKind, - } - - impl RenameFileArgs { - pub async fn rename_one( - RenameOne { - from_file_path_id, - to, - }: RenameOne, - location_path: impl AsRef, - library: &Library, - ) -> Result<(), rspc::Error> { - let location_path = location_path.as_ref(); - let iso_file_path = IsolatedFilePathData::try_from( - library - .db - .file_path() - .find_unique(file_path::id::equals(from_file_path_id)) - .select(file_path_to_isolate::select()) - .exec() - .await? - .ok_or(LocationError::FilePath(FilePathError::IdNotFound( - from_file_path_id, - )))?, - ) - .map_err(LocationError::MissingField)?; - - if iso_file_path.full_name() == to { - return Ok(()); - } - - let (new_file_name, new_extension) = - IsolatedFilePathData::separate_name_and_extension_from_str(&to) - .map_err(LocationError::FilePath)?; - - let mut new_file_full_path = location_path.join(iso_file_path.parent()); - if !new_extension.is_empty() { - new_file_full_path.push(format!("{}.{}", new_file_name, new_extension)); - } else { - new_file_full_path.push(new_file_name); - } - - match fs::metadata(&new_file_full_path).await { - Ok(_) => { - return Err(rspc::Error::new( - ErrorCode::Conflict, - "Renaming would overwrite a file".to_string(), - )); - } - - Err(e) => { - if e.kind() != std::io::ErrorKind::NotFound { - return Err(rspc::Error::with_cause( - ErrorCode::InternalServerError, - "Failed to check if file exists".to_string(), - e, - )); - } - - fs::rename(location_path.join(&iso_file_path), new_file_full_path) - .await - .map_err(|e| { - rspc::Error::with_cause( - ErrorCode::InternalServerError, - "Failed to rename file".to_string(), - e, - ) - })?; - } - } - - Ok(()) - } - - pub async fn rename_many( - RenameMany { - from_pattern, - to_pattern, - from_file_path_ids, - }: RenameMany, - location_path: impl AsRef, - library: &Library, - ) -> Result<(), rspc::Error> { - let location_path = location_path.as_ref(); - - let Ok(from_regex) = Regex::new(&from_pattern.pattern) else { - return Err(rspc::Error::new( - rspc::ErrorCode::BadRequest, - "Invalid `from` regex pattern".into(), - )); - }; - - let errors = join_all( - library - .db - .file_path() - .find_many(vec![file_path::id::in_vec(from_file_path_ids)]) - .select(file_path_to_isolate_with_id::select()) - .exec() - .await? - .into_iter() - .flat_map(IsolatedFilePathData::try_from) - .map(|iso_file_path| { - let from = location_path.join(&iso_file_path); - let mut to = location_path.join(iso_file_path.parent()); - let full_name = iso_file_path.full_name(); - let replaced_full_name = if from_pattern.replace_all { - from_regex.replace_all(&full_name, &to_pattern) - } else { - from_regex.replace(&full_name, &to_pattern) - } - .to_string(); - - to.push(&replaced_full_name); - - async move { - if !IsolatedFilePathData::accept_file_name(&replaced_full_name) - { - Err(rspc::Error::new( - ErrorCode::BadRequest, - "Invalid file name".to_string(), - )) - } else { - fs::rename(&from, &to).await.map_err(|e| { - error!( - from = %from.display(), - to = %to.display(), - ?e, - "Failed to rename file;", - ); - rspc::Error::with_cause( - ErrorCode::Conflict, - "Failed to rename file".to_string(), - e, - ) - }) - } - } - }), - ) - .await - .into_iter() - .filter_map(Result::err) - .collect::>(); - - if !errors.is_empty() { - return Err(rspc::Error::new( - rspc::ErrorCode::Conflict, - errors - .into_iter() - .map(|e| e.to_string()) - .collect::>() - .join("\n"), - )); - } - - Ok(()) - } - } - - R.with2(library()).mutation( - |(_, library), RenameFileArgs { location_id, kind }: RenameFileArgs| async move { - let location_path = - get_location_path_from_location_id(&library.db, location_id).await?; - - let res = match kind { - RenameKind::One(one) => { - RenameFileArgs::rename_one(one, location_path, &library).await - } - RenameKind::Many(many) => { - RenameFileArgs::rename_many(many, location_path, &library).await - } - }; - - invalidate_query!(library, "search.paths"); - invalidate_query!(library, "search.objects"); - - res - }, - ) - }) -} - -pub(super) async fn create_directory( - mut target_path: PathBuf, - library: &Library, -) -> Result { - match fs::metadata(&target_path).await { - Ok(metadata) if metadata.is_dir() => { - target_path = find_available_filename_for_duplicate(&target_path).await?; - } - Ok(_) => { - return Err(FileSystemJobsError::WouldOverwrite(target_path.into_boxed_path()).into()) - } - Err(e) if e.kind() == io::ErrorKind::NotFound => { - // Everything is awesome! - } - Err(e) => { - return Err(FileIOError::from(( - target_path, - e, - "Failed to access file system and get metadata on directory to be created", - )) - .into()) - } - }; - - fs::create_dir(&target_path) - .await - .map_err(|e| FileIOError::from((&target_path, e, "Failed to create directory")))?; - - invalidate_query!(library, "search.objects"); - invalidate_query!(library, "search.paths"); - invalidate_query!(library, "search.ephemeralPaths"); - - Ok(target_path - .file_name() - .expect("Failed to get file name") - .to_string_lossy() - .to_string()) -} - -pub(super) async fn create_file( - mut target_path: PathBuf, - library: &Library, -) -> Result { - match fs::metadata(&target_path).await { - Ok(metadata) if metadata.is_file() => { - target_path = find_available_filename_for_duplicate(&target_path).await?; - } - Ok(_) => { - return Err(FileSystemJobsError::WouldOverwrite(target_path.into_boxed_path()).into()) - } - Err(e) if e.kind() == io::ErrorKind::NotFound => { - // Everything is awesome! - } - Err(e) => { - return Err(FileIOError::from(( - target_path, - e, - "Failed to access file system and get metadata on file to be created", - )) - .into()) - } - }; - - fs::File::create(&target_path) - .await - .map_err(|e| FileIOError::from((&target_path, e, "Failed to create file")))?; - - invalidate_query!(library, "search.objects"); - invalidate_query!(library, "search.paths"); - invalidate_query!(library, "search.ephemeralPaths"); - - Ok(target_path - .file_name() - .expect("Failed to get file name") - .to_string_lossy() - .to_string()) -} - -#[derive(Type, Deserialize)] -pub struct FromPattern { - pub pattern: String, - pub replace_all: bool, -} diff --git a/core/src/api/jobs.rs b/core/src/api/jobs.rs deleted file mode 100644 index 0cddeefda..000000000 --- a/core/src/api/jobs.rs +++ /dev/null @@ -1,444 +0,0 @@ -use crate::{ - context::NodeContext, - invalidate_query, - location::{find_location, LocationError}, - object::validation::old_validator_job::OldObjectValidatorJobInit, - old_job::{JobStatus, OldJob, OldJobReport}, -}; - -use sd_core_heavy_lifting::{ - file_identifier::FileIdentifier, job_system::report, media_processor::job::MediaProcessor, - JobId, JobSystemError, Report, -}; - -use sd_prisma::prisma::{job, location, SortOrder}; - -use std::{ - collections::{hash_map::Entry, BTreeMap, HashMap, VecDeque}, - path::PathBuf, - sync::Arc, - time::Instant, -}; - -use chrono::{DateTime, Utc}; -use prisma_client_rust::or; -use rspc::alpha::AlphaRouter; -use serde::{Deserialize, Serialize}; -use specta::Type; -use tokio::time::Duration; -use tracing::{info, trace}; -use uuid::Uuid; - -use super::{utils::library, CoreEvent, Ctx, R}; - -const TEN_MINUTES: Duration = Duration::from_secs(60 * 10); - -pub(crate) fn mount() -> AlphaRouter { - R.router() - .procedure("progress", { - // Listen for updates from the job manager - // - the client listens for events containing an updated JobReport - // - the client replaces its local copy of the JobReport using the index provided by the reports procedure - // - this should be used with the ephemeral sync engine - R.with2(library()) - .subscription(|(node, _), _: ()| async move { - let mut event_bus_rx = node.event_bus.0.subscribe(); - // debounce per-job - let mut intervals = BTreeMap::::new(); - - async_stream::stream! { - loop { - let progress_event = loop { - if let Ok(CoreEvent::JobProgress(progress_event)) = event_bus_rx.recv().await { - break progress_event; - } - }; - - let instant = intervals.entry(progress_event.id).or_insert_with( - Instant::now - ); - - if instant.elapsed() <= Duration::from_secs_f64(1.0 / 30.0) { - continue; - } - - yield progress_event; - - *instant = Instant::now(); - - // remove stale jobs that didn't receive a progress for more than 10 minutes - intervals.retain(|_, instant| instant.elapsed() < TEN_MINUTES); - } - } - }) - }) - .procedure("reports", { - // Reports provides the client with a list of JobReports - // - we query with a custom select! to avoid returning paused job cache `job.data` - // - results must include running jobs, and be combined with the in-memory state - // this is to ensure the client will always get the correct initial state - // - jobs are sorted into groups by their action - // - TODO: refactor grouping system to a many-to-many table - #[derive(Debug, Clone, Serialize, Type)] - pub struct JobGroup { - id: JobId, - running_job_id: Option, - action: Option, - status: report::Status, - created_at: DateTime, - jobs: VecDeque, - } - - R.with2(library()) - .query(|(node, library), _: ()| async move { - // Check if the job system is active - let is_active = node - .job_system - .has_active_jobs(NodeContext { - node: Arc::clone(&node), - library: library.clone(), - }) - .await || node.old_jobs.has_active_workers(library.id).await; - - let mut groups: HashMap = HashMap::new(); - - let job_reports: Vec = library - .db - .job() - .find_many(vec![]) - .order_by(job::date_created::order(SortOrder::Desc)) - .take(100) - .exec() - .await? - .into_iter() - .flat_map(|job| { - if let Ok(report) = Report::try_from(job.clone()) { - Some(report) - } else { - // TODO: this is a temporary fix for the old job system - OldJobReport::try_from(job).map(Into::into).ok() - } - }) - .collect(); - - let mut active_reports_by_id = node.job_system.get_active_reports().await; - active_reports_by_id.extend( - node.old_jobs - .get_active_reports_with_id() - .await - .into_iter() - .map(|(id, old_report)| (id, old_report.into())), - ); - - for job in job_reports { - // Skip running jobs if the job system is not active. Temporary fix - if !is_active && job.status == report::Status::Running { - continue; - } - - // action name and group key are computed from the job data - let (action_name, group_key) = job.get_action_name_and_group_key(); - - trace!(?job, %action_name, ?group_key); - - // if the job is running, use the in-memory report - let report = active_reports_by_id.get(&job.id).unwrap_or(&job); - - // if we have a group key, handle grouping - if let Some(group_key) = group_key { - match groups.entry(group_key) { - // Create new job group with metadata - Entry::Vacant(entry) => { - entry.insert(JobGroup { - id: job.parent_id.unwrap_or(job.id), - running_job_id: (job.status == report::Status::Running - || job.status == report::Status::Paused) - .then_some(job.id), - action: Some(action_name), - status: job.status, - jobs: [report.clone()].into_iter().collect(), - created_at: job.created_at.unwrap_or(Utc::now()), - }); - } - // Add to existing job group - Entry::Occupied(mut entry) => { - let group = entry.get_mut(); - - if report.status == report::Status::Running - || report.status == report::Status::Paused - { - group.running_job_id = Some(report.id); - group.status = report.status; - } - - group.jobs.push_front(report.clone()); - } - } - } else { - // insert individual job as group - groups.insert( - job.id.to_string(), - JobGroup { - id: job.id, - running_job_id: Some(job.id), - action: None, - status: job.status, - jobs: [report.clone()].into_iter().collect(), - created_at: job.created_at.unwrap_or(Utc::now()), - }, - ); - } - } - - let mut groups_vec = groups.into_values().collect::>(); - groups_vec.sort_by(|a, b| b.created_at.cmp(&a.created_at)); - - Ok(groups_vec) - }) - }) - .procedure("isActive", { - R.with2(library()) - .query(|(node, library), _: ()| async move { - let library_id = library.id; - Ok(node - .job_system - .has_active_jobs(NodeContext { - node: Arc::clone(&node), - library, - }) - .await || node.old_jobs.has_active_workers(library_id).await) - }) - }) - .procedure("clear", { - R.with2(library()) - .mutation(|(_, library), id: Uuid| async move { - library - .db - .job() - .delete(job::id::equals(id.as_bytes().to_vec())) - .exec() - .await?; - - invalidate_query!(library, "jobs.reports"); - Ok(()) - }) - }) - .procedure("clearAll", { - R.with2(library()) - .mutation(|(_, library), _: ()| async move { - info!("Clearing all jobs"); - library - .db - .job() - .delete_many(vec![or![ - job::status::equals(Some(JobStatus::Canceled as i32)), - job::status::equals(Some(JobStatus::Failed as i32)), - job::status::equals(Some(JobStatus::Completed as i32)), - job::status::equals(Some(JobStatus::CompletedWithErrors as i32)), - ]]) - .exec() - .await?; - - invalidate_query!(library, "jobs.reports"); - Ok(()) - }) - }) - // pause job - .procedure("pause", { - R.with2(library()) - .mutation(|(node, library), job_id: JobId| async move { - if let Err(e) = node.job_system.pause(job_id).await { - if matches!(e, JobSystemError::NotFound(_)) { - // If the job is not found, it can be a job from the old job system - node.old_jobs.pause(job_id).await?; - } else { - return Err(e.into()); - } - } - - invalidate_query!(library, "jobs.isActive"); - invalidate_query!(library, "jobs.reports"); - - Ok(()) - }) - }) - .procedure("resume", { - R.with2(library()) - .mutation(|(node, library), job_id: JobId| async move { - if let Err(e) = node.job_system.resume(job_id).await { - if matches!(e, JobSystemError::NotFound(_)) { - // If the job is not found, it can be a job from the old job system - node.old_jobs.resume(job_id).await?; - } else { - return Err(e.into()); - } - } - - invalidate_query!(library, "jobs.isActive"); - invalidate_query!(library, "jobs.reports"); - - Ok(()) - }) - }) - .procedure("cancel", { - R.with2(library()) - .mutation(|(node, library), job_id: JobId| async move { - if let Err(e) = node.job_system.cancel(job_id).await { - if matches!(e, JobSystemError::NotFound(_)) { - // If the job is not found, it can be a job from the old job system - node.old_jobs.cancel(job_id).await?; - } else { - return Err(e.into()); - } - } - - invalidate_query!(library, "jobs.isActive"); - invalidate_query!(library, "jobs.reports"); - - Ok(()) - }) - }) - .procedure("generateThumbsForLocation", { - #[derive(Type, Deserialize)] - pub struct GenerateThumbsForLocationArgs { - pub id: location::id::Type, - pub path: PathBuf, - #[serde(default)] - pub regenerate: bool, - } - - R.with2(library()).mutation( - |(node, library), - GenerateThumbsForLocationArgs { - id, - path, - regenerate, - }: GenerateThumbsForLocationArgs| async move { - let Some(location) = find_location(&library, id).exec().await? else { - return Err(LocationError::IdNotFound(id).into()); - }; - - node.job_system - .dispatch( - MediaProcessor::new(location, Some(path), regenerate)?, - id, - NodeContext { - node: Arc::clone(&node), - library, - }, - ) - .await - .map_err(Into::into) - }, - ) - }) - // .procedure("generateLabelsForLocation", { - // #[derive(Type, Deserialize)] - // pub struct GenerateLabelsForLocationArgs { - // pub id: location::id::Type, - // pub path: PathBuf, - // #[serde(default)] - // pub regenerate: bool, - // } - // R.with2(library()).mutation( - // |(node, library), - // GenerateLabelsForLocationArgs { - // id, - // path, - // regenerate, - // }: GenerateLabelsForLocationArgs| async move { - // let Some(location) = find_location(&library, id).exec().await? else { - // return Err(LocationError::IdNotFound(id).into()); - // }; - // OldJob::new(OldMediaProcessorJobInit { - // location, - // sub_path: Some(path), - // regenerate_thumbnails: false, - // regenerate_labels: regenerate, - // }) - // .spawn(&node, &library) - // .await - // .map_err(Into::into) - // }, - // ) - // }) - .procedure("objectValidator", { - #[derive(Type, Deserialize)] - pub struct ObjectValidatorArgs { - pub id: location::id::Type, - pub path: PathBuf, - } - - R.with2(library()) - .mutation(|(node, library), args: ObjectValidatorArgs| async move { - let Some(location) = find_location(&library, args.id).exec().await? else { - return Err(LocationError::IdNotFound(args.id).into()); - }; - - OldJob::new(OldObjectValidatorJobInit { - location, - sub_path: Some(args.path), - }) - .spawn(&node, &library) - .await - .map_err(Into::into) - }) - }) - .procedure("identifyUniqueFiles", { - #[derive(Type, Deserialize)] - pub struct IdentifyUniqueFilesArgs { - pub id: location::id::Type, - pub path: PathBuf, - } - - R.with2(library()).mutation( - |(node, library), IdentifyUniqueFilesArgs { id, path }: IdentifyUniqueFilesArgs| async move { - let Some(location) = find_location(&library, id).exec().await? else { - return Err(LocationError::IdNotFound(id).into()); - }; - - node.job_system - .dispatch( - FileIdentifier::new(location, Some(path))?, - id, - NodeContext { - node: Arc::clone(&node), - library, - }, - ) - .await - .map_err(Into::into) - }, - ) - }) - .procedure("newThumbnail", { - R.with2(library()) - .subscription(|(node, _), _: ()| async move { - // TODO: Only return event for the library that was subscribed to - let mut event_bus_rx = node.event_bus.0.subscribe(); - async_stream::stream! { - while let Ok(event) = event_bus_rx.recv().await { - match event { - CoreEvent::NewThumbnail { thumb_key } => yield thumb_key, - _ => {} - } - } - } - }) - }) - .procedure("newFilePathIdentified", { - R.with2(library()) - .subscription(|(node, _), _: ()| async move { - // TODO: Only return event for the library that was subscribed to - let mut event_bus_rx = node.event_bus.0.subscribe(); - async_stream::stream! { - while let Ok(event) = event_bus_rx.recv().await { - match event { - CoreEvent::NewIdentifiedObjects { file_path_ids } => yield file_path_ids, - _ => {} - } - } - } - }) - }) -} diff --git a/core/src/api/keys.rs b/core/src/api/keys.rs deleted file mode 100644 index 3423a6ccc..000000000 --- a/core/src/api/keys.rs +++ /dev/null @@ -1,447 +0,0 @@ -use super::utils::library; -use super::{Ctx, SanitizedNodeConfig, R}; -use once_cell::sync::Lazy; -use rspc::{alpha::AlphaRouter, ErrorCode}; -use sd_crypto::cookie::CookieCipher; -use serde_json::{json, Map, Value}; -use std::path::Path; -use std::sync::Arc; -use tokio::io::AsyncWriteExt; -use tokio::sync::RwLock; -use tracing::{debug, error}; - -static CACHE: Lazy>> = Lazy::new(|| RwLock::new(None)); - -#[derive(Clone)] -struct CipherCache { - uuid: String, - cipher: CookieCipher, -} - -async fn get_cipher( - node: &Ctx, - cache: Arc>>, -) -> Result { - let config = SanitizedNodeConfig::from(node.config.get().await); - let uuid = config.id.to_string(); - - { - let cache_read = cache.read().await; - if let Some(ref cache) = *cache_read { - if cache.uuid == uuid { - return Ok(cache.cipher.clone()); - } - } - } - - let uuid_key = CookieCipher::generate_key_from_string(&uuid).map_err(|e| { - error!("Failed to generate key: {:?}", e.to_string()); - rspc::Error::new( - ErrorCode::InternalServerError, - "Failed to generate key".to_string(), - ) - })?; - - let cipher = CookieCipher::new(&uuid_key).map_err(|e| { - error!("Failed to create cipher: {:?}", e.to_string()); - rspc::Error::new( - ErrorCode::InternalServerError, - "Failed to create cipher".to_string(), - ) - })?; - - { - let mut cache_write = cache.write().await; - *cache_write = Some(CipherCache { - uuid, - cipher: cipher.clone(), - }); - } - - Ok(cipher) -} - -async fn read_file(path: &Path) -> Result, rspc::Error> { - tokio::fs::read(path).await.map_err(|e| { - error!("Failed to read file: {:?}", e.to_string()); - rspc::Error::new( - ErrorCode::InternalServerError, - format!("Failed to read file {:?}", path), - ) - }) -} - -async fn write_file(path: &Path, data: &[u8]) -> Result<(), rspc::Error> { - let mut file = tokio::fs::OpenOptions::new() - .write(true) - .create(true) - .truncate(true) - .open(path) - .await - .map_err(|e| { - error!("Failed to open file: {:?}", e.to_string()); - rspc::Error::new( - ErrorCode::InternalServerError, - format!("Failed to open file {:?}", path), - ) - })?; - file.write_all(data).await.map_err(|e| { - error!("Failed to write to file: {:?}", e.to_string()); - rspc::Error::new( - ErrorCode::InternalServerError, - format!("Failed to write to file {:?}", path), - ) - }) -} - -// fn sanitize_path(base_dir: &Path, path: &Path) -> Result { -// let abs_base = base_dir.canonicalize().map_err(|e| { -// error!("Failed to canonicalize base directory: {:?}", e.to_string()); -// rspc::Error::new( -// ErrorCode::InternalServerError, -// "Failed to canonicalize base directory".to_string(), -// ) -// })?; -// let abs_path = abs_base.join(path).canonicalize().map_err(|e| { -// error!("Failed to canonicalize path: {:?}", e.to_string()); -// rspc::Error::new( -// ErrorCode::InternalServerError, -// "Failed to canonicalize path".to_string(), -// ) -// })?; -// if abs_path.starts_with(&abs_base) { -// Ok(abs_path) -// } else { -// error!("Path injection attempt detected: {:?}", abs_path); -// Err(rspc::Error::new( -// ErrorCode::InternalServerError, -// "Invalid path".to_string(), -// )) -// } -// } - -pub(crate) fn mount() -> AlphaRouter { - let cipher_cache = Arc::new(RwLock::new(None)); - - R.router() - .procedure("get", { - let cipher_cache = cipher_cache.clone(); - R.query(move |node, _: ()| { - let cipher_cache = cipher_cache.clone(); - async move { - let cache_guard = CACHE.read().await; - if let Some(cached_data) = cache_guard.clone().as_ref() { - debug!("Returning cached data"); - return Ok(cached_data.clone()); - } - - let base_dir = node.config.data_directory(); - // Remove /dev from Path - let base_dir = if base_dir.ends_with("dev") { - base_dir.parent().unwrap_or(&base_dir).to_path_buf() - } else { - base_dir - }; - // let path = sanitize_path(&base_dir, Path::new(".sdks"))?; - let path = base_dir.join(".sdks"); - let data = read_file(&path).await?; - let cipher = get_cipher(&node, cipher_cache).await?; - - let data_str = String::from_utf8(data).map_err(|e| { - error!("Failed to convert data to string: {:?}", e.to_string()); - rspc::Error::new( - ErrorCode::InternalServerError, - "Failed to convert data to string".to_string(), - ) - })?; - let data = CookieCipher::base64_decode(&data_str).map_err(|e| { - error!("Failed to decode data: {:?}", e.to_string()); - rspc::Error::new( - ErrorCode::InternalServerError, - "Failed to decode data".to_string(), - ) - })?; - let de_data = cipher.decrypt(&data).map_err(|e| { - error!("Failed to decrypt data: {:?}", e.to_string()); - rspc::Error::new( - ErrorCode::InternalServerError, - "Failed to decrypt data".to_string(), - ) - })?; - let de_data = String::from_utf8(de_data).map_err(|e| { - error!("Failed to convert data to string: {:?}", e.to_string()); - rspc::Error::new( - ErrorCode::InternalServerError, - "Failed to convert data to string".to_string(), - ) - })?; - Ok(de_data) - } - }) - }) - .procedure("save", { - let cipher_cache = cipher_cache.clone(); - R.mutation(move |node, args: String| { - let cipher_cache = cipher_cache.clone(); - async move { - let base_dir = node.config.data_directory(); - // let path = sanitize_path(&base_dir, Path::new(".sdks"))?; - let path = base_dir.join(".sdks"); - - // Read and decrypt existing data if it exists - let existing_decrypted = if let Ok(existing_data) = read_file(&path).await { - let cipher = get_cipher(&node, cipher_cache.clone()).await?; - let data_str = String::from_utf8(existing_data).map_err(|e| { - rspc::Error::new( - ErrorCode::InternalServerError, - "Failed to convert data to string".to_string(), - ) - })?; - let decoded = CookieCipher::base64_decode(&data_str).map_err(|e| { - rspc::Error::new( - ErrorCode::InternalServerError, - "Failed to decode data".to_string(), - ) - })?; - let decrypted = cipher.decrypt(&decoded).map_err(|e| { - rspc::Error::new( - ErrorCode::InternalServerError, - "Failed to decrypt data".to_string(), - ) - })?; - String::from_utf8(decrypted).ok() - } else { - None - }; - - // Compare unencrypted data - if let Some(existing) = existing_decrypted { - if existing == args { - debug!("Data unchanged, skipping write operation"); - return Ok(()); - } - } - - // Only encrypt and write if data changed - let cipher = get_cipher(&node, cipher_cache).await?; - let en_data = cipher.encrypt(args.as_bytes()).map_err(|e| { - rspc::Error::new( - ErrorCode::InternalServerError, - "Failed to encrypt data".to_string(), - ) - })?; - - let en_data = CookieCipher::base64_encode(&en_data); - - write_file(&path, en_data.as_bytes()).await?; - let mut cache_guard = CACHE.write().await; - *cache_guard = Some(args.clone()); - debug!("Written to read cache"); - - debug!("Saved data to {:?}", path); - Ok(()) - } - }) - }) - .procedure("saveEmailAddress", { - R.with2(library()) - .mutation(move |(node, library), args: String| async move { - let path = node - .libraries - .libraries_dir - .join(format!("{}.sdlibrary", library.id)); - - let mut config = serde_json::from_slice::>( - &tokio::fs::read(path.clone()).await.map_err(|e| { - rspc::Error::new( - ErrorCode::InternalServerError, - format!("Failed to read library config: {:?}", e.to_string()), - ) - })?, - ) - .map_err(|e: serde_json::Error| { - rspc::Error::new( - ErrorCode::InternalServerError, - format!("Failed to parse library config: {:?}", e.to_string()), - ) - })?; - - // Decrypt existing email if present - let existing_email = if let Some(encrypted) = config.get("cloud_email_address") - { - if let Some(encrypted_str) = encrypted.as_str() { - let uuid_key = CookieCipher::generate_key_from_string( - library.id.to_string().as_str(), - ) - .map_err(|e| { - error!("Failed to generate key: {:?}", e.to_string()); - rspc::Error::new( - ErrorCode::InternalServerError, - "Failed to generate key".to_string(), - ) - })?; - let cipher = CookieCipher::new(&uuid_key).map_err(|e| { - error!("Failed to create cipher: {:?}", e.to_string()); - rspc::Error::new( - ErrorCode::InternalServerError, - "Failed to create cipher".to_string(), - ) - })?; - let decoded = - CookieCipher::base64_decode(encrypted_str).map_err(|e| { - error!("Failed to decode data: {:?}", e.to_string()); - rspc::Error::new( - ErrorCode::InternalServerError, - "Failed to decode data".to_string(), - ) - })?; - let decrypted = cipher.decrypt(&decoded).map_err(|e| { - error!("Failed to decrypt data: {:?}", e.to_string()); - rspc::Error::new( - ErrorCode::InternalServerError, - "Failed to decrypt data".to_string(), - ) - })?; - String::from_utf8(decrypted).ok() - } else { - None - } - } else { - None - }; - - // Compare unencrypted data - if let Some(existing) = existing_email { - if existing == args { - debug!("Email unchanged, skipping write operation"); - return Ok(()); - } - } - - // Only encrypt and write if email changed - let uuid_key = - CookieCipher::generate_key_from_string(library.id.to_string().as_str()) - .map_err(|e| { - error!("Failed to generate key: {:?}", e.to_string()); - rspc::Error::new( - ErrorCode::InternalServerError, - "Failed to generate key".to_string(), - ) - })?; - let cipher = CookieCipher::new(&uuid_key).map_err(|e| { - error!("Failed to create cipher: {:?}", e.to_string()); - rspc::Error::new( - ErrorCode::InternalServerError, - "Failed to create cipher".to_string(), - ) - })?; - let en_data = cipher.encrypt(args.as_bytes()).map_err(|e| { - rspc::Error::new( - ErrorCode::InternalServerError, - "Failed to encrypt data".to_string(), - ) - })?; - let en_data = CookieCipher::base64_encode(&en_data); - - config.insert("cloud_email_address".to_string(), json!(en_data)); - - let config_vec = serde_json::to_vec(&config).map_err(|e| { - rspc::Error::new( - ErrorCode::InternalServerError, - format!("Failed to serialize library config: {:?}", e.to_string()), - ) - })?; - - tokio::fs::write(path, config_vec).await.map_err(|e| { - rspc::Error::new( - ErrorCode::InternalServerError, - format!("Failed to write library config: {:?}", e.to_string()), - ) - })?; - - Ok(()) - }) - }) - .procedure("getEmailAddress", { - R.with2(library()) - .query(move |(node, library), _: ()| async move { - let path = node - .libraries - .libraries_dir - .join(format!("{}.sdlibrary", library.id)); - - let config = serde_json::from_slice::>( - &tokio::fs::read(path.clone()).await.map_err(|e| { - rspc::Error::new( - ErrorCode::InternalServerError, - format!("Failed to read library config: {:?}", e.to_string()), - ) - })?, - ) - .map_err(|e: serde_json::Error| { - rspc::Error::new( - ErrorCode::InternalServerError, - format!("Failed to parse library config: {:?}", e.to_string()), - ) - })?; - - let en_data = config.get("cloud_email_address").ok_or_else(|| { - rspc::Error::new( - ErrorCode::InternalServerError, - "Failed to get cloud_email_address".to_string(), - ) - })?; - - let en_data = en_data.as_str().ok_or_else(|| { - rspc::Error::new( - ErrorCode::InternalServerError, - "Failed to get cloud_email_address".to_string(), - ) - })?; - - let en_data = CookieCipher::base64_decode(en_data).map_err(|e| { - error!("Failed to decode data: {:?}", e.to_string()); - rspc::Error::new( - ErrorCode::InternalServerError, - "Failed to decode data".to_string(), - ) - })?; - - let uuid_key = - CookieCipher::generate_key_from_string(library.id.to_string().as_str()) - .map_err(|e| { - error!("Failed to generate key: {:?}", e.to_string()); - rspc::Error::new( - ErrorCode::InternalServerError, - "Failed to generate key".to_string(), - ) - })?; - - let cipher = CookieCipher::new(&uuid_key).map_err(|e| { - error!("Failed to create cipher: {:?}", e.to_string()); - rspc::Error::new( - ErrorCode::InternalServerError, - "Failed to create cipher".to_string(), - ) - })?; - - let de_data = cipher.decrypt(&en_data).map_err(|e| { - error!("Failed to decrypt data: {:?}", e.to_string()); - rspc::Error::new( - ErrorCode::InternalServerError, - "Failed to decrypt data".to_string(), - ) - })?; - - let de_data = String::from_utf8(de_data).map_err(|e| { - error!("Failed to convert data to string: {:?}", e.to_string()); - rspc::Error::new( - ErrorCode::InternalServerError, - "Failed to convert data to string".to_string(), - ) - })?; - - Ok(de_data) - }) - }) -} diff --git a/core/src/api/labels.rs b/core/src/api/labels.rs deleted file mode 100644 index eed08d8d3..000000000 --- a/core/src/api/labels.rs +++ /dev/null @@ -1,175 +0,0 @@ -use crate::{invalidate_query, library::Library}; - -use sd_core_heavy_lifting::media_processor::ThumbKey; -use sd_core_prisma_helpers::{label_with_objects, CasId}; - -use sd_prisma::{ - prisma::{label, label_on_object, object, SortOrder}, - prisma_sync, -}; -use sd_sync::OperationFactory; - -use std::collections::BTreeMap; - -use rspc::alpha::AlphaRouter; - -use super::{locations::ExplorerItem, utils::library, Ctx, R}; - -pub(crate) fn mount() -> AlphaRouter { - R.router() - .procedure("list", { - R.with2(library()).query(|(_, library), _: ()| async move { - Ok(library.db.label().find_many(vec![]).exec().await?) - }) - }) - .procedure("listWithThumbnails", { - R.with2(library()) - .query(|(_, library), cursor: label::name::Type| async move { - Ok(library - .db - .label() - .find_many(vec![label::name::gt(cursor)]) - .order_by(label::name::order(SortOrder::Asc)) - .include(label_with_objects::include(4)) - .exec() - .await? - .into_iter() - .map(|label| ExplorerItem::Label { - item: label.clone(), - // map the first 4 objects to thumbnails - thumbnails: label - .label_objects - .into_iter() - .take(10) - .filter_map(|label_object| { - label_object.object.file_paths.into_iter().next() - }) - .filter_map(|file_path_data| { - file_path_data - .cas_id - .as_ref() - .map(CasId::from) - .map(CasId::into_owned) - .map(|cas_id| ThumbKey::new_indexed(cas_id, library.id)) - }) // Filter out None values and transform each element to Vec> - .collect::>(), // Collect into Vec>> - }) - .collect::>()) - }) - }) - .procedure("count", { - R.with2(library()).query(|(_, library), _: ()| async move { - Ok(library.db.label().count(vec![]).exec().await? as i32) - }) - }) - .procedure("getForObject", { - R.with2(library()) - .query(|(_, library), object_id: i32| async move { - Ok(library - .db - .label() - .find_many(vec![label::label_objects::some(vec![ - label_on_object::object_id::equals(object_id), - ])]) - .exec() - .await?) - }) - }) - .procedure("getWithObjects", { - R.with2(library()).query( - |(_, library), object_ids: Vec| async move { - let Library { db, .. } = library.as_ref(); - let labels_with_objects = db - .label() - .find_many(vec![label::label_objects::some(vec![ - label_on_object::object_id::in_vec(object_ids.clone()), - ])]) - .select(label::select!({ - id - label_objects(vec![label_on_object::object_id::in_vec(object_ids.clone())]): select { - date_created - object: select { - id - } - } - })) - .exec() - .await?; - Ok(labels_with_objects - .into_iter() - .map(|label| (label.id, label.label_objects)) - .collect::>()) - }, - ) - }) - .procedure("get", { - R.with2(library()) - .query(|(_, library), label_id: i32| async move { - Ok(library - .db - .label() - .find_unique(label::id::equals(label_id)) - .exec() - .await?) - }) - }) - .procedure( - "delete", - R.with2(library()) - .mutation(|(_, library), label_id: label::id::Type| async move { - let Library { db, sync, .. } = library.as_ref(); - - let label = db - .label() - .find_unique(label::id::equals(label_id)) - .exec() - .await? - .ok_or_else(|| { - rspc::Error::new( - rspc::ErrorCode::NotFound, - "Label not found".to_string(), - ) - })?; - - let delete_ops = db - .label_on_object() - .find_many(vec![label_on_object::label_id::equals(label_id)]) - .select(label_on_object::select!({ object: select { pub_id } })) - .exec() - .await? - .into_iter() - .map(|label_on_object| { - sync.relation_delete(prisma_sync::label_on_object::SyncId { - label: prisma_sync::label::SyncId { - name: label.name.clone(), - }, - object: prisma_sync::object::SyncId { - pub_id: label_on_object.object.pub_id, - }, - }) - }) - .collect::>(); - - sync.write_ops( - db, - ( - delete_ops, - db.label_on_object() - .delete_many(vec![label_on_object::label_id::equals(label_id)]), - ), - ) - .await?; - - sync.write_op( - db, - sync.shared_delete(prisma_sync::label::SyncId { name: label.name }), - db.label().delete(label::id::equals(label_id)), - ) - .await?; - - invalidate_query!(library, "labels.list"); - - Ok(()) - }), - ) -} diff --git a/core/src/api/libraries.rs b/core/src/api/libraries.rs deleted file mode 100644 index 2f9712b70..000000000 --- a/core/src/api/libraries.rs +++ /dev/null @@ -1,667 +0,0 @@ -use crate::{ - api::CoreEvent, - invalidate_query, - library::{update_library_statistics, Library, LibraryConfig, LibraryName}, - location::{scan_location, LocationCreateArgs, ScanState}, - util::MaybeUndefined, - Node, -}; - -use sd_core_heavy_lifting::JobId; - -use sd_file_ext::kind::ObjectKind; -use sd_old_p2p::RemoteIdentity; -use sd_prisma::prisma::{file_path, indexer_rule, object, object_kind_statistics, statistics}; -use sd_utils::{db::size_in_bytes_from_db, u64_to_frontend}; - -use std::{ - collections::{hash_map::Entry, HashMap}, - convert::identity, - pin::pin, - sync::{ - atomic::{AtomicBool, Ordering}, - Arc, LazyLock, - }, - time::Duration, -}; - -use async_channel as chan; -use directories::UserDirs; -use futures::{FutureExt, StreamExt}; -use futures_concurrency::{ - future::{Join, TryJoin}, - stream::Merge, -}; -use prisma_client_rust::{raw, QueryError}; -use rspc::{alpha::AlphaRouter, ErrorCode}; -use serde::{Deserialize, Serialize}; -use specta::Type; -use strum::IntoEnumIterator; -use tokio::{ - spawn, - sync::{Mutex, RwLock}, - time::{interval, Instant}, -}; -use tokio_stream::wrappers::IntervalStream; -use tracing::{debug, error, info, warn}; -use uuid::Uuid; - -use super::{utils::library, Ctx, R}; - -const ONE_MINUTE: Duration = Duration::from_secs(60); -const TWO_MINUTES: Duration = Duration::from_secs(60 * 2); -const FIVE_MINUTES: Duration = Duration::from_secs(60 * 5); - -static IS_COMPUTING_KIND_STATISTICS: AtomicBool = AtomicBool::new(false); -static LAST_KIND_STATISTICS_UPDATE: LazyLock> = - LazyLock::new(RwLock::default); - -static STATISTICS_UPDATERS: LazyLock>>> = - LazyLock::new(|| Mutex::new(HashMap::new())); - -struct TotalFilesStatistics { - updated_at: Instant, - total_identified_files: u64, - total_unidentified_files: u64, -} - -impl Default for TotalFilesStatistics { - fn default() -> Self { - Self { - updated_at: Instant::now() - FIVE_MINUTES, - total_identified_files: 0, - total_unidentified_files: 0, - } - } -} - -// TODO(@Oscar): Replace with `specta::json` -#[derive(Serialize, Type)] -pub struct LibraryConfigWrapped { - pub uuid: Uuid, - pub instance_id: Uuid, - pub instance_public_key: RemoteIdentity, - pub config: LibraryConfig, -} - -impl LibraryConfigWrapped { - pub async fn from_library(library: &Library) -> Self { - Self { - uuid: library.id, - instance_id: library.instance_uuid, - instance_public_key: library.identity.to_remote_identity(), - config: library.config().await, - } - } -} - -#[derive(Debug, Serialize, Deserialize, Type, Default, Clone)] -pub struct KindStatistic { - kind: i32, - name: String, - count: (u32, u32), - total_bytes: (u32, u32), -} - -pub(crate) fn mount() -> AlphaRouter { - R.router() - .procedure("list", { - R.query(|node, _: ()| async move { - Ok(node - .libraries - .get_all() - .await - .into_iter() - .map(|lib| async move { - LibraryConfigWrapped { - uuid: lib.id, - instance_id: lib.instance_uuid, - instance_public_key: lib.identity.to_remote_identity(), - config: lib.config().await, - } - }) - .collect::>() - .join() - .await) - }) - }) - .procedure("statistics", { - #[derive(Serialize, Deserialize, Type)] - pub struct StatisticsResponse { - statistics: Option, - } - R.with2(library()) - .query(|(node, library), _: ()| async move { - debug!("Getting statistics for library"); - let statistics = library - .db - .statistics() - .find_unique(statistics::id::equals(1)) - .exec() - .await?; - debug!(?statistics, "Got statistics for library"); - - match STATISTICS_UPDATERS.lock().await.entry(library.id) { - Entry::Occupied(entry) => { - if entry.get().send(Instant::now()).await.is_err() { - error!("Failed to send statistics update request;"); - } - debug!("Sent statistics update request;"); - } - Entry::Vacant(entry) => { - let (tx, rx) = chan::bounded(1); - entry.insert(tx); - - debug!("Spawning statistics updater loop;"); - spawn(update_statistics_loop(node, library, rx)); - } - } - - Ok(StatisticsResponse { statistics }) - }) - }) - .procedure("kindStatistics", { - #[derive(Debug, Serialize, Deserialize, Type, Default)] - pub struct KindStatistics { - statistics: HashMap, - total_identified_files: i32, - total_unidentified_files: i32, - } - - R.with2(library()) - .query(|(node, library), _: ()| async move { - let last_kind_statistics = LAST_KIND_STATISTICS_UPDATE.read().await; - - let (total_unidentified_files, total_identified_files) = if last_kind_statistics - .updated_at - .elapsed() - < ONE_MINUTE - { - ( - last_kind_statistics.total_unidentified_files, - last_kind_statistics.total_identified_files, - ) - } else { - drop(last_kind_statistics); - let mut last_kind_statistics = LAST_KIND_STATISTICS_UPDATE.write().await; - - if let Ok(false) = IS_COMPUTING_KIND_STATISTICS.compare_exchange( - false, - true, - Ordering::Release, - Ordering::Relaxed, - ) { - spawn({ - let library = Arc::clone(&library); - async move { - if let Err(e) = update_kind_statistics(node, library).await { - error!(?e, "Failed to update kind statistics;"); - } - } - }); - } - - let (total_unidentified_files, total_identified_files) = ( - library - .db - .file_path() - .count(vec![file_path::object_id::equals(None)]) - .exec() - .map(|count_res| count_res.map(|count| count as u64)), - library - .db - .file_path() - .count(vec![file_path::object_id::not(None)]) - .exec() - .map(|count_res| count_res.map(|count| count as u64)), - ) - .try_join() - .await?; - - last_kind_statistics.total_unidentified_files = total_unidentified_files; - last_kind_statistics.total_identified_files = total_identified_files; - last_kind_statistics.updated_at = Instant::now(); - - (total_unidentified_files, total_identified_files) - }; - - let statistics = ObjectKind::iter() - .map(|kind| { - let library = Arc::clone(&library); - async move { - let int_kind = kind as i32; - library - .db - .object_kind_statistics() - .find_unique(object_kind_statistics::kind::equals(int_kind)) - .select( - object_kind_statistics::select!({ total_bytes files_count }), - ) - .exec() - .await - .map(|maybe_data| { - let (files_count, total_bytes) = - maybe_data.map_or((0, 0), |data| { - (data.files_count as u64, data.total_bytes as u64) - }); - - ( - int_kind, - KindStatistic { - kind: int_kind, - name: kind.to_string(), - count: u64_to_frontend(files_count), - total_bytes: u64_to_frontend(total_bytes), - }, - ) - }) - } - }) - .collect::>() - .try_join() - .await? - .into_iter() - .collect(); - - Ok(KindStatistics { - statistics, - total_identified_files: total_identified_files as i32, - total_unidentified_files: total_unidentified_files as i32, - }) - }) - }) - .procedure("updatedKindStatistic", { - R.with2(library()) - .subscription(|(node, library), _: ()| async move { - let mut event_bus_rx = node.event_bus.0.subscribe(); - async_stream::stream! { - while let Ok(event) = event_bus_rx.recv().await { - match event { - CoreEvent::UpdatedKindStatistic(stat, library_id) if library_id == library.id => { - yield stat - }, - _ => {} - } - } - } - }) - }) - .procedure("create", { - #[derive(Deserialize, Type, Default)] - pub struct DefaultLocations { - desktop: bool, - documents: bool, - downloads: bool, - pictures: bool, - music: bool, - videos: bool, - } - - #[derive(Deserialize, Type)] - pub struct CreateLibraryArgs { - name: LibraryName, - default_locations: Option, - } - - async fn create_default_locations_on_library_creation( - DefaultLocations { - desktop, - documents, - downloads, - pictures, - music, - videos, - }: DefaultLocations, - node: Arc, - library: Arc, - ) -> Result, rspc::Error> { - // If all of them are false, we skip - if [!desktop, !documents, !downloads, !pictures, !music, !videos] - .into_iter() - .all(identity) - { - return Ok(None); - } - - let Some(default_locations_paths) = UserDirs::new() else { - return Err(rspc::Error::new( - ErrorCode::NotFound, - "Didn't find any system locations".to_string(), - )); - }; - - let default_rules_ids = library - .db - .indexer_rule() - .find_many(vec![indexer_rule::default::equals(Some(true))]) - .select(indexer_rule::select!({ id })) - .exec() - .await - .map_err(|e| { - rspc::Error::with_cause( - ErrorCode::InternalServerError, - "Failed to get default indexer rules for default locations".to_string(), - e, - ) - })? - .into_iter() - .map(|rule| rule.id) - .collect::>(); - - let mut maybe_error = None; - - [ - (desktop, default_locations_paths.desktop_dir()), - (documents, default_locations_paths.document_dir()), - (downloads, default_locations_paths.download_dir()), - (pictures, default_locations_paths.picture_dir()), - (music, default_locations_paths.audio_dir()), - (videos, default_locations_paths.video_dir()), - ] - .into_iter() - .filter_map(|entry| { - if let (true, Some(path)) = entry { - let node = Arc::clone(&node); - let library = Arc::clone(&library); - let indexer_rules_ids = default_rules_ids.clone(); - let path = path.to_path_buf(); - Some(spawn(async move { - let Some(location) = LocationCreateArgs { - path, - dry_run: false, - indexer_rules_ids, - } - .create(&node, &library) - .await - .map_err(rspc::Error::from)? - else { - return Ok(None); - }; - - let scan_state = ScanState::try_from(location.scan_state)?; - - scan_location(&node, &library, location, scan_state) - .await - .map_err(rspc::Error::from) - })) - } else { - None - } - }) - .collect::>() - .join() - .await - .into_iter() - .map(|spawn_res| { - spawn_res - .map_err(|_| { - rspc::Error::new( - ErrorCode::InternalServerError, - "A task to create a default location failed".to_string(), - ) - }) - .and_then(identity) - }) - .fold(&mut maybe_error, |maybe_error, res| { - if let Err(e) = res { - error!(?e, "Failed to create default location;"); - *maybe_error = Some(e); - } - maybe_error - }); - - if let Some(e) = maybe_error { - return Err(e); - } - - debug!("Created default locations"); - - Ok(None) - } - - R.mutation( - |node, - CreateLibraryArgs { - name, - default_locations, - }: CreateLibraryArgs| async move { - debug!("Creating library"); - - let library = node.libraries.create(name, None, &node).await?; - - debug!(%library.id, "Created library;"); - - if let Some(locations) = default_locations { - create_default_locations_on_library_creation( - locations, - node, - Arc::clone(&library), - ) - .await?; - } - - Ok(LibraryConfigWrapped::from_library(&library).await) - }, - ) - }) - .procedure("edit", { - #[derive(Type, Deserialize)] - pub struct EditLibraryArgs { - pub id: Uuid, - pub name: Option, - pub description: MaybeUndefined, - } - - R.mutation( - |node, - EditLibraryArgs { - id, - name, - description, - }: EditLibraryArgs| async move { - Ok(node - .libraries - .edit(id, name, description, MaybeUndefined::Undefined, None) - .await?) - }, - ) - }) - .procedure( - "delete", - R.mutation(|node, id: Uuid| async move { - node.libraries.delete(&id).await.map_err(Into::into) - }), - ) - .procedure( - "actors", - R.with2(library()).subscription(|(_, library), _: ()| { - let mut rx = library.cloud_sync_actors.invalidate_rx.resubscribe(); - - async_stream::stream! { - let actors = library.cloud_sync_actors.get_state().await; - yield actors; - - while let Ok(()) = rx.recv().await { - let actors = library.cloud_sync_actors.get_state().await; - yield actors; - } - } - }), - ) - .procedure( - "vacuumDb", - R.with2(library()) - .mutation(|(_, library), _: ()| async move { - // We retry a few times because if the DB is being actively used, the vacuum will fail - for _ in 0..5 { - match library.db._execute_raw(raw!("VACUUM;")).exec().await { - Ok(_) => break, - Err(e) => { - warn!( - %library.id, - ?e, - "Failed to vacuum DB for library, retrying...;", - ); - tokio::time::sleep(Duration::from_millis(500)).await; - } - } - } - - info!(%library.id, "Successfully vacuumed DB;"); - - Ok(()) - }), - ) -} - -async fn update_kind_statistics(node: Arc, library: Arc) -> Result<(), QueryError> { - async fn inner(node: Arc, library: Arc) -> Result<(), QueryError> { - // We're processing each kind in a sequentially manner, so we can update the frontend step by step - for kind in ObjectKind::iter() { - let int_kind = kind as i32; - let mut last_object_id = 0; - - let mut total_size = 0; - let mut files_count = 0; - - loop { - let objects = library - .db - .object() - .find_many(vec![ - object::id::gt(last_object_id), - object::kind::equals(Some(int_kind)), - ]) - .take(10_000) - .select(object::select!({ id file_paths: select { size_in_bytes_bytes } })) - .exec() - .await?; - - if let Some(last) = objects.last() { - last_object_id = last.id; - } else { - break; // No more objects - } - - for object in objects { - files_count += object.file_paths.len() as u64; - if kind != ObjectKind::Folder { - // We're skipping folders because their size is the sum of all their children and - // we don't want to count them twice - total_size += object - .file_paths - .into_iter() - .map(|file_path| { - file_path - .size_in_bytes_bytes - .map(|size| size_in_bytes_from_db(&size)) - .unwrap_or(0) - }) - .sum::(); - } - } - } - - let (old_files_count, old_total_bytes) = library - .db - .object_kind_statistics() - .find_unique(object_kind_statistics::kind::equals(int_kind)) - .select(object_kind_statistics::select!({ files_count total_bytes })) - .exec() - .await? - .map_or((0, 0), |stats| { - (stats.files_count as u64, stats.total_bytes as u64) - }); - - // Only update if the statistics changed - if files_count != old_files_count || total_size != old_total_bytes { - let update_params = { - #[allow(clippy::cast_possible_wrap)] - { - // SAFETY: we had to store using i64 due to SQLite limitations - vec![ - object_kind_statistics::files_count::set(files_count as i64), - object_kind_statistics::total_bytes::set(total_size as i64), - ] - } - }; - - library - .db - .object_kind_statistics() - .upsert( - object_kind_statistics::kind::equals(int_kind), - object_kind_statistics::Create { - kind: int_kind, - _params: update_params.clone(), - }, - update_params, - ) - .select(object_kind_statistics::select!({ kind })) - .exec() - .await?; - - // Sending back the updated statistics to the frontend - node.emit(CoreEvent::UpdatedKindStatistic( - KindStatistic { - kind: int_kind, - name: kind.to_string(), - count: u64_to_frontend(files_count), - total_bytes: u64_to_frontend(total_size), - }, - library.id, - )); - } - } - - Ok(()) - } - - let res = inner(node, library).await; - - // We finished and we can reset the flag - IS_COMPUTING_KIND_STATISTICS.store(false, Ordering::Release); - - res -} - -async fn update_statistics_loop( - node: Arc, - library: Arc, - last_requested_rx: chan::Receiver, -) { - let mut last_received_at = Instant::now(); - - let tick = interval(ONE_MINUTE); - - enum Message { - Tick, - Requested(Instant), - } - - let mut msg_stream = pin!(( - IntervalStream::new(tick).map(|_| Message::Tick), - last_requested_rx.map(Message::Requested) - ) - .merge()); - - while let Some(msg) = msg_stream.next().await { - match msg { - Message::Tick => { - if last_received_at.elapsed() < FIVE_MINUTES { - if let Err(e) = update_library_statistics(&node, &library).await { - error!(?e, "Failed to update library statistics;"); - } else { - invalidate_query!(&library, "library.statistics"); - } - } - } - Message::Requested(instant) => { - if instant - last_received_at > TWO_MINUTES { - debug!("Updating last received at"); - last_received_at = instant; - } - } - } - } -} diff --git a/core/src/api/locations.rs b/core/src/api/locations.rs deleted file mode 100644 index a3f4a55f5..000000000 --- a/core/src/api/locations.rs +++ /dev/null @@ -1,561 +0,0 @@ -use crate::{ - invalidate_query, - location::{ - delete_location, find_location, light_scan_location, non_indexed::NonIndexedPathItem, - relink_location, scan_location, scan_location_sub_path, LocationCreateArgs, LocationError, - LocationUpdateArgs, ScanState, - }, - old_p2p::PeerMetadata, - util::AbortOnDrop, -}; - -use sd_core_heavy_lifting::{media_processor::ThumbKey, JobName}; -use sd_core_indexer_rules::IndexerRuleCreateArgs; -use sd_core_prisma_helpers::{ - file_path_for_frontend, label_with_objects, location_with_indexer_rules, object_with_file_paths, -}; - -use sd_prisma::prisma::{file_path, indexer_rule, indexer_rules_in_location, location, SortOrder}; - -use std::path::{Path, PathBuf}; - -use chrono::{DateTime, FixedOffset, Utc}; -use directories::UserDirs; -use rspc::{alpha::AlphaRouter, ErrorCode}; -use serde::{Deserialize, Serialize}; -use specta::Type; -use tracing::{debug, error}; - -use super::{utils::library, Ctx, R}; - -#[derive(Serialize, Type, Debug)] -#[serde(tag = "type")] -pub enum ExplorerItem { - Path { - // provide the frontend with the thumbnail key explicitly - thumbnail: Option, - // this tells the frontend if a thumbnail actually exists or not - has_created_thumbnail: bool, - // we can't actually modify data from PCR types, thats why computed properties are used on ExplorerItem - item: Box, - }, - Object { - thumbnail: Option, - has_created_thumbnail: bool, - item: object_with_file_paths::Data, - }, - NonIndexedPath { - thumbnail: Option, - has_created_thumbnail: bool, - item: NonIndexedPathItem, - }, - Location { - item: location::Data, - }, - SpacedropPeer { - item: PeerMetadata, - }, - Label { - thumbnails: Vec, - item: label_with_objects::Data, - }, -} - -impl ExplorerItem { - pub fn id(&self) -> String { - let ty = match self { - ExplorerItem::Path { .. } => "FilePath", - ExplorerItem::Object { .. } => "Object", - ExplorerItem::Location { .. } => "Location", - ExplorerItem::NonIndexedPath { .. } => "NonIndexedPath", - ExplorerItem::SpacedropPeer { .. } => "SpacedropPeer", - ExplorerItem::Label { .. } => "Label", - }; - match self { - ExplorerItem::Path { item, .. } => format!("{ty}:{}", item.id), - ExplorerItem::Object { item, .. } => format!("{ty}:{}", item.id), - ExplorerItem::Location { item, .. } => format!("{ty}:{}", item.id), - ExplorerItem::NonIndexedPath { item, .. } => format!("{ty}:{}", item.path), - ExplorerItem::SpacedropPeer { item, .. } => format!("{ty}:{}", item.name), // TODO: Use a proper primary key - ExplorerItem::Label { item, .. } => format!("{ty}:{}", item.name), - } - } -} - -#[derive(Serialize, Type, Debug)] -pub struct SystemLocations { - desktop: Option, - documents: Option, - downloads: Option, - pictures: Option, - music: Option, - videos: Option, -} - -impl From for SystemLocations { - fn from(value: UserDirs) -> Self { - Self { - desktop: value.desktop_dir().map(Path::to_path_buf), - documents: value.document_dir().map(Path::to_path_buf), - downloads: value.download_dir().map(Path::to_path_buf), - pictures: value.picture_dir().map(Path::to_path_buf), - music: value.audio_dir().map(Path::to_path_buf), - videos: value.video_dir().map(Path::to_path_buf), - } - } -} - -impl ExplorerItem { - pub fn name(&self) -> &str { - match self { - ExplorerItem::Path { item, .. } => item.name.as_deref().unwrap_or(""), - ExplorerItem::Location { - item: location::Data { name, .. }, - .. - } => name.as_deref().unwrap_or(""), - ExplorerItem::NonIndexedPath { item, .. } => item.name.as_str(), - _ => "", - } - } - - pub fn size_in_bytes(&self) -> u64 { - match self { - ExplorerItem::Path { item, .. } => item - .size_in_bytes_bytes - .as_ref() - .map(|size| { - u64::from_be_bytes([ - size[0], size[1], size[2], size[3], size[4], size[5], size[6], size[7], - ]) - }) - .unwrap_or(0), - - ExplorerItem::NonIndexedPath { - item: NonIndexedPathItem { - size_in_bytes_bytes, - .. - }, - .. - } => u64::from_be_bytes([ - size_in_bytes_bytes[0], - size_in_bytes_bytes[1], - size_in_bytes_bytes[2], - size_in_bytes_bytes[3], - size_in_bytes_bytes[4], - size_in_bytes_bytes[5], - size_in_bytes_bytes[6], - size_in_bytes_bytes[7], - ]), - _ => 0, - } - } - - pub fn date_created(&self) -> DateTime { - match self { - ExplorerItem::Path { item, .. } => { - item.date_created.map(Into::into).unwrap_or_default() - } - ExplorerItem::Object { - item: object_with_file_paths::Data { date_created, .. }, - .. - } - | ExplorerItem::Location { - item: location::Data { date_created, .. }, - .. - } => date_created.map(Into::into).unwrap_or_default(), - - ExplorerItem::NonIndexedPath { item, .. } => item.date_created, - _ => Default::default(), - } - } - - pub fn date_modified(&self) -> DateTime { - match self { - ExplorerItem::Path { item, .. } => { - item.date_modified.map(Into::into).unwrap_or_default() - } - ExplorerItem::NonIndexedPath { item, .. } => item.date_modified, - _ => Default::default(), - } - } -} - -pub(crate) fn mount() -> AlphaRouter { - R.router() - .procedure("list", { - R.with2(library()).query(|(_, library), _: ()| async move { - Ok(library - .db - .location() - .find_many(vec![]) - .order_by(location::date_created::order(SortOrder::Desc)) - .exec() - .await?) - }) - }) - .procedure("get", { - R.with2(library()) - .query(|(_, library), location_id: location::id::Type| async move { - Ok(library - .db - .location() - .find_unique(location::id::equals(location_id)) - .exec() - .await?) - }) - }) - .procedure("getWithRules", { - #[derive(Type, Serialize)] - struct LocationWithIndexerRule { - pub id: i32, - pub pub_id: Vec, - pub name: Option, - pub path: Option, - pub total_capacity: Option, - pub available_capacity: Option, - pub size_in_bytes: Option>, - pub is_archived: Option, - pub generate_preview_media: Option, - pub sync_preview_media: Option, - pub hidden: Option, - pub date_created: Option>, - pub instance_id: Option, - pub indexer_rules: Vec, - } - - impl LocationWithIndexerRule { - pub fn from_db(value: location_with_indexer_rules::Data) -> Self { - Self { - id: value.id, - pub_id: value.pub_id, - name: value.name, - path: value.path, - total_capacity: value.total_capacity, - available_capacity: value.available_capacity, - size_in_bytes: value.size_in_bytes, - is_archived: value.is_archived, - generate_preview_media: value.generate_preview_media, - sync_preview_media: value.sync_preview_media, - hidden: value.hidden, - date_created: value.date_created, - instance_id: value.instance_id, - indexer_rules: value - .indexer_rules - .into_iter() - .map(|i| i.indexer_rule) - .collect::>(), - } - } - } - - R.with2(library()) - .query(|(_, library), location_id: location::id::Type| async move { - Ok(library - .db - .location() - .find_unique(location::id::equals(location_id)) - .include(location_with_indexer_rules::include()) - .exec() - .await? - .map(LocationWithIndexerRule::from_db)) - }) - }) - .procedure("create", { - R.with2(library()) - .mutation(|(node, library), args: LocationCreateArgs| async move { - if let Some(location) = args.create(&node, &library).await? { - let id = Some(location.id); - scan_location(&node, &library, location, ScanState::Pending).await?; - invalidate_query!(library, "locations.list"); - Ok(id) - } else { - Ok(None) - } - }) - }) - .procedure("update", { - R.with2(library()) - .mutation(|(node, library), args: LocationUpdateArgs| async move { - let ret = args.update(&node, &library).await.map_err(Into::into); - invalidate_query!(library, "locations.list"); - ret - }) - }) - .procedure("delete", { - R.with2(library()).mutation( - |(node, library), location_id: location::id::Type| async move { - delete_location(&node, &library, location_id).await?; - invalidate_query!(library, "locations.list"); - Ok(()) - }, - ) - }) - .procedure("relink", { - R.with2(library()) - .mutation(|(_, library), location_path: PathBuf| async move { - relink_location(&library, location_path) - .await - .map_err(Into::into) - }) - }) - .procedure("addLibrary", { - R.with2(library()) - .mutation(|(node, library), args: LocationCreateArgs| async move { - if let Some(location) = args.add_library(&node, &library).await? { - let id = location.id; - let location_scan_state = ScanState::try_from(location.scan_state)?; - scan_location(&node, &library, location, location_scan_state).await?; - invalidate_query!(library, "locations.list"); - Ok(Some(id)) - } else { - Ok(None) - } - }) - }) - .procedure("fullRescan", { - #[derive(Type, Deserialize)] - pub struct FullRescanArgs { - pub location_id: location::id::Type, - pub reidentify_objects: bool, - } - R.with2(library()).mutation( - |(node, library), - FullRescanArgs { - location_id, - reidentify_objects, - }| async move { - if reidentify_objects { - let count = library - .db - .file_path() - .update_many( - vec![ - file_path::location_id::equals(Some(location_id)), - file_path::object_id::not(None), - file_path::cas_id::not(None), - ], - vec![ - file_path::object::disconnect(), - file_path::cas_id::set(None), - ], - ) - .exec() - .await?; - - debug!(%count, "Disconnected file paths from objects;"); - - // library.orphan_remover.invoke().await; - } - - let location = find_location(&library, location_id) - .include(location_with_indexer_rules::include()) - .exec() - .await? - .ok_or(LocationError::IdNotFound(location_id))?; - - let location_scan_state = ScanState::try_from(location.scan_state)?; - - // rescan location - scan_location(&node, &library, location, location_scan_state) - .await - .map_err(Into::into) - }, - ) - }) - .procedure("subPathRescan", { - #[derive(Clone, Serialize, Deserialize, Type, Debug)] - pub struct RescanArgs { - pub location_id: location::id::Type, - pub sub_path: String, - } - - R.with2(library()).mutation( - |(node, library), - RescanArgs { - location_id, - sub_path, - }: RescanArgs| async move { - scan_location_sub_path( - &node, - &library, - find_location(&library, location_id) - .include(location_with_indexer_rules::include()) - .exec() - .await? - .ok_or(LocationError::IdNotFound(location_id))?, - sub_path, - ) - .await - .map_err(Into::into) - }, - ) - }) - .procedure("quickRescan", { - #[derive(Clone, Serialize, Deserialize, Type, Debug)] - pub struct LightScanArgs { - pub location_id: location::id::Type, - pub sub_path: String, - } - - R.with2(library()).subscription( - |(node, library), - LightScanArgs { - location_id, - sub_path, - }: LightScanArgs| async move { - if node - .job_system - .check_running_jobs( - vec![ - JobName::Indexer, - JobName::FileIdentifier, - JobName::MediaProcessor, - ], - location_id, - ) - .await - { - return Err(rspc::Error::new( - ErrorCode::Conflict, - "We're still indexing this location, pleases wait a bit...".to_string(), - )); - } - - let location = find_location(&library, location_id) - .include(location_with_indexer_rules::include()) - .exec() - .await? - .ok_or(LocationError::IdNotFound(location_id))?; - - let handle = tokio::spawn(async move { - if let Err(e) = light_scan_location(node, library, location, sub_path).await - { - error!(?e, "Light scan error;"); - } - }); - - Ok(AbortOnDrop(handle)) - }, - ) - }) - .procedure( - "online", - R.subscription(|node, _: ()| async move { - let mut rx = node.locations.online_rx(); - - async_stream::stream! { - let online = node.locations.get_online().await; - - yield online; - - while let Ok(locations) = rx.recv().await { - yield locations; - } - } - }), - ) - .procedure("systemLocations", { - R.query(|_, _: ()| async move { - UserDirs::new().map(SystemLocations::from).ok_or_else(|| { - rspc::Error::new( - ErrorCode::NotFound, - "Didn't find any system locations".to_string(), - ) - }) - }) - }) - .merge("indexer_rules.", mount_indexer_rule_routes()) -} - -fn mount_indexer_rule_routes() -> AlphaRouter { - R.router() - .procedure("create", { - R.with2(library()) - .mutation(|(_, library), args: IndexerRuleCreateArgs| async move { - if args.create(&library.db).await?.is_some() { - invalidate_query!(library, "locations.indexer_rules.list"); - } - - Ok(()) - }) - }) - .procedure("delete", { - R.with2(library()) - .mutation(|(_, library), indexer_rule_id: i32| async move { - let indexer_rule_db = library.db.indexer_rule(); - - if let Some(indexer_rule) = indexer_rule_db - .to_owned() - .find_unique(indexer_rule::id::equals(indexer_rule_id)) - .exec() - .await? - { - if indexer_rule.default.unwrap_or_default() { - return Err(rspc::Error::new( - ErrorCode::Forbidden, - format!("Indexer rule can't be deleted"), - )); - } - } else { - return Err(rspc::Error::new( - ErrorCode::NotFound, - format!("Indexer rule not found"), - )); - } - - library - .db - .indexer_rules_in_location() - .delete_many(vec![indexer_rules_in_location::indexer_rule_id::equals( - indexer_rule_id, - )]) - .exec() - .await?; - - indexer_rule_db - .delete(indexer_rule::id::equals(indexer_rule_id)) - .exec() - .await?; - - invalidate_query!(library, "locations.indexer_rules.list"); - - Ok(()) - }) - }) - .procedure("get", { - R.with2(library()) - .query(|(_, library), indexer_rule_id: i32| async move { - library - .db - .indexer_rule() - .find_unique(indexer_rule::id::equals(indexer_rule_id)) - .exec() - .await? - .ok_or_else(|| { - rspc::Error::new( - ErrorCode::NotFound, - format!("Indexer rule not found"), - ) - }) - }) - }) - .procedure("list", { - R.with2(library()).query(|(_, library), _: ()| async move { - Ok(library.db.indexer_rule().find_many(vec![]).exec().await?) - }) - }) - // list indexer rules for location, returning the indexer rule - .procedure("listForLocation", { - R.with2(library()) - .query(|(_, library), location_id: location::id::Type| async move { - Ok(library - .db - .indexer_rule() - .find_many(vec![indexer_rule::locations::some(vec![ - indexer_rules_in_location::location_id::equals(location_id), - ])]) - .exec() - .await?) - }) - }) -} diff --git a/core/src/api/mod.rs b/core/src/api/mod.rs deleted file mode 100644 index 186cff645..000000000 --- a/core/src/api/mod.rs +++ /dev/null @@ -1,258 +0,0 @@ -use crate::{ - invalidate_query, - library::LibraryId, - node::{ - config::{is_in_docker, NodeConfig, NodeConfigP2P, NodePreferences}, - HardwareModel, - }, - old_job::JobProgressEvent, - Node, -}; - -use sd_core_heavy_lifting::media_processor::ThumbKey; -use sd_core_sync::DevicePubId; - -use sd_cloud_schema::devices::DeviceOS; -use sd_old_p2p::RemoteIdentity; -use sd_prisma::prisma::file_path; - -use std::sync::Arc; - -use itertools::Itertools; -use rspc::{alpha::Rspc, Config, ErrorCode}; -use serde::{Deserialize, Serialize}; -use specta::Type; -use tracing::warn; - -mod backups; -mod cloud; -mod devices; -mod ephemeral_files; -mod files; -mod jobs; -mod keys; -mod labels; -mod libraries; -pub mod locations; -mod models; -mod nodes; -pub mod notifications; -mod p2p; -mod preferences; -pub(crate) mod search; -mod sync; -mod tags; -pub mod utils; -pub mod volumes; -mod web_api; - -use libraries::KindStatistic; -use utils::{InvalidRequests, InvalidateOperationEvent}; - -#[allow(non_upper_case_globals)] -pub(crate) const R: Rspc = Rspc::new(); - -pub type Ctx = Arc; -pub type Router = rspc::Router; - -/// Represents an internal core event, these are exposed to client via a rspc subscription. -#[derive(Debug, Clone, Serialize, Type)] -pub enum CoreEvent { - NewThumbnail { - thumb_key: ThumbKey, - }, - NewIdentifiedObjects { - file_path_ids: Vec, - }, - UpdatedKindStatistic(KindStatistic, LibraryId), - JobProgress(JobProgressEvent), - InvalidateOperation(InvalidateOperationEvent), -} - -/// All of the feature flags provided by the core itself. The frontend has it's own set of feature flags! -/// -/// If you want a variant of this to show up on the frontend it must be added to `backendFeatures` in `useFeatureFlag.tsx` -#[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize, Type)] -#[serde(rename_all = "camelCase")] -pub enum BackendFeature {} - -// impl BackendFeature { -// pub fn restore(&self, node: &Node) { -// match self { -// BackendFeature::CloudSync => { -// node.cloud_sync_flag.store(true, Ordering::Relaxed); -// } -// } -// } -// } - -/// A version of [`NodeConfig`] that is safe to share with the frontend -#[derive(Debug, Serialize, Deserialize, Clone, Type)] -pub struct SanitizedNodeConfig { - /// id is a unique identifier for the current node. Each node has a public identifier (this one) and is given a local id for each library (done within the library code). - pub id: DevicePubId, - /// name is the display name of the current node. This is set by the user and is shown in the UI. // TODO: Length validation so it can fit in DNS record - pub name: String, - pub identity: RemoteIdentity, - pub p2p: NodeConfigP2P, - pub features: Vec, - pub preferences: NodePreferences, - pub os: DeviceOS, - pub hardware_model: HardwareModel, -} - -impl From for SanitizedNodeConfig { - fn from(value: NodeConfig) -> Self { - Self { - id: value.id, - name: value.name, - identity: value.identity.to_remote_identity(), - p2p: value.p2p, - features: value.features, - preferences: value.preferences, - os: value.os, - hardware_model: value.hardware_model, - } - } -} - -#[derive(Serialize, Debug, Type)] -struct NodeState { - #[serde(flatten)] - config: SanitizedNodeConfig, - data_path: String, - device_model: Option, - is_in_docker: bool, -} - -pub(crate) fn mount() -> Arc { - let r = R - .router() - .procedure("buildInfo", { - #[derive(Serialize, Type)] - pub struct BuildInfo { - version: &'static str, - commit: &'static str, - } - - R.query(|_, _: ()| { - Ok(BuildInfo { - version: env!("CARGO_PKG_VERSION"), - commit: env!("GIT_HASH"), - }) - }) - }) - .procedure("nodeState", { - R.query(|node, _: ()| async move { - let config = SanitizedNodeConfig::from(node.config.get().await); - - Ok(NodeState { - device_model: Some(config.hardware_model.to_string()), - config, - // We are taking the assumption here that this value is only used on the frontend for display purposes - data_path: node - .config - .data_directory() - .to_str() - .expect("Found non-UTF-8 path") - .to_string(), - is_in_docker: is_in_docker(), - }) - }) - }) - .procedure("toggleFeatureFlag", { - R.mutation(|node, feature: BackendFeature| async move { - let config = node.config.get().await; - - let enabled = if config.features.iter().contains(&feature) { - node.config - .write(|cfg| { - cfg.features.retain(|f| *f != feature); - }) - .await - .map(|_| false) - } else { - node.config - .write(|cfg| { - cfg.features.push(feature.clone()); - }) - .await - .map(|_| true) - } - .map_err(|e| rspc::Error::new(ErrorCode::InternalServerError, e.to_string()))?; - - warn!("Feature {:?} is now {}", feature, enabled); - - // match feature { - // BackendFeature::CloudSync => { - // node.cloud_sync_flag.store(enabled, Ordering::Relaxed); - // } - // } - - invalidate_query!(node; node, "nodeState"); - - Ok(()) - }) - }) - .merge("api.", web_api::mount()) - .merge("cloud.", cloud::mount()) - .merge("devices.", devices::mount()) - .merge("search.", search::mount()) - .merge("library.", libraries::mount()) - .merge("volumes.", volumes::mount()) - .merge("tags.", tags::mount()) - .merge("labels.", labels::mount()) - .merge("locations.", locations::mount()) - .merge("ephemeralFiles.", ephemeral_files::mount()) - .merge("files.", files::mount()) - .merge("jobs.", jobs::mount()) - .merge("p2p.", p2p::mount()) - .merge("models.", models::mount()) - .merge("nodes.", nodes::mount()) - .merge("sync.", sync::mount()) - .merge("preferences.", preferences::mount()) - .merge("notifications.", notifications::mount()) - .merge("backups.", backups::mount()) - .merge("keys.", keys::mount()) - .merge("invalidation.", utils::mount_invalidate()) - .sd_patch_types_dangerously(|type_map| { - let def = - ::definition_named_data_type( - type_map, - ); - type_map.insert( - ::sid(), - def, - ); - }); - - let r = r - .build( - #[allow(clippy::let_and_return)] - { - let config = Config::new().set_ts_bindings_header("/* eslint-disable */"); - - #[cfg(all(debug_assertions, not(feature = "mobile")))] - let config = config.export_ts_bindings( - std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("../packages/client/src/core.ts"), - ); - - config - }, - ) - .arced(); - - InvalidRequests::validate(r.clone()); // This validates all invalidation calls. - - r -} - -#[cfg(test)] -mod tests { - /// This test will ensure the rspc router and all calls to `invalidate_query` are valid and also export an updated version of the Typescript bindings. - #[test] - fn test_and_export_rspc_bindings() { - super::mount(); - } -} diff --git a/core/src/api/models.rs b/core/src/api/models.rs deleted file mode 100644 index b0e052872..000000000 --- a/core/src/api/models.rs +++ /dev/null @@ -1,23 +0,0 @@ -use rspc::alpha::AlphaRouter; - -use super::{Ctx, R}; - -pub(crate) fn mount() -> AlphaRouter { - R.router().procedure("image_detection.list", { - R.query( - |_, _: ()| -> std::result::Result, rspc::Error> { - #[cfg(not(feature = "ai"))] - return Err(rspc::Error::new( - rspc::ErrorCode::MethodNotSupported, - "AI feature is not available".to_string(), - )); - - #[cfg(feature = "ai")] - { - use sd_ai::old_image_labeler::{Model, YoloV8}; - Ok(YoloV8::versions()) - } - }, - ) - }) -} diff --git a/core/src/api/nodes.rs b/core/src/api/nodes.rs deleted file mode 100644 index 9083d4e70..000000000 --- a/core/src/api/nodes.rs +++ /dev/null @@ -1,135 +0,0 @@ -use std::collections::HashSet; - -use crate::{ - invalidate_query, - node::config::{P2PDiscoveryState, Port}, -}; - -use sd_prisma::prisma::{device, location}; - -use rspc::{alpha::AlphaRouter, ErrorCode}; -use sd_utils::uuid_to_bytes; -use serde::Deserialize; -use specta::Type; -use tracing::error; -use uuid::Uuid; - -use super::{locations::ExplorerItem, utils::library, Ctx, R}; - -pub(crate) fn mount() -> AlphaRouter { - R.router() - .procedure("edit", { - #[derive(Deserialize, Type)] - pub struct ChangeNodeNameArgs { - pub name: Option, - pub p2p_port: Option, - pub p2p_disabled: Option, - pub p2p_ipv6_disabled: Option, - pub p2p_relay_disabled: Option, - pub p2p_discovery: Option, - pub p2p_remote_access: Option, - pub p2p_manual_peers: Option>, - } - R.mutation(|node, args: ChangeNodeNameArgs| async move { - if let Some(name) = &args.name { - if name.is_empty() || name.len() > 250 { - return Err(rspc::Error::new( - ErrorCode::BadRequest, - "invalid node name".into(), - )); - } - } - - node.config - .write(|config| { - if let Some(name) = args.name { - config.name = name; - } - - if let Some(port) = args.p2p_port { - config.p2p.port = port; - }; - if let Some(enabled) = args.p2p_disabled { - config.p2p.disabled = enabled; - }; - if let Some(enabled) = args.p2p_ipv6_disabled { - config.p2p.disable_ipv6 = enabled; - }; - if let Some(enabled) = args.p2p_relay_disabled { - config.p2p.disable_relay = enabled; - }; - if let Some(discovery) = args.p2p_discovery { - config.p2p.discovery = discovery; - }; - if let Some(remote_access) = args.p2p_remote_access { - config.p2p.enable_remote_access = remote_access; - }; - if let Some(manual_peers) = args.p2p_manual_peers { - config.p2p.manual_peers = manual_peers; - }; - }) - .await - .map_err(|e| { - error!(?e, "Failed to write config;"); - rspc::Error::new( - ErrorCode::InternalServerError, - "error updating config".into(), - ) - })?; - - // This is a no-op if the config didn't change - node.p2p.on_node_config_change().await; - - invalidate_query!(node; node, "nodeState"); - - Ok(()) - }) - }) - // TODO: add pagination!! and maybe ordering etc - .procedure("listLocations", { - R.with2(library()) - // TODO: I don't like this. `node_id` should probs be a machine hash or something cause `node_id` is dynamic in the context of P2P and what does it mean for removable media to be owned by a node? - .query(|(_, library), device_pub_id: Option| async move { - Ok(library - .db - .location() - .find_many( - device_pub_id - .map(|id| { - vec![location::device::is(vec![device::pub_id::equals( - uuid_to_bytes(&id), - )])] - }) - .unwrap_or_default(), - ) - .exec() - .await? - .into_iter() - .map(|location| ExplorerItem::Location { item: location }) - .collect::>()) - }) - }) - .procedure("updateThumbnailerPreferences", { - #[derive(Deserialize, Type)] - pub struct UpdateThumbnailerPreferences { - // pub background_processing_percentage: u8, // 0-100 - } - R.mutation( - |node, UpdateThumbnailerPreferences { .. }: UpdateThumbnailerPreferences| async move { - node.config - .update_preferences(|_| { - // TODO(fogodev): introduce configurable workers count to task system - }) - .await - .map_err(|e| { - error!(?e, "Failed to update thumbnailer preferences;"); - rspc::Error::with_cause( - ErrorCode::InternalServerError, - "Failed to update thumbnailer preferences".to_string(), - e, - ) - }) - }, - ) - }) -} diff --git a/core/src/api/notifications.rs b/core/src/api/notifications.rs deleted file mode 100644 index ff8b7b076..000000000 --- a/core/src/api/notifications.rs +++ /dev/null @@ -1,167 +0,0 @@ -use sd_prisma::prisma::notification; - -use crate::api::{Ctx, R}; -use async_stream::stream; -use chrono::{DateTime, Utc}; -use futures::future::join_all; -use rspc::{alpha::AlphaRouter, ErrorCode}; -use serde::{Deserialize, Serialize}; -use specta::Type; -use uuid::Uuid; - -/// Represents a single notification. -#[derive(Debug, Clone, Serialize, Deserialize, Type)] -pub struct Notification { - #[serde(flatten)] - pub id: NotificationId, - pub data: NotificationData, - pub read: bool, - pub expires: Option>, -} - -#[derive(Debug, Clone, Serialize, Deserialize, Type, PartialEq, Eq)] -#[serde(tag = "type", content = "id", rename_all = "camelCase")] -pub enum NotificationId { - Library(Uuid, u32), - Node(u32), -} -#[derive(Debug, Clone, Serialize, Deserialize, Type)] -#[serde(rename_all = "camelCase")] -pub enum NotificationKind { - Info, - Success, - Error, - Warning, -} - -/// Represents the data of a single notification. -/// This data is used by the frontend to properly display the notification. -#[derive(Debug, Clone, Serialize, Deserialize, Type)] -pub struct NotificationData { - pub title: String, - pub content: String, - pub kind: NotificationKind, -} - -pub(crate) fn mount() -> AlphaRouter { - R.router() - .procedure("get", { - R.query(|node, _: ()| async move { - let mut notifications = node.config.get().await.notifications; - for lib_notifications in join_all(node.libraries.get_all().await.into_iter().map( - |library| async move { - library - .db - .notification() - .find_many(vec![]) - .exec() - .await - .map_err(|e| { - rspc::Error::new( - ErrorCode::InternalServerError, - format!( - "Failed to get notifications for library '{}': {}", - library.id, e - ), - ) - })? - .into_iter() - .map(|n| { - Ok(Notification { - id: NotificationId::Library(library.id, n.id as u32), - data: rmp_serde::from_slice(&n.data).map_err(|e| { - rspc::Error::new( - ErrorCode::InternalServerError, - format!( - "Failed to get notifications for library '{}': {}", - library.id, e - ), - ) - })?, - read: false, - expires: n.expires_at.map(Into::into), - }) - }) - .collect::, rspc::Error>>() - }, - )) - .await - { - notifications.extend(lib_notifications?); - } - - Ok(notifications) - }) - }) - .procedure("dismiss", { - R.query(|node, id: NotificationId| async move { - match id { - NotificationId::Library(library_id, id) => { - node.libraries - .get_library(&library_id) - .await - .ok_or_else(|| { - rspc::Error::new(ErrorCode::NotFound, "Library not found".into()) - })? - .db - .notification() - .delete_many(vec![notification::id::equals(id as i32)]) - .exec() - .await - .map_err(|e| { - rspc::Error::new(ErrorCode::InternalServerError, e.to_string()) - })?; - } - NotificationId::Node(id) => { - node.config - .write(|cfg| { - cfg.notifications - .retain(|n| n.id != NotificationId::Node(id)); - }) - .await - .map_err(|e| { - rspc::Error::new(ErrorCode::InternalServerError, e.to_string()) - })?; - } - } - - Ok(()) - }) - }) - .procedure("dismissAll", { - R.query(|node, _: ()| async move { - node.config - .write(|cfg| { - cfg.notifications = vec![]; - }) - .await - .map_err(|e| rspc::Error::new(ErrorCode::InternalServerError, e.to_string()))?; - - join_all( - node.libraries - .get_all() - .await - .into_iter() - .map(|library| async move { - library.db.notification().delete_many(vec![]).exec().await - }), - ) - .await - .into_iter() - .collect::, _>>()?; - - Ok(()) - }) - }) - .procedure("listen", { - R.subscription(|node, _: ()| async move { - let mut sub = node.notifications.subscribe(); - - stream! { - while let Ok(notification) = sub.recv().await { - yield notification; - } - } - }) - }) -} diff --git a/core/src/api/p2p.rs b/core/src/api/p2p.rs deleted file mode 100644 index dd16ba0ac..000000000 --- a/core/src/api/p2p.rs +++ /dev/null @@ -1,154 +0,0 @@ -//! DEPRICATED FOR NEW SYSTEM. DO NOT USE THIS API -use crate::old_p2p::{ - operations, ConnectionMethod, DiscoveryMethod, Header, P2PEvent, PeerMetadata, -}; - -use sd_old_p2p::{PeerConnectionCandidate, RemoteIdentity}; - -use rspc::{alpha::AlphaRouter, ErrorCode}; -use serde::Deserialize; -use specta::Type; -use std::{path::PathBuf, sync::PoisonError}; -use tokio::io::AsyncWriteExt; -use uuid::Uuid; - -use super::{Ctx, R}; - -pub(crate) fn mount() -> AlphaRouter { - R.router() - .procedure("events", { - R.subscription(|node, _: ()| async move { - let mut rx = node.p2p.events.subscribe(); - - let mut queued = Vec::new(); - - for (_, peer, metadata) in node.p2p.p2p.peers().iter().filter_map(|(i, p)| { - PeerMetadata::from_hashmap(&p.metadata()) - .ok() - .map(|m| (i, p, m)) - }) { - queued.push(P2PEvent::PeerChange { - identity: peer.identity(), - connection: if peer.is_connected() { - if node.p2p.quic.is_relayed(peer.identity()) { - ConnectionMethod::Relay - } else { - ConnectionMethod::Local - } - } else { - ConnectionMethod::Disconnected - }, - discovery: if peer - .connection_candidates() - .iter() - .any(|c| matches!(c, PeerConnectionCandidate::Manual(_))) - { - DiscoveryMethod::Manual - } else if peer - .connection_candidates() - .iter() - .all(|c| *c == PeerConnectionCandidate::Relay) - { - DiscoveryMethod::Relay - } else { - DiscoveryMethod::Local - }, - metadata, - addrs: peer.addrs(), - }); - } - - Ok(async_stream::stream! { - for event in queued.drain(..queued.len()) { - yield event; - } - - while let Ok(event) = rx.recv().await { - yield event; - } - }) - }) - }) - .procedure("state", { - R.query(|node, _: ()| async move { Ok(node.p2p.state().await) }) - }) - .procedure("listeners", { - R.query(|node, _: ()| async move { - Ok(node - .p2p - .listeners - .lock() - .unwrap_or_else(PoisonError::into_inner) - .clone()) - }) - }) - .procedure("debugConnect", { - R.mutation(|node, identity: RemoteIdentity| async move { - let peer = { node.p2p.p2p.peers().get(&identity).cloned() }; - let mut stream = peer - .ok_or(rspc::Error::new( - ErrorCode::InternalServerError, - "big man, not found".into(), - ))? - .new_stream() - .await - .map_err(|e| { - rspc::Error::new( - ErrorCode::InternalServerError, - format!("error in peer.new_stream: {:?}", e), - ) - })?; - - stream - .write_all(&Header::Ping.to_bytes()) - .await - .map_err(|e| { - rspc::Error::new( - ErrorCode::InternalServerError, - format!("error sending ping header: {:?}", e), - ) - })?; - - Ok("connected") - }) - }) - .procedure("spacedrop", { - #[derive(Type, Deserialize)] - pub struct SpacedropArgs { - identity: RemoteIdentity, - file_path: Vec, - } - - R.mutation(|node, args: SpacedropArgs| async move { - operations::spacedrop( - node.p2p.clone(), - args.identity, - args.file_path - .into_iter() - .map(PathBuf::from) - .collect::>(), - ) - .await - .map_err(|spacedrop_err| { - rspc::Error::new(ErrorCode::InternalServerError, spacedrop_err.to_string()) - }) - }) - }) - .procedure("acceptSpacedrop", { - R.mutation(|node, (id, path): (Uuid, Option)| async move { - match path { - Some(path) => node.p2p.accept_spacedrop(id, path).await, - None => node.p2p.reject_spacedrop(id).await, - }; - - Ok(()) - }) - }) - .procedure("cancelSpacedrop", { - R.mutation(|node, id: Uuid| async move { - node.p2p.cancel_spacedrop(id).await; - - Ok(()) - }) - }) -} diff --git a/core/src/api/preferences.rs b/core/src/api/preferences.rs deleted file mode 100644 index 2722fecf3..000000000 --- a/core/src/api/preferences.rs +++ /dev/null @@ -1,22 +0,0 @@ -use crate::preferences::LibraryPreferences; - -use rspc::alpha::AlphaRouter; - -use super::{utils::library, Ctx, R}; - -pub(crate) fn mount() -> AlphaRouter { - R.router() - .procedure("update", { - R.with2(library()) - .mutation(|(_, library), args: LibraryPreferences| async move { - args.write(&library.db).await?; - - Ok(()) - }) - }) - .procedure("get", { - R.with2(library()).query(|(_, library), _: ()| async move { - Ok(LibraryPreferences::read(&library.db).await?) - }) - }) -} diff --git a/core/src/api/search/exif_data.rs b/core/src/api/search/exif_data.rs deleted file mode 100644 index 61b8f6cd8..000000000 --- a/core/src/api/search/exif_data.rs +++ /dev/null @@ -1,29 +0,0 @@ -use sd_prisma::prisma::{self, exif_data}; - -use serde::{Deserialize, Serialize}; -use specta::Type; - -use super::utils::*; - -#[derive(Serialize, Deserialize, Type, Debug, Clone)] -#[serde(rename_all = "camelCase", tag = "field", content = "value")] -pub enum ExifDataOrder { - EpochTime(SortOrder), -} - -impl ExifDataOrder { - pub fn get_sort_order(&self) -> prisma::SortOrder { - (*match self { - Self::EpochTime(v) => v, - }) - .into() - } - - pub fn into_param(self) -> exif_data::OrderByWithRelationParam { - let dir = self.get_sort_order(); - use exif_data::*; - match self { - Self::EpochTime(_) => epoch_time::order(dir), - } - } -} diff --git a/core/src/api/search/file_path.rs b/core/src/api/search/file_path.rs deleted file mode 100644 index 7e7a4f29f..000000000 --- a/core/src/api/search/file_path.rs +++ /dev/null @@ -1,296 +0,0 @@ -use crate::location::LocationError; - -use sd_core_file_path_helper::{check_file_path_exists, IsolatedFilePathData}; - -use sd_prisma::prisma::{self, file_path}; - -use chrono::{DateTime, FixedOffset, Utc}; -use prisma_client_rust::{OrderByQuery, PaginatedQuery, WhereQuery}; -use rspc::ErrorCode; -use serde::{Deserialize, Serialize}; -use specta::Type; - -use super::{ - object::*, - utils::{self, *}, -}; - -#[derive(Serialize, Deserialize, Type, Debug, Clone)] -#[serde(rename_all = "camelCase", tag = "field", content = "value")] -pub enum FilePathOrder { - Name(SortOrder), - SizeInBytes(SortOrder), - DateCreated(SortOrder), - DateModified(SortOrder), - DateIndexed(SortOrder), - Object(Box), -} - -impl FilePathOrder { - pub fn get_sort_order(&self) -> prisma::SortOrder { - (*match self { - Self::Name(v) => v, - Self::SizeInBytes(v) => v, - Self::DateCreated(v) => v, - Self::DateModified(v) => v, - Self::DateIndexed(v) => v, - Self::Object(v) => return v.get_sort_order(), - }) - .into() - } - - pub fn into_param(self) -> file_path::OrderByWithRelationParam { - let dir = self.get_sort_order(); - use file_path::*; - match self { - Self::Name(_) => name::order(dir), - Self::SizeInBytes(_) => size_in_bytes_bytes::order(dir), - Self::DateCreated(_) => date_created::order(dir), - Self::DateModified(_) => date_modified::order(dir), - Self::DateIndexed(_) => date_indexed::order(dir), - Self::Object(v) => object::order(vec![v.into_param()]), - } - } -} - -#[derive(Serialize, Deserialize, Type, Debug, Clone)] -#[serde(rename_all = "camelCase")] -pub enum FilePathFilterArgs { - Locations(InOrNotIn), - Path { - location_id: prisma::location::id::Type, - path: String, - include_descendants: bool, - }, - // #[deprecated] - // Search(String), - Name(TextMatch), - Extension(InOrNotIn), - CreatedAt(Range>), - ModifiedAt(Range>), - IndexedAt(Range>), - Hidden(bool), -} - -impl FilePathFilterArgs { - pub async fn into_params( - self, - db: &prisma::PrismaClient, - ) -> Result, rspc::Error> { - use file_path::*; - - Ok(match self { - Self::Locations(v) => v - .into_param( - file_path::location_id::in_vec, - file_path::location_id::not_in_vec, - ) - .map(|v| vec![v]) - .unwrap_or_default(), - Self::Path { - location_id, - path, - include_descendants, - } => { - let directory_materialized_path_str = if !path.is_empty() && path != "/" { - let parent_iso_file_path = - IsolatedFilePathData::from_relative_str(location_id, &path); - - if !check_file_path_exists::(&parent_iso_file_path, db).await? { - return Err(rspc::Error::new( - ErrorCode::NotFound, - "Directory not found".into(), - )); - } - - parent_iso_file_path.materialized_path_for_children() - } else { - Some("/".into()) - }; - - directory_materialized_path_str - .map(Some) - .map(|materialized_path| { - vec![if include_descendants { - materialized_path::starts_with( - materialized_path.unwrap_or_else(|| "/".into()), - ) - } else { - materialized_path::equals(materialized_path) - }] - }) - .unwrap_or_default() - } - Self::Name(v) => v - .into_param(name::contains, name::starts_with, name::ends_with, |s| { - name::equals(Some(s)) - }) - .map(|v| vec![v]) - .unwrap_or_default(), - Self::Extension(v) => v - .into_param(extension::in_vec, extension::not_in_vec) - .map(|v| vec![v]) - .unwrap_or_default(), - Self::CreatedAt(v) => { - vec![match v { - Range::From(v) => date_created::gte(v.into()), - Range::To(v) => date_created::lte(v.into()), - }] - } - Self::ModifiedAt(v) => { - vec![match v { - Range::From(v) => date_modified::gte(v.into()), - Range::To(v) => date_modified::lte(v.into()), - }] - } - Self::IndexedAt(v) => { - vec![match v { - Range::From(v) => date_indexed::gte(v.into()), - Range::To(v) => date_indexed::lte(v.into()), - }] - } - Self::Hidden(v) => { - vec![hidden::equals(Some(v))] - } - }) - } -} - -#[derive(Deserialize, Type, Debug)] -#[serde(rename_all = "camelCase")] -pub enum FilePathObjectCursor { - DateAccessed(CursorOrderItem>), - Kind(CursorOrderItem), -} - -impl FilePathObjectCursor { - fn apply(self, query: &mut file_path::FindManyQuery) { - macro_rules! arm { - ($field:ident, $item:ident) => {{ - let item = $item; - - query.add_where(match item.order { - SortOrder::Asc => { - prisma::file_path::object::is(vec![prisma::object::$field::gt(item.data)]) - } - SortOrder::Desc => { - prisma::file_path::object::is(vec![prisma::object::$field::lt(item.data)]) - } - }); - - query.add_order_by(prisma::file_path::object::order(vec![ - prisma::object::$field::order(item.order.into()), - ])); - }}; - } - - match self { - FilePathObjectCursor::Kind(item) => arm!(kind, item), - FilePathObjectCursor::DateAccessed(item) => { - arm!(date_accessed, item) - } - }; - } -} - -#[derive(Deserialize, Type, Debug)] -#[serde(rename_all = "camelCase")] -pub enum FilePathCursorVariant { - None, - Name(CursorOrderItem), - SizeInBytes(SortOrder), - DateCreated(CursorOrderItem>), - DateModified(CursorOrderItem>), - DateIndexed(CursorOrderItem>), - Object(FilePathObjectCursor), -} - -impl FilePathCursorVariant { - pub fn apply(self, query: &mut file_path::FindManyQuery, id: i32) { - macro_rules! arm { - ($field:ident, $item:ident) => {{ - let item = $item; - - let data = item.data.clone(); - - query.add_where(prisma_client_rust::or![ - match item.order { - SortOrder::Asc => prisma::file_path::$field::gt(data), - SortOrder::Desc => prisma::file_path::$field::lt(data), - }, - prisma_client_rust::and![ - prisma::file_path::$field::equals(Some(item.data)), - match item.order { - SortOrder::Asc => prisma::file_path::id::gt(id), - SortOrder::Desc => prisma::file_path::id::lt(id), - } - ] - ]); - - query.add_order_by(prisma::file_path::$field::order(item.order.into())); - }}; - } - - match self { - Self::None => { - query.add_where(prisma::file_path::id::gt(id)); - } - Self::SizeInBytes(order) => { - query.add_order_by(prisma::file_path::size_in_bytes_bytes::order(order.into())); - } - Self::Name(item) => arm!(name, item), - Self::DateCreated(item) => { - arm!(date_created, item) - } - Self::DateModified(item) => { - arm!(date_modified, item) - } - Self::DateIndexed(item) => { - arm!(date_indexed, item) - } - Self::Object(obj) => obj.apply(query), - }; - } -} - -#[derive(Deserialize, Type, Debug)] -#[serde(rename_all = "camelCase")] -pub struct FilePathCursor { - pub is_dir: bool, - pub variant: FilePathCursorVariant, -} - -pub type OrderAndPagination = - utils::OrderAndPagination; - -impl OrderAndPagination { - pub fn apply(self, query: &mut file_path::FindManyQuery, group_directories: bool) { - match self { - Self::OrderOnly(order) => { - query.add_order_by(order.into_param()); - } - Self::Offset { offset, order } => { - query.set_skip(offset as i64); - - if let Some(order) = order { - query.add_order_by(order.into_param()) - } - } - Self::Cursor { id, cursor } => { - // This may seem dumb but it's vital! - // If we're grouping by directories + all directories have been fetched, - // we don't want to include them in the results. - // It's important to keep in mind that since the `order_by` for - // `group_directories` comes before all other orderings, - // all other orderings will be applied independently to directories and paths. - if group_directories && !cursor.is_dir { - query.add_where(prisma::file_path::is_dir::not(Some(true))) - } - - cursor.variant.apply(query, id); - - query.add_order_by(prisma::file_path::id::order(prisma::SortOrder::Asc)); - } - } - } -} diff --git a/core/src/api/search/mod.rs b/core/src/api/search/mod.rs deleted file mode 100644 index 90abc8a1d..000000000 --- a/core/src/api/search/mod.rs +++ /dev/null @@ -1,424 +0,0 @@ -use crate::{ - api::{locations::ExplorerItem, utils::library}, - library::Library, - location::{non_indexed, LocationError}, - util::{unsafe_streamed_query, BatchedStream}, -}; - -use prisma_client_rust::Operator; -use sd_core_heavy_lifting::media_processor::ThumbKey; -use sd_core_prisma_helpers::{file_path_for_frontend, object_with_file_paths, CasId}; -use sd_prisma::prisma::{self, PrismaClient}; - -use std::path::PathBuf; - -use async_stream::stream; -use futures::StreamExt; -use itertools::Either; -use rspc::{alpha::AlphaRouter, ErrorCode}; -use serde::{Deserialize, Serialize}; -use specta::Type; - -pub mod exif_data; -pub mod file_path; -pub mod object; -pub mod saved; -mod utils; - -pub use self::{file_path::*, object::*, utils::*}; - -use super::{Ctx, R}; - -const MAX_TAKE: u8 = 100; - -#[derive(Serialize, Type, Debug)] -struct SearchData { - cursor: Option>, - items: Vec, -} - -#[derive(Serialize, Deserialize, Type, Debug, Clone)] -#[serde(rename_all = "camelCase")] -pub enum SearchFilterArgs { - FilePath(FilePathFilterArgs), - Object(ObjectFilterArgs), -} - -impl SearchFilterArgs { - async fn into_params( - self, - db: &PrismaClient, - file_path: &mut Vec, - object: &mut Vec, - ) -> Result<(), rspc::Error> { - match self { - Self::FilePath(v) => file_path.extend(v.into_params(db).await?), - Self::Object(v) => object.extend(v.into_params()), - }; - Ok(()) - } -} - -pub fn mount() -> AlphaRouter { - R.router() - .procedure("ephemeralPaths", { - #[derive(Serialize, Deserialize, Type, Debug, Clone)] - #[serde(rename_all = "camelCase", tag = "field", content = "value")] - enum EphemeralPathOrder { - Name(SortOrder), - SizeInBytes(SortOrder), - DateCreated(SortOrder), - DateModified(SortOrder), - } - - #[derive(Deserialize, Type, Debug)] - #[serde(rename_all = "camelCase")] - struct EphemeralPathSearchArgs { - path: PathBuf, - with_hidden_files: bool, - #[specta(optional)] - order: Option, - } - #[derive(Serialize, Type, Debug)] - struct EphemeralPathsResultItem { - pub entries: Vec, - pub errors: Vec, - } - - R.with2(library()).subscription( - |(node, library), - EphemeralPathSearchArgs { - path, - with_hidden_files, - order, - }| async move { - let paths = - non_indexed::walk(path, with_hidden_files, node, library, |entries| { - macro_rules! order_match { - ($order:ident, [$(($variant:ident, |$i:ident| $func:expr)),+]) => {{ - match $order { - $(EphemeralPathOrder::$variant(order) => { - entries.sort_unstable_by(|path1, path2| { - let func = |$i: &non_indexed::Entry| $func; - - let one = func(path1); - let two = func(path2); - - match order { - SortOrder::Desc => two.cmp(&one), - SortOrder::Asc => one.cmp(&two), - } - }); - })+ - } - }}; - } - - if let Some(order) = order { - order_match!( - order, - [ - (Name, |p| p.name().to_lowercase()), - (SizeInBytes, |p| p.size_in_bytes()), - (DateCreated, |p| p.date_created()), - (DateModified, |p| p.date_modified()) - ] - ) - } - }) - .await?; - - let mut stream = BatchedStream::new(paths); - Ok(unsafe_streamed_query(stream! { - while let Some(result) = stream.next().await { - // We optimize for the case of no errors because it should be way more common. - let mut entries = Vec::with_capacity(result.len()); - let mut errors = Vec::with_capacity(0); - - for item in result { - match item { - Ok(item) => entries.push(item), - Err(e) => match e { - Either::Left(e) => errors.push(e), - Either::Right(e) => errors.push(e.into()), - }, - } - } - - yield EphemeralPathsResultItem { - entries, - errors, - }; - } - })) - }, - ) - }) - .procedure("paths", { - #[derive(Deserialize, Type, Debug)] - #[serde(rename_all = "camelCase")] - struct FilePathSearchArgs { - #[specta(optional)] - take: Option, - #[specta(optional)] - order_and_pagination: Option, - #[serde(default)] - filters: Vec, - #[serde(default = "default_group_directories")] - group_directories: bool, - } - - fn default_group_directories() -> bool { - true - } - - R.with2(library()).query( - |(node, library), - FilePathSearchArgs { - take, - order_and_pagination, - filters, - group_directories, - }| async move { - let Library { db, .. } = library.as_ref(); - - let params = { - let (mut fp, obj) = merge_filters(filters, db).await?; - - if !obj.is_empty() { - fp.push(prisma::file_path::object::is(obj)); - } - - fp - }; - - let mut query = db.file_path().find_many(andify(params)); - - if let Some(take) = take { - query = query.take(take as i64); - } - - // WARN: this order_by for grouping directories MUST always come before the other order_by - if group_directories { - query = query - .order_by(prisma::file_path::is_dir::order(prisma::SortOrder::Desc)); - } - - // WARN: this order_by for sorting data MUST always come after the other order_by - if let Some(order_and_pagination) = order_and_pagination { - order_and_pagination.apply(&mut query, group_directories) - } - - let file_paths = query - .include(file_path_for_frontend::include()) - .exec() - .await?; - - let mut items = Vec::with_capacity(file_paths.len()); - - for file_path in file_paths { - let has_created_thumbnail = - if let Some(cas_id) = file_path.cas_id.as_ref().map(CasId::from) { - library - .thumbnail_exists(&node, &cas_id) - .await - .map_err(LocationError::from)? - } else { - false - }; - - items.push(ExplorerItem::Path { - thumbnail: file_path - .cas_id - .as_ref() - .map(CasId::from) - .map(CasId::into_owned) - .map(|cas_id| ThumbKey::new_indexed(cas_id, library.id)), - has_created_thumbnail, - item: Box::new(file_path), - }) - } - - Ok(SearchData { - items, - cursor: None, - }) - }, - ) - }) - .procedure("pathsCount", { - #[derive(Deserialize, Type, Debug)] - #[serde(rename_all = "camelCase")] - #[specta(inline)] - struct Args { - #[specta(default)] - filters: Vec, - } - - R.with2(library()) - .query(|(_, library), Args { filters }| async move { - let Library { db, .. } = library.as_ref(); - - Ok(db - .file_path() - .count({ - let (mut fp, obj) = merge_filters(filters, db).await?; - - if !obj.is_empty() { - fp.push(prisma::file_path::object::is(obj)); - } - - fp - }) - .exec() - .await? as u32) - }) - }) - .procedure("objects", { - #[derive(Deserialize, Type, Debug)] - #[serde(rename_all = "camelCase")] - struct ObjectSearchArgs { - take: u8, - #[specta(optional)] - order_and_pagination: Option, - #[serde(default)] - filters: Vec, - } - - R.with2(library()).query( - |(node, library), - ObjectSearchArgs { - take, - order_and_pagination, - filters, - }| async move { - let Library { db, .. } = library.as_ref(); - - let take = take.max(MAX_TAKE); - - let mut query = db - .object() - .find_many({ - let (fp, mut obj) = merge_filters(filters, db).await?; - - if !fp.is_empty() { - obj.push(prisma::object::file_paths::some(fp)); - } - - andify(obj) - }) - .take(take as i64); - - if let Some(order_and_pagination) = order_and_pagination { - order_and_pagination.apply(&mut query); - } - - let (objects, cursor) = { - let mut objects = query - .include(object_with_file_paths::include()) - .exec() - .await?; - - let cursor = (objects.len() as u8 > take) - .then(|| objects.pop()) - .flatten() - .map(|r| r.pub_id); - - (objects, cursor) - }; - - let mut items = Vec::with_capacity(objects.len()); - - for object in objects { - let cas_id = object - .file_paths - .iter() - .map(|fp| fp.cas_id.as_ref()) - .find_map(|c| c) - .map(CasId::from) - .map(|cas_id| cas_id.to_owned()); - - let has_created_thumbnail = if let Some(cas_id) = &cas_id { - library.thumbnail_exists(&node, cas_id).await.map_err(|e| { - rspc::Error::with_cause( - ErrorCode::InternalServerError, - "Failed to check that thumbnail exists".to_string(), - e, - ) - })? - } else { - false - }; - - items.push(ExplorerItem::Object { - thumbnail: cas_id - .map(|cas_id| ThumbKey::new_indexed(cas_id, library.id)), - item: object, - has_created_thumbnail, - }); - } - - Ok(SearchData { items, cursor }) - }, - ) - }) - .procedure("objectsCount", { - #[derive(Deserialize, Type, Debug)] - #[serde(rename_all = "camelCase")] - #[specta(inline)] - struct Args { - #[serde(default)] - filters: Vec, - } - - R.with2(library()) - .query(|(_, library), Args { filters }| async move { - let Library { db, .. } = library.as_ref(); - - Ok(db - .object() - .count({ - let (fp, mut obj) = merge_filters(filters, db).await?; - - if !fp.is_empty() { - obj.push(prisma::object::file_paths::some(fp)); - } - - obj - }) - .exec() - .await? as u32) - }) - }) - .merge("saved.", saved::mount()) -} - -async fn merge_filters( - filters: Vec, - db: &PrismaClient, -) -> Result< - ( - Vec, - Vec, - ), - rspc::Error, -> { - let mut obj = vec![]; - let mut fp = vec![]; - - for filter in filters { - filter.into_params(db, &mut fp, &mut obj).await?; - } - - Ok((fp, obj)) -} - -/// PCR 0.6.x's AND does { AND: [{ ...}] } instead of { AND: [{ ... }, { ... }, { ... }] }, -/// this works around it. -fn andify>>(params: Vec) -> Vec { - params.into_iter().fold(vec![], |mut params, param| { - params.push(param); - - vec![prisma_client_rust::operator::and(params)] - }) -} diff --git a/core/src/api/search/object.rs b/core/src/api/search/object.rs deleted file mode 100644 index 5804b3917..000000000 --- a/core/src/api/search/object.rs +++ /dev/null @@ -1,181 +0,0 @@ -// use crate::library::Category; - -use sd_prisma::prisma::{self, label_on_object, object, tag_on_object}; - -use chrono::{DateTime, FixedOffset}; -use prisma_client_rust::{not, or, OrderByQuery, PaginatedQuery, WhereQuery}; -use serde::{Deserialize, Serialize}; -use specta::Type; - -use super::{ - exif_data::*, - utils::{self, *}, -}; - -#[derive(Deserialize, Type, Debug)] -#[serde(rename_all = "camelCase")] -pub enum ObjectCursor { - None, - DateAccessed(CursorOrderItem>), - Kind(CursorOrderItem), -} - -impl ObjectCursor { - fn apply(self, query: &mut object::FindManyQuery, id: i32) { - macro_rules! arm { - ($field:ident, $item:ident) => {{ - let item = $item; - - let data = item.data.clone(); - - query.add_where(or![ - match item.order { - SortOrder::Asc => object::$field::gt(data), - SortOrder::Desc => object::$field::lt(data), - }, - prisma_client_rust::and![ - object::$field::equals(Some(item.data)), - match item.order { - SortOrder::Asc => object::id::gt(id), - SortOrder::Desc => object::id::lt(id), - } - ] - ]); - - query.add_order_by(object::$field::order(item.order.into())); - }}; - } - - match self { - Self::None => { - query.add_where(object::id::gt(id)); - } - Self::Kind(item) => arm!(kind, item), - Self::DateAccessed(item) => arm!(date_accessed, item), - } - } -} - -#[derive(Serialize, Deserialize, Type, Debug, Clone)] -#[serde(rename_all = "camelCase", tag = "field", content = "value")] -pub enum ObjectOrder { - DateAccessed(SortOrder), - Kind(SortOrder), - MediaData(Box), -} - -impl ObjectOrder { - pub fn get_sort_order(&self) -> prisma::SortOrder { - (*match self { - Self::DateAccessed(v) => v, - Self::Kind(v) => v, - Self::MediaData(v) => return v.get_sort_order(), - }) - .into() - } - - pub fn into_param(self) -> object::OrderByWithRelationParam { - let dir = self.get_sort_order(); - use object::*; - - match self { - Self::DateAccessed(_) => date_accessed::order(dir), - Self::Kind(_) => kind::order(dir), - Self::MediaData(v) => exif_data::order(vec![v.into_param()]), - } - } -} - -#[derive(Serialize, Deserialize, Type, Debug, Default, Clone, Copy)] -#[serde(rename_all = "camelCase")] -pub enum ObjectHiddenFilter { - #[default] - Exclude, - Include, -} - -impl ObjectHiddenFilter { - pub fn to_param(self) -> Option { - match self { - ObjectHiddenFilter::Exclude => Some(or![ - object::hidden::equals(None), - object::hidden::not(Some(true)) - ]), - ObjectHiddenFilter::Include => None, - } - } -} - -#[derive(Serialize, Deserialize, Type, Debug, Clone)] -#[serde(rename_all = "camelCase")] -pub enum ObjectFilterArgs { - Favorite(bool), - Hidden(ObjectHiddenFilter), - Kind(InOrNotIn), - Tags(InOrNotIn), - Labels(InOrNotIn), - DateAccessed(Range>), -} - -impl ObjectFilterArgs { - pub fn into_params(self) -> Vec { - use object::*; - - match self { - Self::Favorite(v) => vec![favorite::equals(Some(v))], - Self::Hidden(v) => v.to_param().map(|v| vec![v]).unwrap_or_default(), - Self::Tags(v) => v - .into_param( - |v| tags::some(vec![tag_on_object::tag_id::in_vec(v)]), - |v| tags::none(vec![tag_on_object::tag_id::in_vec(v)]), - ) - .map(|v| vec![v]) - .unwrap_or_default(), - Self::Labels(v) => v - .into_param( - |v| labels::some(vec![label_on_object::label_id::in_vec(v)]), - |v| labels::none(vec![label_on_object::label_id::in_vec(v)]), - ) - .map(|v| vec![v]) - .unwrap_or_default(), - Self::Kind(v) => v - .into_param(kind::in_vec, kind::not_in_vec) - .map(|v| vec![v]) - .unwrap_or_default(), - Self::DateAccessed(v) => { - vec![ - not![date_accessed::equals(None)], - match v { - Range::From(v) => date_accessed::gte(v), - Range::To(v) => date_accessed::lte(v), - }, - ] - } - } - } -} - -pub type OrderAndPagination = - utils::OrderAndPagination; - -impl OrderAndPagination { - pub fn apply(self, query: &mut object::FindManyQuery) { - match self { - Self::OrderOnly(order) => { - query.add_order_by(order.into_param()); - } - Self::Offset { offset, order } => { - query.set_skip(offset as i64); - - if let Some(order) = order { - query.add_order_by(order.into_param()) - } - } - Self::Cursor { id, cursor } => { - cursor.apply(query, id); - - query.add_order_by(object::pub_id::order(prisma::SortOrder::Asc)) - } - } - } -} diff --git a/core/src/api/search/saved.rs b/core/src/api/search/saved.rs deleted file mode 100644 index 957474c49..000000000 --- a/core/src/api/search/saved.rs +++ /dev/null @@ -1,232 +0,0 @@ -use std::str::FromStr; - -use crate::{api::utils::library, invalidate_query, library::Library}; - -use sd_prisma::{prisma::saved_search, prisma_sync}; -use sd_sync::{option_sync_db_entry, sync_db_entry, OperationFactory}; -use sd_utils::chain_optional_iter; - -use chrono::{DateTime, FixedOffset, Utc}; -use rspc::alpha::AlphaRouter; -use serde::{de::IgnoredAny, Deserialize}; -use specta::Type; -use tracing::error; -use uuid::Uuid; - -use super::{Ctx, R}; - -#[derive(Type, Deserialize, Clone, Debug, Default)] -#[serde(rename_all = "camelCase")] -enum SearchTarget { - #[default] - Paths, - Objects, -} - -impl std::fmt::Display for SearchTarget { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - SearchTarget::Paths => write!(f, "paths"), - SearchTarget::Objects => write!(f, "objects"), - } - } -} - -impl FromStr for SearchTarget { - type Err = String; - - fn from_str(s: &str) -> Result { - match s { - "paths" => Ok(SearchTarget::Paths), - "objects" => Ok(SearchTarget::Objects), - _ => Err(format!("invalid search target: {s}")), - } - } -} - -pub(crate) fn mount() -> AlphaRouter { - R.router() - .procedure("create", { - R.with2(library()).mutation({ - #[derive(Type, Deserialize, Clone, Debug)] - #[specta(inline)] - pub struct Args { - pub name: String, - #[serde(default)] - pub target: SearchTarget, - #[specta(optional)] - pub search: Option, - #[specta(optional)] - pub filters: Option, - #[specta(optional)] - pub description: Option, - #[specta(optional)] - pub icon: Option, - } - - |(_, library), args: Args| async move { - let Library { db, sync, .. } = library.as_ref(); - let pub_id = Uuid::now_v7().as_bytes().to_vec(); - let date_created: DateTime = Utc::now().into(); - - let (sync_params, db_params) = chain_optional_iter( - [ - sync_db_entry!(date_created, saved_search::date_created), - sync_db_entry!(args.name, saved_search::name), - sync_db_entry!(args.target.to_string(), saved_search::target), - ], - [ - option_sync_db_entry!( - args.filters.and_then(|s| { - // https://github.com/serde-rs/json/issues/579 - // https://docs.rs/serde/latest/serde/de/struct.IgnoredAny.html - - if let Err(e) = serde_json::from_str::(&s) { - error!(?e, "Failed to parse filters;"); - None - } else { - Some(s) - } - }), - saved_search::filters - ), - option_sync_db_entry!(args.search, saved_search::search), - option_sync_db_entry!(args.description, saved_search::description), - option_sync_db_entry!(args.icon, saved_search::icon), - ], - ) - .into_iter() - .unzip::<_, _, Vec<_>, Vec<_>>(); - - sync.write_op( - db, - sync.shared_create( - prisma_sync::saved_search::SyncId { - pub_id: pub_id.clone(), - }, - sync_params, - ), - db.saved_search() - .create(pub_id, db_params) - .select(saved_search::select!({ id })), - ) - .await?; - - invalidate_query!(library, "search.saved.list"); - - Ok(()) - } - }) - }) - .procedure("get", { - R.with2(library()) - .query(|(_, library), search_id: i32| async move { - Ok(library - .db - .saved_search() - .find_unique(saved_search::id::equals(search_id)) - .exec() - .await?) - }) - }) - .procedure("list", { - R.with2(library()).query(|(_, library), _: ()| async move { - Ok(library - .db - .saved_search() - .find_many(vec![]) - // .order_by(saved_search::order::order(prisma::SortOrder::Desc)) - .exec() - .await?) - }) - }) - .procedure("update", { - R.with2(library()).mutation({ - saved_search::partial_unchecked!(Args { - name - description - icon - search - filters - }); - - |(_, library), (id, args): (saved_search::id::Type, Args)| async move { - let Library { db, sync, .. } = library.as_ref(); - let updated_at = Utc::now(); - - let search = db - .saved_search() - .find_unique(saved_search::id::equals(id)) - .select(saved_search::select!({ pub_id })) - .exec() - .await? - .ok_or_else(|| { - rspc::Error::new(rspc::ErrorCode::NotFound, "search not found".into()) - })?; - - let (sync_params, db_params) = chain_optional_iter( - [sync_db_entry!(updated_at, saved_search::date_modified)], - [ - option_sync_db_entry!(args.name.flatten(), saved_search::name), - option_sync_db_entry!(args.description.flatten(), saved_search::name), - option_sync_db_entry!(args.icon.flatten(), saved_search::icon), - option_sync_db_entry!(args.search.flatten(), saved_search::search), - option_sync_db_entry!(args.filters.flatten(), saved_search::filters), - ], - ) - .into_iter() - .unzip::<_, _, Vec<_>, Vec<_>>(); - - sync.write_op( - db, - sync.shared_update( - prisma_sync::saved_search::SyncId { - pub_id: search.pub_id.clone(), - }, - sync_params, - ), - db.saved_search() - .update_unchecked(saved_search::id::equals(id), db_params), - ) - .await?; - - invalidate_query!(library, "search.saved.list"); - invalidate_query!(library, "search.saved.get"); - - Ok(()) - } - }) - }) - .procedure("delete", { - R.with2(library()) - .mutation(|(_, library), search_id: i32| async move { - let Library { db, sync, .. } = library.as_ref(); - - let search = db - .saved_search() - .find_unique(saved_search::id::equals(search_id)) - .select(saved_search::select!({ pub_id })) - .exec() - .await? - .ok_or_else(|| { - rspc::Error::new(rspc::ErrorCode::NotFound, "search not found".into()) - })?; - - sync.write_op( - db, - sync.shared_delete(prisma_sync::saved_search::SyncId { - pub_id: search.pub_id, - }), - db.saved_search() - .delete(saved_search::id::equals(search_id)), - ) - .await?; - - invalidate_query!(library, "search.saved.list"); - // disabled as it's messing with pre-delete navigation - // invalidate_query!(library, "search.saved.get"); - - Ok(()) - }) - }) -} diff --git a/core/src/api/search/utils.rs b/core/src/api/search/utils.rs deleted file mode 100644 index 1206f398c..000000000 --- a/core/src/api/search/utils.rs +++ /dev/null @@ -1,125 +0,0 @@ -use sd_prisma::prisma; - -use serde::{Deserialize, Serialize}; -use specta::Type; - -#[derive(Serialize, Deserialize, Type, Debug, Clone)] -#[serde(rename_all = "camelCase")] -pub enum Range { - From(T), - To(T), -} - -#[derive(Serialize, Deserialize, Type, Debug, Clone, Copy)] -#[serde(rename_all = "PascalCase")] -pub enum SortOrder { - Asc, - Desc, -} - -impl From for prisma::SortOrder { - fn from(value: SortOrder) -> prisma::SortOrder { - match value { - SortOrder::Asc => prisma::SortOrder::Asc, - SortOrder::Desc => prisma::SortOrder::Desc, - } - } -} - -// #[derive(Deserialize, Type, Debug, Clone)] -// #[serde(untagged)] -// pub enum MaybeNot { -// None(T), -// Not { not: T }, -// } - -// impl MaybeNot { -// pub fn into_prisma>>(self, param: fn(T) -> R) -> R { -// match self { -// Self::None(v) => param(v), -// Self::Not { not } => prisma_client_rust::not![param(not)], -// } -// } -// } - -#[derive(Deserialize, Type, Debug)] -#[serde(rename_all = "camelCase")] -pub struct CursorOrderItem { - pub order: SortOrder, - pub data: T, -} - -#[derive(Deserialize, Type, Debug)] -#[serde(rename_all = "camelCase")] -pub enum OrderAndPagination { - OrderOnly(TOrder), - Offset { offset: i32, order: Option }, - Cursor { id: TId, cursor: TCursor }, -} - -#[derive(Serialize, Deserialize, Type, Debug, Clone)] -#[serde(rename_all = "camelCase")] -pub enum InOrNotIn { - In(Vec), - NotIn(Vec), -} - -impl InOrNotIn { - pub fn is_empty(&self) -> bool { - match self { - Self::In(v) => v.is_empty(), - Self::NotIn(v) => v.is_empty(), - } - } - - pub fn into_param( - self, - in_fn: fn(Vec) -> TParam, - not_in_fn: fn(Vec) -> TParam, - ) -> Option { - self.is_empty() - .then_some(None) - .unwrap_or_else(|| match self { - Self::In(v) => Some(in_fn(v)), - Self::NotIn(v) => Some(not_in_fn(v)), - }) - } -} - -#[derive(Serialize, Deserialize, Type, Debug, Clone)] -#[serde(rename_all = "camelCase")] -pub enum TextMatch { - Contains(String), - StartsWith(String), - EndsWith(String), - Equals(String), -} - -impl TextMatch { - pub fn is_empty(&self) -> bool { - match self { - Self::Contains(v) => v.is_empty(), - Self::StartsWith(v) => v.is_empty(), - Self::EndsWith(v) => v.is_empty(), - Self::Equals(v) => v.is_empty(), - } - } - - // 3. Update the to_param method of TextMatch - pub fn into_param( - self, - contains_fn: fn(String) -> TParam, - starts_with_fn: fn(String) -> TParam, - ends_with_fn: fn(String) -> TParam, - equals_fn: fn(String) -> TParam, - ) -> Option { - self.is_empty() - .then_some(None) - .unwrap_or_else(|| match self { - Self::Contains(v) => Some(contains_fn(v)), - Self::StartsWith(v) => Some(starts_with_fn(v)), - Self::EndsWith(v) => Some(ends_with_fn(v)), - Self::Equals(v) => Some(equals_fn(v)), - }) - } -} diff --git a/core/src/api/sync.rs b/core/src/api/sync.rs deleted file mode 100644 index 5880f8fa7..000000000 --- a/core/src/api/sync.rs +++ /dev/null @@ -1,78 +0,0 @@ -use rspc::alpha::AlphaRouter; -use std::sync::atomic::Ordering; - -use crate::util::MaybeUndefined; - -use super::{utils::library, Ctx, R}; - -pub(crate) fn mount() -> AlphaRouter { - R.router() - .procedure("backfill", { - R.with2(library()) - .mutation(|(node, library), _: ()| async move { - if library - .config() - .await - .generate_sync_operations - .load(Ordering::Relaxed) - { - return Ok(()); - } - - sd_core_sync::backfill::backfill_operations(&library.sync).await?; - - node.libraries - .edit( - library.id, - None, - MaybeUndefined::Undefined, - MaybeUndefined::Undefined, - Some(true), - ) - .await?; - - Ok(()) - }) - }) - .procedure("enabled", { - R.with2(library()).query(|(_, library), _: ()| async move { - Ok(library - .config() - .await - .generate_sync_operations - .load(Ordering::Relaxed)) - }) - }) - .procedure("active", { - R.with2(library()) - .subscription(|(_, library), _: ()| async move { - #[derive(serde::Serialize, specta::Type)] - #[specta(rename = "SyncStatus")] - struct Data { - ingest: bool, - cloud_send: bool, - cloud_receive: bool, - cloud_ingest: bool, - } - - async_stream::stream! { - let cloud_sync_state = &library.cloud_sync_state; - let sync = &library.sync; - - loop { - yield Data { - ingest: sync.active.load(Ordering::Relaxed), - cloud_send: cloud_sync_state.send_active.load(Ordering::Relaxed), - cloud_receive: cloud_sync_state.receive_active.load(Ordering::Relaxed), - cloud_ingest: cloud_sync_state.ingest_active.load(Ordering::Relaxed), - }; - - tokio::select! { - _ = cloud_sync_state.state_change_notifier.notified() => {}, - _ = sync.active_notify.notified() => {} - } - } - } - }) - }) -} diff --git a/core/src/api/tags.rs b/core/src/api/tags.rs deleted file mode 100644 index 0035ea592..000000000 --- a/core/src/api/tags.rs +++ /dev/null @@ -1,396 +0,0 @@ -use crate::{invalidate_query, library::Library, object::tag::TagCreateArgs}; - -use sd_prisma::{ - prisma::{device, file_path, object, tag, tag_on_object}, - prisma_sync, -}; -use sd_sync::{option_sync_db_entry, sync_db_entry, sync_entry, OperationFactory}; - -use std::collections::BTreeMap; - -use chrono::{DateTime, Utc}; -use itertools::{Either, Itertools}; -use rspc::{alpha::AlphaRouter, ErrorCode}; -use serde::{Deserialize, Serialize}; -use specta::Type; - -use super::{utils::library, Ctx, R}; - -pub(crate) fn mount() -> AlphaRouter { - R.router() - .procedure("list", { - R.with2(library()).query(|(_, library), _: ()| async move { - Ok(library.db.tag().find_many(vec![]).exec().await?) - }) - }) - .procedure("getForObject", { - R.with2(library()) - .query(|(_, library), object_id: i32| async move { - Ok(library - .db - .tag() - .find_many(vec![tag::tag_objects::some(vec![ - tag_on_object::object_id::equals(object_id), - ])]) - .exec() - .await?) - }) - }) - .procedure("getWithObjects", { - #[derive(Serialize, Type)] - pub struct ObjectWithDateCreated { - object: object::Data, - date_created: DateTime, - } - - R.with2(library()).query( - |(_, library), object_ids: Vec| async move { - let Library { db, .. } = library.as_ref(); - - let tags_with_objects = db - .tag() - .find_many(vec![tag::tag_objects::some(vec![ - tag_on_object::object_id::in_vec(object_ids.clone()), - ])]) - .select(tag::select!({ - id - tag_objects(vec![tag_on_object::object_id::in_vec(object_ids.clone())]): select { - date_created - object: select { - id - } - } - })) - .exec() - .await?; - - // This doesn't need normalised caching because it doesn't return whole models. - Ok(tags_with_objects - .into_iter() - .map(|tag| (tag.id, tag.tag_objects)) - .collect::>()) - }, - ) - }) - .procedure("get", { - R.with2(library()) - .query(|(_, library), tag_id: i32| async move { - Ok(library - .db - .tag() - .find_unique(tag::id::equals(tag_id)) - .exec() - .await?) - }) - }) - .procedure("create", { - R.with2(library()) - .mutation(|(_, library), args: TagCreateArgs| async move { - // Check if tag with the same name already exists - let existing_tag = library - .db - .tag() - .find_many(vec![tag::name::equals(Some(args.name.clone()))]) - .select(tag::select!({ id })) - .exec() - .await?; - - if !existing_tag.is_empty() { - return Err(rspc::Error::new( - ErrorCode::Conflict, - "Tag with the same name already exists".to_string(), - )); - } - - let created_tag = args.exec(&library).await?; - - invalidate_query!(library, "tags.list"); - - Ok(created_tag) - }) - }) - .procedure("assign", { - #[derive(Debug, Type, Deserialize)] - #[specta(inline)] - enum Target { - Object(object::id::Type), - FilePath(file_path::id::Type), - } - - #[derive(Debug, Type, Deserialize)] - #[specta(inline)] - struct TagAssignArgs { - targets: Vec, - tag_id: i32, - unassign: bool, - } - - R.with2(library()) - .mutation(|(_, library), args: TagAssignArgs| async move { - let Library { db, sync, .. } = library.as_ref(); - - let device_id = library - .db - .device() - .find_unique(device::pub_id::equals(sync.device_pub_id.to_db())) - .select(device::select!({ id })) - .exec() - .await? - .ok_or_else(|| { - rspc::Error::new( - ErrorCode::NotFound, - "Local device not found".to_string(), - ) - })? - .id; - - let tag = db - .tag() - .find_unique(tag::id::equals(args.tag_id)) - .select(tag::select!({ pub_id })) - .exec() - .await? - .ok_or_else(|| { - rspc::Error::new(ErrorCode::NotFound, "Tag not found".to_string()) - })?; - - let (objects, file_paths) = db - ._batch({ - let (objects, file_paths): (Vec<_>, Vec<_>) = args - .targets - .into_iter() - .partition_map(|target| match target { - Target::Object(id) => Either::Left(id), - Target::FilePath(id) => Either::Right(id), - }); - - ( - db.object() - .find_many(vec![object::id::in_vec(objects)]) - .select(object::select!({ - id - pub_id - })), - db.file_path() - .find_many(vec![file_path::id::in_vec(file_paths)]) - .select(file_path::select!({ - id - pub_id - is_dir - object: select { id pub_id } - })), - ) - }) - .await?; - - if args.unassign { - let query = db.tag_on_object().delete_many(vec![ - tag_on_object::tag_id::equals(args.tag_id), - tag_on_object::object_id::in_vec( - objects - .iter() - .map(|o| o.id) - .chain( - file_paths - .iter() - .filter_map(|fp| fp.object.as_ref().map(|o| o.id)), - ) - .collect(), - ), - ]); - - let ops = objects - .into_iter() - .map(|o| o.pub_id) - .chain( - file_paths - .into_iter() - .filter_map(|fp| fp.object.map(|o| o.pub_id)), - ) - .map(|pub_id| { - sync.relation_delete(prisma_sync::tag_on_object::SyncId { - tag: prisma_sync::tag::SyncId { - pub_id: tag.pub_id.clone(), - }, - object: prisma_sync::object::SyncId { pub_id }, - }) - }) - .collect::>(); - - if !ops.is_empty() { - sync.write_ops(db, (ops, query)).await?; - } - } else { - let (sync_ops, db_creates) = objects - .into_iter() - .map(|o| (o.id, o.pub_id)) - .chain( - file_paths - .into_iter() - .filter_map(|fp| fp.object.map(|o| (o.id, o.pub_id))), - ) - .map(|(id, pub_id)| { - ( - sync.relation_create( - prisma_sync::tag_on_object::SyncId { - tag: prisma_sync::tag::SyncId { - pub_id: tag.pub_id.clone(), - }, - object: prisma_sync::object::SyncId { pub_id }, - }, - [sync_entry!( - prisma_sync::device::SyncId { - pub_id: sync.device_pub_id.to_db(), - }, - tag_on_object::device - )], - ), - tag_on_object::CreateUnchecked { - tag_id: args.tag_id, - object_id: id, - _params: vec![ - tag_on_object::date_created::set(Some( - Utc::now().into(), - )), - tag_on_object::device_id::set(Some(device_id)), - ], - }, - ) - }) - .unzip::<_, _, Vec<_>, Vec<_>>(); - - if !sync_ops.is_empty() && !db_creates.is_empty() { - sync.write_ops( - db, - ( - sync_ops, - db.tag_on_object().create_many(db_creates).skip_duplicates(), - ), - ) - .await?; - } - } - - invalidate_query!(library, "tags.getForObject"); - invalidate_query!(library, "tags.getWithObjects"); - invalidate_query!(library, "search.objects"); - - Ok(()) - }) - }) - .procedure("update", { - #[derive(Type, Deserialize)] - pub struct TagUpdateArgs { - pub id: i32, - pub name: Option, - pub color: Option, - } - - R.with2(library()).mutation( - |(_, library), TagUpdateArgs { id, name, color }: TagUpdateArgs| async move { - if name.is_none() && color.is_none() { - return Ok(()); - } - - let Library { sync, db, .. } = library.as_ref(); - - let tag = db - .tag() - .find_unique(tag::id::equals(id)) - .select(tag::select!({ pub_id })) - .exec() - .await? - .ok_or(rspc::Error::new( - ErrorCode::NotFound, - "Error finding tag in db".into(), - ))?; - - let (sync_params, db_params) = [ - option_sync_db_entry!(name, tag::name), - option_sync_db_entry!(color, tag::color), - Some(sync_db_entry!(Utc::now(), tag::date_modified)), - ] - .into_iter() - .flatten() - .unzip::<_, _, Vec<_>, Vec<_>>(); - - sync.write_op( - db, - sync.shared_update( - prisma_sync::tag::SyncId { - pub_id: tag.pub_id.clone(), - }, - sync_params, - ), - db.tag() - .update(tag::id::equals(id), db_params) - .select(tag::select!({ id })), - ) - .await?; - - invalidate_query!(library, "tags.list"); - - Ok(()) - }, - ) - }) - .procedure( - "delete", - R.with2(library()) - .mutation(|(_, library), tag_id: tag::id::Type| async move { - let Library { sync, db, .. } = &*library; - - let tag_pub_id = db - .tag() - .find_unique(tag::id::equals(tag_id)) - .select(tag::select!({ pub_id })) - .exec() - .await? - .ok_or(rspc::Error::new( - rspc::ErrorCode::NotFound, - "Tag not found".to_string(), - ))? - .pub_id; - - let delete_ops = db - .tag_on_object() - .find_many(vec![tag_on_object::tag_id::equals(tag_id)]) - .select(tag_on_object::select!({ object: select { pub_id } })) - .exec() - .await? - .into_iter() - .map(|tag_on_object| { - sync.relation_delete(prisma_sync::tag_on_object::SyncId { - tag: prisma_sync::tag::SyncId { - pub_id: tag_pub_id.clone(), - }, - object: prisma_sync::object::SyncId { - pub_id: tag_on_object.object.pub_id, - }, - }) - }) - .collect::>(); - - sync.write_ops( - db, - ( - delete_ops, - db.tag_on_object() - .delete_many(vec![tag_on_object::tag_id::equals(tag_id)]), - ), - ) - .await?; - - sync.write_op( - db, - sync.shared_delete(prisma_sync::tag::SyncId { pub_id: tag_pub_id }), - db.tag().delete(tag::id::equals(tag_id)), - ) - .await?; - - invalidate_query!(library, "tags.list"); - - Ok(()) - }), - ) -} diff --git a/core/src/api/utils/invalidate.rs b/core/src/api/utils/invalidate.rs deleted file mode 100644 index e888b08a2..000000000 --- a/core/src/api/utils/invalidate.rs +++ /dev/null @@ -1,431 +0,0 @@ -use crate::api::{CoreEvent, Ctx, Router, R}; - -use async_stream::stream; -use rspc::alpha::AlphaRouter; -use serde::Serialize; -use serde_hashkey::to_key; -use serde_json::Value; -use specta::{DataType, Type}; -use std::{ - collections::HashMap, - sync::{ - atomic::{AtomicBool, Ordering}, - Arc, - }, - time::Duration, -}; -use tokio::sync::broadcast; -use tracing::{debug, warn}; - -#[cfg(debug_assertions)] -use std::sync::Mutex; - -/// holds information about all invalidation queries done with the [`invalidate_query!`] macro so we can check they are valid when building the router. -#[cfg(debug_assertions)] -pub(crate) static INVALIDATION_REQUESTS: Mutex = - Mutex::new(InvalidRequests::new()); - -// fwi: This exists to keep the enum fields private. -#[derive(Debug, Clone, Serialize, Type)] -pub struct SingleInvalidateOperationEvent { - /// This fields are intentionally private. - pub key: &'static str, - arg: Value, - result: Option, -} - -#[derive(Debug, Clone, Serialize, Type)] -#[serde(tag = "type", content = "data", rename_all = "camelCase")] -pub enum InvalidateOperationEvent { - Single(SingleInvalidateOperationEvent), - // TODO: A temporary hack used with Brendan's sync system until the v2 invalidation system is implemented. - All, -} - -impl InvalidateOperationEvent { - /// If you are using this function, your doing it wrong. - pub fn dangerously_create(key: &'static str, arg: Value, result: Option) -> Self { - Self::Single(SingleInvalidateOperationEvent { key, arg, result }) - } - - pub fn all() -> Self { - Self::All - } -} - -/// a request to invalidate a specific resource -#[derive(Debug)] -pub(crate) struct InvalidationRequest { - pub key: &'static str, - pub arg_ty: Option, - pub result_ty: Option, - pub macro_src: &'static str, -} - -/// invalidation request for a specific resource -#[derive(Debug, Default)] -pub(crate) struct InvalidRequests { - pub queries: Vec, -} - -impl InvalidRequests { - const fn new() -> Self { - Self { - queries: Vec::new(), - } - } - - #[allow(unused_variables, clippy::panic)] - pub(crate) fn validate(r: Arc) { - #[cfg(debug_assertions)] - { - let invalidate_requests = INVALIDATION_REQUESTS - .lock() - .expect("Failed to lock the mutex for invalidation requests"); - - let queries = r.queries(); - for req in &invalidate_requests.queries { - // This is a subscription in Rust but is query in React where it needs revalidation. - // We also don't check it's arguments are valid because we can't, lol. - if req.key == "search.ephemeralPaths" { - continue; - } - - if let Some(query_ty) = queries.get(req.key) { - if let Some(arg) = &req.arg_ty { - if &query_ty.ty.input != arg { - panic!( - "Error at '{}': Attempted to invalid query '{}' but the argument type does not match the type defined on the router.", - req.macro_src, req.key - ); - } - } - - if let Some(result) = &req.result_ty { - if &query_ty.ty.result != result { - panic!( - "Error at '{}': Attempted to invalid query '{}' but the data type does not match the type defined on the router.", - req.macro_src, req.key - ); - } - } - } else { - panic!( - "Error at '{}': Attempted to invalid query '{}' which was not found in the router", - req.macro_src, req.key - ); - } - } - } - } -} - -/// `invalidate_query` is a macro which stores a list of all of it's invocations so it can ensure all of the queries match the queries attached to the router. -/// -/// This allows invalidate to be type-safe even when the router keys are stringly typed. -/// ```ignore -/// invalidate_query!( -/// library, // crate::library::LibraryContext -/// "version": (), // Name of the query and the type of it -/// () // The arguments -/// ); -/// ``` -#[macro_export] -// #[allow(clippy::crate_in_macro_def)] -macro_rules! invalidate_query { - - ($ctx:expr, $query:ident) => {{ - let ctx: &$crate::library::Library = &$ctx; // Assert the context is the correct type - let query: &'static str = $query; - - ::tracing::trace!(target: "sd_core::invalidate-query", "invalidate_query!(\"{}\") at {}", query, concat!(file!(), ":", line!())); - - // The error are ignored here because they aren't mission critical. If they fail the UI might be outdated for a bit. - ctx.emit($crate::api::CoreEvent::InvalidateOperation( - $crate::api::utils::InvalidateOperationEvent::dangerously_create(query, serde_json::Value::Null, None) - )) - }}; - - ($ctx:expr, $key:literal) => {{ - let ctx: &$crate::library::Library = &$ctx; // Assert the context is the correct type - - #[cfg(debug_assertions)] - { - #[ctor::ctor] - fn invalidate() { - $crate::api::utils::INVALIDATION_REQUESTS - .lock() - .unwrap() - .queries - .push($crate::api::utils::InvalidationRequest { - key: $key, - arg_ty: None, - result_ty: None, - macro_src: concat!(file!(), ":", line!()), - }) - } - } - - ::tracing::trace!(target: "sd_core::invalidate-query", "invalidate_query!(\"{}\") at {}", $key, concat!(file!(), ":", line!())); - - // The error are ignored here because they aren't mission critical. If they fail the UI might be outdated for a bit. - ctx.emit($crate::api::CoreEvent::InvalidateOperation( - $crate::api::utils::InvalidateOperationEvent::dangerously_create($key, serde_json::Value::Null, None) - )) - }}; - (node; $ctx:expr, $key:literal) => {{ - let ctx: &$crate::Node = &$ctx; // Assert the context is the correct type - - #[cfg(debug_assertions)] - { - #[ctor::ctor] - fn invalidate() { - $crate::api::utils::INVALIDATION_REQUESTS - .lock() - .unwrap() - .queries - .push($crate::api::utils::InvalidationRequest { - key: $key, - arg_ty: None, - result_ty: None, - macro_src: concat!(file!(), ":", line!()), - }) - } - } - - ::tracing::trace!(target: "sd_core::invalidate-query", "invalidate_query!(\"{}\") at {}", $key, concat!(file!(), ":", line!())); - - // The error are ignored here because they aren't mission critical. If they fail the UI might be outdated for a bit. - ctx.event_bus.0.send($crate::api::CoreEvent::InvalidateOperation( - $crate::api::utils::InvalidateOperationEvent::dangerously_create($key, serde_json::Value::Null, None) - )).ok(); - }}; - ($ctx:expr, $key:literal: $arg_ty:ty, $arg:expr $(,)?) => {{ - let _: $arg_ty = $arg; // Assert the type the user provided is correct - let ctx: &$crate::library::Library = &$ctx; // Assert the context is the correct type - - #[cfg(debug_assertions)] - { - #[ctor::ctor] - fn invalidate() { - $crate::api::utils::INVALIDATION_REQUESTS - .lock() - .unwrap() - .queries - .push($crate::api::utils::InvalidationRequest { - key: $key, - arg_ty: <$arg_ty as specta::Type>::reference(specta::DefOpts { - parent_inline: false, - type_map: &mut specta::TypeDefs::new(), - }, &[]).map_err(|e| { - ::tracing::error!( - "Failed to get type reference for invalidate query '{}': {:?}", - $key, - e - ) - }).ok(), - result_ty: None, - macro_src: concat!(file!(), ":", line!()), - }) - } - } - - ::tracing::trace!(target: "sd_core::invalidate-query", "invalidate_query!(\"{}\") at {}", $key, concat!(file!(), ":", line!())); - - // The error are ignored here because they aren't mission critical. If they fail the UI might be outdated for a bit. - let _ = serde_json::to_value($arg) - .map(|v| - ctx.emit($crate::api::CoreEvent::InvalidateOperation( - $crate::api::utils::InvalidateOperationEvent::dangerously_create($key, v, None), - )) - ) - .map_err(|_| { - tracing::warn!("Failed to serialize invalidate query event!"); - }); - }}; - ($ctx:expr, $key:literal: $arg_ty:ty, $arg:expr, $result_ty:ty: $result:expr $(,)?) => {{ - let _: $arg_ty = $arg; // Assert the type the user provided is correct - let ctx: &$crate::library::Library = &$ctx; // Assert the context is the correct type - - #[cfg(debug_assertions)] - { - #[ctor::ctor] - fn invalidate() { - $crate::api::utils::INVALIDATION_REQUESTS - .lock() - .unwrap() - .queries - .push($crate::api::utils::InvalidationRequest { - key: $key, - arg_ty: Some(<$arg_ty as rspc::internal::specta::Type>::reference(rspc::internal::specta::DefOpts { - parent_inline: false, - type_map: &mut rspc::internal::specta::TypeDefs::new(), - }, &[])), - result_ty: Some(<$result_ty as rspc::internal::specta::Type>::reference(rspc::internal::specta::DefOpts { - parent_inline: false, - type_map: &mut rspc::internal::specta::TypeDefs::new(), - }, &[])), - macro_src: concat!(file!(), ":", line!()), - }) - } - } - - ::tracing::trace!(target: "sd_core::invalidate-query", "invalidate_query!(\"{}\") at {}", $key, concat!(file!(), ":", line!())); - - // The error are ignored here because they aren't mission critical. If they fail the UI might be outdated for a bit. - let _ = serde_json::to_value($arg) - .and_then(|arg| - serde_json::to_value($result) - .map(|result| - ctx.emit($crate::api::CoreEvent::InvalidateOperation( - $crate::api::utils::InvalidateOperationEvent::dangerously_create($key, arg, Some(result)), - )) - ) - ) - .map_err(|_| { - tracing::warn!("Failed to serialize invalidate query event!"); - }); - }}; -} - -pub(crate) fn mount_invalidate() -> AlphaRouter { - let (tx, _) = broadcast::channel(100); - let manager_thread_active = Arc::new(AtomicBool::new(false)); - - // TODO: Scope the invalidate queries to a specific library (filtered server side) - let r = if cfg!(debug_assertions) { - let count = Arc::new(std::sync::atomic::AtomicU16::new(0)); - - R.router() - .procedure( - "test-invalidate", - R.query(move |_, _: ()| Ok(count.fetch_add(1, Ordering::SeqCst))), - ) - .procedure( - "test-invalidate-mutation", - R.with2(super::library()).mutation(|(_, library), _: ()| { - invalidate_query!(library, "invalidation.test-invalidate"); - Ok(()) - }), - ) - } else { - R.router() - }; - - r.procedure("listen", { - R.subscription(move |ctx, _: ()| { - // This thread is used to deal with batching and deduplication. - // Their is only ever one of these management threads per Node but we spawn it like this so we can steal the event bus from the rspc context. - // Batching is important because when refetching data on the frontend rspc can fetch all invalidated queries in a single round trip. - if !manager_thread_active.swap(true, Ordering::Relaxed) { - let mut event_bus_rx = ctx.event_bus.0.subscribe(); - let tx = tx.clone(); - let manager_thread_active = manager_thread_active.clone(); - - tokio::spawn(async move { - loop { - let Ok(CoreEvent::InvalidateOperation(first_event)) = - event_bus_rx.recv().await - else { - continue; - }; - - let mut buf = - match &first_event { - InvalidateOperationEvent::All => None, - InvalidateOperationEvent::Single( - SingleInvalidateOperationEvent { key, arg, .. }, - ) => { - let key = match to_key(&(key, arg)) { - Ok(key) => key, - Err(e) => { - warn!( - ?first_event, - ?e, - "Error deriving key for invalidate operation;" - ); - continue; - } - }; - - let mut map = HashMap::with_capacity(20); - map.insert(key, first_event); - - Some(map) - } - }; - let batch_time = tokio::time::Instant::now() + Duration::from_millis(10); - - loop { - tokio::select! { - _ = tokio::time::sleep_until(batch_time) => { - break; - } - event = event_bus_rx.recv() => { - let Ok(event) = event else { - warn!( - "Shutting down invalidation manager thread \ - due to the core event bus being dropped!" - ); - break; - }; - - let CoreEvent::InvalidateOperation(op) = event else { continue; }; - - match (&op, &mut buf) { - (InvalidateOperationEvent::All, Some(_)) => buf = None, - (InvalidateOperationEvent::Single(SingleInvalidateOperationEvent { key, arg, .. }), Some(buf)) => { - // Newer data replaces older data in the buffer - match to_key(&(key, &arg)) { - Ok(key) => { - buf.insert(key, op); - }, - Err(e) => { - warn!( - ?op, - ?e, - "Error deriving key for invalidate operation;", - ); - }, - } - }, - _ => {} - } - }, - } - } - - let events = match buf { - None => vec![InvalidateOperationEvent::all()], - Some(buf) => buf.into_values().collect::>(), - }; - - if events.is_empty() { - break; - } - - match tx.send(events) { - Ok(_) => {} - // All receivers are shutdown means that all clients are disconnected. - Err(_) => { - debug!( - "Shutting down invalidation manager! \ - This is normal if all clients disconnects." - ); - manager_thread_active.swap(false, Ordering::Relaxed); - break; - } - } - } - }); - } - - let mut rx = tx.subscribe(); - stream! { - while let Ok(msg) = rx.recv().await { - yield msg; - } - } - }) - }) -} diff --git a/core/src/api/utils/library.rs b/core/src/api/utils/library.rs deleted file mode 100644 index 001943f55..000000000 --- a/core/src/api/utils/library.rs +++ /dev/null @@ -1,53 +0,0 @@ -use std::sync::Arc; - -use rspc::{ - alpha::{ - unstable::{MwArgMapper, MwArgMapperMiddleware}, - MwV3, - }, - ErrorCode, -}; -use serde::{de::DeserializeOwned, Deserialize, Serialize}; -use specta::Type; -use uuid::Uuid; - -use crate::{api::Ctx, library::Library}; - -/// Can wrap a query argument to require it to contain a `library_id` and provide helpers for working with libraries. -#[derive(Clone, Serialize, Deserialize, Type)] -pub(crate) struct LibraryArgs { - library_id: Uuid, - arg: T, -} - -pub(crate) struct LibraryArgsLike; -impl MwArgMapper for LibraryArgsLike { - type Input - = LibraryArgs - where - T: Type + DeserializeOwned + 'static; - type State = Uuid; - - fn map( - arg: Self::Input, - ) -> (T, Self::State) { - (arg.arg, arg.library_id) - } -} - -pub(crate) fn library() -> impl MwV3)> { - MwArgMapperMiddleware::::new().mount(|mw, ctx: Ctx, library_id| async move { - let library = ctx - .libraries - .get_library(&library_id) - .await - .ok_or_else(|| { - rspc::Error::new( - ErrorCode::BadRequest, - "You must specify a valid library to use this operation.".to_string(), - ) - })?; - - Ok(mw.next((ctx, library))) - }) -} diff --git a/core/src/api/utils/mod.rs b/core/src/api/utils/mod.rs deleted file mode 100644 index b12c1ec7e..000000000 --- a/core/src/api/utils/mod.rs +++ /dev/null @@ -1,109 +0,0 @@ -use std::path::Path; - -// #[cfg(not(any(target_os = "ios", target_os = "android")))] -// use keyring::Entry; - -use tokio::{fs, io}; - -mod invalidate; -mod library; - -pub use invalidate::*; -pub(crate) use library::*; - -/// Returns the size of the file or directory -pub async fn get_size(path: impl AsRef) -> Result { - let path = path.as_ref(); - let metadata = fs::metadata(path).await?; - - if metadata.is_dir() { - let mut result = 0; - let mut to_walk = vec![path.to_path_buf()]; - - while let Some(path) = to_walk.pop() { - let mut read_dir = fs::read_dir(&path).await?; - - while let Some(entry) = read_dir.next_entry().await? { - let metadata = entry.metadata().await?; - if metadata.is_dir() { - to_walk.push(entry.path()) - } else { - result += metadata.len() - } - } - } - - Ok(result) - } else { - Ok(metadata.len()) - } -} - -// pub fn get_access_token() -> Result { -// // If target is ios or android, return an error as this function is not supported on those platforms -// if cfg!(any(target_os = "ios", target_os = "android")) { -// return Err(rspc::Error::new( -// rspc::ErrorCode::InternalServerError, -// "Function not supported on this platform".to_string(), -// )); -// } else { -// let username = whoami::username(); -// let entry = match Entry::new("spacedrive-auth-service", username.as_str()) { -// Ok(entry) => entry, -// Err(e) => { -// error!("Error creating entry: {}", e); -// return Err(rspc::Error::new( -// rspc::ErrorCode::InternalServerError, -// "Error creating entry".to_string(), -// )); -// } -// }; - -// let data = match entry.get_password() { -// Ok(key) => key, -// Err(e) => { -// error!("Error retrieving key: {}. Does the key exist yet?", e); -// return Err(rspc::Error::new( -// rspc::ErrorCode::InternalServerError, -// "Error retrieving key".to_string(), -// )); -// } -// }; - -// let re = match Regex::new(r#"st-access-token=([^;]+)"#) { -// Ok(re) => re, -// Err(e) => { -// error!("Error creating regex: {}", e); -// return Err(rspc::Error::new( -// rspc::ErrorCode::InternalServerError, -// "Error creating regex".to_string(), -// )); -// } -// }; - -// let token = match re.captures(&data) { -// Some(captures) => match captures.get(1) { -// Some(token) => token.as_str(), -// None => { -// error!("Error parsing Cookie String value: {}", "No token found"); -// return Err(rspc::Error::new( -// rspc::ErrorCode::InternalServerError, -// "Error parsing Cookie String value".to_string(), -// )); -// } -// }, -// None => { -// error!( -// "Error parsing Cookie String value: {}", -// "No token cookie string found" -// ); -// return Err(rspc::Error::new( -// rspc::ErrorCode::InternalServerError, -// "Error parsing Cookie String value".to_string(), -// )); -// } -// }; - -// Ok(token.to_string()) -// } -// } diff --git a/core/src/api/volumes.rs b/core/src/api/volumes.rs deleted file mode 100644 index 4ecb14dc3..000000000 --- a/core/src/api/volumes.rs +++ /dev/null @@ -1,82 +0,0 @@ -use super::{utils::library, Ctx, R}; -use crate::volume::{VolumeEvent, VolumeFingerprint}; -use rspc::alpha::AlphaRouter; -use serde::Deserialize; -use specta::Type; - -pub(crate) fn mount() -> AlphaRouter { - R.router() - .procedure( - "list", - R.with2(library()) - .query(|(node, library), _: ()| async move { - match node.volumes.list_system_volumes(library).await { - Ok(volumes) => Ok(volumes), - Err(e) => { - tracing::error!("Error listing volumes: {:?}", e); - Err(e.into()) - } - } - }), - ) - .procedure( - "track", - R.with2(library()).mutation( - |(node, library), fingerprint: VolumeFingerprint| async move { - tracing::debug!( - "Handling track volume request for volume_id={:?}", - fingerprint - ); - - node.volumes - .track_volume(fingerprint, library) - .await - .map_err(|e| { - tracing::error!("Failed to track volume: {:?}", e); - e.into() - }) - }, - ), - ) - .procedure( - "listForLibrary", - R.with2(library()) - .query(|(node, library), _: ()| async move { - node.volumes - .list_library_volumes(library) - .await - .map_err(Into::into) - }), - ) - // .procedure( - // "listByDevice", - // R.with2(library()) - // .query(|(node, library), _: ()| async move { - // node.volumes - // .list_by_device(library) - // .await - // .map_err(Into::into) - // }), - // ) - .procedure( - "unmount", - R.with2(library()) - .mutation(|(node, _), fingerprint: Vec| async move { - node.volumes - .unmount_volume(VolumeFingerprint(fingerprint).into()) - .await - .map_err(Into::into) - }), - ) - .procedure("events", { - R.with2(library()).subscription(|(node, library), _: ()| { - Ok(async_stream::stream! { - let mut event_bus_rx = node.volumes.subscribe(); - - while let Ok(event) = event_bus_rx.recv().await { - yield event; - } - }) - }) - }) -} diff --git a/core/src/api/web_api.rs b/core/src/api/web_api.rs deleted file mode 100644 index 81cb410d3..000000000 --- a/core/src/api/web_api.rs +++ /dev/null @@ -1,29 +0,0 @@ -use rspc::alpha::AlphaRouter; -use serde::{Deserialize, Serialize}; -use specta::Type; - -use super::{Ctx, R}; - -pub(crate) fn mount() -> AlphaRouter { - R.router().procedure( - "sendFeedback", - R.mutation({ - #[derive(Debug, Type, Serialize, Deserialize)] - struct Feedback { - message: String, - emoji: u8, - } - - |_node, _args: Feedback| async move { - // sd_cloud_api::feedback::send( - // node.cloud_api_config().await, - // args.message, - // args.emoji, - // ) - // .await?; - - Ok(()) - } - }), - ) -} diff --git a/core-new/src/bin/cli.rs b/core/src/bin/cli.rs similarity index 100% rename from core-new/src/bin/cli.rs rename to core/src/bin/cli.rs diff --git a/core-new/src/config/app_config.rs b/core/src/config/app_config.rs similarity index 100% rename from core-new/src/config/app_config.rs rename to core/src/config/app_config.rs diff --git a/core-new/src/config/migration.rs b/core/src/config/migration.rs similarity index 100% rename from core-new/src/config/migration.rs rename to core/src/config/migration.rs diff --git a/core-new/src/config/mod.rs b/core/src/config/mod.rs similarity index 100% rename from core-new/src/config/mod.rs rename to core/src/config/mod.rs diff --git a/core/src/context.rs b/core/src/context.rs index 519034d30..bb656e2a0 100644 --- a/core/src/context.rs +++ b/core/src/context.rs @@ -1,231 +1,78 @@ -use crate::{api::CoreEvent, invalidate_query, library::Library, old_job::JobProgressEvent, Node}; +//! Shared context providing access to core application components. -use sd_core_heavy_lifting::{ - job_system::report::{Report, Status}, - OuterContext, ProgressUpdate, UpdateEvent, +//! Shared context providing access to core application components. + +use crate::{ + config::JobLoggingConfig, + device::DeviceManager, infrastructure::events::EventBus, + keys::library_key_manager::LibraryKeyManager, library::LibraryManager, + infrastructure::actions::manager::ActionManager, + services::networking::NetworkingService, volume::VolumeManager, }; -use sd_core_sync::SyncManager; - -use std::{ - ops::{Deref, DerefMut}, - sync::{ - atomic::{AtomicU8, Ordering}, - Arc, - }, -}; - -use chrono::{DateTime, Utc}; -use tokio::{spawn, sync::RwLock}; -use tracing::{error, trace}; -use uuid::Uuid; +use std::{path::PathBuf, sync::Arc}; +use tokio::sync::RwLock; +/// Shared context providing access to core application components. #[derive(Clone)] -pub struct NodeContext { - pub node: Arc, - pub library: Arc, +pub struct CoreContext { + pub events: Arc, + pub device_manager: Arc, + pub library_manager: Arc, + pub volume_manager: Arc, + pub library_key_manager: Arc, + // This is wrapped in an RwLock to allow it to be set after initialization + pub action_manager: Arc>>>, + pub networking: Arc>>>, + // Job logging configuration + pub job_logging_config: Option, + pub job_logs_dir: Option, } -pub trait NodeContextExt: sealed::Sealed { - fn library(&self) -> &Arc; -} - -mod sealed { - pub trait Sealed {} -} - -impl sealed::Sealed for NodeContext {} - -impl NodeContextExt for NodeContext { - fn library(&self) -> &Arc { - &self.library - } -} - -impl OuterContext for NodeContext { - fn id(&self) -> Uuid { - self.library.id - } - - fn db(&self) -> &Arc { - &self.library.db - } - - fn sync(&self) -> &SyncManager { - &self.library.sync - } - - fn invalidate_query(&self, query: &'static str) { - invalidate_query!(self.library, query) - } - - fn query_invalidator(&self) -> impl Fn(&'static str) + Send + Sync { - |query| { - invalidate_query!(self.library, query); - } - } - - fn report_update(&self, update: UpdateEvent) { - // FIX-ME: Remove this conversion once we have a proper atomic updates system - let event = match update { - UpdateEvent::NewThumbnail { thumb_key } => CoreEvent::NewThumbnail { thumb_key }, - UpdateEvent::NewIdentifiedObjects { file_path_ids } => { - CoreEvent::NewIdentifiedObjects { file_path_ids } - } - }; - self.node.emit(event); - } - - fn get_data_directory(&self) -> &std::path::Path { - &self.node.data_dir - } -} - -#[derive(Clone)] -pub struct JobContext { - outer_ctx: OuterCtx, - report: Arc>, - start_time: DateTime, - report_update_counter: Arc, -} - -impl OuterContext for JobContext { - fn id(&self) -> Uuid { - self.outer_ctx.id() - } - - fn db(&self) -> &Arc { - self.outer_ctx.db() - } - - fn sync(&self) -> &SyncManager { - self.outer_ctx.sync() - } - - fn invalidate_query(&self, query: &'static str) { - self.outer_ctx.invalidate_query(query); - } - - fn query_invalidator(&self) -> impl Fn(&'static str) + Send + Sync { - self.outer_ctx.query_invalidator() - } - - fn report_update(&self, update: UpdateEvent) { - self.outer_ctx.report_update(update); - } - - fn get_data_directory(&self) -> &std::path::Path { - self.outer_ctx.get_data_directory() - } -} - -impl sd_core_heavy_lifting::JobContext - for JobContext -{ - fn new(report: Report, outer_ctx: OuterCtx) -> Self { +impl CoreContext { + /// Create a new context with the given components + pub fn new( + events: Arc, + device_manager: Arc, + library_manager: Arc, + volume_manager: Arc, + library_key_manager: Arc, + ) -> Self { Self { - report: Arc::new(RwLock::new(report)), - outer_ctx, - start_time: Utc::now(), - report_update_counter: Arc::new(AtomicU8::new(0)), + events, + device_manager, + library_manager, + volume_manager, + library_key_manager, + action_manager: Arc::new(RwLock::new(None)), + networking: Arc::new(RwLock::new(None)), + job_logging_config: None, + job_logs_dir: None, } } - - async fn progress(&self, updates: impl IntoIterator + Send) { - let mut report = self.report.write().await; - - // protect against updates if job is not running - if report.status != Status::Running { - return; - }; - - let mut changed_phase = false; - - for update in updates { - match update { - ProgressUpdate::TaskCount(task_count) => { - report.task_count = task_count as i32; - } - ProgressUpdate::CompletedTaskCount(completed_task_count) => { - report.completed_task_count = completed_task_count as i32; - } - - ProgressUpdate::Message(message) => { - trace!(job_id = %report.id, %message, "job message;"); - report.message = message; - } - ProgressUpdate::Phase(phase) => { - trace!( - job_id = %report.id, - "changing phase: {} -> {phase};", - report.phase - ); - report.phase = phase; - changed_phase = true; - } - } - } - - // Calculate elapsed time - let elapsed = Utc::now() - self.start_time; - - // Calculate remaining time - let task_count = report.task_count as usize; - let completed_task_count = report.completed_task_count as usize; - let remaining_task_count = task_count.saturating_sub(completed_task_count); - - // Adding 1 to avoid division by zero - let remaining_time_per_task = elapsed / (completed_task_count + 1) as i32; - - let remaining_time = remaining_time_per_task * remaining_task_count as i32; - - // Update the report with estimated remaining time - report.estimated_completion = Utc::now() - .checked_add_signed(remaining_time) - .unwrap_or(Utc::now()); - - let library = self.outer_ctx.library(); - - let counter = self.report_update_counter.fetch_add(1, Ordering::AcqRel); - - if counter == 50 || counter == 0 || changed_phase { - self.report_update_counter.store(1, Ordering::Release); - - spawn({ - let db = Arc::clone(&library.db); - let report = report.clone(); - async move { - if let Err(e) = report.update(&db).await { - error!( - ?e, - "Failed to update job report on debounced job progress event;" - ); - } - } - }); - } - - // emit a CoreEvent - library.emit(CoreEvent::JobProgress(JobProgressEvent { - id: report.id, - library_id: library.id, - task_count: report.task_count, - completed_task_count: report.completed_task_count, - info: report.info.clone(), - estimated_completion: report.estimated_completion, - phase: report.phase.clone(), - message: report.message.clone(), - })); + + /// Set job logging configuration + pub fn set_job_logging(&mut self, config: JobLoggingConfig, logs_dir: PathBuf) { + self.job_logging_config = Some(config); + self.job_logs_dir = Some(logs_dir); } - async fn report(&self) -> impl Deref { - Arc::clone(&self.report).read_owned().await + /// Helper method for services to get the networking service + pub async fn get_networking(&self) -> Option> { + self.networking.read().await.clone() } - async fn report_mut(&self) -> impl DerefMut { - Arc::clone(&self.report).write_owned().await + /// Method for Core to set networking after it's initialized + pub async fn set_networking(&self, networking: Arc) { + *self.networking.write().await = Some(networking); } - fn get_outer_ctx(&self) -> OuterCtx { - self.outer_ctx.clone() + /// Helper method to get the action manager + pub async fn get_action_manager(&self) -> Option> { + self.action_manager.read().await.clone() + } + + /// Method for Core to set action manager after it's initialized + pub async fn set_action_manager(&self, action_manager: Arc) { + *self.action_manager.write().await = Some(action_manager); } } diff --git a/core/src/custom_uri/mod.rs b/core/src/custom_uri/mod.rs deleted file mode 100644 index 531e4345e..000000000 --- a/core/src/custom_uri/mod.rs +++ /dev/null @@ -1,592 +0,0 @@ -use crate::{ - api::{utils::InvalidateOperationEvent, CoreEvent}, - library::Library, - old_p2p::operations::{self, request_file}, - util::InfallibleResponse, - Node, -}; - -use sd_core_file_path_helper::IsolatedFilePathData; -use sd_core_heavy_lifting::media_processor::WEBP_EXTENSION; -use sd_core_prisma_helpers::file_path_to_handle_custom_uri; - -use sd_file_ext::text::is_text; -use sd_old_p2p::{RemoteIdentity, P2P}; -use sd_old_p2p_block::Range; -use sd_prisma::prisma::{file_path, location}; -use sd_utils::db::maybe_missing; -use tokio_util::sync::PollSender; - -use std::{ - cmp::min, - ffi::OsStr, - fmt::Debug, - fs::Metadata, - path::{Path, PathBuf}, - str::FromStr, - sync::Arc, -}; - -use async_stream::stream; -use axum::{ - body::Body, - extract::{self, State}, - http::{HeaderMap, HeaderValue, Request, Response, StatusCode}, - middleware, - response::IntoResponse, - routing::get, - Router, -}; -use bytes::Bytes; -use hyper::{header, upgrade::OnUpgrade}; -use hyper_util::rt::TokioIo; -use mini_moka::sync::Cache; -use tokio::{ - fs::{self, File}, - io::{self, copy_bidirectional, AsyncReadExt, AsyncSeekExt, SeekFrom}, -}; -use tracing::{error, warn}; -use uuid::Uuid; - -use self::{serve_file::serve_file, utils::*}; - -mod mpsc_to_async_write; -mod serve_file; -mod utils; - -use mpsc_to_async_write::MpscToAsyncWrite; - -type CacheKey = (Uuid, file_path::id::Type); - -#[derive(Debug, Clone)] -struct CacheValue { - name: PathBuf, - ext: String, - file_path_pub_id: Uuid, - serve_from: ServeFrom, -} - -const MAX_TEXT_READ_LENGTH: usize = 10 * 1024; // 10KB - -#[derive(Debug, Clone)] -pub enum ServeFrom { - /// Serve from the local filesystem - Local, - /// Serve from a specific instance - Remote { - library_identity: Box, - node_identity: Box, - library: Arc, - }, -} - -#[derive(Clone)] -pub struct LocalState { - node: Arc, - - // This LRU cache allows us to avoid doing a DB lookup on every request. - // The main advantage of this LRU Cache is for video files. Video files are fetch in multiple chunks and the cache prevents a DB lookup on every chunk reducing the request time from 15-25ms to 1-10ms. - // TODO: We should listen to events when deleting or moving a location and evict the cache accordingly. - file_metadata_cache: Arc>, -} - -type ExtractedPath = extract::Path<(String, String, String)>; - -async fn request_to_remote_node( - p2p: Arc, - identity: RemoteIdentity, - mut request: Request, -) -> Response { - let request_upgrade_header = request.headers().get(header::UPGRADE).cloned(); - let maybe_client_upgrade = request.extensions_mut().remove::(); - - let mut response = match operations::remote_rspc(p2p.clone(), identity, request).await { - Ok(v) => v, - Err(e) => { - warn!(%identity, ?e, "Error doing remote rspc query with;"); - return StatusCode::BAD_GATEWAY.into_response(); - } - }; - if response.status() == StatusCode::SWITCHING_PROTOCOLS { - if response.headers().get(header::UPGRADE) != request_upgrade_header.as_ref() { - return StatusCode::BAD_REQUEST.into_response(); - } - - let Some(request_upgraded) = maybe_client_upgrade else { - return StatusCode::BAD_REQUEST.into_response(); - }; - let Some(response_upgraded) = response.extensions_mut().remove::() else { - return StatusCode::BAD_REQUEST.into_response(); - }; - - tokio::spawn(async move { - let Ok(request_upgraded) = request_upgraded.await.map_err(|e| { - warn!(?e, "Error upgrading websocket request;"); - }) else { - return; - }; - let Ok(response_upgraded) = response_upgraded.await.map_err(|e| { - warn!(?e, "Error upgrading websocket response;"); - }) else { - return; - }; - - let mut request_upgraded = TokioIo::new(request_upgraded); - let mut response_upgraded = TokioIo::new(response_upgraded); - - copy_bidirectional(&mut request_upgraded, &mut response_upgraded) - .await - .map_err(|e| { - warn!(?e, "Error upgrading websocket response;"); - }) - .ok(); - }); - } - - response.into_response() -} - -async fn get_or_init_lru_entry( - state: &LocalState, - extract::Path((lib_id, loc_id, path_id)): ExtractedPath, -) -> Result<(CacheValue, Arc), Response> { - let library_id = Uuid::from_str(&lib_id).map_err(bad_request)?; - let location_id = loc_id.parse::().map_err(bad_request)?; - let file_path_id = path_id - .parse::() - .map_err(bad_request)?; - - let lru_cache_key = (library_id, file_path_id); - let library = state - .node - .libraries - .get_library(&library_id) - .await - .ok_or_else(|| internal_server_error(()))?; - - if let Some(entry) = state.file_metadata_cache.get(&lru_cache_key) { - Ok((entry, library)) - } else { - let file_path = library - .db - .file_path() - .find_unique(file_path::id::equals(file_path_id)) - // TODO: This query could be seen as a security issue as it could load the private key (`identity`) when we 100% don't need it. We are gonna wanna fix that! - .select(file_path_to_handle_custom_uri::select()) - .exec() - .await - .map_err(internal_server_error)? - .ok_or_else(|| not_found(()))?; - - let location = maybe_missing(&file_path.location, "file_path.location") - .map_err(internal_server_error)?; - let path = maybe_missing(&location.path, "file_path.location.path") - .map_err(internal_server_error)?; - let instance = maybe_missing(&location.instance, "file_path.location.instance") - .map_err(internal_server_error)?; - - let path = Path::new(path) - .join(IsolatedFilePathData::try_from((location_id, &file_path)).map_err(not_found)?); - - let library_identity = - RemoteIdentity::from_bytes(&instance.remote_identity).map_err(internal_server_error)?; - - let node_identity = RemoteIdentity::from_bytes( - instance - .node_remote_identity - .as_ref() - .expect("node_remote_identity is required"), - ) - .map_err(internal_server_error)?; - - let lru_entry = CacheValue { - name: path, - ext: maybe_missing(file_path.extension, "extension").map_err(not_found)?, - file_path_pub_id: Uuid::from_slice(&file_path.pub_id).map_err(internal_server_error)?, - serve_from: if library_identity == library.identity.to_remote_identity() { - ServeFrom::Local - } else { - ServeFrom::Remote { - library_identity: Box::new(library_identity), - node_identity: Box::new(node_identity), - library: library.clone(), - } - }, - }; - - state - .file_metadata_cache - .insert(lru_cache_key, lru_entry.clone()); - - Ok((lru_entry, library)) - } -} - -pub fn base_router() -> Router { - Router::new() - .route( - "/thumbnail/*path", - get( - |State(state): State, - extract::Path(path): extract::Path, - request: Request| async move { - let thumbnail_path = state.node.config.data_directory().join("thumbnails"); - let path = thumbnail_path.join(path); - - // Prevent directory traversal attacks (Eg. requesting `../../../etc/passwd`) - // For now we only support `webp` thumbnails. - (path.starts_with(&thumbnail_path) - && path.extension() == Some(WEBP_EXTENSION.as_ref())) - .then_some(()) - .ok_or_else(|| not_found(()))?; - - let file = File::open(&path).await.map_err(|e| { - InfallibleResponse::builder() - .status(if e.kind() == io::ErrorKind::NotFound { - StatusCode::NOT_FOUND - } else { - StatusCode::INTERNAL_SERVER_ERROR - }) - .body(Body::from("")) - })?; - let metadata = file.metadata().await; - serve_file( - file, - metadata, - request.into_parts().0, - InfallibleResponse::builder() - .header("Content-Type", HeaderValue::from_static("image/webp")), - ) - .await - }, - ), - ) - .route( - "/file/:lib_id/:loc_id/:path_id", - get( - |State(state): State, path: ExtractedPath, request: Request| async move { - let ( - CacheValue { - name: file_path_full_path, - ext: extension, - file_path_pub_id, - serve_from, - .. - }, - _library, - ) = get_or_init_lru_entry(&state, path).await?; - - match serve_from { - ServeFrom::Local => { - let metadata = fs::metadata(&file_path_full_path) - .await - .map_err(internal_server_error)?; - (!metadata.is_dir()) - .then_some(()) - .ok_or_else(|| not_found(()))?; - - let mut file = File::open(&file_path_full_path).await.map_err(|e| { - InfallibleResponse::builder() - .status(if e.kind() == io::ErrorKind::NotFound { - StatusCode::NOT_FOUND - } else { - StatusCode::INTERNAL_SERVER_ERROR - }) - .body(Body::from("")) - })?; - - let resp = InfallibleResponse::builder().header( - "Content-Type", - HeaderValue::from_str( - &infer_the_mime_type(&extension, &mut file, &metadata).await?, - ) - .map_err(|e| { - error!(?e, "Error converting mime-type into header value;"); - internal_server_error(()) - })?, - ); - - serve_file(file, Ok(metadata), request.into_parts().0, resp).await - } - ServeFrom::Remote { - library_identity: _, - node_identity, - library, - } => { - // TODO: Support `Range` requests and `ETag` headers - - let (tx, mut rx) = tokio::sync::mpsc::channel::>(150); - request_file( - state.node.p2p.p2p.clone(), - *node_identity, - &library.identity, - file_path_pub_id, - Range::Full, - MpscToAsyncWrite::new(PollSender::new(tx)), - ) - .await - .map_err(|e| { - error!( - %file_path_pub_id, - node_identity = ?library.identity.to_remote_identity(), - ?e, - "Error requesting file from other node;", - ); - internal_server_error(()) - })?; - - // TODO: Content Type - Ok(InfallibleResponse::builder().status(StatusCode::OK).body( - Body::from_stream(stream! { - while let Some(item) = rx.recv().await { - yield item; - } - }), - )) - } - } - }, - ), - ) - .route( - "/local-file-by-path/:path", - get( - |extract::Path(path): extract::Path, request: Request| async move { - let path = PathBuf::from(path); - - let metadata = fs::metadata(&path).await.map_err(internal_server_error)?; - (!metadata.is_dir()) - .then_some(()) - .ok_or_else(|| not_found(()))?; - - let mut file = File::open(&path).await.map_err(|e| { - InfallibleResponse::builder() - .status(if e.kind() == io::ErrorKind::NotFound { - StatusCode::NOT_FOUND - } else { - StatusCode::INTERNAL_SERVER_ERROR - }) - .body(Body::from("")) - })?; - - let resp = InfallibleResponse::builder().header( - "Content-Type", - HeaderValue::from_str(&match path.extension().and_then(OsStr::to_str) { - None => "text/plain".to_string(), - Some(ext) => infer_the_mime_type(ext, &mut file, &metadata).await?, - }) - .map_err(|e| { - error!(?e, "Error converting mime-type into header value;"); - internal_server_error(()) - })?, - ); - - serve_file(file, Ok(metadata), request.into_parts().0, resp).await - }, - ), - ) -} - -pub fn with_state(node: Arc) -> LocalState { - let file_metadata_cache = Arc::new(Cache::new(150)); - - tokio::spawn({ - let file_metadata_cache = file_metadata_cache.clone(); - let mut tx = node.event_bus.0.subscribe(); - async move { - while let Ok(event) = tx.recv().await { - if let CoreEvent::InvalidateOperation(e) = event { - match e { - InvalidateOperationEvent::Single(event) => { - // TODO: This is inefficient as any change will invalidate who cache. We need the new invalidation system!!! - // TODO: It's also error prone and a fine-grained resource based invalidation system would avoid that. - if event.key == "search.objects" || event.key == "search.paths" { - file_metadata_cache.invalidate_all(); - } - } - InvalidateOperationEvent::All => { - file_metadata_cache.invalidate_all(); - } - } - } - } - } - }); - - LocalState { - node, - file_metadata_cache, - } -} - -// We are using Axum on all platforms because Tauri's custom URI protocols can't be async! -pub fn router(node: Arc) -> Router<()> { - Router::new() - .route( - "/remote/:identity/*path", - get( - |State(state): State, - extract::Path((identity, rest)): extract::Path<(String, String)>, - mut request: Request| async move { - let identity = match RemoteIdentity::from_str(&identity) { - Ok(identity) => identity, - Err(e) => { - warn!(%identity, ?e, "Error parsing identity;"); - return (StatusCode::BAD_REQUEST, HeaderMap::new(), vec![]) - .into_response(); - } - }; - - *request.uri_mut() = format!("/{rest}") - .parse() - .expect("url was validated by Axum"); - - request_to_remote_node(state.node.p2p.p2p.clone(), identity, request).await - }, - ), - ) - .merge(base_router()) - .route_layer(middleware::from_fn(cors_middleware)) - .with_state(with_state(node)) -} - -// TODO: This should possibly be determined from magic bytes when the file is indexed and stored it in the DB on the file path -async fn infer_the_mime_type( - ext: &str, - file: &mut File, - metadata: &Metadata, -) -> Result> { - let ext = ext.to_lowercase(); - let mime_type = match ext.as_str() { - // AAC audio - "aac" => "audio/aac", - // Musical Instrument Digital Interface (MIDI) - "mid" | "midi" => "audio/midi, audio/x-midi", - // MP3 audio - "mp3" => "audio/mpeg", - // MP4 audio - "m4a" => "audio/mp4", - // OGG audio - "oga" => "audio/ogg", - // Opus audio - "opus" => "audio/opus", - // Waveform Audio Format - "wav" => "audio/wav", - // WEBM audio - "weba" => "audio/webm", - // AVI: Audio Video Interleave - "avi" => "video/x-msvideo", - // MP4 video - "mp4" | "m4v" => "video/mp4", - // TODO: Bruh - #[cfg(not(target_os = "macos"))] - // TODO: Bruh - // FIX-ME: This media types break macOS video rendering - // MPEG transport stream - "ts" => "video/mp2t", - // TODO: Bruh - #[cfg(not(target_os = "macos"))] - // FIX-ME: This media types break macOS video rendering - // MPEG Video - "mpeg" => "video/mpeg", - // OGG video - "ogv" => "video/ogg", - // WEBM video - "webm" => "video/webm", - // 3GPP audio/video container (TODO: audio/3gpp if it doesn't contain video) - "3gp" => "video/3gpp", - // 3GPP2 audio/video container (TODO: audio/3gpp2 if it doesn't contain video) - "3g2" => "video/3gpp2", - // Quicktime movies - "mov" => "video/quicktime", - // Windows OS/2 Bitmap Graphics - "bmp" => "image/bmp", - // Graphics Interchange Format (GIF) - "gif" => "image/gif", - // Icon format - "ico" => "image/vnd.microsoft.icon", - // JPEG images - "jpeg" | "jpg" => "image/jpeg", - // Portable Network Graphics - "png" => "image/png", - // Scalable Vector Graphics (SVG) - "svg" => "image/svg+xml", - // Tagged Image File Format (TIFF) - "tif" | "tiff" => "image/tiff", - // WEBP image - "webp" => "image/webp", - // PDF document - "pdf" => "application/pdf", - // HEIF images - "heif" => "image/heif", - // HEIF images sequence (animated) - "heifs" => "image/heif-sequence", - // HEIC images - "heic" | "hif" => "image/heic", - // HEIC images sequence (animated) - "heics" => "image/heic-sequence", - // AV1 in HEIF images - "avif" => "image/avif", - // AV1 in HEIF images sequence (DEPRECATED: https://github.com/AOMediaCodec/av1-avif/pull/86/files) - "avifs" => "image/avif-sequence", - // AVC in HEIF images - "avci" => "image/avci", - // AVC in HEIF images sequence (animated) - "avcs" => "image/avcs", - _ => "text/plain", - }; - - Ok(if mime_type == "text/plain" { - let mut text_buf = vec![ - 0; - min( - metadata.len().try_into().unwrap_or(usize::MAX), - MAX_TEXT_READ_LENGTH - ) - ]; - if !text_buf.is_empty() { - file.read_exact(&mut text_buf) - .await - .map_err(internal_server_error)?; - file.seek(SeekFrom::Start(0)) - .await - .map_err(internal_server_error)?; - } - - let charset = is_text(&text_buf, text_buf.len() == (metadata.len() as usize)).unwrap_or(""); - - // Only browser recognized types, everything else should be text/plain - // https://www.iana.org/assignments/media-types/media-types.xhtml#table-text - let mime_type = match ext.as_str() { - // HyperText Markup Language - "html" | "htm" => "text/html", - // Cascading Style Sheets - "css" => "text/css", - // Javascript - "js" | "mjs" => "text/javascript", - // Comma-separated values - "csv" => "text/csv", - // Markdown - "md" | "markdown" => "text/markdown", - // Rich text format - "rtf" => "text/rtf", - // Web Video Text Tracks - "vtt" => "text/vtt", - // Extensible Markup Language - "xml" => "text/xml", - // Text - "txt" => "text/plain", - _ => { - if charset.is_empty() { - // "TODO: This filetype is not supported because of the missing mime type!", - return Err(not_implemented(())); - }; - mime_type - } - }; - - format!("{mime_type}; charset={charset}") - } else { - mime_type.to_string() - }) -} diff --git a/core/src/custom_uri/mpsc_to_async_write.rs b/core/src/custom_uri/mpsc_to_async_write.rs deleted file mode 100644 index 90bd764c1..000000000 --- a/core/src/custom_uri/mpsc_to_async_write.rs +++ /dev/null @@ -1,44 +0,0 @@ -use std::{ - pin::Pin, - task::{Context, Poll}, -}; - -use bytes::Bytes; -use tokio::io::{self, AsyncWrite}; -use tokio_util::sync::PollSender; - -/// Allowing wrapping an `mpsc::Sender` into an `AsyncWrite` -pub struct MpscToAsyncWrite(PollSender>); - -impl MpscToAsyncWrite { - #[allow(dead_code)] - pub fn new(sender: PollSender>) -> Self { - Self(sender) - } -} - -impl AsyncWrite for MpscToAsyncWrite { - fn poll_write( - mut self: Pin<&mut Self>, - cx: &mut Context<'_>, - buf: &[u8], - ) -> Poll> { - #[allow(clippy::unwrap_used)] - match self.0.poll_reserve(cx) { - Poll::Ready(Ok(())) => { - self.0.send_item(Ok(Bytes::from(buf.to_vec()))).unwrap(); - Poll::Ready(Ok(buf.len())) - } - Poll::Ready(Err(_)) => todo!(), - Poll::Pending => Poll::Pending, - } - } - - fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { - Poll::Ready(Ok(())) - } - - fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { - Poll::Ready(Ok(())) - } -} diff --git a/core/src/custom_uri/serve_file.rs b/core/src/custom_uri/serve_file.rs deleted file mode 100644 index 460684cf5..000000000 --- a/core/src/custom_uri/serve_file.rs +++ /dev/null @@ -1,150 +0,0 @@ -use crate::util::InfallibleResponse; - -use std::{fs::Metadata, time::UNIX_EPOCH}; - -use axum::{ - body::Body, - http::{header, request, HeaderValue, Method, Response, StatusCode}, -}; -use http_range::HttpRange; -use tokio::{ - fs::File, - io::{self, AsyncReadExt, AsyncSeekExt, SeekFrom}, -}; -use tokio_util::io::ReaderStream; -use tracing::error; - -use super::utils::*; - -// default capacity 64KiB -const DEFAULT_CAPACITY: usize = 65536; - -/// Serve a Tokio file as a HTTP response. -/// -/// This function takes care of: -/// - 304 Not Modified using ETag's -/// - Range requests for partial content -/// -/// BE AWARE this function does not do any path traversal protection so that's up to the caller! -pub(crate) async fn serve_file( - mut file: File, - metadata: io::Result, - req: request::Parts, - mut resp: InfallibleResponse, -) -> Result, Response> { - if let Ok(metadata) = metadata { - // We only accept range queries if `files.metadata() == Ok(_)` - // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Accept-Ranges - resp = resp - .header("Accept-Ranges", HeaderValue::from_static("bytes")) - .header( - "Content-Length", - HeaderValue::from_str(&metadata.len().to_string()) - .expect("number won't fail conversion"), - ); - - // Empty files - if metadata.len() == 0 { - return Ok(resp - .status(StatusCode::OK) - .header("Content-Length", HeaderValue::from_static("0")) - .body(Body::from(""))); - } - - // ETag - let mut status_code = StatusCode::PARTIAL_CONTENT; - if let Ok(time) = metadata.modified() { - let etag_header = - format!( - r#""{}""#, - // The ETag's can be any value so we just use the modified time to make it easy. - time.duration_since(UNIX_EPOCH) - .expect("are you a time traveler? cause that's the only explanation for this error") - .as_millis() - ); - - // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/ETag - if let Ok(etag_header) = HeaderValue::from_str(&etag_header) { - resp = resp.header("etag", etag_header); - } else { - error!("Failed to convert ETag into header value!"); - } - - // Used for normal requests - if let Some(etag) = req.headers.get("If-None-Match") { - if etag.as_bytes() == etag_header.as_bytes() { - return Ok(resp.status(StatusCode::NOT_MODIFIED).body(Body::from(""))); - } - } - - // Used checking if the resource has been modified since starting the download - if let Some(if_range) = req.headers.get("If-Range") { - // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/If-Range - if if_range.as_bytes() != etag_header.as_bytes() { - status_code = StatusCode::OK - } - } - }; - - // https://developer.mozilla.org/en-US/docs/Web/HTTP/Range_requests - if req.method == Method::GET { - if let Some(range) = req.headers.get("range") { - // TODO: Error handling - let ranges = HttpRange::parse(range.to_str().map_err(bad_request)?, metadata.len()) - .map_err(bad_request)?; - - // TODO: Multipart requests are not support, yet - if ranges.len() != 1 { - return Ok(resp - .header( - header::CONTENT_RANGE, - HeaderValue::from_str(&format!("bytes */{}", metadata.len())) - .map_err(internal_server_error)?, - ) - .status(StatusCode::RANGE_NOT_SATISFIABLE) - .body(Body::from(""))); - } - let range = ranges.first().expect("checked above"); - - if (range.start + range.length) > metadata.len() { - return Ok(resp - .header( - header::CONTENT_RANGE, - HeaderValue::from_str(&format!("bytes */{}", metadata.len())) - .map_err(internal_server_error)?, - ) - .status(StatusCode::RANGE_NOT_SATISFIABLE) - .body(Body::from(""))); - } - - file.seek(SeekFrom::Start(range.start)) - .await - .map_err(internal_server_error)?; - - return Ok(resp - .status(status_code) - .header( - "Content-Range", - HeaderValue::from_str(&format!( - "bytes {}-{}/{}", - range.start, - range.start + range.length - 1, - metadata.len() - )) - .map_err(internal_server_error)?, - ) - .header( - "Content-Length", - HeaderValue::from_str(&range.length.to_string()) - .map_err(internal_server_error)?, - ) - .body(Body::from_stream(ReaderStream::with_capacity( - file.take(range.length), - DEFAULT_CAPACITY, - )))); - } - } - } - - Ok(resp.body(Body::from_stream(ReaderStream::new(file)))) -} diff --git a/core/src/custom_uri/utils.rs b/core/src/custom_uri/utils.rs deleted file mode 100644 index cb54b815f..000000000 --- a/core/src/custom_uri/utils.rs +++ /dev/null @@ -1,84 +0,0 @@ -use crate::util::InfallibleResponse; - -use std::{fmt::Debug, panic::Location}; - -use axum::{ - body::Body, - http::{self, HeaderValue, Method, Request, Response, StatusCode}, - middleware::Next, -}; -use tracing::debug; - -#[track_caller] -pub(crate) fn bad_request(e: impl Debug) -> http::Response { - debug!(caller = %Location::caller(), ?e, "400: Bad Request;"); - - InfallibleResponse::builder() - .status(StatusCode::BAD_REQUEST) - .body(Body::from("")) -} - -#[track_caller] -pub(crate) fn not_found(e: impl Debug) -> http::Response { - debug!(caller = %Location::caller(), ?e, "404: Not Found;"); - - InfallibleResponse::builder() - .status(StatusCode::NOT_FOUND) - .body(Body::from("")) -} - -#[track_caller] -pub(crate) fn internal_server_error(e: impl Debug) -> http::Response { - debug!(caller = %Location::caller(), ?e, "500: Internal Server Error;"); - - InfallibleResponse::builder() - .status(StatusCode::INTERNAL_SERVER_ERROR) - .body(Body::from("")) -} - -#[track_caller] -pub(crate) fn not_implemented(e: impl Debug) -> http::Response { - debug!(caller = %Location::caller(), ?e, "501: Not Implemented;"); - - InfallibleResponse::builder() - .status(StatusCode::NOT_IMPLEMENTED) - .body(Body::from("")) -} - -pub(crate) async fn cors_middleware(req: Request, next: Next) -> Response { - if req.method() == Method::OPTIONS { - return Response::builder() - .header("Access-Control-Allow-Methods", "GET, HEAD, POST, OPTIONS") - .header("Access-Control-Allow-Origin", "*") - .header("Access-Control-Allow-Headers", "*") - .header("Access-Control-Max-Age", "86400") - .status(StatusCode::OK) - .body(Body::from("")) - .expect("Invalid static response!"); - } - - let is_upgrade_request = req.headers().get("Upgrade").is_some(); - - let mut response = next.run(req).await; - - { - let headers = response.headers_mut(); - - headers.insert("Access-Control-Allow-Origin", HeaderValue::from_static("*")); - - headers.insert( - "Access-Control-Allow-Headers", - HeaderValue::from_static("*"), - ); - - // With websocket requests, setting this causes the browser to loose it's shit. - if !is_upgrade_request { - // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Connection - headers.insert("Connection", HeaderValue::from_static("Keep-Alive")); - } - - headers.insert("Server", HeaderValue::from_static("Spacedrive")); - } - - response -} diff --git a/core-new/src/device/config.rs b/core/src/device/config.rs similarity index 100% rename from core-new/src/device/config.rs rename to core/src/device/config.rs diff --git a/core-new/src/device/manager.rs b/core/src/device/manager.rs similarity index 100% rename from core-new/src/device/manager.rs rename to core/src/device/manager.rs diff --git a/core-new/src/device/mod.rs b/core/src/device/mod.rs similarity index 100% rename from core-new/src/device/mod.rs rename to core/src/device/mod.rs diff --git a/core-new/src/domain/addressing.rs b/core/src/domain/addressing.rs similarity index 100% rename from core-new/src/domain/addressing.rs rename to core/src/domain/addressing.rs diff --git a/core-new/src/domain/content_identity.rs b/core/src/domain/content_identity.rs similarity index 100% rename from core-new/src/domain/content_identity.rs rename to core/src/domain/content_identity.rs diff --git a/core-new/src/domain/device.rs b/core/src/domain/device.rs similarity index 100% rename from core-new/src/domain/device.rs rename to core/src/domain/device.rs diff --git a/core-new/src/domain/entry.rs b/core/src/domain/entry.rs similarity index 100% rename from core-new/src/domain/entry.rs rename to core/src/domain/entry.rs diff --git a/core-new/src/domain/location.rs b/core/src/domain/location.rs similarity index 100% rename from core-new/src/domain/location.rs rename to core/src/domain/location.rs diff --git a/core-new/src/domain/mod.rs b/core/src/domain/mod.rs similarity index 100% rename from core-new/src/domain/mod.rs rename to core/src/domain/mod.rs diff --git a/core-new/src/domain/user_metadata.rs b/core/src/domain/user_metadata.rs similarity index 100% rename from core-new/src/domain/user_metadata.rs rename to core/src/domain/user_metadata.rs diff --git a/core-new/src/domain/volume.rs b/core/src/domain/volume.rs similarity index 100% rename from core-new/src/domain/volume.rs rename to core/src/domain/volume.rs diff --git a/core-new/src/file_type/builtin.rs b/core/src/file_type/builtin.rs similarity index 100% rename from core-new/src/file_type/builtin.rs rename to core/src/file_type/builtin.rs diff --git a/core-new/src/file_type/definitions/archives.toml b/core/src/file_type/definitions/archives.toml similarity index 100% rename from core-new/src/file_type/definitions/archives.toml rename to core/src/file_type/definitions/archives.toml diff --git a/core-new/src/file_type/definitions/audio.toml b/core/src/file_type/definitions/audio.toml similarity index 100% rename from core-new/src/file_type/definitions/audio.toml rename to core/src/file_type/definitions/audio.toml diff --git a/core-new/src/file_type/definitions/code.toml b/core/src/file_type/definitions/code.toml similarity index 100% rename from core-new/src/file_type/definitions/code.toml rename to core/src/file_type/definitions/code.toml diff --git a/core-new/src/file_type/definitions/documents.toml b/core/src/file_type/definitions/documents.toml similarity index 100% rename from core-new/src/file_type/definitions/documents.toml rename to core/src/file_type/definitions/documents.toml diff --git a/core-new/src/file_type/definitions/images.toml b/core/src/file_type/definitions/images.toml similarity index 100% rename from core-new/src/file_type/definitions/images.toml rename to core/src/file_type/definitions/images.toml diff --git a/core-new/src/file_type/definitions/misc.toml b/core/src/file_type/definitions/misc.toml similarity index 100% rename from core-new/src/file_type/definitions/misc.toml rename to core/src/file_type/definitions/misc.toml diff --git a/core-new/src/file_type/definitions/video.toml b/core/src/file_type/definitions/video.toml similarity index 100% rename from core-new/src/file_type/definitions/video.toml rename to core/src/file_type/definitions/video.toml diff --git a/core-new/src/file_type/magic.rs b/core/src/file_type/magic.rs similarity index 100% rename from core-new/src/file_type/magic.rs rename to core/src/file_type/magic.rs diff --git a/core-new/src/file_type/mod.rs b/core/src/file_type/mod.rs similarity index 100% rename from core-new/src/file_type/mod.rs rename to core/src/file_type/mod.rs diff --git a/core-new/src/file_type/registry.rs b/core/src/file_type/registry.rs similarity index 100% rename from core-new/src/file_type/registry.rs rename to core/src/file_type/registry.rs diff --git a/core-new/src/infrastructure/actions/BUILDER_REFACTOR_PLAN.md b/core/src/infrastructure/actions/BUILDER_REFACTOR_PLAN.md similarity index 100% rename from core-new/src/infrastructure/actions/BUILDER_REFACTOR_PLAN.md rename to core/src/infrastructure/actions/BUILDER_REFACTOR_PLAN.md diff --git a/core-new/src/infrastructure/actions/builder.rs b/core/src/infrastructure/actions/builder.rs similarity index 100% rename from core-new/src/infrastructure/actions/builder.rs rename to core/src/infrastructure/actions/builder.rs diff --git a/core-new/src/infrastructure/actions/error.rs b/core/src/infrastructure/actions/error.rs similarity index 100% rename from core-new/src/infrastructure/actions/error.rs rename to core/src/infrastructure/actions/error.rs diff --git a/core-new/src/infrastructure/actions/handler.rs b/core/src/infrastructure/actions/handler.rs similarity index 100% rename from core-new/src/infrastructure/actions/handler.rs rename to core/src/infrastructure/actions/handler.rs diff --git a/core-new/src/infrastructure/actions/manager.rs b/core/src/infrastructure/actions/manager.rs similarity index 100% rename from core-new/src/infrastructure/actions/manager.rs rename to core/src/infrastructure/actions/manager.rs diff --git a/core-new/src/infrastructure/actions/mod.rs b/core/src/infrastructure/actions/mod.rs similarity index 100% rename from core-new/src/infrastructure/actions/mod.rs rename to core/src/infrastructure/actions/mod.rs diff --git a/core-new/src/infrastructure/actions/output.rs b/core/src/infrastructure/actions/output.rs similarity index 100% rename from core-new/src/infrastructure/actions/output.rs rename to core/src/infrastructure/actions/output.rs diff --git a/core-new/src/infrastructure/actions/receipt.rs b/core/src/infrastructure/actions/receipt.rs similarity index 100% rename from core-new/src/infrastructure/actions/receipt.rs rename to core/src/infrastructure/actions/receipt.rs diff --git a/core-new/src/infrastructure/actions/registry.rs b/core/src/infrastructure/actions/registry.rs similarity index 100% rename from core-new/src/infrastructure/actions/registry.rs rename to core/src/infrastructure/actions/registry.rs diff --git a/core-new/src/infrastructure/actions/tests.rs b/core/src/infrastructure/actions/tests.rs similarity index 100% rename from core-new/src/infrastructure/actions/tests.rs rename to core/src/infrastructure/actions/tests.rs diff --git a/core-new/src/infrastructure/cli/README.md b/core/src/infrastructure/cli/README.md similarity index 100% rename from core-new/src/infrastructure/cli/README.md rename to core/src/infrastructure/cli/README.md diff --git a/core-new/src/infrastructure/cli/adapters/copy.rs b/core/src/infrastructure/cli/adapters/copy.rs similarity index 100% rename from core-new/src/infrastructure/cli/adapters/copy.rs rename to core/src/infrastructure/cli/adapters/copy.rs diff --git a/core-new/src/infrastructure/cli/adapters/mod.rs b/core/src/infrastructure/cli/adapters/mod.rs similarity index 100% rename from core-new/src/infrastructure/cli/adapters/mod.rs rename to core/src/infrastructure/cli/adapters/mod.rs diff --git a/core-new/src/infrastructure/cli/commands/daemon.rs b/core/src/infrastructure/cli/commands/daemon.rs similarity index 100% rename from core-new/src/infrastructure/cli/commands/daemon.rs rename to core/src/infrastructure/cli/commands/daemon.rs diff --git a/core-new/src/infrastructure/cli/commands/file.rs b/core/src/infrastructure/cli/commands/file.rs similarity index 100% rename from core-new/src/infrastructure/cli/commands/file.rs rename to core/src/infrastructure/cli/commands/file.rs diff --git a/core-new/src/infrastructure/cli/commands/job.rs b/core/src/infrastructure/cli/commands/job.rs similarity index 100% rename from core-new/src/infrastructure/cli/commands/job.rs rename to core/src/infrastructure/cli/commands/job.rs diff --git a/core-new/src/infrastructure/cli/commands/library.rs b/core/src/infrastructure/cli/commands/library.rs similarity index 100% rename from core-new/src/infrastructure/cli/commands/library.rs rename to core/src/infrastructure/cli/commands/library.rs diff --git a/core-new/src/infrastructure/cli/commands/location.rs b/core/src/infrastructure/cli/commands/location.rs similarity index 100% rename from core-new/src/infrastructure/cli/commands/location.rs rename to core/src/infrastructure/cli/commands/location.rs diff --git a/core-new/src/infrastructure/cli/commands/mod.rs b/core/src/infrastructure/cli/commands/mod.rs similarity index 100% rename from core-new/src/infrastructure/cli/commands/mod.rs rename to core/src/infrastructure/cli/commands/mod.rs diff --git a/core-new/src/infrastructure/cli/commands/network.rs b/core/src/infrastructure/cli/commands/network.rs similarity index 100% rename from core-new/src/infrastructure/cli/commands/network.rs rename to core/src/infrastructure/cli/commands/network.rs diff --git a/core-new/src/infrastructure/cli/commands/system.rs b/core/src/infrastructure/cli/commands/system.rs similarity index 100% rename from core-new/src/infrastructure/cli/commands/system.rs rename to core/src/infrastructure/cli/commands/system.rs diff --git a/core-new/src/infrastructure/cli/commands/volume.rs b/core/src/infrastructure/cli/commands/volume.rs similarity index 100% rename from core-new/src/infrastructure/cli/commands/volume.rs rename to core/src/infrastructure/cli/commands/volume.rs diff --git a/core-new/src/infrastructure/cli/daemon/client.rs b/core/src/infrastructure/cli/daemon/client.rs similarity index 100% rename from core-new/src/infrastructure/cli/daemon/client.rs rename to core/src/infrastructure/cli/daemon/client.rs diff --git a/core-new/src/infrastructure/cli/daemon/config.rs b/core/src/infrastructure/cli/daemon/config.rs similarity index 100% rename from core-new/src/infrastructure/cli/daemon/config.rs rename to core/src/infrastructure/cli/daemon/config.rs diff --git a/core-new/src/infrastructure/cli/daemon/handlers/core.rs b/core/src/infrastructure/cli/daemon/handlers/core.rs similarity index 100% rename from core-new/src/infrastructure/cli/daemon/handlers/core.rs rename to core/src/infrastructure/cli/daemon/handlers/core.rs diff --git a/core-new/src/infrastructure/cli/daemon/handlers/file.rs b/core/src/infrastructure/cli/daemon/handlers/file.rs similarity index 100% rename from core-new/src/infrastructure/cli/daemon/handlers/file.rs rename to core/src/infrastructure/cli/daemon/handlers/file.rs diff --git a/core-new/src/infrastructure/cli/daemon/handlers/job.rs b/core/src/infrastructure/cli/daemon/handlers/job.rs similarity index 100% rename from core-new/src/infrastructure/cli/daemon/handlers/job.rs rename to core/src/infrastructure/cli/daemon/handlers/job.rs diff --git a/core-new/src/infrastructure/cli/daemon/handlers/library.rs b/core/src/infrastructure/cli/daemon/handlers/library.rs similarity index 100% rename from core-new/src/infrastructure/cli/daemon/handlers/library.rs rename to core/src/infrastructure/cli/daemon/handlers/library.rs diff --git a/core-new/src/infrastructure/cli/daemon/handlers/location.rs b/core/src/infrastructure/cli/daemon/handlers/location.rs similarity index 100% rename from core-new/src/infrastructure/cli/daemon/handlers/location.rs rename to core/src/infrastructure/cli/daemon/handlers/location.rs diff --git a/core-new/src/infrastructure/cli/daemon/handlers/mod.rs b/core/src/infrastructure/cli/daemon/handlers/mod.rs similarity index 100% rename from core-new/src/infrastructure/cli/daemon/handlers/mod.rs rename to core/src/infrastructure/cli/daemon/handlers/mod.rs diff --git a/core-new/src/infrastructure/cli/daemon/handlers/network.rs b/core/src/infrastructure/cli/daemon/handlers/network.rs similarity index 100% rename from core-new/src/infrastructure/cli/daemon/handlers/network.rs rename to core/src/infrastructure/cli/daemon/handlers/network.rs diff --git a/core-new/src/infrastructure/cli/daemon/handlers/system.rs b/core/src/infrastructure/cli/daemon/handlers/system.rs similarity index 100% rename from core-new/src/infrastructure/cli/daemon/handlers/system.rs rename to core/src/infrastructure/cli/daemon/handlers/system.rs diff --git a/core-new/src/infrastructure/cli/daemon/handlers/volume.rs b/core/src/infrastructure/cli/daemon/handlers/volume.rs similarity index 100% rename from core-new/src/infrastructure/cli/daemon/handlers/volume.rs rename to core/src/infrastructure/cli/daemon/handlers/volume.rs diff --git a/core-new/src/infrastructure/cli/daemon/mod.rs b/core/src/infrastructure/cli/daemon/mod.rs similarity index 100% rename from core-new/src/infrastructure/cli/daemon/mod.rs rename to core/src/infrastructure/cli/daemon/mod.rs diff --git a/core-new/src/infrastructure/cli/daemon/services/helpers.rs b/core/src/infrastructure/cli/daemon/services/helpers.rs similarity index 100% rename from core-new/src/infrastructure/cli/daemon/services/helpers.rs rename to core/src/infrastructure/cli/daemon/services/helpers.rs diff --git a/core-new/src/infrastructure/cli/daemon/services/mod.rs b/core/src/infrastructure/cli/daemon/services/mod.rs similarity index 100% rename from core-new/src/infrastructure/cli/daemon/services/mod.rs rename to core/src/infrastructure/cli/daemon/services/mod.rs diff --git a/core-new/src/infrastructure/cli/daemon/services/state.rs b/core/src/infrastructure/cli/daemon/services/state.rs similarity index 100% rename from core-new/src/infrastructure/cli/daemon/services/state.rs rename to core/src/infrastructure/cli/daemon/services/state.rs diff --git a/core-new/src/infrastructure/cli/daemon/types/commands.rs b/core/src/infrastructure/cli/daemon/types/commands.rs similarity index 100% rename from core-new/src/infrastructure/cli/daemon/types/commands.rs rename to core/src/infrastructure/cli/daemon/types/commands.rs diff --git a/core-new/src/infrastructure/cli/daemon/types/common.rs b/core/src/infrastructure/cli/daemon/types/common.rs similarity index 100% rename from core-new/src/infrastructure/cli/daemon/types/common.rs rename to core/src/infrastructure/cli/daemon/types/common.rs diff --git a/core-new/src/infrastructure/cli/daemon/types/mod.rs b/core/src/infrastructure/cli/daemon/types/mod.rs similarity index 100% rename from core-new/src/infrastructure/cli/daemon/types/mod.rs rename to core/src/infrastructure/cli/daemon/types/mod.rs diff --git a/core-new/src/infrastructure/cli/daemon/types/responses.rs b/core/src/infrastructure/cli/daemon/types/responses.rs similarity index 100% rename from core-new/src/infrastructure/cli/daemon/types/responses.rs rename to core/src/infrastructure/cli/daemon/types/responses.rs diff --git a/core-new/src/infrastructure/cli/mod.rs b/core/src/infrastructure/cli/mod.rs similarity index 100% rename from core-new/src/infrastructure/cli/mod.rs rename to core/src/infrastructure/cli/mod.rs diff --git a/core-new/src/infrastructure/cli/output/context.rs b/core/src/infrastructure/cli/output/context.rs similarity index 100% rename from core-new/src/infrastructure/cli/output/context.rs rename to core/src/infrastructure/cli/output/context.rs diff --git a/core-new/src/infrastructure/cli/output/formatters.rs b/core/src/infrastructure/cli/output/formatters.rs similarity index 100% rename from core-new/src/infrastructure/cli/output/formatters.rs rename to core/src/infrastructure/cli/output/formatters.rs diff --git a/core-new/src/infrastructure/cli/output/messages.rs b/core/src/infrastructure/cli/output/messages.rs similarity index 100% rename from core-new/src/infrastructure/cli/output/messages.rs rename to core/src/infrastructure/cli/output/messages.rs diff --git a/core-new/src/infrastructure/cli/output/mod.rs b/core/src/infrastructure/cli/output/mod.rs similarity index 100% rename from core-new/src/infrastructure/cli/output/mod.rs rename to core/src/infrastructure/cli/output/mod.rs diff --git a/core-new/src/infrastructure/cli/output/section.rs b/core/src/infrastructure/cli/output/section.rs similarity index 100% rename from core-new/src/infrastructure/cli/output/section.rs rename to core/src/infrastructure/cli/output/section.rs diff --git a/core-new/src/infrastructure/cli/output/tests.rs b/core/src/infrastructure/cli/output/tests.rs similarity index 100% rename from core-new/src/infrastructure/cli/output/tests.rs rename to core/src/infrastructure/cli/output/tests.rs diff --git a/core-new/src/infrastructure/cli/pairing_ui.rs b/core/src/infrastructure/cli/pairing_ui.rs similarity index 100% rename from core-new/src/infrastructure/cli/pairing_ui.rs rename to core/src/infrastructure/cli/pairing_ui.rs diff --git a/core-new/src/infrastructure/cli/state.rs b/core/src/infrastructure/cli/state.rs similarity index 100% rename from core-new/src/infrastructure/cli/state.rs rename to core/src/infrastructure/cli/state.rs diff --git a/core-new/src/infrastructure/cli/tui.rs b/core/src/infrastructure/cli/tui.rs similarity index 100% rename from core-new/src/infrastructure/cli/tui.rs rename to core/src/infrastructure/cli/tui.rs diff --git a/core-new/src/infrastructure/cli/utils.rs b/core/src/infrastructure/cli/utils.rs similarity index 100% rename from core-new/src/infrastructure/cli/utils.rs rename to core/src/infrastructure/cli/utils.rs diff --git a/core-new/src/infrastructure/database/entities/audit_log.rs b/core/src/infrastructure/database/entities/audit_log.rs similarity index 100% rename from core-new/src/infrastructure/database/entities/audit_log.rs rename to core/src/infrastructure/database/entities/audit_log.rs diff --git a/core-new/src/infrastructure/database/entities/collection.rs b/core/src/infrastructure/database/entities/collection.rs similarity index 100% rename from core-new/src/infrastructure/database/entities/collection.rs rename to core/src/infrastructure/database/entities/collection.rs diff --git a/core-new/src/infrastructure/database/entities/collection_entry.rs b/core/src/infrastructure/database/entities/collection_entry.rs similarity index 100% rename from core-new/src/infrastructure/database/entities/collection_entry.rs rename to core/src/infrastructure/database/entities/collection_entry.rs diff --git a/core-new/src/infrastructure/database/entities/content_identity.rs b/core/src/infrastructure/database/entities/content_identity.rs similarity index 100% rename from core-new/src/infrastructure/database/entities/content_identity.rs rename to core/src/infrastructure/database/entities/content_identity.rs diff --git a/core-new/src/infrastructure/database/entities/content_kind.rs b/core/src/infrastructure/database/entities/content_kind.rs similarity index 100% rename from core-new/src/infrastructure/database/entities/content_kind.rs rename to core/src/infrastructure/database/entities/content_kind.rs diff --git a/core-new/src/infrastructure/database/entities/device.rs b/core/src/infrastructure/database/entities/device.rs similarity index 100% rename from core-new/src/infrastructure/database/entities/device.rs rename to core/src/infrastructure/database/entities/device.rs diff --git a/core-new/src/infrastructure/database/entities/directory_paths.rs b/core/src/infrastructure/database/entities/directory_paths.rs similarity index 100% rename from core-new/src/infrastructure/database/entities/directory_paths.rs rename to core/src/infrastructure/database/entities/directory_paths.rs diff --git a/core-new/src/infrastructure/database/entities/entry.rs b/core/src/infrastructure/database/entities/entry.rs similarity index 100% rename from core-new/src/infrastructure/database/entities/entry.rs rename to core/src/infrastructure/database/entities/entry.rs diff --git a/core-new/src/infrastructure/database/entities/entry_closure.rs b/core/src/infrastructure/database/entities/entry_closure.rs similarity index 100% rename from core-new/src/infrastructure/database/entities/entry_closure.rs rename to core/src/infrastructure/database/entities/entry_closure.rs diff --git a/core-new/src/infrastructure/database/entities/indexer_rule.rs b/core/src/infrastructure/database/entities/indexer_rule.rs similarity index 100% rename from core-new/src/infrastructure/database/entities/indexer_rule.rs rename to core/src/infrastructure/database/entities/indexer_rule.rs diff --git a/core-new/src/infrastructure/database/entities/label.rs b/core/src/infrastructure/database/entities/label.rs similarity index 100% rename from core-new/src/infrastructure/database/entities/label.rs rename to core/src/infrastructure/database/entities/label.rs diff --git a/core-new/src/infrastructure/database/entities/location.rs b/core/src/infrastructure/database/entities/location.rs similarity index 100% rename from core-new/src/infrastructure/database/entities/location.rs rename to core/src/infrastructure/database/entities/location.rs diff --git a/core-new/src/infrastructure/database/entities/metadata_label.rs b/core/src/infrastructure/database/entities/metadata_label.rs similarity index 100% rename from core-new/src/infrastructure/database/entities/metadata_label.rs rename to core/src/infrastructure/database/entities/metadata_label.rs diff --git a/core-new/src/infrastructure/database/entities/metadata_tag.rs b/core/src/infrastructure/database/entities/metadata_tag.rs similarity index 100% rename from core-new/src/infrastructure/database/entities/metadata_tag.rs rename to core/src/infrastructure/database/entities/metadata_tag.rs diff --git a/core-new/src/infrastructure/database/entities/mime_type.rs b/core/src/infrastructure/database/entities/mime_type.rs similarity index 100% rename from core-new/src/infrastructure/database/entities/mime_type.rs rename to core/src/infrastructure/database/entities/mime_type.rs diff --git a/core-new/src/infrastructure/database/entities/mod.rs b/core/src/infrastructure/database/entities/mod.rs similarity index 100% rename from core-new/src/infrastructure/database/entities/mod.rs rename to core/src/infrastructure/database/entities/mod.rs diff --git a/core-new/src/infrastructure/database/entities/sidecar.rs b/core/src/infrastructure/database/entities/sidecar.rs similarity index 100% rename from core-new/src/infrastructure/database/entities/sidecar.rs rename to core/src/infrastructure/database/entities/sidecar.rs diff --git a/core-new/src/infrastructure/database/entities/sidecar_availability.rs b/core/src/infrastructure/database/entities/sidecar_availability.rs similarity index 100% rename from core-new/src/infrastructure/database/entities/sidecar_availability.rs rename to core/src/infrastructure/database/entities/sidecar_availability.rs diff --git a/core-new/src/infrastructure/database/entities/tag.rs b/core/src/infrastructure/database/entities/tag.rs similarity index 100% rename from core-new/src/infrastructure/database/entities/tag.rs rename to core/src/infrastructure/database/entities/tag.rs diff --git a/core-new/src/infrastructure/database/entities/user_metadata.rs b/core/src/infrastructure/database/entities/user_metadata.rs similarity index 100% rename from core-new/src/infrastructure/database/entities/user_metadata.rs rename to core/src/infrastructure/database/entities/user_metadata.rs diff --git a/core-new/src/infrastructure/database/entities/volume.rs b/core/src/infrastructure/database/entities/volume.rs similarity index 100% rename from core-new/src/infrastructure/database/entities/volume.rs rename to core/src/infrastructure/database/entities/volume.rs diff --git a/core-new/src/infrastructure/database/migration/m20240101_000001_initial_schema.rs b/core/src/infrastructure/database/migration/m20240101_000001_initial_schema.rs similarity index 100% rename from core-new/src/infrastructure/database/migration/m20240101_000001_initial_schema.rs rename to core/src/infrastructure/database/migration/m20240101_000001_initial_schema.rs diff --git a/core-new/src/infrastructure/database/migration/m20240102_000001_populate_lookups.rs b/core/src/infrastructure/database/migration/m20240102_000001_populate_lookups.rs similarity index 100% rename from core-new/src/infrastructure/database/migration/m20240102_000001_populate_lookups.rs rename to core/src/infrastructure/database/migration/m20240102_000001_populate_lookups.rs diff --git a/core-new/src/infrastructure/database/migration/m20240107_000001_create_collections.rs b/core/src/infrastructure/database/migration/m20240107_000001_create_collections.rs similarity index 100% rename from core-new/src/infrastructure/database/migration/m20240107_000001_create_collections.rs rename to core/src/infrastructure/database/migration/m20240107_000001_create_collections.rs diff --git a/core-new/src/infrastructure/database/migration/m20250109_000001_create_sidecars.rs b/core/src/infrastructure/database/migration/m20250109_000001_create_sidecars.rs similarity index 100% rename from core-new/src/infrastructure/database/migration/m20250109_000001_create_sidecars.rs rename to core/src/infrastructure/database/migration/m20250109_000001_create_sidecars.rs diff --git a/core-new/src/infrastructure/database/migration/m20250110_000001_refactor_volumes_table.rs b/core/src/infrastructure/database/migration/m20250110_000001_refactor_volumes_table.rs similarity index 100% rename from core-new/src/infrastructure/database/migration/m20250110_000001_refactor_volumes_table.rs rename to core/src/infrastructure/database/migration/m20250110_000001_refactor_volumes_table.rs diff --git a/core-new/src/infrastructure/database/migration/m20250112_000001_create_indexer_rules.rs b/core/src/infrastructure/database/migration/m20250112_000001_create_indexer_rules.rs similarity index 100% rename from core-new/src/infrastructure/database/migration/m20250112_000001_create_indexer_rules.rs rename to core/src/infrastructure/database/migration/m20250112_000001_create_indexer_rules.rs diff --git a/core-new/src/infrastructure/database/migration/mod.rs b/core/src/infrastructure/database/migration/mod.rs similarity index 100% rename from core-new/src/infrastructure/database/migration/mod.rs rename to core/src/infrastructure/database/migration/mod.rs diff --git a/core-new/src/infrastructure/database/mod.rs b/core/src/infrastructure/database/mod.rs similarity index 100% rename from core-new/src/infrastructure/database/mod.rs rename to core/src/infrastructure/database/mod.rs diff --git a/core-new/src/infrastructure/events/mod.rs b/core/src/infrastructure/events/mod.rs similarity index 100% rename from core-new/src/infrastructure/events/mod.rs rename to core/src/infrastructure/events/mod.rs diff --git a/core-new/src/infrastructure/jobs/context.rs b/core/src/infrastructure/jobs/context.rs similarity index 100% rename from core-new/src/infrastructure/jobs/context.rs rename to core/src/infrastructure/jobs/context.rs diff --git a/core-new/src/infrastructure/jobs/database.rs b/core/src/infrastructure/jobs/database.rs similarity index 100% rename from core-new/src/infrastructure/jobs/database.rs rename to core/src/infrastructure/jobs/database.rs diff --git a/core-new/src/infrastructure/jobs/error.rs b/core/src/infrastructure/jobs/error.rs similarity index 100% rename from core-new/src/infrastructure/jobs/error.rs rename to core/src/infrastructure/jobs/error.rs diff --git a/core-new/src/infrastructure/jobs/executor.rs b/core/src/infrastructure/jobs/executor.rs similarity index 100% rename from core-new/src/infrastructure/jobs/executor.rs rename to core/src/infrastructure/jobs/executor.rs diff --git a/core-new/src/infrastructure/jobs/generic_progress.rs b/core/src/infrastructure/jobs/generic_progress.rs similarity index 100% rename from core-new/src/infrastructure/jobs/generic_progress.rs rename to core/src/infrastructure/jobs/generic_progress.rs diff --git a/core-new/src/infrastructure/jobs/handle.rs b/core/src/infrastructure/jobs/handle.rs similarity index 100% rename from core-new/src/infrastructure/jobs/handle.rs rename to core/src/infrastructure/jobs/handle.rs diff --git a/core-new/src/infrastructure/jobs/logger.rs b/core/src/infrastructure/jobs/logger.rs similarity index 100% rename from core-new/src/infrastructure/jobs/logger.rs rename to core/src/infrastructure/jobs/logger.rs diff --git a/core-new/src/infrastructure/jobs/manager.rs b/core/src/infrastructure/jobs/manager.rs similarity index 100% rename from core-new/src/infrastructure/jobs/manager.rs rename to core/src/infrastructure/jobs/manager.rs diff --git a/core-new/src/infrastructure/jobs/mod.rs b/core/src/infrastructure/jobs/mod.rs similarity index 100% rename from core-new/src/infrastructure/jobs/mod.rs rename to core/src/infrastructure/jobs/mod.rs diff --git a/core-new/src/infrastructure/jobs/output.rs b/core/src/infrastructure/jobs/output.rs similarity index 100% rename from core-new/src/infrastructure/jobs/output.rs rename to core/src/infrastructure/jobs/output.rs diff --git a/core-new/src/infrastructure/jobs/progress.rs b/core/src/infrastructure/jobs/progress.rs similarity index 100% rename from core-new/src/infrastructure/jobs/progress.rs rename to core/src/infrastructure/jobs/progress.rs diff --git a/core-new/src/infrastructure/jobs/registry.rs b/core/src/infrastructure/jobs/registry.rs similarity index 100% rename from core-new/src/infrastructure/jobs/registry.rs rename to core/src/infrastructure/jobs/registry.rs diff --git a/core-new/src/infrastructure/jobs/traits.rs b/core/src/infrastructure/jobs/traits.rs similarity index 100% rename from core-new/src/infrastructure/jobs/traits.rs rename to core/src/infrastructure/jobs/traits.rs diff --git a/core-new/src/infrastructure/jobs/types.rs b/core/src/infrastructure/jobs/types.rs similarity index 100% rename from core-new/src/infrastructure/jobs/types.rs rename to core/src/infrastructure/jobs/types.rs diff --git a/core-new/src/infrastructure/mod.rs b/core/src/infrastructure/mod.rs similarity index 100% rename from core-new/src/infrastructure/mod.rs rename to core/src/infrastructure/mod.rs diff --git a/core-new/src/keys/device_key_manager.rs b/core/src/keys/device_key_manager.rs similarity index 100% rename from core-new/src/keys/device_key_manager.rs rename to core/src/keys/device_key_manager.rs diff --git a/core-new/src/keys/library_key_manager.rs b/core/src/keys/library_key_manager.rs similarity index 100% rename from core-new/src/keys/library_key_manager.rs rename to core/src/keys/library_key_manager.rs diff --git a/core-new/src/keys/mod.rs b/core/src/keys/mod.rs similarity index 100% rename from core-new/src/keys/mod.rs rename to core/src/keys/mod.rs diff --git a/core/src/lib.rs b/core/src/lib.rs index e91eb7fb1..649adfe4d 100644 --- a/core/src/lib.rs +++ b/core/src/lib.rs @@ -1,437 +1,499 @@ -#![recursion_limit = "256"] -#![warn(clippy::unwrap_used, clippy::panic)] +#![allow(warnings)] +//! Spacedrive Core v2 +//! +//! A unified, simplified architecture for cross-platform file management. -use crate::{ - api::{CoreEvent, Router}, - location::LocationManagerError, -}; - -use sd_core_cloud_services::CloudServices; -use sd_core_heavy_lifting::{media_processor::ThumbnailKind, JobSystem}; -use sd_core_prisma_helpers::CasId; - -use sd_crypto::CryptoRng; -use sd_task_system::TaskSystem; -use sd_utils::error::FileIOError; -use volume::VolumeManagerActor; - -use std::{ - fmt, - path::{Path, PathBuf}, - sync::Arc, -}; - -use chrono::{DateTime, Utc}; -use futures_concurrency::future::Join; -use thiserror::Error; -use tokio::{ - fs, io, - sync::{broadcast, Mutex}, -}; -use tracing::{error, info, warn}; -use tracing_appender::{ - non_blocking::{NonBlocking, WorkerGuard}, - rolling::{RollingFileAppender, Rotation}, -}; -use tracing_subscriber::{ - filter::FromEnvError, fmt::format::Format, prelude::*, registry, EnvFilter, -}; - -pub mod api; -mod context; -pub mod custom_uri; +pub mod config; +pub mod context; +pub mod device; +pub mod domain; +pub mod file_type; +pub mod infrastructure; +pub mod keys; pub mod library; -pub(crate) mod location; -pub(crate) mod node; -pub(crate) mod notifications; -pub(crate) mod object; -pub(crate) mod old_job; -pub(crate) mod old_p2p; -pub(crate) mod preferences; -#[doc(hidden)] // TODO(@Oscar): Make this private when breaking out `utils` into `sd-utils` -pub mod util; -pub(crate) mod volume; +pub mod location; +pub mod operations; +pub mod services; +pub mod shared; +pub mod test_framework; +pub mod volume; -use api::notifications::{Notification, NotificationData, NotificationId}; -use context::{JobContext, NodeContext}; -use node::config; -use notifications::Notifications; -use sd_core_cloud_services::AUTH_SERVER_URL; +use services::networking::protocols::PairingProtocolHandler; +use services::networking::utils::logging::NetworkLogger; -/// Represents a single running instance of the Spacedrive core. -/// Holds references to all the services that make up the Spacedrive core. -pub struct Node { - pub data_dir: PathBuf, - pub config: Arc, - pub libraries: Arc, - pub volumes: Arc, - pub locations: location::Locations, - pub p2p: Arc, - pub event_bus: (broadcast::Sender, broadcast::Receiver), - pub notifications: Notifications, - pub task_system: TaskSystem, - pub job_system: JobSystem>, - pub cloud_services: Arc, - /// This should only be used to generate the seed of local instances of [`CryptoRng`]. - /// Don't use this as a common RNG, it will fuck up Core's performance due to this Mutex. - pub master_rng: Arc>, - pub old_jobs: Arc, +// Compatibility module for legacy networking references +pub mod networking { + pub use crate::services::networking::*; } -impl fmt::Debug for Node { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("Node") - .field("data_dir", &self.data_dir) - .finish() +use crate::config::AppConfig; +use crate::context::CoreContext; +use crate::device::DeviceManager; +use crate::infrastructure::actions::manager::ActionManager; +use crate::infrastructure::events::{Event, EventBus}; +use crate::library::LibraryManager; +use crate::services::Services; +use crate::volume::{VolumeDetectionConfig, VolumeManager}; +use std::path::PathBuf; +use std::sync::Arc; +use tokio::sync::{mpsc, RwLock}; +use tracing::{error, info}; + +/// Pending pairing request information +#[derive(Debug, Clone)] +pub struct PendingPairingRequest { + pub request_id: uuid::Uuid, + pub device_id: uuid::Uuid, + pub device_name: String, + pub received_at: chrono::DateTime, +} + +/// Spacedrop request message +#[derive(serde::Serialize, serde::Deserialize)] +struct SpacedropRequest { + transfer_id: uuid::Uuid, + file_path: String, + sender_name: String, + message: Option, + file_size: u64, +} + +// NOTE: SimplePairingUI has been moved to CLI infrastructure +// See: src/infrastructure/cli/pairing_ui.rs for CLI-specific implementations + +/// Bridge between networking events and core events +pub struct NetworkEventBridge { + network_events: mpsc::UnboundedReceiver, + core_events: Arc, +} + +impl NetworkEventBridge { + pub fn new( + network_events: mpsc::UnboundedReceiver, + core_events: Arc, + ) -> Self { + Self { + network_events, + core_events, + } + } + + pub async fn run(mut self) { + while let Some(event) = self.network_events.recv().await { + if let Some(core_event) = self.translate_event(event) { + self.core_events.emit(core_event); + } + } + } + + fn translate_event(&self, event: networking::NetworkEvent) -> Option { + match event { + networking::NetworkEvent::ConnectionEstablished { device_id, .. } => { + Some(Event::DeviceConnected { + device_id, + device_name: "Connected Device".to_string(), + }) + } + networking::NetworkEvent::ConnectionLost { device_id, .. } => { + Some(Event::DeviceDisconnected { device_id }) + } + networking::NetworkEvent::PairingCompleted { + device_id, + device_info, + } => Some(Event::DeviceConnected { + device_id, + device_name: device_info.device_name, + }), + _ => None, // Some events don't map to core events + } } } -impl Node { - pub async fn new(data_dir: impl AsRef) -> Result<(Arc, Arc), NodeError> { - let data_dir = data_dir.as_ref(); +/// The main context for all core operations +pub struct Core { + /// Application configuration + pub config: Arc>, - info!(data_directory = %data_dir.display(), "Starting core;"); + /// Device manager + pub device: Arc, - #[cfg(debug_assertions)] - let init_data = util::debug_initializer::InitConfig::load(data_dir).await?; + /// Library manager + pub libraries: Arc, - // This error is ignored because it's throwing on mobile despite the folder existing. - let _ = fs::create_dir_all(&data_dir).await; + /// Volume manager + pub volumes: Arc, - let event_bus = broadcast::channel(1024); - let config = config::Manager::new(data_dir.to_path_buf()) - .await - .map_err(NodeError::FailedToInitializeConfig)?; + /// Event bus for state changes + pub events: Arc, - let (locations, locations_actor) = location::Locations::new(); - let (old_jobs, jobs_actor) = old_job::OldJobs::new(); - let libraries = library::Libraries::new(data_dir.join("libraries")).await?; + /// Container for high-level services + pub services: Services, - let ( - get_cloud_api_address, - cloud_p2p_relay_url, - cloud_p2p_dns_origin_name, - cloud_p2p_dns_pkarr_url, - cloud_services_domain_name, - ) = { - #[cfg(debug_assertions)] - { - ( - std::env::var("SD_CLOUD_API_ADDRESS_URL").unwrap_or_else(|_| { - format!("{AUTH_SERVER_URL}/cloud-api-address").to_string() - }), - std::env::var("SD_CLOUD_P2P_RELAY_URL") - // .unwrap_or_else(|_| "https://use1-1.relay.iroh.network/".to_string()), - // .unwrap_or_else(|_| "http://localhost:8081/".to_string()), - .unwrap_or_else(|_| "https://relay.spacedrive.com:4433/".to_string()), - std::env::var("SD_CLOUD_P2P_DNS_ORIGIN_NAME") - // .unwrap_or_else(|_| "dns.iroh.link/".to_string()), - // .unwrap_or_else(|_| "irohdns.localhost".to_string()), - .unwrap_or_else(|_| "irohdns.spacedrive.com".to_string()), - std::env::var("SD_CLOUD_P2P_DNS_PKARR_URL") - // .unwrap_or_else(|_| "https://dns.iroh.link/pkarr".to_string()), - // .unwrap_or_else(|_| "http://localhost:8080/pkarr".to_string()), - .unwrap_or_else(|_| "https://irohdns.spacedrive.com/pkarr".to_string()), - std::env::var("SD_CLOUD_API_DOMAIN_NAME") - // .unwrap_or_else(|_| "localhost".to_string()), - .unwrap_or_else(|_| "cloud.spacedrive.com".to_string()), - ) + /// Shared context for core components + pub context: Arc, +} + +impl Core { + /// Initialize a new Core instance with default data directory + pub async fn new() -> Result> { + let data_dir = crate::config::default_data_dir()?; + Self::new_with_config(data_dir).await + } + + /// Initialize a new Core instance with custom data directory + pub async fn new_with_config(data_dir: PathBuf) -> Result> { + info!("Initializing Spacedrive Core at {:?}", data_dir); + + // 1. Load or create app config + let config = AppConfig::load_or_create(&data_dir)?; + config.ensure_directories()?; + let config = Arc::new(RwLock::new(config)); + + // 2. Initialize device manager + let device = Arc::new(DeviceManager::init_with_path(&data_dir)?); + // Set the global device ID for legacy compatibility + shared::utils::set_current_device_id(device.device_id()?); + + // 3. Create event bus + let events = Arc::new(EventBus::default()); + + // 4. Initialize volume manager + let volume_config = VolumeDetectionConfig::default(); + let device_id = device.device_id()?; + let volumes = Arc::new(VolumeManager::new(device_id, volume_config, events.clone())); + + // 5. Initialize volume detection + // info!("Initializing volume detection..."); + // match volumes.initialize().await { + // Ok(()) => info!("Volume manager initialized"), + // Err(e) => error!("Failed to initialize volume manager: {}", e), + // } + + // 6. Initialize library manager with libraries directory + let libraries_dir = config.read().await.libraries_dir(); + let libraries = Arc::new(LibraryManager::new_with_dir(libraries_dir, events.clone())); + + // 7. Initialize library key manager + let library_key_manager = + Arc::new(crate::keys::library_key_manager::LibraryKeyManager::new()?); + + // 8. Register all job types + info!("Registering job types..."); + crate::operations::register_all_jobs(); + info!("Job types registered"); + + // 9. Create the context that will be shared with services + let mut context_inner = CoreContext::new( + events.clone(), + device.clone(), + libraries.clone(), + volumes.clone(), + library_key_manager.clone(), + ); + + // Set job logging configuration if enabled + let app_config = config.read().await; + if app_config.job_logging.enabled { + context_inner + .set_job_logging(app_config.job_logging.clone(), app_config.job_logs_dir()); + } + drop(app_config); + + let context = Arc::new(context_inner); + + // 10. Initialize services first, passing them the context + let services = Services::new(context.clone()); + + // 11. Auto-load all libraries with context for job manager initialization + info!("Loading existing libraries..."); + let loaded_libraries: Vec> = + match libraries.load_all_with_context(context.clone()).await { + Ok(count) => { + info!("Loaded {} libraries", count); + libraries.list().await + } + Err(e) => { + error!("Failed to load libraries: {}", e); + vec![] + } + }; + + // Initialize sidecar manager for each loaded library + for library in &loaded_libraries { + info!("Initializing sidecar manager for library {}", library.id()); + if let Err(e) = services.sidecar_manager.init_library(&library).await { + error!( + "Failed to initialize sidecar manager for library {}: {}", + library.id(), + e + ); + } else { + // Run bootstrap scan + if let Err(e) = services.sidecar_manager.bootstrap_scan(&library).await { + error!( + "Failed to run sidecar bootstrap scan for library {}: {}", + library.id(), + e + ); + } } - #[cfg(not(debug_assertions))] - { - ( - "https://auth.spacedrive.com/cloud-api-address".to_string(), - "https://relay.spacedrive.com/".to_string(), - "https://irohdns.spacedrive.com".to_string(), - "https://irohdns.spacedrive.com/pkarr".to_string(), - "cloud.spacedrive.com".to_string(), - ) - } - }; + } - let task_system = TaskSystem::new(); + info!("Starting background services..."); + match services.start_all().await { + Ok(()) => info!("Background services started"), + Err(e) => error!("Failed to start services: {}", e), + } - let (p2p, start_p2p) = old_p2p::P2PManager::new(config.clone(), libraries.clone()) - .await - .map_err(NodeError::P2PManager)?; + // 12. Initialize ActionManager and set it in context + let action_manager = Arc::new(crate::infrastructure::actions::manager::ActionManager::new( + context.clone(), + )); + context.set_action_manager(action_manager).await; - let device_id = config.get().await.id; - let volume_ctx = volume::VolumeManagerContext { - device_id: device_id.clone().into(), - library_event_tx: libraries.rx.clone(), - }; + // 13. Emit startup event + events.emit(Event::CoreStarted); - let (volumes, volume_manager_actor) = VolumeManagerActor::new(Arc::new(volume_ctx)).await?; - - let volumes = Arc::new(volumes); - - let node = Arc::new(Node { - data_dir: data_dir.to_path_buf(), - job_system: JobSystem::new(task_system.get_dispatcher(), data_dir), - task_system, - volumes, - locations, - notifications: notifications::Notifications::new(), - p2p, + Ok(Self { config, - event_bus, + device, libraries, - cloud_services: Arc::new( - CloudServices::new( - &get_cloud_api_address, - cloud_p2p_relay_url, - cloud_p2p_dns_pkarr_url, - cloud_p2p_dns_origin_name, - cloud_services_domain_name, - ) - .await?, - ), - master_rng: Arc::new(Mutex::new(CryptoRng::new()?)), - old_jobs, - }); + volumes, + events, + services, + context, + }) + } - // Setup start actors that depend on the `Node` - #[cfg(debug_assertions)] - if let Some(init_data) = init_data { - init_data.apply(&node.libraries, &node).await?; - } + /// Get the application configuration + pub fn config(&self) -> Arc> { + self.config.clone() + } - // Create the .sdks file if it doesn't exist - let temp_data_dir_clone = data_dir.to_path_buf(); - let data_directory = temp_data_dir_clone - .parent() - .expect("Config path must have a parent directory"); - let sdks_file = data_directory.join(".sdks"); - if !sdks_file.exists() { - fs::write(&sdks_file, b"") - .await - .map_err(|e| { - FileIOError::from((sdks_file.clone(), e, "Failed to create .sdks file")) - }) - .map_err(NodeError::FileIO)?; - } + /// Initialize networking using master key + pub async fn init_networking(&mut self) -> Result<(), Box> { + self.init_networking_with_logger(Arc::new(networking::SilentLogger)) + .await + } - let router = api::mount(); + /// Initialize networking with custom logger + pub async fn init_networking_with_logger( + &mut self, + logger: Arc, + ) -> Result<(), Box> { + logger.info("Initializing networking...").await; - // Be REALLY careful about ordering here or you'll get unreliable deadlock's! - locations_actor.start(node.clone()); - node.libraries.init(&node).await?; - jobs_actor.start(node.clone()); - volume_manager_actor.start(device_id).await; - - node.job_system - .init( - &node - .libraries - .get_all() - .await - .into_iter() - .map(|library| { - ( - library.id, - NodeContext { - library, - node: Arc::clone(&node), - }, - ) - }) - .collect(), + // Initialize networking service through the services container + let data_dir = self.config.read().await.data_dir.clone(); + self.services + .init_networking( + self.device.clone(), + self.services.library_key_manager.clone(), + data_dir, ) .await?; - start_p2p( - node.clone(), - axum::Router::new() - .nest( - "/uri", - custom_uri::base_router().with_state(custom_uri::with_state(node.clone())), - ) - .nest( - "/rspc", - router - .clone() - .endpoint({ - let node = node.clone(); - move |_| node.clone() - }) - .axum::<()>(), - ) - .into_make_service(), - ); + // Start the networking service + self.services.start_networking().await?; - // save_storage_statistics(&node); + // Get the networking service for protocol registration + if let Some(networking_service) = self.services.networking() { + // Register default protocol handlers + self.register_default_protocols(&networking_service).await?; - info!("Spacedrive online!"); - Ok((node, router)) - } - - pub fn init_logger(data_dir: impl AsRef) -> Result { - let (logfile, guard) = NonBlocking::new( - RollingFileAppender::builder() - .filename_prefix("sd.log") - .rotation(Rotation::DAILY) - .max_log_files(4) - .build(data_dir.as_ref().join("logs")) - .expect("Error setting up log file!"), - ); - - // Set a default if the user hasn't set an override - if std::env::var("RUST_LOG") == Err(std::env::VarError::NotPresent) { - let level = if cfg!(debug_assertions) { - "debug" - } else { - "info" - }; - - std::env::set_var( - "RUST_LOG", - format!( - "info,\ - iroh_net=info,\ - sd_core={level},\ - sd_p2p={level},\ - sd_core_heavy_lifting={level},\ - sd_task_system={level},\ - sd_ai={level}" - ), + // Set up event bridge to integrate with core event system + let event_bridge = NetworkEventBridge::new( + networking_service + .subscribe_events() + .await + .unwrap_or_else(|| { + let (_, rx) = tokio::sync::mpsc::unbounded_channel(); + rx + }), + self.events.clone(), ); + tokio::spawn(event_bridge.run()); + + // Make networking service available to the context for other services + self.context.set_networking(networking_service).await; } - let registry = registry(); - - let registry = registry - .with( - tracing_subscriber::fmt::layer() - .with_file(true) - .with_line_number(true) - .with_ansi(false) - .with_target(true) - .with_writer(logfile) - .with_filter(EnvFilter::from_default_env()), - ) - .with( - tracing_subscriber::fmt::layer() - .with_file(true) - .with_line_number(true) - .with_writer(std::io::stdout) - .event_format(Format::default().pretty()) - .with_filter(EnvFilter::from_default_env()), - ); - - #[cfg(target_os = "android")] - let registry = registry.with(tracing_android::layer("com.spacedrive.app").unwrap()); - - registry.init(); - - std::panic::set_hook(Box::new(move |panic| { - use std::backtrace::{Backtrace, BacktraceStatus}; - let backtrace = Backtrace::capture(); - if let Some(location) = panic.location() { - tracing::error!( - message = %panic, - panic.file = format!("{}:{}", location.file(), location.line()), - panic.column = location.column(), - ); - if backtrace.status() == BacktraceStatus::Captured { - // NOTE(matheus-consoli): it seems that `tauri` is messing up the stack-trace - // and it doesn't capture anything, even when `RUST_BACKTRACE=full`, - // so in the current architecture, this is emitting an empty event. - tracing::error!(message = %backtrace); - } - } else { - tracing::error!(message = %panic); - } - })); - - Ok(guard) + logger.info("Networking initialized successfully").await; + Ok(()) } - pub async fn shutdown(&self) { - info!("Spacedrive shutting down..."); - - // Let's shutdown the task system first, as the job system will receive tasks to save - self.task_system.shutdown().await; - - ( - self.old_jobs.shutdown(), - self.p2p.shutdown(), - self.job_system.shutdown(), - ) - .join() - .await; - - info!("Spacedrive Core shutdown successful!"); - } - - pub(crate) fn emit(&self, event: CoreEvent) { - if let Err(e) = self.event_bus.0.send(event) { - warn!(?e, "Error sending event to event bus;"); - } - } - - pub async fn ephemeral_thumbnail_exists( + /// Register default protocol handlers + async fn register_default_protocols( &self, - cas_id: &CasId<'_>, - ) -> Result { - let thumb_path = - ThumbnailKind::Ephemeral.compute_path(self.config.data_directory(), cas_id); + networking: &networking::NetworkingService, + ) -> Result<(), Box> { + let logger = std::sync::Arc::new(networking::utils::logging::ConsoleLogger); - match fs::metadata(&thumb_path).await { - Ok(_) => Ok(true), - Err(e) if e.kind() == io::ErrorKind::NotFound => Ok(false), - Err(e) => Err(FileIOError::from((thumb_path, e))), - } - } + // Get command sender for the pairing handler's state machine + let command_sender = networking + .command_sender() + .ok_or("NetworkingEventLoop command sender not available")? + .clone(); - pub async fn emit_notification(&self, data: NotificationData, expires: Option>) { - let notification = Notification { - id: NotificationId::Node(self.notifications._internal_next_id()), - data, - read: false, - expires, + // Get data directory from config + let data_dir = { + let config = self.config.read().await; + config.data_dir.clone() }; - match self - .config - .write(|cfg| cfg.notifications.push(notification.clone())) - .await - { - Ok(_) => { - self.notifications._internal_send(notification); - } - Err(e) => { - error!(?e, "Error saving notification to config;"); - } + let pairing_handler = Arc::new(networking::protocols::PairingProtocolHandler::new_with_persistence( + networking.identity().clone(), + networking.device_registry(), + logger.clone(), + command_sender, + data_dir, + )); + + // Try to load persisted sessions, but don't fail if there's an error + if let Err(e) = pairing_handler.load_persisted_sessions().await { + logger.warn(&format!("Failed to load persisted pairing sessions: {}. Starting with empty sessions.", e)).await; } + + // Start the state machine task for pairing + networking::protocols::PairingProtocolHandler::start_state_machine_task( + pairing_handler.clone(), + ); + + // Start cleanup task for expired sessions + networking::protocols::PairingProtocolHandler::start_cleanup_task(pairing_handler.clone()); + + let messaging_handler = networking::protocols::MessagingProtocolHandler::new(); + let mut file_transfer_handler = + networking::protocols::FileTransferProtocolHandler::new_default(logger.clone()); + + // Inject device registry into file transfer handler for encryption + file_transfer_handler.set_device_registry(networking.device_registry()); + + let protocol_registry = networking.protocol_registry(); + { + let mut registry = protocol_registry.write().await; + registry.register_handler(pairing_handler)?; + registry.register_handler(Arc::new(messaging_handler))?; + registry.register_handler(Arc::new(file_transfer_handler))?; + } + + Ok(()) + } + + /// Initialize networking from Arc - for daemon use + pub async fn init_networking_shared( + core: Arc, + ) -> Result, Box> { + info!("Initializing networking for shared core..."); + + // Create a new Core with networking enabled + let mut new_core = + Core::new_with_config(core.config().read().await.data_dir.clone()).await?; + + // Initialize networking on the new core + new_core.init_networking().await?; + + info!("Networking initialized successfully for shared core"); + Ok(Arc::new(new_core)) + } + + /// Get the networking service (if initialized) + pub fn networking(&self) -> Option> { + self.services.networking() + } + + /// Get list of connected devices + pub async fn get_connected_devices( + &self, + ) -> Result, Box> { + Ok(self.services.device.get_connected_devices().await?) + } + + /// Get detailed information about connected devices + pub async fn get_connected_devices_info( + &self, + ) -> Result, Box> { + Ok(self.services.device.get_connected_devices_info().await?) + } + + /// Add a location to the file system watcher + pub async fn add_watched_location( + &self, + location_id: uuid::Uuid, + library_id: uuid::Uuid, + path: std::path::PathBuf, + enabled: bool, + ) -> Result<(), Box> { + use crate::services::location_watcher::WatchedLocation; + + let watched_location = WatchedLocation { + id: location_id, + library_id, + path, + enabled, + }; + + Ok(self + .services + .location_watcher + .add_location(watched_location) + .await?) + } + + /// Remove a location from the file system watcher + pub async fn remove_watched_location( + &self, + location_id: uuid::Uuid, + ) -> Result<(), Box> { + Ok(self + .services + .location_watcher + .remove_location(location_id) + .await?) + } + + /// Update file watching settings for a location + pub async fn update_watched_location( + &self, + location_id: uuid::Uuid, + enabled: bool, + ) -> Result<(), Box> { + Ok(self + .services + .location_watcher + .update_location(location_id, enabled) + .await?) + } + + /// Get all currently watched locations + pub async fn get_watched_locations( + &self, + ) -> Vec { + self.services.location_watcher.get_watched_locations().await + } + + /// Shutdown the core gracefully + pub async fn shutdown(&self) -> Result<(), Box> { + info!("Shutting down Spacedrive Core..."); + + // Networking service is stopped by services.stop_all() + + // Stop all services + self.services.stop_all().await?; + + // Stop volume monitoring + self.volumes.stop_monitoring().await; + + // Close all libraries + self.libraries.close_all().await?; + + // Save configuration + self.config.write().await.save()?; + + // Emit shutdown event + self.events.emit(Event::CoreShutdown); + + info!("Spacedrive Core shutdown complete"); + Ok(()) } } - -/// Error type for Node related errors. -#[derive(Error, Debug)] -pub enum NodeError { - #[error("NodeError::FailedToInitializeConfig({0})")] - FailedToInitializeConfig(config::NodeConfigError), - #[error("failed to initialize library manager: {0}")] - FailedToInitializeLibraryManager(#[from] library::LibraryManagerError), - #[error("failed to initialize location manager: {0}")] - LocationManager(#[from] LocationManagerError), - #[error("failed to initialize p2p manager: {0}")] - P2PManager(String), - #[error("invalid platform integer: {0}")] - InvalidPlatformInt(u8), - #[cfg(debug_assertions)] - #[error("init config error: {0}")] - InitConfig(#[from] util::debug_initializer::InitConfigError), - #[error("logger error: {0}")] - Logger(#[from] FromEnvError), - #[error(transparent)] - JobSystem(#[from] sd_core_heavy_lifting::JobSystemError), - #[error(transparent)] - CloudServices(#[from] sd_core_cloud_services::Error), - #[error(transparent)] - Crypto(#[from] sd_crypto::Error), - #[error(transparent)] - Volume(#[from] volume::VolumeError), - #[error(transparent)] - FileIO(#[from] FileIOError), -} diff --git a/core/src/library/config.rs b/core/src/library/config.rs index f8a3ae982..207483d25 100644 --- a/core/src/library/config.rs +++ b/core/src/library/config.rs @@ -1,469 +1,208 @@ -use crate::{ - node::config::NodeConfig, - util::version_manager::{Kind, ManagedVersion, VersionManager, VersionManagerError}, -}; +//! Library configuration types -use sd_old_p2p::{Identity, RemoteIdentity}; -use sd_prisma::prisma::{file_path, indexer_rule, instance, location, PrismaClient}; -use sd_utils::{db::maybe_missing, error::FileIOError}; - -use std::{ - path::{Path, PathBuf}, - sync::{atomic::AtomicBool, Arc}, -}; - -use int_enum::IntEnum; -use prisma_client_rust::not; +use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; -use serde_json::{json, Map, Value}; -use serde_repr::{Deserialize_repr, Serialize_repr}; -use specta::Type; -use thiserror::Error; -use tokio::fs; -use tracing::error; use uuid::Uuid; -use super::name::LibraryName; - -/// LibraryConfig holds the configuration for a specific library. This is stored as a '{uuid}.sdlibrary' file. -#[derive(Debug, Clone, Serialize, Deserialize, Type)] +/// Library configuration stored in library.json +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct LibraryConfig { - /// name is the display name of the library. This is used in the UI and is set by the user. - pub name: LibraryName, - /// description is a user set description of the library. This is used in the UI and is set by the user. + /// Version of the configuration format + pub version: u32, + + /// Unique identifier for this library + pub id: Uuid, + + /// Human-readable name + pub name: String, + + /// Optional description pub description: Option, - /// id of the current instance so we know who this `.db` is. This can be looked up within the `Instance` table. - pub instance_id: i32, - /// cloud_id is the ID of the cloud library this library is linked to. - /// If this is set we can assume the library is synced with the Cloud. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub cloud_id: Option, - // false = library is old and sync hasn't been enabled - // true = sync is enabled as either the library is new or it has been manually toggled on + + /// When the library was created + pub created_at: DateTime, + + /// When the library was last modified + pub updated_at: DateTime, + + /// Library-specific settings + pub settings: LibrarySettings, + + /// Library statistics + pub statistics: LibraryStatistics, +} + +/// Library-specific settings +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct LibrarySettings { + /// Whether to generate thumbnails for media files + pub generate_thumbnails: bool, + + /// Thumbnail quality (0-100) + pub thumbnail_quality: u8, + + /// Whether to enable AI-powered tagging + pub enable_ai_tagging: bool, + + /// Whether sync is enabled for this library + pub sync_enabled: bool, + + /// Whether the library is encrypted at rest + pub encryption_enabled: bool, + + /// Custom thumbnail sizes to generate + pub thumbnail_sizes: Vec, + + /// File extensions to ignore during indexing + pub ignored_extensions: Vec, + + /// Maximum file size to index (in bytes) + pub max_file_size: Option, + + /// Whether to automatically track system volumes + pub auto_track_system_volumes: bool, + + /// Whether to automatically track external volumes when connected + pub auto_track_external_volumes: bool, + + /// Indexer settings (rule toggles and related) #[serde(default)] - pub generate_sync_operations: Arc, - version: LibraryConfigVersion, - - #[serde(skip, default)] - pub config_path: PathBuf, - /// cloud_email_address is the email address of the user who owns the cloud library this library is linked to. - pub cloud_email_address: Option, -} - -#[derive( - IntEnum, - Debug, - Clone, - Copy, - Eq, - PartialEq, - strum::Display, - Serialize_repr, - Deserialize_repr, - Type, -)] -#[repr(u64)] -pub enum LibraryConfigVersion { - V0 = 0, - V1 = 1, - V2 = 2, - V3 = 3, - V4 = 4, - V5 = 5, - V6 = 6, - V7 = 7, - V8 = 8, - V9 = 9, - V10 = 10, - V11 = 11, - V12 = 12, -} - -impl ManagedVersion for LibraryConfig { - const LATEST_VERSION: LibraryConfigVersion = LibraryConfigVersion::V12; - - const KIND: Kind = Kind::Json("version"); - - type MigrationError = LibraryConfigError; + pub indexer: IndexerSettings, } impl LibraryConfig { - pub(crate) async fn new( - name: LibraryName, - description: Option, - instance_id: i32, - path: impl AsRef, - ) -> Result { - let this = Self { - name, - description, - instance_id, - version: Self::LATEST_VERSION, - cloud_id: None, - generate_sync_operations: Arc::new(AtomicBool::new(false)), - config_path: path.as_ref().to_path_buf(), - cloud_email_address: None, - }; - - this.save(path).await.map(|()| this) - } - - pub(crate) async fn load( - path: impl AsRef, - _node_config: &NodeConfig, - db: &PrismaClient, - ) -> Result { - let path = path.as_ref(); - - let mut loaded_config = VersionManager::::migrate_and_load( - path, - |current, next| async move { - match (current, next) { - (LibraryConfigVersion::V0, LibraryConfigVersion::V1) => { - let rules = vec![ - String::from("No OS protected"), - String::from("No Hidden"), - String::from("No Git"), - String::from("Only Images"), - ]; - - db._batch( - rules - .into_iter() - .enumerate() - .map(|(i, name)| { - db.indexer_rule().update_many( - vec![indexer_rule::name::equals(Some(name))], - vec![indexer_rule::pub_id::set(sd_utils::uuid_to_bytes( - &Uuid::from_u128(i as u128), - ))], - ) - }) - .collect::>(), - ) - .await?; - } - - (LibraryConfigVersion::V1, LibraryConfigVersion::V2) => { - let mut config = serde_json::from_slice::>( - &fs::read(path).await.map_err(|e| { - VersionManagerError::FileIO(FileIOError::from((path, e))) - })?, - ) - .map_err(VersionManagerError::SerdeJson)?; - - config.insert( - String::from("identity"), - Value::Array( - Identity::new() - .to_bytes() - .into_iter() - .map(Into::into) - .collect(), - ), - ); - - fs::write( - path, - &serde_json::to_vec(&config).map_err(VersionManagerError::SerdeJson)?, - ) - .await - .map_err(|e| VersionManagerError::FileIO(FileIOError::from((path, e))))?; - } - - (LibraryConfigVersion::V2, LibraryConfigVersion::V3) => { - // Removed, can't be automatically updated - return Err(LibraryConfigError::CriticalUpdateError); - } - - (LibraryConfigVersion::V3, LibraryConfigVersion::V4) => { - // -_- - } - - (LibraryConfigVersion::V4, LibraryConfigVersion::V5) => loop { - let paths = db - .file_path() - .find_many(vec![not![file_path::size_in_bytes::equals(None)]]) - .take(500) - .select(file_path::select!({ id size_in_bytes })) - .exec() - .await?; - - if paths.is_empty() { - break; - } - - db._batch( - paths - .into_iter() - .filter_map(|path| { - maybe_missing(path.size_in_bytes, "file_path.size_in_bytes") - .map_or_else( - |e| { - error!(?e); - None - }, - Some, - ) - .map(|size_in_bytes| { - let size = - if let Ok(size) = size_in_bytes.parse::() { - Some(size.to_be_bytes().to_vec()) - } else { - error!( - file_path_id = %path.id, - size = %size_in_bytes, - "File path had invalid size;", - ); - - None - }; - - db.file_path().update( - file_path::id::equals(path.id), - vec![ - file_path::size_in_bytes_bytes::set(size), - file_path::size_in_bytes::set(None), - ], - ) - }) - }) - .collect::>(), - ) - .await?; - }, - - (LibraryConfigVersion::V5, LibraryConfigVersion::V6) => { - // Removed, can't be automatically updated - return Err(LibraryConfigError::CriticalUpdateError); - } - - (LibraryConfigVersion::V6, LibraryConfigVersion::V7) => { - let instances = db.instance().find_many(vec![]).exec().await?; - - if instances.len() > 1 { - error!("7 - More than one instance found in the DB... This can't be automatically reconciled!"); - return Err(LibraryConfigError::TooManyInstances); - } - - let Some(instance) = instances.first() else { - error!("7 - No instance found... How did you even get this far?!"); - return Err(LibraryConfigError::MissingInstance); - }; - - let mut config = serde_json::from_slice::>( - &fs::read(path).await.map_err(|e| { - VersionManagerError::FileIO(FileIOError::from((path, e))) - })?, - ) - .map_err(VersionManagerError::SerdeJson)?; - - config.remove("instance_id"); - config.insert(String::from("instance_id"), json!(instance.id)); - - fs::write( - path, - &serde_json::to_vec(&config).map_err(VersionManagerError::SerdeJson)?, - ) - .await - .map_err(|e| VersionManagerError::FileIO(FileIOError::from((path, e))))?; - - // We are relinking all locations to the current instance. - // If you have more than one node in your database and you're not @Oscar, something went horribly wrong so this is fine. - db.location() - .update_many( - vec![], - vec![location::instance_id::set(Some(instance.id))], - ) - .exec() - .await?; - } - - (LibraryConfigVersion::V7, LibraryConfigVersion::V8) => { - let instances = db.device().find_many(vec![]).exec().await?; - let Some(instance) = instances.first() else { - error!("8 - No nodes found... How did you even get this far?!"); - return Err(LibraryConfigError::MissingInstance); - }; - - // This should be in 7 but it's added to ensure to hell it runs. - let mut config = serde_json::from_slice::>( - &fs::read(path).await.map_err(|e| { - VersionManagerError::FileIO(FileIOError::from((path, e))) - })?, - ) - .map_err(VersionManagerError::SerdeJson)?; - - config.remove("instance_id"); - config.insert(String::from("instance_id"), json!(instance.id)); - - fs::write( - path, - &serde_json::to_vec(&config).map_err(VersionManagerError::SerdeJson)?, - ) - .await - .map_err(|e| VersionManagerError::FileIO(FileIOError::from((path, e))))?; - } - - (LibraryConfigVersion::V8, LibraryConfigVersion::V9) => { - db._batch( - db.instance() - .find_many(vec![]) - .exec() - .await? - .into_iter() - .map(|i| { - db.instance().update( - instance::id::equals(i.id), - vec![ - // In earlier versions of the app this migration would convert an `Identity` in the `identity` column to a `IdentityOrRemoteIdentity::Identity`. - // We have removed the `IdentityOrRemoteIdentity` type so we have disabled this change and the V9 -> V10 will take care of it. - // instance::identity::set( - // // This code is assuming you only have the current node. - // // If you've paired your node with another node, reset your db. - // IdentityOrRemoteIdentity::Identity( - // Identity::from_bytes(&i.identity).expect( - // "Invalid identity detected in DB during migrations", - // ), - // ) - // .to_bytes(), - // ), - ], - ) - }) - .collect::>(), - ) - .await?; - } - - (LibraryConfigVersion::V9, LibraryConfigVersion::V10) => { - db._batch( - db.instance() - .find_many(vec![]) - .exec() - .await? - .into_iter() - .filter_map(|i| { - let identity = i.identity?; - - let (remote_identity, identity) = if identity[0] == b'I' { - // We have an `IdentityOrRemoteIdentity::Identity` - let identity = Identity::from_bytes(&identity[1..]).expect( - "Invalid identity detected in DB during migrations - 1", - ); - - (identity.to_remote_identity(), Some(identity)) - } else if identity[0] == b'R' { - // We have an `IdentityOrRemoteIdentity::RemoteIdentity` - let identity = RemoteIdentity::from_bytes(&identity[1..]) - .expect( - "Invalid identity detected in DB during migrations - 2", - ); - - (identity, None) - } else { - // We have an `Identity` or an invalid column. - let identity = Identity::from_bytes(&identity).expect( - "Invalid identity detected in DB during migrations - 3", - ); - - (identity.to_remote_identity(), Some(identity)) - }; - - Some(db.instance().update( - instance::id::equals(i.id), - vec![ - instance::identity::set(identity.map(|i| i.to_bytes())), - instance::remote_identity::set( - remote_identity.get_bytes().to_vec(), - ), - ], - )) - }) - .collect::>(), - ) - .await?; - } - - (LibraryConfigVersion::V10, LibraryConfigVersion::V11) => { - db.instance() - .update_many( - vec![], - vec![instance::node_remote_identity::set(Some( - // This is a remote identity that doesn't exist. The expectation is that: - // - The current node will update it's own and notice the change causing it to push the updated id to the cloud - // - All other instances will be updated when the regular sync process with the cloud happens - "SaEhml9thV088ocsOXZ17BrNjFaROB0ojwBvnPHhztI".into(), - ))], - ) - .exec() - .await?; - } - - (LibraryConfigVersion::V11, LibraryConfigVersion::V12) => { - // Add the `cloud_email_address` field to the library config. - let mut config = serde_json::from_slice::>( - &fs::read(path).await.map_err(|e| { - VersionManagerError::FileIO(FileIOError::from((path, e))) - })?, - ) - .map_err(VersionManagerError::SerdeJson)?; - - config.insert(String::from("cloud_email_address"), Value::Null); - - fs::write( - path, - &serde_json::to_vec(&config).map_err(VersionManagerError::SerdeJson)?, - ) - .await - .map_err(|e| VersionManagerError::FileIO(FileIOError::from((path, e))))?; - } - - _ => { - error!(current_version = ?current, "Library config version is not handled;"); - - return Err(VersionManagerError::UnexpectedMigration { - current_version: current.int_value(), - next_version: next.int_value(), - } - .into()); - } - } - Ok(()) - }, - ) - .await?; - - loaded_config.config_path = path.to_path_buf(); - - Ok(loaded_config) - } - - pub(crate) async fn save(&self, path: impl AsRef) -> Result<(), LibraryConfigError> { - let path = path.as_ref(); - fs::write(path, &serde_json::to_vec(self)?) + /// Load library configuration from a JSON file + pub async fn load(path: &std::path::Path) -> Result { + let config_data = tokio::fs::read_to_string(path) .await - .map_err(|e| FileIOError::from((path, e)).into()) + .map_err(|e| super::error::LibraryError::IoError(e))?; + let config: LibraryConfig = serde_json::from_str(&config_data) + .map_err(|e| super::error::LibraryError::JsonError(e))?; + Ok(config) } } -#[derive(Error, Debug)] -pub enum LibraryConfigError { - #[error("database error: {0}")] - Database(#[from] prisma_client_rust::QueryError), - #[error("there are too many nodes in the database, this should not happen!")] - TooManyNodes, - #[error("there are too many instances in the database, this should not happen!")] - TooManyInstances, - #[error("missing instances")] - MissingInstance, - #[error("your library version can't be automatically updated, please recreate your library")] - CriticalUpdateError, - - #[error(transparent)] - SerdeJson(#[from] serde_json::Error), - #[error(transparent)] - VersionManager(#[from] VersionManagerError), - #[error(transparent)] - FileIO(#[from] FileIOError), +impl Default for LibrarySettings { + fn default() -> Self { + Self { + generate_thumbnails: true, + thumbnail_quality: 85, + enable_ai_tagging: false, + sync_enabled: false, + encryption_enabled: false, + thumbnail_sizes: vec![128, 256, 512], + ignored_extensions: vec![ + ".tmp".to_string(), + ".temp".to_string(), + ".cache".to_string(), + ".part".to_string(), + ], + max_file_size: Some(100 * 1024 * 1024 * 1024), // 100GB + auto_track_system_volumes: true, // Default to true for user convenience + auto_track_external_volumes: false, // Default to false for privacy + indexer: IndexerSettings::default(), + } + } +} + +/// Indexer settings controlling rule toggles +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct IndexerSettings { + #[serde(default = "IndexerSettings::default_true")] + pub no_system_files: bool, + #[serde(default = "IndexerSettings::default_true")] + pub no_git: bool, + #[serde(default = "IndexerSettings::default_true")] + pub no_dev_dirs: bool, + #[serde(default)] + pub no_hidden: bool, + #[serde(default = "IndexerSettings::default_true")] + pub gitignore: bool, + #[serde(default)] + pub only_images: bool, +} + +impl IndexerSettings { + fn default_true() -> bool { + true + } +} + +impl Default for IndexerSettings { + fn default() -> Self { + Self { + no_system_files: true, + no_git: true, + no_dev_dirs: true, + no_hidden: false, + gitignore: true, + only_images: false, + } + } +} + +/// Library statistics +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct LibraryStatistics { + /// Total number of files indexed + pub total_files: u64, + + /// Total size of all files in bytes + pub total_size: u64, + + /// Number of locations in this library + pub location_count: u32, + + /// Number of tags created + pub tag_count: u32, + + /// Number of thumbnails generated + pub thumbnail_count: u64, + + /// Last time the library was fully indexed + pub last_indexed: Option>, + + /// When these statistics were last updated + pub updated_at: DateTime, +} + +impl Default for LibraryStatistics { + fn default() -> Self { + Self { + total_files: 0, + total_size: 0, + location_count: 0, + tag_count: 0, + thumbnail_count: 0, + last_indexed: None, + updated_at: Utc::now(), + } + } +} + +/// Thumbnail generation metadata +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ThumbnailMetadata { + /// Version of the thumbnail format + pub version: u32, + + /// Quality setting used for generation + pub quality: u8, + + /// Sizes that were generated + pub sizes: Vec, + + /// When this metadata was created + pub created_at: DateTime, +} + +impl Default for ThumbnailMetadata { + fn default() -> Self { + Self { + version: 1, + quality: 85, + sizes: vec![128, 256, 512], + created_at: Utc::now(), + } + } } diff --git a/core-new/src/library/error.rs b/core/src/library/error.rs similarity index 100% rename from core-new/src/library/error.rs rename to core/src/library/error.rs diff --git a/core/src/library/library.rs b/core/src/library/library.rs deleted file mode 100644 index 62d132b20..000000000 --- a/core/src/library/library.rs +++ /dev/null @@ -1,218 +0,0 @@ -use crate::{api::CoreEvent, Node}; - -use sd_core_cloud_services::{declare_cloud_sync, CloudSyncActors, CloudSyncActorsState}; -use sd_core_file_path_helper::IsolatedFilePathData; -use sd_core_heavy_lifting::media_processor::ThumbnailKind; -use sd_core_prisma_helpers::{file_path_to_full_path, CasId}; -use sd_core_sync::{backfill::backfill_operations, SyncManager}; - -use sd_actors::ActorsCollection; -use sd_cloud_schema::sync::groups; -use sd_crypto::{CryptoRng, SeedableRng}; -use sd_old_p2p::Identity; -use sd_prisma::prisma::{file_path, location, PrismaClient}; -use sd_utils::{db::maybe_missing, error::FileIOError}; - -use std::{ - collections::HashMap, - fmt::{Debug, Formatter}, - path::{Path, PathBuf}, - sync::{atomic::Ordering, Arc}, -}; - -use futures_concurrency::future::Join; -use tokio::{fs, io, sync::broadcast, sync::RwLock}; -use tracing::{debug, warn}; -use uuid::Uuid; - -use super::{LibraryConfig, LibraryManagerError}; - -pub struct Library { - /// id holds the ID of the current library. - pub id: Uuid, - /// config holds the configuration of the current library. - /// KEEP PRIVATE: Access through `Self::config` method. - config: RwLock, - /// db holds the database client for the current library. - pub db: Arc, - pub sync: SyncManager, - - /// key manager that provides encryption keys to functions that require them - // pub key_manager: Arc, - /// p2p identity - pub identity: Arc, - // pub orphan_remover: OrphanRemoverActor, - // The UUID which matches `config.instance_id`'s primary key. - pub instance_uuid: Uuid, - - // Look, I think this shouldn't be here but our current invalidation system needs it. - // TODO(@Oscar): Get rid of this with the new invalidation system. - event_bus_tx: broadcast::Sender, - - pub cloud_sync_state: CloudSyncActorsState, - pub cloud_sync_actors: ActorsCollection, -} - -impl Debug for Library { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - // Rolling out this implementation because `NodeContext` contains a DynJob which is - // troublesome to implement Debug trait - f.debug_struct("LibraryContext") - .field("id", &self.id) - .field("instance_uuid", &self.instance_uuid) - .field("config", &self.config) - .field("db", &self.db) - .finish() - } -} - -impl Library { - pub async fn new( - id: Uuid, - config: LibraryConfig, - instance_uuid: Uuid, - identity: Arc, - db: Arc, - node: &Arc, - sync: SyncManager, - ) -> Arc { - Arc::new(Self { - id, - config: RwLock::new(config), - sync, - db: db.clone(), - identity, - // orphan_remover: OrphanRemoverActor::spawn(db), - instance_uuid, - event_bus_tx: node.event_bus.0.clone(), - cloud_sync_state: CloudSyncActorsState::default(), - cloud_sync_actors: ActorsCollection::default(), - }) - } - - pub async fn init_cloud_sync( - &self, - node: &Node, - sync_group_pub_id: groups::PubId, - ) -> Result<(), LibraryManagerError> { - let rng = CryptoRng::from_seed(node.master_rng.lock().await.generate_fixed()); - - self.update_config(|config| { - config - .generate_sync_operations - .store(true, Ordering::Relaxed) - }) - .await?; - - // If this library doesn't have any sync operations, it means that it had sync activated - // for the first time, so we need to backfill the operations from existing db data - if self.db.crdt_operation().count(vec![]).exec().await? == 0 { - backfill_operations(&self.sync).await?; - } - - declare_cloud_sync( - node.data_dir.clone().into_boxed_path(), - node.cloud_services.clone(), - &self.cloud_sync_actors, - &self.cloud_sync_state, - sync_group_pub_id, - self.sync.clone(), - rng, - ) - .await?; - - ( - self.cloud_sync_actors.start(CloudSyncActors::Sender), - self.cloud_sync_actors.start(CloudSyncActors::Receiver), - self.cloud_sync_actors.start(CloudSyncActors::Ingester), - ) - .join() - .await; - - debug!(library_id = %self.id, "Started cloud sync actors"); - - Ok(()) - } - - pub async fn config(&self) -> LibraryConfig { - self.config.read().await.clone() - } - - pub async fn update_config( - &self, - update_fn: impl FnOnce(&mut LibraryConfig), - ) -> Result<(), LibraryManagerError> { - let mut config = self.config.write().await; - - update_fn(&mut config); - - config.save(&config.config_path).await.map_err(Into::into) - } - - // TODO: Remove this once we replace the old invalidation system - pub(crate) fn emit(&self, event: CoreEvent) { - if let Err(e) = self.event_bus_tx.send(event) { - warn!(?e, "Error sending event to event bus;"); - } - } - - pub async fn thumbnail_exists( - &self, - node: &Node, - cas_id: &CasId<'_>, - ) -> Result { - let thumb_path = - ThumbnailKind::Indexed(self.id).compute_path(node.config.data_directory(), cas_id); - - match fs::metadata(&thumb_path).await { - Ok(_) => Ok(true), - Err(e) if e.kind() == io::ErrorKind::NotFound => Ok(false), - Err(e) => Err(FileIOError::from((thumb_path, e))), - } - } - - /// Returns the full path of a file - pub async fn get_file_paths( - &self, - ids: Vec, - ) -> Result>, LibraryManagerError> { - let mut out = ids - .iter() - .copied() - .map(|id| (id, None)) - .collect::>(); - - out.extend( - self.db - .file_path() - .find_many(vec![ - // TODO(N): This isn't gonna work with removable media and this will likely permanently break if the DB is restored from a backup. - file_path::location::is(vec![location::instance_id::equals(Some( - self.config().await.instance_id, - ))]), - file_path::id::in_vec(ids), - ]) - .select(file_path_to_full_path::select()) - .exec() - .await? - .into_iter() - .flat_map(|file_path| { - let location = maybe_missing(&file_path.location, "file_path.location")?; - - Ok::<_, LibraryManagerError>(( - file_path.id, - location - .path - .as_ref() - .map(|location_path| { - IsolatedFilePathData::try_from((location.id, &file_path)) - .map(|data| Path::new(&location_path).join(data)) - }) - .transpose()?, - )) - }), - ); - - Ok(out) - } -} diff --git a/core-new/src/library/lock.rs b/core/src/library/lock.rs similarity index 100% rename from core-new/src/library/lock.rs rename to core/src/library/lock.rs diff --git a/core-new/src/library/manager.rs b/core/src/library/manager.rs similarity index 100% rename from core-new/src/library/manager.rs rename to core/src/library/manager.rs diff --git a/core/src/library/manager/error.rs b/core/src/library/manager/error.rs deleted file mode 100644 index de5aef2cb..000000000 --- a/core/src/library/manager/error.rs +++ /dev/null @@ -1,65 +0,0 @@ -use crate::{library::LibraryConfigError, location::LocationManagerError, volume}; - -use sd_core_indexer_rules::seed::SeederError; -use sd_core_sync::DevicePubId; - -use sd_old_p2p::IdentityErr; -use sd_utils::{ - db::{self, MissingFieldError}, - error::{FileIOError, NonUtf8PathError}, -}; - -use tracing::error; - -#[derive(thiserror::Error, Debug)] -pub enum LibraryManagerError { - #[error("error serializing or deserializing the JSON in the config file: {0}")] - Json(#[from] serde_json::Error), - #[error("database error: {0}")] - Database(#[from] prisma_client_rust::QueryError), - #[error("library not found error")] - LibraryNotFound, - #[error("failed to parse uuid: {0}")] - Uuid(#[from] uuid::Error), - #[error("failed to run indexer rules seeder: {0}")] - IndexerRulesSeeder(#[from] SeederError), - #[error("error migrating the library: {0}")] - MigrationError(#[from] db::MigrationError), - #[error("invalid library configuration: {0}")] - InvalidConfig(String), - #[error(transparent)] - NonUtf8Path(#[from] NonUtf8PathError), - #[error("failed to watch locations: {0}")] - LocationWatcher(#[from] LocationManagerError), - #[error("failed to parse library p2p identity: {0}")] - Identity(#[from] IdentityErr), - #[error("failed to load private key for instance p2p identity")] - InvalidIdentity, - #[error("current instance with id '{0}' was not found in the database")] - CurrentInstanceNotFound(String), - #[error("current device with pub id '{0}' was not found in the database")] - CurrentDeviceNotFound(DevicePubId), - #[error("missing-field: {0}")] - MissingField(#[from] MissingFieldError), - #[error("Error in volumes: {0}")] - VolumeError(#[from] volume::VolumeError), - - #[error(transparent)] - FileIO(#[from] FileIOError), - #[error(transparent)] - LibraryConfig(#[from] LibraryConfigError), - #[error(transparent)] - CloudServices(#[from] sd_core_cloud_services::Error), - #[error(transparent)] - Sync(#[from] sd_core_sync::Error), -} - -impl From for rspc::Error { - fn from(error: LibraryManagerError) -> Self { - rspc::Error::with_cause( - rspc::ErrorCode::InternalServerError, - error.to_string(), - error, - ) - } -} diff --git a/core/src/library/manager/mod.rs b/core/src/library/manager/mod.rs deleted file mode 100644 index 8a7dc5ae2..000000000 --- a/core/src/library/manager/mod.rs +++ /dev/null @@ -1,697 +0,0 @@ -use crate::{ - api::{utils::InvalidateOperationEvent, CoreEvent}, - invalidate_query, - location::metadata::{LocationMetadataError, SpacedriveLocationMetadataFile}, - object::tag, - old_p2p, - util::{mpscrr, MaybeUndefined}, - Node, -}; - -use sd_core_sync::{DevicePubId, SyncEvent, SyncManager}; - -use sd_old_p2p::{Identity, RemoteIdentity}; -use sd_prisma::{ - prisma::{self, device, instance, location, PrismaClient}, - prisma_sync, -}; -use sd_sync::ModelId; -use sd_utils::{ - db, - error::{FileIOError, NonUtf8PathError}, -}; - -use std::{ - collections::HashMap, - path::{Path, PathBuf}, - str::FromStr, - sync::{ - atomic::{AtomicBool, Ordering}, - Arc, - }, -}; - -use chrono::Utc; -use futures_concurrency::future::{Join, TryJoin}; -use prisma_client_rust::Raw; -use tokio::{ - fs, io, spawn, - sync::{broadcast, RwLock}, -}; -use tracing::{debug, error, info, instrument, warn}; -use uuid::Uuid; - -use super::{Library, LibraryConfig, LibraryName}; - -mod error; - -pub mod pragmas; - -use pragmas::configure_pragmas; - -pub use error::*; - -/// Event that is emitted to subscribers of the library manager. -#[derive(Debug, Clone)] -pub enum LibraryManagerEvent { - Load(Arc), - Edit(Arc), - // TODO(@Oscar): Replace this with pairing -> ready state transitions - InstancesModified(Arc), - Delete(Arc), -} - -/// is a singleton that manages all libraries for a node. -pub struct Libraries { - /// libraries_dir holds the path to the directory where libraries are stored. - pub libraries_dir: PathBuf, - /// libraries holds the list of libraries which are currently loaded into the node. - libraries: RwLock>>, - // Transmit side of `self.rx` channel - tx: mpscrr::Sender, - /// A channel for receiving events from the library manager. - pub rx: mpscrr::Receiver, - pub emit_messages_flag: Arc, -} - -impl Libraries { - pub(crate) async fn new(libraries_dir: PathBuf) -> Result, LibraryManagerError> { - fs::create_dir_all(&libraries_dir) - .await - .map_err(|e| FileIOError::from((&libraries_dir, e)))?; - - let (tx, rx) = mpscrr::unbounded_channel(); - Ok(Arc::new(Self { - libraries_dir, - libraries: Default::default(), - tx, - rx, - emit_messages_flag: Arc::new(AtomicBool::new(false)), - })) - } - - /// Loads the initial libraries from disk. - /// - /// `Arc` is constructed and passed to other managers for them to subscribe (`self.rx.subscribe`) then this method is run to load the initial libraries and trigger the subscriptions. - pub async fn init(self: &Arc, node: &Arc) -> Result<(), LibraryManagerError> { - let mut read_dir = fs::read_dir(&self.libraries_dir) - .await - .map_err(|e| FileIOError::from((&self.libraries_dir, e)))?; - - while let Some(entry) = read_dir - .next_entry() - .await - .map_err(|e| FileIOError::from((&self.libraries_dir, e)))? - { - let config_path = entry.path(); - if config_path - .extension() - .map(|ext| ext == "sdlibrary") - .unwrap_or(false) - && entry - .metadata() - .await - .map_err(|e| FileIOError::from((&config_path, e)))? - .is_file() - { - let Some(Ok(library_id)) = config_path - .file_stem() - .and_then(|v| v.to_str().map(Uuid::from_str)) - else { - warn!( - config_path = %config_path.display(), - "Attempted to load library from path \ - but it has an invalid filename. Skipping...;", - ); - continue; - }; - - let db_path = config_path.with_extension("db"); - match fs::metadata(&db_path).await { - Ok(_) => {} - Err(e) if e.kind() == io::ErrorKind::NotFound => { - warn!( - config_path = %config_path.display(), - "Found library but no matching database file was found. Skipping...;", - ); - - continue; - } - Err(e) => return Err(FileIOError::from((db_path, e)).into()), - } - - let _library_arc = self - .load(library_id, &db_path, config_path, None, None, true, node) - .await?; - } - } - - Ok(()) - } - - /// create creates a new library with the given config and mounts it into the running [LibraryManager]. - pub async fn create( - self: &Arc, - name: LibraryName, - description: Option, - node: &Arc, - ) -> Result, LibraryManagerError> { - self.create_with_uuid(Uuid::now_v7(), name, description, true, None, node) - .await - } - - #[instrument(skip(self, instance, node), err)] - pub(crate) async fn create_with_uuid( - self: &Arc, - id: Uuid, - name: LibraryName, - description: Option, - should_seed: bool, - // `None` will fallback to default as library must be created with at least one instance - instance: Option, - node: &Arc, - ) -> Result, LibraryManagerError> { - if name.as_ref().is_empty() || name.as_ref().chars().all(|x| x.is_whitespace()) { - return Err(LibraryManagerError::InvalidConfig( - "name cannot be empty".to_string(), - )); - } - - let config_path = self.libraries_dir.join(format!("{id}.sdlibrary")); - - let config = LibraryConfig::new( - name, - description, - // First instance will be zero - 0, - &config_path, - ) - .await?; - - debug!( - config_path = %config_path.display(), - "Created library;", - ); - - let node_cfg = node.config.get().await; - let now = Utc::now().fixed_offset(); - let library = self - .load( - id, - self.libraries_dir.join(format!("{id}.db")), - config_path, - Some(device::Create { - pub_id: node_cfg.id.to_db(), - _params: vec![ - device::name::set(Some(node_cfg.name.clone())), - device::os::set(Some(node_cfg.os as i32)), - device::hardware_model::set(Some(node_cfg.hardware_model as i32)), - device::date_created::set(Some(now)), - ], - }), - Some({ - let identity = Identity::new(); - let mut create = instance.unwrap_or_else(|| instance::Create { - pub_id: Uuid::now_v7().as_bytes().to_vec(), - remote_identity: identity.to_remote_identity().get_bytes().to_vec(), - node_id: node_cfg.id.to_db(), - last_seen: now, - date_created: now, - _params: vec![ - instance::identity::set(Some(identity.to_bytes())), - instance::metadata::set(Some( - serde_json::to_vec(&node.p2p.peer_metadata()) - .expect("invalid node metadata"), - )), - ], - }); - create._params.push(instance::id::set(config.instance_id)); - create - }), - should_seed, - node, - ) - .await?; - - debug!("Loaded library"); - - if should_seed { - tag::seed::new_library(&library).await?; - sd_core_indexer_rules::seed::new_or_existing_library(&library.db).await?; - debug!("Seeded library"); - } - - invalidate_query!(library, "library.list"); - - Ok(library) - } - - /// `LoadedLibrary.id` can be used to get the library's id. - pub async fn get_all(&self) -> Vec> { - self.libraries - .read() - .await - .iter() - .map(|v| v.1.clone()) - .collect() - } - - pub(crate) async fn edit( - &self, - id: Uuid, - name: Option, - description: MaybeUndefined, - cloud_id: MaybeUndefined, - enable_sync: Option, - ) -> Result<(), LibraryManagerError> { - // check library is valid - let libraries = self.libraries.read().await; - let library = Arc::clone( - libraries - .get(&id) - .ok_or(LibraryManagerError::LibraryNotFound)?, - ); - - library - .update_config(|config| { - // update the library - if let Some(name) = name { - config.name = name; - } - match description { - MaybeUndefined::Undefined => {} - MaybeUndefined::Null => config.description = None, - MaybeUndefined::Value(description) => config.description = Some(description), - } - match cloud_id { - MaybeUndefined::Undefined => {} - MaybeUndefined::Null => config.cloud_id = None, - MaybeUndefined::Value(cloud_id) => config.cloud_id = Some(cloud_id), - } - match enable_sync { - None => {} - Some(value) => config - .generate_sync_operations - .store(value, Ordering::SeqCst), - } - }) - .await?; - - self.tx - .emit(LibraryManagerEvent::Edit(Arc::clone(&library))) - .await; - - invalidate_query!(library, "library.list"); - - Ok(()) - } - - pub async fn delete(&self, id: &Uuid) -> Result<(), LibraryManagerError> { - // As we're holding a write lock here, we know nothing will change during this function - let mut libraries_write_guard = self.libraries.write().await; - - // TODO: Library go into "deletion" state until it's finished! - - let library = libraries_write_guard - .get(id) - .ok_or(LibraryManagerError::LibraryNotFound)?; - - self.tx - .emit(LibraryManagerEvent::Delete(library.clone())) - .await; - - if let Ok(location_paths) = library - .db - .location() - .find_many(vec![]) - .select(location::select!({ path })) - .exec() - .await - .map(|locations| locations.into_iter().filter_map(|location| location.path)) - .map_err(|e| error!(?e, "Failed to fetch locations for library deletion;")) - { - location_paths - .map(|location_path| async move { - if let Some(mut sd_metadata) = - SpacedriveLocationMetadataFile::try_load(location_path).await? - { - sd_metadata.remove_library(*id).await?; - } - - Ok::<_, LocationMetadataError>(()) - }) - .collect::>() - .join() - .await - .into_iter() - .for_each(|res| { - if let Err(e) = res { - error!(?e, "Failed to remove library from location metadata;"); - } - }); - } - - let db_path = self.libraries_dir.join(format!("{}.db", library.id)); - let sd_lib_path = self.libraries_dir.join(format!("{}.sdlibrary", library.id)); - - ( - async { - fs::remove_file(&db_path) - .await - .map_err(|e| LibraryManagerError::FileIO(FileIOError::from((db_path, e)))) - }, - async { - fs::remove_file(&sd_lib_path) - .await - .map_err(|e| LibraryManagerError::FileIO(FileIOError::from((sd_lib_path, e)))) - }, - ) - .try_join() - .await?; - - // We only remove here after files deletion - let library = libraries_write_guard - .remove(id) - .expect("we have exclusive access and checked it exists!"); - - info!(%library.id, "Removed Library;"); - - invalidate_query!(library, "library.list"); - - Ok(()) - } - - // get_ctx will return the library context for the given library id. - pub async fn get_library(&self, library_id: &Uuid) -> Option> { - self.libraries.read().await.get(library_id).cloned() - } - - // will return the library context for the given instance - pub async fn get_library_for_instance( - &self, - instance: &RemoteIdentity, - ) -> Option> { - self.libraries - .read() - .await - .iter() - .map(|(_, library)| async move { - library - .db - .instance() - .find_many(vec![instance::remote_identity::equals( - instance.get_bytes().to_vec(), - )]) - .exec() - .await - .ok() - .iter() - .flatten() - .filter_map(|i| RemoteIdentity::from_bytes(&i.remote_identity).ok()) - .any(|i| i == *instance) - .then(|| Arc::clone(library)) - }) - .collect::>() - .join() - .await - .into_iter() - .find_map(|v| v) - } - - // get_ctx will return the library context for the given library id. - pub async fn hash_library(&self, library_id: &Uuid) -> bool { - self.libraries.read().await.get(library_id).is_some() - } - - #[allow(clippy::too_many_arguments)] // TODO: remove this when we remove instance stuff - #[instrument( - skip_all, - fields( - library_id = %id, - db_path = %db_path.as_ref().display(), - config_path = %config_path.as_ref().display(), - %should_seed, - ), - err, - )] - /// load the library from a given path. - pub async fn load( - self: &Arc, - id: Uuid, - db_path: impl AsRef, - config_path: impl AsRef, - maybe_create_device: Option, - maybe_create_instance: Option, // Deprecated - should_seed: bool, - node: &Arc, - ) -> Result, LibraryManagerError> { - let db_path = db_path.as_ref(); - let config_path = config_path.as_ref(); - - let db_url = format!( - "file:{}?socket_timeout=15&connection_limit=1", - db_path.as_os_str().to_str().ok_or_else(|| { - LibraryManagerError::NonUtf8Path(NonUtf8PathError(db_path.into())) - })? - ); - let db = Arc::new(db::load_and_migrate(&db_url).await?); - - // Configure database - configure_pragmas(&db).await?; - special_sync_indexes(&db).await?; - - if let Some(create) = maybe_create_device { - create.to_query(&db).exec().await?; - } - - // TODO: remove instances from locations - if let Some(create) = maybe_create_instance { - create.to_query(&db).exec().await?; - } - - let node_config = node.config.get().await; - let device_pub_id: DevicePubId = node_config.id.clone(); - let config = LibraryConfig::load(config_path, &node_config, &db).await?; - - let instances = db.instance().find_many(vec![]).exec().await?; - - let instance = instances - .iter() - .find(|i| i.id == config.instance_id) - .ok_or_else(|| { - LibraryManagerError::CurrentInstanceNotFound(config.instance_id.to_string()) - })? - .clone(); - - let devices = db.device().find_many(vec![]).exec().await?; - - let device_pub_id_to_db = device_pub_id.to_db(); - if !devices - .iter() - .any(|device| device.pub_id == device_pub_id_to_db) - { - // return Err(LibraryManagerError::CurrentDeviceNotFound(device_pub_id)); - // Create the device if it doesn't exist - device::Create { - pub_id: device_pub_id_to_db, - _params: vec![ - device::name::set(Some(node_config.name.clone())), - device::os::set(Some(node_config.os as i32)), - device::hardware_model::set(Some(node_config.hardware_model as i32)), - device::date_created::set(Some(Utc::now().fixed_offset())), - ], - } - .to_query(&db) - .exec() - .await?; - } - - let identity = match instance.identity.as_ref() { - Some(b) => Arc::new(Identity::from_bytes(b)?), - // We are not this instance, so we don't have the private key. - None => return Err(LibraryManagerError::InvalidIdentity), - }; - - let instance_id = Uuid::from_slice(&instance.pub_id)?; - let curr_metadata: Option> = instance - .metadata - .as_ref() - .map(|metadata| serde_json::from_slice(metadata).expect("invalid metadata")); - let instance_node_id = Uuid::from_slice(&instance.node_id)?; - let instance_node_remote_identity = instance - .node_remote_identity - .as_ref() - .and_then(|v| RemoteIdentity::from_bytes(v).ok()); - if instance_node_id != Uuid::from(&node_config.id) - || instance_node_remote_identity != Some(node_config.identity.to_remote_identity()) - || curr_metadata != Some(node.p2p.peer_metadata()) - { - info!( - old_node_id = %instance_node_id, - new_node_id = %node_config.id, - "Detected that the library has changed nodes. Reconciling node data...", - ); - - // ensure - - db.instance() - .update( - instance::id::equals(instance.id), - vec![ - instance::node_id::set(node_config.id.to_db()), - instance::node_remote_identity::set(Some( - node_config - .identity - .to_remote_identity() - .get_bytes() - .to_vec(), - )), - instance::metadata::set(Some( - serde_json::to_vec(&node.p2p.peer_metadata()) - .expect("invalid peer metadata"), - )), - ], - ) - .select(instance::select!({ id })) - .exec() - .await?; - } - - // TODO: Move this reconciliation into P2P and do reconciliation of both local and remote nodes. - - let (sync, sync_rx) = SyncManager::with_existing_devices( - Arc::clone(&db), - &device_pub_id, - Arc::clone(&config.generate_sync_operations), - &devices, - ) - .await?; - - let library = Library::new(id, config, instance_id, identity, db, node, sync).await; - - // This is an exception. Generally subscribe to this by `self.tx.subscribe`. - spawn(sync_rx_actor(library.clone(), node.clone(), sync_rx)); - - self.tx - .emit(LibraryManagerEvent::Load(library.clone())) - .await; - - self.libraries - .write() - .await - .insert(library.id, Arc::clone(&library)); - - if should_seed { - // library.orphan_remover.invoke().await; - sd_core_indexer_rules::seed::new_or_existing_library(&library.db).await?; - } - - for location in library - .db - .location() - .find_many(vec![ - // TODO(N): This isn't gonna work with removable media and this will likely permanently break if the DB is restored from a backup. - location::instance_id::equals(Some(instance.id)), - ]) - .exec() - .await? - { - if let Err(e) = node.locations.add(location.id, library.clone()).await { - error!(?e, "Failed to watch location on startup;"); - }; - } - - if let Err(e) = node.old_jobs.clone().cold_resume(node, &library).await { - error!(?e, "Failed to resume jobs for library;"); - } - - Ok(library) - } - - pub async fn update_instances(&self, library: Arc) { - self.tx - .emit(LibraryManagerEvent::InstancesModified(library)) - .await; - } - - pub async fn update_instances_by_id(&self, library_id: Uuid) { - let Some(library) = self.libraries.read().await.get(&library_id).cloned() else { - warn!("Failed to find instance to update by id"); - return; - }; - - self.tx - .emit(LibraryManagerEvent::InstancesModified(library)) - .await; - } -} - -async fn sync_rx_actor( - library: Arc, - node: Arc, - mut sync_rx: broadcast::Receiver, -) { - loop { - let Ok(msg) = sync_rx.recv().await else { - continue; - }; - - match msg { - // TODO: Any sync event invalidates the entire React Query cache this is a hacky workaround until the new invalidation system. - SyncEvent::Ingested => node.emit(CoreEvent::InvalidateOperation( - InvalidateOperationEvent::all(), - )), - SyncEvent::Created => { - old_p2p::sync::originator(library.clone(), &library.sync, &node.p2p).await - } - } - } -} - -async fn special_sync_indexes(db: &PrismaClient) -> Result<(), LibraryManagerError> { - async fn create_index( - db: &PrismaClient, - model_id: ModelId, - model_name: &str, - ) -> Result<(), LibraryManagerError> { - db._execute_raw(Raw::new( - &format!( - "CREATE INDEX IF NOT EXISTS partial_index_model_{model_name} \ - ON crdt_operation(model,record_id,kind,timestamp) \ - WHERE model = {model_id} - " - ), - vec![], - )) - .exec() - .await?; - - debug!(model_name, "Created sync partial index"); - - Ok(()) - } - - for (model_id, model_name) in [ - (prisma_sync::device::MODEL_ID, prisma::device::NAME), - (prisma_sync::volume::MODEL_ID, prisma::volume::NAME), - (prisma_sync::tag::MODEL_ID, prisma::tag::NAME), - (prisma_sync::location::MODEL_ID, prisma::location::NAME), - (prisma_sync::object::MODEL_ID, prisma::object::NAME), - (prisma_sync::label::MODEL_ID, prisma::label::NAME), - (prisma_sync::exif_data::MODEL_ID, prisma::exif_data::NAME), - (prisma_sync::file_path::MODEL_ID, prisma::file_path::NAME), - ( - prisma_sync::tag_on_object::MODEL_ID, - prisma::tag_on_object::NAME, - ), - ( - prisma_sync::label_on_object::MODEL_ID, - prisma::label_on_object::NAME, - ), - ] { - // Creating indexes sequentially just in case - create_index(db, model_id, model_name).await?; - } - - Ok(()) -} diff --git a/core/src/library/manager/pragmas.rs b/core/src/library/manager/pragmas.rs deleted file mode 100644 index 792846e2f..000000000 --- a/core/src/library/manager/pragmas.rs +++ /dev/null @@ -1,109 +0,0 @@ -use crate::library::LibraryManagerError; -use prisma_client_rust::raw; -use sd_prisma::prisma::PrismaClient; -use tracing::{error, info}; - -// ************************************** -// pragmas to optimize SQLite performance -// ************************************** - -#[derive(serde::Deserialize, Debug)] -#[allow(dead_code)] -struct JournalMode { - journal_mode: String, -} - -#[derive(serde::Deserialize, Debug)] -#[allow(dead_code)] -struct CacheSize { - cache_size: i32, -} - -#[derive(serde::Deserialize, Debug)] -#[allow(dead_code)] -struct MmapSize { - mmap_size: i64, -} - -#[derive(serde::Deserialize, Debug)] -#[allow(dead_code)] -struct JournalSizeLimit { - journal_size_limit: i64, -} - -async fn execute_pragma( - db: &PrismaClient, - query: &str, - description: &str, -) -> Result<(), LibraryManagerError> { - match db._query_raw::(raw!(query)).exec().await { - Ok(_) => { - info!("{}", description); - Ok(()) - } - Err(e) => { - error!("Failed to execute '{}': {:?}", description, e); - Err(e.into()) - } - } -} - -pub async fn configure_pragmas(db: &PrismaClient) -> Result<(), LibraryManagerError> { - let pragmas = vec![ - ( - // WAL (Write-Ahead Logging) mode allows SQLite to perform better in situations with concurrent reads and writes. - // It uses a separate write-ahead log file instead of overwriting the database file directly, enabling better performance and durability. - // This mode is commonly used to optimize database performance in environments where multiple transactions happen simultaneously. - "PRAGMA journal_mode = WAL;", - "Set journal mode to WAL", - Some("JournalMode"), - ), - ( - // The synchronous mode controls how often SQLite waits for data to be physically written to disk. - // Setting it to NORMAL means SQLite will wait for the data to be flushed to the disk at key points but not after every transaction. - // This mode balances durability with performance. It's faster than FULL (which waits on every transaction) and safer than OFF. - "PRAGMA synchronous = NORMAL;", - "Set synchronous to NORMAL", - None, - ), - ( - // mmap_size sets the maximum number of bytes that SQLite will map into memory when using memory-mapped I/O. - // 512MB (536870912 bytes) is a reasonable amount for systems with enough RAM, allowing SQLite to access data directly from memory, improving performance for large databases. - // A larger mmap_size typically results in better performance, especially for read-heavy operations, but it should not exceed available system memory. - "PRAGMA mmap_size = 536870912;", - "Set mmap_size to 512MB", - Some("MmapSize"), - ), - ( - // journal_size_limit sets a maximum size for the write-ahead log (WAL) file. - // Limiting it to 64MB (67108864 bytes) ensures that the WAL file doesn’t grow too large, which could otherwise consume excessive disk space. - // A smaller limit might cause SQLite to checkpoint (merge the WAL file back into the database) more often, which could slightly impact performance. - "PRAGMA journal_size_limit = 67108864;", - "Set journal size limit to 64MB", - Some("JournalSizeLimit"), - ), - ( - // cache_size defines how much memory SQLite should allocate for storing frequently accessed database pages in memory. - // Setting it to 10,000 pages allows SQLite to cache more pages in RAM, reducing the need to hit the disk frequently. - // Each page is typically 4KB, so 10,000 pages would equal around 40MB of cache, which can significantly improve performance for frequently accessed data. - "PRAGMA cache_size = 10000;", - "Set cache size to 10k pages", - Some("CacheSize"), - ), - ]; - - for (query, description, result_type) in pragmas { - match result_type { - Some("JournalMode") => execute_pragma::(db, query, description).await?, - Some("CacheSize") => execute_pragma::(db, query, description).await?, - Some("MmapSize") => execute_pragma::(db, query, description).await?, - Some("JournalSizeLimit") => { - execute_pragma::(db, query, description).await? - } - None => execute_pragma::<()>(db, query, description).await?, - _ => unreachable!(), - } - } - - Ok(()) -} diff --git a/core/src/library/mod.rs b/core/src/library/mod.rs index 49057177f..6e8740e21 100644 --- a/core/src/library/mod.rs +++ b/core/src/library/mod.rs @@ -1,14 +1,238 @@ +//! Library management system +//! +//! This module provides the core library functionality for Spacedrive. +//! Each library is a self-contained directory with its own database, +//! thumbnails, and other data. + mod config; -#[allow(clippy::module_inception)] -mod library; +mod error; +mod lock; mod manager; -mod name; -mod statistics; -pub use config::*; -pub use library::*; -pub use manager::*; -pub use name::*; -pub use statistics::*; +pub use config::{LibraryConfig, LibrarySettings, LibraryStatistics}; +pub use error::{LibraryError, Result}; +pub use lock::LibraryLock; +pub use manager::{LibraryManager, DiscoveredLibrary}; -pub type LibraryId = uuid::Uuid; +use crate::infrastructure::{ + database::Database, + jobs::manager::JobManager, +}; +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use tokio::sync::RwLock; +use uuid::Uuid; + +/// Represents an open Spacedrive library +pub struct Library { + /// Root directory of the library (the .sdlibrary folder) + path: PathBuf, + + /// Library configuration + config: RwLock, + + /// Database connection + db: Arc, + + /// Job manager for this library + jobs: Arc, + + /// Lock preventing concurrent access + _lock: LibraryLock, +} + +impl Library { + /// Get the library ID + pub fn id(&self) -> Uuid { + // Config is immutable for ID, so we can use try_read + self.config.try_read().map(|c| c.id).unwrap_or_else(|_| { + // This should never happen in practice + panic!("Failed to read library config for ID") + }) + } + + /// Get the library name + pub async fn name(&self) -> String { + self.config.read().await.name.clone() + } + + /// Get the library path + pub fn path(&self) -> &Path { + &self.path + } + + /// Get the database + pub fn db(&self) -> &Arc { + &self.db + } + + /// Get the job manager + pub fn jobs(&self) -> &Arc { + &self.jobs + } + + /// Get a copy of the current configuration + pub async fn config(&self) -> LibraryConfig { + self.config.read().await.clone() + } + + /// Update library configuration + pub async fn update_config(&self, f: F) -> Result<()> + where + F: FnOnce(&mut LibraryConfig), + { + let mut config = self.config.write().await; + f(&mut config); + config.updated_at = chrono::Utc::now(); + + // Save to disk + let config_path = self.path.join("library.json"); + let json = serde_json::to_string_pretty(&*config)?; + tokio::fs::write(config_path, json).await?; + + Ok(()) + } + + /// Save library configuration to disk + pub async fn save_config(&self, config: &LibraryConfig) -> Result<()> { + let config_path = self.path.join("library.json"); + let json = serde_json::to_string_pretty(config)?; + tokio::fs::write(config_path, json).await?; + Ok(()) + } + + /// Get the thumbnail directory for this library + pub fn thumbnails_dir(&self) -> PathBuf { + self.path.join("thumbnails") + } + + /// Get the path for a specific thumbnail with size + pub fn thumbnail_path(&self, cas_id: &str, size: u32) -> PathBuf { + if cas_id.len() < 4 { + // Fallback for short IDs + return self.thumbnails_dir().join(format!("{}_{}.webp", cas_id, size)); + } + + // Two-level sharding based on first four characters + let shard1 = &cas_id[0..2]; + let shard2 = &cas_id[2..4]; + + self.thumbnails_dir() + .join(shard1) + .join(shard2) + .join(format!("{}_{}.webp", cas_id, size)) + } + + /// Get the path for any thumbnail size (legacy compatibility) + pub fn thumbnail_path_legacy(&self, cas_id: &str) -> PathBuf { + self.thumbnail_path(cas_id, 256) // Default to 256px + } + + /// Save a thumbnail with specific size + pub async fn save_thumbnail(&self, cas_id: &str, size: u32, data: &[u8]) -> Result<()> { + let path = self.thumbnail_path(cas_id, size); + + // Ensure parent directory exists + if let Some(parent) = path.parent() { + tokio::fs::create_dir_all(parent).await?; + } + + // Write thumbnail + tokio::fs::write(path, data).await?; + + Ok(()) + } + + /// Check if a thumbnail exists for a specific size + pub async fn has_thumbnail(&self, cas_id: &str, size: u32) -> bool { + tokio::fs::metadata(self.thumbnail_path(cas_id, size)) + .await + .is_ok() + } + + /// Shutdown the library, gracefully stopping all jobs + pub async fn shutdown(&self) -> Result<()> { + // Shutdown the job manager, which will pause all running jobs + self.jobs.shutdown().await?; + + // Save config to ensure any updates are persisted + let config = self.config.read().await; + self.save_config(&*config).await?; + + Ok(()) + } + + /// Check if thumbnails exist for all specified sizes + pub async fn has_all_thumbnails(&self, cas_id: &str, sizes: &[u32]) -> bool { + for &size in sizes { + if !self.has_thumbnail(cas_id, size).await { + return false; + } + } + true + } + + /// Get thumbnail data for specific size + pub async fn get_thumbnail(&self, cas_id: &str, size: u32) -> Result> { + let path = self.thumbnail_path(cas_id, size); + Ok(tokio::fs::read(path).await?) + } + + /// Get the best available thumbnail (largest size available) + pub async fn get_best_thumbnail(&self, cas_id: &str, preferred_sizes: &[u32]) -> Result)>> { + // Try sizes in descending order + let mut sizes = preferred_sizes.to_vec(); + sizes.sort_by(|a, b| b.cmp(a)); + + for &size in &sizes { + if self.has_thumbnail(cas_id, size).await { + let data = self.get_thumbnail(cas_id, size).await?; + return Ok(Some((size, data))); + } + } + + Ok(None) + } + + /// Start thumbnail generation job + pub async fn generate_thumbnails(&self, entry_ids: Option>) -> Result { + use crate::operations::media::thumbnail::{ThumbnailJob, ThumbnailJobConfig}; + + let config = ThumbnailJobConfig { + sizes: self.config().await.settings.thumbnail_sizes.clone(), + quality: self.config().await.settings.thumbnail_quality, + regenerate: false, + batch_size: 50, + max_concurrent: 4, + }; + + let job = if let Some(ids) = entry_ids { + ThumbnailJob::for_entries(ids, config) + } else { + ThumbnailJob::new(config) + }; + + self.jobs().dispatch(job).await + .map_err(|e| LibraryError::JobError(e)) + } + + /// Update library statistics + pub async fn update_statistics(&self, f: F) -> Result<()> + where + F: FnOnce(&mut LibraryStatistics), + { + self.update_config(|config| { + f(&mut config.statistics); + config.statistics.updated_at = chrono::Utc::now(); + }).await + } +} + +// Note: Library does not implement Clone due to the exclusive lock +// Use Arc when you need shared access + +/// Current library configuration version +pub const LIBRARY_CONFIG_VERSION: u32 = 2; + +/// Library directory extension +pub const LIBRARY_EXTENSION: &str = "sdlibrary"; \ No newline at end of file diff --git a/core/src/library/name.rs b/core/src/library/name.rs deleted file mode 100644 index 3e4350236..000000000 --- a/core/src/library/name.rs +++ /dev/null @@ -1,61 +0,0 @@ -use std::ops::Deref; - -use serde::{Deserialize, Serialize}; -use specta::Type; -use thiserror::Error; - -#[derive(Debug, Serialize, Clone, Type)] -pub struct LibraryName(String); - -#[derive(Debug, Error)] -pub enum LibraryNameError { - #[error("empty")] - Empty, - #[error("needs-trim")] - NeedsTrim, -} - -impl LibraryName { - pub fn new(name: impl Into) -> Result { - let name = name.into(); - - if name.is_empty() { - return Err(LibraryNameError::Empty); - } - - if name.starts_with(' ') || name.ends_with(' ') { - return Err(LibraryNameError::NeedsTrim); - } - - Ok(Self(name)) - } -} - -impl<'de> Deserialize<'de> for LibraryName { - fn deserialize(deserializer: D) -> Result - where - D: serde::Deserializer<'de>, - { - LibraryName::new(String::deserialize(deserializer)?).map_err(serde::de::Error::custom) - } -} - -impl AsRef for LibraryName { - fn as_ref(&self) -> &str { - &self.0 - } -} - -impl Deref for LibraryName { - type Target = String; - - fn deref(&self) -> &Self::Target { - &self.0 - } -} - -impl From for String { - fn from(name: LibraryName) -> Self { - name.0 - } -} diff --git a/core/src/library/statistics.rs b/core/src/library/statistics.rs deleted file mode 100644 index 8e376e6c7..000000000 --- a/core/src/library/statistics.rs +++ /dev/null @@ -1,122 +0,0 @@ -use crate::{api::utils::get_size, invalidate_query, library::Library, Node}; - -use sd_prisma::prisma::{statistics, volume}; -use sd_utils::db::size_in_bytes_from_db; - -use chrono::Utc; -use tracing::{debug, error, info}; - -use super::LibraryManagerError; - -pub async fn update_library_statistics( - node: &Node, - library: &Library, -) -> Result { - let (mut total_capacity, mut available_capacity) = library - .db - .volume() - .find_many(vec![]) - .select(volume::select!({ total_bytes_capacity total_bytes_available })) - .exec() - .await? - .into_iter() - .fold((0, 0), |(mut total, mut available), vol| { - total += vol - .total_bytes_capacity - .map(|c| c.parse::().unwrap_or(0)) - .unwrap_or(0); - available += vol - .total_bytes_available - .map(|a| a.parse::().unwrap_or(0)) - .unwrap_or(0); - (total, available) - }); - debug!( - ?total_capacity, - ?available_capacity, - "Fetched volume statistics;" - ); - - if total_capacity == 0 && available_capacity == 0 { - // Failed to fetch volume statistics from database, so we compute from local volumes - #[cfg(any(target_os = "linux", target_os = "macos"))] - let volumes = crate::volume::get_volumes().await?; - - #[cfg(target_os = "windows")] - let volumes = crate::volume::get_volumes().await; - - let mut local_total_capacity: u64 = 0; - let mut local_available_capacity: u64 = 0; - for volume in volumes { - local_total_capacity += volume.total_bytes_capacity; - local_available_capacity += volume.total_bytes_available; - } - - total_capacity = local_total_capacity; - available_capacity = local_available_capacity; - } - - let total_bytes_used = total_capacity - available_capacity; - - let library_db_size = get_size( - node.config - .data_directory() - .join("libraries") - .join(format!("{}.db", library.id)), - ) - .await - .unwrap_or(0); - - let total_library_bytes = library - .db - .location() - .find_many(vec![]) - .exec() - .await - .unwrap_or_else(|e| { - error!(?e, "Failed to get locations;"); - vec![] - }) - .into_iter() - .map(|location| { - location - .size_in_bytes - .map(|size| size_in_bytes_from_db(&size)) - .unwrap_or(0) - }) - .sum::(); - - let thumbnail_folder_size = get_size(node.config.data_directory().join("thumbnails")) - .await - .unwrap_or(0); - - use statistics::*; - let params = vec![ - id::set(1), // Each library is a database so only one of these ever exists - date_captured::set(Utc::now().into()), - total_object_count::set(0), - library_db_size::set(library_db_size.to_string()), - total_library_bytes::set(total_library_bytes.to_string()), - total_local_bytes_used::set(total_bytes_used.to_string()), - total_local_bytes_capacity::set(total_capacity.to_string()), - total_local_bytes_free::set(available_capacity.to_string()), - total_library_preview_media_bytes::set(thumbnail_folder_size.to_string()), - ]; - - let stats = library - .db - .statistics() - .upsert( - statistics::id::equals(1), - statistics::create(params.clone()), - params, - ) - .exec() - .await?; - - info!(?stats, "Updated library statistics;"); - - invalidate_query!(&library, "library.statistics"); - - Ok(stats) -} diff --git a/core/src/location/archive/archive_job.rs b/core/src/location/archive/archive_job.rs deleted file mode 100644 index e69de29bb..000000000 diff --git a/core/src/location/archive/mod.rs b/core/src/location/archive/mod.rs deleted file mode 100644 index e69de29bb..000000000 diff --git a/core/src/location/error.rs b/core/src/location/error.rs deleted file mode 100644 index 3e0f2b80e..000000000 --- a/core/src/location/error.rs +++ /dev/null @@ -1,115 +0,0 @@ -use sd_core_file_path_helper::FilePathError; - -use sd_prisma::prisma::location; -use sd_utils::{ - db::MissingFieldError, - error::{FileIOError, NonUtf8PathError}, -}; - -use std::path::Path; - -use rspc::ErrorCode; -use thiserror::Error; -use uuid::Uuid; - -use super::{manager::LocationManagerError, metadata::LocationMetadataError}; - -/// Error type for location related errors -#[derive(Error, Debug)] -pub enum LocationError { - // Not Found errors - #[error("location not found ", .0.display())] - PathNotFound(Box), - #[error("location not found ")] - UuidNotFound(Uuid), - #[error("location not found ")] - IdNotFound(location::id::Type), - - // User errors - #[error("location not a directory ", .0.display())] - NotDirectory(Box), - #[error("could not find directory in location ", .0.display())] - DirectoryNotFound(Box), - #[error( - "library exists in the location metadata file, must relink ", - .old_path.display(), - .new_path.display(), - )] - NeedRelink { - old_path: Box, - new_path: Box, - }, - #[error( - "this location belongs to another library, must update .spacedrive file ", - .0.display() - )] - AddLibraryToMetadata(Box), - #[error("location metadata file not found ", .0.display())] - MetadataNotFound(Box), - #[error("location already exists in database ", .0.display())] - LocationAlreadyExists(Box), - #[error("nested location currently not supported ", .0.display())] - NestedLocation(Box), - #[error(transparent)] - NonUtf8Path(#[from] NonUtf8PathError), - - // Internal Errors - #[error(transparent)] - LocationMetadata(#[from] LocationMetadataError), - #[error("failed to read location path metadata info: {0}")] - LocationPathFilesystemMetadataAccess(FileIOError), - #[error("missing metadata file for location ", .0.display())] - MissingMetadataFile(Box), - #[error("failed to open file from local OS: {0}")] - FileRead(FileIOError), - #[error("failed to read mounted volumes from local OS: {0}")] - VolumeReadError(String), - #[error("database error: {0}")] - Database(#[from] prisma_client_rust::QueryError), - #[error(transparent)] - LocationManager(#[from] LocationManagerError), - #[error(transparent)] - FilePath(#[from] FilePathError), - #[error(transparent)] - FileIO(#[from] FileIOError), - #[error("location missing path ")] - MissingPath(location::id::Type), - #[error("missing-field: {0}")] - MissingField(#[from] MissingFieldError), - #[error("invalid location scan state value: {0}")] - InvalidScanStateValue(i32), - #[error(transparent)] - Sync(#[from] sd_core_sync::Error), -} - -impl From for rspc::Error { - fn from(e: LocationError) -> Self { - use LocationError::*; - - match e { - // Not found errors - PathNotFound(_) - | UuidNotFound(_) - | IdNotFound(_) - | FilePath(FilePathError::IdNotFound(_) | FilePathError::NotFound(_)) => { - Self::with_cause(ErrorCode::NotFound, e.to_string(), e) - } - - // User's fault errors - NotDirectory(_) | NestedLocation(_) | LocationAlreadyExists(_) => { - Self::with_cause(ErrorCode::BadRequest, e.to_string(), e) - } - - // Custom error message is used to differentiate these errors in the frontend - // TODO: A better solution would be for rspc to support sending custom data alongside errors - NeedRelink { .. } => Self::with_cause(ErrorCode::Conflict, "NEED_RELINK".to_owned(), e), - AddLibraryToMetadata(_) => { - Self::with_cause(ErrorCode::Conflict, "ADD_LIBRARY".to_owned(), e) - } - - // Internal errors - MissingField(missing_error) => missing_error.into(), - _ => Self::with_cause(ErrorCode::InternalServerError, e.to_string(), e), - } - } -} diff --git a/core-new/src/location/manager.rs b/core/src/location/manager.rs similarity index 100% rename from core-new/src/location/manager.rs rename to core/src/location/manager.rs diff --git a/core/src/location/manager/mod.rs b/core/src/location/manager/mod.rs deleted file mode 100644 index bfeaa6b1f..000000000 --- a/core/src/location/manager/mod.rs +++ /dev/null @@ -1,451 +0,0 @@ -use crate::{ - library::{Library, LibraryManagerEvent}, - Node, -}; - -use sd_core_file_path_helper::FilePathError; - -use sd_prisma::prisma::location; -use sd_utils::{db::MissingFieldError, error::FileIOError}; - -use std::{ - collections::BTreeSet, - path::{Path, PathBuf}, - sync::Arc, -}; - -use async_channel as chan; -use futures::executor::block_on; -use thiserror::Error; -use tokio::{ - spawn, - sync::{ - broadcast::{self, Receiver}, - oneshot, RwLock, - }, -}; -use tracing::{debug, error, instrument, trace}; -use uuid::Uuid; - -mod runner; -mod watcher; - -#[derive(Clone, Copy, Debug)] -enum ManagementMessageAction { - Add, - Remove, -} - -#[derive(Debug)] -pub struct LocationManagementMessage { - location_id: location::id::Type, - library: Arc, - action: ManagementMessageAction, - ack: oneshot::Sender>, -} - -#[derive(Debug)] -enum WatcherManagementMessageAction { - Pause, - Resume, - IgnoreEventsForPath { path: PathBuf, ignore: bool }, -} - -#[derive(Debug)] -pub struct WatcherManagementMessage { - location_id: location::id::Type, - library: Arc, - action: WatcherManagementMessageAction, - ack: oneshot::Sender>, -} - -#[derive(Error, Debug)] -pub enum LocationManagerError { - #[error("location not found in database: ")] - LocationNotFound(location::id::Type), - - #[error("watcher error: {0}")] - Watcher(#[from] notify::Error), - - #[error("non local location: ")] - NonLocalLocation(location::id::Type), - - #[error("file still exists on disk after remove event received: ", .0.display())] - FileStillExistsOnDisk(Box), - - #[error("failed to move file '{}' for reason: {reason}", .path.display())] - MoveError { - path: Box, - reason: &'static str, - }, - - #[error("database error: {0}")] - Database(#[from] prisma_client_rust::QueryError), - #[error("corrupted location pub_id on database: {0}")] - CorruptedLocationPubId(#[from] uuid::Error), - #[error("missing field: {0}")] - MissingField(#[from] MissingFieldError), - - #[error(transparent)] - FilePath(#[from] FilePathError), - #[error(transparent)] - IndexerRuler(#[from] sd_core_indexer_rules::Error), - #[error(transparent)] - JobSystem(#[from] sd_core_heavy_lifting::Error), - #[error(transparent)] - FileIO(#[from] FileIOError), - #[error(transparent)] - Sync(#[from] sd_core_sync::Error), -} - -type OnlineLocations = BTreeSet>; - -#[must_use = "'LocationManagerActor::start' must be used to start the actor"] -pub struct LocationManagerActor { - location_management_rx: chan::Receiver, - watcher_management_rx: chan::Receiver, - stop_rx: chan::Receiver<()>, -} - -impl LocationManagerActor { - pub fn start(self, node: Arc) { - spawn({ - let node = node.clone(); - let rx = node.libraries.rx.clone(); - async move { - if let Err(e) = rx - .subscribe(|event| { - let node = node.clone(); - async move { - match event { - LibraryManagerEvent::Load(library) => { - for location in library - .db - .location() - .find_many(vec![]) - .exec() - .await - .unwrap_or_else(|e| { - error!( - ?e, - "Failed to get locations from database for location manager;", - ); - - vec![] - }) { - if let Err(e) = - node.locations.add(location.id, library.clone()).await - { - error!( - ?e, - "Failed to add location to location manager;", - ); - } - } - } - LibraryManagerEvent::Edit(_) => {} - LibraryManagerEvent::InstancesModified(_) => {} - LibraryManagerEvent::Delete(_) => { - #[cfg(debug_assertions)] - error!("TODO: Remove locations from location manager"); // TODO - } - } - } - }) - .await - { - error!( - ?e, - "Core may become unstable! LocationManager's \ - library manager subscription aborted with error;", - ); - } - } - }); - - spawn({ - let node = Arc::clone(&node); - let Self { - location_management_rx, - watcher_management_rx, - stop_rx, - } = self; - - async move { - while let Err(e) = spawn({ - runner::run( - location_management_rx.clone(), - watcher_management_rx.clone(), - stop_rx.clone(), - Arc::clone(&node), - ) - }) - .await - { - if e.is_panic() { - error!(?e, "Location manager panicked;"); - } else { - trace!("Location manager received shutdown signal and will exit..."); - break; - } - trace!("Restarting location manager processing task..."); - } - - debug!("Location manager gracefully shutdown"); - } - }); - } -} - -pub struct Locations { - online_locations: RwLock, - pub online_tx: broadcast::Sender, - - location_management_tx: chan::Sender, - - watcher_management_tx: chan::Sender, - stop_tx: chan::Sender<()>, -} - -impl Locations { - pub fn new() -> (Self, LocationManagerActor) { - let (location_management_tx, location_management_rx) = chan::bounded(128); - let (watcher_management_tx, watcher_management_rx) = chan::bounded(128); - let (stop_tx, stop_rx) = chan::bounded(1); - - debug!("Starting location manager actor"); - - ( - Self { - online_locations: Default::default(), - online_tx: broadcast::channel(16).0, - location_management_tx, - watcher_management_tx, - stop_tx, - }, - LocationManagerActor { - location_management_rx, - watcher_management_rx, - stop_rx, - }, - ) - } - - #[instrument(skip(self, library), fields(library_id = %library.id), err)] - #[inline] - async fn location_management_message( - &self, - location_id: location::id::Type, - library: Arc, - action: ManagementMessageAction, - ) -> Result<(), LocationManagerError> { - let (tx, rx) = oneshot::channel(); - trace!("Sending location management message to location manager actor"); - - self.location_management_tx - .send(LocationManagementMessage { - location_id, - library, - action, - ack: tx, - }) - .await - .expect("Location manager actor channel closed sending new location message"); - - rx.await - .expect("Ack channel closed for location management message response") - } - - #[instrument(skip(self, library), fields(library_id = %library.id), err)] - #[inline] - #[allow(unused_variables)] - async fn watcher_management_message( - &self, - location_id: location::id::Type, - library: Arc, - action: WatcherManagementMessageAction, - ) -> Result<(), LocationManagerError> { - let (tx, rx) = oneshot::channel(); - trace!("Sending watcher management message to location manager actor"); - - self.watcher_management_tx - .send(WatcherManagementMessage { - location_id, - library, - action, - ack: tx, - }) - .await - .expect("Location manager actor channel closed sending new watcher message"); - - rx.await - .expect("Ack channel closed for watcher management message response") - } - - pub async fn add( - &self, - location_id: location::id::Type, - library: Arc, - ) -> Result<(), LocationManagerError> { - self.location_management_message(location_id, library, ManagementMessageAction::Add) - .await - } - - pub async fn remove( - &self, - location_id: location::id::Type, - library: Arc, - ) -> Result<(), LocationManagerError> { - self.location_management_message(location_id, library, ManagementMessageAction::Remove) - .await - } - - pub async fn pause_watcher( - &self, - location_id: location::id::Type, - library: Arc, - ) -> Result<(), LocationManagerError> { - self.watcher_management_message(location_id, library, WatcherManagementMessageAction::Pause) - .await - } - - pub async fn resume_watcher( - &self, - location_id: location::id::Type, - library: Arc, - ) -> Result<(), LocationManagerError> { - self.watcher_management_message( - location_id, - library, - WatcherManagementMessageAction::Resume, - ) - .await - } - - pub async fn temporary_watcher_pause( - &self, - location_id: location::id::Type, - library: Arc, - ) -> Result, LocationManagerError> { - self.pause_watcher(location_id, library.clone()).await?; - - Ok(PauseWatcherGuard { - location_id, - library: Some(library), - manager: self, - }) - } - - pub async fn temporary_ignore_events_for_path( - &self, - location_id: location::id::Type, - library: Arc, - path: impl AsRef + Send, - ) -> Result, LocationManagerError> { - let path = path.as_ref().to_path_buf(); - - self.watcher_management_message( - location_id, - library.clone(), - WatcherManagementMessageAction::IgnoreEventsForPath { - path: path.clone(), - ignore: true, - }, - ) - .await?; - - Ok(IgnoreEventsForPathGuard { - location_id, - library: Some(library), - manager: self, - path: Some(path), - }) - } - - pub async fn is_online(&self, id: &Uuid) -> bool { - let online_locations = self.online_locations.read().await; - online_locations.iter().any(|v| v == id.as_bytes()) - } - - pub async fn get_online(&self) -> OnlineLocations { - self.online_locations.read().await.clone() - } - - async fn broadcast_online(&self) { - self.online_tx.send(self.get_online().await).ok(); - } - - pub async fn add_online(&self, id: Uuid) { - { - self.online_locations - .write() - .await - .insert(id.as_bytes().to_vec()); - } - self.broadcast_online().await; - } - - pub async fn remove_online(&self, id: &Uuid) { - { - let mut online_locations = self.online_locations.write().await; - online_locations.retain(|v| v != id.as_bytes()); - } - self.broadcast_online().await; - } - - pub fn online_rx(&self) -> Receiver { - self.online_tx.subscribe() - } -} - -impl Drop for Locations { - fn drop(&mut self) { - // SAFETY: This will never block as we only have 1 sender and this channel has 1 slot - if self.stop_tx.send_blocking(()).is_err() { - error!("Failed to send stop signal to location manager"); - } - } -} - -#[must_use = "this `StopWatcherGuard` must be held for some time, so the watcher is stopped"] -pub struct PauseWatcherGuard<'m> { - manager: &'m Locations, - location_id: location::id::Type, - library: Option>, -} - -impl Drop for PauseWatcherGuard<'_> { - fn drop(&mut self) { - // FIXME: change this Drop to async drop in the future - if let Err(e) = block_on(self.manager.resume_watcher( - self.location_id, - self.library.take().expect("library should be set"), - )) { - error!(?e, "Failed to resume watcher on stop watcher guard drop;"); - } - } -} - -#[must_use = "this `IgnoreEventsForPathGuard` must be held for some time, so the watcher can ignore events for the desired path"] -pub struct IgnoreEventsForPathGuard<'m> { - manager: &'m Locations, - path: Option, - location_id: location::id::Type, - library: Option>, -} - -impl Drop for IgnoreEventsForPathGuard<'_> { - fn drop(&mut self) { - // FIXME: change this Drop to async drop in the future - if let Err(e) = block_on(self.manager.watcher_management_message( - self.location_id, - self.library.take().expect("library should be set"), - WatcherManagementMessageAction::IgnoreEventsForPath { - path: self.path.take().expect("path should be set"), - ignore: false, - }, - )) { - error!(?e, "Failed to un-ignore path on watcher guard drop;"); - } - } -} diff --git a/core/src/location/manager/runner.rs b/core/src/location/manager/runner.rs deleted file mode 100644 index bf769c191..000000000 --- a/core/src/location/manager/runner.rs +++ /dev/null @@ -1,459 +0,0 @@ -use crate::{ - library::{Library, LibraryId}, - Node, -}; - -use sd_core_prisma_helpers::location_ids_and_path; - -use sd_prisma::prisma::location; -use sd_utils::db::maybe_missing; - -use std::{ - collections::{HashMap, HashSet}, - io::ErrorKind, - path::PathBuf, - pin::pin, - sync::Arc, - time::Duration, -}; - -use async_channel as chan; -use futures::stream::StreamExt; -use futures_concurrency::stream::Merge; -use tokio::{ - fs, - sync::oneshot, - time::{interval, MissedTickBehavior}, -}; -use tokio_stream::wrappers::IntervalStream; -use tracing::{debug, error, instrument, trace, warn}; -use uuid::Uuid; - -use super::{ - watcher::LocationWatcher, LocationManagementMessage, LocationManagerError, - ManagementMessageAction, WatcherManagementMessage, WatcherManagementMessageAction, -}; - -type LocationIdAndLibraryId = (location::id::Type, LibraryId); - -struct Runner { - node: Arc, - device_pub_id_to_db: Vec, - locations_to_check: HashMap>, - locations_watched: HashMap, - locations_unwatched: HashMap, - forced_unwatch: HashSet, -} -impl Runner { - async fn new(node: Arc) -> Self { - Self { - device_pub_id_to_db: node.config.get().await.id.to_db(), - node, - locations_to_check: HashMap::new(), - locations_watched: HashMap::new(), - locations_unwatched: HashMap::new(), - forced_unwatch: HashSet::new(), - } - } - - fn check_same_device(&self, location: &location_ids_and_path::Data) -> bool { - location - .device - .as_ref() - .is_some_and(|device| device.pub_id == self.device_pub_id_to_db) - } - - async fn add_location( - &mut self, - location_id: i32, - library: Arc, - ) -> Result<(), LocationManagerError> { - if let Some(location) = get_location(location_id, &library).await? { - check_online(&location, &self.node, &library, &self.device_pub_id_to_db) - .await - .and_then(|is_online| { - LocationWatcher::new(location, Arc::clone(&library), Arc::clone(&self.node)) - .map(|mut watcher| { - if is_online { - trace!(%location_id, "Location is online, watching it!;"); - watcher.watch(); - self.locations_watched - .insert((location_id, library.id), watcher); - } else { - self.locations_unwatched - .insert((location_id, library.id), watcher); - } - - self.locations_to_check - .insert(location_id, Arc::clone(&library)); - }) - }) - } else { - Err(LocationManagerError::LocationNotFound(location_id)) - } - } - - async fn remove_location( - &mut self, - location_id: i32, - library: Arc, - ) -> Result<(), LocationManagerError> { - let key = (location_id, library.id); - - if let Some(location) = get_location(location_id, &library).await? { - if self.check_same_device(&location) { - self.unwatch_location(location, library.id); - self.locations_unwatched.remove(&key); - self.forced_unwatch.remove(&key); - } else { - self.drop_location( - location_id, - library.id, - "Dropping location from location manager, because it isn't from this device", - ); - } - } else { - self.drop_location( - location_id, - library.id, - "Removing location from location manager, as we failed to fetch from db", - ); - } - - // Removing location from checker - self.locations_to_check.remove(&location_id); - - Ok(()) - } - - #[instrument(skip(self, reason))] - fn drop_location( - &mut self, - location_id: location::id::Type, - library_id: LibraryId, - reason: &'static str, - ) { - warn!(%reason); - if let Some(mut watcher) = self.locations_watched.remove(&(location_id, library_id)) { - watcher.unwatch(); - } else { - self.locations_unwatched.remove(&(location_id, library_id)); - } - } - - fn watch_location( - &mut self, - location_ids_and_path::Data { - id: location_id, - path: maybe_location_path, - .. - }: location_ids_and_path::Data, - library_id: LibraryId, - ) { - if let Some(location_path) = maybe_location_path { - if let Some(mut watcher) = self.locations_unwatched.remove(&(location_id, library_id)) { - if watcher.check_path(location_path) { - watcher.watch(); - } - - self.locations_watched - .insert((location_id, library_id), watcher); - } - } - } - - fn unwatch_location( - &mut self, - location_ids_and_path::Data { - id: location_id, - path: maybe_location_path, - .. - }: location_ids_and_path::Data, - library_id: LibraryId, - ) { - if let Some(location_path) = maybe_location_path { - if let Some(mut watcher) = self.locations_watched.remove(&(location_id, library_id)) { - if watcher.check_path(location_path) { - watcher.unwatch(); - } - - self.locations_unwatched - .insert((location_id, library_id), watcher); - } - } - } - - #[instrument(skip(self, library), fields(library_id = %library.id), err)] - async fn pause_watcher( - &mut self, - location_id: location::id::Type, - library: Arc, - ) -> Result<(), LocationManagerError> { - let key = (location_id, library.id); - - if !self.forced_unwatch.contains(&key) && self.locations_watched.contains_key(&key) { - get_location(location_id, &library) - .await? - .ok_or(LocationManagerError::LocationNotFound(location_id)) - .map(|location| { - self.unwatch_location(location, library.id); - self.forced_unwatch.insert(key); - }) - } else { - Ok(()) - } - } - - #[instrument(skip(self, library), fields(library_id = %library.id), err)] - async fn resume_watcher( - &mut self, - location_id: location::id::Type, - library: Arc, - ) -> Result<(), LocationManagerError> { - let key = (location_id, library.id); - - if self.forced_unwatch.contains(&key) && self.locations_unwatched.contains_key(&key) { - get_location(location_id, &library) - .await? - .ok_or(LocationManagerError::LocationNotFound(location_id)) - .map(|location| { - self.watch_location(location, library.id); - self.forced_unwatch.remove(&key); - }) - } else { - Ok(()) - } - } - - async fn ignore_events_for_path( - &self, - location_id: location::id::Type, - library: Arc, - path: PathBuf, - ignore: bool, - ) { - if let Some(watcher) = self.locations_watched.get(&(location_id, library.id)) { - watcher.ignore_path(path, ignore).await - } - } - - async fn handle_location_management_message( - &mut self, - location_id: location::id::Type, - library: Arc, - action: ManagementMessageAction, - ack: oneshot::Sender>, - ) { - ack.send(match action { - ManagementMessageAction::Add => self.add_location(location_id, library).await, - ManagementMessageAction::Remove => self.remove_location(location_id, library).await, - }) - .expect("Ack channel closed") - } - - async fn handle_watcher_management_message( - &mut self, - location_id: location::id::Type, - library: Arc, - action: WatcherManagementMessageAction, - ack: oneshot::Sender>, - ) { - ack.send(match action { - WatcherManagementMessageAction::Pause => self.pause_watcher(location_id, library).await, - WatcherManagementMessageAction::Resume => { - self.resume_watcher(location_id, library).await - } - WatcherManagementMessageAction::IgnoreEventsForPath { path, ignore } => { - self.ignore_events_for_path(location_id, library, path, ignore) - .await; - Ok(()) - } - }) - .expect("Ack channel closed") - } - - async fn check_locations( - &mut self, - locations_to_check_buffer: &mut Vec<(location::id::Type, Arc)>, - ) -> Result<(), Vec> { - let mut errors = vec![]; - locations_to_check_buffer.clear(); - locations_to_check_buffer.extend(self.locations_to_check.drain()); - - for (location_id, library) in locations_to_check_buffer.drain(..) { - if let Err(e) = self - .check_single_location(location_id, Arc::clone(&library)) - .await - { - self.drop_location( - location_id, - library.id, - "Removing location from manager, as we failed to check if it was online", - ); - self.forced_unwatch.remove(&(location_id, library.id)); - errors.push(e); - } - } - - Ok(()) - } - - async fn check_single_location( - &mut self, - location_id: i32, - library: Arc, - ) -> Result<(), LocationManagerError> { - let key = (location_id, library.id); - - if let Some(location) = get_location(location_id, &library).await? { - if self.check_same_device(&location) { - if check_online(&location, &self.node, &library, &self.device_pub_id_to_db).await? - && !self.forced_unwatch.contains(&key) - { - self.watch_location(location, library.id); - } else { - self.unwatch_location(location, library.id); - } - - self.locations_to_check.insert(location_id, library); - } else { - self.drop_location( - location_id, - library.id, - "Dropping location from location manager, because \ - it isn't a location in the current device", - ); - self.forced_unwatch.remove(&key); - } - - Ok(()) - } else { - Err(LocationManagerError::LocationNotFound(location_id)) - } - } -} - -pub(super) async fn run( - location_management_rx: chan::Receiver, - watcher_management_rx: chan::Receiver, - stop_rx: chan::Receiver<()>, - node: Arc, -) { - enum StreamMessage { - LocationManagementMessage(LocationManagementMessage), - WatcherManagementMessage(WatcherManagementMessage), - CheckLocations, - Stop, - } - - let mut locations_to_check_buffer = vec![]; - - let mut check_locations_interval = interval(Duration::from_secs(2)); - check_locations_interval.set_missed_tick_behavior(MissedTickBehavior::Skip); - - let mut runner = Runner::new(node).await; - - let mut msg_stream = pin!(( - location_management_rx.map(StreamMessage::LocationManagementMessage), - watcher_management_rx.map(StreamMessage::WatcherManagementMessage), - IntervalStream::new(check_locations_interval).map(|_| StreamMessage::CheckLocations), - stop_rx.map(|()| StreamMessage::Stop), - ) - .merge()); - - while let Some(msg) = msg_stream.next().await { - match msg { - StreamMessage::LocationManagementMessage(LocationManagementMessage { - location_id, - library, - action, - ack, - }) => { - runner - .handle_location_management_message(location_id, library, action, ack) - .await - } - // Watcher management messages - StreamMessage::WatcherManagementMessage(WatcherManagementMessage { - location_id, - library, - action, - ack, - }) => { - runner - .handle_watcher_management_message(location_id, library, action, ack) - .await - } - StreamMessage::CheckLocations => { - if let Err(errors) = runner.check_locations(&mut locations_to_check_buffer).await { - warn!(?errors, "Errors while checking locations;"); - } - } - StreamMessage::Stop => { - debug!("Stopping location manager"); - break; - } - } - } -} - -#[instrument(skip(library), fields(library_id = %library.id), err)] -async fn get_location( - location_id: location::id::Type, - library: &Library, -) -> Result, LocationManagerError> { - library - .db - .location() - .find_unique(location::id::equals(location_id)) - .select(location_ids_and_path::select()) - .exec() - .await - .map_err(Into::into) -} - -#[instrument( - skip_all, - fields(%location_id, library_id = %library.id), - err, -)] -async fn check_online( - location_ids_and_path::Data { - id: location_id, - pub_id, - device, - path, - }: &location_ids_and_path::Data, - node: &Node, - library: &Library, - device_pub_id_to_db: &[u8], -) -> Result { - let pub_id = Uuid::from_slice(pub_id)?; - - if device - .as_ref() - .is_some_and(|device| device.pub_id == device_pub_id_to_db) - { - match fs::metadata(maybe_missing(path, "location.path")?).await { - Ok(_) => { - node.locations.add_online(pub_id).await; - Ok(true) - } - Err(e) if e.kind() == ErrorKind::NotFound => { - node.locations.remove_online(&pub_id).await; - Ok(false) - } - Err(e) => { - error!( - ?e, - "Failed to check if location is online, will consider as offline;" - ); - Ok(false) - } - } - } else { - // In this case, we don't have a `local_path`, but this location was marked as online - node.locations.remove_online(&pub_id).await; - Err(LocationManagerError::NonLocalLocation(*location_id)) - } -} diff --git a/core/src/location/manager/watcher/android.rs b/core/src/location/manager/watcher/android.rs deleted file mode 100644 index 723f2e076..000000000 --- a/core/src/location/manager/watcher/android.rs +++ /dev/null @@ -1,303 +0,0 @@ -//! Android file system watcher implementation. -//! Just basically linux.rs - -use crate::{invalidate_query, library::Library, location::manager::LocationManagerError, Node}; - -use sd_prisma::prisma::location; -use sd_utils::error::FileIOError; - -use std::{ - collections::{BTreeMap, HashMap}, - path::{Path, PathBuf}, - sync::Arc, -}; - -use notify::{ - event::{CreateKind, DataChange, ModifyKind, RenameMode}, - Event, EventKind, -}; -use tokio::{fs, time::Instant}; -use tracing::{error, instrument, trace}; - -use super::{ - utils::{create_dir, recalculate_directories_size, remove, rename, update_file}, - HUNDRED_MILLIS, ONE_SECOND, -}; - -#[derive(Debug)] -pub(super) struct EventHandler { - location_id: location::id::Type, - location_pub_id: location::pub_id::Type, - library: Arc, - node: Arc, - last_events_eviction_check: Instant, - rename_from: HashMap, - recently_renamed_from: BTreeMap, - files_to_update: HashMap, - reincident_to_update_files: HashMap, - to_recalculate_size: HashMap, - - path_and_instant_buffer: Vec<(PathBuf, Instant)>, -} - -impl super::EventHandler for EventHandler { - fn new( - location_id: location::id::Type, - location_pub_id: location::pub_id::Type, - library: Arc, - node: Arc, - ) -> Self - where - Self: Sized, - { - Self { - location_id, - location_pub_id, - library, - node, - last_events_eviction_check: Instant::now(), - rename_from: HashMap::new(), - recently_renamed_from: BTreeMap::new(), - files_to_update: HashMap::new(), - reincident_to_update_files: HashMap::new(), - to_recalculate_size: HashMap::new(), - path_and_instant_buffer: Vec::new(), - } - } - - #[instrument( - skip_all, - fields( - location_id = %self.location_id, - library_id = %self.library.id, - waiting_rename_count = %self.recently_renamed_from.len(), - waiting_update_count = %self.files_to_update.len(), - reincident_to_update_files_count = %self.reincident_to_update_files.len(), - waiting_size_count = %self.to_recalculate_size.len(), - ), - )] - async fn handle_event(&mut self, event: Event) -> Result<(), LocationManagerError> { - trace!("Received Android event"); - - let Event { - kind, mut paths, .. - } = event; - - match kind { - EventKind::Create(CreateKind::File) - | EventKind::Modify(ModifyKind::Data(DataChange::Any)) => { - // When we receive a create, modify data or metadata events of the above kinds - // we just mark the file to be updated in a near future - // each consecutive event of these kinds that we receive for the same file - // we just store the path again in the map below, with a new instant - // that effectively resets the timer for the file to be updated - let path = paths.remove(0); - if self.files_to_update.contains_key(&path) { - if let Some(old_instant) = - self.files_to_update.insert(path.clone(), Instant::now()) - { - self.reincident_to_update_files - .entry(path) - .or_insert(old_instant); - } - } else { - self.files_to_update.insert(path, Instant::now()); - } - } - - EventKind::Create(CreateKind::Folder) => { - let path = &paths[0]; - - // Don't need to dispatch a recalculate directory event as `create_dir` dispatches - // a `scan_location_sub_path` function, which recalculates the size already - - create_dir( - self.location_id, - path, - &fs::metadata(path) - .await - .map_err(|e| FileIOError::from((path, e)))?, - &self.node, - &self.library, - ) - .await?; - } - - EventKind::Modify(ModifyKind::Name(RenameMode::From)) => { - // Just in case we can't guarantee that we receive the Rename From event before the - // Rename Both event. Just a safeguard - if self.recently_renamed_from.remove(&paths[0]).is_none() { - self.rename_from.insert(paths.remove(0), Instant::now()); - } - } - - EventKind::Modify(ModifyKind::Name(RenameMode::Both)) => { - let to_path = paths.remove(1); - let from_path = paths.remove(0); - - self.rename_from.remove(&from_path); - - rename( - self.location_id, - &to_path, - &from_path, - fs::metadata(&to_path) - .await - .map_err(|e| FileIOError::from((&to_path, e)))?, - &self.library, - ) - .await?; - - self.recently_renamed_from.insert(from_path, Instant::now()); - } - - EventKind::Remove(_) => { - let path = paths.remove(0); - if let Some(parent) = path.parent() { - if parent != Path::new("") { - self.to_recalculate_size - .insert(parent.to_path_buf(), Instant::now()); - } - } - - remove(self.location_id, &path, &self.library).await?; - } - - _ => { - trace!("Other Android event that we don't handle for now"); - } - } - - Ok(()) - } - - async fn tick(&mut self) { - if self.last_events_eviction_check.elapsed() > HUNDRED_MILLIS { - if let Err(e) = self.handle_to_update_eviction().await { - error!( - ?e, - "Error while handling recently created or update files eviction;" - ); - } - - if let Err(e) = self.handle_rename_from_eviction().await { - error!(?e, "Failed to remove file_path;"); - } - - self.recently_renamed_from - .retain(|_, instant| instant.elapsed() < HUNDRED_MILLIS); - - if !self.to_recalculate_size.is_empty() { - if let Err(e) = recalculate_directories_size( - &mut self.to_recalculate_size, - &mut self.path_and_instant_buffer, - self.location_id, - self.location_pub_id.clone(), - &self.library, - ) - .await - { - error!(?e, "Failed to recalculate directories size;"); - } - } - - self.last_events_eviction_check = Instant::now(); - } - } -} - -impl EventHandler { - async fn handle_to_update_eviction(&mut self) -> Result<(), LocationManagerError> { - self.path_and_instant_buffer.clear(); - - let mut should_invalidate = false; - - for (path, created_at) in self.files_to_update.drain() { - if created_at.elapsed() < HUNDRED_MILLIS * 5 { - self.path_and_instant_buffer.push((path, created_at)); - } else { - if let Some(parent) = path.parent() { - if parent != Path::new("") { - self.to_recalculate_size - .insert(parent.to_path_buf(), Instant::now()); - } - } - - self.reincident_to_update_files.remove(&path); - - update_file(self.location_id, &path, &self.node, &self.library).await?; - - should_invalidate = true; - } - } - - self.files_to_update - .extend(self.path_and_instant_buffer.drain(..)); - - self.path_and_instant_buffer.clear(); - - // We have to check if we have any reincident files to update and update them after a bigger - // timeout, this way we keep track of files being update frequently enough to bypass our - // eviction check above - for (path, created_at) in self.reincident_to_update_files.drain() { - if created_at.elapsed() < ONE_SECOND * 10 { - self.path_and_instant_buffer.push((path, created_at)); - } else { - if let Some(parent) = path.parent() { - if parent != Path::new("") { - self.to_recalculate_size - .insert(parent.to_path_buf(), Instant::now()); - } - } - - self.files_to_update.remove(&path); - - update_file(self.location_id, &path, &self.node, &self.library).await?; - - should_invalidate = true; - } - } - - if should_invalidate { - invalidate_query!(&self.library, "search.paths"); - } - - self.reincident_to_update_files - .extend(self.path_and_instant_buffer.drain(..)); - - Ok(()) - } - - async fn handle_rename_from_eviction(&mut self) -> Result<(), LocationManagerError> { - self.path_and_instant_buffer.clear(); - let mut should_invalidate = false; - - for (path, instant) in self.rename_from.drain() { - if instant.elapsed() > HUNDRED_MILLIS { - if let Some(parent) = path.parent() { - if parent != Path::new("") { - self.to_recalculate_size - .insert(parent.to_path_buf(), Instant::now()); - } - } - - remove(self.location_id, &path, &self.library).await?; - - should_invalidate = true; - - trace!(path = %path.display(), "Removed file_path due timeout;"); - } else { - self.path_and_instant_buffer.push((path, instant)); - } - } - - if should_invalidate { - invalidate_query!(&self.library, "search.paths"); - } - - self.rename_from - .extend(self.path_and_instant_buffer.drain(..)); - - Ok(()) - } -} diff --git a/core/src/location/manager/watcher/ios.rs b/core/src/location/manager/watcher/ios.rs deleted file mode 100644 index 25f0a49fd..000000000 --- a/core/src/location/manager/watcher/ios.rs +++ /dev/null @@ -1,452 +0,0 @@ -//! iOS file system watcher implementation. - -use crate::{invalidate_query, library::Library, location::manager::LocationManagerError, Node}; - -use sd_core_file_path_helper::{ - check_file_path_exists, get_inode, FilePathError, IsolatedFilePathData, -}; - -use sd_prisma::prisma::location; -use sd_utils::error::FileIOError; - -use std::{ - collections::HashMap, - path::{Path, PathBuf}, - sync::Arc, -}; - -use notify::{ - event::{CreateKind, DataChange, MetadataKind, ModifyKind, RenameMode}, - Event, EventKind, -}; -use tokio::{fs, io, time::Instant}; -use tracing::{error, instrument, trace, warn}; - -use super::{ - utils::{ - create_dir, create_file, extract_inode_from_path, extract_location_path, - recalculate_directories_size, remove, rename, update_file, - }, - INode, InstantAndPath, HUNDRED_MILLIS, ONE_SECOND, -}; - -#[derive(Debug)] -pub(super) struct EventHandler { - location_id: location::id::Type, - location_pub_id: location::pub_id::Type, - library: Arc, - node: Arc, - last_events_eviction_check: Instant, - latest_created_dir: Option, - old_paths_map: HashMap, - new_paths_map: HashMap, - files_to_update: HashMap, - reincident_to_update_files: HashMap, - to_recalculate_size: HashMap, - - path_and_instant_buffer: Vec<(PathBuf, Instant)>, - paths_map_buffer: Vec<(INode, InstantAndPath)>, -} - -impl super::EventHandler for EventHandler { - fn new( - location_id: location::id::Type, - location_pub_id: location::pub_id::Type, - library: Arc, - node: Arc, - ) -> Self - where - Self: Sized, - { - Self { - location_id, - location_pub_id, - library, - node, - last_events_eviction_check: Instant::now(), - latest_created_dir: None, - old_paths_map: HashMap::new(), - new_paths_map: HashMap::new(), - files_to_update: HashMap::new(), - reincident_to_update_files: HashMap::new(), - to_recalculate_size: HashMap::new(), - path_and_instant_buffer: Vec::new(), - paths_map_buffer: Vec::new(), - } - } - - #[instrument( - skip_all, - fields( - location_id = %self.location_id, - library_id = %self.library.id, - latest_created_dir = ?self.latest_created_dir, - old_paths_map_count = %self.old_paths_map.len(), - new_paths_map = %self.new_paths_map.len(), - waiting_update_count = %self.files_to_update.len(), - reincident_to_update_files_count = %self.reincident_to_update_files.len(), - waiting_size_count = %self.to_recalculate_size.len(), - ), - )] - async fn handle_event(&mut self, event: Event) -> Result<(), LocationManagerError> { - trace!("Received iOS event"); - - let Event { - kind, mut paths, .. - } = event; - - match kind { - EventKind::Create(CreateKind::Folder) => { - let path = paths.remove(0); - - create_dir( - self.location_id, - &path, - &fs::metadata(&path) - .await - .map_err(|e| FileIOError::from((&path, e)))?, - &self.node, - &self.library, - ) - .await?; - - self.latest_created_dir = Some(path); - } - - EventKind::Create(CreateKind::File) - | EventKind::Modify(ModifyKind::Data(DataChange::Content)) - | EventKind::Modify(ModifyKind::Metadata( - MetadataKind::WriteTime | MetadataKind::Extended, - )) => { - // When we receive a create, modify data or metadata events of the above kinds - // we just mark the file to be updated in a near future - // each consecutive event of these kinds that we receive for the same file - // we just store the path again in the map below, with a new instant - // that effectively resets the timer for the file to be updated <- Copied from macos.rs - let path = paths.remove(0); - - if self.files_to_update.contains_key(&path) { - if let Some(old_instant) = - self.files_to_update.insert(path.clone(), Instant::now()) - { - self.reincident_to_update_files - .entry(path) - .or_insert(old_instant); - } - } else { - self.files_to_update.insert(path, Instant::now()); - } - } - - EventKind::Modify(ModifyKind::Name(RenameMode::Any)) => { - self.handle_single_rename_event(paths.remove(0)).await?; - } - - // For some reason, iOS doesn't have a Delete Event, so the vent type comes up as this. - // Delete Event - EventKind::Modify(ModifyKind::Metadata(MetadataKind::Any)) => { - let path = paths.remove(0); - - trace!(path = %path.display(), "File has been deleted;"); - - if let Some(parent) = path.parent() { - if parent != Path::new("") { - self.to_recalculate_size - .insert(parent.to_path_buf(), Instant::now()); - } - } - - remove(self.location_id, &path, &self.library).await?; //FIXME: Find out why this freezes the watcher - } - - _ => { - trace!("Other iOS event that we don't handle for now"); - } - } - - Ok(()) - } - - async fn tick(&mut self) { - if self.last_events_eviction_check.elapsed() > HUNDRED_MILLIS { - if let Err(e) = self.handle_to_update_eviction().await { - error!( - ?e, - "Error while handling recently created or update files eviction;" - ); - } - - // Cleaning out recently renamed files that are older than 100 milliseconds - if let Err(e) = self.handle_rename_create_eviction().await { - error!(?e, "Failed to create file_path on iOS;"); - } - - if let Err(e) = self.handle_rename_remove_eviction().await { - error!(?e, "Failed to remove file_path;"); - } - - if !self.to_recalculate_size.is_empty() { - if let Err(e) = recalculate_directories_size( - &mut self.to_recalculate_size, - &mut self.path_and_instant_buffer, - self.location_id, - self.location_pub_id.clone(), - &self.library, - ) - .await - { - error!(?e, "Failed to recalculate directories size;"); - } - } - - self.last_events_eviction_check = Instant::now(); - } - } -} - -impl EventHandler { - async fn handle_to_update_eviction(&mut self) -> Result<(), LocationManagerError> { - self.path_and_instant_buffer.clear(); - let mut should_invalidate = false; - - for (path, created_at) in self.files_to_update.drain() { - if created_at.elapsed() < HUNDRED_MILLIS * 5 { - self.path_and_instant_buffer.push((path, created_at)); - } else { - if let Some(parent) = path.parent() { - if parent != Path::new("") { - self.to_recalculate_size - .insert(parent.to_path_buf(), Instant::now()); - } - } - - self.reincident_to_update_files.remove(&path); - - update_file(self.location_id, &path, &self.node, &self.library).await?; - - should_invalidate = true; - } - } - - self.files_to_update - .extend(self.path_and_instant_buffer.drain(..)); - - self.path_and_instant_buffer.clear(); - - // We have to check if we have any reincident files to update and update them after a bigger - // timeout, this way we keep track of files being update frequently enough to bypass our - // eviction check above - for (path, created_at) in self.reincident_to_update_files.drain() { - if created_at.elapsed() < ONE_SECOND * 10 { - self.path_and_instant_buffer.push((path, created_at)); - } else { - if let Some(parent) = path.parent() { - if parent != Path::new("") { - self.to_recalculate_size - .insert(parent.to_path_buf(), Instant::now()); - } - } - - self.files_to_update.remove(&path); - - update_file(self.location_id, &path, &self.node, &self.library).await?; - - should_invalidate = true; - } - } - - if should_invalidate { - invalidate_query!(self.library, "search.paths"); - } - - self.reincident_to_update_files - .extend(self.path_and_instant_buffer.drain(..)); - - Ok(()) - } - - async fn handle_rename_create_eviction(&mut self) -> Result<(), LocationManagerError> { - // Just to make sure that our buffer is clean - self.paths_map_buffer.clear(); - let mut should_invalidate = false; - - for (inode, (instant, path)) in self.new_paths_map.drain() { - if instant.elapsed() > HUNDRED_MILLIS { - if !self.files_to_update.contains_key(&path) { - let metadata = fs::metadata(&path) - .await - .map_err(|e| FileIOError::from((&path, e)))?; - - if metadata.is_dir() { - // Don't need to dispatch a recalculate directory event as `create_dir` dispatches - // a `scan_location_sub_path` function, which recalculates the size already - create_dir( - self.location_id, - &path, - &metadata, - &self.node, - &self.library, - ) - .await?; - } else { - if let Some(parent) = path.parent() { - if parent != Path::new("") { - self.to_recalculate_size - .insert(parent.to_path_buf(), Instant::now()); - } - } - - create_file( - self.location_id, - &path, - &metadata, - &self.node, - &self.library, - ) - .await?; - } - - trace!(path = %path.display(), "Created file_path due timeout;"); - - should_invalidate = true; - } - } else { - self.paths_map_buffer.push((inode, (instant, path))); - } - } - - if should_invalidate { - invalidate_query!(self.library, "search.paths"); - } - - self.new_paths_map.extend(self.paths_map_buffer.drain(..)); - - Ok(()) - } - - async fn handle_rename_remove_eviction(&mut self) -> Result<(), LocationManagerError> { - // Just to make sure that our buffer is clean - self.paths_map_buffer.clear(); - let mut should_invalidate = false; - - for (inode, (instant, path)) in self.old_paths_map.drain() { - if instant.elapsed() > HUNDRED_MILLIS { - if let Some(parent) = path.parent() { - if parent != Path::new("") { - self.to_recalculate_size - .insert(parent.to_path_buf(), Instant::now()); - } - } - - remove(self.location_id, &path, &self.library).await?; - - trace!(path = %path.display(), "Removed file_path due timeout;"); - - should_invalidate = true; - } else { - self.paths_map_buffer.push((inode, (instant, path))); - } - } - - if should_invalidate { - invalidate_query!(self.library, "search.paths"); - } - - self.old_paths_map.extend(self.paths_map_buffer.drain(..)); - - Ok(()) - } - - async fn handle_single_rename_event( - &mut self, - path: PathBuf, // this is used internally only once, so we can use just PathBuf - ) -> Result<(), LocationManagerError> { - match fs::metadata(&path).await { - Ok(meta) => { - // File or directory exists, so this can be a "new path" to an actual rename/move or a creation - trace!(path = %path.display(), "Path exists;"); - - let inode = get_inode(&meta); - let location_path = extract_location_path(self.location_id, &self.library).await?; - - if !check_file_path_exists::( - &IsolatedFilePathData::new( - self.location_id, - &location_path, - &path, - meta.is_dir(), - )?, - &self.library.db, - ) - .await? - { - if let Some((_, old_path)) = self.old_paths_map.remove(&inode) { - trace!( - old_path = %old_path.display(), - new_path = %path.display(), - "Got a match new -> old;", - ); - - // We found a new path for this old path, so we can rename it - rename(self.location_id, &path, &old_path, meta, &self.library).await?; - } else { - trace!(path = %path.display(), "No match for new path yet;"); - - self.new_paths_map.insert(inode, (Instant::now(), path)); - } - } else { - warn!( - path = %path.display(), - "Received rename event for a file that already exists in the database;", - ); - } - } - Err(e) if e.kind() == io::ErrorKind::NotFound => { - // File or directory does not exist in the filesystem, if it exists in the database, - // then we try pairing it with the old path from our map - - trace!(path = %path.display(), "Path doesn't exists;"); - - let inode = - match extract_inode_from_path(self.location_id, &path, &self.library).await { - Ok(inode) => inode, - - Err(LocationManagerError::FilePath(FilePathError::NotFound(_))) => { - // temporary file, we can ignore it - return Ok(()); - } - - Err(e) => return Err(e), - }; - - if let Some((_, new_path)) = self.new_paths_map.remove(&inode) { - trace!( - old_path = %path.display(), - new_path = %new_path.display(), - "Got a match old -> new;", - ); - - // We found a new path for this old path, so we can rename it - rename( - self.location_id, - &new_path, - &path, - fs::metadata(&new_path) - .await - .map_err(|e| FileIOError::from((&new_path, e)))?, - &self.library, - ) - .await?; - } else { - trace!(path = %path.display(), "No match for old path yet;"); - - // We didn't find a new path for this old path, so we store ir for later - self.old_paths_map.insert(inode, (Instant::now(), path)); - } - } - Err(e) => return Err(FileIOError::from((path, e)).into()), - } - - Ok(()) - } -} diff --git a/core/src/location/manager/watcher/linux.rs b/core/src/location/manager/watcher/linux.rs deleted file mode 100644 index 34d37ed15..000000000 --- a/core/src/location/manager/watcher/linux.rs +++ /dev/null @@ -1,309 +0,0 @@ -//! Linux has the best behaving file system events, with just some small caveats: -//! When we move files or directories, we receive 3 events: Rename From, Rename To and Rename Both. -//! But when we move a file or directory to the outside from the watched location, we just receive -//! the Rename From event, so we have to keep track of all rename events to match them against each -//! other. If we have dangling Rename From events, we have to remove them after some time. -//! Aside from that, when a directory is moved to our watched location from the outside, we receive -//! a Create Dir event, this one is actually ok at least. - -use crate::{invalidate_query, library::Library, location::manager::LocationManagerError, Node}; - -use sd_prisma::prisma::location; -use sd_utils::error::FileIOError; - -use std::{ - collections::{BTreeMap, HashMap}, - path::{Path, PathBuf}, - sync::Arc, -}; - -use notify::{ - event::{CreateKind, DataChange, ModifyKind, RenameMode}, - Event, EventKind, -}; -use tokio::{fs, time::Instant}; -use tracing::{error, instrument, trace}; - -use super::{ - utils::{create_dir, recalculate_directories_size, remove, rename, update_file}, - HUNDRED_MILLIS, ONE_SECOND, -}; - -#[derive(Debug)] -pub(super) struct EventHandler { - location_id: location::id::Type, - location_pub_id: location::pub_id::Type, - library: Arc, - node: Arc, - last_events_eviction_check: Instant, - rename_from: HashMap, - recently_renamed_from: BTreeMap, - files_to_update: HashMap, - reincident_to_update_files: HashMap, - to_recalculate_size: HashMap, - - path_and_instant_buffer: Vec<(PathBuf, Instant)>, -} - -impl super::EventHandler for EventHandler { - fn new( - location_id: location::id::Type, - location_pub_id: location::pub_id::Type, - library: Arc, - node: Arc, - ) -> Self - where - Self: Sized, - { - Self { - location_id, - location_pub_id, - library, - node, - last_events_eviction_check: Instant::now(), - rename_from: HashMap::new(), - recently_renamed_from: BTreeMap::new(), - files_to_update: HashMap::new(), - reincident_to_update_files: HashMap::new(), - to_recalculate_size: HashMap::new(), - path_and_instant_buffer: Vec::new(), - } - } - - #[instrument( - skip_all, - fields( - location_id = %self.location_id, - library_id = %self.library.id, - waiting_rename_count = %self.recently_renamed_from.len(), - waiting_update_count = %self.files_to_update.len(), - reincident_to_update_files_count = %self.reincident_to_update_files.len(), - waiting_size_count = %self.to_recalculate_size.len(), - ), - )] - async fn handle_event(&mut self, event: Event) -> Result<(), LocationManagerError> { - trace!("Received Linux event"); - - let Event { - kind, mut paths, .. - } = event; - - match kind { - EventKind::Create(CreateKind::File) - | EventKind::Modify(ModifyKind::Data(DataChange::Any)) => { - // When we receive a create, modify data or metadata events of the above kinds - // we just mark the file to be updated in a near future - // each consecutive event of these kinds that we receive for the same file - // we just store the path again in the map below, with a new instant - // that effectively resets the timer for the file to be updated - let path = paths.remove(0); - - if self.files_to_update.contains_key(&path) { - if let Some(old_instant) = - self.files_to_update.insert(path.clone(), Instant::now()) - { - self.reincident_to_update_files - .entry(path) - .or_insert(old_instant); - } - } else { - self.files_to_update.insert(path, Instant::now()); - } - } - - EventKind::Create(CreateKind::Folder) => { - let path = paths.remove(0); - - // Don't need to dispatch a recalculate directory event as `create_dir` dispatches - // a `scan_location_sub_path` function, which recalculates the size already - - create_dir( - self.location_id, - &path, - &fs::metadata(&path) - .await - .map_err(|e| FileIOError::from((&path, e)))?, - &self.node, - &self.library, - ) - .await?; - } - - EventKind::Modify(ModifyKind::Name(RenameMode::From)) => { - // Just in case we can't guarantee that we receive the Rename From event before the - // Rename Both event. Just a safeguard - if self.recently_renamed_from.remove(&paths[0]).is_none() { - self.rename_from.insert(paths.remove(0), Instant::now()); - } - } - - EventKind::Modify(ModifyKind::Name(RenameMode::Both)) => { - let to_path = paths.remove(1); - let from_path = paths.remove(0); - - self.rename_from.remove(&from_path); - rename( - self.location_id, - &to_path, - &from_path, - fs::metadata(&to_path) - .await - .map_err(|e| FileIOError::from((&to_path, e)))?, - &self.library, - ) - .await?; - - self.recently_renamed_from.insert(from_path, Instant::now()); - } - - EventKind::Remove(_) => { - let path = paths.remove(0); - if let Some(parent) = path.parent() { - if parent != Path::new("") { - self.to_recalculate_size - .insert(parent.to_path_buf(), Instant::now()); - } - } - - remove(self.location_id, &path, &self.library).await?; - } - - _ => { - trace!("Other Linux event that we don't handle for now"); - } - } - - Ok(()) - } - - async fn tick(&mut self) { - if self.last_events_eviction_check.elapsed() > HUNDRED_MILLIS { - if let Err(e) = self.handle_to_update_eviction().await { - error!( - ?e, - "Error while handling recently created or update files eviction;" - ); - } - - if let Err(e) = self.handle_rename_from_eviction().await { - error!(?e, "Failed to remove file_path;"); - } - - self.recently_renamed_from - .retain(|_, instant| instant.elapsed() < HUNDRED_MILLIS); - - if !self.to_recalculate_size.is_empty() { - if let Err(e) = recalculate_directories_size( - &mut self.to_recalculate_size, - &mut self.path_and_instant_buffer, - self.location_id, - self.location_pub_id.clone(), - &self.library, - ) - .await - { - error!(?e, "Failed to recalculate directories size;"); - } - } - - self.last_events_eviction_check = Instant::now(); - } - } -} - -impl EventHandler { - async fn handle_to_update_eviction(&mut self) -> Result<(), LocationManagerError> { - self.path_and_instant_buffer.clear(); - - let mut should_invalidate = false; - - for (path, created_at) in self.files_to_update.drain() { - if created_at.elapsed() < HUNDRED_MILLIS * 5 { - self.path_and_instant_buffer.push((path, created_at)); - } else { - if let Some(parent) = path.parent() { - if parent != Path::new("") { - self.to_recalculate_size - .insert(parent.to_path_buf(), Instant::now()); - } - } - - self.reincident_to_update_files.remove(&path); - - update_file(self.location_id, &path, &self.node, &self.library).await?; - - should_invalidate = true; - } - } - - self.files_to_update - .extend(self.path_and_instant_buffer.drain(..)); - - self.path_and_instant_buffer.clear(); - - // We have to check if we have any reincident files to update and update them after a bigger - // timeout, this way we keep track of files being update frequently enough to bypass our - // eviction check above - for (path, created_at) in self.reincident_to_update_files.drain() { - if created_at.elapsed() < ONE_SECOND * 10 { - self.path_and_instant_buffer.push((path, created_at)); - } else { - if let Some(parent) = path.parent() { - if parent != Path::new("") { - self.to_recalculate_size - .insert(parent.to_path_buf(), Instant::now()); - } - } - - self.files_to_update.remove(&path); - - update_file(self.location_id, &path, &self.node, &self.library).await?; - - should_invalidate = true; - } - } - - if should_invalidate { - invalidate_query!(self.library, "search.paths"); - } - - self.reincident_to_update_files - .extend(self.path_and_instant_buffer.drain(..)); - - Ok(()) - } - - async fn handle_rename_from_eviction(&mut self) -> Result<(), LocationManagerError> { - self.path_and_instant_buffer.clear(); - - let mut should_invalidate = false; - - for (path, instant) in self.rename_from.drain() { - if instant.elapsed() > HUNDRED_MILLIS { - if let Some(parent) = path.parent() { - if parent != Path::new("") { - self.to_recalculate_size - .insert(parent.to_path_buf(), Instant::now()); - } - } - - remove(self.location_id, &path, &self.library).await?; - - should_invalidate = true; - - trace!(path = %path.display(), "Removed file_path due timeout;"); - } else { - self.path_and_instant_buffer.push((path, instant)); - } - } - - if should_invalidate { - invalidate_query!(self.library, "search.paths"); - } - - self.rename_from - .extend(self.path_and_instant_buffer.drain(..)); - - Ok(()) - } -} diff --git a/core/src/location/manager/watcher/macos.rs b/core/src/location/manager/watcher/macos.rs deleted file mode 100644 index 4d3b1ffec..000000000 --- a/core/src/location/manager/watcher/macos.rs +++ /dev/null @@ -1,478 +0,0 @@ -//! On MacOS, we use the FSEvents backend of notify-rs and Rename events are pretty complicated; -//! There are just ModifyKind::Name(RenameMode::Any) events and nothing else. -//! This means that we have to link the old path with the new path to know which file was renamed. -//! But you can't forget that renames events aren't always the case that I file name was modified, -//! but its path was modified. So we have to check if the file was moved. When a file is moved -//! inside the same location, we received 2 events: one for the old path and one for the new path. -//! But when a file is moved to another location, we only receive the old path event... This -//! way we have to handle like a file deletion, and the same applies for when a file is moved to our -//! current location from anywhere else, we just receive the new path rename event, which means a -//! creation. - -use crate::{invalidate_query, library::Library, location::manager::LocationManagerError, Node}; - -use sd_core_file_path_helper::{ - check_file_path_exists, get_inode, FilePathError, IsolatedFilePathData, -}; - -use sd_prisma::prisma::location; -use sd_utils::error::FileIOError; - -use std::{ - collections::HashMap, - path::{Path, PathBuf}, - sync::Arc, -}; - -use notify::{ - event::{CreateKind, DataChange, MetadataKind, ModifyKind, RenameMode}, - Event, EventKind, -}; -use tokio::{fs, io, time::Instant}; -use tracing::{error, instrument, trace, warn}; - -use super::{ - utils::{ - create_dir, create_file, extract_inode_from_path, extract_location_path, - recalculate_directories_size, remove, rename, update_file, - }, - INode, InstantAndPath, HUNDRED_MILLIS, ONE_SECOND, -}; - -#[derive(Debug)] -pub(super) struct EventHandler { - location_id: location::id::Type, - location_pub_id: location::pub_id::Type, - library: Arc, - node: Arc, - last_events_eviction_check: Instant, - latest_created_dir: Option, - old_paths_map: HashMap, - new_paths_map: HashMap, - files_to_update: HashMap, - reincident_to_update_files: HashMap, - to_recalculate_size: HashMap, - - path_and_instant_buffer: Vec<(PathBuf, Instant)>, - paths_map_buffer: Vec<(INode, InstantAndPath)>, -} - -impl super::EventHandler for EventHandler { - fn new( - location_id: location::id::Type, - location_pub_id: location::pub_id::Type, - library: Arc, - node: Arc, - ) -> Self - where - Self: Sized, - { - Self { - location_id, - location_pub_id, - library, - node, - last_events_eviction_check: Instant::now(), - latest_created_dir: None, - old_paths_map: HashMap::new(), - new_paths_map: HashMap::new(), - files_to_update: HashMap::new(), - reincident_to_update_files: HashMap::new(), - to_recalculate_size: HashMap::new(), - path_and_instant_buffer: Vec::new(), - paths_map_buffer: Vec::new(), - } - } - - #[instrument( - skip_all, - fields( - location_id = %self.location_id, - library_id = %self.library.id, - latest_created_dir = ?self.latest_created_dir, - old_paths_map_count = %self.old_paths_map.len(), - new_paths_map = %self.new_paths_map.len(), - waiting_update_count = %self.files_to_update.len(), - reincident_to_update_files_count = %self.reincident_to_update_files.len(), - waiting_size_count = %self.to_recalculate_size.len(), - ), - )] - async fn handle_event(&mut self, event: Event) -> Result<(), LocationManagerError> { - trace!("Received MacOS event"); - - let Event { - kind, mut paths, .. - } = event; - - match kind { - EventKind::Create(CreateKind::Folder) => { - let path = paths.remove(0); - - if let Some(latest_created_dir) = self.latest_created_dir.take() { - if path == latest_created_dir { - // NOTE: This is a MacOS specific event that happens when a folder is created - // trough Finder. It creates a folder but 2 events are triggered in - // FSEvents. So we store and check the latest created folder to avoid - // hitting a unique constraint in the database - return Ok(()); - } - } - - // Don't need to dispatch a recalculate directory event as `create_dir` dispatches - // a `scan_location_sub_path` function, which recalculates the size already - - let metadata = match fs::metadata(&path).await { - Ok(metadata) => metadata, - Err(e) if e.kind() == io::ErrorKind::NotFound => { - // temporary file, bailing out - return Ok(()); - } - Err(e) => return Err(FileIOError::from((&path, e)).into()), - }; - - create_dir( - self.location_id, - &path, - &metadata, - &self.node, - &self.library, - ) - .await?; - - self.latest_created_dir = Some(path); - } - - EventKind::Create(CreateKind::File) - | EventKind::Modify(ModifyKind::Data(DataChange::Content)) - | EventKind::Modify(ModifyKind::Metadata( - MetadataKind::WriteTime | MetadataKind::Extended, - )) => { - // When we receive a create, modify data or metadata events of the above kinds - // we just mark the file to be updated in a near future - // each consecutive event of these kinds that we receive for the same file - // we just store the path again in the map below, with a new instant - // that effectively resets the timer for the file to be updated - let path = paths.remove(0); - - if self.files_to_update.contains_key(&path) { - if let Some(old_instant) = - self.files_to_update.insert(path.clone(), Instant::now()) - { - self.reincident_to_update_files - .entry(path) - .or_insert(old_instant); - } - } else { - self.files_to_update.insert(path, Instant::now()); - } - } - - EventKind::Modify(ModifyKind::Name(RenameMode::Any)) => { - self.handle_single_rename_event(paths.remove(0)).await?; - } - - EventKind::Remove(_) => { - let path = paths.remove(0); - - if let Some(parent) = path.parent() { - if parent != Path::new("") { - self.to_recalculate_size - .insert(parent.to_path_buf(), Instant::now()); - } - } - remove(self.location_id, &path, &self.library).await?; - } - _ => { - trace!("Other MacOS event that we don't handle for now"); - } - } - - Ok(()) - } - - async fn tick(&mut self) { - if self.last_events_eviction_check.elapsed() > HUNDRED_MILLIS { - if let Err(e) = self.handle_to_update_eviction().await { - error!( - ?e, - "Error while handling recently created or update files eviction;" - ); - } - - // Cleaning out recently renamed files that are older than 100 milliseconds - if let Err(e) = self.handle_rename_create_eviction().await { - error!(?e, "Failed to create file_path on MacOS;"); - } - - if let Err(e) = self.handle_rename_remove_eviction().await { - error!(?e, "Failed to remove file_path;"); - } - - if !self.to_recalculate_size.is_empty() { - if let Err(e) = recalculate_directories_size( - &mut self.to_recalculate_size, - &mut self.path_and_instant_buffer, - self.location_id, - self.location_pub_id.clone(), - &self.library, - ) - .await - { - error!(?e, "Failed to recalculate directories size;"); - } - } - - self.last_events_eviction_check = Instant::now(); - } - } -} - -impl EventHandler { - async fn handle_to_update_eviction(&mut self) -> Result<(), LocationManagerError> { - self.path_and_instant_buffer.clear(); - - let mut should_invalidate = false; - - for (path, created_at) in self.files_to_update.drain() { - if created_at.elapsed() < HUNDRED_MILLIS * 5 { - self.path_and_instant_buffer.push((path, created_at)); - } else { - if let Some(parent) = path.parent() { - if parent != Path::new("") { - self.to_recalculate_size - .insert(parent.to_path_buf(), Instant::now()); - } - } - self.reincident_to_update_files.remove(&path); - update_file(self.location_id, &path, &self.node, &self.library).await?; - should_invalidate = true; - } - } - - self.files_to_update - .extend(self.path_and_instant_buffer.drain(..)); - - self.path_and_instant_buffer.clear(); - - // We have to check if we have any reincident files to update and update them after a bigger - // timeout, this way we keep track of files being update frequently enough to bypass our - // eviction check above - for (path, created_at) in self.reincident_to_update_files.drain() { - if created_at.elapsed() < ONE_SECOND * 10 { - self.path_and_instant_buffer.push((path, created_at)); - } else { - if let Some(parent) = path.parent() { - if parent != Path::new("") { - self.to_recalculate_size - .insert(parent.to_path_buf(), Instant::now()); - } - } - self.files_to_update.remove(&path); - update_file(self.location_id, &path, &self.node, &self.library).await?; - should_invalidate = true; - } - } - - if should_invalidate { - invalidate_query!(self.library, "search.paths"); - } - - self.reincident_to_update_files - .extend(self.path_and_instant_buffer.drain(..)); - - Ok(()) - } - - async fn handle_rename_create_eviction(&mut self) -> Result<(), LocationManagerError> { - // Just to make sure that our buffer is clean - self.paths_map_buffer.clear(); - - let mut should_invalidate = false; - - for (inode, (instant, path)) in self.new_paths_map.drain() { - if instant.elapsed() > HUNDRED_MILLIS { - if !self.files_to_update.contains_key(&path) { - let metadata = match fs::metadata(&path).await { - Ok(metadata) => metadata, - Err(e) if e.kind() == io::ErrorKind::NotFound => { - // temporary file, bailing out - return Ok(()); - } - Err(e) => return Err(FileIOError::from((&path, e)).into()), - }; - - if metadata.is_dir() { - // Don't need to dispatch a recalculate directory event as `create_dir` dispatches - // a `scan_location_sub_path` function, which recalculates the size already - create_dir( - self.location_id, - &path, - &metadata, - &self.node, - &self.library, - ) - .await?; - } else { - if let Some(parent) = path.parent() { - if parent != Path::new("") { - self.to_recalculate_size - .insert(parent.to_path_buf(), Instant::now()); - } - } - create_file( - self.location_id, - &path, - &metadata, - &self.node, - &self.library, - ) - .await?; - } - - trace!(path = %path.display(), "Created file_path due timeout;"); - - should_invalidate = true; - } - } else { - self.paths_map_buffer.push((inode, (instant, path))); - } - } - - if should_invalidate { - invalidate_query!(self.library, "search.paths"); - } - - self.new_paths_map.extend(self.paths_map_buffer.drain(..)); - - Ok(()) - } - - async fn handle_rename_remove_eviction(&mut self) -> Result<(), LocationManagerError> { - // Just to make sure that our buffer is clean - self.paths_map_buffer.clear(); - - let mut should_invalidate = false; - - for (inode, (instant, path)) in self.old_paths_map.drain() { - if instant.elapsed() > HUNDRED_MILLIS { - if let Some(parent) = path.parent() { - if parent != Path::new("") { - self.to_recalculate_size - .insert(parent.to_path_buf(), Instant::now()); - } - } - - remove(self.location_id, &path, &self.library).await?; - - trace!(path = %path.display(), "Removed file_path due timeout;"); - - should_invalidate = true; - } else { - self.paths_map_buffer.push((inode, (instant, path))); - } - } - - if should_invalidate { - invalidate_query!(self.library, "search.paths"); - } - - self.old_paths_map.extend(self.paths_map_buffer.drain(..)); - - Ok(()) - } - - async fn handle_single_rename_event( - &mut self, - path: PathBuf, // this is used internally only once, so we can use just PathBuf - ) -> Result<(), LocationManagerError> { - match fs::metadata(&path).await { - Ok(meta) => { - // File or directory exists, so this can be a "new path" to an actual rename/move or a creation - trace!(path = %path.display(), "Path exists;"); - - let inode = get_inode(&meta); - let location_path = extract_location_path(self.location_id, &self.library).await?; - - if !check_file_path_exists::( - &IsolatedFilePathData::new( - self.location_id, - &location_path, - &path, - meta.is_dir(), - )?, - &self.library.db, - ) - .await? - { - if let Some((_, old_path)) = self.old_paths_map.remove(&inode) { - trace!( - new_path = %path.display(), - old_path = %old_path.display(), - "Got a match new -> old;", - ); - - // We found a new path for this old path, so we can rename it - rename(self.location_id, &path, &old_path, meta, &self.library).await?; - } else { - trace!(path = %path.display(), "No match for new path yet;"); - - self.new_paths_map.insert(inode, (Instant::now(), path)); - } - } else { - warn!( - path = %path.display(), - "Received rename event for a file that already exists in the database;", - ); - } - } - - Err(e) if e.kind() == io::ErrorKind::NotFound => { - // File or directory does not exist in the filesystem, if it exists in the database, - // then we try pairing it with the old path from our map - - trace!(path = %path.display(), "Path doesn't exists;"); - - let inode = - match extract_inode_from_path(self.location_id, &path, &self.library).await { - Ok(inode) => inode, - - Err(LocationManagerError::FilePath(FilePathError::NotFound(_))) => { - // temporary file, we can ignore it - return Ok(()); - } - - Err(e) => return Err(e), - }; - - if let Some((_, new_path)) = self.new_paths_map.remove(&inode) { - trace!( - old_path = %path.display(), - new_path = %new_path.display(), - "Got a match old -> new;", - ); - - // We found a new path for this old path, so we can rename it - rename( - self.location_id, - &new_path, - &path, - fs::metadata(&new_path) - .await - .map_err(|e| FileIOError::from((&new_path, e)))?, - &self.library, - ) - .await?; - } else { - trace!(path = %path.display(), "No match for old path yet;"); - - // We didn't find a new path for this old path, so we store ir for later - self.old_paths_map.insert(inode, (Instant::now(), path)); - } - } - - Err(e) => return Err(FileIOError::from((path, e)).into()), - } - - Ok(()) - } -} diff --git a/core/src/location/manager/watcher/mod.rs b/core/src/location/manager/watcher/mod.rs deleted file mode 100644 index d63709740..000000000 --- a/core/src/location/manager/watcher/mod.rs +++ /dev/null @@ -1,885 +0,0 @@ -use crate::{library::Library, Node}; - -use sd_core_indexer_rules::{IndexerRule, IndexerRuler}; -use sd_core_prisma_helpers::{location_ids_and_path, location_with_indexer_rules}; - -use sd_prisma::prisma::{location, PrismaClient}; -use sd_utils::{db::maybe_missing, uuid_to_bytes}; - -use std::{ - collections::HashSet, - future::Future, - path::{Path, PathBuf}, - pin::pin, - sync::Arc, - time::Duration, -}; - -use async_channel as chan; -use futures::StreamExt; -use futures_concurrency::stream::Merge; -use notify::{Config, Event, RecommendedWatcher, RecursiveMode, Watcher}; -use tokio::{ - spawn, - task::JoinHandle, - time::{interval_at, Instant, MissedTickBehavior}, -}; -use tokio_stream::wrappers::IntervalStream; -use tracing::{debug, error, info, instrument, trace, warn, Instrument}; -use uuid::Uuid; - -use super::LocationManagerError; - -#[cfg(target_os = "linux")] -mod linux; - -#[cfg(target_os = "macos")] -mod macos; - -#[cfg(target_os = "windows")] -mod windows; - -#[cfg(target_os = "ios")] -mod ios; - -#[cfg(target_os = "android")] -mod android; - -mod utils; - -use utils::reject_event; - -#[cfg(target_os = "linux")] -type Handler = linux::EventHandler; - -#[cfg(target_os = "macos")] -type Handler = macos::EventHandler; - -#[cfg(target_os = "windows")] -type Handler = windows::EventHandler; - -#[cfg(target_os = "android")] -type Handler = android::EventHandler; - -#[cfg(target_os = "ios")] -type Handler = ios::EventHandler; - -pub(super) type IgnorePath = (PathBuf, bool); - -type INode = u64; - -#[cfg(any(target_os = "ios", target_os = "macos", target_os = "windows"))] -type InstantAndPath = (Instant, PathBuf); - -const ONE_SECOND: Duration = Duration::from_secs(1); -const THIRTY_SECONDS: Duration = Duration::from_secs(30); -const HUNDRED_MILLIS: Duration = Duration::from_millis(100); - -trait EventHandler: 'static { - fn new( - location_id: location::id::Type, - location_pub_id: location::pub_id::Type, - library: Arc, - node: Arc, - ) -> Self - where - Self: Sized; - - /// Handle a file system event. - fn handle_event( - &mut self, - event: Event, - ) -> impl Future> + Send; - - /// As Event Handlers have some inner state, from time to time we need to call this tick method - /// so the event handler can update its state. - fn tick(&mut self) -> impl Future + Send; -} - -#[derive(Debug)] -pub(super) struct LocationWatcher { - location_id: location::id::Type, - location_path: PathBuf, - watcher: RecommendedWatcher, - ignore_path_tx: chan::Sender, - handle: Option>, - stop_tx: chan::Sender<()>, -} - -impl LocationWatcher { - #[instrument( - name = "location_watcher", - skip(pub_id, maybe_location_path, library, node), - fields( - library_id = %library.id, - location_path = ?maybe_location_path, - ), - )] - pub(super) fn new( - location_ids_and_path::Data { - id: location_id, - pub_id, - path: maybe_location_path, - .. - }: location_ids_and_path::Data, - library: Arc, - node: Arc, - ) -> Result { - let location_pub_id = Uuid::from_slice(&pub_id)?; - let location_path = maybe_missing(maybe_location_path, "location.path")?.into(); - - let (events_tx, events_rx) = chan::unbounded(); - let (ignore_path_tx, ignore_path_rx) = chan::bounded(8); - let (stop_tx, stop_rx) = chan::bounded(1); - - let watcher = RecommendedWatcher::new( - move |result| { - if !events_tx.is_closed() { - // SAFETY: we are not blocking the thread as this is an unbounded channel - if events_tx.send_blocking(result).is_err() { - error!(%location_id, "Unable to send watcher event to location manager;"); - } - } else { - error!(%location_id, "Tried to send file system events to a closed channel;"); - } - }, - Config::default(), - )?; - - let handle = spawn({ - let events_rx = events_rx.clone(); - let ignore_path_rx = ignore_path_rx.clone(); - let stop_rx = stop_rx.clone(); - async move { - while let Err(e) = spawn( - Self::handle_watch_events( - location_id, - location_pub_id, - Arc::clone(&node), - Arc::clone(&library), - events_rx.clone(), - ignore_path_rx.clone(), - stop_rx.clone(), - ) - .in_current_span(), - ) - .await - { - if e.is_panic() { - error!(?e, "Location watcher panicked;"); - } else { - trace!("Location watcher received shutdown signal and will exit..."); - break; - } - trace!("Restarting location watcher processing task..."); - } - - info!("Location watcher gracefully shutdown"); - } - .in_current_span() - }); - - Ok(Self { - location_id, - location_path, - watcher, - ignore_path_tx, - handle: Some(handle), - stop_tx, - }) - } - - async fn handle_watch_events( - location_id: location::id::Type, - location_pub_id: Uuid, - node: Arc, - library: Arc, - events_rx: chan::Receiver>, - ignore_path_rx: chan::Receiver, - stop_rx: chan::Receiver<()>, - ) { - enum StreamMessage { - NewEvent(notify::Result), - NewIgnorePath(IgnorePath), - Tick, - Stop, - } - - let mut event_handler = Handler::new( - location_id, - uuid_to_bytes(&location_pub_id), - Arc::clone(&library), - Arc::clone(&node), - ); - - let mut last_event_at = Instant::now(); - - let mut cached_indexer_ruler = None; - let mut cached_location_path = None; - - let mut paths_to_ignore = HashSet::new(); - - let mut handler_tick_interval = - interval_at(Instant::now() + HUNDRED_MILLIS, HUNDRED_MILLIS); - // In case of doubt check: https://docs.rs/tokio/latest/tokio/time/enum.MissedTickBehavior.html - handler_tick_interval.set_missed_tick_behavior(MissedTickBehavior::Delay); - - let mut msg_stream = pin!(( - events_rx.map(StreamMessage::NewEvent), - ignore_path_rx.map(StreamMessage::NewIgnorePath), - IntervalStream::new(handler_tick_interval).map(|_| StreamMessage::Tick), - stop_rx.map(|()| StreamMessage::Stop), - ) - .merge()); - - while let Some(msg) = msg_stream.next().await { - match msg { - StreamMessage::NewEvent(Ok(event)) => { - if let Err(e) = get_cached_indexer_ruler_and_location_path( - location_id, - &mut cached_indexer_ruler, - &mut cached_location_path, - &last_event_at, - &library.db, - ) - .await - { - error!(?e, "Failed to get indexer ruler;"); - } - - last_event_at = Instant::now(); - - if let Err(e) = Self::handle_single_event( - location_pub_id, - cached_location_path.as_deref(), - event, - &mut event_handler, - &node, - &paths_to_ignore, - cached_indexer_ruler.as_ref(), - ) - .await - { - error!(?e, "Failed to handle location file system event;"); - } - } - - StreamMessage::NewEvent(Err(e)) => error!(?e, "Watcher error;"), - - StreamMessage::NewIgnorePath((path, should_ignore)) => { - if should_ignore { - paths_to_ignore.insert(path); - } else { - paths_to_ignore.remove(&path); - } - } - - StreamMessage::Tick => event_handler.tick().await, - - StreamMessage::Stop => { - debug!("Stopping Location Manager event handler for location"); - break; - } - } - } - } - - #[instrument(skip_all, fields(?event, ?ignore_paths, ?location_path))] - async fn handle_single_event( - location_pub_id: Uuid, - location_path: Option<&Path>, - event: Event, - event_handler: &mut impl EventHandler, - node: &Node, - ignore_paths: &HashSet, - indexer_ruler: Option<&IndexerRuler>, - ) -> Result<(), LocationManagerError> { - if reject_event(&event, ignore_paths, location_path, indexer_ruler).await { - return Ok(()); - } - - if !node.locations.is_online(&location_pub_id).await { - warn!("Tried to handle event for offline location"); - return Ok(()); - } - - event_handler.handle_event(event).await - } - - #[instrument( - skip(self, path), - fields( - location_id = %self.location_id, - location_path = %self.location_path.display(), - path = %path.display(), - ), - )] - pub(super) async fn ignore_path(&self, path: PathBuf, ignore: bool) { - self.ignore_path_tx - .send((path, ignore)) - .await - .expect("Location watcher ignore path channel closed"); - } - - pub(super) fn check_path(&self, path: impl AsRef) -> bool { - self.location_path == path.as_ref() - } - - #[instrument( - skip(self), - fields( - location_id = %self.location_id, - location_path = %self.location_path.display(), - ), - )] - pub(super) fn watch(&mut self) { - trace!("Start watching location"); - - if let Err(e) = self - .watcher - .watch(self.location_path.as_path(), RecursiveMode::Recursive) - { - error!(?e, "Unable to watch location;"); - } else { - trace!("Now watching location"); - } - } - - #[instrument( - skip(self), - fields( - location_id = %self.location_id, - location_path = %self.location_path.display(), - ), - )] - pub(super) fn unwatch(&mut self) { - if let Err(e) = self.watcher.unwatch(self.location_path.as_path()) { - /**************************************** TODO: **************************************** - * According to an unit test, this error may occur when a subdirectory is removed * - * and we try to unwatch the parent directory then we have to check the implications * - * of unwatch error for this case. * - **************************************************************************************/ - error!(?e, "Unable to unwatch location;"); - } else { - trace!("Stop watching location"); - } - } -} - -impl Drop for LocationWatcher { - fn drop(&mut self) { - // FIXME: change this Drop to async drop in the future - if let Some(handle) = self.handle.take() { - let stop_tx = self.stop_tx.clone(); - spawn(async move { - stop_tx - .send(()) - .await - .expect("Location watcher stop channel closed"); - - if let Err(e) = handle.await { - error!(?e, "Failed to join watcher task;"); - } - }); - } - } -} - -async fn get_cached_indexer_ruler_and_location_path( - location_id: location::id::Type, - cached_indexer_ruler: &mut Option, - location_path: &mut Option, - last_event_at: &Instant, - db: &PrismaClient, -) -> Result<(), LocationManagerError> { - if cached_indexer_ruler.is_none() || last_event_at.elapsed() > THIRTY_SECONDS { - if let Some(location_with_indexer_rules::Data { - path, - indexer_rules, - .. - }) = db - .location() - .find_unique(location::id::equals(location_id)) - .include(location_with_indexer_rules::include()) - .exec() - .await? - { - *cached_indexer_ruler = Some( - indexer_rules - .iter() - .map(|rule| IndexerRule::try_from(&rule.indexer_rule)) - .collect::, _>>() - .map(IndexerRuler::new)?, - ); - - *location_path = path.map(Into::into); - } - } - - Ok(()) -} - -/*************************************************************************************************** -* Some tests to validate our assumptions of events through different file systems * -**************************************************************************************************** -* Events dispatched on Linux: * -* Create File: * -* 1) EventKind::Create(CreateKind::File) * -* 2) EventKind::Modify(ModifyKind::Metadata(MetadataKind::Any)) * -* or EventKind::Modify(ModifyKind::Data(DataChange::Any)) * -* 3) EventKind::Access(AccessKind::Close(AccessMode::Write))) * -* Create Directory: * -* 1) EventKind::Create(CreateKind::Folder) * -* Update File: * -* 1) EventKind::Modify(ModifyKind::Data(DataChange::Any)) * -* 2) EventKind::Access(AccessKind::Close(AccessMode::Write))) * -* Update File (rename): * -* 1) EventKind::Modify(ModifyKind::Name(RenameMode::From)) * -* 2) EventKind::Modify(ModifyKind::Name(RenameMode::To)) * -* 3) EventKind::Modify(ModifyKind::Name(RenameMode::Both)) * -* Update Directory (rename): * -* 1) EventKind::Modify(ModifyKind::Name(RenameMode::From)) * -* 2) EventKind::Modify(ModifyKind::Name(RenameMode::To)) * -* 3) EventKind::Modify(ModifyKind::Name(RenameMode::Both)) * -* Delete File: * -* 1) EventKind::Remove(RemoveKind::File) * -* Delete Directory: * -* 1) EventKind::Remove(RemoveKind::Folder) * -* * -* Events dispatched on MacOS: * -* Create File: * -* 1) EventKind::Create(CreateKind::File) * -* 2) EventKind::Modify(ModifyKind::Data(DataChange::Content)) * -* Create Directory: * -* 1) EventKind::Create(CreateKind::Folder) * -* Update File: * -* 1) EventKind::Modify(ModifyKind::Data(DataChange::Content)) * -* Update File (rename): * -* 1) EventKind::Modify(ModifyKind::Name(RenameMode::Any)) -- From * -* 2) EventKind::Modify(ModifyKind::Name(RenameMode::Any)) -- To * -* Update Directory (rename): * -* 1) EventKind::Modify(ModifyKind::Name(RenameMode::Any)) -- From * -* 2) EventKind::Modify(ModifyKind::Name(RenameMode::Any)) -- To * -* Delete File: * -* 1) EventKind::Remove(RemoveKind::File) * -* Delete Directory: * -* 1) EventKind::Remove(RemoveKind::Folder) * -* * -* Events dispatched on Windows: * -* Create File: * -* 1) EventKind::Create(CreateKind::Any) * -* 2) EventKind::Modify(ModifyKind::Any) * -* Create Directory: * -* 1) EventKind::Create(CreateKind::Any) * -* Update File: * -* 1) EventKind::Modify(ModifyKind::Any) * -* Update File (rename): * -* 1) EventKind::Modify(ModifyKind::Name(RenameMode::From)) * -* 2) EventKind::Modify(ModifyKind::Name(RenameMode::To)) * -* Update Directory (rename): * -* 1) EventKind::Modify(ModifyKind::Name(RenameMode::From)) * -* 2) EventKind::Modify(ModifyKind::Name(RenameMode::To)) * -* Delete File: * -* 1) EventKind::Remove(RemoveKind::Any) * -* Delete Directory: * -* 1) EventKind::Remove(RemoveKind::Any) * -* * -* Events dispatched on Android: * -* TODO * -* * -* Events dispatched on iOS: * -* TODO * -* * -***************************************************************************************************/ -#[cfg(test)] -#[allow(clippy::unwrap_used, clippy::panic)] -mod tests { - use std::{ - io::ErrorKind, - path::{Path, PathBuf}, - time::Duration, - }; - - use notify::{ - event::{CreateKind, ModifyKind, RemoveKind, RenameMode}, - Config, Event, EventKind, RecommendedWatcher, Watcher, - }; - use tempfile::{tempdir, TempDir}; - use tokio::{fs, io::AsyncWriteExt, sync::mpsc, time::sleep}; - use tracing::{debug, error}; - // use tracing_test::traced_test; - - #[cfg(any(target_os = "macos", target_os = "ios"))] - use notify::event::DataChange; - - #[cfg(target_os = "linux")] - use notify::event::{AccessKind, AccessMode}; - - async fn setup_watcher() -> ( - TempDir, - RecommendedWatcher, - mpsc::UnboundedReceiver>, - ) { - let (events_tx, events_rx) = mpsc::unbounded_channel(); - - let watcher = RecommendedWatcher::new( - move |result| { - events_tx - .send(result) - .expect("Unable to send watcher event"); - }, - Config::default(), - ) - .expect("Failed to create watcher"); - - (tempdir().unwrap(), watcher, events_rx) - } - - async fn expect_event( - mut events_rx: mpsc::UnboundedReceiver>, - path: impl AsRef, - expected_event: EventKind, - ) { - let path = path.as_ref(); - debug!(?expected_event, path = %path.display()); - let mut tries = 0; - loop { - match events_rx.try_recv() { - Ok(maybe_event) => { - let event = maybe_event.expect("Failed to receive event"); - debug!(?event, "Received event;"); - // Using `ends_with` and removing root path here due to a weird edge case on CI tests at MacOS - if event.paths[0].ends_with(path.iter().skip(1).collect::()) - && event.kind == expected_event - { - debug!("Received expected event"); - break; - } - } - Err(e) => { - debug!(?e, "No event yet;"); - tries += 1; - sleep(Duration::from_millis(100)).await; - } - } - - if tries == 10 { - panic!("No expected event received after 10 tries"); - } - } - } - - #[tokio::test] - // #[traced_test] - async fn create_file_event() { - let (root_dir, mut watcher, events_rx) = setup_watcher().await; - - watcher - .watch(root_dir.path(), notify::RecursiveMode::Recursive) - .expect("Failed to watch root directory"); - debug!(root = %root_dir.path().display(), "Now watching;"); - - let file_path = root_dir.path().join("test.txt"); - fs::write(&file_path, "test").await.unwrap(); - - #[cfg(target_os = "windows")] - expect_event(events_rx, &file_path, EventKind::Modify(ModifyKind::Any)).await; - - #[cfg(any(target_os = "macos", target_os = "ios"))] - expect_event( - events_rx, - &file_path, - EventKind::Modify(ModifyKind::Data(DataChange::Content)), - ) - .await; - - #[cfg(target_os = "linux")] - expect_event( - events_rx, - &file_path, - EventKind::Access(AccessKind::Close(AccessMode::Write)), - ) - .await; - - debug!(root = %root_dir.path().display(), "Unwatching root directory;"); - if let Err(e) = watcher.unwatch(root_dir.path()) { - error!(?e, "Failed to unwatch root directory;"); - } - } - - #[tokio::test] - // #[traced_test] - async fn create_dir_event() { - let (root_dir, mut watcher, events_rx) = setup_watcher().await; - - watcher - .watch(root_dir.path(), notify::RecursiveMode::Recursive) - .expect("Failed to watch root directory"); - debug!(root = %root_dir.path().display(), "Now watching;"); - - let dir_path = root_dir.path().join("inner"); - fs::create_dir(&dir_path) - .await - .expect("Failed to create directory"); - - #[cfg(target_os = "windows")] - expect_event(events_rx, &dir_path, EventKind::Create(CreateKind::Any)).await; - - #[cfg(any(target_os = "macos", target_os = "ios"))] - expect_event(events_rx, &dir_path, EventKind::Create(CreateKind::Folder)).await; - - #[cfg(target_os = "linux")] - expect_event(events_rx, &dir_path, EventKind::Create(CreateKind::Folder)).await; - - debug!(root = %root_dir.path().display(), "Unwatching root directory;"); - if let Err(e) = watcher.unwatch(root_dir.path()) { - error!(?e, "Failed to unwatch root directory;"); - } - } - - #[tokio::test] - // #[traced_test] - async fn update_file_event() { - let (root_dir, mut watcher, events_rx) = setup_watcher().await; - - let file_path = root_dir.path().join("test.txt"); - fs::write(&file_path, "test").await.unwrap(); - - watcher - .watch(root_dir.path(), notify::RecursiveMode::Recursive) - .expect("Failed to watch root directory"); - debug!(root = %root_dir.path().display(), "Now watching;"); - - let mut file = fs::OpenOptions::new() - .append(true) - .open(&file_path) - .await - .expect("Failed to open file"); - - // Writing then sync data before closing the file - file.write_all(b"\nanother test") - .await - .expect("Failed to write to file"); - file.sync_all().await.expect("Failed to flush file"); - drop(file); - - #[cfg(target_os = "windows")] - expect_event(events_rx, &file_path, EventKind::Modify(ModifyKind::Any)).await; - - #[cfg(any(target_os = "macos", target_os = "ios"))] - expect_event( - events_rx, - &file_path, - EventKind::Modify(ModifyKind::Data(DataChange::Content)), - ) - .await; - - #[cfg(target_os = "linux")] - expect_event( - events_rx, - &file_path, - EventKind::Access(AccessKind::Close(AccessMode::Write)), - ) - .await; - - debug!(root = %root_dir.path().display(), "Unwatching root directory;"); - if let Err(e) = watcher.unwatch(root_dir.path()) { - error!(?e, "Failed to unwatch root directory;"); - } - } - - #[tokio::test] - // #[traced_test] - async fn update_file_rename_event() { - let (root_dir, mut watcher, events_rx) = setup_watcher().await; - - let file_path = root_dir.path().join("test.txt"); - fs::write(&file_path, "test").await.unwrap(); - - watcher - .watch(root_dir.path(), notify::RecursiveMode::Recursive) - .expect("Failed to watch root directory"); - debug!(root = %root_dir.path().display(), "Now watching;"); - - let new_file_name = root_dir.path().join("test2.txt"); - - fs::rename(&file_path, &new_file_name) - .await - .expect("Failed to rename file"); - - #[cfg(target_os = "windows")] - expect_event( - events_rx, - &new_file_name, - EventKind::Modify(ModifyKind::Name(RenameMode::To)), - ) - .await; - - #[cfg(any(target_os = "macos", target_os = "ios"))] - expect_event( - events_rx, - &file_path, - EventKind::Modify(ModifyKind::Name(RenameMode::Any)), - ) - .await; - - #[cfg(target_os = "linux")] - expect_event( - events_rx, - &file_path, - EventKind::Modify(ModifyKind::Name(RenameMode::Both)), - ) - .await; - - debug!(root = %root_dir.path().display(), "Unwatching root directory;"); - if let Err(e) = watcher.unwatch(root_dir.path()) { - error!(?e, "Failed to unwatch root directory;"); - } - } - - #[tokio::test] - // #[traced_test] - async fn update_dir_event() { - let (root_dir, mut watcher, events_rx) = setup_watcher().await; - - let dir_path = root_dir.path().join("inner"); - fs::create_dir(&dir_path) - .await - .expect("Failed to create directory"); - - watcher - .watch(root_dir.path(), notify::RecursiveMode::Recursive) - .expect("Failed to watch root directory"); - debug!(root = %root_dir.path().display(), "Now watching;"); - - let new_dir_name = root_dir.path().join("inner2"); - - fs::rename(&dir_path, &new_dir_name) - .await - .expect("Failed to rename directory"); - - #[cfg(target_os = "windows")] - expect_event( - events_rx, - &new_dir_name, - EventKind::Modify(ModifyKind::Name(RenameMode::To)), - ) - .await; - - #[cfg(any(target_os = "macos", target_os = "ios"))] - expect_event( - events_rx, - &dir_path, - EventKind::Modify(ModifyKind::Name(RenameMode::Any)), - ) - .await; - - #[cfg(target_os = "linux")] - expect_event( - events_rx, - &dir_path, - EventKind::Modify(ModifyKind::Name(RenameMode::Both)), - ) - .await; - - debug!(root = %root_dir.path().display(), "Unwatching root directory;"); - if let Err(e) = watcher.unwatch(root_dir.path()) { - error!(?e, "Failed to unwatch root directory;"); - } - } - - #[tokio::test] - // #[traced_test] - async fn delete_file_event() { - let (root_dir, mut watcher, events_rx) = setup_watcher().await; - - let file_path = root_dir.path().join("test.txt"); - fs::write(&file_path, "test").await.unwrap(); - - watcher - .watch(root_dir.path(), notify::RecursiveMode::Recursive) - .expect("Failed to watch root directory"); - debug!(root = %root_dir.path().display(), "Now watching;"); - - fs::remove_file(&file_path) - .await - .expect("Failed to remove file"); - - #[cfg(target_os = "windows")] - expect_event(events_rx, &file_path, EventKind::Remove(RemoveKind::Any)).await; - - #[cfg(target_os = "macos")] - expect_event(events_rx, &file_path, EventKind::Remove(RemoveKind::File)).await; - - #[cfg(target_os = "linux")] - expect_event(events_rx, &file_path, EventKind::Remove(RemoveKind::File)).await; - - #[cfg(target_os = "ios")] - expect_event( - events_rx, - &file_path, - EventKind::Modify(ModifyKind::Metadata(MetadataKind::Any)), - ) - .await; - - debug!(root = %root_dir.path().display(), "Unwatching root directory;"); - if let Err(e) = watcher.unwatch(root_dir.path()) { - error!(?e, "Failed to unwatch root directory;"); - } - } - - #[tokio::test] - // #[traced_test] - async fn delete_dir_event() { - let (root_dir, mut watcher, events_rx) = setup_watcher().await; - - let dir_path = root_dir.path().join("inner"); - fs::create_dir(&dir_path) - .await - .expect("Failed to create directory"); - - if let Err(e) = fs::metadata(&dir_path).await { - if e.kind() == ErrorKind::NotFound { - panic!("Directory not found"); - } else { - panic!("{e}"); - } - } - - watcher - .watch(root_dir.path(), notify::RecursiveMode::Recursive) - .expect("Failed to watch root directory"); - debug!(root = %root_dir.path().display(), "Now watching;"); - - debug!("First unwatching the inner directory before removing it"); - if let Err(e) = watcher.unwatch(&dir_path) { - error!(?e, "Failed to unwatch inner directory;"); - } - - fs::remove_dir(&dir_path) - .await - .expect("Failed to remove directory"); - - #[cfg(target_os = "windows")] - expect_event(events_rx, &dir_path, EventKind::Remove(RemoveKind::Any)).await; - - #[cfg(target_os = "macos")] - expect_event(events_rx, &dir_path, EventKind::Remove(RemoveKind::Folder)).await; - - #[cfg(target_os = "linux")] - expect_event(events_rx, &dir_path, EventKind::Remove(RemoveKind::Folder)).await; - - #[cfg(target_os = "ios")] - expect_event( - events_rx, - &file_path, - EventKind::Modify(ModifyKind::Metadata(MetadataKind::Any)), - ) - .await; - - debug!(root = %root_dir.path().display(), "Unwatching root directory;"); - if let Err(e) = watcher.unwatch(root_dir.path()) { - error!(?e, "Failed to unwatch root directory;"); - } - } -} diff --git a/core/src/location/manager/watcher/utils.rs b/core/src/location/manager/watcher/utils.rs deleted file mode 100644 index 88b065810..000000000 --- a/core/src/location/manager/watcher/utils.rs +++ /dev/null @@ -1,1241 +0,0 @@ -use crate::{ - invalidate_query, - library::Library, - location::{ - create_file_path, delete_directory, find_location, - indexer::reverse_update_directories_sizes, location_with_indexer_rules, - manager::LocationManagerError, scan_location_sub_path, update_location_size, - }, - object::validation::hash::file_checksum, - Node, -}; - -use sd_core_file_path_helper::{ - check_file_path_exists, filter_existing_file_path_params, - isolated_file_path_data::extract_normalized_materialized_path_str, - loose_find_existing_file_path_params, path_is_hidden, FilePathError, FilePathMetadata, - IsolatedFilePathData, MetadataExt, -}; -use sd_core_heavy_lifting::{ - file_identifier::FileMetadata, - media_processor::{ - exif_media_data, ffmpeg_media_data, generate_single_thumbnail, get_thumbnails_directory, - ThumbnailKind, - }, -}; -use sd_core_indexer_rules::{ - seed::{GitIgnoreRules, GITIGNORE}, - IndexerRuler, RulerDecision, -}; -use sd_core_prisma_helpers::{ - file_path_watcher_remove, file_path_with_object, object_ids, CasId, ObjectPubId, -}; - -use sd_file_ext::{ - extensions::{AudioExtension, ImageExtension, VideoExtension}, - kind::ObjectKind, -}; -use sd_prisma::{ - prisma::{device, file_path, location, object}, - prisma_sync, -}; -use sd_sync::{option_sync_db_entry, sync_db_entry, sync_entry, OperationFactory}; -use sd_utils::{ - chain_optional_iter, - db::{inode_from_db, inode_to_db, maybe_missing, size_in_bytes_to_db}, - error::FileIOError, -}; - -#[cfg(target_family = "unix")] -use sd_core_file_path_helper::get_inode; - -#[cfg(target_family = "windows")] -use sd_core_file_path_helper::get_inode_from_path; - -use std::{ - collections::{HashMap, HashSet}, - ffi::OsStr, - fs::Metadata, - path::{Path, PathBuf}, - str::FromStr, - sync::Arc, -}; - -use chrono::{DateTime, FixedOffset, Local, Utc}; -use futures_concurrency::future::Join; -use notify::Event; -use tokio::{ - fs, - io::{self, ErrorKind}, - spawn, - time::{sleep, Instant}, -}; -use tracing::{error, instrument, trace, warn}; - -use super::{INode, HUNDRED_MILLIS, ONE_SECOND}; - -pub(super) async fn reject_event( - event: &Event, - ignore_paths: &HashSet, - location_path: Option<&Path>, - indexer_ruler: Option<&IndexerRuler>, -) -> bool { - // if path includes .DS_Store, .spacedrive file creation or is in the `ignore_paths` set, we ignore - if event.paths.iter().any(|p| { - p.file_name() - .and_then(OsStr::to_str) - .map_or(false, |name| name == ".DS_Store" || name == ".spacedrive") - || ignore_paths.contains(p) - }) { - trace!("Rejected by ignored paths"); - return true; - } - - if let Some(indexer_ruler) = indexer_ruler { - let ruler_decisions = event - .paths - .iter() - .map(|path| async move { (path, fs::metadata(path).await) }) - .collect::>() - .join() - .await - .into_iter() - .filter_map(|(path, res)| { - res.map(|metadata| (path, metadata)) - .map_err(|e| { - if e.kind() != ErrorKind::NotFound { - error!(?e, path = %path.display(), "Failed to get metadata for path;"); - } - }) - .ok() - }) - .map(|(path, metadata)| { - let mut independent_ruler = indexer_ruler.clone(); - - async move { - let path_to_check_gitignore = if metadata.is_dir() { - Some(path.as_path()) - } else { - path.parent() - }; - - if let (Some(path_to_check_gitignore), Some(location_path)) = - (path_to_check_gitignore, location_path.as_ref()) - { - if independent_ruler.has_system(&GITIGNORE) { - if let Some(rules) = GitIgnoreRules::get_rules_if_in_git_repo( - location_path, - path_to_check_gitignore, - ) - .await - { - trace!("Found gitignore rules to follow"); - independent_ruler.extend(rules.map(Into::into)); - } - } - } - - independent_ruler.evaluate_path(path, &metadata).await - } - }) - .collect::>() - .join() - .await; - - if !ruler_decisions.is_empty() - && ruler_decisions.into_iter().all(|res| { - matches!( - res.map_err(|e| trace!(?e, "Failed to evaluate path;")) - // In case of error, we accept the path as a safe default - .unwrap_or(RulerDecision::Accept), - RulerDecision::Reject - ) - }) { - trace!("Rejected by indexer ruler"); - return true; - } - } - - false -} - -#[instrument(skip_all, fields(path = %path.as_ref().display()), err)] -pub(super) async fn create_dir( - location_id: location::id::Type, - path: impl AsRef + Send, - metadata: &Metadata, - node: &Arc, - library: &Arc, -) -> Result<(), LocationManagerError> { - let location = find_location(library, location_id) - .include(location_with_indexer_rules::include()) - .exec() - .await? - .ok_or(LocationManagerError::LocationNotFound(location_id))?; - - let path = path.as_ref(); - - let location_path = maybe_missing(&location.path, "location.path")?; - - trace!(new_directory = %path.display(), "Creating directory;"); - - let iso_file_path = IsolatedFilePathData::new(location.id, location_path, path, true)?; - - let parent_iso_file_path = iso_file_path.parent(); - if !parent_iso_file_path.is_root() - && !check_file_path_exists::(&parent_iso_file_path, &library.db).await? - { - warn!(%iso_file_path, "Watcher found a directory without parent;"); - - return Ok(()); - }; - - let children_materialized_path = iso_file_path - .materialized_path_for_children() - .expect("We're in the create dir function lol"); - - create_file_path( - library, - iso_file_path.to_parts(), - None, - FilePathMetadata::from_path(path, metadata)?, - ) - .await?; - - spawn({ - let node = Arc::clone(node); - let library = Arc::clone(library); - - async move { - // Wait a bit for any files being moved into the new directory to be indexed by the watcher - sleep(ONE_SECOND).await; - - trace!(%iso_file_path, "Scanning new directory;"); - - // scan the new directory - if let Err(e) = - scan_location_sub_path(&node, &library, location, &children_materialized_path).await - { - error!(?e, "Failed to scan new directory;"); - } - } - }); - - invalidate_query!(library, "search.paths"); - invalidate_query!(library, "search.objects"); - - Ok(()) -} - -#[instrument(skip_all, fields(path = %path.as_ref().display()), err)] -pub(super) async fn create_file( - location_id: location::id::Type, - path: impl AsRef + Send, - metadata: &Metadata, - node: &Arc, - library: &Arc, -) -> Result<(), LocationManagerError> { - inner_create_file( - location_id, - extract_location_path(location_id, library).await?, - path, - metadata, - node, - library, - ) - .await -} - -async fn inner_create_file( - location_id: location::id::Type, - location_path: impl AsRef + Send, - path: impl AsRef + Send, - metadata: &Metadata, - node: &Arc, - library @ Library { - id: library_id, - db, - sync, - .. - }: &Library, -) -> Result<(), LocationManagerError> { - let path = path.as_ref(); - let location_path = location_path.as_ref(); - - trace!(new_file = %path.display(), "Creating file;"); - - let iso_file_path = IsolatedFilePathData::new(location_id, location_path, path, false)?; - let iso_file_path_parts = iso_file_path.to_parts(); - let extension = iso_file_path_parts.extension.to_string(); - - let metadata = FilePathMetadata::from_path(path, metadata)?; - - // First we check if already exist a file with this same inode number - // if it does, we just update it - if let Some(file_path) = db - .file_path() - .find_unique(file_path::location_id_inode( - location_id, - inode_to_db(metadata.inode), - )) - .include(file_path_with_object::include()) - .exec() - .await? - { - trace!(%iso_file_path, "File already exists with that inode;"); - - return inner_update_file(location_path, &file_path, path, node, library, None).await; - - // If we can't find an existing file with the same inode, we check if there is a file with the same path - } else if let Some(file_path) = db - .file_path() - .find_unique(file_path::location_id_materialized_path_name_extension( - location_id, - iso_file_path_parts.materialized_path.to_string(), - iso_file_path_parts.name.to_string(), - iso_file_path_parts.extension.to_string(), - )) - .include(file_path_with_object::include()) - .exec() - .await? - { - trace!(%iso_file_path, "File already exists with that iso_file_path;"); - - return inner_update_file( - location_path, - &file_path, - path, - node, - library, - Some(metadata.inode), - ) - .await; - } - - let parent_iso_file_path = iso_file_path.parent(); - if !parent_iso_file_path.is_root() - && !check_file_path_exists::(&parent_iso_file_path, db).await? - { - warn!(%iso_file_path, "Watcher found a file without parent;"); - - return Ok(()); - }; - - // generate provisional object - let FileMetadata { - cas_id, - kind, - fs_metadata, - } = FileMetadata::new(&location_path, &iso_file_path).await?; - - let created_file = - create_file_path(library, iso_file_path_parts, cas_id.clone(), metadata).await?; - - let existing_object = db - .object() - .find_first(vec![object::file_paths::some(vec![ - file_path::cas_id::equals(cas_id.clone().map(Into::into)), - file_path::pub_id::not(created_file.pub_id.clone()), - ])]) - .select(object_ids::select()) - .exec() - .await?; - - let is_new_file = existing_object.is_none(); - - let object_ids::Data { - id: object_id, - pub_id: object_pub_id, - } = if let Some(object) = existing_object { - object - } else { - let pub_id: ObjectPubId = ObjectPubId::new(); - let date_created: DateTime = - DateTime::::from(fs_metadata.created_or_now()).into(); - let int_kind = kind as i32; - - let device_pub_id = sync.device_pub_id.to_db(); - - let (sync_params, db_params) = [ - sync_db_entry!(date_created, object::date_created), - sync_db_entry!(int_kind, object::kind), - ( - sync_entry!( - prisma_sync::device::SyncId { - pub_id: device_pub_id.clone() - }, - object::device - ), - object::device::connect(device::pub_id::equals(device_pub_id)), - ), - ] - .into_iter() - .unzip::<_, _, Vec<_>, Vec<_>>(); - - sync.write_op( - db, - sync.shared_create( - prisma_sync::object::SyncId { - pub_id: pub_id.to_db(), - }, - sync_params, - ), - db.object() - .create(pub_id.into(), db_params) - .select(object_ids::select()), - ) - .await? - }; - - sync.write_op( - db, - sync.shared_update( - prisma_sync::location::SyncId { - pub_id: created_file.pub_id.clone(), - }, - [sync_entry!( - prisma_sync::object::SyncId { - pub_id: object_pub_id.clone() - }, - file_path::object - )], - ), - db.file_path() - .update( - file_path::pub_id::equals(created_file.pub_id.clone()), - vec![file_path::object::connect(object::pub_id::equals( - object_pub_id.clone(), - ))], - ) - .select(file_path::select!({ id })), - ) - .await?; - - // If the file is a duplicate of an existing file, we don't need to generate thumbnails nor extract media data - if is_new_file - && !extension.is_empty() - && matches!( - kind, - ObjectKind::Image | ObjectKind::Video | ObjectKind::Audio - ) { - // Running in a detached task as thumbnail generation can take a while and we don't want to block the watcher - if matches!(kind, ObjectKind::Image | ObjectKind::Video) { - if let Some(cas_id) = cas_id { - spawn({ - let extension = extension.clone(); - let path = path.to_path_buf(); - let thumbnails_directory = - get_thumbnails_directory(node.config.data_directory()); - let library_id = *library_id; - - async move { - if let Err(e) = generate_single_thumbnail( - &thumbnails_directory, - extension, - cas_id, - path, - ThumbnailKind::Indexed(library_id), - ) - .await - { - error!(?e, "Failed to generate thumbnail in the watcher;"); - } - } - }); - } - } - - match kind { - ObjectKind::Image => { - if let Ok(image_extension) = ImageExtension::from_str(&extension) { - if exif_media_data::can_extract(image_extension) { - if let Ok(Some(exif_data)) = exif_media_data::extract(path) - .await - .map_err(|e| error!(?e, "Failed to extract image media data;")) - { - exif_media_data::save( - [(exif_data, object_id, object_pub_id.into())], - db, - sync, - ) - .await?; - } - } - } - } - - ObjectKind::Audio => { - if let Ok(audio_extension) = AudioExtension::from_str(&extension) { - if ffmpeg_media_data::can_extract_for_audio(audio_extension) { - if let Ok(ffmpeg_data) = ffmpeg_media_data::extract(path) - .await - .map_err(|e| error!(?e, "Failed to extract audio media data;")) - { - ffmpeg_media_data::save([(ffmpeg_data, object_id)], db).await?; - } - } - } - } - - ObjectKind::Video => { - if let Ok(video_extension) = VideoExtension::from_str(&extension) { - if ffmpeg_media_data::can_extract_for_video(video_extension) { - if let Ok(ffmpeg_data) = ffmpeg_media_data::extract(path) - .await - .map_err(|e| error!(?e, "Failed to extract video media data;")) - { - ffmpeg_media_data::save([(ffmpeg_data, object_id)], db).await?; - } - } - } - } - - _ => { - // Do nothing - } - } - } - - invalidate_query!(library, "search.paths"); - invalidate_query!(library, "search.objects"); - - Ok(()) -} - -#[instrument(skip_all, fields(path = %path.as_ref().display()), err)] -pub(super) async fn update_file( - location_id: location::id::Type, - path: impl AsRef + Send, - node: &Arc, - library: &Arc, -) -> Result<(), LocationManagerError> { - let full_path = path.as_ref(); - - let metadata = match fs::metadata(full_path).await { - Ok(metadata) => metadata, - Err(e) if e.kind() == io::ErrorKind::NotFound => { - // If the file doesn't exist anymore, it was just a temporary file - return Ok(()); - } - Err(e) => return Err(FileIOError::from((full_path, e)).into()), - }; - - let location_path = extract_location_path(location_id, library).await?; - - if let Some(ref file_path) = library - .db - .file_path() - .find_first(filter_existing_file_path_params( - &IsolatedFilePathData::new(location_id, &location_path, full_path, false)?, - )) - // include object for orphan check - .include(file_path_with_object::include()) - .exec() - .await? - { - inner_update_file(location_path, file_path, full_path, node, library, None).await - } else { - inner_create_file( - location_id, - location_path, - full_path, - &metadata, - node, - library, - ) - .await - } - .map(|()| { - invalidate_query!(library, "search.paths"); - invalidate_query!(library, "search.objects"); - }) -} - -async fn inner_update_file( - location_path: impl AsRef + Send, - file_path: &file_path_with_object::Data, - full_path: impl AsRef + Send, - node: &Arc, - library @ Library { db, sync, .. }: &Library, - maybe_new_inode: Option, -) -> Result<(), LocationManagerError> { - let full_path = full_path.as_ref(); - let location_path = location_path.as_ref(); - - let current_inode = - inode_from_db(&maybe_missing(file_path.inode.as_ref(), "file_path.inode")?[0..8]); - - trace!( - location_path = %location_path.display(), - path = %full_path.display(), - "Updating file;", - ); - - let iso_file_path = IsolatedFilePathData::try_from(file_path)?; - - let FileMetadata { - cas_id, - fs_metadata, - kind, - } = FileMetadata::new(&location_path, &iso_file_path).await?; - - let inode = if let Some(inode) = maybe_new_inode { - inode - } else { - #[cfg(target_family = "unix")] - { - get_inode(&fs_metadata) - } - - #[cfg(target_family = "windows")] - { - // FIXME: This is a workaround for Windows, because we can't get the inode from the metadata - get_inode_from_path(full_path).await? - } - }; - - let is_hidden = path_is_hidden(full_path, &fs_metadata); - if file_path.cas_id.as_deref() != cas_id.as_ref().map(CasId::as_str) { - let (sync_params, db_params) = chain_optional_iter( - [ - sync_db_entry!( - size_in_bytes_to_db(fs_metadata.len()), - file_path::size_in_bytes_bytes - ), - sync_db_entry!( - DateTime::::from(fs_metadata.modified_or_now()), - file_path::date_modified - ), - ], - [ - option_sync_db_entry!(file_path.cas_id.clone(), file_path::cas_id), - option_sync_db_entry!( - if file_path.integrity_checksum.is_some() { - // TODO: Should this be a skip rather than a null-set? - // If a checksum was already computed, we need to recompute it - Some( - file_checksum(full_path) - .await - .map_err(|e| FileIOError::from((full_path, e)))?, - ) - } else { - None - }, - file_path::integrity_checksum - ), - option_sync_db_entry!( - (current_inode != inode).then(|| inode_to_db(inode)), - file_path::inode - ), - option_sync_db_entry!( - (is_hidden != file_path.hidden.unwrap_or_default()).then_some(is_hidden), - file_path::hidden - ), - ], - ) - .into_iter() - .unzip::<_, _, Vec<_>, Vec<_>>(); - - // file content changed - sync.write_op( - db, - sync.shared_update( - prisma_sync::file_path::SyncId { - pub_id: file_path.pub_id.clone(), - }, - sync_params, - ), - db.file_path() - .update( - file_path::pub_id::equals(file_path.pub_id.clone()), - db_params, - ) - .select(file_path::select!({ id })), - ) - .await?; - - if let Some(ref object) = file_path.object { - let int_kind = kind as i32; - - if db - .file_path() - .count(vec![file_path::object_id::equals(Some(object.id))]) - .exec() - .await? == 1 - { - if object.kind.map(|k| k != int_kind).unwrap_or_default() { - let (sync_param, db_param) = sync_db_entry!(int_kind, object::kind); - sync.write_op( - db, - sync.shared_update( - prisma_sync::object::SyncId { - pub_id: object.pub_id.clone(), - }, - [sync_param], - ), - db.object() - .update(object::id::equals(object.id), vec![db_param]) - .select(object::select!({ id })), - ) - .await?; - } - } else { - let pub_id = ObjectPubId::new(); - let date_created: DateTime = - DateTime::::from(fs_metadata.created_or_now()).into(); - - let device_pub_id = sync.device_pub_id.to_db(); - - let (sync_params, db_params) = [ - sync_db_entry!(date_created, object::date_created), - sync_db_entry!(int_kind, object::kind), - ( - sync_entry!( - prisma_sync::device::SyncId { - pub_id: device_pub_id.clone() - }, - object::device - ), - object::device::connect(device::pub_id::equals(device_pub_id)), - ), - ] - .into_iter() - .unzip::<_, _, Vec<_>, Vec<_>>(); - - sync.write_op( - db, - sync.shared_create( - prisma_sync::object::SyncId { - pub_id: pub_id.to_db(), - }, - sync_params, - ), - db.object().create(pub_id.to_db(), db_params), - ) - .await?; - - sync.write_op( - db, - sync.shared_update( - prisma_sync::location::SyncId { - pub_id: file_path.pub_id.clone(), - }, - [sync_entry!( - prisma_sync::object::SyncId { - pub_id: pub_id.to_db() - }, - file_path::object - )], - ), - db.file_path() - .update( - file_path::pub_id::equals(file_path.pub_id.clone()), - vec![file_path::object::connect(object::pub_id::equals( - pub_id.into(), - ))], - ) - .select(file_path::select!({ id })), - ) - .await?; - } - - if let Some(old_cas_id) = file_path.cas_id.as_ref().map(CasId::from) { - // if this file had a thumbnail previously, we update it to match the new content - if library.thumbnail_exists(node, &old_cas_id).await? { - if let Some(ext) = file_path.extension.clone() { - // Running in a detached task as thumbnail generation can take a while and we don't want to block the watcher - if let Some(cas_id) = cas_id { - let node = Arc::clone(node); - let path = full_path.to_path_buf(); - let library_id = library.id; - let old_cas_id = old_cas_id.to_owned(); - - spawn(async move { - let thumbnails_directory = - get_thumbnails_directory(node.config.data_directory()); - - let was_overwritten = old_cas_id == cas_id; - if let Err(e) = generate_single_thumbnail( - &thumbnails_directory, - ext.clone(), - cas_id, - path, - ThumbnailKind::Indexed(library_id), - ) - .await - { - error!(?e, "Failed to generate thumbnail in the watcher;"); - } - - // If only a few bytes changed, cas_id will probably remains intact - // so we overwrote our previous thumbnail, so we can't remove it - if !was_overwritten { - // remove the old thumbnail as we're generating a new one - let thumb_path = ThumbnailKind::Indexed(library_id) - .compute_path(node.config.data_directory(), &old_cas_id); - if let Err(e) = fs::remove_file(&thumb_path).await { - error!( - e = ?FileIOError::from((thumb_path, e)), - "Failed to remove old thumbnail;", - ); - } - } - }); - } - } - } - } - - if let Some(extension) = &file_path.extension { - match kind { - ObjectKind::Image => { - if let Ok(image_extension) = ImageExtension::from_str(extension) { - if exif_media_data::can_extract(image_extension) { - if let Ok(Some(exif_data)) = exif_media_data::extract(full_path) - .await - .map_err(|e| error!(?e, "Failed to extract media data;")) - { - exif_media_data::save( - [(exif_data, object.id, object.pub_id.as_slice().into())], - db, - sync, - ) - .await?; - } - } - } - } - - ObjectKind::Audio => { - if let Ok(audio_extension) = AudioExtension::from_str(extension) { - if ffmpeg_media_data::can_extract_for_audio(audio_extension) { - if let Ok(ffmpeg_data) = ffmpeg_media_data::extract(full_path) - .await - .map_err(|e| error!(?e, "Failed to extract media data;")) - { - ffmpeg_media_data::save([(ffmpeg_data, object.id)], db).await?; - } - } - } - } - - ObjectKind::Video => { - if let Ok(video_extension) = VideoExtension::from_str(extension) { - if ffmpeg_media_data::can_extract_for_video(video_extension) { - if let Ok(ffmpeg_data) = ffmpeg_media_data::extract(full_path) - .await - .map_err(|e| error!(?e, "Failed to extract media data;")) - { - ffmpeg_media_data::save([(ffmpeg_data, object.id)], db).await?; - } - } - } - } - - _ => { - // Do nothing - } - } - } - } - - invalidate_query!(library, "search.paths"); - invalidate_query!(library, "search.objects"); - } else if is_hidden != file_path.hidden.unwrap_or_default() { - let (sync_param, db_param) = sync_db_entry!(is_hidden, file_path::hidden); - - sync.write_op( - db, - sync.shared_update( - prisma_sync::file_path::SyncId { - pub_id: file_path.pub_id.clone(), - }, - [sync_param], - ), - db.file_path() - .update( - file_path::pub_id::equals(file_path.pub_id.clone()), - vec![db_param], - ) - .select(file_path::select!({ id })), - ) - .await?; - - invalidate_query!(library, "search.paths"); - } - - Ok(()) -} - -#[instrument( - skip_all, - fields(new_path = %new_path.as_ref().display(), old_path = %old_path.as_ref().display()), - err, -)] -pub(super) async fn rename( - location_id: location::id::Type, - new_path: impl AsRef + Send, - old_path: impl AsRef + Send, - new_path_metadata: Metadata, - library: &Library, -) -> Result<(), LocationManagerError> { - let location_path = extract_location_path(location_id, library).await?; - let old_path = old_path.as_ref(); - let new_path = new_path.as_ref(); - let Library { db, sync, .. } = library; - - let old_path_materialized_str = - extract_normalized_materialized_path_str(location_id, &location_path, old_path)?; - - let new_path_materialized_str = - extract_normalized_materialized_path_str(location_id, &location_path, new_path)?; - - // Renaming a file could potentially be a move to another directory, - // so we check if our parent changed - if old_path_materialized_str != new_path_materialized_str - && !check_file_path_exists::( - &IsolatedFilePathData::new(location_id, &location_path, new_path, true)?.parent(), - db, - ) - .await? - { - return Err(LocationManagerError::MoveError { - path: new_path.into(), - reason: "parent directory does not exist", - }); - } - - if let Some(file_path) = db - .file_path() - .find_first(loose_find_existing_file_path_params( - location_id, - &location_path, - old_path, - )?) - .exec() - .await? - { - let is_dir = maybe_missing(file_path.is_dir, "file_path.is_dir")?; - - let new = IsolatedFilePathData::new(location_id, &location_path, new_path, is_dir)?; - let new_parts = new.to_parts(); - - // If the renamed path is a directory, we have to update every successor - if is_dir { - let old = IsolatedFilePathData::new(location_id, &location_path, old_path, is_dir)?; - let old_parts = old.to_parts(); - - let starts_with = format!("{}/{}/", old_parts.materialized_path, old_parts.name); - let paths = db - .file_path() - .find_many(vec![ - file_path::location_id::equals(Some(location_id)), - file_path::materialized_path::starts_with(starts_with.clone()), - ]) - .select(file_path::select!({ - id - pub_id - materialized_path - })) - .exec() - .await?; - - let total_paths_count = paths.len(); - let (sync_params, db_params) = paths - .into_iter() - .filter_map(|path| path.materialized_path.map(|mp| (path.id, path.pub_id, mp))) - .map(|(id, pub_id, mp)| { - let new_path = mp.replace( - &starts_with, - &format!("{}/{}/", new_parts.materialized_path, new_parts.name), - ); - - let (sync_param, db_param) = - sync_db_entry!(new_path, file_path::materialized_path); - - ( - sync.shared_update( - sd_prisma::prisma_sync::file_path::SyncId { pub_id }, - [sync_param], - ), - db.file_path() - .update(file_path::id::equals(id), vec![db_param]) - .select(file_path::select!({ id })), - ) - }) - .unzip::<_, _, Vec<_>, Vec<_>>(); - - if !sync_params.is_empty() && !db_params.is_empty() { - sync.write_ops(db, (sync_params, db_params)).await?; - } - - trace!(%total_paths_count, "Updated file_paths;"); - } - - let (sync_params, db_params) = [ - sync_db_entry!(new_path_materialized_str, file_path::materialized_path), - sync_db_entry!(new_parts.name.to_string(), file_path::name), - sync_db_entry!(new_parts.extension.to_string(), file_path::extension), - sync_db_entry!( - DateTime::::from(new_path_metadata.modified_or_now()), - file_path::date_modified - ), - sync_db_entry!( - path_is_hidden(new_path, &new_path_metadata), - file_path::hidden - ), - ] - .into_iter() - .unzip::<_, _, Vec<_>, Vec<_>>(); - - sync.write_op( - db, - sync.shared_update( - prisma_sync::file_path::SyncId { - pub_id: file_path.pub_id.clone(), - }, - sync_params, - ), - db.file_path() - .update(file_path::pub_id::equals(file_path.pub_id), db_params) - .select(file_path::select!({ id })), - ) - .await?; - - invalidate_query!(library, "search.paths"); - invalidate_query!(library, "search.objects"); - } - - Ok(()) -} - -#[instrument(skip_all, fields(path = %path.as_ref().display()), err)] -pub(super) async fn remove( - location_id: location::id::Type, - path: impl AsRef + Send, - library: &Library, -) -> Result<(), LocationManagerError> { - let full_path = path.as_ref(); - let location_path = extract_location_path(location_id, library).await?; - - // if it doesn't exist either way, then we don't care - let Some(file_path) = library - .db - .file_path() - .find_first(loose_find_existing_file_path_params( - location_id, - &location_path, - full_path, - )?) - .select(file_path_watcher_remove::select()) - .exec() - .await? - else { - return Ok(()); - }; - - remove_by_file_path(location_id, full_path, file_path, library).await -} - -async fn remove_by_file_path( - location_id: location::id::Type, - path: impl AsRef + Send, - file_path: file_path_watcher_remove::Data, - library: &Library, -) -> Result<(), LocationManagerError> { - // check file still exists on disk - match fs::metadata(path.as_ref()).await { - Ok(_) => { - // It's possible that in the interval of time between the removal file event being - // received and we reaching this point, the file has been created again for some - // external reason, so we just error out and hope to receive this new create event - // later - return Err(LocationManagerError::FileStillExistsOnDisk( - path.as_ref().into(), - )); - } - Err(e) if e.kind() == ErrorKind::NotFound => { - let Library { sync, db, .. } = library; - - let is_dir = maybe_missing(file_path.is_dir, "file_path.is_dir")?; - - // if is doesn't, we can remove it safely from our db - if is_dir { - delete_directory( - library, - location_id, - Some(&IsolatedFilePathData::try_from(&file_path)?), - ) - .await?; - } else { - sync.write_op( - db, - sync.shared_delete(prisma_sync::file_path::SyncId { - pub_id: file_path.pub_id, - }), - db.file_path().delete(file_path::id::equals(file_path.id)), - ) - .await?; - - if let Some(object) = file_path.object { - // If this object doesn't have any other file paths, delete it - if db - .object() - .count(vec![ - object::id::equals(object.id), - // https://www.prisma.io/docs/reference/api-reference/prisma-client-reference#none - object::file_paths::none(vec![]), - ]) - .exec() - .await? == 1 - { - sync.write_op( - db, - sync.shared_delete(prisma_sync::object::SyncId { - pub_id: object.pub_id, - }), - db.object() - .delete(object::id::equals(object.id)) - .select(object::select!({ id })), - ) - .await?; - } - } - } - } - Err(e) => return Err(FileIOError::from((path, e)).into()), - } - - invalidate_query!(library, "search.paths"); - invalidate_query!(library, "search.objects"); - - Ok(()) -} - -#[instrument(skip_all, fields(path = %path.as_ref().display()), err)] -pub(super) async fn extract_inode_from_path( - location_id: location::id::Type, - path: impl AsRef + Send, - library: &Library, -) -> Result { - let path = path.as_ref(); - let location = find_location(library, location_id) - .select(location::select!({ path })) - .exec() - .await? - .ok_or(LocationManagerError::LocationNotFound(location_id))?; - - let location_path = maybe_missing(&location.path, "location.path")?; - - library - .db - .file_path() - .find_first(loose_find_existing_file_path_params( - location_id, - location_path, - path, - )?) - .select(file_path::select!({ inode })) - .exec() - .await? - .map_or( - Err(FilePathError::NotFound(path.into()).into()), - |file_path| { - Ok(inode_from_db( - &maybe_missing(file_path.inode.as_ref(), "file_path.inode")?[0..8], - )) - }, - ) -} - -#[instrument(skip_all, err)] -pub(super) async fn extract_location_path( - location_id: location::id::Type, - library: &Library, -) -> Result { - find_location(library, location_id) - .select(location::select!({ path })) - .exec() - .await? - .map_or( - Err(LocationManagerError::LocationNotFound(location_id)), - // NOTE: The following usage of `PathBuf` doesn't incur a new allocation so it's fine - |location| Ok(maybe_missing(location.path, "location.path")?.into()), - ) -} -#[instrument(skip_all, err)] -pub(super) async fn recalculate_directories_size( - candidates: &mut HashMap, - buffer: &mut Vec<(PathBuf, Instant)>, - location_id: location::id::Type, - location_pub_id: location::pub_id::Type, - library: &Library, -) -> Result<(), LocationManagerError> { - let mut location_path_cache = None; - let mut should_invalidate = false; - let mut should_update_location_size = false; - buffer.clear(); - - for (path, instant) in candidates.drain() { - if instant.elapsed() > HUNDRED_MILLIS * 5 { - if location_path_cache.is_none() { - location_path_cache = Some(PathBuf::from(maybe_missing( - find_location(library, location_id) - .select(location::select!({ path })) - .exec() - .await? - .ok_or(LocationManagerError::LocationNotFound(location_id))? - .path, - "location.path", - )?)) - } - - if let Some(location_path) = &location_path_cache { - if path != *location_path { - trace!( - start_directory = %path.display(), - end_directory = %location_path.display(), - "Reverse calculating directory sizes;", - ); - let mut non_critical_errors = vec![]; - reverse_update_directories_sizes( - path, - location_id, - location_path, - &library.db, - &library.sync, - &mut non_critical_errors, - ) - .await - .map_err(sd_core_heavy_lifting::Error::from)?; - - if !non_critical_errors.is_empty() { - error!( - ?non_critical_errors, - "Reverse calculating directory sizes finished errors;", - ); - } - - should_invalidate = true; - } else { - should_update_location_size = true; - } - } - } else { - buffer.push((path, instant)); - } - } - - if should_update_location_size { - update_location_size(location_id, location_pub_id, library).await?; - } - - if should_invalidate { - invalidate_query!(library, "search.paths"); - invalidate_query!(library, "search.objects"); - } - - candidates.extend(buffer.drain(..)); - - Ok(()) -} diff --git a/core/src/location/manager/watcher/windows.rs b/core/src/location/manager/watcher/windows.rs deleted file mode 100644 index bd85693e8..000000000 --- a/core/src/location/manager/watcher/windows.rs +++ /dev/null @@ -1,419 +0,0 @@ -//! Windows file system event handler implementation has some caveats die to how -//! file system events are emitted on Windows. -//! -//! For example: When a file is moved to another -//! directory, we receive a remove event and then a create event, so to avoid having to actually -//! remove and create the `file_path` in the database, we have to wait some time after receiving -//! a remove event to see if a create event is emitted. If it is, we just update the `file_path` -//! in the database. If not, we remove the file from the database. - -use crate::{invalidate_query, library::Library, location::manager::LocationManagerError, Node}; - -use sd_core_file_path_helper::{get_inode_from_path, FilePathError}; - -use sd_prisma::prisma::location; -use sd_utils::error::FileIOError; - -use std::{ - collections::{BTreeMap, HashMap}, - path::{Path, PathBuf}, - sync::Arc, -}; - -use notify::{ - event::{CreateKind, ModifyKind, RenameMode}, - Event, EventKind, -}; -use tokio::{fs, time::Instant}; -use tracing::{error, instrument, trace}; - -use super::{ - utils::{ - create_dir, extract_inode_from_path, recalculate_directories_size, remove, rename, - update_file, - }, - INode, InstantAndPath, HUNDRED_MILLIS, ONE_SECOND, -}; - -/// Windows file system event handler -#[derive(Debug)] -pub(super) struct EventHandler { - location_id: location::id::Type, - location_pub_id: location::pub_id::Type, - library: Arc, - node: Arc, - last_events_eviction_check: Instant, - rename_from_map: BTreeMap, - rename_to_map: BTreeMap, - files_to_remove: HashMap, - files_to_update: HashMap, - reincident_to_update_files: HashMap, - to_recalculate_size: HashMap, - - path_and_instant_buffer: Vec<(PathBuf, Instant)>, - files_to_remove_buffer: Vec<(INode, InstantAndPath)>, -} - -impl super::EventHandler for EventHandler { - fn new( - location_id: location::id::Type, - location_pub_id: location::pub_id::Type, - library: Arc, - node: Arc, - ) -> Self - where - Self: Sized, - { - Self { - location_id, - location_pub_id, - library, - node, - last_events_eviction_check: Instant::now(), - rename_from_map: BTreeMap::new(), - rename_to_map: BTreeMap::new(), - files_to_remove: HashMap::new(), - files_to_update: HashMap::new(), - reincident_to_update_files: HashMap::new(), - to_recalculate_size: HashMap::new(), - path_and_instant_buffer: Vec::new(), - files_to_remove_buffer: Vec::new(), - } - } - - #[instrument( - skip_all, - fields( - location_id = %self.location_id, - library_id = %self.library.id, - rename_from_map_count = %self.rename_from_map.len(), - rename_to_map_count = %self.rename_to_map.len(), - files_to_remove_map = %self.files_to_remove.len(), - waiting_update_count = %self.files_to_update.len(), - reincident_to_update_files_count = %self.reincident_to_update_files.len(), - waiting_size_count = %self.to_recalculate_size.len(), - ), - )] - async fn handle_event(&mut self, event: Event) -> Result<(), LocationManagerError> { - trace!("Received Windows event"); - - let Event { - kind, mut paths, .. - } = event; - - match kind { - EventKind::Create(CreateKind::Any) => { - let path = paths.remove(0); - - let inode = match get_inode_from_path(&path).await { - Ok(inode) => inode, - - Err(FilePathError::FileIO(FileIOError { source, .. })) - if source.raw_os_error() == Some(32) => - { - // This is still being manipulated by another process, so we can just ignore it for now - // as we will probably receive update events later - self.files_to_update.insert(path, Instant::now()); - - return Ok(()); - } - - Err(e) => { - return Err(e.into()); - } - }; - - if let Some((_, old_path)) = self.files_to_remove.remove(&inode) { - // if previously we added a file to be removed with the same inode - // of this "newly created" created file, it means that the file was just moved to another location - // so we can treat if just as a file rename, like in other OSes - - trace!( - old_path = %old_path.display(), - new_path = %path.display(), - "Got a rename instead of remove/create;", - ); - - // We found a new path for this old path, so we can rename it instead of removing and creating it - rename( - self.location_id, - &path, - &old_path, - fs::metadata(&path) - .await - .map_err(|e| FileIOError::from((&path, e)))?, - &self.library, - ) - .await?; - } else { - let metadata = fs::metadata(&path) - .await - .map_err(|e| FileIOError::from((&path, e)))?; - - if metadata.is_dir() { - // Don't need to dispatch a recalculate directory event as `create_dir` dispatches - // a `scan_location_sub_path` function, which recalculates the size already - create_dir(self.location_id, path, &metadata, &self.node, &self.library) - .await?; - } else if self.files_to_update.contains_key(&path) { - if let Some(old_instant) = - self.files_to_update.insert(path.clone(), Instant::now()) - { - self.reincident_to_update_files - .entry(path) - .or_insert(old_instant); - } - } else { - self.files_to_update.insert(path, Instant::now()); - } - } - } - - EventKind::Modify(ModifyKind::Any) => { - let path = paths.remove(0); - - if self.files_to_update.contains_key(&path) { - if let Some(old_instant) = - self.files_to_update.insert(path.clone(), Instant::now()) - { - self.reincident_to_update_files - .entry(path) - .or_insert(old_instant); - } - } else { - self.files_to_update.insert(path, Instant::now()); - } - } - - EventKind::Modify(ModifyKind::Name(RenameMode::From)) => { - let path = paths.remove(0); - - let inode = extract_inode_from_path(self.location_id, &path, &self.library).await?; - - if let Some((_, new_path)) = self.rename_to_map.remove(&inode) { - // We found a new path for this old path, so we can rename it - rename( - self.location_id, - &new_path, - &path, - fs::metadata(&new_path) - .await - .map_err(|e| FileIOError::from((&new_path, e)))?, - &self.library, - ) - .await?; - } else { - self.rename_from_map.insert(inode, (Instant::now(), path)); - } - } - - EventKind::Modify(ModifyKind::Name(RenameMode::To)) => { - let path = paths.remove(0); - - let inode = get_inode_from_path(&path).await?; - - if let Some((_, old_path)) = self.rename_from_map.remove(&inode) { - // We found a old path for this new path, so we can rename it - rename( - self.location_id, - &path, - &old_path, - fs::metadata(&path) - .await - .map_err(|e| FileIOError::from((&path, e)))?, - &self.library, - ) - .await?; - } else { - self.rename_to_map.insert(inode, (Instant::now(), path)); - } - } - - EventKind::Remove(_) => { - let path = paths.remove(0); - - self.files_to_remove.insert( - extract_inode_from_path(self.location_id, &path, &self.library).await?, - (Instant::now(), path), - ); - } - - _ => { - trace!("Other Windows event that we don't handle for now"); - } - } - - Ok(()) - } - - async fn tick(&mut self) { - if self.last_events_eviction_check.elapsed() > HUNDRED_MILLIS { - if let Err(e) = self.handle_to_update_eviction().await { - error!( - ?e, - "Error while handling recently created or update files eviction;" - ); - } - - self.rename_from_map.retain(|_, (created_at, path)| { - let to_retain = created_at.elapsed() < HUNDRED_MILLIS; - - if !to_retain { - trace!(path = %path.display(), "Removing from rename from map;") - } - - to_retain - }); - - self.rename_to_map.retain(|_, (created_at, path)| { - let to_retain = created_at.elapsed() < HUNDRED_MILLIS; - - if !to_retain { - trace!(path = %path.display(), "Removing from rename to map;") - } - - to_retain - }); - - if let Err(e) = self.handle_removes_eviction().await { - error!(?e, "Failed to remove file_path;"); - } - - if !self.to_recalculate_size.is_empty() { - if let Err(e) = recalculate_directories_size( - &mut self.to_recalculate_size, - &mut self.path_and_instant_buffer, - self.location_id, - self.location_pub_id.clone(), - &self.library, - ) - .await - { - error!(?e, "Failed to recalculate directories size;"); - } - } - - self.last_events_eviction_check = Instant::now(); - } - } -} - -impl EventHandler { - async fn handle_to_update_eviction(&mut self) -> Result<(), LocationManagerError> { - self.path_and_instant_buffer.clear(); - - let mut should_invalidate = false; - - for (path, created_at) in self.files_to_update.drain() { - if created_at.elapsed() < HUNDRED_MILLIS * 5 { - self.path_and_instant_buffer.push((path, created_at)); - } else { - self.reincident_to_update_files.remove(&path); - - handle_update( - self.location_id, - &path, - &self.node, - &mut self.to_recalculate_size, - &self.library, - ) - .await?; - - should_invalidate = true; - } - } - - self.files_to_update - .extend(self.path_and_instant_buffer.drain(..)); - - self.path_and_instant_buffer.clear(); - - // We have to check if we have any reincident files to update and update them after a bigger - // timeout, this way we keep track of files being update frequently enough to bypass our - // eviction check above - for (path, created_at) in self.reincident_to_update_files.drain() { - if created_at.elapsed() < ONE_SECOND * 10 { - self.path_and_instant_buffer.push((path, created_at)); - } else { - self.files_to_update.remove(&path); - - handle_update( - self.location_id, - &path, - &self.node, - &mut self.to_recalculate_size, - &self.library, - ) - .await?; - - should_invalidate = true; - } - } - - if should_invalidate { - invalidate_query!(self.library, "search.paths"); - } - - self.reincident_to_update_files - .extend(self.path_and_instant_buffer.drain(..)); - - Ok(()) - } - - async fn handle_removes_eviction(&mut self) -> Result<(), LocationManagerError> { - self.files_to_remove_buffer.clear(); - - let mut should_invalidate = false; - - for (inode, (instant, path)) in self.files_to_remove.drain() { - if instant.elapsed() > HUNDRED_MILLIS { - if let Some(parent) = path.parent() { - if parent != Path::new("") { - self.to_recalculate_size - .insert(parent.to_path_buf(), Instant::now()); - } - } - - remove(self.location_id, &path, &self.library).await?; - - should_invalidate = true; - - trace!(path = %path.display(), "Removed file_path due timeout;"); - } else { - self.files_to_remove_buffer.push((inode, (instant, path))); - } - } - if should_invalidate { - invalidate_query!(self.library, "search.paths"); - } - - self.files_to_remove - .extend(self.files_to_remove_buffer.drain(..)); - - Ok(()) - } -} - -async fn handle_update( - location_id: location::id::Type, - path: &PathBuf, - node: &Arc, - to_recalculate_size: &mut HashMap, - library: &Arc, -) -> Result<(), LocationManagerError> { - let metadata = fs::metadata(&path) - .await - .map_err(|e| FileIOError::from((&path, e)))?; - - if metadata.is_file() { - if let Some(parent) = path.parent() { - if parent != Path::new("") { - to_recalculate_size.insert(parent.to_path_buf(), Instant::now()); - } - } - - update_file(location_id, path, node, library).await?; - } - - Ok(()) -} diff --git a/core/src/location/metadata.rs b/core/src/location/metadata.rs deleted file mode 100644 index f5138ee35..000000000 --- a/core/src/location/metadata.rs +++ /dev/null @@ -1,295 +0,0 @@ -use crate::library::LibraryId; - -use std::{ - collections::{HashMap, HashSet}, - path::{Path, PathBuf}, -}; - -use chrono::{DateTime, Utc}; -use serde::{Deserialize, Serialize}; -use thiserror::Error; -use tokio::{ - fs::{self, OpenOptions}, - io::{self, AsyncWriteExt}, -}; -use tracing::error; -use uuid::Uuid; - -use super::LocationPubId; - -static SPACEDRIVE_LOCATION_METADATA_FILE: &str = ".spacedrive"; - -#[derive(Serialize, Deserialize, Default, Debug)] -struct LocationMetadata { - pub_id: LocationPubId, - name: String, - path: PathBuf, - created_at: DateTime, - updated_at: DateTime, -} - -#[derive(Serialize, Deserialize, Default, Debug)] -struct SpacedriveLocationMetadata { - libraries: HashMap, - created_at: DateTime, - updated_at: DateTime, -} - -pub struct SpacedriveLocationMetadataFile { - path: PathBuf, - metadata: SpacedriveLocationMetadata, -} - -impl SpacedriveLocationMetadataFile { - pub async fn try_load( - location_path: impl AsRef, - ) -> Result, LocationMetadataError> { - let metadata_file_name = location_path - .as_ref() - .join(SPACEDRIVE_LOCATION_METADATA_FILE); - - match fs::read(&metadata_file_name).await { - Ok(data) => Ok(Some(Self { - metadata: match serde_json::from_slice(&data) { - Ok(data) => data, - Err(e) => { - #[cfg(debug_assertions)] - { - error!( - metadata_file_name = %metadata_file_name.display(), - ?e, - "Failed to deserialize corrupted metadata file, \ - we will remove it and create a new one;", - ); - - fs::remove_file(&metadata_file_name).await.map_err(|e| { - LocationMetadataError::Delete( - e, - location_path.as_ref().to_path_buf(), - ) - })?; - - return Ok(None); - } - - #[cfg(not(debug_assertions))] - return Err(LocationMetadataError::Deserialize( - e, - location_path.as_ref().to_path_buf(), - )); - } - }, - path: metadata_file_name, - })), - Err(e) if e.kind() == io::ErrorKind::NotFound => Ok(None), - Err(e) => Err(LocationMetadataError::Read( - e, - location_path.as_ref().to_path_buf(), - )), - } - } - - pub async fn create_and_save( - library_id: LibraryId, - location_pub_id: Uuid, - location_path: impl AsRef, - location_name: String, - ) -> Result<(), LocationMetadataError> { - Self { - path: location_path - .as_ref() - .join(SPACEDRIVE_LOCATION_METADATA_FILE), - metadata: SpacedriveLocationMetadata { - libraries: [( - library_id, - LocationMetadata { - pub_id: location_pub_id, - name: location_name, - path: location_path.as_ref().to_path_buf(), - created_at: Utc::now(), - updated_at: Utc::now(), - }, - )] - .into_iter() - .collect(), - created_at: Utc::now(), - updated_at: Utc::now(), - }, - } - .write_metadata() - .await - } - - pub async fn relink( - &mut self, - library_id: LibraryId, - location_path: impl AsRef, - ) -> Result<(), LocationMetadataError> { - let location_metadata = self - .metadata - .libraries - .get_mut(&library_id) - .ok_or(LocationMetadataError::LibraryNotFound(library_id))?; - - let new_path = location_path.as_ref().to_path_buf(); - if location_metadata.path == new_path { - return Err(LocationMetadataError::RelinkSamePath(new_path)); - } - - location_metadata.path = new_path; - location_metadata.updated_at = Utc::now(); - self.path = location_path - .as_ref() - .join(SPACEDRIVE_LOCATION_METADATA_FILE); - - self.write_metadata().await - } - - pub async fn update( - &mut self, - library_id: LibraryId, - location_name: String, - ) -> Result<(), LocationMetadataError> { - let location_metadata = self - .metadata - .libraries - .get_mut(&library_id) - .ok_or(LocationMetadataError::LibraryNotFound(library_id))?; - - location_metadata.name = location_name; - location_metadata.updated_at = Utc::now(); - - self.write_metadata().await - } - - pub async fn add_library( - &mut self, - library_id: LibraryId, - location_pub_id: Uuid, - location_path: impl AsRef, - location_name: String, - ) -> Result<(), LocationMetadataError> { - self.metadata.libraries.insert( - library_id, - LocationMetadata { - pub_id: location_pub_id, - name: location_name, - path: location_path.as_ref().to_path_buf(), - created_at: Utc::now(), - updated_at: Utc::now(), - }, - ); - - self.metadata.updated_at = Utc::now(); - self.write_metadata().await - } - - pub fn has_library(&self, library_id: LibraryId) -> bool { - self.metadata.libraries.contains_key(&library_id) - } - - pub fn location_path(&self, library_id: LibraryId) -> Option<&Path> { - self.metadata - .libraries - .get(&library_id) - .map(|l| l.path.as_path()) - } - - pub fn is_empty(&self) -> bool { - self.metadata.libraries.is_empty() - } - - pub async fn remove_library( - &mut self, - library_id: LibraryId, - ) -> Result<(), LocationMetadataError> { - self.metadata - .libraries - .remove(&library_id) - .ok_or(LocationMetadataError::LibraryNotFound(library_id))?; - - self.metadata.updated_at = Utc::now(); - - if !self.metadata.libraries.is_empty() { - self.write_metadata().await - } else { - fs::remove_file(&self.path) - .await - .map_err(|e| LocationMetadataError::Delete(e, self.path.clone())) - } - } - - pub async fn clean_stale_libraries( - &mut self, - existing_libraries_ids: &HashSet, - ) -> Result<(), LocationMetadataError> { - let previous_libraries_count = self.metadata.libraries.len(); - self.metadata - .libraries - .retain(|library_id, _| existing_libraries_ids.contains(library_id)); - - if self.metadata.libraries.len() != previous_libraries_count { - self.metadata.updated_at = Utc::now(); - - if !self.metadata.libraries.is_empty() { - self.write_metadata().await - } else { - fs::remove_file(&self.path) - .await - .map_err(|e| LocationMetadataError::Delete(e, self.path.clone())) - } - } else { - Ok(()) - } - } - - pub fn location_pub_id(&self, library_id: LibraryId) -> Result { - self.metadata - .libraries - .get(&library_id) - .ok_or(LocationMetadataError::LibraryNotFound(library_id)) - .map(|m| m.pub_id) - } - - async fn write_metadata(&self) -> Result<(), LocationMetadataError> { - let mut file_options = OpenOptions::new(); - - // we want to write the file if it exists, otherwise create it - file_options.create(true).write(true); - - #[cfg(target_os = "windows")] - { - use windows::Win32::Storage::FileSystem::FILE_ATTRIBUTE_HIDDEN; - file_options.attributes(FILE_ATTRIBUTE_HIDDEN.0); - } - - let metadata_contents = serde_json::to_vec(&self.metadata) - .map_err(|e| LocationMetadataError::Serialize(e, self.path.clone()))?; - - file_options - .open(&self.path) - .await - .map_err(|e| LocationMetadataError::Write(e, self.path.clone()))? - .write_all(&metadata_contents) - .await - .map_err(|e| LocationMetadataError::Write(e, self.path.clone())) - } -} - -#[derive(Error, Debug)] -pub enum LocationMetadataError { - #[error("Library not found: {0}")] - LibraryNotFound(LibraryId), - #[error("Failed to read location metadata file (path: {1:?}); (error: {0:?})")] - Read(io::Error, PathBuf), - #[error("Failed to delete location metadata file (path: {1:?}); (error: {0:?})")] - Delete(io::Error, PathBuf), - #[error("Failed to serialize metadata file for location (at path: {1:?}); (error: {0:?})")] - Serialize(serde_json::Error, PathBuf), - #[error("Failed to write location metadata file (path: {1:?}); (error: {0:?})")] - Write(io::Error, PathBuf), - #[error("Failed to deserialize metadata file for location (at path: {1:?}); (error: {0:?})")] - Deserialize(serde_json::Error, PathBuf), - #[error("Failed to relink, as the new location path is the same as the old path: {0}")] - RelinkSamePath(PathBuf), -} diff --git a/core/src/location/mod.rs b/core/src/location/mod.rs index e89639285..983d53c72 100644 --- a/core/src/location/mod.rs +++ b/core/src/location/mod.rs @@ -1,1155 +1,563 @@ -use crate::{context::NodeContext, invalidate_query, library::Library, Node}; +//! Location management - simplified implementation matching core patterns -use sd_core_file_path_helper::{ - filter_existing_file_path_params, IsolatedFilePathData, IsolatedFilePathDataParts, -}; -use sd_core_heavy_lifting::{ - file_identifier::{self, FileIdentifier}, - indexer::{self, job::Indexer}, - job_system::report::ReportInputMetadata, - media_processor::{self, job::MediaProcessor}, - JobEnqueuer, JobId, -}; -use sd_core_prisma_helpers::{location_with_indexer_rules, CasId}; +pub mod manager; -use sd_prisma::{ - prisma::{device, file_path, indexer_rules_in_location, instance, location, PrismaClient}, - prisma_sync, -}; -use sd_sync::*; -use sd_utils::{ - db::{maybe_missing, size_in_bytes_from_db, size_in_bytes_to_db}, - error::{FileIOError, NonUtf8PathError}, - uuid_to_bytes, +use crate::{ + infrastructure::{ + database::entities::{self, entry::EntryKind}, + events::{Event, EventBus}, + jobs::{handle::JobHandle, output::IndexedOutput, types::JobStatus}, + }, + library::Library, + operations::indexing::{IndexMode as JobIndexMode, IndexerJob, IndexerJobConfig, PathResolver, rules::RuleToggles}, + domain::addressing::SdPath, }; -use std::{ - collections::HashSet, - path::{Component, Path, PathBuf}, - sync::Arc, -}; - -use chrono::Utc; -use futures::future::TryFutureExt; -use normpath::PathExt; -use prisma_client_rust::{operator::and, or, QueryError}; +use sea_orm::{ActiveModelTrait, ActiveValue::Set, ColumnTrait, EntityTrait, QueryFilter, TransactionTrait}; use serde::{Deserialize, Serialize}; -use specta::Type; -use tokio::{fs, io, time::Instant}; -use tracing::{debug, error, info, instrument, warn}; +use std::{path::PathBuf, sync::Arc}; +use tokio::fs; +use tracing::{error, info, warn}; use uuid::Uuid; -mod error; -mod manager; -pub mod metadata; -pub mod non_indexed; +pub use manager::LocationManager; -pub use error::LocationError; -pub use manager::{LocationManagerError, Locations}; -use metadata::SpacedriveLocationMetadataFile; - -pub type LocationPubId = Uuid; - -#[repr(i32)] -#[derive(Debug, Clone, Copy, Serialize, Deserialize, Type, Eq, PartialEq)] -pub enum ScanState { - Pending = 0, - Indexed = 1, - FilesIdentified = 2, - Completed = 3, -} - -impl TryFrom for ScanState { - type Error = LocationError; - - fn try_from(value: i32) -> Result { - Ok(match value { - 0 => Self::Pending, - 1 => Self::Indexed, - 2 => Self::FilesIdentified, - 3 => Self::Completed, - _ => return Err(LocationError::InvalidScanStateValue(value)), - }) - } -} - -/// `LocationCreateArgs` is the argument received from the client using `rspc` to create a new location. -/// It has the actual path and a vector of indexer rules ids, to create many-to-many relationships -/// between the location and indexer rules. -#[derive(Debug, Type, Deserialize)] +/// Location creation arguments (simplified from production version) +#[derive(Debug, Serialize, Deserialize)] pub struct LocationCreateArgs { pub path: PathBuf, - pub dry_run: bool, - pub indexer_rules_ids: Vec, + pub name: Option, + pub index_mode: IndexMode, } -impl LocationCreateArgs { - #[instrument(skip(node, library), err)] - pub async fn create( - self, - node: &Node, - library: &Arc, - ) -> Result, LocationError> { - let Some(path_str) = self.path.to_str().map(str::to_string) else { - return Err(LocationError::NonUtf8Path(NonUtf8PathError( - self.path.into_boxed_path(), - ))); - }; +/// Location indexing mode +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +pub enum IndexMode { + /// Only scan file/directory structure + Shallow, + /// Quick scan (metadata only) + Quick, + /// Include content hashing for deduplication + Content, + /// Full indexing with content analysis and metadata + Deep, + /// Full indexing with all features + Full, +} - let path_metadata = match fs::metadata(&self.path).await { - Ok(metadata) => metadata, - Err(e) if e.kind() == io::ErrorKind::NotFound => { - return Err(LocationError::PathNotFound(self.path.into_boxed_path())) - } - Err(e) => { - return Err(LocationError::LocationPathFilesystemMetadataAccess( - FileIOError::from((self.path, e)), - )); - } - }; - - if !path_metadata.is_dir() { - return Err(LocationError::NotDirectory(self.path.into_boxed_path())); - } - - if let Some(mut metadata) = SpacedriveLocationMetadataFile::try_load(&self.path).await? { - metadata - .clean_stale_libraries( - &node - .libraries - .get_all() - .await - .into_iter() - .map(|library| library.id) - .collect(), - ) - .await?; - - if !metadata.is_empty() { - if let Some(old_path) = metadata.location_path(library.id) { - if old_path == self.path { - if library - .db - .location() - .count(vec![location::path::equals(Some(path_str))]) - .exec() - .await? > 0 - { - // Location already exists in this library - return Err(LocationError::LocationAlreadyExists( - self.path.into_boxed_path(), - )); - } - } else { - return Err(LocationError::NeedRelink { - old_path: old_path.into(), - new_path: self.path.into_boxed_path(), - }); - } - } else { - return Err(LocationError::AddLibraryToMetadata( - self.path.into_boxed_path(), - )); - }; - } - } - - debug!( - "{} new location", - if self.dry_run { - "Dry run: Would create" - } else { - "Trying to create" - } - ); - - let uuid = Uuid::now_v7(); - - let location = create_location( - library, - uuid, - &self.path, - &self.indexer_rules_ids, - self.dry_run, - ) - .await?; - - if let Some(location) = location { - info!(location_name = ?location.name, "Created location;"); - - // Write location metadata to a .spacedrive file - if let Err(e) = SpacedriveLocationMetadataFile::create_and_save( - library.id, - uuid, - &self.path, - location.name, - ) - .err_into::() - .and_then(|()| async move { - node.locations - .add(location.data.id, library.clone()) - .await - .map_err(Into::into) - }) - .await - { - // DISABLED TO FAIL SILENTLY - HOTFIX FOR LACK OF WRITE PERMISSION PREVENTING LOCATION CREATION - error!(?e, "Failed to write .spacedrive file;"); - // delete_location(node, library, location.data.id).await?; - // Err(e)?; - } - - Ok(Some(location.data)) - } else { - Ok(None) +impl From for JobIndexMode { + fn from(mode: IndexMode) -> Self { + match mode { + IndexMode::Shallow => JobIndexMode::Shallow, + IndexMode::Quick => JobIndexMode::Content, + IndexMode::Content => JobIndexMode::Content, + IndexMode::Deep => JobIndexMode::Deep, + IndexMode::Full => JobIndexMode::Deep, } } +} - #[instrument(skip(node, library), fields(library_id = %library.id), err)] - pub async fn add_library( - self, - node: &Node, - library: &Arc, - ) -> Result, LocationError> { - let Some(mut metadata) = SpacedriveLocationMetadataFile::try_load(&self.path).await? else { - return Err(LocationError::MetadataNotFound(self.path.into_boxed_path())); - }; +impl From<&str> for IndexMode { + fn from(s: &str) -> Self { + match s.to_lowercase().as_str() { + "shallow" => IndexMode::Shallow, + "quick" => IndexMode::Quick, + "content" => IndexMode::Content, + "deep" => IndexMode::Deep, + "full" => IndexMode::Full, + _ => IndexMode::Full, + } + } +} - metadata - .clean_stale_libraries( - &node - .libraries - .get_all() - .await - .into_iter() - .map(|library| library.id) - .collect(), - ) - .await?; +impl std::fmt::Display for IndexMode { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + IndexMode::Shallow => write!(f, "shallow"), + IndexMode::Quick => write!(f, "quick"), + IndexMode::Content => write!(f, "content"), + IndexMode::Deep => write!(f, "deep"), + IndexMode::Full => write!(f, "full"), + } + } +} - if metadata.has_library(library.id) { - return Err(LocationError::NeedRelink { - old_path: metadata - .location_path(library.id) - .expect("We checked that we have this library_id") - .into(), - new_path: self.path.into_boxed_path(), +/// Managed location representation +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ManagedLocation { + pub id: Uuid, + pub name: String, + pub path: PathBuf, + pub device_id: i32, + pub library_id: Uuid, + pub indexing_enabled: bool, + pub index_mode: IndexMode, + pub watch_enabled: bool, +} + +/// Location management errors +#[derive(Debug, thiserror::Error)] +pub enum LocationError { + #[error("Database error: {0}")] + Database(#[from] sea_orm::DbErr), + #[error("Database error: {0}")] + DatabaseError(String), + #[error("Path does not exist: {path}")] + PathNotFound { path: PathBuf }, + #[error("Path not accessible: {path}")] + PathNotAccessible { path: PathBuf }, + #[error("Location already exists: {path}")] + LocationExists { path: PathBuf }, + #[error("Location not found: {id}")] + LocationNotFound { id: Uuid }, + #[error("IO error: {0}")] + Io(#[from] std::io::Error), + #[error("Invalid path: {0}")] + InvalidPath(String), + #[error("Job error: {0}")] + Job(#[from] crate::infrastructure::jobs::error::JobError), + #[error("Other error: {0}")] + Other(String), +} + +pub type LocationResult = Result; + +/// Create a new location and start indexing (production pattern) +pub async fn create_location( + library: Arc, + events: &EventBus, + args: LocationCreateArgs, + device_id: i32, +) -> LocationResult { + let path_str = args + .path + .to_str() + .ok_or_else(|| LocationError::InvalidPath("Non-UTF8 path".to_string()))?; + + // Validate path exists + if !args.path.exists() { + return Err(LocationError::PathNotFound { path: args.path }); + } + + if !args.path.is_dir() { + return Err(LocationError::InvalidPath( + "Path must be a directory".to_string(), + )); + } + + // Begin transaction to ensure atomicity + let txn = library.db().conn().begin().await + .map_err(|e| LocationError::DatabaseError(e.to_string()))?; + + // First, check if an entry already exists for this path + // We need to create a root entry for the location directory + let directory_name = args.path + .file_name() + .and_then(|n| n.to_str()) + .unwrap_or("Unknown") + .to_string(); + + // Create entry for the location directory + let entry_model = entities::entry::ActiveModel { + uuid: Set(Some(Uuid::new_v4())), + name: Set(directory_name.clone()), + kind: Set(EntryKind::Directory as i32), + extension: Set(None), + metadata_id: Set(None), + content_id: Set(None), + size: Set(0), + aggregate_size: Set(0), + child_count: Set(0), + file_count: Set(0), + created_at: Set(chrono::Utc::now()), + modified_at: Set(chrono::Utc::now()), + accessed_at: Set(None), + permissions: Set(None), + inode: Set(None), + parent_id: Set(None), // Location root has no parent + ..Default::default() + }; + + let entry_record = entry_model.insert(&txn).await + .map_err(|e| LocationError::DatabaseError(e.to_string()))?; + let entry_id = entry_record.id; + + // Add self-reference to closure table + let self_closure = entities::entry_closure::ActiveModel { + ancestor_id: Set(entry_id), + descendant_id: Set(entry_id), + depth: Set(0), + ..Default::default() + }; + self_closure.insert(&txn).await + .map_err(|e| LocationError::DatabaseError(e.to_string()))?; + + // Add to directory_paths table + let dir_path_entry = entities::directory_paths::ActiveModel { + entry_id: Set(entry_id), + path: Set(path_str.to_string()), + ..Default::default() + }; + dir_path_entry.insert(&txn).await + .map_err(|e| LocationError::DatabaseError(e.to_string()))?; + + // Check if a location already exists for this entry + let existing = entities::location::Entity::find() + .filter(entities::location::Column::EntryId.eq(entry_id)) + .one(&txn) + .await + .map_err(|e| LocationError::DatabaseError(e.to_string()))?; + + if existing.is_some() { + // Rollback transaction + txn.rollback().await + .map_err(|e| LocationError::DatabaseError(e.to_string()))?; + return Err(LocationError::LocationExists { path: args.path }); + } + + // Create location record + let location_id = Uuid::new_v4(); + let name = args.name.unwrap_or_else(|| { + args.path + .file_name() + .and_then(|n| n.to_str()) + .unwrap_or("Unknown") + .to_string() + }); + + let location_model = entities::location::ActiveModel { + id: Set(0), // Auto-increment + uuid: Set(location_id), + device_id: Set(device_id), + entry_id: Set(entry_id), + name: Set(Some(name.clone())), + index_mode: Set(args.index_mode.to_string()), + scan_state: Set("pending".to_string()), + last_scan_at: Set(None), + error_message: Set(None), + total_file_count: Set(0), + total_byte_size: Set(0), + created_at: Set(chrono::Utc::now()), + updated_at: Set(chrono::Utc::now()), + }; + + let location_record = location_model.insert(&txn).await + .map_err(|e| LocationError::DatabaseError(e.to_string()))?; + let location_db_id = location_record.id; + + // Commit transaction + txn.commit().await + .map_err(|e| LocationError::DatabaseError(e.to_string()))?; + + info!("Created location '{}' with ID: {}", name, location_db_id); + + // Emit location added event + events.emit(Event::LocationAdded { + library_id: library.id(), + location_id, + path: args.path.clone(), + }); + + // Start indexing (simplified - in production this goes through proper job manager) + start_location_indexing( + library.clone(), + events, + location_db_id, + location_id, + args.path, + args.index_mode, + ) + .await?; + + Ok(location_db_id) +} + +/// Start indexing for a location (production implementation) +async fn start_location_indexing( + library: Arc, + events: &EventBus, + location_db_id: i32, + location_uuid: Uuid, + path: PathBuf, + index_mode: IndexMode, +) -> LocationResult<()> { + info!("Starting indexing for location: {}", path.display()); + + // Update scan state to "running" + update_location_scan_state(library.clone(), location_db_id, "running", None).await?; + + // Emit indexing started event + events.emit(Event::IndexingStarted { + location_id: location_uuid, + }); + + // Get device UUID for SdPath + let device_uuid = get_device_uuid(library.clone()).await?; + let location_sd_path = SdPath::new(device_uuid, path.clone()); + + // Create and dispatch indexer job through the proper job manager + let lib_cfg = library.config().await; + let idx_cfg = lib_cfg.settings.indexer; + let mut config = IndexerJobConfig::new(location_uuid, location_sd_path, index_mode.into()); + config.rule_toggles = RuleToggles { + no_system_files: idx_cfg.no_system_files, + no_hidden: idx_cfg.no_hidden, + no_git: idx_cfg.no_git, + gitignore: idx_cfg.gitignore, + only_images: idx_cfg.only_images, + no_dev_dirs: idx_cfg.no_dev_dirs, + }; + let indexer_job = IndexerJob::new(config); + + match library.jobs().dispatch(indexer_job).await { + Ok(job_handle) => { + info!( + "Successfully dispatched indexer job {} for location: {}", + job_handle.id(), + path.display() + ); + + // Monitor job progress asynchronously + let events_clone = events.clone(); + let library_clone = library.clone(); + let handle_clone = job_handle.clone(); + + tokio::spawn(async move { + monitor_indexing_job( + handle_clone, + events_clone, + library_clone, + location_db_id, + location_uuid, + path, + ) + .await; }); } + Err(e) => { + error!( + "Failed to dispatch indexer job for {}: {}", + path.display(), + e + ); - debug!( - "{} a new Library to an already existing location", - if self.dry_run { - "Dry run: Would add" + // Update scan state to failed + if let Err(update_err) = update_location_scan_state( + library.clone(), + location_db_id, + "failed", + Some(e.to_string()), + ) + .await + { + error!("Failed to update scan state: {}", update_err); + } + + events.emit(Event::IndexingFailed { + location_id: location_uuid, + error: e.to_string(), + }); + + return Err(LocationError::Other(format!( + "Failed to start indexing: {}", + e + ))); + } + } + + Ok(()) +} + +/// Monitor indexing job progress and update location state accordingly +async fn monitor_indexing_job( + job_handle: JobHandle, + events: EventBus, + library: Arc, + location_db_id: i32, + location_uuid: Uuid, + path: PathBuf, +) { + info!( + "Monitoring indexer job {} for location: {}", + job_handle.id(), + path.display() + ); + + // Wait for job completion + let job_result = job_handle.wait().await; + + match job_result { + Ok(output) => { + info!( + "Indexing completed successfully for location: {}", + path.display() + ); + + // Parse output to get statistics + if let Some(indexer_output) = output.as_indexed() { + // Update location stats + if let Err(e) = update_location_stats( + library.clone(), + location_db_id, + indexer_output.total_files, + indexer_output.total_bytes, + ) + .await + { + error!("Failed to update location stats: {}", e); + } + + // Update scan state to completed + if let Err(e) = + update_location_scan_state(library.clone(), location_db_id, "completed", None) + .await + { + error!("Failed to update scan state: {}", e); + } + + // Emit completion events + events.emit(Event::IndexingCompleted { + location_id: location_uuid, + total_files: indexer_output.total_files, + total_dirs: indexer_output.total_dirs, + }); + + events.emit(Event::FilesIndexed { + library_id: library.id(), + location_id: location_uuid, + count: indexer_output.total_files as usize, + }); + + info!( + "Location indexing completed: {} ({} files, {} dirs, {} bytes)", + path.display(), + indexer_output.total_files, + indexer_output.total_dirs, + indexer_output.total_bytes + ); } else { - "Trying to add" - }, - ); + warn!("Job completed but output format was unexpected"); - let uuid = Uuid::now_v7(); + // Update scan state to completed anyway + if let Err(e) = + update_location_scan_state(library.clone(), location_db_id, "completed", None) + .await + { + error!("Failed to update scan state: {}", e); + } + } + } + Err(e) => { + error!("Indexing failed for {}: {}", path.display(), e); - let location = create_location( - library, - uuid, - &self.path, - &self.indexer_rules_ids, - self.dry_run, - ) - .await?; + // Update scan state to failed + if let Err(update_err) = update_location_scan_state( + library.clone(), + location_db_id, + "failed", + Some(e.to_string()), + ) + .await + { + error!("Failed to update scan state: {}", update_err); + } - if let Some(location) = location { - metadata - .add_library(library.id, uuid, &self.path, location.name) - .await?; - - node.locations - .add(location.data.id, library.clone()) - .await?; - - info!(location_id = %location.data.id, "Added library to location;"); - - Ok(Some(location.data)) - } else { - Ok(None) + events.emit(Event::IndexingFailed { + location_id: location_uuid, + error: e.to_string(), + }); } } } -/// `LocationUpdateArgs` is the argument received from the client using `rspc` to update a location. -/// It contains the id of the location to be updated, possible a name to change the current location's name -/// and a vector of indexer rules ids to add or remove from the location. -/// -/// It is important to note that only the indexer rule ids in this vector will be used from now on. -/// Old rules that aren't in this vector will be purged. -#[derive(Type, Deserialize)] -pub struct LocationUpdateArgs { - id: location::id::Type, - name: Option, - generate_preview_media: Option, - sync_preview_media: Option, - hidden: Option, - indexer_rules_ids: Vec, - path: Option, -} +/// Scan directory to get basic stats +async fn scan_directory_stats(path: &PathBuf) -> Result<(u64, u64), std::io::Error> { + let mut file_count = 0u64; + let mut total_size = 0u64; -impl LocationUpdateArgs { - pub async fn update(self, node: &Node, library: &Arc) -> Result<(), LocationError> { - let Library { sync, db, .. } = &**library; + let mut stack = vec![path.clone()]; - let location = find_location(library, self.id) - .include(location_with_indexer_rules::include()) - .exec() - .await? - .ok_or(LocationError::IdNotFound(self.id))?; - - let name = self.name.clone(); - - let (sync_params, db_params) = [ - option_sync_db_entry!( - self.name - .filter(|name| location.name.as_ref() != Some(name)), - location::name - ), - option_sync_db_entry!( - self.generate_preview_media, - location::generate_preview_media - ), - option_sync_db_entry!(self.sync_preview_media, location::sync_preview_media), - option_sync_db_entry!(self.hidden, location::hidden), - option_sync_db_entry!(self.path.clone(), location::path), - ] - .into_iter() - .flatten() - .unzip::<_, _, Vec<_>, Vec<_>>(); - - if !sync_params.is_empty() { - sync.write_op( - db, - sync.shared_update( - prisma_sync::location::SyncId { - pub_id: location.pub_id.clone(), - }, - sync_params, - ), - db.location() - .update(location::id::equals(self.id), db_params) - .select(location::select!({ id })), - ) - .await?; - - // TODO(N): This will probs fall apart with removable media. - if location.instance_id == Some(library.config().await.instance_id) { - if let Some(path) = &location.path { - if let Some(mut metadata) = - SpacedriveLocationMetadataFile::try_load(path).await? - { - metadata - .update(library.id, maybe_missing(name, "location.name")?) - .await?; + while let Some(current_path) = stack.pop() { + if let Ok(mut entries) = fs::read_dir(¤t_path).await { + while let Ok(Some(entry)) = entries.next_entry().await { + if let Ok(metadata) = entry.metadata().await { + if metadata.is_file() { + file_count += 1; + total_size += metadata.len(); + } else if metadata.is_dir() { + stack.push(entry.path()); } } } - - if self.path.is_some() { - node.locations.remove(self.id, library.clone()).await?; - node.locations.add(self.id, library.clone()).await?; - } } - - let current_rules_ids = location - .indexer_rules - .iter() - .map(|r| r.indexer_rule.id) - .collect::>(); - - let new_rules_ids = self.indexer_rules_ids.into_iter().collect::>(); - - if current_rules_ids != new_rules_ids { - let rule_ids_to_add = new_rules_ids - .difference(¤t_rules_ids) - .copied() - .collect::>(); - let rule_ids_to_remove = current_rules_ids - .difference(&new_rules_ids) - .copied() - .collect::>(); - - if !rule_ids_to_remove.is_empty() { - library - .db - .indexer_rules_in_location() - .delete_many(vec![ - indexer_rules_in_location::location_id::equals(self.id), - indexer_rules_in_location::indexer_rule_id::in_vec(rule_ids_to_remove), - ]) - .exec() - .await?; - } - - if !rule_ids_to_add.is_empty() { - link_location_and_indexer_rules(library, self.id, &rule_ids_to_add).await?; - } - } - - Ok(()) - } -} - -pub fn find_location( - library: &Library, - location_id: location::id::Type, -) -> location::FindUniqueQuery { - library - .db - .location() - .find_unique(location::id::equals(location_id)) -} - -async fn link_location_and_indexer_rules( - library: &Library, - location_id: location::id::Type, - rules_ids: &[i32], -) -> Result<(), LocationError> { - library - .db - .indexer_rules_in_location() - .create_many( - rules_ids - .iter() - .map(|id| indexer_rules_in_location::create_unchecked(location_id, *id, vec![])) - .collect(), - ) - .exec() - .await?; - - Ok(()) -} - -#[instrument( - skip(node, library, location), - fields(library_id = %library.id, location_id = %location.id), - err, -)] -pub async fn scan_location( - node: &Arc, - library: &Arc, - location: location_with_indexer_rules::Data, - location_scan_state: ScanState, -) -> Result, sd_core_heavy_lifting::Error> { - // TODO(N): This isn't gonna work with removable media and this will likely permanently break if the DB is restored from a backup. - if location.instance_id != Some(library.config().await.instance_id) { - warn!("Tried to scan a location on a different instance"); - return Ok(None); } - let location_id = location.id; - let ctx = NodeContext { - node: Arc::clone(node), - library: Arc::clone(library), - }; - - let location_base_data = location::Data::from(&location); - - debug!("Scanning location"); - - let job_id = match location_scan_state { - ScanState::Pending | ScanState::Completed => { - node.job_system - .dispatch( - JobEnqueuer::new(Indexer::new(location, None)?) - .with_action("scan_location") - .with_metadata(ReportInputMetadata::Location(location_base_data.clone())) - .enqueue_next(FileIdentifier::new(location_base_data.clone(), None)?) - .enqueue_next(MediaProcessor::new(location_base_data, None, false)?), - location_id, - ctx.clone(), - ) - .await? - } - - ScanState::Indexed => { - node.job_system - .dispatch( - JobEnqueuer::new(FileIdentifier::new(location_base_data.clone(), None)?) - .with_action("scan_location_already_indexed") - .with_metadata(ReportInputMetadata::Location(location_base_data.clone())) - .enqueue_next(MediaProcessor::new(location_base_data, None, false)?), - location_id, - ctx.clone(), - ) - .await? - } - - ScanState::FilesIdentified => { - node.job_system - .dispatch( - JobEnqueuer::new(MediaProcessor::new( - location_base_data.clone(), - None, - false, - )?) - .with_action("scan_location_files_already_identified") - .with_metadata(ReportInputMetadata::Location(location_base_data)), - location_id, - ctx.clone(), - ) - .await? - } - }; - - Ok(Some(job_id)) + Ok((file_count, total_size)) } -#[instrument( - skip_all, - fields( - library_id = %library.id, - location_id = %location.id, - sub_path = %sub_path.as_ref().display(), - ), - err, -)] -pub async fn scan_location_sub_path( - node: &Arc, - library: &Arc, - location: location_with_indexer_rules::Data, - sub_path: impl AsRef + Send, -) -> Result, sd_core_heavy_lifting::Error> { - let sub_path = sub_path.as_ref().to_path_buf(); - - // TODO(N): This isn't gonna work with removable media and this will likely permanently break if the DB is restored from a backup. - if location.instance_id != Some(library.config().await.instance_id) { - warn!("Tried to scan a location on a different instance"); - return Ok(None); - } - - let location_id = location.id; - let ctx = NodeContext { - node: Arc::clone(node), - library: Arc::clone(library), - }; - - let location_base_data = location::Data::from(&location); - - debug!("Scanning location on a sub path"); - - node.job_system - .dispatch( - JobEnqueuer::new(Indexer::new(location, Some(sub_path.clone()))?) - .with_action("scan_location") - .with_metadata(ReportInputMetadata::Location(location_base_data.clone())) - .with_metadata(ReportInputMetadata::SubPath(sub_path.clone())) - .enqueue_next(FileIdentifier::new( - location_base_data.clone(), - Some(sub_path.clone()), - )?) - .enqueue_next(MediaProcessor::new( - location_base_data, - Some(sub_path), - false, - )?), - location_id, - ctx.clone(), - ) - .await - .map_err(Into::into) - .map(Some) -} - -#[instrument( - skip_all, - fields( - library_id = %library.id, - location_id = %location.id, - sub_path = %sub_path.as_ref().display(), - ), - err, -)] -pub async fn light_scan_location( - node: Arc, +/// Update location scan state +async fn update_location_scan_state( library: Arc, - location: location_with_indexer_rules::Data, - sub_path: impl AsRef, -) -> Result<(), sd_core_heavy_lifting::Error> { - let sub_path = sub_path.as_ref().to_path_buf(); + location_id: i32, + state: &str, + error_message: Option, +) -> LocationResult<()> { + let location = entities::location::Entity::find_by_id(location_id) + .one(library.db().conn()) + .await? + .ok_or_else(|| LocationError::LocationNotFound { id: Uuid::nil() })?; - // TODO(N): This isn't gonna work with removable media and this will likely permanently break if the DB is restored from a backup. - if location.instance_id != Some(library.config().await.instance_id) { - warn!("Tried to scan a location on a different instance"); - return Ok(()); - } - - let location_base_data = location::Data::from(&location); - - let dispatcher = node.task_system.get_dispatcher(); - let ctx = NodeContext { node, library }; - - for e in indexer::shallow(location, &sub_path, &dispatcher, &ctx).await? { - error!(?e, "Shallow indexer errors;"); - } - - for e in - file_identifier::shallow(location_base_data.clone(), &sub_path, &dispatcher, &ctx).await? - { - error!(?e, "Shallow file identifier errors;"); - } - - for e in media_processor::shallow(location_base_data, &sub_path, &dispatcher, &ctx).await? { - error!(?e, "Shallow media processor errors;"); + let mut active_location: entities::location::ActiveModel = location.into(); + active_location.scan_state = Set(state.to_string()); + active_location.error_message = Set(error_message); + active_location.updated_at = Set(chrono::Utc::now()); + + if state == "running" { + active_location.last_scan_at = Set(Some(chrono::Utc::now())); } + active_location.update(library.db().conn()).await?; Ok(()) } -#[instrument( - skip_all, - fields( - library_id = %id, - location_path = %location_path.as_ref().display(), - ), - err, -)] -pub async fn relink_location( - Library { db, id, sync, .. }: &Library, - location_path: impl AsRef, -) -> Result { - let location_path = location_path.as_ref(); - let mut metadata = SpacedriveLocationMetadataFile::try_load(&location_path) +/// Update location statistics +async fn update_location_stats( + library: Arc, + location_id: i32, + file_count: u64, + total_size: u64, +) -> LocationResult<()> { + let location = entities::location::Entity::find_by_id(location_id) + .one(library.db().conn()) .await? - .ok_or_else(|| LocationError::MissingMetadataFile(location_path.into()))?; + .ok_or_else(|| LocationError::LocationNotFound { id: Uuid::nil() })?; - metadata.relink(*id, location_path).await?; - - let pub_id = uuid_to_bytes(&metadata.location_pub_id(*id)?); - let path = location_path - .to_str() - .map(str::to_string) - .ok_or_else(|| NonUtf8PathError(location_path.into()))?; - - let (sync_param, db_param) = sync_db_entry!(path, location::path); - - let location_id = sync - .write_op( - db, - sync.shared_update( - prisma_sync::location::SyncId { - pub_id: pub_id.clone(), - }, - [sync_param], - ), - db.location() - .update(location::pub_id::equals(pub_id.clone()), vec![db_param]) - .select(location::select!({ id })), - ) - .await? - .id; - - Ok(location_id) -} - -#[derive(Debug)] -pub struct CreatedLocationResult { - pub name: String, - pub data: location_with_indexer_rules::Data, -} - -pub(crate) fn normalize_path(path: impl AsRef) -> io::Result<(String, String)> { - let mut path = path.as_ref().to_path_buf(); - let (location_path, normalized_path) = path - // Normalize path and also check if it exists - .normalize() - .and_then(|normalized_path| { - if cfg!(windows) { - // Use normalized path as main path on Windows - // This ensures we always receive a valid windows formatted path - // ex: /Users/JohnDoe/Downloads will become C:\Users\JohnDoe\Downloads - // Internally `normalize` calls `GetFullPathNameW` on Windows - // https://learn.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-getfullpathnamew - path = normalized_path.as_path().to_path_buf(); - } - - Ok(( - // TODO: Maybe save the path bytes instead of the string representation to avoid depending on UTF-8 - path.to_str().map(str::to_string).ok_or(io::Error::new( - io::ErrorKind::InvalidInput, - "Found non-UTF-8 path", - ))?, - normalized_path, - )) - })?; - - // Not needed on Windows because the normalization already handles it - if cfg!(not(windows)) { - // Replace location_path with normalize_path, when the first one ends in `.` or `..` - // This is required so localize_name doesn't panic - if let Some(component) = path.components().next_back() { - if matches!(component, Component::CurDir | Component::ParentDir) { - path = normalized_path.as_path().to_path_buf(); - } - } - } - - // Use `to_string_lossy` because a partially corrupted but identifiable name is better than nothing - let mut name = path.localize_name().to_string_lossy().to_string(); - - // Windows doesn't have a root directory - if cfg!(not(windows)) && name == "/" { - name = "Root".to_string() - } - - if name.replace(char::REPLACEMENT_CHARACTER, "") == "" { - name = "Unknown".to_string() - } - - Ok((location_path, name)) -} - -async fn create_location( - library @ Library { db, sync, .. }: &Library, - location_pub_id: Uuid, - location_path: impl AsRef, - indexer_rules_ids: &[i32], - dry_run: bool, -) -> Result, LocationError> { - let location_path = location_path.as_ref(); - let (path, name) = normalize_path(location_path) - .map_err(|_| LocationError::DirectoryNotFound(location_path.into()))?; - - if db - .location() - .count(vec![location::path::equals(Some(path.clone()))]) - .exec() - .await? - > 0 - { - return Err(LocationError::LocationAlreadyExists(location_path.into())); - } - - if check_nested_location(&location_path, db).await? { - return Err(LocationError::NestedLocation(location_path.into())); - } - - if dry_run { - return Ok(None); - } - - let (sync_values, mut db_params) = [ - sync_db_entry!(&name, location::name), - sync_db_entry!(path, location::path), - sync_db_entry!(Utc::now(), location::date_created), - ( - sync_entry!( - prisma_sync::device::SyncId { - pub_id: sync.device_pub_id.to_db() - }, - location::device - ), - location::device::connect(device::pub_id::equals(sync.device_pub_id.to_db())), - ), - ] - .into_iter() - .unzip::<_, _, Vec<_>, Vec<_>>(); - - // temporary workaround until we remove instances from locations - db_params.push(location::instance::connect(instance::id::equals( - library.config().await.instance_id, - ))); - - let location_id = sync - .write_op( - db, - sync.shared_create( - prisma_sync::location::SyncId { - pub_id: uuid_to_bytes(&location_pub_id), - }, - sync_values, - ), - db.location() - .create(uuid_to_bytes(&location_pub_id), db_params) - .select(location::select!({ id })), - ) - .await? - .id; - - debug!("New location created in db"); - - if !indexer_rules_ids.is_empty() { - link_location_and_indexer_rules(library, location_id, indexer_rules_ids).await?; - } - - // Updating our location variable to include information about the indexer rules - let location = find_location(library, location_id) - .include(location_with_indexer_rules::include()) - .exec() - .await? - .ok_or(LocationError::IdNotFound(location_id))?; - - invalidate_query!(library, "locations.list"); - - Ok(Some(CreatedLocationResult { - data: location, - name, - })) -} - -#[instrument(skip(node, library), fields(library_id = %library.id), err)] -pub async fn delete_location( - node: &Node, - library: &Arc, - location_id: location::id::Type, -) -> Result<(), LocationError> { - let Library { db, sync, .. } = library.as_ref(); - - let start = Instant::now(); - node.locations.remove(location_id, library.clone()).await?; - debug!(elapsed_time = ?start.elapsed(), "Removed location from node;"); - - let start = Instant::now(); - delete_directory(library, location_id, None).await?; - debug!(elapsed_time = ?start.elapsed(), "Deleted location file paths;"); - - let location = library - .db - .location() - .find_unique(location::id::equals(location_id)) - .exec() - .await? - .ok_or(LocationError::IdNotFound(location_id))?; - - let start = Instant::now(); - // TODO: This should really be queued to the proper node so it will always run - // TODO: Deal with whether a location is online or not - // TODO(N): This isn't gonna work with removable media and this will likely permanently break if the DB is restored from a backup. - if location.instance_id == Some(library.config().await.instance_id) { - if let Some(path) = &location.path { - if let Ok(Some(mut metadata)) = SpacedriveLocationMetadataFile::try_load(path).await { - metadata - .clean_stale_libraries( - &node - .libraries - .get_all() - .await - .into_iter() - .map(|library| library.id) - .collect(), - ) - .await?; - - metadata.remove_library(library.id).await?; - } - } - } - debug!(elapsed_time = ?start.elapsed(), "Removed location metadata;"); - - let start = Instant::now(); - - library - .db - .indexer_rules_in_location() - .delete_many(vec![indexer_rules_in_location::location_id::equals( - location_id, - )]) - .exec() - .await?; - debug!(elapsed_time = ?start.elapsed(), "Deleted indexer rules in location;"); - - let start = Instant::now(); - - sync.write_op( - db, - sync.shared_delete(prisma_sync::location::SyncId { - pub_id: location.pub_id, - }), - db.location().delete(location::id::equals(location_id)), - ) - .await?; - - debug!(elapsed_time = ?start.elapsed(), "Deleted location from db;"); - - invalidate_query!(library, "locations.list"); - - info!("Location deleted"); + let mut active_location: entities::location::ActiveModel = location.into(); + active_location.total_file_count = Set(file_count as i64); + active_location.total_byte_size = Set(total_size as i64); + active_location.updated_at = Set(chrono::Utc::now()); + active_location.update(library.db().conn()).await?; Ok(()) } -/// Will delete a directory recursively with Objects if left as orphans -/// this function is used to delete a location and when ingesting directory deletion events -#[instrument(skip_all, err)] -pub async fn delete_directory( - library: &Library, - location_id: location::id::Type, - parent_iso_file_path: Option<&IsolatedFilePathData<'_>>, -) -> Result<(), sd_core_sync::Error> { - let Library { db, .. } = library; +/// Get device UUID for current device +async fn get_device_uuid(_library: Arc) -> LocationResult { + // Get the current device ID from the global state + let device_uuid = crate::shared::utils::get_current_device_id(); - let children_params = sd_utils::chain_optional_iter( - [file_path::location_id::equals(Some(location_id))], - [parent_iso_file_path.and_then(|parent| { - parent - .materialized_path_for_children() - .map(|materialized_path| { - or![ - and(filter_existing_file_path_params(parent)), - file_path::materialized_path::starts_with(materialized_path), - ] - }) - })], - ); - - let pub_ids = library - .db - .file_path() - .find_many(children_params.clone()) - .select(file_path::select!({ pub_id })) - .exec() - .await? - .into_iter() - .map(|fp| fp.pub_id) - .collect::>(); - - if pub_ids.is_empty() { - debug!("No file paths to delete"); - return Ok(()); + if device_uuid.is_nil() { + return Err(LocationError::InvalidPath("Current device ID not initialized".to_string())); } - library - .sync - .write_ops( - &library.db, - ( - pub_ids - .into_iter() - .map(|pub_id| { - library - .sync - .shared_delete(prisma_sync::file_path::SyncId { pub_id }) - }) - .collect(), - db.file_path().delete_many(children_params), - ), - ) - .await?; - - // library.orphan_remover.invoke().await; - - invalidate_query!(library, "search.paths"); - invalidate_query!(library, "search.objects"); - - Ok(()) + Ok(device_uuid) } -#[instrument(skip_all, err)] -async fn check_nested_location( - location_path: impl AsRef, - db: &PrismaClient, -) -> Result { - let location_path = location_path.as_ref(); - - let (parents_count, potential_children) = db - ._batch(( - db.location().count(vec![location::path::in_vec( - location_path - .ancestors() - .skip(1) // skip the actual location_path, we only want the parents - .map(|p| { - p.to_str() - .map(str::to_string) - .expect("Found non-UTF-8 path") - }) - .collect(), - )]), - db.location().find_many(vec![location::path::starts_with( - location_path - .to_str() - .map(str::to_string) - .expect("Found non-UTF-8 path"), - )]), - )) - .await?; - - let comps = location_path.components().collect::>(); - let is_a_child_location = potential_children.into_iter().any(|v| { - let Some(location_path) = v.path else { - warn!( - location_id = %v.id, - "Missing location path on location at check nested location", - ); - return false; - }; - let comps2 = PathBuf::from(location_path); - let comps2 = comps2.components().collect::>(); - - if comps.len() > comps2.len() { - return false; - } - - for (a, b) in comps.iter().zip(comps2.iter()) { - if a != b { - return false; - } - } - - true - }); - - Ok(parents_count > 0 || is_a_child_location) -} - -#[instrument(skip_all, err)] -pub async fn update_location_size( - location_id: location::id::Type, - location_pub_id: location::pub_id::Type, - library: &Library, -) -> Result<(), sd_core_sync::Error> { - let Library { db, sync, .. } = library; - - let total_size = size_in_bytes_to_db( - db.file_path() - .find_many(vec![ - file_path::location_id::equals(Some(location_id)), - file_path::materialized_path::equals(Some("/".to_string())), - ]) - .select(file_path::select!({ size_in_bytes_bytes })) - .exec() - .await? - .into_iter() - .filter_map(|file_path| { - file_path - .size_in_bytes_bytes - .map(|size_in_bytes_bytes| size_in_bytes_from_db(&size_in_bytes_bytes)) - }) - .sum::(), - ); - - let (sync_param, db_param) = sync_db_entry!(total_size, location::size_in_bytes); - - sync.write_op( - db, - sync.shared_update( - prisma_sync::location::SyncId { - pub_id: location_pub_id, - }, - [sync_param], - ), - db.location() - .update(location::id::equals(location_id), vec![db_param]) - .select(location::select!({ id })), - ) - .await?; - - invalidate_query!(library, "locations.list"); - invalidate_query!(library, "locations.get"); - - Ok(()) -} - -#[instrument(skip_all, err)] -pub async fn get_location_path_from_location_id( - db: &PrismaClient, - location_id: file_path::id::Type, -) -> Result { - db.location() - .find_unique(location::id::equals(location_id)) - .exec() - .await - .map_err(Into::into) - .and_then(|maybe_location| { - maybe_location - .ok_or(LocationError::IdNotFound(location_id)) - .and_then(|location| { - location - .path - .map(PathBuf::from) - .ok_or(LocationError::MissingPath(location_id)) - }) - }) -} - -#[instrument(skip_all, err)] -pub async fn create_file_path( - crate::location::Library { db, sync, .. }: &crate::location::Library, - IsolatedFilePathDataParts { - materialized_path, - is_dir, - location_id, - name, - extension, - .. - }: IsolatedFilePathDataParts<'_>, - cas_id: Option>, - metadata: sd_core_file_path_helper::FilePathMetadata, -) -> Result { - use sd_utils::db::inode_to_db; - - use sd_prisma::prisma; - - let indexed_at = Utc::now(); - - let location = db - .location() - .find_unique(location::id::equals(location_id)) - .select(location::select!({ id pub_id })) - .exec() - .await? - .ok_or(sd_core_file_path_helper::FilePathError::LocationNotFound( - location_id, - ))?; - - let device_pub_id = sync.device_pub_id.to_db(); - - let (sync_params, db_params) = [ - ( - sync_entry!( - prisma_sync::location::SyncId { - pub_id: location.pub_id - }, - file_path::location - ), - file_path::location::connect(prisma::location::id::equals(location.id)), - ), - ( - sync_entry!(cas_id, file_path::cas_id), - file_path::cas_id::set(cas_id.map(Into::into)), - ), - sync_db_entry!(materialized_path, file_path::materialized_path), - sync_db_entry!(name, file_path::name), - sync_db_entry!(extension, file_path::extension), - sync_db_entry!( - size_in_bytes_to_db(metadata.size_in_bytes), - file_path::size_in_bytes_bytes - ), - sync_db_entry!(inode_to_db(metadata.inode), file_path::inode), - sync_db_entry!(is_dir, file_path::is_dir), - sync_db_entry!(metadata.created_at, file_path::date_created), - sync_db_entry!(metadata.modified_at, file_path::date_modified), - sync_db_entry!(indexed_at, file_path::date_indexed), - sync_db_entry!(metadata.hidden, file_path::hidden), - ( - sync_entry!( - prisma_sync::device::SyncId { - pub_id: device_pub_id.clone() - }, - file_path::device - ), - file_path::device::connect(prisma::device::pub_id::equals(device_pub_id)), - ), - ] - .into_iter() - .unzip::<_, _, Vec<_>, Vec<_>>(); - - let pub_id = sd_utils::uuid_to_bytes(&Uuid::now_v7()); - - sync.write_op( - db, - sync.shared_create( - prisma_sync::file_path::SyncId { - pub_id: pub_id.clone(), - }, - sync_params, - ), - db.file_path().create(pub_id, db_params), - ) - .await - .map_err(Into::into) +/// List all locations for a library +pub async fn list_locations( + library: Arc, +) -> LocationResult> { + Ok(entities::location::Entity::find() + .all(library.db().conn()) + .await?) } diff --git a/core/src/location/non_indexed.rs b/core/src/location/non_indexed.rs deleted file mode 100644 index 5ad43989e..000000000 --- a/core/src/location/non_indexed.rs +++ /dev/null @@ -1,414 +0,0 @@ -use crate::{api::locations::ExplorerItem, context::NodeContext, library::Library, Node}; - -use sd_core_file_path_helper::{path_is_hidden, MetadataExt}; -use sd_core_heavy_lifting::{ - file_identifier::generate_cas_id, - media_processor::{ - self, get_thumbnails_directory, thumbnailer::NewThumbnailReporter, GenerateThumbnailArgs, - NewThumbnailsReporter, ThumbKey, - }, -}; -use sd_core_indexer_rules::{ - seed::{NO_HIDDEN, NO_SYSTEM_FILES}, - IndexerRule, IndexerRuler, RulerDecision, -}; - -use sd_file_ext::{extensions::Extension, kind::ObjectKind}; -use sd_prisma::prisma::location; -use sd_utils::{chain_optional_iter, error::FileIOError}; - -use std::{ - collections::HashMap, - io::ErrorKind, - ops::Deref, - path::{Path, PathBuf}, - sync::Arc, -}; - -use chrono::{DateTime, Utc}; -use futures::Stream; -use itertools::{Either, Itertools}; -use rspc::ErrorCode; -use serde::Serialize; -use specta::Type; -use thiserror::Error; -use tokio::{io, spawn, sync::mpsc, task::JoinError}; -use tokio_stream::wrappers::ReceiverStream; -use tracing::{debug, error, span, warn, Level}; - -use super::normalize_path; - -#[derive(Debug, Error)] -pub enum NonIndexedLocationError { - #[error("path not found: {}", .0.display())] - NotFound(PathBuf), - - #[error(transparent)] - FileIO(#[from] FileIOError), - - #[error("database error: {0}")] - Database(#[from] prisma_client_rust::QueryError), - - #[error("error joining tokio task: {0}")] - TaskJoinError(#[from] JoinError), - - #[error("receiver shutdown error")] - SendError, -} - -impl From> for NonIndexedLocationError { - fn from(_: mpsc::error::SendError) -> Self { - Self::SendError - } -} - -impl From for rspc::Error { - fn from(e: NonIndexedLocationError) -> Self { - match e { - NonIndexedLocationError::NotFound(_) => { - rspc::Error::with_cause(ErrorCode::NotFound, e.to_string(), e) - } - _ => rspc::Error::with_cause(ErrorCode::InternalServerError, e.to_string(), e), - } - } -} - -impl> From<(P, io::Error)> for NonIndexedLocationError { - fn from((path, source): (P, io::Error)) -> Self { - if source.kind() == io::ErrorKind::NotFound { - Self::NotFound(path.as_ref().into()) - } else { - Self::FileIO(FileIOError::from((path, source))) - } - } -} - -#[derive(Serialize, Type, Debug)] -pub struct NonIndexedPathItem { - pub path: String, - pub name: String, - pub extension: String, - pub kind: i32, - pub is_dir: bool, - pub date_created: DateTime, - pub date_modified: DateTime, - pub size_in_bytes_bytes: Vec, - pub hidden: bool, -} - -// #[instrument(name = "non_indexed::walk", skip(sort_fn))] -pub async fn walk( - path: PathBuf, - with_hidden_files: bool, - node: Arc, - library: Arc, - sort_fn: impl FnOnce(&mut Vec) + Send, -) -> Result< - impl Stream>> + Send, - NonIndexedLocationError, -> { - let mut entries = get_all_entries(path.clone()).await?; - - { - let span = span!(Level::INFO, "sort_fn"); - let _enter = span.enter(); - - sort_fn(&mut entries); - } - - let (tx, rx) = mpsc::channel(128); - let tx2 = tx.clone(); - - // We wanna process and let the caller use the stream. - let task = spawn(async move { - let path = &path; - let indexer_ruler = IndexerRuler::new(chain_optional_iter( - [IndexerRule::from(NO_SYSTEM_FILES.deref())], - [(!with_hidden_files).then(|| IndexerRule::from(NO_HIDDEN.deref()))], - )); - - let mut thumbnails_to_generate = vec![]; - // Generating thumbnails for PDFs is kinda slow, so we're leaving them for last in the batch - let mut document_thumbnails_to_generate = vec![]; - let mut directories = vec![]; - - for entry in entries.into_iter() { - let (entry_path, name) = match normalize_path(entry.path) { - Ok(v) => v, - Err(e) => { - tx.send(Err(Either::Left( - NonIndexedLocationError::from((path, e)).into(), - ))) - .await?; - continue; - } - }; - - match indexer_ruler - .evaluate_path(&entry_path, &entry.metadata) - .await - { - Ok(RulerDecision::Accept) => { /* Everything is awesome! */ } - - Ok(RulerDecision::Reject) => { - continue; - } - - Err(e) => { - tx.send(Err(Either::Left(e.into()))).await?; - continue; - } - } - - if entry.metadata.is_dir() { - directories.push((entry_path, name, entry.metadata)); - } else { - let path = Path::new(&entry_path); - - let Some(name) = path - .file_stem() - .and_then(|s| s.to_str().map(str::to_string)) - else { - warn!(%entry_path, "Failed to extract name from path;"); - continue; - }; - - let extension = path - .extension() - .and_then(|s| s.to_str().map(str::to_string)) - .unwrap_or_default(); - - let kind = Extension::resolve_conflicting(&path, false) - .await - .map(Into::into) - .unwrap_or(ObjectKind::Unknown); - - let should_generate_thumbnail = { - #[cfg(feature = "ffmpeg")] - { - matches!( - kind, - ObjectKind::Image | ObjectKind::Video | ObjectKind::Document - ) - } - - #[cfg(not(feature = "ffmpeg"))] - { - matches!(kind, ObjectKind::Image | ObjectKind::Document) - } - }; - - let (thumbnail_key, has_created_thumbnail) = if should_generate_thumbnail { - if let Ok(cas_id) = - generate_cas_id(&path, entry.metadata.len()) - .await - .map_err(|e| { - tx.send(Err(Either::Left( - NonIndexedLocationError::from((path, e)).into(), - ))) - }) { - if kind == ObjectKind::Document { - document_thumbnails_to_generate.push(GenerateThumbnailArgs::new( - extension.clone(), - cas_id.clone(), - path.to_path_buf(), - )); - } else { - thumbnails_to_generate.push(GenerateThumbnailArgs::new( - extension.clone(), - cas_id.clone(), - path.to_path_buf(), - )); - } - - let thumb_exists = node - .ephemeral_thumbnail_exists(&cas_id) - .await - .map_err(NonIndexedLocationError::from)?; - - (Some(ThumbKey::new_ephemeral(cas_id)), thumb_exists) - } else { - (None, false) - } - } else { - (None, false) - }; - - tx.send(Ok(ExplorerItem::NonIndexedPath { - thumbnail: thumbnail_key, - item: NonIndexedPathItem { - hidden: path_is_hidden(Path::new(&entry_path), &entry.metadata), - path: entry_path, - name, - extension, - kind: kind as i32, - is_dir: false, - date_created: entry.metadata.created_or_now().into(), - date_modified: entry.metadata.modified_or_now().into(), - size_in_bytes_bytes: entry.metadata.len().to_be_bytes().to_vec(), - }, - has_created_thumbnail, - })) - .await?; - } - } - - thumbnails_to_generate.extend(document_thumbnails_to_generate); - - let thumbnails_directory = Arc::new(get_thumbnails_directory(node.config.data_directory())); - let reporter: Arc = Arc::new(NewThumbnailsReporter { - ctx: NodeContext { - node: Arc::clone(&node), - library: Arc::clone(&library), - }, - }); - - if node - .task_system - .dispatch_many( - thumbnails_to_generate - .into_iter() - .chunks(10) - .into_iter() - .map(|chunk| { - media_processor::Thumbnailer::new_ephemeral( - Arc::clone(&thumbnails_directory), - chunk.collect(), - Arc::clone(&reporter), - ) - }) - .collect::>(), - ) - .await - .is_err() - { - debug!("Task system shutting down"); - } - - let mut locations = library - .db - .location() - .find_many(vec![location::path::in_vec( - directories - .iter() - .map(|(path, _, _)| path.clone()) - .collect(), - )]) - .exec() - .await? - .into_iter() - .flat_map(|location| { - location - .path - .clone() - .map(|location_path| (location_path, location)) - }) - .collect::>(); - - for (directory, name, metadata) in directories { - if let Some(location) = locations.remove(&directory) { - tx.send(Ok(ExplorerItem::Location { item: location })) - .await?; - } else { - tx.send(Ok(ExplorerItem::NonIndexedPath { - thumbnail: None, - item: NonIndexedPathItem { - hidden: path_is_hidden(Path::new(&directory), &metadata), - path: directory, - name, - extension: String::new(), - kind: ObjectKind::Folder as i32, - is_dir: true, - date_created: metadata.created_or_now().into(), - date_modified: metadata.modified_or_now().into(), - size_in_bytes_bytes: metadata.len().to_be_bytes().to_vec(), - }, - has_created_thumbnail: false, - })) - .await?; - } - } - - Ok::<_, NonIndexedLocationError>(()) - }); - - spawn(async move { - match task.await { - Ok(Ok(())) => {} - Ok(Err(e)) => { - let _ = tx2.send(Err(Either::Left(e.into()))).await; - } - Err(e) => error!(?e, "error joining tokio task"), - } - }); - - Ok(ReceiverStream::new(rx)) -} - -#[derive(Debug)] -pub struct Entry { - path: PathBuf, - name: String, - // size_in_bytes: u64, - // date_created: - metadata: std::fs::Metadata, -} - -impl Entry { - pub fn name(&self) -> &str { - &self.name - } - - pub fn size_in_bytes(&self) -> u64 { - self.metadata.len() - } - - pub fn date_created(&self) -> DateTime { - self.metadata.created_or_now().into() - } - - pub fn date_modified(&self) -> DateTime { - self.metadata.modified_or_now().into() - } -} - -/// We get all of the FS entries first before we start processing on each of them. -/// -/// From my M1 Macbook Pro this: -/// - takes 11ms per 10 000 files -/// -/// and -/// -/// - consumes 0.16MB of RAM per 10 000 entries. -/// -/// The reason we collect these all up is so we can apply ordering, and then begin streaming the data as it's processed to the frontend. -// #[instrument(name = "get_all_entries")] -pub async fn get_all_entries(path: PathBuf) -> Result, NonIndexedLocationError> { - tokio::task::spawn_blocking(move || { - let path = &path; - let dir = std::fs::read_dir(path).map_err(|e| (path, e))?; - let mut entries = Vec::new(); - for entry in dir { - let entry = entry.map_err(|e| (path, e))?; - - // We must not keep `entry` around as we will quickly hit the OS limit on open file descriptors - entries.push(Entry { - path: entry.path(), - name: entry - .file_name() - .to_str() - .ok_or_else(|| { - ( - path, - io::Error::new(ErrorKind::Other, "error non UTF-8 path"), - ) - })? - .to_string(), - metadata: entry.metadata().map_err(|e| (path, e))?, - }); - } - - Ok(entries) - }) - .await? -} diff --git a/core/src/node/config.rs b/core/src/node/config.rs deleted file mode 100644 index 8fb0f4c13..000000000 --- a/core/src/node/config.rs +++ /dev/null @@ -1,572 +0,0 @@ -use crate::{ - api::{notifications::Notification, BackendFeature}, - /*object::media::old_thumbnail::preferences::ThumbnailerPreferences,*/ - util::version_manager::{Kind, ManagedVersion, VersionManager, VersionManagerError}, -}; - -use sd_cloud_schema::devices::DeviceOS; -use sd_core_sync::DevicePubId; -use sd_old_p2p::Identity; -use sd_utils::error::FileIOError; - -use std::{ - collections::HashSet, - path::{Path, PathBuf}, - sync::Arc, -}; - -use int_enum::IntEnum; -use serde::{Deserialize, Serialize}; -use serde_json::{json, Map, Value}; -use serde_repr::{Deserialize_repr, Serialize_repr}; -use specta::Type; -use thiserror::Error; -use tokio::{ - fs, - sync::{watch, RwLock}, -}; -use tracing::error; -use uuid::Uuid; - -use super::HardwareModel; - -/// NODE_STATE_CONFIG_NAME is the name of the file which stores the NodeState -pub const NODE_STATE_CONFIG_NAME: &str = "node_state.sdconfig"; - -#[derive(Debug, Default, Clone, Copy, Eq, PartialEq, Serialize, Deserialize, Type)] -pub enum P2PDiscoveryState { - #[default] - Everyone, - ContactsOnly, - Disabled, -} - -#[derive(Debug, Default, Clone, Copy, Serialize, Deserialize, Type)] -#[serde(rename_all = "snake_case", tag = "type", content = "value")] -pub enum Port { - #[default] - Random, - Discrete(u16), -} - -impl Port { - pub fn get(&self) -> u16 { - if is_in_docker() { - return 7373; - } - - match self { - Port::Random => 0, - Port::Discrete(port) => *port, - } - } - - pub fn is_default(&self) -> bool { - matches!(self, Port::Random) - } -} - -pub fn is_in_docker() -> bool { - std::env::var("SD_DOCKER").as_deref() == Ok("true") -} - -fn skip_if_false(value: &bool) -> bool { - !*value -} - -#[derive(Debug, Clone, Serialize, Deserialize, Type)] -pub struct NodeConfigP2P { - #[serde(default)] - pub discovery: P2PDiscoveryState, - #[serde(default, skip_serializing_if = "Port::is_default")] - pub port: Port, - #[serde(default, skip_serializing_if = "skip_if_false")] - pub disabled: bool, - #[serde(default, skip_serializing_if = "skip_if_false")] - pub disable_ipv6: bool, - #[serde(default, skip_serializing_if = "skip_if_false")] - pub disable_relay: bool, - #[serde(default, skip_serializing_if = "skip_if_false")] - pub enable_remote_access: bool, - /// A list of peer addresses to try and manually connect to, instead of relying on discovery. - /// - /// All of these are valid values: - /// - `localhost` - /// - `spacedrive.com` or `spacedrive.com:3000` - /// - `127.0.0.1` or `127.0.0.1:300` - /// - `[::1]` or `[::1]:3000` - /// - /// which is why we use `String` not `SocketAddr` - #[serde(default)] - pub manual_peers: HashSet, -} - -impl Default for NodeConfigP2P { - fn default() -> Self { - Self { - discovery: P2PDiscoveryState::Everyone, - port: Port::Random, - disabled: true, - disable_ipv6: true, - disable_relay: true, - enable_remote_access: false, - manual_peers: Default::default(), - } - } -} - -/// NodeConfig is the configuration for a node. -/// This is shared between all libraries and is stored in a JSON file on disk. -#[derive(Debug, Clone, Serialize, Deserialize)] // If you are adding `specta::Type` on this your probably about to leak the P2P private key -pub struct NodeConfig { - /// id is a unique identifier for the current node. Each node has a public identifier (this one) and is given a local id for each library (done within the library code). - pub id: DevicePubId, - /// name is the display name of the current node. This is set by the user and is shown in the UI. // TODO: Length validation so it can fit in DNS record - pub name: String, - /// core level notifications - #[serde(default)] - pub notifications: Vec, - /// The p2p identity keypair for this node. This is used to identify the node on the network. - /// This keypair does effectively nothing except for provide libp2p with a stable peer_id. - #[serde(with = "identity_serde")] - // TODO(@fogodev): remove these from here, we must not store secret keys in plaintext... - // Put then on secret storage when we have a keyring compatible with all our supported platforms - pub identity: Identity, - /// P2P config - #[serde(default)] - pub p2p: NodeConfigP2P, - /// Feature flags enabled on the node - #[serde(default)] - pub features: Vec, - /// The aggregation of many different preferences for the node - pub preferences: NodePreferences, - /// Operating System of the node - pub os: DeviceOS, - /// Hardware model of the node - pub hardware_model: HardwareModel, - - version: NodeConfigVersion, -} - -mod identity_serde { - use sd_old_p2p::Identity; - use serde::{Deserialize, Deserializer, Serialize, Serializer}; - - pub fn serialize(identity: &Identity, serializer: S) -> Result - where - S: Serializer, - { - to_string(identity).serialize(serializer) - } - - pub fn deserialize<'de, D>(deserializer: D) -> Result - where - D: Deserializer<'de>, - { - let s = String::deserialize(deserializer)?; - Identity::from_bytes(&base91::slice_decode(s.as_bytes())).map_err(serde::de::Error::custom) - } - - pub fn to_string(identity: &Identity) -> String { - String::from_utf8_lossy(&base91::slice_encode(&identity.to_bytes())).to_string() - } -} - -#[derive(Debug, Clone, Serialize, Deserialize, Default, PartialEq, Eq, Type)] -pub struct NodePreferences { - // pub thumbnailer: ThumbnailerPreferences, - // TODO(fogodev): introduce preferences to choose how many worker the task system should have -} - -#[derive( - IntEnum, Debug, Clone, Copy, Eq, PartialEq, strum::Display, Serialize_repr, Deserialize_repr, -)] -#[repr(u64)] -pub enum NodeConfigVersion { - V0 = 0, - V1 = 1, - V2 = 2, - V3 = 3, - V4 = 4, - V5 = 5, -} - -impl ManagedVersion for NodeConfig { - const LATEST_VERSION: NodeConfigVersion = NodeConfigVersion::V5; - const KIND: Kind = Kind::Json("version"); - type MigrationError = NodeConfigError; - - fn from_latest_version() -> Option { - #[cfg(not(any(target_os = "ios", target_os = "android")))] - let mut name = whoami::devicename(); - - #[cfg(target_os = "ios")] - let mut name = "iOS Device".to_string(); - - #[cfg(target_os = "android")] - let mut name = "Android Device".to_string(); - - name.truncate(255); - - let os = DeviceOS::from_env(); - let hardware_model = HardwareModel::try_get().unwrap_or_else(|e| { - error!(?e, "Failed to get hardware model"); - HardwareModel::Other - }); - - // Create .sdks file in the data directory if it doesn't exist - let data_directory = Path::new(NODE_STATE_CONFIG_NAME) - .parent() - .expect("Config path must have a parent directory"); - let sdks_file = data_directory.join(".sdks"); - if !sdks_file.exists() { - std::fs::write(&sdks_file, b"") - .map_err(|e| { - FileIOError::from((sdks_file.clone(), e, "Failed to create .sdks file")) - }) - .expect("Panicked to initialize .sdks file"); - } - - Some(Self { - id: Uuid::now_v7().into(), - name, - identity: Identity::default(), - p2p: NodeConfigP2P::default(), - version: Self::LATEST_VERSION, - features: vec![], - notifications: vec![], - preferences: NodePreferences::default(), - os, - hardware_model, - }) - } -} - -impl NodeConfig { - pub async fn load(path: impl AsRef) -> Result { - let path = path.as_ref(); - VersionManager::::migrate_and_load( - path, - |current, next| async move { - match (current, next) { - (NodeConfigVersion::V0, NodeConfigVersion::V1) => { - let mut config: Map = - serde_json::from_slice(&fs::read(path).await.map_err(|e| { - FileIOError::from(( - path, - e, - "Failed to read node config file for migration", - )) - })?) - .map_err(VersionManagerError::SerdeJson)?; - - // All were never hooked up to the UI - config.remove("p2p_email"); - config.remove("p2p_img_url"); - config.remove("p2p_port"); - - // In a recent PR I screwed up Serde `default` so P2P was disabled by default, prior it was always enabled. - // Given the config for it is behind a feature flag (so no one would have changed it) this fixes the default. - if let Some(Value::Object(obj)) = config.get_mut("p2p") { - obj.insert("enabled".into(), Value::Bool(true)); - } - - fs::write( - path, - serde_json::to_vec(&config).map_err(VersionManagerError::SerdeJson)?, - ) - .await - .map_err(|e| FileIOError::from((path, e)))?; - } - - (NodeConfigVersion::V1, NodeConfigVersion::V2) => { - let mut config: Map = - serde_json::from_slice(&fs::read(path).await.map_err(|e| { - FileIOError::from(( - path, - e, - "Failed to read node config file for migration", - )) - })?) - .map_err(VersionManagerError::SerdeJson)?; - - config.insert( - String::from("preferences"), - json!(NodePreferences::default()), - ); - - let a = - serde_json::to_vec(&config).map_err(VersionManagerError::SerdeJson)?; - - fs::write(path, a) - .await - .map_err(|e| FileIOError::from((path, e)))?; - } - - (NodeConfigVersion::V2, NodeConfigVersion::V3) => { - let mut config: Map = - serde_json::from_slice(&fs::read(path).await.map_err(|e| { - FileIOError::from(( - path, - e, - "Failed to read node config file for migration", - )) - })?) - .map_err(VersionManagerError::SerdeJson)?; - - config.remove("keypair"); - config.remove("p2p"); - - config.insert( - String::from("identity"), - json!(identity_serde::to_string(&Default::default())), - ); - - let a = - serde_json::to_vec(&config).map_err(VersionManagerError::SerdeJson)?; - - fs::write(path, a) - .await - .map_err(|e| FileIOError::from((path, e)))?; - } - - (NodeConfigVersion::V3, NodeConfigVersion::V4) => { - let mut config: Map = - serde_json::from_slice(&fs::read(path).await.map_err(|e| { - FileIOError::from(( - path, - e, - "Failed to read node config file for migration", - )) - })?) - .map_err(VersionManagerError::SerdeJson)?; - - config.remove("id"); - config.insert( - String::from("id"), - serde_json::to_value(Uuid::now_v7()) - .map_err(VersionManagerError::SerdeJson)?, - ); - - config.remove("name"); - - #[cfg(not(any(target_os = "ios", target_os = "android")))] - config.insert( - String::from("name"), - serde_json::to_value(whoami::devicename()) - .map_err(VersionManagerError::SerdeJson)?, - ); - - #[cfg(target_os = "ios")] - config.insert( - String::from("name"), - serde_json::to_value("iOS Device") - .map_err(VersionManagerError::SerdeJson)?, - ); - - #[cfg(target_os = "android")] - config.insert( - String::from("name"), - serde_json::to_value("Android Device") - .map_err(VersionManagerError::SerdeJson)?, - ); - - config.insert( - String::from("os"), - serde_json::to_value(std::env::consts::OS) - .map_err(VersionManagerError::SerdeJson)?, - ); - - let a = - serde_json::to_vec(&config).map_err(VersionManagerError::SerdeJson)?; - - fs::write(path, a) - .await - .map_err(|e| FileIOError::from((path, e)))?; - } - - (NodeConfigVersion::V4, NodeConfigVersion::V5) => { - let mut config: Map = - serde_json::from_slice(&fs::read(path).await.map_err(|e| { - FileIOError::from(( - path, - e, - "Failed to read node config file for migration", - )) - })?) - .map_err(VersionManagerError::SerdeJson)?; - - config.insert( - String::from("os"), - serde_json::to_value(DeviceOS::from_env()) - .map_err(VersionManagerError::SerdeJson)?, - ); - config.insert( - String::from("hardware_model"), - serde_json::to_value( - HardwareModel::try_get().unwrap_or(HardwareModel::Other), - ) - .map_err(VersionManagerError::SerdeJson)?, - ); - - config.remove("features"); - config.remove("auth_token"); - config.remove("sd_api_origin"); - config.remove("image_labeler_version"); - - // Verify that the ID isn't already set to a UUID v7. If it is, we don't want to overwrite it. - // Get the current ID, if it's a string, parse it as a UUID and check if it's a UUID v7. - // If it's not a UUID v7, set it to a UUID v7. - let id = config - .get("id") - .and_then(|v| v.as_str()) - .and_then(|s| Uuid::parse_str(s).ok()); - if let Some(id) = id { - if id.get_version() != Some(uuid::Version::Md5) { - config.remove("id"); - config.insert( - String::from("id"), - serde_json::to_value(DevicePubId::from(Uuid::now_v7())) - .map_err(VersionManagerError::SerdeJson)?, - ); - } - } - - // Create a .sdks file in the data directory if it doesn't exist - let data_directory = path - .parent() - .expect("Config path must have a parent directory"); - let sdks_file = data_directory.join(".sdks"); - if !sdks_file.exists() { - fs::write(&sdks_file, b"").await.map_err(|e| { - FileIOError::from(( - sdks_file.clone(), - e, - "Failed to create .sdks file", - )) - })?; - } - - // Write the updated config back to disk - fs::write( - path, - serde_json::to_vec(&config).map_err(VersionManagerError::SerdeJson)?, - ) - .await - .map_err(|e| { - FileIOError::from((path, e, "Failed to write back updated config")) - })?; - } - - _ => { - error!(current_version = ?current, "Node config version is not handled;"); - return Err(VersionManagerError::UnexpectedMigration { - current_version: current.int_value(), - next_version: next.int_value(), - } - .into()); - } - } - - Ok(()) - }, - ) - .await - } - - async fn save(&self, path: impl AsRef) -> Result<(), NodeConfigError> { - let path = path.as_ref(); - fs::write(path, serde_json::to_vec(self)?) - .await - .map_err(|e| FileIOError::from((path, e)))?; - - Ok(()) - } -} - -pub struct Manager { - config: RwLock, - data_directory_path: PathBuf, - config_file_path: PathBuf, - preferences_watcher_tx: watch::Sender, -} - -impl Manager { - /// new will create a new NodeConfigManager with the given path to the config file. - pub(crate) async fn new( - data_directory_path: impl AsRef, - ) -> Result, NodeConfigError> { - let data_directory_path = data_directory_path.as_ref().to_path_buf(); - let config_file_path = data_directory_path.join(NODE_STATE_CONFIG_NAME); - - let config = NodeConfig::load(&config_file_path).await?; - - let (preferences_watcher_tx, _preferences_watcher_rx) = - watch::channel(config.preferences.clone()); - - Ok(Arc::new(Self { - config: RwLock::new(config), - data_directory_path, - config_file_path, - preferences_watcher_tx, - })) - } - - /// get will return the current NodeConfig in a read only state. - pub(crate) async fn get(&self) -> NodeConfig { - self.config.read().await.clone() - } - - /// data_directory returns the path to the directory storing the configuration data. - pub(crate) fn data_directory(&self) -> PathBuf { - self.data_directory_path.clone() - } - - /// write allows the user to update the configuration. This is done in a closure while a Mutex lock is held so that the user can't cause a race condition if the config were to be updated in multiple parts of the app at the same time. - pub(crate) async fn write( - &self, - mutation_fn: F, - ) -> Result { - let mut config = self.config.write().await; - - mutation_fn(&mut config); - - self.preferences_watcher_tx.send_if_modified(|current| { - let modified = current != &config.preferences; - if modified { - *current = config.preferences.clone(); - } - modified - }); - - config - .save(&self.config_file_path) - .await - .map(|()| config.clone()) - } - - /// update_preferences allows the user to update the preferences of the node - pub(crate) async fn update_preferences( - &self, - update_fn: impl FnOnce(&mut NodePreferences), - ) -> Result<(), NodeConfigError> { - let mut config = self.config.write().await; - - update_fn(&mut config.preferences); - - self.preferences_watcher_tx - .send_replace(config.preferences.clone()); - - config.save(&self.config_file_path).await - } -} - -#[derive(Error, Debug)] -pub enum NodeConfigError { - #[error(transparent)] - SerdeJson(#[from] serde_json::Error), - #[error(transparent)] - VersionManager(#[from] VersionManagerError), - #[error(transparent)] - FileIO(#[from] FileIOError), -} diff --git a/core/src/node/hardware.rs b/core/src/node/hardware.rs deleted file mode 100644 index b0d6625cf..000000000 --- a/core/src/node/hardware.rs +++ /dev/null @@ -1,210 +0,0 @@ -use std::io; - -use serde::{Deserialize, Serialize}; -use specta::Type; -use strum::IntoEnumIterator; -use strum_macros::{Display, EnumIter}; - -#[repr(i32)] -#[derive(Debug, Clone, Display, Copy, EnumIter, Type, Serialize, Deserialize, Eq, PartialEq)] -#[specta(rename = "CoreHardwareModel")] -pub enum HardwareModel { - Other = 0, - MacStudio = 1, - MacBookAir = 2, - MacBookPro = 3, - MacBook = 4, - MacMini = 5, - MacPro = 6, - IMac = 7, - IMacPro = 8, - IPad = 9, - IPhone = 10, - Simulator = 11, - Android = 12, -} - -impl From for HardwareModel { - fn from(value: i32) -> Self { - match value { - 1 => Self::MacStudio, - 2 => Self::MacBookAir, - 3 => Self::MacBookPro, - 4 => Self::MacBook, - 5 => Self::MacMini, - 6 => Self::MacPro, - 7 => Self::IMac, - 8 => Self::IMacPro, - 9 => Self::IPad, - 10 => Self::IPhone, - 11 => Self::Simulator, - 12 => Self::Android, - _ => Self::Other, - } - } -} - -impl From for sd_cloud_schema::devices::HardwareModel { - fn from(model: HardwareModel) -> Self { - match model { - HardwareModel::MacStudio => Self::MacStudio, - HardwareModel::MacBookAir => Self::MacBookAir, - HardwareModel::MacBookPro => Self::MacBookPro, - HardwareModel::MacBook => Self::MacBook, - HardwareModel::MacMini => Self::MacMini, - HardwareModel::MacPro => Self::MacPro, - HardwareModel::IMac => Self::IMac, - HardwareModel::IMacPro => Self::IMacPro, - HardwareModel::IPad => Self::IPad, - HardwareModel::IPhone => Self::IPhone, - HardwareModel::Simulator => Self::Simulator, - HardwareModel::Android => Self::Android, - HardwareModel::Other => Self::Other, - } - } -} - -impl From for HardwareModel { - fn from(model: sd_cloud_schema::devices::HardwareModel) -> Self { - match model { - sd_cloud_schema::devices::HardwareModel::MacStudio => Self::MacStudio, - sd_cloud_schema::devices::HardwareModel::MacBookAir => Self::MacBookAir, - sd_cloud_schema::devices::HardwareModel::MacBookPro => Self::MacBookPro, - sd_cloud_schema::devices::HardwareModel::MacBook => Self::MacBook, - sd_cloud_schema::devices::HardwareModel::MacMini => Self::MacMini, - sd_cloud_schema::devices::HardwareModel::MacPro => Self::MacPro, - sd_cloud_schema::devices::HardwareModel::IMac => Self::IMac, - sd_cloud_schema::devices::HardwareModel::IMacPro => Self::IMacPro, - sd_cloud_schema::devices::HardwareModel::IPad => Self::IPad, - sd_cloud_schema::devices::HardwareModel::IPhone => Self::IPhone, - sd_cloud_schema::devices::HardwareModel::Simulator => Self::Simulator, - sd_cloud_schema::devices::HardwareModel::Android => Self::Android, - sd_cloud_schema::devices::HardwareModel::Other => Self::Other, - } - } -} - -impl From<&str> for HardwareModel { - fn from(name: &str) -> Self { - Self::iter() - .find(|&model| { - model.to_string().to_lowercase().replace(' ', "") - == name.to_lowercase().replace(' ', "") - }) - .unwrap_or(Self::Other) - } -} - -impl HardwareModel { - pub fn try_get() -> Result { - #[cfg(target_os = "macos")] - { - use std::process::Command; - - let output = Command::new("system_profiler") - .arg("SPHardwareDataType") - .output()?; - - if output.status.success() { - let output_str = std::str::from_utf8(&output.stdout).unwrap_or_default(); - let hardware_model = output_str - .lines() - .find(|line| line.to_lowercase().contains("model name")) - .and_then(|line| line.split_once(':')) - .map(|(_, model_name)| model_name.trim().into()) - .unwrap_or(Self::Other); - - Ok(hardware_model) - } else { - Err(io::Error::new( - io::ErrorKind::Other, - format!( - "Failed to get hardware model name: {}", - String::from_utf8_lossy(&output.stderr) - ), - )) - } - } - #[cfg(target_os = "ios")] - { - use std::ffi::CString; - use std::io::Error; - use std::ptr; - - extern "C" { - fn sysctlbyname( - name: *const libc::c_char, - oldp: *mut libc::c_void, - oldlenp: *mut usize, - newp: *mut libc::c_void, - newlen: usize, - ) -> libc::c_int; - } - - fn get_device_type() -> Option { - let mut size: usize = 0; - let name = CString::new("hw.machine").expect("CString::new failed"); - - // First, get the size of the buffer needed - unsafe { - sysctlbyname( - name.as_ptr(), - ptr::null_mut(), - &mut size, - ptr::null_mut(), - 0, - ); - } - - // Allocate a buffer with the correct size - let mut buffer: Vec = vec![0; size]; - - // Get the actual machine type - unsafe { - sysctlbyname( - name.as_ptr(), - buffer.as_mut_ptr() as *mut libc::c_void, - &mut size, - ptr::null_mut(), - 0, - ); - } - - // Convert the buffer to a String - let machine_type = String::from_utf8_lossy(&buffer).trim().to_string(); - - // Check if the device is an iPad or iPhone - if machine_type.starts_with("iPad") { - Some("iPad".to_string()) - } else if machine_type.starts_with("iPhone") { - Some("iPhone".to_string()) - } else if machine_type.starts_with("arm") { - Some("Simulator".to_string()) - } else { - None - } - } - - if let Some(device_type) = get_device_type() { - let hardware_model = HardwareModel::from(device_type.as_str()); - - Ok(hardware_model) - } else { - Err(Error::new( - std::io::ErrorKind::Other, - "Failed to get hardware model name", - )) - } - } - - #[cfg(target_os = "android")] - { - Ok(Self::Android) - } - - #[cfg(not(any(target_os = "macos", target_os = "ios", target_os = "android")))] - { - Ok(Self::Other) - } - } -} diff --git a/core/src/node/mod.rs b/core/src/node/mod.rs deleted file mode 100644 index dbc77094b..000000000 --- a/core/src/node/mod.rs +++ /dev/null @@ -1,6 +0,0 @@ -pub mod config; -mod hardware; -mod platform; - -pub use hardware::*; -pub use platform::*; diff --git a/core/src/node/platform.rs b/core/src/node/platform.rs deleted file mode 100644 index b1566900e..000000000 --- a/core/src/node/platform.rs +++ /dev/null @@ -1,62 +0,0 @@ -use crate::NodeError; - -use serde::{Deserialize, Serialize}; -use specta::Type; - -#[allow(clippy::upper_case_acronyms)] -#[repr(u8)] -#[derive(Debug, Clone, Copy, Serialize, Deserialize, Type, Eq, PartialEq)] -pub enum Platform { - Unknown = 0, - Windows = 1, - MacOS = 2, - Linux = 3, - IOS = 4, - Android = 5, -} - -impl Platform { - #[allow(unreachable_code)] - pub fn current() -> Self { - #[cfg(target_os = "windows")] - return Self::Windows; - - #[cfg(target_os = "macos")] - return Self::MacOS; - - #[cfg(target_os = "linux")] - return Self::Linux; - - #[cfg(target_os = "ios")] - return Self::IOS; - - #[cfg(target_os = "android")] - return Self::Android; - - Self::Unknown - } -} - -impl TryFrom for Platform { - type Error = NodeError; - - fn try_from(value: u8) -> Result { - let s = match value { - 0 => Self::Unknown, - 1 => Self::Windows, - 2 => Self::MacOS, - 3 => Self::Linux, - 4 => Self::IOS, - 5 => Self::Android, - _ => return Err(NodeError::InvalidPlatformInt(value)), - }; - - Ok(s) - } -} - -impl From for u8 { - fn from(platform: Platform) -> Self { - platform as u8 - } -} diff --git a/core/src/notifications.rs b/core/src/notifications.rs deleted file mode 100644 index aa8f50162..000000000 --- a/core/src/notifications.rs +++ /dev/null @@ -1,34 +0,0 @@ -use crate::api::notifications::Notification; - -use std::sync::{atomic::AtomicU32, Arc}; - -use tokio::sync::broadcast; - -#[derive(Clone)] -pub struct Notifications( - // Keep this private and use `Node::emit_notification` or `Library::emit_notification` instead. - broadcast::Sender, - // Counter for `NotificationId::Node(_)`. NotificationId::Library(_, _)` is autogenerated by the DB. - Arc, -); - -impl Notifications { - #[allow(clippy::new_without_default)] - pub fn new() -> Self { - let (tx, _) = broadcast::channel(30); - Self(tx, Arc::new(AtomicU32::new(0))) - } - - pub fn subscribe(&self) -> broadcast::Receiver { - self.0.subscribe() - } - - /// DO NOT USE THIS. Use `Node::emit_notification` or `Library::emit_notification` instead. - pub fn _internal_send(&self, notification: Notification) { - self.0.send(notification).ok(); - } - - pub fn _internal_next_id(&self) -> u32 { - self.1.fetch_add(1, std::sync::atomic::Ordering::SeqCst) - } -} diff --git a/core/src/object/fs/archive.rs b/core/src/object/fs/archive.rs deleted file mode 100644 index e69de29bb..000000000 diff --git a/core/src/object/fs/convert.rs b/core/src/object/fs/convert.rs deleted file mode 100644 index e69de29bb..000000000 diff --git a/core/src/object/fs/error.rs b/core/src/object/fs/error.rs deleted file mode 100644 index 2f610eab7..000000000 --- a/core/src/object/fs/error.rs +++ /dev/null @@ -1,51 +0,0 @@ -use crate::location::LocationError; - -use sd_core_file_path_helper::FilePathError; - -use sd_prisma::prisma::file_path; -use sd_utils::{ - db::MissingFieldError, - error::{FileIOError, NonUtf8PathError}, -}; - -use std::path::Path; - -use prisma_client_rust::QueryError; -use thiserror::Error; - -/// Error type for file system related jobs errors -#[derive(Error, Debug)] -pub enum FileSystemJobsError { - #[error("Location error: {0}")] - Location(#[from] LocationError), - #[error("file_path not in database: ", .0.display())] - FilePathNotFound(Box), - #[error("file_path id not in database: ")] - FilePathIdNotFound(file_path::id::Type), - #[error("failed to create file or folder on disk")] - CreateFileOrFolder(FileIOError), - #[error("database error: {0}")] - Database(#[from] QueryError), - #[error(transparent)] - FilePath(#[from] FilePathError), - #[error("action would overwrite another file: {}", .0.display())] - WouldOverwrite(Box), - #[error("missing-field: {0}")] - MissingField(#[from] MissingFieldError), - #[error("no parent for path, which is supposed to be directory: ", .0.display())] - MissingParentPath(Box), - #[error("no stem on file path, but it's supposed to be a file: ", .0.display())] - MissingFileStem(Box), - #[error(transparent)] - FileIO(#[from] FileIOError), - #[error(transparent)] - NonUTF8Path(#[from] NonUtf8PathError), - #[error("failed to find an available name to avoid duplication: ", .0.display())] - FailedToFindAvailableName(Box), -} - -impl From for rspc::Error { - fn from(e: FileSystemJobsError) -> Self { - Self::with_cause(rspc::ErrorCode::InternalServerError, e.to_string(), e) - } -} diff --git a/core/src/object/fs/mod.rs b/core/src/object/fs/mod.rs deleted file mode 100644 index 93f748651..000000000 --- a/core/src/object/fs/mod.rs +++ /dev/null @@ -1,222 +0,0 @@ -use crate::location::LocationError; - -use sd_core_file_path_helper::IsolatedFilePathData; -use sd_core_prisma_helpers::file_path_with_object; - -use sd_prisma::prisma::{file_path, location, PrismaClient}; -use sd_utils::{ - db::maybe_missing, - error::{FileIOError, NonUtf8PathError}, -}; -use tracing::trace; - -use std::{ - ffi::OsStr, - path::{Path, PathBuf}, - sync::LazyLock, -}; - -use regex::Regex; -use serde::{Deserialize, Serialize}; - -pub mod old_delete; -pub mod old_erase; - -pub mod old_copy; -pub mod old_cut; - -// pub mod decrypt; -// pub mod encrypt; - -pub mod error; - -use error::FileSystemJobsError; -use tokio::{fs, io}; - -static DUPLICATE_PATTERN: LazyLock = - LazyLock::new(|| Regex::new(r" \(\d+\)").expect("Failed to compile hardcoded regex")); - -// pub const BYTES_EXT: &str = ".bytes"; - -#[derive(Serialize, Deserialize, Debug, Clone, Eq, PartialEq)] -pub enum ObjectType { - File, - Directory, -} - -#[derive(Serialize, Deserialize, Debug, Clone)] -pub struct FileData { - pub file_path: file_path_with_object::Data, - pub full_path: PathBuf, -} - -/// Get the [`FileData`] related to every `file_path_id` -pub async fn get_many_files_datas( - db: &PrismaClient, - location_path: impl AsRef, - file_path_ids: &[file_path::id::Type], -) -> Result, FileSystemJobsError> { - let location_path = location_path.as_ref(); - - db._batch( - file_path_ids - .iter() - .map(|file_path_id| { - db.file_path() - .find_unique(file_path::id::equals(*file_path_id)) - .include(file_path_with_object::include()) - }) - // FIXME:(fogodev -> Brendonovich) this collect is a workaround to a weird higher ranker lifetime error on - // the _batch function, it should be removed once the error is fixed - .collect::>(), - ) - .await? - .into_iter() - .zip(file_path_ids.iter()) - .map(|(maybe_file_path, file_path_id)| { - maybe_file_path - .ok_or(FileSystemJobsError::FilePathIdNotFound(*file_path_id)) - .and_then(|path_data| { - Ok(FileData { - full_path: location_path.join(IsolatedFilePathData::try_from(&path_data)?), - file_path: path_data, - }) - }) - }) - .collect() -} - -pub async fn get_file_data_from_isolated_file_path( - db: &PrismaClient, - location_path: impl AsRef, - iso_file_path: &IsolatedFilePathData<'_>, -) -> Result { - let location_path = location_path.as_ref(); - db.file_path() - .find_unique(iso_file_path.into()) - .include(file_path_with_object::include()) - .exec() - .await? - .ok_or_else(|| { - FileSystemJobsError::FilePathNotFound( - location_path.join(iso_file_path).into_boxed_path(), - ) - }) - .and_then(|path_data| { - Ok(FileData { - full_path: location_path.join(IsolatedFilePathData::try_from(&path_data)?), - file_path: path_data, - }) - }) -} - -pub async fn fetch_source_and_target_location_paths( - db: &PrismaClient, - source_location_id: location::id::Type, - target_location_id: location::id::Type, -) -> Result<(PathBuf, PathBuf), FileSystemJobsError> { - match db - ._batch(( - db.location() - .find_unique(location::id::equals(source_location_id)), - db.location() - .find_unique(location::id::equals(target_location_id)), - )) - .await? - { - (Some(source_location), Some(target_location)) => Ok(( - maybe_missing(source_location.path.map(PathBuf::from), "location.path")?, - maybe_missing(target_location.path.map(PathBuf::from), "location.path")?, - )), - (None, _) => Err(LocationError::IdNotFound(source_location_id))?, - (_, None) => Err(LocationError::IdNotFound(target_location_id))?, - } -} - -fn construct_target_filename(source_file_data: &FileData) -> Result { - // extension wizardry for cloning and such - // if no suffix has been selected, just use the file name - // if a suffix is provided and it's a directory, use the directory name + suffix - // if a suffix is provided and it's a file, use the (file name + suffix).extension - - Ok( - if *maybe_missing(&source_file_data.file_path.is_dir, "file_path.is_dir")? - || source_file_data.file_path.extension.is_none() - || source_file_data.file_path.extension == Some(String::new()) - { - maybe_missing(&source_file_data.file_path.name, "file_path.name")?.clone() - } else { - format!( - "{}.{}", - maybe_missing(&source_file_data.file_path.name, "file_path.name")?, - maybe_missing(&source_file_data.file_path.extension, "file_path.extension")? - ) - }, - ) -} - -pub fn append_digit_to_filename( - final_path: &mut PathBuf, - file_name: &str, - ext: Option<&str>, - current_int: u32, -) { - let new_file_name = if let Some(found) = DUPLICATE_PATTERN.find_iter(file_name).last() { - &file_name[..found.start()] - } else { - file_name - } - .to_string(); - - if let Some(ext) = ext { - final_path.push(format!("{} ({current_int}).{}", new_file_name, ext)); - } else { - final_path.push(format!("{new_file_name} ({current_int})")); - } -} - -pub async fn find_available_filename_for_duplicate( - target_path: impl AsRef, -) -> Result { - let target_path = target_path.as_ref(); - - let new_file_name = target_path - .file_stem() - .ok_or_else(|| { - FileSystemJobsError::MissingFileStem(target_path.to_path_buf().into_boxed_path()) - })? - .to_str() - .ok_or_else(|| NonUtf8PathError(target_path.to_path_buf().into_boxed_path()))?; - - let new_file_full_path_without_suffix = - target_path.parent().map(Path::to_path_buf).ok_or_else(|| { - FileSystemJobsError::MissingParentPath(target_path.to_path_buf().into_boxed_path()) - })?; - - for i in 1..u32::MAX { - let mut new_file_full_path_candidate = new_file_full_path_without_suffix.clone(); - - append_digit_to_filename( - &mut new_file_full_path_candidate, - new_file_name, - target_path.extension().and_then(OsStr::to_str), - i, - ); - - match fs::metadata(&new_file_full_path_candidate).await { - Ok(_) => { - // This candidate already exists, so we try the next one - continue; - } - Err(e) if e.kind() == io::ErrorKind::NotFound => { - trace!(old_name=?target_path, new_name=?new_file_full_path_candidate, "duplicated file name, file renamed"); - return Ok(new_file_full_path_candidate); - } - Err(e) => return Err(FileIOError::from((new_file_full_path_candidate, e)).into()), - } - } - - Err(FileSystemJobsError::FailedToFindAvailableName( - target_path.to_path_buf().into_boxed_path(), - )) -} diff --git a/core/src/object/fs/old_copy.rs b/core/src/object/fs/old_copy.rs deleted file mode 100644 index 2d7b0fb70..000000000 --- a/core/src/object/fs/old_copy.rs +++ /dev/null @@ -1,669 +0,0 @@ -use crate::{ - invalidate_query, - library::Library, - old_job::{ - CurrentStep, JobError, JobInitOutput, JobReportUpdate, JobResult, JobRunMetadata, - JobStepOutput, StatefulJob, WorkerContext, - }, -}; - -use sd_core_file_path_helper::{join_location_relative_path, IsolatedFilePathData}; - -use sd_prisma::prisma::{file_path, location}; -use sd_utils::{db::maybe_missing, error::FileIOError}; - -use std::{ - collections::HashSet, - hash::Hash, - path::{Path, PathBuf}, - sync::{Arc, Mutex}, - time::Duration, -}; - -use futures_concurrency::future::{Race, TryJoin}; -use serde::{Deserialize, Serialize}; -use serde_json::json; -use specta::Type; -use tokio::fs; -use tracing::debug; - -use super::{ - construct_target_filename, error::FileSystemJobsError, fetch_source_and_target_location_paths, - find_available_filename_for_duplicate, get_file_data_from_isolated_file_path, - get_many_files_datas, FileData, -}; - -#[derive(Serialize, Deserialize, Debug, Clone)] -pub struct OldFileCopierJobData { - sources_location_path: PathBuf, - total_size: u64, - steps_len: usize, -} - -#[derive(Serialize, Deserialize, Hash, Type, Debug)] -pub struct OldFileCopierJobInit { - pub source_location_id: location::id::Type, - pub target_location_id: location::id::Type, - pub sources_file_path_ids: Vec, - pub target_location_relative_directory_path: PathBuf, -} - -#[derive(Serialize, Deserialize, Debug, Clone, Copy)] -enum CopierStepKind { - CreateDirs(CreateDirs), - CopyFiles(CopyFiles), -} - -#[derive(Serialize, Deserialize, Debug, Clone, Copy)] -struct CopyFiles; - -impl CopyFiles { - async fn copy_files( - files: &[Copy], - jobmeta: Arc>, - ) -> Result<(), JobError> { - // NOTE(matheus-consoli): if a step contains multiple files with the same name, - // for example `[file (1), file (2), file (3)]`, they all may be get renamed to the same - // new file (`file (4)`) as the decision for the new file name and the creation of such file - // all happens concurrently. - // this HashSet introduces a point of synchronization so we guarantee that all files have - // unique names - this is not the ideal solution, but it's a quick one. - let renamed_files_in_this_step = Arc::new(Mutex::new(HashSet::new())); - - files - .iter() - .map( - |Copy { - source, - source_size, - target_full_path, - }| { - let jobmeta = Arc::clone(&jobmeta); - let renamed_files_in_this_step = Arc::clone(&renamed_files_in_this_step); - async move { - let target = loop { - let target = - OldFileCopierJobStep::find_available_name(&target_full_path) - .await?; - - let mut cache = renamed_files_in_this_step - .lock() - .expect("failed to get lock for internal cache"); - if cache.contains(&target) { - // file name is taken, try again - continue; - } else { - cache.insert(target.clone()); - break target; - } - }; - fs::copy(&source.full_path, &target).await.map_err(|e| { - let source = source.full_path.clone(); - FileIOError::from((source, e)) - })?; - - let mut meta = jobmeta - .lock() - .expect("failed to get the lock for the list of files to copy"); - let accumulated_copied_size = meta.accumulated_copied_size + source_size; - let copied_files_count = meta.copied_files_count + 1; - meta.update(OldFileCopierJobMetadata { - accumulated_copied_size, - copied_files_count, - }); - - Ok::<_, JobError>(()) - } - }, - ) - .collect::>() - .try_join() - .await?; - - Ok(()) - } -} - -#[derive(Serialize, Deserialize, Debug, Clone, Copy)] -struct CreateDirs; - -impl CreateDirs { - /// Create the directories - async fn create_dir_structure(files: &[Copy]) -> Result<(), JobError> { - // TODO(matheus-consoli): when the directory name conflicts, what should we do? - // same as find_available...? - files - .iter() - .map(|file| async move { - fs::create_dir_all(&file.target_full_path) - .await - .map_err(|e| FileIOError::from((file.target_full_path.clone(), e))) - }) - .collect::>() - .try_join() - .await?; - Ok(()) - } -} - -#[derive(Serialize, Deserialize, Debug)] -pub struct Copy { - source: FileData, - source_size: u64, - target_full_path: Box, -} - -#[derive(Serialize, Deserialize, Debug)] -pub struct OldFileCopierJobStep { - files: Box<[Copy]>, - target_location_path: Box, - copy_kind: CopierStepKind, -} - -impl OldFileCopierJobStep { - async fn find_available_name(path: impl AsRef) -> Result { - let path = path.as_ref(); - match fs::try_exists(&path).await { - Ok(true) => { - // file already exists, try finding a better name - find_available_filename_for_duplicate(&path) - .await - .map_err(Into::into) - } - Ok(false) => { - // nothing todo, file name is available - Ok(path.to_owned()) - } - Err(e) => Err(FileIOError::from((path, e)).into()), - } - } -} - -fn progress(ctx: &WorkerContext, msgs: impl IntoIterator) { - let updates = msgs.into_iter().map(Into::into).collect(); - - ctx.progress(updates); -} - -#[derive(Debug, Clone)] -enum CopierUpdate { - Start, - TotalSize(String), - FileCount(usize), - TotalProgress(u64), - ProgressPerFile(String), - FinishedWithPercetage(u64), -} - -impl From for JobReportUpdate { - fn from(value: CopierUpdate) -> Self { - match value { - CopierUpdate::Start => { - const HUNDRED_PERCENT: usize = 100; - JobReportUpdate::TaskCount(HUNDRED_PERCENT) - } - CopierUpdate::FinishedWithPercetage(task_progress) => JobReportUpdate::TaskCount( - task_progress - .try_into() - .expect("should be able to convert a `u64` to `usize`"), - ), - CopierUpdate::TotalSize(size) => JobReportUpdate::Message(size.to_owned()), - CopierUpdate::FileCount(count) => JobReportUpdate::Info(count.to_string()), - CopierUpdate::ProgressPerFile(per_file) => JobReportUpdate::Phase(per_file.to_owned()), - CopierUpdate::TotalProgress(progressed_tasks) => JobReportUpdate::CompletedTaskCount( - progressed_tasks - .try_into() - .expect("should be able to convert a `u64` to `usize`"), - ), - } - } -} - -#[derive(Serialize, Deserialize, Debug, Clone, Default)] -pub struct OldFileCopierJobMetadata { - accumulated_copied_size: u64, - copied_files_count: u64, -} - -impl JobRunMetadata for OldFileCopierJobMetadata { - fn update(&mut self, metadata: Self) { - *self = metadata; - } -} - -#[async_trait::async_trait] -impl StatefulJob for OldFileCopierJobInit { - type Data = OldFileCopierJobData; - type Step = OldFileCopierJobStep; - type RunMetadata = OldFileCopierJobMetadata; - - const NAME: &'static str = "file_copier"; - const IS_BATCHED: bool = true; - - fn target_location(&self) -> location::id::Type { - self.target_location_id - } - - async fn init( - &self, - ctx: &WorkerContext, - data: &mut Option, - ) -> Result, JobError> { - let init = self; - let Library { db, .. } = &*ctx.library; - - let (sources_location_path, targets_location_path) = - fetch_source_and_target_location_paths( - db, - init.source_location_id, - init.target_location_id, - ) - .await?; - - let files = - get_many_files_datas(db, &sources_location_path, &init.sources_file_path_ids).await?; - - if let Some(missing_field) = files - .iter() - .find_map(|file| maybe_missing(file.file_path.is_dir, "file.is_dir").err()) - { - return Err(missing_field.into()); - } - - let archives = files - .into_iter() - .map(|file_data| async { - // add the currently viewed subdirectory to the location root - let mut full_target_path = join_location_relative_path( - &targets_location_path, - &init.target_location_relative_directory_path, - ); - - full_target_path.push(construct_target_filename(&file_data)?); - - Ok::<_, FileSystemJobsError>((file_data, full_target_path)) - }) - .collect::>() - .try_join() - .await?; - - let (mut dirs, mut files): (Vec<_>, _) = archives.into_iter().partition(|file| { - file.0 - .file_path - .is_dir - .expect("we tested that all file paths have the `is_dir` field") - }); - - let mut steps = Vec::with_capacity(dirs.len() + files.len()); - - // first step: create all directories using a single job - if !dirs.is_empty() { - let (more_dirs, more_files) = unfold_diretory(&dirs).await?; - - let more_dirs = more_dirs - .into_iter() - .map(|dir| async { - let iso = IsolatedFilePathData::new( - init.source_location_id, - &sources_location_path, - dir.source, - true, // is dir - ) - .map_err(FileSystemJobsError::from)?; - let file_data = get_file_data_from_isolated_file_path( - &ctx.library.db, - &sources_location_path, - &iso, - ) - .await?; - Ok::<_, JobError>((file_data, dir.dest)) - }) - .collect::>() - .try_join() - .await?; - dirs.extend(more_dirs); - - let (dir_source_file_data, dir_target_full_path) = - dirs.into_iter().unzip::<_, _, Vec<_>, Vec<_>>(); - - let step_files = dir_source_file_data - .into_iter() - .zip(dir_target_full_path.into_iter()) - .map(|(source, target_full_path)| Copy { - source, - source_size: 0, - target_full_path: target_full_path.into_boxed_path(), - }) - .collect(); - - let create_dirs_step = OldFileCopierJobStep { - files: step_files, - target_location_path: targets_location_path.clone().into_boxed_path(), - copy_kind: CopierStepKind::CreateDirs(CreateDirs), - }; - steps.push(create_dirs_step); - - let more_files = more_files - .into_iter() - .map(|file| async { - let iso = IsolatedFilePathData::new( - init.source_location_id, - &sources_location_path, - file.source, - false, // is dir - ) - .map_err(FileSystemJobsError::from)?; - let file_data = get_file_data_from_isolated_file_path( - &ctx.library.db, - &sources_location_path, - &iso, - ) - .await?; - Ok::<_, JobError>((file_data, file.dest)) - }) - .collect::>() - .try_join() - .await; - - if let Err(e) = more_files.as_ref() { - // the file is not indexed - tracing::error!(?e, "job init failed"); - } - - files.extend(more_files?); - }; - - // remaining steps: delegate to the copy strategist to decide how to organize - // the steps that copies the files - let steps_to_create_files = file_copy_strategist(files, &targets_location_path).await?; - steps.extend(steps_to_create_files); - - let total_size = steps - .iter() - .filter(|step| matches!(step.copy_kind, CopierStepKind::CopyFiles(_))) - .map(|step| step.files.iter().map(|file| file.source_size).sum::()) - .sum::(); - - let file_count = steps - .iter() - .filter(|step| matches!(step.copy_kind, CopierStepKind::CopyFiles(_))) - .map(|step| step.files.len()) - .sum::(); - - let updates = [ - CopierUpdate::Start, - CopierUpdate::FileCount(file_count), - CopierUpdate::TotalSize(total_size.to_string()), - ]; - progress(ctx, updates); - - *data = Some(OldFileCopierJobData { - sources_location_path, - total_size, - steps_len: steps.len(), - }); - - Ok(steps.into()) - } - - #[tracing::instrument( - skip_all, - fields( - step.kind = ?step.step.copy_kind, - step.n = step.step_number, - progress = jobmeta.accumulated_copied_size - ) - )] - async fn execute_step( - &self, - ctx: &WorkerContext, - step: CurrentStep<'_, Self::Step>, - data: &Self::Data, - jobmeta: &Self::RunMetadata, - ) -> Result, JobError> { - let files = &step.step.files; - let acc_copied_size = jobmeta.accumulated_copied_size; - let total_size = data.total_size; - let jobmeta = Arc::new(Mutex::new(jobmeta.clone())); - - let transfer = { - let jobmeta = Arc::clone(&jobmeta); - async move { - match step.step.copy_kind { - CopierStepKind::CreateDirs(CreateDirs) => { - CreateDirs::create_dir_structure(&step.step.files).await?; - } - CopierStepKind::CopyFiles(CopyFiles) => { - CopyFiles::copy_files(&step.step.files, jobmeta).await?; - } - }; - Ok::<_, JobError>(()) - } - }; - - let report = async move { - let mut finished = vec![false; files.len()]; - let mut step_copied = vec![0; files.len()]; - let relative_paths: Vec<&Path> = files - .iter() - .map(|f| { - f.target_full_path - .strip_prefix(&step.step.target_location_path) - .unwrap_or(&f.target_full_path) - }) - .collect(); - - loop { - for (((file, relative_path), copied), is_file_done) in files - .iter() - .zip(relative_paths.iter()) - .zip(step_copied.iter_mut()) - .zip(finished.iter_mut()) - .filter(|(_, is_file_done)| !**is_file_done) - { - let Ok(transfering) = fs::metadata(&file.target_full_path).await else { - // file may not have been created yet - continue; - }; - - let file_percentage = - (transfering.len() as f64 / file.source_size as f64) * 100.0; - let file_percentage = file_percentage.round(); - - let msg = format!("{file_percentage}% of {:?}", relative_path); - progress(ctx, [CopierUpdate::ProgressPerFile(msg)]); - - *copied = transfering.len(); - if transfering.len() == file.source_size { - *is_file_done = true; - } - } - - let copied_in_step = step_copied.iter().sum::(); - let total_percentage = - ((copied_in_step + acc_copied_size) as f64 / total_size as f64) * 100.; - let per = total_percentage.round() as u64; - progress(ctx, [CopierUpdate::TotalProgress(per)]); - - // wait for progress - tokio::time::sleep(Duration::from_millis(200)).await; - } - }; - - (transfer, report).race().await?; - - if data.steps_len == step.step_number + 1 { - let jobmeta = jobmeta - .lock() - .expect("failed to get the lock for job metadata"); - progress( - ctx, - [CopierUpdate::FinishedWithPercetage( - jobmeta.copied_files_count, - )], - ); - } - - let jobmeta = Arc::into_inner(jobmeta) - .expect("all the other copies should have been dropped by this point") - .into_inner() - .expect("the Mutex shouldn't be poisoned"); - - // we've calculated all steps up ahead - Ok(jobmeta.into()) - } - - async fn finalize( - &self, - ctx: &WorkerContext, - _data: &Option, - _run_metadata: &Self::RunMetadata, - ) -> JobResult { - let init = self; - - invalidate_query!(ctx.library, "search.paths"); - - Ok(Some(json!({ "init": init }))) - } -} - -/// Gather information about the list of files and decide what is the best -/// approach to organize them into steps. -async fn file_copy_strategist( - files: Vec<(FileData, PathBuf)>, - location_path: &Path, -) -> Result, JobError> { - // maximum size in bytes per step (800) - const MAX_TOTAL_SIZE_PER_STEP: u64 = 1024 * 1024 * 800; - // max quantity of files per step - const MAX_FILES_PER_STEP: usize = 20; - - debug!("generating steps to copy files"); - - let mut metadata = files - .into_iter() - .map(|(data, path)| async move { - let meta = tokio::fs::metadata(&data.full_path) - .await - .map_err(|e| FileIOError::from((data.full_path.clone(), e)))?; - Ok::<_, JobError>((meta.len(), data, path)) - }) - .collect::>() - .try_join() - .await?; - - // sort by size - metadata.sort_unstable_by_key(|m| m.0); - - let mut metadata = metadata.into_iter().peekable(); - let mut steps = Vec::new(); - - loop { - let mut sum = 0; - let mut files = Vec::with_capacity(MAX_FILES_PER_STEP); - - while let Some((source_size, source, path)) = metadata.next_if(|(len, _, _)| { - files.len() < MAX_FILES_PER_STEP && len + sum <= MAX_TOTAL_SIZE_PER_STEP || sum == 0 - }) { - sum += source_size; - files.push(Copy { - source, - source_size, - target_full_path: path.into_boxed_path(), - }); - } - - steps.push(OldFileCopierJobStep { - files: files.into_boxed_slice(), - target_location_path: location_path.into(), - copy_kind: CopierStepKind::CopyFiles(CopyFiles), - }); - - if metadata.peek().is_none() { - // nothing left to do, all files are grouped into steps - break; - } - } - - Ok(steps) -} - -async fn unfold_diretory( - dirs: &[(FileData, PathBuf)], -) -> Result<(Vec, Vec), JobError> { - let mut unfolded_dirs = Vec::new(); - let mut unfolded_files = Vec::new(); - - let mut dirs = Vec::from_iter( - dirs.iter() - .map(|(file_data, path)| (file_data.full_path.clone(), path.clone())), - ); - - loop { - if dirs.is_empty() { - break; - } - let unfolds = dirs - .iter() - .map(|(file_data, target_full_path)| async move { - let target_full_path = target_full_path.clone(); - - let mut to_look = Vec::new(); - let mut more_dirs = Vec::new(); - let mut more_files = Vec::new(); - let mut read_dir = fs::read_dir(file_data) - .await - .map_err(|e| FileIOError::from((file_data.clone(), e)))?; - - while let Some(children_entry) = read_dir - .next_entry() - .await - .map_err(|e| FileIOError::from((file_data.clone(), e)))? - { - let children_path = &children_entry.path(); - let relative_path = children_path.strip_prefix(file_data) - .expect("We got the children path from the `read_dir`, so it should be a child of the source path"); - let target_children_full_path = target_full_path.join(relative_path); - let metadata = fs::metadata(children_path) - .await - .map_err(|e| FileIOError::from((file_data.clone(), e)))?; - if metadata.is_dir() { - to_look.push((children_path.clone(), target_children_full_path.clone())); - let dir = NewEntry { - source: children_path.clone(), - dest: target_children_full_path, - }; - more_dirs.push(dir); - } else { - let file = NewEntry { - source: children_path.clone(), - dest: target_children_full_path, - }; - more_files.push(file); - } - } - - Ok::<_, JobError>((to_look, more_dirs, more_files)) - }) - .collect::>() - .try_join() - .await?; - - dirs.clear(); - unfolds - .into_iter() - .for_each(|(keep_looking, more_dirs, more_files)| { - dirs.extend(keep_looking); - unfolded_dirs.extend(more_dirs); - unfolded_files.extend(more_files); - }); - } - - Ok((unfolded_dirs, unfolded_files)) -} - -#[derive(Debug)] -struct NewEntry { - source: PathBuf, - dest: PathBuf, -} diff --git a/core/src/object/fs/old_cut.rs b/core/src/object/fs/old_cut.rs deleted file mode 100644 index 887d92b02..000000000 --- a/core/src/object/fs/old_cut.rs +++ /dev/null @@ -1,142 +0,0 @@ -use crate::{ - invalidate_query, - library::Library, - object::fs::{construct_target_filename, error::FileSystemJobsError}, - old_job::{ - CurrentStep, JobError, JobInitOutput, JobResult, JobRunErrors, JobStepOutput, StatefulJob, - WorkerContext, - }, -}; - -use sd_core_file_path_helper::push_location_relative_path; - -use sd_prisma::prisma::{file_path, location}; -use sd_utils::error::FileIOError; - -use std::{hash::Hash, path::PathBuf}; - -use serde::{Deserialize, Serialize}; -use serde_json::json; -use specta::Type; -use tokio::{fs, io}; -use tracing::{trace, warn}; - -use super::{fetch_source_and_target_location_paths, get_many_files_datas, FileData}; - -#[derive(Serialize, Deserialize, Hash, Type, Debug)] -pub struct OldFileCutterJobInit { - pub source_location_id: location::id::Type, - pub target_location_id: location::id::Type, - pub sources_file_path_ids: Vec, - pub target_location_relative_directory_path: PathBuf, -} - -#[derive(Serialize, Deserialize, Debug)] -pub struct OldFileCutterJobData { - full_target_directory_path: PathBuf, -} - -#[async_trait::async_trait] -impl StatefulJob for OldFileCutterJobInit { - type Data = OldFileCutterJobData; - type Step = FileData; - type RunMetadata = (); - - const NAME: &'static str = "file_cutter"; - - fn target_location(&self) -> location::id::Type { - self.target_location_id - } - - async fn init( - &self, - ctx: &WorkerContext, - data: &mut Option, - ) -> Result, JobError> { - let init = self; - let Library { db, .. } = &*ctx.library; - - let (sources_location_path, targets_location_path) = - fetch_source_and_target_location_paths( - db, - init.source_location_id, - init.target_location_id, - ) - .await?; - - let full_target_directory_path = push_location_relative_path( - targets_location_path, - &init.target_location_relative_directory_path, - ); - - *data = Some(OldFileCutterJobData { - full_target_directory_path, - }); - - let steps = - get_many_files_datas(db, &sources_location_path, &init.sources_file_path_ids).await?; - - Ok(steps.into()) - } - - async fn execute_step( - &self, - _: &WorkerContext, - CurrentStep { - step: file_data, .. - }: CurrentStep<'_, Self::Step>, - data: &Self::Data, - _: &Self::RunMetadata, - ) -> Result, JobError> { - let full_output = data - .full_target_directory_path - .join(construct_target_filename(file_data)?); - - if file_data.full_path == full_output { - // File is already here, do nothing - Ok(().into()) - } else { - match fs::metadata(&full_output).await { - Ok(_) => { - warn!( - output_path = %full_output.display(), - "Skipping as it would be overwritten;", - ); - - Ok(JobRunErrors(vec![FileSystemJobsError::WouldOverwrite( - full_output.into_boxed_path(), - ) - .to_string()]) - .into()) - } - Err(e) if e.kind() == io::ErrorKind::NotFound => { - trace!( - source = %file_data.full_path.display(), - target = %full_output.display(), - "Cutting source -> target;", - ); - - fs::rename(&file_data.full_path, &full_output) - .await - .map_err(|e| FileIOError::from((&file_data.full_path, e)))?; - - Ok(().into()) - } - - Err(e) => return Err(FileIOError::from((&full_output, e)).into()), - } - } - } - - async fn finalize( - &self, - ctx: &WorkerContext, - _data: &Option, - _run_metadata: &Self::RunMetadata, - ) -> JobResult { - let init = self; - invalidate_query!(ctx.library, "search.paths"); - - Ok(Some(json!({ "init": init }))) - } -} diff --git a/core/src/object/fs/old_delete.rs b/core/src/object/fs/old_delete.rs deleted file mode 100644 index 0b05a6103..000000000 --- a/core/src/object/fs/old_delete.rs +++ /dev/null @@ -1,122 +0,0 @@ -use crate::{ - invalidate_query, - library::Library, - location::get_location_path_from_location_id, - old_job::{ - CurrentStep, JobError, JobInitOutput, JobResult, JobStepOutput, StatefulJob, WorkerContext, - }, -}; - -use sd_prisma::{ - prisma::{file_path, location}, - prisma_sync, -}; -use sd_sync::OperationFactory; -use sd_utils::{db::maybe_missing, error::FileIOError}; - -use std::hash::Hash; - -use serde::{Deserialize, Serialize}; -use serde_json::json; -use specta::Type; -use tokio::{fs, io}; -use tracing::warn; - -use super::{error::FileSystemJobsError, get_many_files_datas, FileData}; - -#[derive(Serialize, Deserialize, Hash, Type, Debug)] -pub struct OldFileDeleterJobInit { - pub location_id: location::id::Type, - pub file_path_ids: Vec, -} - -#[async_trait::async_trait] -impl StatefulJob for OldFileDeleterJobInit { - type Data = (); - type Step = FileData; - type RunMetadata = (); - - const NAME: &'static str = "file_deleter"; - - fn target_location(&self) -> location::id::Type { - self.location_id - } - - async fn init( - &self, - ctx: &WorkerContext, - data: &mut Option, - ) -> Result, JobError> { - let init = self; - let Library { db, .. } = &*ctx.library; - - let steps = get_many_files_datas( - db, - get_location_path_from_location_id(db, init.location_id).await?, - &init.file_path_ids, - ) - .await - .map_err(FileSystemJobsError::from)?; - - // Must fill in the data, otherwise the job will not run - *data = Some(()); - - Ok(steps.into()) - } - - async fn execute_step( - &self, - ctx: &WorkerContext, - CurrentStep { step, .. }: CurrentStep<'_, Self::Step>, - _: &Self::Data, - _: &Self::RunMetadata, - ) -> Result, JobError> { - // need to handle stuff such as querying prisma for all paths of a file, and deleting all of those if requested (with a checkbox in the ui) - // maybe a files.countOccurrences/and or files.getPath(location_id, path_id) to show how many of these files would be deleted (and where?) - - let Library { db, sync, .. } = ctx.library.as_ref(); - - match if maybe_missing(step.file_path.is_dir, "file_path.is_dir")? { - fs::remove_dir_all(&step.full_path).await - } else { - fs::remove_file(&step.full_path).await - } { - Ok(()) => { /* Everything is awesome! */ } - Err(e) if e.kind() == io::ErrorKind::NotFound => { - warn!( - path = %step.full_path.display(), - "File not found in the file system, will remove from database;", - ); - - sync.write_op( - db, - sync.shared_delete(prisma_sync::file_path::SyncId { - pub_id: step.file_path.pub_id.clone(), - }), - db.file_path() - .delete(file_path::id::equals(step.file_path.id)), - ) - .await?; - } - Err(e) => { - return Err(JobError::from(FileIOError::from((&step.full_path, e)))); - } - } - - Ok(().into()) - } - - async fn finalize( - &self, - ctx: &WorkerContext, - _data: &Option, - _run_metadata: &Self::RunMetadata, - ) -> JobResult { - let init = self; - invalidate_query!(ctx.library, "search.paths"); - - // ctx.library.orphan_remover.invoke().await; - - Ok(Some(json!({ "init": init }))) - } -} diff --git a/core/src/object/fs/old_erase.rs b/core/src/object/fs/old_erase.rs deleted file mode 100644 index d04970590..000000000 --- a/core/src/object/fs/old_erase.rs +++ /dev/null @@ -1,205 +0,0 @@ -use crate::{ - invalidate_query, - library::Library, - location::get_location_path_from_location_id, - old_job::{ - CurrentStep, JobError, JobInitOutput, JobResult, JobRunMetadata, JobStepOutput, - StatefulJob, WorkerContext, - }, -}; - -use sd_core_file_path_helper::IsolatedFilePathData; - -use sd_prisma::prisma::{file_path, location}; -use sd_utils::{db::maybe_missing, error::FileIOError}; - -use std::{hash::Hash, path::PathBuf}; - -use futures::future::try_join_all; -use serde::{Deserialize, Serialize}; -use serde_with::{serde_as, DisplayFromStr}; -use specta::Type; -use tokio::{ - fs::{self, OpenOptions}, - io::AsyncWriteExt, -}; -use tracing::trace; - -use super::{ - error::FileSystemJobsError, get_file_data_from_isolated_file_path, get_many_files_datas, - FileData, -}; - -#[serde_as] -#[derive(Serialize, Deserialize, Hash, Type, Debug)] -pub struct OldFileEraserJobInit { - pub location_id: location::id::Type, - pub file_path_ids: Vec, - #[specta(type = String)] - #[serde_as(as = "DisplayFromStr")] - pub passes: usize, -} - -#[derive(Serialize, Deserialize, Debug)] -pub struct OldFileEraserJobData { - location_path: PathBuf, -} - -#[derive(Serialize, Deserialize, Default, Debug)] -pub struct FileEraserJobRunMetadata { - directories_to_remove: Vec, -} - -impl JobRunMetadata for FileEraserJobRunMetadata { - fn update(&mut self, new_data: Self) { - self.directories_to_remove - .extend(new_data.directories_to_remove); - } -} - -#[async_trait::async_trait] -impl StatefulJob for OldFileEraserJobInit { - type Data = OldFileEraserJobData; - type Step = FileData; - type RunMetadata = FileEraserJobRunMetadata; - - const NAME: &'static str = "file_eraser"; - - fn target_location(&self) -> location::id::Type { - self.location_id - } - - async fn init( - &self, - ctx: &WorkerContext, - data: &mut Option, - ) -> Result, JobError> { - let init = self; - let Library { db, .. } = &*ctx.library; - - let location_path = get_location_path_from_location_id(db, init.location_id) - .await - .map_err(FileSystemJobsError::from)?; - - let steps = get_many_files_datas(db, &location_path, &init.file_path_ids).await?; - - *data = Some(OldFileEraserJobData { location_path }); - - Ok((Default::default(), steps).into()) - } - - async fn execute_step( - &self, - ctx: &WorkerContext, - CurrentStep { step, .. }: CurrentStep<'_, Self::Step>, - data: &Self::Data, - _: &Self::RunMetadata, - ) -> Result, JobError> { - let init = self; - - // need to handle stuff such as querying prisma for all paths of a file, and deleting all of those if requested (with a checkbox in the ui) - // maybe a files.countOccurrences/and or files.getPath(location_id, path_id) to show how many of these files would be erased (and where?) - - let mut new_metadata = Self::RunMetadata::default(); - - if maybe_missing(step.file_path.is_dir, "file_path.is_dir")? { - let mut more_steps = Vec::new(); - - let mut dir = tokio::fs::read_dir(&step.full_path) - .await - .map_err(|e| FileIOError::from((&step.full_path, e)))?; - - while let Some(children_entry) = dir - .next_entry() - .await - .map_err(|e| FileIOError::from((&step.full_path, e)))? - { - let children_path = children_entry.path(); - - more_steps.push( - get_file_data_from_isolated_file_path( - &ctx.library.db, - &data.location_path, - &IsolatedFilePathData::new( - init.location_id, - &data.location_path, - &children_path, - children_entry - .metadata() - .await - .map_err(|e| FileIOError::from((&children_path, e)))? - .is_dir(), - ) - .map_err(FileSystemJobsError::from)?, - ) - .await?, - ); - } - new_metadata - .directories_to_remove - .push(step.full_path.clone()); - - Ok((more_steps, new_metadata).into()) - } else { - { - let mut file = OpenOptions::new() - .read(true) - .write(true) - .open(&step.full_path) - .await - .map_err(|e| FileIOError::from((&step.full_path, e)))?; - // let file_len = file - // .metadata() - // .await - // .map_err(|e| FileIOError::from((&step.full_path, e)))? - // .len(); - - trace!( - path = %step.full_path.display(), - passes = init.passes, - "Overwriting file;", - ); - - // TODO: File is only being truncated and not actually erased, - // we should provide a way for securely overwriting the file with random data - file.set_len(0) - .await - .map_err(|e| FileIOError::from((&step.full_path, e)))?; - file.flush() - .await - .map_err(|e| FileIOError::from((&step.full_path, e)))?; - } - - fs::remove_file(&step.full_path) - .await - .map_err(|e| FileIOError::from((&step.full_path, e)))?; - - Ok(None.into()) - } - } - - async fn finalize( - &self, - ctx: &WorkerContext, - _data: &Option, - run_metadata: &Self::RunMetadata, - ) -> JobResult { - let init = self; - try_join_all( - run_metadata - .directories_to_remove - .iter() - .cloned() - .map(|data| async { - fs::remove_dir_all(&data) - .await - .map_err(|e| FileIOError::from((data, e))) - }), - ) - .await?; - - invalidate_query!(ctx.library, "search.paths"); - - Ok(Some(serde_json::to_value(init)?)) - } -} diff --git a/core/src/object/fs/sync.rs b/core/src/object/fs/sync.rs deleted file mode 100644 index e69de29bb..000000000 diff --git a/core/src/object/media/old_thumbnail/clean_up.rs b/core/src/object/media/old_thumbnail/clean_up.rs deleted file mode 100644 index 0d65409be..000000000 --- a/core/src/object/media/old_thumbnail/clean_up.rs +++ /dev/null @@ -1,183 +0,0 @@ -use crate::library::LibraryId; - -use sd_prisma::prisma::{file_path, PrismaClient}; -use sd_utils::error::FileIOError; - -use std::{collections::HashSet, ffi::OsString, path::PathBuf, sync::Arc}; - -use futures_concurrency::future::Join; -use tokio::{fs, spawn}; -use tracing::{debug, error}; - -use super::{ThumbnailerError, EPHEMERAL_DIR, WEBP_EXTENSION}; - -// TODO(fogodev) Introduce a task using the new task system to clean up the thumbnails from time to time. - -pub(super) async fn process_ephemeral_clean_up( - thumbnails_directory: Arc, - existing_ephemeral_thumbs: HashSet, -) { - let ephemeral_thumbs_dir = thumbnails_directory.join(EPHEMERAL_DIR); - - spawn(async move { - let mut to_remove = vec![]; - - let mut read_ephemeral_thumbs_dir = fs::read_dir(&ephemeral_thumbs_dir) - .await - .map_err(|e| FileIOError::from((&ephemeral_thumbs_dir, e)))?; - - while let Some(shard_entry) = read_ephemeral_thumbs_dir - .next_entry() - .await - .map_err(|e| FileIOError::from((&ephemeral_thumbs_dir, e)))? - { - let shard_path = shard_entry.path(); - if shard_entry - .file_type() - .await - .map_err(|e| FileIOError::from((&shard_path, e)))? - .is_dir() - { - let mut read_shard_dir = fs::read_dir(&shard_path) - .await - .map_err(|e| FileIOError::from((&shard_path, e)))?; - - while let Some(thumb_entry) = read_shard_dir - .next_entry() - .await - .map_err(|e| FileIOError::from((&shard_path, e)))? - { - let thumb_path = thumb_entry.path(); - if thumb_path.extension() == Some(WEBP_EXTENSION.as_ref()) - && !existing_ephemeral_thumbs.contains(&thumb_entry.file_name()) - { - to_remove.push(async move { - debug!( - thumb_path = %thumb_path.display(), - "Removing stale ephemeral thumbnail;", - ); - - fs::remove_file(&thumb_path).await.map_err(|e| { - ThumbnailerError::FileIO(FileIOError::from((thumb_path, e))) - }) - }); - } - } - } - } - - Ok::<_, ThumbnailerError>(to_remove.join().await) - }) - .await - .map_or_else( - |e| error!(?e, "Join error on ephemeral clean up;",), - |fetching_res| { - fetching_res.map_or_else( - |e| error!(?e, "Error fetching ephemeral thumbs to be removed;"), - |remove_results| { - remove_results.into_iter().for_each(|remove_res| { - if let Err(e) = remove_res { - error!(?e, "Error on ephemeral clean up;"); - } - }) - }, - ) - }, - ) -} - -pub(super) async fn process_indexed_clean_up( - thumbnails_directory: Arc, - libraries_ids_and_databases: Vec<(LibraryId, Arc)>, -) { - libraries_ids_and_databases - .into_iter() - .map(|(library_id, db)| { - let library_thumbs_dir = thumbnails_directory.join(library_id.to_string()); - spawn(async move { - let existing_thumbs = db - .file_path() - .find_many(vec![file_path::cas_id::not(None)]) - .select(file_path::select!({ cas_id })) - .exec() - .await? - .into_iter() - .map(|file_path| { - OsString::from(format!( - "{}.webp", - file_path.cas_id.expect("we filtered right") - )) - }) - .collect::>(); - - let mut read_library_thumbs_dir = fs::read_dir(&library_thumbs_dir) - .await - .map_err(|e| FileIOError::from((&library_thumbs_dir, e)))?; - - let mut to_remove = vec![]; - - while let Some(shard_entry) = read_library_thumbs_dir - .next_entry() - .await - .map_err(|e| FileIOError::from((&library_thumbs_dir, e)))? - { - let shard_path = shard_entry.path(); - if shard_entry - .file_type() - .await - .map_err(|e| FileIOError::from((&shard_path, e)))? - .is_dir() - { - let mut read_shard_dir = fs::read_dir(&shard_path) - .await - .map_err(|e| FileIOError::from((&shard_path, e)))?; - - while let Some(thumb_entry) = read_shard_dir - .next_entry() - .await - .map_err(|e| FileIOError::from((&shard_path, e)))? - { - let thumb_path = thumb_entry.path(); - if thumb_path.extension() == Some(WEBP_EXTENSION.as_ref()) - && !existing_thumbs.contains(&thumb_entry.file_name()) - { - to_remove.push(async move { - debug!( - thumb_path = %thumb_path.display(), - "Removing stale indexed thumbnail;", - ); - - fs::remove_file(&thumb_path).await.map_err(|e| { - ThumbnailerError::FileIO(FileIOError::from((thumb_path, e))) - }) - }); - } - } - } - } - - Ok::<_, ThumbnailerError>(to_remove.join().await) - }) - }) - .collect::>() - .join() - .await - .into_iter() - .filter_map(|join_res| { - join_res - .map_err(|e| error!(?e, "Join error on indexed clean up;")) - .ok() - }) - .filter_map(|fetching_res| { - fetching_res - .map_err(|e| error!(?e, "Error fetching indexed thumbs to be removed;")) - .ok() - }) - .for_each(|remove_results| { - remove_results.into_iter().for_each(|remove_res| { - if let Err(e) = remove_res { - error!(?e, "Error on indexed clean up;"); - } - }) - }) -} diff --git a/core/src/object/media/old_thumbnail/directory.rs b/core/src/object/media/old_thumbnail/directory.rs deleted file mode 100644 index d6d00caca..000000000 --- a/core/src/object/media/old_thumbnail/directory.rs +++ /dev/null @@ -1,385 +0,0 @@ -use crate::{ - library::{Libraries, LibraryId}, - object::media::old_thumbnail::ONE_SEC, - util::version_manager::{Kind, ManagedVersion, VersionManager, VersionManagerError}, -}; - -use sd_prisma::prisma::{file_path, PrismaClient}; -use sd_utils::error::FileIOError; -use serde_repr::{Deserialize_repr, Serialize_repr}; - -use std::{ - collections::{HashMap, HashSet}, - path::{Path, PathBuf}, - sync::Arc, -}; - -use futures_concurrency::future::{Join, TryJoin}; -use int_enum::IntEnum; -use tokio::{ - fs, io, spawn, - time::{sleep, timeout}, -}; -use tracing::{debug, error, info, trace, warn}; - -use super::{ - get_shard_hex, ThumbnailerError, EPHEMERAL_DIR, THIRTY_SECS, THUMBNAIL_CACHE_DIR_NAME, - VERSION_FILE, WEBP_EXTENSION, -}; - -// TODO(fogodev): Move this logic to be used alongside the NodeConfig or other Node part to run at app startup - -#[derive( - IntEnum, Debug, Clone, Copy, Eq, PartialEq, strum::Display, Serialize_repr, Deserialize_repr, -)] -#[repr(u64)] -pub enum ThumbnailVersion { - V1 = 1, - V2 = 2, - V3 = 3, -} - -impl ManagedVersion for ThumbnailVersion { - const LATEST_VERSION: Self = Self::V3; - - const KIND: Kind = Kind::PlainText; - - type MigrationError = ThumbnailerError; - - fn from_latest_version() -> Option { - Some(Self::LATEST_VERSION) - } -} - -pub(super) async fn init_thumbnail_dir( - data_dir: impl AsRef, - libraries_manager: Arc, -) -> Result { - debug!("Initializing thumbnail directory"); - let thumbnails_directory = data_dir.as_ref().join(THUMBNAIL_CACHE_DIR_NAME); - - debug!(thumbnails_directory = %thumbnails_directory.display()); - - // create thumbnails base directory - fs::create_dir_all(&thumbnails_directory) - .await - .map_err(|e| FileIOError::from((&thumbnails_directory, e)))?; - - spawn({ - let thumbnails_directory = thumbnails_directory.clone(); - async move { - let Ok(databases) = timeout(THIRTY_SECS, async move { - loop { - let libraries = libraries_manager.get_all().await; - if !libraries.is_empty() { - break libraries - .into_iter() - .map(|library| (library.id, Arc::clone(&library.db))) - .collect::>(); - } - - sleep(ONE_SEC).await; - } - }) - .await - else { - warn!( - "Failed to get libraries after 30 seconds, thumbnailer migration will not work; \ - Ignore this warning if you don't created libraries yet." - ); - return; - }; - - if let Err(e) = process_migration(thumbnails_directory, databases).await { - error!(?e, "Failed to migrate thumbnails;"); - } - } - }); - - Ok(thumbnails_directory) -} - -async fn process_migration( - thumbnails_directory: impl AsRef, - databases: HashMap>, -) -> Result<(), ThumbnailerError> { - let thumbnails_directory = thumbnails_directory.as_ref(); - - // create all other directories, for each library and for ephemeral thumbnails - databases - .keys() - .map(|library_id| thumbnails_directory.join(library_id.to_string())) - .chain([thumbnails_directory.join(EPHEMERAL_DIR)]) - .map(|path| async move { - fs::create_dir_all(&path) - .await - .map_err(|e| FileIOError::from((&path, e))) - }) - .collect::>() - .join() - .await - .into_iter() - .collect::, _>>()?; - - VersionManager::::migrate_and_load( - thumbnails_directory.join(VERSION_FILE), - |current, next| { - let databases = &databases; - async move { - match (current, next) { - (ThumbnailVersion::V1, ThumbnailVersion::V2) => { - move_to_shards(thumbnails_directory).await - } - (ThumbnailVersion::V2, ThumbnailVersion::V3) => { - segregate_thumbnails_by_library(thumbnails_directory, databases).await - } - - _ => { - error!(current_version = ?current, "Thumbnail version is not handled;"); - - Err(VersionManagerError::UnexpectedMigration { - current_version: current.int_value(), - next_version: next.int_value(), - } - .into()) - } - } - } - }, - ) - .await - .map(|_| ()) -} - -/// This function moves all webp files in the thumbnail directory to their respective shard folders. -/// It is used to migrate from V1 to V2. -async fn move_to_shards(thumbnails_directory: impl AsRef) -> Result<(), ThumbnailerError> { - let thumbnails_directory = thumbnails_directory.as_ref(); - - let mut dir_entries = fs::read_dir(thumbnails_directory) - .await - .map_err(|source| FileIOError::from((thumbnails_directory, source)))?; - - let mut count = 0; - - while let Ok(Some(entry)) = dir_entries.next_entry().await { - if entry - .file_type() - .await - .map_err(|e| FileIOError::from((entry.path(), e)))? - .is_file() - { - let path = entry.path(); - if path.extension() == Some(WEBP_EXTENSION.as_ref()) { - let file_name = entry.file_name(); - - // we know they're cas_id's, so they're valid utf8 - let shard_folder = get_shard_hex(file_name.to_str().expect("Failed to parse UTF8")); - - let new_dir = thumbnails_directory.join(shard_folder); - fs::create_dir_all(&new_dir) - .await - .map_err(|source| FileIOError::from((new_dir.clone(), source)))?; - - let new_path = new_dir.join(file_name); - fs::rename(&path, &new_path) - .await - .map_err(|source| FileIOError::from((path.clone(), source)))?; - count += 1; - } - } - } - - info!(%count, "Moved webp files to their respective shard folders;"); - - Ok(()) -} - -async fn segregate_thumbnails_by_library( - thumbnails_directory: impl AsRef, - databases: &HashMap>, -) -> Result<(), ThumbnailerError> { - // We already created the library folders in init_thumbnail_dir, so we can just move the files - // to their respective folders - - let thumbnails_directory = thumbnails_directory.as_ref(); - - databases - .iter() - .map(|(library_id, db)| (*library_id, Arc::clone(db))) - .map(|(library_id, db)| { - let library_thumbs_dir = thumbnails_directory.join(library_id.to_string()); - let old_thumbs_dir = thumbnails_directory.to_path_buf(); - spawn(async move { - let mut shards_to_create = HashSet::new(); - - let to_move = db - .file_path() - .find_many(vec![file_path::cas_id::not(None)]) - .select(file_path::select!({ cas_id })) - .exec() - .await? - .into_iter() - .filter_map(|file_path| file_path.cas_id) - .map(|cas_id| { - let new_shard = get_shard_hex(&cas_id).to_string(); - let new_sharded_filename = format!("{new_shard}/{cas_id}.webp"); - let old_sharded_filename = format!("{}/{cas_id}.webp", &cas_id[0..2]); - - (new_shard, new_sharded_filename, old_sharded_filename) - }) - .map(|(new_shard, new_sharded_filename, old_sharded_filename)| { - let old = old_thumbs_dir.join(old_sharded_filename); - let new = library_thumbs_dir.join(new_sharded_filename); - let new_shard_dir = library_thumbs_dir.join(new_shard); - - shards_to_create.insert(new_shard_dir); - - async move { - trace!( - old_location = %old.display(), - new_location = %new.display(), - "Moving thumbnail from old location to new location;", - ); - - match fs::rename(&old, new).await { - Ok(_) => Ok(1), - Err(e) if e.kind() == io::ErrorKind::NotFound => { - // Thumbnail not found, it probably wasn't processed yet - Ok(0) - } - Err(e) => { - Err(ThumbnailerError::FileIO(FileIOError::from((old, e)))) - } - } - } - }) - .collect::>(); - - let shards_created_count = shards_to_create - .into_iter() - .map(|path| async move { - fs::create_dir_all(&path) - .await - .map_err(|e| FileIOError::from((path, e))) - }) - .collect::>() - .try_join() - .await? - .len(); - - let moved_count = to_move.try_join().await?.into_iter().sum::(); - - info!( - %shards_created_count, - %moved_count, - %library_id - "Created shards and moved thumbnails to library folder;", - ); - - Ok::<_, ThumbnailerError>(()) - }) - }) - .collect::>() - .try_join() - .await? - .into_iter() - .collect::>()?; - - // Now that we moved all files from all databases, everything else should be ephemeral thumbnails - // so we can just move all of them to the ephemeral directory - let ephemeral_thumbs_dir = thumbnails_directory.join(EPHEMERAL_DIR); - - let mut shards_to_create = HashSet::new(); - let mut to_move = vec![]; - - let mut read_thumbs_dir = fs::read_dir(thumbnails_directory) - .await - .map_err(|e| FileIOError::from((thumbnails_directory, e)))?; - - let mut empty_shards = vec![]; - - while let Some(shard_entry) = read_thumbs_dir - .next_entry() - .await - .map_err(|e| FileIOError::from((thumbnails_directory, e)))? - { - let old_shard_path = shard_entry.path(); - if shard_entry - .file_type() - .await - .map_err(|e| FileIOError::from((&old_shard_path, e)))? - .is_dir() - { - let mut read_shard_dir = fs::read_dir(&old_shard_path) - .await - .map_err(|e| FileIOError::from((&old_shard_path, e)))?; - - while let Some(thumb_entry) = read_shard_dir - .next_entry() - .await - .map_err(|e| FileIOError::from((&old_shard_path, e)))? - { - let thumb_path = thumb_entry.path(); - if thumb_path.extension() == Some(WEBP_EXTENSION.as_ref()) { - let thumb_filename = thumb_entry.file_name(); - - let mut new_ephemeral_shard = ephemeral_thumbs_dir.join(get_shard_hex( - thumb_filename.to_str().expect("cas_ids are utf-8"), - )); - - shards_to_create.insert(new_ephemeral_shard.clone()); - - new_ephemeral_shard.push(thumb_filename); - - to_move.push(async move { - trace!( - old_location = %thumb_path.display(), - new_location = %new_ephemeral_shard.display(), - "Moving thumbnail from old location to new location;" - ); - - fs::rename(&thumb_path, &new_ephemeral_shard) - .await - .map_err(|e| FileIOError::from((thumb_path, e))) - }); - } - } - - empty_shards.push(old_shard_path); - } - } - - shards_to_create - .into_iter() - .map(|path| async move { - fs::create_dir_all(&path) - .await - .map_err(|e| FileIOError::from((path, e))) - }) - .collect::>() - .try_join() - .await?; - - let moved_shard = to_move.try_join().await?.len(); - - info!(%moved_shards, "Moved shards to the ephemeral directory;"); - - empty_shards - .into_iter() - .filter_map(|path| { - path.file_name() - .map_or(false, |name| name.len() == 2) - .then_some(async move { - trace!(path = path.display(), "Removing empty shard directory;"); - fs::remove_dir(&path) - .await - .map_err(|e| FileIOError::from((path, e))) - }) - }) - .collect::>() - .try_join() - .await?; - - Ok(()) -} diff --git a/core/src/object/mod.rs b/core/src/object/mod.rs deleted file mode 100644 index e4de76f43..000000000 --- a/core/src/object/mod.rs +++ /dev/null @@ -1,3 +0,0 @@ -pub mod fs; -pub mod tag; -pub mod validation; diff --git a/core/src/object/old_orphan_remover.rs b/core/src/object/old_orphan_remover.rs deleted file mode 100644 index 25acf9852..000000000 --- a/core/src/object/old_orphan_remover.rs +++ /dev/null @@ -1,106 +0,0 @@ -use sd_prisma::prisma::{object, tag_on_object, PrismaClient}; - -use std::{sync::Arc, time::Duration}; - -use tokio::{ - select, - sync::mpsc, - time::{interval_at, Instant, MissedTickBehavior}, -}; -use tracing::{error, trace}; - -// TODO(fogodev): To be rewritten using new task system - -const TEN_SECONDS: Duration = Duration::from_secs(10); -const ONE_MINUTE: Duration = Duration::from_secs(60); - -// Actor that can be invoked to find and delete objects with no matching file paths -#[derive(Clone)] -#[allow(dead_code)] -pub struct OrphanRemoverActor { - tx: mpsc::Sender<()>, -} - -impl OrphanRemoverActor { - #[allow(dead_code)] - pub fn spawn(db: Arc) -> Self { - let (tx, mut rx) = mpsc::channel(4); - - tokio::spawn(async move { - let mut last_checked = Instant::now(); - - let mut check_interval = interval_at(Instant::now() + ONE_MINUTE, ONE_MINUTE); - check_interval.set_missed_tick_behavior(MissedTickBehavior::Skip); - - loop { - // Here we wait for a signal or for the tick interval to be reached - select! { - _ = check_interval.tick() => {} - signal = rx.recv() => { - if signal.is_none() { - break; - } - } - } - - // For any of them we process a clean up if a time since the last one already passed - if last_checked.elapsed() > TEN_SECONDS { - Self::process_clean_up(&db).await; - last_checked = Instant::now(); - } - } - }); - - Self { tx } - } - - #[allow(dead_code)] - pub async fn invoke(&self) { - self.tx.send(()).await.ok(); - } - - #[allow(dead_code)] - async fn process_clean_up(db: &PrismaClient) { - loop { - let Ok(objects_ids) = db - .object() - .find_many(vec![object::file_paths::none(vec![])]) - .take(512) - .select(object::select!({ id })) - .exec() - .await - .map(|objects| { - objects - .into_iter() - .map(|object| object.id) - .collect::>() - }) - .map_err(|e| error!(?e, "Failed to fetch orphaned objects;")) - else { - break; - }; - - if objects_ids.is_empty() { - break; - } - - trace!( - orphans_count = objects_ids.len(), - "Removing orphaned objects;" - ); - - if let Err(e) = db - ._batch(( - db.tag_on_object() - .delete_many(vec![tag_on_object::object_id::in_vec(objects_ids.clone())]), - db.object() - .delete_many(vec![object::id::in_vec(objects_ids)]), - )) - .await - { - error!(?e, "Failed to remove orphaned objects;"); - break; - } - } - } -} diff --git a/core/src/object/tag/mod.rs b/core/src/object/tag/mod.rs deleted file mode 100644 index 34b609a83..000000000 --- a/core/src/object/tag/mod.rs +++ /dev/null @@ -1,47 +0,0 @@ -use crate::library::Library; - -use sd_prisma::{prisma::tag, prisma_sync}; -use sd_sync::*; - -use chrono::Utc; -use serde::Deserialize; -use specta::Type; -use uuid::Uuid; - -pub mod seed; - -#[derive(Type, Deserialize, Clone)] -pub struct TagCreateArgs { - pub name: String, - pub color: String, -} - -impl TagCreateArgs { - pub async fn exec( - self, - Library { db, sync, .. }: &Library, - ) -> Result { - let pub_id = Uuid::now_v7().as_bytes().to_vec(); - - let (sync_params, db_params) = [ - sync_db_entry!(self.name, tag::name), - sync_db_entry!(self.color, tag::color), - sync_db_entry!(false, tag::is_hidden), - sync_db_entry!(Utc::now(), tag::date_created), - ] - .into_iter() - .unzip::<_, _, Vec<_>, Vec<_>>(); - - sync.write_op( - db, - sync.shared_create( - prisma_sync::tag::SyncId { - pub_id: pub_id.clone(), - }, - sync_params, - ), - db.tag().create(pub_id, db_params), - ) - .await - } -} diff --git a/core/src/object/tag/seed.rs b/core/src/object/tag/seed.rs deleted file mode 100644 index 41dbf8e9a..000000000 --- a/core/src/object/tag/seed.rs +++ /dev/null @@ -1,34 +0,0 @@ -use crate::library::Library; - -use super::TagCreateArgs; - -/// Seeds tags in a new library. -/// Shouldn't be called more than once! -pub async fn new_library(library: &Library) -> Result<(), sd_core_sync::Error> { - // remove type after tags are added - - let tags = [ - TagCreateArgs { - name: "Keepsafe".to_string(), - color: "#D9188E".to_string(), - }, - TagCreateArgs { - name: "Hidden".to_string(), - color: "#646278".to_string(), - }, - TagCreateArgs { - name: "Projects".to_string(), - color: "#42D097".to_string(), - }, - TagCreateArgs { - name: "Memes".to_string(), - color: "#A718D9".to_string(), - }, - ]; - - for tag in tags { - tag.exec(library).await?; - } - - Ok(()) -} diff --git a/core/src/object/validation/hash.rs b/core/src/object/validation/hash.rs deleted file mode 100644 index 9b8b4b8cd..000000000 --- a/core/src/object/validation/hash.rs +++ /dev/null @@ -1,25 +0,0 @@ -use std::path::Path; - -use blake3::Hasher; -use tokio::{ - fs::File, - io::{self, AsyncReadExt}, -}; - -const BLOCK_LEN: usize = 1048576; - -pub async fn file_checksum(path: impl AsRef) -> Result { - let mut reader = File::open(path).await?; - let mut context = Hasher::new(); - let mut buffer = vec![0; BLOCK_LEN].into_boxed_slice(); - loop { - let read_count = reader.read(&mut buffer).await?; - context.update(&buffer[..read_count]); - if read_count != BLOCK_LEN { - break; - } - } - let hex = context.finalize().to_hex(); - - Ok(hex.to_string()) -} diff --git a/core/src/object/validation/mod.rs b/core/src/object/validation/mod.rs deleted file mode 100644 index cda0b12c6..000000000 --- a/core/src/object/validation/mod.rs +++ /dev/null @@ -1,23 +0,0 @@ -use sd_core_file_path_helper::FilePathError; -use sd_utils::error::FileIOError; - -use std::path::Path; - -use thiserror::Error; - -pub mod hash; -pub mod old_validator_job; - -#[derive(Error, Debug)] -pub enum ValidatorError { - #[error("sub path not found: ", .0.display())] - SubPathNotFound(Box), - - // Internal errors - #[error("database error: {0}")] - Database(#[from] prisma_client_rust::QueryError), - #[error(transparent)] - FilePath(#[from] FilePathError), - #[error(transparent)] - FileIO(#[from] FileIOError), -} diff --git a/core/src/object/validation/old_validator_job.rs b/core/src/object/validation/old_validator_job.rs deleted file mode 100644 index 7ddd42938..000000000 --- a/core/src/object/validation/old_validator_job.rs +++ /dev/null @@ -1,203 +0,0 @@ -use crate::{ - library::Library, - old_job::{ - CurrentStep, JobError, JobInitOutput, JobResult, JobStepOutput, StatefulJob, WorkerContext, - }, -}; - -use sd_core_file_path_helper::{ - ensure_file_path_exists, ensure_sub_path_is_directory, ensure_sub_path_is_in_location, - IsolatedFilePathData, -}; -use sd_core_prisma_helpers::file_path_for_object_validator; - -use sd_prisma::{ - prisma::{file_path, location}, - prisma_sync, -}; -use sd_sync::{sync_db_entry, OperationFactory}; -use sd_utils::{db::maybe_missing, error::FileIOError}; - -use std::{ - hash::{Hash, Hasher}, - path::{Path, PathBuf}, -}; - -use serde::{Deserialize, Serialize}; -use serde_json::json; -use tracing::info; - -use super::{hash::file_checksum, ValidatorError}; - -#[derive(Serialize, Deserialize, Debug)] -pub struct OldObjectValidatorJobData { - pub location_path: PathBuf, - pub task_count: usize, -} - -// The validator can -#[derive(Serialize, Deserialize, Debug)] -pub struct OldObjectValidatorJobInit { - pub location: location::Data, - pub sub_path: Option, -} - -impl Hash for OldObjectValidatorJobInit { - fn hash(&self, state: &mut H) { - self.location.id.hash(state); - if let Some(ref sub_path) = self.sub_path { - sub_path.hash(state); - } - } -} - -// The Validator is able to: -// - generate a full byte checksum for Objects in a Location -// - generate checksums for all Objects missing without one -// - compare two objects and return true if they are the same -#[async_trait::async_trait] -impl StatefulJob for OldObjectValidatorJobInit { - type Data = OldObjectValidatorJobData; - type Step = file_path_for_object_validator::Data; - type RunMetadata = (); - - const NAME: &'static str = "object_validator"; - - fn target_location(&self) -> location::id::Type { - self.location.id - } - - async fn init( - &self, - ctx: &WorkerContext, - data: &mut Option, - ) -> Result, JobError> { - let init = self; - let Library { db, .. } = &*ctx.library; - - let location_id = init.location.id; - - let location_path = - maybe_missing(&init.location.path, "location.path").map(PathBuf::from)?; - - let maybe_sub_iso_file_path = match &init.sub_path { - Some(sub_path) if sub_path != Path::new("") => { - let full_path = ensure_sub_path_is_in_location(&location_path, sub_path) - .await - .map_err(ValidatorError::from)?; - ensure_sub_path_is_directory(&location_path, sub_path) - .await - .map_err(ValidatorError::from)?; - - let sub_iso_file_path = - IsolatedFilePathData::new(location_id, &location_path, &full_path, true) - .map_err(ValidatorError::from)?; - - ensure_file_path_exists( - sub_path, - &sub_iso_file_path, - db, - ValidatorError::SubPathNotFound, - ) - .await?; - - Some(sub_iso_file_path) - } - _ => None, - }; - - let steps = db - .file_path() - .find_many(sd_utils::chain_optional_iter( - [ - file_path::location_id::equals(Some(init.location.id)), - file_path::is_dir::equals(Some(false)), - file_path::integrity_checksum::equals(None), - ], - [maybe_sub_iso_file_path.and_then(|iso_sub_path| { - iso_sub_path - .materialized_path_for_children() - .map(file_path::materialized_path::starts_with) - })], - )) - .select(file_path_for_object_validator::select()) - .exec() - .await?; - - *data = Some(OldObjectValidatorJobData { - location_path, - task_count: steps.len(), - }); - - Ok(steps.into()) - } - - async fn execute_step( - &self, - ctx: &WorkerContext, - CurrentStep { - step: file_path, .. - }: CurrentStep<'_, Self::Step>, - data: &Self::Data, - _: &Self::RunMetadata, - ) -> Result, JobError> { - let init = self; - let Library { db, sync, .. } = &*ctx.library; - - // this is to skip files that already have checksums - // i'm unsure what the desired behavior is in this case - // we can also compare old and new checksums here - // This if is just to make sure, we already queried objects where integrity_checksum is null - if file_path.integrity_checksum.is_none() { - let full_path = data.location_path.join(IsolatedFilePathData::try_from(( - init.location.id, - file_path, - ))?); - let checksum = file_checksum(&full_path) - .await - .map_err(|e| ValidatorError::FileIO(FileIOError::from((full_path, e))))?; - - let (sync_param, db_param) = sync_db_entry!(checksum, file_path::integrity_checksum); - - sync.write_op( - db, - sync.shared_update( - prisma_sync::file_path::SyncId { - pub_id: file_path.pub_id.clone(), - }, - [sync_param], - ), - db.file_path() - .update( - file_path::pub_id::equals(file_path.pub_id.clone()), - vec![db_param], - ) - .select(file_path::select!({ id })), - ) - .await?; - } - - Ok(().into()) - } - - async fn finalize( - &self, - _: &WorkerContext, - data: &Option, - _run_metadata: &Self::RunMetadata, - ) -> JobResult { - let init = self; - let data = data - .as_ref() - .expect("critical error: missing data on job state"); - - info!( - location_path = %data.location_path.display(), - sub_path = ?init.sub_path.as_ref().map(|p| p.display()), - task_count = data.task_count, - "finalizing validator job;", - ); - - Ok(Some(json!({ "init": init }))) - } -} diff --git a/core/src/old_job/error.rs b/core/src/old_job/error.rs deleted file mode 100644 index a629fb88d..000000000 --- a/core/src/old_job/error.rs +++ /dev/null @@ -1,119 +0,0 @@ -use crate::{ - location::{/*indexer::IndexerError,*/ LocationError}, - object::{ - fs::error::FileSystemJobsError, /*media::old_media_processor::MediaProcessorError,*/ - /*old_file_identifier::FileIdentifierJobError,*/ validation::ValidatorError, - }, -}; - -// use sd_crypto::Error as CryptoError; -use sd_utils::{db::MissingFieldError, error::FileIOError}; - -use std::time::Duration; - -use prisma_client_rust::QueryError; -use rmp_serde::{decode::Error as DecodeError, encode::Error as EncodeError}; -use thiserror::Error; -use tokio::sync::oneshot; -use uuid::Uuid; - -#[derive(Error, Debug)] -pub enum JobError { - // General errors - #[error("database error: {0}")] - Database(#[from] QueryError), - #[error("Failed to join Tokio spawn blocking: {0}")] - JoinTask(#[from] tokio::task::JoinError), - #[error("job state encode error: {0}")] - StateEncode(#[from] EncodeError), - #[error("job state decode error: {0}")] - StateDecode(#[from] DecodeError), - #[error("job metadata serialization error: {0}")] - MetadataSerialization(#[from] serde_json::Error), - #[error("tried to resume a job with unknown name: job ")] - UnknownJobName(Uuid, String), - #[error( - "Tried to resume a job that doesn't have saved state data: job " - )] - MissingJobDataState(Uuid, String), - #[error("missing report field: job ")] - MissingReport { id: Uuid, name: String }, - #[error("missing some job data: '{value}'")] - MissingData { value: String }, - #[error("invalid job status integer: {0}")] - InvalidJobStatusInt(i32), - #[error(transparent)] - FileIO(#[from] FileIOError), - #[error("Location error: {0}")] - Location(#[from] LocationError), - #[error("missing-field: {0}")] - MissingField(#[from] MissingFieldError), - #[error("item of type '{0}' with id '{1}' is missing from the db")] - MissingFromDb(&'static str, String), - #[error("job timed out after {0:?} without updates")] - Timeout(Duration), - #[error("critical job error: {0}")] - Critical(&'static str), - #[error(transparent)] - Sync(#[from] sd_core_sync::Error), - - // Specific job errors - #[error(transparent)] - Validator(#[from] ValidatorError), - #[error(transparent)] - FileSystemJobsError(#[from] FileSystemJobsError), - // #[error(transparent)] - // CryptoError(#[from] CryptoError), - - // Not errors - #[error("job had a early finish: ")] - EarlyFinish { name: String, reason: String }, - #[error("data needed for job execution not found: job ")] - JobDataNotFound(String), - #[error("job paused")] - Paused(Vec, oneshot::Sender<()>), - #[error("job canceled")] - Canceled(oneshot::Sender<()>), -} - -#[derive(Error, Debug)] -pub enum JobManagerError { - #[error("Tried to dispatch a job that is already running: Job ")] - AlreadyRunningJob { name: &'static str, hash: u64 }, - - #[error("Failed to fetch job data from database: {0}")] - Database(#[from] prisma_client_rust::QueryError), - - #[error("job not found: {0}")] - NotFound(Uuid), - - #[error("missing-field: {0}")] - MissingField(#[from] MissingFieldError), -} - -impl From for rspc::Error { - fn from(value: JobManagerError) -> Self { - match value { - JobManagerError::AlreadyRunningJob { .. } => Self::with_cause( - rspc::ErrorCode::BadRequest, - "Tried to spawn a job that is already running!".to_string(), - value, - ), - JobManagerError::Database(_) => Self::with_cause( - rspc::ErrorCode::InternalServerError, - "Error accessing the database".to_string(), - value, - ), - JobManagerError::NotFound(_) => Self::with_cause( - rspc::ErrorCode::NotFound, - "Job not found".to_string(), - value, - ), - JobManagerError::MissingField(_) => Self::with_cause( - rspc::ErrorCode::InternalServerError, - "Missing field".to_string(), - value, - ), - } - } -} diff --git a/core/src/old_job/manager.rs b/core/src/old_job/manager.rs deleted file mode 100644 index c9e5cc892..000000000 --- a/core/src/old_job/manager.rs +++ /dev/null @@ -1,416 +0,0 @@ -use crate::{ - library::Library, - object::{ - fs::{ - old_copy::OldFileCopierJobInit, old_cut::OldFileCutterJobInit, - old_delete::OldFileDeleterJobInit, old_erase::OldFileEraserJobInit, - }, - validation::old_validator_job::OldObjectValidatorJobInit, - }, - old_job::{worker::Worker, DynJob, JobError, OldJob}, - Node, -}; - -use sd_prisma::prisma::job; - -use std::{ - collections::{HashMap, HashSet, VecDeque}, - sync::Arc, -}; - -use futures::future::join_all; -use prisma_client_rust::operator::or; -use tokio::sync::{mpsc, oneshot, RwLock}; -use tracing::{debug, error, info, instrument, warn}; -use uuid::Uuid; - -use super::{JobIdentity, JobManagerError, JobStatus, OldJobReport, StatefulJob}; - -const MAX_WORKERS: usize = 5; - -pub enum JobManagerEvent { - IngestJob(Arc, Box), - Shutdown(oneshot::Sender<()>, Arc), -} - -#[must_use = "'job::manager::Actor::start' must be called to start the actor"] -pub struct Actor { - jobs: Arc, - internal_receiver: mpsc::UnboundedReceiver, -} - -impl Actor { - pub fn start(mut self, node: Arc) { - tokio::spawn(async move { - // FIXME: if this task crashes, the entire application is unusable - while let Some(event) = self.internal_receiver.recv().await { - match event { - JobManagerEvent::IngestJob(library, job) => { - self.jobs.clone().dispatch(&node, &library, job).await - } - // When the app shuts down, we need to gracefully shutdown all - // active workers and preserve their state - JobManagerEvent::Shutdown(signal_tx, this) => { - info!("Shutting down job manager"); - let running_workers = this.running_workers.read().await; - join_all(running_workers.values().map(|worker| worker.shutdown())).await; - - signal_tx.send(()).ok(); - } - } - } - }); - } -} - -/// JobManager handles queueing and executing jobs using the [`DynJob`] -/// Handling persisting JobReports to the database, pause/resuming -pub struct OldJobs { - current_jobs_hashes: RwLock>, - job_queue: RwLock>>, - running_workers: RwLock>, - internal_sender: mpsc::UnboundedSender, -} - -impl OldJobs { - /// Initializes the JobManager and spawns the internal event loop to listen for ingest. - pub fn new() -> (Arc, Actor) { - // allow the job manager to control its workers - let (internal_sender, internal_receiver) = mpsc::unbounded_channel(); - let this = Arc::new(Self { - current_jobs_hashes: RwLock::new(HashSet::new()), - job_queue: RwLock::new(VecDeque::new()), - running_workers: RwLock::new(HashMap::new()), - internal_sender, - }); - - ( - this.clone(), - Actor { - jobs: this, - internal_receiver, - }, - ) - } - - #[instrument( - skip_all, - fields(library_id = %library.id, job_name = %job.name(), job_hash = %job.hash()), - err, - )] - /// Ingests a new job and dispatches it if possible, queues it otherwise. - pub async fn ingest( - self: Arc, - node: &Arc, - library: &Arc, - job: Box>, - ) -> Result<(), JobManagerError> { - let job_hash = job.hash(); - - if self.current_jobs_hashes.read().await.contains(&job_hash) { - return Err(JobManagerError::AlreadyRunningJob { - name: job.name(), - hash: job_hash, - }); - } - - debug!("Ingesting job;"); - - self.current_jobs_hashes.write().await.insert(job_hash); - self.dispatch(node, library, job).await; - Ok(()) - } - - #[instrument( - skip_all, - fields(library_id = %library.id, job_name = %job.name(), job_hash = %job.hash()), - )] - /// Dispatches a job to a worker if under MAX_WORKERS limit, queues it otherwise. - async fn dispatch( - self: Arc, - node: &Arc, - library: &Arc, - mut job: Box, - ) { - let mut running_workers = self.running_workers.write().await; - let mut job_report = job - .report_mut() - .take() - .expect("critical error: missing job on worker"); - - if running_workers.len() < MAX_WORKERS { - info!("Running job"); - - let worker_id = job_report.parent_id.unwrap_or(job_report.id); - - Worker::new( - worker_id, - job, - job_report, - library.clone(), - node.clone(), - self.clone(), - ) - .await - .map_or_else( - |e| { - error!(?e, "Error spawning worker;"); - }, - |worker| { - running_workers.insert(worker_id, worker); - }, - ); - } else { - debug!("Queueing job"); - if let Err(e) = job_report.create(library).await { - // It's alright to just log here, as will try to create the report on run if it wasn't created before - error!(?e, "Error creating job report;"); - } - - // Put the report back, or it will be lost forever - *job.report_mut() = Some(job_report); - - self.job_queue.write().await.push_back(job); - } - } - - pub async fn complete( - self: Arc, - library: &Arc, - worker_id: Uuid, - job_hash: u64, - next_job: Option>, - ) { - // remove worker from running workers and from current jobs hashes - self.current_jobs_hashes.write().await.remove(&job_hash); - self.running_workers.write().await.remove(&worker_id); - // continue queue - let job = if next_job.is_some() { - next_job - } else { - self.job_queue.write().await.pop_front() - }; - - if let Some(job) = job { - // We can't directly execute `self.ingest` here because it would cause an async cycle. - self.internal_sender - .send(JobManagerEvent::IngestJob(library.clone(), job)) - .unwrap_or_else(|_| { - error!("Failed to ingest job!"); - }); - } - } - - /// Shutdown the job manager, signaled by core on shutdown. - pub async fn shutdown(self: &Arc) { - let (tx, rx) = oneshot::channel(); - self.internal_sender - .send(JobManagerEvent::Shutdown(tx, self.clone())) - .unwrap_or_else(|_| { - error!("Failed to send shutdown event to job manager!"); - }); - - rx.await.unwrap_or_else(|_| { - error!("Failed to receive shutdown event response from job manager!"); - }); - } - - #[instrument(skip(self))] - /// Pause a specific job. - pub async fn pause(&self, job_id: Uuid) -> Result<(), JobManagerError> { - // Look up the worker for the given job ID. - if let Some(worker) = self.running_workers.read().await.get(&job_id) { - debug!(report = ?worker.report(), "Pausing job;"); - - // Set the pause signal in the worker. - worker.pause().await; - - Ok(()) - } else { - Err(JobManagerError::NotFound(job_id)) - } - } - /// Resume a specific job. - pub async fn resume(&self, job_id: Uuid) -> Result<(), JobManagerError> { - // Look up the worker for the given job ID. - if let Some(worker) = self.running_workers.read().await.get(&job_id) { - debug!(report = ?worker.report(), "Resuming job;"); - - // Set the pause signal in the worker. - worker.resume().await; - - Ok(()) - } else { - Err(JobManagerError::NotFound(job_id)) - } - } - - /// Cancel a specific job. - pub async fn cancel(&self, job_id: Uuid) -> Result<(), JobManagerError> { - // Look up the worker for the given job ID. - if let Some(worker) = self.running_workers.read().await.get(&job_id) { - debug!(report = ?worker.report(), "Canceling job;"); - - // Set the cancel signal in the worker. - worker.cancel().await; - - Ok(()) - } else { - Err(JobManagerError::NotFound(job_id)) - } - } - - /// This is called at startup to resume all paused jobs or jobs that were running - /// when the core was shut down. - /// - It will resume jobs that contain data and cancel jobs that do not. - /// - Prevents jobs from being stuck in a paused/running state - pub async fn cold_resume( - self: Arc, - node: &Arc, - library: &Arc, - ) -> Result<(), JobManagerError> { - // Include the Queued status in the initial find condition - let find_condition = vec![or(vec![ - job::status::equals(Some(JobStatus::Paused as i32)), - job::status::equals(Some(JobStatus::Running as i32)), - job::status::equals(Some(JobStatus::Queued as i32)), - ])]; - - let all_jobs = library - .db - .job() - .find_many(find_condition) - .exec() - .await? - .into_iter() - .map(OldJobReport::try_from); - - for job in all_jobs { - let job = job?; - - match initialize_resumable_job(job.clone(), None) { - Ok(resumable_job) => { - info!(%job.name, %job.id, "Resuming job;"); - Arc::clone(&self) - .dispatch(node, library, resumable_job) - .await; - } - Err(JobError::UnknownJobName(_, job_name)) - if matches!( - job_name.as_str(), - "indexer" | "file_identifier" | "media_processor" - ) => - { - debug!(%job_name, "Moved to new job system"); - } - Err(e) => { - warn!( - %job.name, - %job.id, - ?e, - "Failed to initialize job;", - ); - - info!(%job.name, %job.id, "Cancelling job;"); - - library - .db - .job() - .update( - job::id::equals(job.id.as_bytes().to_vec()), - vec![job::status::set(Some(JobStatus::Canceled as i32))], - ) - .select(job::select!({ id })) - .exec() - .await?; - } - } - } - Ok(()) - } - - // get all active jobs, including paused jobs organized by job id - pub async fn get_active_reports_with_id(&self) -> HashMap { - self.running_workers - .read() - .await - .values() - .map(|worker| { - let report = worker.report(); - (report.id, report) - }) - .collect() - } - - // get all running jobs, excluding paused jobs organized by action - pub async fn get_running_reports(&self) -> HashMap { - self.running_workers - .read() - .await - .values() - .filter(|&worker| !worker.is_paused()) - .map(|worker| { - let report = worker.report(); - (report.get_meta().0, report) - }) - .collect() - } - - /// Check if the manager currently has some active workers. - pub async fn has_active_workers(&self, library_id: Uuid) -> bool { - self.running_workers - .read() - .await - .values() - .any(|worker| worker.library_id == library_id && !worker.is_paused()) - } - - pub async fn has_job_running(&self, predicate: impl Fn(JobIdentity) -> bool) -> bool { - for worker in self.running_workers.read().await.values() { - if worker.who_am_i().await.map(&predicate).unwrap_or(false) { - return true; - } - } - false - } -} - -#[macro_use] -mod macros { - macro_rules! dispatch_call_to_job_by_name { - ($job_name:expr, T -> $call:expr, default = $default:block, jobs = [ $($job:ty),+ $(,)?]) => {{ - match $job_name { - $(<$job as $crate::old_job::StatefulJob>::NAME => { - type T = $job; - $call - },)+ - _ => $default - } - }}; - } -} -/// This function is used to initialize a DynJob from a job report. -fn initialize_resumable_job( - job_report: OldJobReport, - next_jobs: Option>>, -) -> Result, JobError> { - dispatch_call_to_job_by_name!( - job_report.name.as_str(), - T -> OldJob::::new_from_report(job_report, next_jobs), - default = { - error!( - %job_report.name, - %job_report.id, - "Unknown job type;", - ); - Err(JobError::UnknownJobName(job_report.id, job_report.name)) - }, - jobs = [ - OldObjectValidatorJobInit, - OldFileCutterJobInit, - OldFileCopierJobInit, - OldFileDeleterJobInit, - OldFileEraserJobInit, - ] - ) -} diff --git a/core/src/old_job/mod.rs b/core/src/old_job/mod.rs deleted file mode 100644 index 01edd1a47..000000000 --- a/core/src/old_job/mod.rs +++ /dev/null @@ -1,1148 +0,0 @@ -use crate::{library::Library, Node}; - -use sd_prisma::prisma::location; - -use std::{ - collections::{hash_map::DefaultHasher, VecDeque}, - fmt, - hash::{Hash, Hasher}, - mem, - pin::pin, - sync::Arc, - time::Instant, -}; - -use async_channel as chan; -use futures::stream::{self, StreamExt}; -use futures_concurrency::stream::Merge; -use serde::{de::DeserializeOwned, Deserialize, Serialize}; -use tokio::{ - spawn, - task::{JoinError, JoinHandle}, -}; -use tracing::{debug, error, info, instrument, trace, warn}; -use uuid::Uuid; - -mod error; -mod manager; -mod report; -mod worker; - -pub use error::*; -pub use manager::*; -pub use report::*; -pub use worker::*; - -pub type JobResult = Result; -pub type JobMetadata = Option; - -#[derive(Debug)] -pub struct JobIdentity { - pub id: Uuid, - pub name: &'static str, - pub target_location: location::id::Type, - pub status: JobStatus, -} - -#[derive(Debug, Default)] -pub struct JobRunErrors(pub Vec); - -impl JobRunErrors { - pub fn is_empty(&self) -> bool { - self.0.is_empty() - } -} - -impl> From for JobRunErrors { - fn from(errors: I) -> Self { - Self(errors.into_iter().collect()) - } -} - -impl fmt::Display for JobRunErrors { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{}", self.0.join("\n")) - } -} - -pub struct JobRunOutput { - pub metadata: JobMetadata, - pub errors: JobRunErrors, - pub next_job: Option>, -} - -pub trait JobRunMetadata: - Default + Serialize + DeserializeOwned + Send + Sync + fmt::Debug -{ - fn update(&mut self, new_data: Self); -} - -impl JobRunMetadata for () { - fn update(&mut self, _new_data: Self) {} -} - -#[async_trait::async_trait] -pub trait StatefulJob: - Serialize + DeserializeOwned + Hash + fmt::Debug + Send + Sync + Sized + 'static -{ - type Data: Serialize + DeserializeOwned + Send + Sync + fmt::Debug; - type Step: Serialize + DeserializeOwned + Send + Sync + fmt::Debug; - type RunMetadata: JobRunMetadata; - - /// The name of the job is a unique human readable identifier for the job. - const NAME: &'static str; - const IS_BACKGROUND: bool = false; - const IS_BATCHED: bool = false; - - /// initialize the steps for the job - async fn init( - &self, - ctx: &WorkerContext, - data: &mut Option, - ) -> Result, JobError>; - - /// The location id where this job will act upon - fn target_location(&self) -> location::id::Type; - - /// is called for each step in the job. These steps are created in the `Self::init` method. - async fn execute_step( - &self, - ctx: &WorkerContext, - step: CurrentStep<'_, Self::Step>, - data: &Self::Data, - run_metadata: &Self::RunMetadata, - ) -> Result, JobError>; - - /// is called after all steps have been executed - async fn finalize( - &self, - ctx: &WorkerContext, - data: &Option, - run_metadata: &Self::RunMetadata, - ) -> JobResult; - - fn hash(&self) -> u64 { - let mut s = DefaultHasher::new(); - Self::NAME.hash(&mut s); - ::hash(self, &mut s); - s.finish() - } -} - -#[async_trait::async_trait] -pub trait DynJob: Send + Sync { - fn id(&self) -> Uuid; - fn parent_id(&self) -> Option; - fn report(&self) -> &Option; - fn report_mut(&mut self) -> &mut Option; - fn name(&self) -> &'static str; - async fn run( - &mut self, - ctx: WorkerContext, - commands_rx: chan::Receiver, - ) -> Result; - fn hash(&self) -> u64; - fn set_next_jobs(&mut self, next_jobs: VecDeque>); - fn serialize_state(&self) -> Result, JobError>; - async fn register_children(&mut self, library: &Library) -> Result<(), JobError>; - async fn pause_children(&mut self, library: &Library) -> Result<(), JobError>; - async fn cancel_children(&mut self, library: &Library) -> Result<(), JobError>; -} - -pub struct OldJob { - id: Uuid, - hash: u64, - report: Option, - state: Option>, - next_jobs: VecDeque>, -} - -impl OldJob { - pub fn new(init: SJob) -> Box { - let id = Uuid::new_v4(); - Box::new(OldJob:: { - id, - hash: ::hash(&init), - report: Some(JobReportBuilder::new(id, SJob::NAME.to_string()).build()), - state: Some(JobState { - init, - data: None, - steps: VecDeque::new(), - step_number: 0, - run_metadata: Default::default(), - }), - next_jobs: VecDeque::new(), - }) - } - - // this function returns an ingestible job instance from a job report - pub fn new_from_report( - mut report: OldJobReport, - next_jobs: Option>>, - ) -> Result, JobError> { - let state = rmp_serde::from_slice::>( - &report - .data - .take() - .ok_or_else(|| JobError::MissingJobDataState(report.id, report.name.clone()))?, - )?; - - Ok(Box::new(Self { - id: report.id, - hash: ::hash(&state.init), - state: Some(state), - report: Some(report), - next_jobs: next_jobs.unwrap_or_default(), - })) - } - - pub async fn spawn( - self, - node: &Arc, - library: &Arc, - ) -> Result<(), JobManagerError> { - node.old_jobs - .clone() - .ingest(node, library, Box::new(self)) - .await - } -} - -#[derive(Serialize)] -pub struct JobState { - pub init: Job, - pub data: Option, - pub steps: VecDeque, - pub step_number: usize, - pub run_metadata: Job::RunMetadata, -} - -impl<'de, Job: StatefulJob> Deserialize<'de> for JobState { - fn deserialize(deserializer: D) -> Result - where - D: serde::Deserializer<'de>, - { - as Deserialize<'de>>::deserialize::(deserializer).map(|raw| { - JobState { - init: raw.init, - data: raw.data, - steps: raw.steps, - step_number: raw.step_number, - run_metadata: raw.run_metadata, - } - }) - } -} - -/// This is a workaround for a serde bug. -/// Both these generics on this type should point to the same type. -/// -/// https://github.com/serde-rs/serde/issues/2418 -/// https://github.com/rust-lang/rust/issues/34979 -#[derive(Serialize, Deserialize)] -struct JobStateRaw -where - Job: StatefulJob, -{ - pub init: JobInit, - pub data: Option, - pub steps: VecDeque, - pub step_number: usize, - pub run_metadata: Job::RunMetadata, -} - -pub struct JobInitOutput { - run_metadata: RunMetadata, - steps: VecDeque, - errors: JobRunErrors, -} - -impl From<(RunMetadata, Vec)> for JobInitOutput { - fn from((run_metadata, steps): (RunMetadata, Vec)) -> Self { - Self { - run_metadata, - steps: VecDeque::from(steps), - errors: Default::default(), - } - } -} - -impl From> for JobInitOutput -where - RunMetadata: Default, -{ - fn from(steps: Vec) -> Self { - Self { - run_metadata: RunMetadata::default(), - steps: VecDeque::from(steps), - errors: Default::default(), - } - } -} - -impl From<(RunMetadata, Vec, JobRunErrors)> - for JobInitOutput -{ - fn from((run_metadata, steps, errors): (RunMetadata, Vec, JobRunErrors)) -> Self { - Self { - run_metadata, - steps: VecDeque::from(steps), - errors, - } - } -} - -pub struct CurrentStep<'step, Step> { - pub step: &'step Step, - pub step_number: usize, -} - -pub struct JobStepOutput { - maybe_more_steps: Option>, - maybe_more_metadata: Option, - errors: JobRunErrors, -} - -impl From> for JobStepOutput { - fn from(more_steps: Vec) -> Self { - Self { - maybe_more_steps: Some(more_steps), - maybe_more_metadata: None, - errors: Default::default(), - } - } -} - -impl From for JobStepOutput { - fn from(more_metadata: RunMetadata) -> Self { - Self { - maybe_more_steps: None, - maybe_more_metadata: Some(more_metadata), - errors: Default::default(), - } - } -} - -impl From for JobStepOutput { - fn from(errors: JobRunErrors) -> Self { - Self { - maybe_more_steps: None, - maybe_more_metadata: None, - errors, - } - } -} - -impl From<(Vec, RunMetadata)> - for JobStepOutput -{ - fn from((more_steps, more_metadata): (Vec, RunMetadata)) -> Self { - Self { - maybe_more_steps: Some(more_steps), - maybe_more_metadata: Some(more_metadata), - errors: Default::default(), - } - } -} - -impl From<(RunMetadata, JobRunErrors)> - for JobStepOutput -{ - fn from((more_metadata, errors): (RunMetadata, JobRunErrors)) -> Self { - Self { - maybe_more_steps: None, - maybe_more_metadata: Some(more_metadata), - errors, - } - } -} - -impl From<(Vec, RunMetadata, JobRunErrors)> - for JobStepOutput -{ - fn from((more_steps, more_metadata, errors): (Vec, RunMetadata, JobRunErrors)) -> Self { - Self { - maybe_more_steps: Some(more_steps), - maybe_more_metadata: Some(more_metadata), - errors, - } - } -} - -impl From> for JobStepOutput { - fn from(_: Option<()>) -> Self { - Self { - maybe_more_steps: None, - maybe_more_metadata: None, - errors: Vec::new().into(), - } - } -} - -#[async_trait::async_trait] -impl DynJob for OldJob { - fn id(&self) -> Uuid { - self.id - } - - fn parent_id(&self) -> Option { - self.report.as_ref().and_then(|r| r.parent_id) - } - - fn report(&self) -> &Option { - &self.report - } - - fn report_mut(&mut self) -> &mut Option { - &mut self.report - } - - fn name(&self) -> &'static str { - ::NAME - } - - #[instrument(skip_all, fields(job_name = %self.name()), err)] - #[allow(clippy::blocks_in_conditions)] // Due to `err` on instrument above - async fn run( - &mut self, - ctx: WorkerContext, - commands_rx: chan::Receiver, - ) -> Result { - let job_name = self.name(); - let job_id = self.id; - let mut errors = vec![]; - info!("Starting Job"); - - let JobState { - init, - data, - mut steps, - mut step_number, - mut run_metadata, - } = self - .state - .take() - .expect("critical error: missing job state"); - - let target_location = init.target_location(); - - let mut stateful_job = Arc::new(init); - - let mut ctx = Arc::new(ctx); - - let mut job_should_run = true; - let job_init_time = Instant::now(); - - // Checking if we have a brand new job, or if we are resuming an old one. - let working_data = if let Some(data) = data { - Some(data) - } else { - // Job init phase - let init_time = Instant::now(); - let init_task = { - let ctx = Arc::clone(&ctx); - spawn(async move { - let mut new_data = None; - let res = stateful_job.init(&ctx, &mut new_data).await; - - if let Ok(res) = res.as_ref() { - if !::IS_BATCHED { - // tell the reporter how much work there is - ctx.progress(vec![JobReportUpdate::TaskCount(res.steps.len())]); - } - } - - (stateful_job, new_data, res) - }) - }; - - let InitPhaseOutput { - stateful_job: returned_stateful_job, - maybe_data, - output, - } = handle_init_phase::( - JobRunWorkTable { - id: job_id, - name: job_name, - init_time, - target_location, - }, - Arc::clone(&ctx), - init_task, - commands_rx.clone(), - ) - .await?; - - stateful_job = returned_stateful_job; - - match output { - Ok(JobInitOutput { - run_metadata: new_run_metadata, - steps: new_steps, - errors: JobRunErrors(new_errors), - }) => { - steps = new_steps; - errors.extend(new_errors); - run_metadata.update(new_run_metadata); - } - Err(e @ JobError::EarlyFinish { .. }) => { - info!(%e); - job_should_run = false; - } - Err(e) => return Err(e), - } - - maybe_data - }; - - // Run the job until it's done or we get a command - let data = if let Some(working_data) = working_data { - let mut working_data_arc = Arc::new(working_data); - - // Job run phase - while job_should_run && !steps.is_empty() { - let steps_len: usize = steps.len(); - - let mut run_metadata_arc = Arc::new(run_metadata); - let step = Arc::new(steps.pop_front().expect("just checked that we have steps")); - - let init_time = Instant::now(); - - let step_task = { - // Need these bunch of Arcs to be able to move them into the async block of tokio::spawn - let ctx = Arc::clone(&ctx); - let run_metadata = Arc::clone(&run_metadata_arc); - let working_data = Arc::clone(&working_data_arc); - let step = Arc::clone(&step); - let stateful_job = Arc::clone(&stateful_job); - spawn(async move { - stateful_job - .execute_step( - &ctx, - CurrentStep { - step: &step, - step_number, - }, - &working_data, - &run_metadata, - ) - .await - }) - }; - - let JobStepsPhaseOutput { - steps: returned_steps, - output, - step_arcs: - ( - returned_ctx, - returned_run_metadata_arc, - returned_working_data_arc, - returned_stateful_job, - ), - } = handle_single_step::( - JobRunWorkTable { - id: job_id, - name: job_name, - init_time, - target_location, - }, - &job_init_time, - ( - // Must not hold extra references here; moving and getting back on function completion - ctx, - run_metadata_arc, - working_data_arc, - stateful_job, - ), - JobStepDataWorkTable { - step_number, - steps, - step, - step_task, - }, - commands_rx.clone(), - ) - .await?; - - steps = returned_steps; - ctx = returned_ctx; - run_metadata_arc = returned_run_metadata_arc; - working_data_arc = returned_working_data_arc; - stateful_job = returned_stateful_job; - - run_metadata = - Arc::try_unwrap(run_metadata_arc).expect("step already ran, no more refs"); - - match output { - Ok(JobStepOutput { - maybe_more_steps, - maybe_more_metadata, - errors: JobRunErrors(new_errors), - }) => { - let mut events = vec![JobReportUpdate::CompletedTaskCount(step_number + 1)]; - - if let Some(more_steps) = maybe_more_steps { - events.push(JobReportUpdate::TaskCount(steps_len + more_steps.len())); - - steps.extend(more_steps); - } - - if let Some(more_metadata) = maybe_more_metadata { - run_metadata.update(more_metadata); - } - - if !::IS_BATCHED { - ctx.progress(events); - } - - if !new_errors.is_empty() { - warn!("Job had a step with errors"); - new_errors.iter().for_each(|e| { - warn!(?e, "Job error;"); - }); - - errors.extend(new_errors); - } - } - Err(e @ JobError::EarlyFinish { .. }) => { - info!(%e); - break; - } - Err(e) => return Err(e), - } - // remove the step from the queue - step_number += 1; - } - - debug!(job_init_time = ?job_init_time.elapsed(), "Total job run time;"); - - Some(Arc::try_unwrap(working_data_arc).expect("job already ran, no more refs")) - } else { - warn!("Tried to run a job without data"); - None - }; - - let metadata = stateful_job.finalize(&ctx, &data, &run_metadata).await?; - - let mut next_jobs = mem::take(&mut self.next_jobs); - - Ok(JobRunOutput { - metadata, - errors: errors.into(), - next_job: next_jobs.pop_front().map(|mut next_job| { - debug!( - next_job_name = %next_job.name(), - "Job requesting to spawn next job now that it's complete!;", - ); - - next_job.set_next_jobs(next_jobs); - - next_job - }), - }) - } - - fn hash(&self) -> u64 { - self.hash - } - - fn set_next_jobs(&mut self, next_jobs: VecDeque>) { - self.next_jobs = next_jobs; - } - - fn serialize_state(&self) -> Result, JobError> { - rmp_serde::to_vec_named(&self.state).map_err(Into::into) - } - - async fn register_children(&mut self, library: &Library) -> Result<(), JobError> { - for next_job in self.next_jobs.iter_mut() { - if let Some(next_job_report) = next_job.report_mut() { - if next_job_report.created_at.is_none() { - next_job_report.create(library).await? - } - } else { - return Err(JobError::MissingReport { - id: next_job.id(), - name: next_job.name().to_string(), - }); - } - } - - Ok(()) - } - - async fn pause_children(&mut self, library: &Library) -> Result<(), JobError> { - for next_job in self.next_jobs.iter_mut() { - let state = next_job.serialize_state()?; - if let Some(next_job_report) = next_job.report_mut() { - next_job_report.status = JobStatus::Paused; - next_job_report.data = Some(state); - next_job_report.update(library).await?; - } else { - return Err(JobError::MissingReport { - id: next_job.id(), - name: next_job.name().to_string(), - }); - } - } - - Ok(()) - } - - async fn cancel_children(&mut self, library: &Library) -> Result<(), JobError> { - for next_job in self.next_jobs.iter_mut() { - let state = next_job.serialize_state()?; - if let Some(next_job_report) = next_job.report_mut() { - next_job_report.status = JobStatus::Canceled; - next_job_report.data = Some(state); - next_job_report.update(library).await?; - } else { - return Err(JobError::MissingReport { - id: next_job.id(), - name: next_job.name().to_string(), - }); - } - } - - Ok(()) - } -} - -struct InitPhaseOutput { - stateful_job: Arc, - maybe_data: Option, - output: Result, JobError>, -} - -struct JobRunWorkTable { - id: Uuid, - name: &'static str, - init_time: Instant, - target_location: location::id::Type, -} - -type InitTaskOutput = ( - Arc, - Option<::Data>, - Result< - JobInitOutput<::RunMetadata, ::Step>, - JobError, - >, -); - -#[inline] -async fn handle_init_phase( - JobRunWorkTable { - id, - name, - init_time, - target_location, - }: JobRunWorkTable, - worker_ctx: Arc, - init_task: JoinHandle>, - mut commands_rx: chan::Receiver, -) -> Result, JobError> { - enum StreamMessage { - NewCommand(WorkerCommand), - InitResult(Result, JoinError>), - } - - let mut status = JobStatus::Running; - - let init_abort_handle = init_task.abort_handle(); - - let mut msg_stream = pin!(( - stream::once(init_task).map(StreamMessage::::InitResult), - commands_rx.clone().map(StreamMessage::::NewCommand), - ) - .merge()); - - let mut commands_rx = pin!(commands_rx); - - 'messages: while let Some(msg) = msg_stream.next().await { - match msg { - StreamMessage::InitResult(Err(join_error)) => { - error!( - ?join_error, - "Job failed to initialize due to an internal error;" - ); - return Err(join_error.into()); - } - StreamMessage::InitResult(Ok((stateful_job, maybe_data, output))) => { - debug!(init_phase_time = ?init_time.elapsed(), "Init phase completed;"); - - return Ok(InitPhaseOutput { - stateful_job, - maybe_data, - output, - }); - } - StreamMessage::NewCommand(WorkerCommand::IdentifyYourself(tx)) => { - if tx - .send(JobIdentity { - id, - name, - target_location, - status, - }) - .is_err() - { - warn!("Failed to send IdentifyYourself event reply"); - } - } - StreamMessage::NewCommand(WorkerCommand::Pause(when)) => { - debug!(pausing_time = ?when.elapsed(), "Pausing Job at init phase;"); - - // Notify the worker's work task that now we're paused - worker_ctx.pause(); - - status = JobStatus::Paused; - - // In case of a Pause command, we keep waiting for the next command - let paused_time = Instant::now(); - while let Some(command) = commands_rx.next().await { - match command { - WorkerCommand::IdentifyYourself(tx) => { - if tx - .send(JobIdentity { - id, - name, - target_location, - status, - }) - .is_err() - { - warn!("Failed to send IdentifyYourself event reply"); - } - } - WorkerCommand::Resume(when) => { - debug!( - resuming_time = ?when.elapsed(), - paused_time = ?paused_time.elapsed(), - "Resuming Job at init phase;", - ); - - status = JobStatus::Running; - - continue 'messages; - } - // The job can also be shutdown or canceled while paused - WorkerCommand::Shutdown(when, signal_tx) => { - init_abort_handle.abort(); - - debug!( - shutting_down_time = ?when.elapsed(), - paused_time = ?paused_time.elapsed(), - total_running_time = ?init_time.elapsed(), - "Shuting down Job at init phase;", - ); - - // Shutting down at init phase will abort the job - return Err(JobError::Canceled(signal_tx)); - } - WorkerCommand::Cancel(when, signal_tx) => { - init_abort_handle.abort(); - debug!( - canceling_time = ?when.elapsed(), - paused_time = ?paused_time.elapsed(), - total_running_time = ?init_time.elapsed(), - "Canceling Job at init phase;", - ); - - return Err(JobError::Canceled(signal_tx)); - } - WorkerCommand::Pause(_) => { - // We continue paused lol - } - WorkerCommand::Timeout(elapsed, tx) => { - error!(elapsed_time = ?elapsed, "Job timed out at init phase;"); - tx.send(()).ok(); - return Err(JobError::Timeout(elapsed)); - } - } - } - - if commands_rx.is_closed() { - error!("Job closed the command channel while paused"); - return Err(JobError::Critical( - "worker command channel closed while job was paused", - )); - } - } - StreamMessage::NewCommand(WorkerCommand::Resume(_)) => { - // We're already running so we just ignore this command - } - StreamMessage::NewCommand(WorkerCommand::Shutdown(when, signal_tx)) => { - init_abort_handle.abort(); - - debug!( - shutting_down_time = ?when.elapsed(), - total_running_time = ?init_time.elapsed(), - "Shuting down at init phase;", - ); - - // Shutting down at init phase will abort the job - return Err(JobError::Canceled(signal_tx)); - } - StreamMessage::NewCommand(WorkerCommand::Cancel(when, signal_tx)) => { - init_abort_handle.abort(); - - debug!( - canceling_time = ?when.elapsed(), - total_running_time = ?init_time.elapsed(), - "Canceling at init phase;", - ); - - return Err(JobError::Canceled(signal_tx)); - } - StreamMessage::NewCommand(WorkerCommand::Timeout(elapsed, tx)) => { - error!( - elapsed_time = ?elapsed, - "Job timed out at init phase;", - ); - tx.send(()).ok(); - return Err(JobError::Timeout(elapsed)); - } - } - } - - Err(JobError::Critical("unexpected job init end without result")) -} - -type StepTaskOutput = Result< - JobStepOutput<::Step, ::RunMetadata>, - JobError, ->; - -struct JobStepDataWorkTable { - step_number: usize, - steps: VecDeque, - step: Arc, - step_task: JoinHandle>, -} - -struct JobStepsPhaseOutput { - steps: VecDeque, - output: StepTaskOutput, - step_arcs: StepArcs, -} - -type StepArcs = ( - Arc, - Arc<::RunMetadata>, - Arc<::Data>, - Arc, -); - -#[instrument(skip_all, fields(job_id = %id, job_name = %name, %step_number))] -#[inline] -async fn handle_single_step( - JobRunWorkTable { - id, - name, - init_time, - target_location, - }: JobRunWorkTable, - job_init_time: &Instant, - (worker_ctx, run_metadata, working_data, stateful_job): StepArcs, - JobStepDataWorkTable { - step_number, - mut steps, - step, - mut step_task, - }: JobStepDataWorkTable, - commands_rx: chan::Receiver, -) -> Result, JobError> { - enum StreamMessage { - NewCommand(WorkerCommand), - StepResult(Result, JoinError>), - } - - let mut status = JobStatus::Running; - - let mut msg_stream = pin!(( - stream::once(&mut step_task).map(StreamMessage::::StepResult), - commands_rx.clone().map(StreamMessage::::NewCommand), - ) - .merge()); - - let mut commands_rx = pin!(commands_rx); - - 'messages: while let Some(msg) = msg_stream.next().await { - match msg { - StreamMessage::StepResult(Err(join_error)) => { - error!(?join_error, "Failed to run step due to an internal error;"); - return Err(join_error.into()); - } - StreamMessage::StepResult(Ok(output)) => { - trace!(elapsed = ?init_time.elapsed(), "Step finished;"); - - return Ok(JobStepsPhaseOutput { - steps, - output, - step_arcs: (worker_ctx, run_metadata, working_data, stateful_job), - }); - } - StreamMessage::NewCommand(WorkerCommand::IdentifyYourself(tx)) => { - if tx - .send(JobIdentity { - id, - name, - target_location, - status, - }) - .is_err() - { - warn!("Failed to send IdentifyYourself event reply"); - } - } - StreamMessage::NewCommand(WorkerCommand::Pause(when)) => { - debug!(elapsed = ?when.elapsed(), "Pausing Job;"); - - worker_ctx.pause(); - - status = JobStatus::Paused; - - // In case of a Pause command, we keep waiting for the next command - let paused_time = Instant::now(); - while let Some(command) = commands_rx.next().await { - match command { - WorkerCommand::IdentifyYourself(tx) => { - if tx - .send(JobIdentity { - id, - name, - target_location, - status, - }) - .is_err() - { - warn!("Failed to send IdentifyYourself event reply"); - } - } - WorkerCommand::Resume(when) => { - debug!(elapsed = ?when.elapsed(), "Resuming Job;"); - debug!(paused_time = ?paused_time.elapsed(), "Total paused time;"); - status = JobStatus::Running; - - continue 'messages; - } - // The job can also be shutdown or canceled while paused - WorkerCommand::Shutdown(when, signal_tx) => { - step_task.abort(); - let _ = step_task.await; - - debug!( - elapsed_time_to_shutdown = ?when.elapsed(), - job_run_time = ?job_init_time.elapsed(), - "Shuting down;", - ); - debug!(paused_time = ?paused_time.elapsed(), "Total paused time;"); - - // Taking back the last step, so it can run to completion later - steps.push_front( - Arc::try_unwrap(step).expect("step already ran, no more refs"), - ); - - return Err(JobError::Paused( - rmp_serde::to_vec_named(&JobState:: { - init: Arc::try_unwrap(stateful_job) - .expect("handle abort already ran, no more refs"), - data: Some( - Arc::try_unwrap(working_data) - .expect("handle abort already ran, no more refs"), - ), - steps, - step_number, - run_metadata: Arc::try_unwrap(run_metadata) - .expect("handle abort already ran, no more refs"), - })?, - signal_tx, - )); - } - WorkerCommand::Cancel(when, signal_tx) => { - step_task.abort(); - let _ = step_task.await; - debug!( - canceling_time = ?when.elapsed(), - job_run_time = ?job_init_time.elapsed(), - "Canceling Job;", - ); - debug!(paused_time = ?paused_time.elapsed(), "Total paused time;"); - return Err(JobError::Canceled(signal_tx)); - } - WorkerCommand::Pause(_) => { - // We continue paused lol - } - - WorkerCommand::Timeout(elapsed, tx) => { - error!(?elapsed, "Step timed out;"); - tx.send(()).ok(); - return Err(JobError::Timeout(elapsed)); - } - } - } - - if commands_rx.is_closed() { - error!("Closed the command channel while paused"); - return Err(JobError::Critical( - "worker command channel closed while job was paused", - )); - } - } - StreamMessage::NewCommand(WorkerCommand::Resume(_)) => { - // We're already running so we just ignore this command - } - StreamMessage::NewCommand(WorkerCommand::Shutdown(when, signal_tx)) => { - step_task.abort(); - let _ = step_task.await; - - debug!( - elapsed = ?when.elapsed(), - job_run_time = ?job_init_time.elapsed(), - "Shutting down Job;", - ); - - // Taking back the last step, so it can run to completion later - steps.push_front( - Arc::try_unwrap(step).expect("handle abort already ran, no more refs"), - ); - - return Err(JobError::Paused( - rmp_serde::to_vec_named(&JobState:: { - init: Arc::try_unwrap(stateful_job) - .expect("handle abort already ran, no more refs"), - data: Some( - Arc::try_unwrap(working_data) - .expect("handle abort already ran, no more refs"), - ), - steps, - step_number, - run_metadata: Arc::try_unwrap(run_metadata) - .expect("step already ran, no more refs"), - })?, - signal_tx, - )); - } - StreamMessage::NewCommand(WorkerCommand::Cancel(when, signal_tx)) => { - step_task.abort(); - let _ = step_task.await; - debug!( - cancel_time = ?when.elapsed(), - job_run_time = ?job_init_time.elapsed(), - "Canceling Job;", - ); - - return Err(JobError::Canceled(signal_tx)); - } - StreamMessage::NewCommand(WorkerCommand::Timeout(elapsed, tx)) => { - error!(?elapsed, "Job timed out without updates;"); - tx.send(()).ok(); - return Err(JobError::Timeout(elapsed)); - } - } - } - - Err(JobError::Critical("unexpected job step end without result")) -} diff --git a/core/src/old_job/report.rs b/core/src/old_job/report.rs deleted file mode 100644 index af7333267..000000000 --- a/core/src/old_job/report.rs +++ /dev/null @@ -1,504 +0,0 @@ -use crate::{ - library::Library, - object::{ - fs::{ - old_copy::OldFileCopierJobInit, old_cut::OldFileCutterJobInit, - old_delete::OldFileDeleterJobInit, old_erase::OldFileEraserJobInit, - }, - validation::old_validator_job::OldObjectValidatorJobInit, - }, -}; - -use sd_core_prisma_helpers::job_without_data; - -use sd_prisma::prisma::job; -use sd_utils::db::{maybe_missing, MissingFieldError}; - -use std::fmt::{Display, Formatter}; - -use chrono::{DateTime, Utc}; -use serde::{Deserialize, Serialize}; -use specta::Type; -use tracing::error; -use uuid::Uuid; - -use super::JobError; - -#[derive(Debug)] -pub enum JobReportUpdate { - TaskCount(usize), - CompletedTaskCount(usize), - Message(String), - Info(String), - Phase(String), -} - -#[derive(Debug, Serialize, Deserialize, Clone)] -pub struct OldJobReport { - pub id: Uuid, - pub name: String, - pub action: Option, - pub data: Option>, - // // In Typescript `any | null` is just `any` so we don't get prompted for null checks - // // TODO(@Oscar): This will be fixed - // #[specta(type = Option>)] - pub metadata: Option, - pub errors_text: Vec, - - pub created_at: Option>, - pub started_at: Option>, - pub completed_at: Option>, - - pub parent_id: Option, - - pub status: JobStatus, - pub task_count: i32, - pub completed_task_count: i32, - pub info: String, - - pub phase: String, - pub message: String, - pub estimated_completion: DateTime, -} - -impl From for sd_core_heavy_lifting::job_system::report::Report { - fn from( - OldJobReport { - id, - name, - action, - data: _, // Not used in the new job system - metadata, - errors_text: _, // New job system uses type-safe errors - created_at, - started_at, - completed_at, - parent_id, - status, - task_count, - completed_task_count, - info, - phase, - message, - estimated_completion, - }: OldJobReport, - ) -> Self { - use sd_core_heavy_lifting::{job_system::report::ReportOutputMetadata, JobName}; - - let mut new_metadata = Vec::new(); - - if let Some(metadata) = metadata { - if let Some(metadata) = metadata.as_object() { - if let Some(metadata) = metadata.get("output") { - if let Some(metadata) = metadata.as_object() { - if let Some(metadata) = metadata.get("init") { - if let Ok(OldFileCopierJobInit { - source_location_id, - target_location_id, - sources_file_path_ids, - target_location_relative_directory_path, - }) = serde_json::from_value::(metadata.clone()) - { - new_metadata.push( - ReportOutputMetadata::Copier { - source_location_id, - target_location_id, - sources_file_path_ids, - target_location_relative_directory_path, - } - .into(), - ); - } else if let Ok(OldFileCutterJobInit { - source_location_id, - target_location_id, - sources_file_path_ids, - target_location_relative_directory_path, - }) = - serde_json::from_value::(metadata.clone()) - { - new_metadata.push( - ReportOutputMetadata::Mover { - source_location_id, - target_location_id, - sources_file_path_ids, - target_location_relative_directory_path, - } - .into(), - ); - } else if let Ok(OldFileDeleterJobInit { - location_id, - file_path_ids, - }) = - serde_json::from_value::(metadata.clone()) - { - new_metadata.push( - ReportOutputMetadata::Deleter { - location_id, - file_path_ids, - } - .into(), - ); - } else if let Ok(OldFileEraserJobInit { - location_id, - file_path_ids, - passes, - }) = - serde_json::from_value::(metadata.clone()) - { - new_metadata.push( - ReportOutputMetadata::Eraser { - location_id, - file_path_ids, - passes: passes as u32, - } - .into(), - ); - } else if let Ok(OldObjectValidatorJobInit { location, sub_path }) = - serde_json::from_value::( - metadata.clone(), - ) { - new_metadata.push( - ReportOutputMetadata::FileValidator { - location_id: location.id, - sub_path, - } - .into(), - ); - } - } - } - } - } - } - - Self { - id, - name: match name.as_str() { - "file_copier" => JobName::Copy, - "file_cutter" => JobName::Move, - "file_deleter" => JobName::Delete, - "file_eraser" => JobName::Erase, - "object_validator" => JobName::FileValidator, - - // Already implemented in the new job system - "indexer" => JobName::Indexer, - "file_identifier" => JobName::FileIdentifier, - "media_processor" => JobName::MediaProcessor, - - unexpected_job => unimplemented!("Job {unexpected_job} not implemented"), - }, - action, - metadata: new_metadata, - critical_error: None, - non_critical_errors: Vec::new(), - created_at, - started_at, - completed_at, - parent_id, - status: status.into(), - task_count, - completed_task_count, - info, - phase, - message, - estimated_completion, - } - } -} - -impl Display for OldJobReport { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!( - f, - "Job {:#?}", - self.name, self.id, self.status - ) - } -} - -// convert database struct into a resource struct -impl TryFrom for OldJobReport { - type Error = MissingFieldError; - - fn try_from(data: job::Data) -> Result { - Ok(Self { - id: Uuid::from_slice(&data.id).expect("corrupted database"), - name: maybe_missing(data.name, "job.name")?, - action: data.action, - data: data.data, - metadata: data.metadata.and_then(|m| { - serde_json::from_slice(&m).unwrap_or_else(|e| -> Option { - error!(?e, "Failed to deserialize job metadata;"); - None - }) - }), - errors_text: data - .errors_text - .map(|errors_str| errors_str.split("\n\n").map(str::to_string).collect()) - .unwrap_or_default(), - created_at: data.date_created.map(DateTime::into), - started_at: data.date_started.map(DateTime::into), - completed_at: data.date_completed.map(DateTime::into), - parent_id: data - .parent_id - .map(|id| Uuid::from_slice(&id).expect("corrupted database")), - status: JobStatus::try_from(maybe_missing(data.status, "job.status")?) - .expect("corrupted database"), - task_count: data.task_count.unwrap_or(0), - completed_task_count: data.completed_task_count.unwrap_or(0), - info: data.info.unwrap_or_default(), - phase: String::new(), - message: String::new(), - estimated_completion: data - .date_estimated_completion - .map_or(Utc::now(), DateTime::into), - }) - } -} - -// I despise having to write this twice, but it seems to be the only way to -// remove the data field from the struct -// would love to get this DRY'd up -impl TryFrom for OldJobReport { - type Error = MissingFieldError; - - fn try_from(data: job_without_data::Data) -> Result { - Ok(Self { - id: Uuid::from_slice(&data.id).expect("corrupted database"), - name: maybe_missing(data.name, "job.name")?, - action: data.action, - data: None, - metadata: data.metadata.and_then(|m| { - serde_json::from_slice(&m).unwrap_or_else(|e| -> Option { - error!(?e, "Failed to deserialize job metadata;"); - None - }) - }), - errors_text: data - .errors_text - .map(|errors_str| errors_str.split("\n\n").map(str::to_string).collect()) - .unwrap_or_default(), - created_at: data.date_created.map(DateTime::into), - started_at: data.date_started.map(DateTime::into), - completed_at: data.date_completed.map(DateTime::into), - parent_id: data - .parent_id - .map(|id| Uuid::from_slice(&id).expect("corrupted database")), - status: JobStatus::try_from(maybe_missing(data.status, "job.status")?) - .expect("corrupted database"), - task_count: data.task_count.unwrap_or(0), - completed_task_count: data.completed_task_count.unwrap_or(0), - info: data.info.unwrap_or_default(), - phase: String::new(), - message: String::new(), - estimated_completion: data - .date_estimated_completion - .map_or(Utc::now(), DateTime::into), - }) - } -} - -impl OldJobReport { - pub fn new(uuid: Uuid, name: String) -> Self { - Self { - id: uuid, - name, - action: None, - created_at: None, - started_at: None, - completed_at: None, - status: JobStatus::Queued, - errors_text: vec![], - task_count: 0, - data: None, - metadata: None, - parent_id: None, - completed_task_count: 0, - info: String::new(), - phase: String::new(), - message: String::new(), - estimated_completion: Utc::now(), - } - } - - pub fn get_meta(&self) -> (String, Option) { - // actions are formatted like "added_location" or "added_location-1" - let Some(action_name) = self.action.as_ref().map(|action| { - action - .split('-') - .next() - .map(str::to_string) - .unwrap_or_default() - }) else { - return (self.id.to_string(), None); - }; - // create a unique group_key, EG: "added_location-" - let group_key = self.parent_id.map_or_else( - || format!("{}-{}", action_name, &self.id), - |parent_id| format!("{}-{}", action_name, parent_id), - ); - - (action_name, Some(group_key)) - } - - pub async fn create(&mut self, library: &Library) -> Result<(), JobError> { - let now = Utc::now(); - - library - .db - .job() - .create( - self.id.as_bytes().to_vec(), - sd_utils::chain_optional_iter( - [ - job::name::set(Some(self.name.clone())), - job::action::set(self.action.clone()), - job::data::set(self.data.clone()), - job::date_created::set(Some(now.into())), - job::status::set(Some(self.status as i32)), - job::date_started::set(self.started_at.map(|d| d.into())), - job::task_count::set(Some(1)), - job::info::set(Some(self.info.clone())), - job::completed_task_count::set(Some(0)), - ], - [self - .parent_id - .map(|id| job::parent::connect(job::id::equals(id.as_bytes().to_vec())))], - ), - ) - .exec() - .await?; - - // Only setting created_at after we successfully created the job in DB - self.created_at = Some(now); - - Ok(()) - } - - pub async fn update(&mut self, library: &Library) -> Result<(), JobError> { - library - .db - .job() - .update( - job::id::equals(self.id.as_bytes().to_vec()), - vec![ - job::status::set(Some(self.status as i32)), - job::errors_text::set( - (!self.errors_text.is_empty()).then(|| self.errors_text.join("\n\n")), - ), - job::data::set(self.data.clone()), - job::metadata::set(serde_json::to_vec(&self.metadata).ok()), - job::task_count::set(Some(self.task_count)), - job::info::set(Some(self.info.clone())), - job::completed_task_count::set(Some(self.completed_task_count)), - job::date_started::set(self.started_at.map(Into::into)), - job::date_completed::set(self.completed_at.map(Into::into)), - ], - ) - .select(job::select!({ id })) - .exec() - .await?; - Ok(()) - } -} - -#[repr(i32)] -#[derive(Debug, Clone, Copy, Serialize, Deserialize, Type, Eq, PartialEq)] -pub enum JobStatus { - Queued = 0, - Running = 1, - Completed = 2, - Canceled = 3, - Failed = 4, - Paused = 5, - CompletedWithErrors = 6, -} - -impl JobStatus { - pub fn is_finished(self) -> bool { - matches!( - self, - Self::Completed - | Self::Canceled - | Self::Paused - | Self::Failed - | Self::CompletedWithErrors - ) - } -} - -impl TryFrom for JobStatus { - type Error = JobError; - - fn try_from(value: i32) -> Result { - let s = match value { - 0 => Self::Queued, - 1 => Self::Running, - 2 => Self::Completed, - 3 => Self::Canceled, - 4 => Self::Failed, - 5 => Self::Paused, - 6 => Self::CompletedWithErrors, - _ => return Err(JobError::InvalidJobStatusInt(value)), - }; - - Ok(s) - } -} - -// TODO(fogodev): this is temporary until we can get rid of the old job system -impl From for sd_core_heavy_lifting::job_system::report::Status { - fn from(value: JobStatus) -> Self { - match value { - JobStatus::Queued => Self::Queued, - JobStatus::Running => Self::Running, - JobStatus::Completed => Self::Completed, - JobStatus::Canceled => Self::Canceled, - JobStatus::Failed => Self::Failed, - JobStatus::Paused => Self::Paused, - JobStatus::CompletedWithErrors => Self::CompletedWithErrors, - } - } -} - -pub struct JobReportBuilder { - pub id: Uuid, - pub name: String, - pub action: Option, - pub metadata: Option, - pub parent_id: Option, -} - -impl JobReportBuilder { - pub fn build(self) -> OldJobReport { - OldJobReport { - id: self.id, - name: self.name, - action: self.action, - created_at: None, - started_at: None, - completed_at: None, - status: JobStatus::Queued, - errors_text: vec![], - task_count: 0, - data: None, - metadata: self.metadata, - parent_id: self.parent_id, - completed_task_count: 0, - info: String::new(), - phase: String::new(), - message: String::new(), - estimated_completion: Utc::now(), - } - } - - pub fn new(id: Uuid, name: String) -> Self { - Self { - id, - name, - action: None, - metadata: None, - parent_id: None, - } - } -} diff --git a/core/src/old_job/worker.rs b/core/src/old_job/worker.rs deleted file mode 100644 index 5ede4d2d4..000000000 --- a/core/src/old_job/worker.rs +++ /dev/null @@ -1,660 +0,0 @@ -use crate::{api::CoreEvent, invalidate_query, library::Library, Node}; - -use std::{ - fmt, - pin::pin, - sync::{ - atomic::{AtomicBool, Ordering}, - Arc, - }, - time::Duration, -}; - -use async_channel as chan; -use chrono::{DateTime, Utc}; -use futures::stream::{self, StreamExt}; -use futures_concurrency::stream::Merge; -use serde::Serialize; -use serde_json::json; -use specta::Type; -use tokio::{ - spawn, - sync::{oneshot, watch}, - task::JoinError, - time::{interval, timeout, Instant, MissedTickBehavior}, -}; -use tokio_stream::wrappers::IntervalStream; -use tracing::{debug, error, info, instrument, trace, warn}; -use uuid::Uuid; - -use super::{ - DynJob, JobError, JobIdentity, JobReportUpdate, JobRunErrors, JobRunOutput, JobStatus, - OldJobReport, OldJobs, -}; - -const FIVE_SECS: Duration = Duration::from_secs(5); -const FIVE_MINUTES: Duration = Duration::from_secs(10 * 60); - -#[derive(Debug, Clone, Serialize, Type)] -pub struct JobProgressEvent { - pub id: Uuid, - pub library_id: Uuid, - pub task_count: i32, - pub completed_task_count: i32, - pub phase: String, - pub message: String, - pub info: String, - pub estimated_completion: DateTime, -} - -/// used to update the worker state from inside the worker thread -#[derive(Debug)] -pub enum WorkerEvent { - Progressed(Vec), - Paused, - Stop, -} - -/// used to send commands to the worker thread from the manager -#[derive(Debug)] -pub enum WorkerCommand { - Pause(Instant), - Resume(Instant), - IdentifyYourself(oneshot::Sender), - Cancel(Instant, oneshot::Sender<()>), - Shutdown(Instant, oneshot::Sender<()>), - Timeout(Duration, oneshot::Sender<()>), -} - -pub struct WorkerContext { - pub library: Arc, - pub node: Arc, - pub(super) events_tx: chan::Sender, -} - -impl fmt::Debug for WorkerContext { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("WorkerContext").finish() - } -} - -impl Drop for WorkerContext { - fn drop(&mut self) { - // This send blocking is fine as this sender is unbounded - if !self.events_tx.is_closed() && self.events_tx.send_blocking(WorkerEvent::Stop).is_err() { - error!("Error sending worker context stop event"); - } - } -} -impl WorkerContext { - pub fn pause(&self) { - if self.events_tx.send_blocking(WorkerEvent::Paused).is_err() { - error!("Error sending worker context pause event"); - } - } - - pub fn progress_msg(&self, msg: String) { - self.progress(vec![JobReportUpdate::Message(msg)]); - } - - pub fn progress(&self, updates: Vec) { - if !self.events_tx.is_closed() - && self - .events_tx - // This send blocking is fine as this sender is unbounded - .send_blocking(WorkerEvent::Progressed(updates)) - .is_err() - { - error!("Error sending worker context progress event"); - } - } -} - -// a worker is a dedicated task that runs a single job -// once the job is complete the worker will exit -pub struct Worker { - pub(super) library_id: Uuid, - commands_tx: chan::Sender, - report_watch_tx: Arc>, - report_watch_rx: watch::Receiver, - paused: AtomicBool, -} - -impl Worker { - pub async fn new( - id: Uuid, - mut job: Box, - mut report: OldJobReport, - library: Arc, - node: Arc, - job_manager: Arc, - ) -> Result { - let (commands_tx, commands_rx) = chan::bounded(8); - - let job_hash = job.hash(); - - let start_time = Utc::now(); - - report.status = JobStatus::Running; - if report.started_at.is_none() { - report.started_at = Some(start_time); - } - - // If the report doesn't have a created_at date, it's a new report - if report.created_at.is_none() { - report.create(&library).await?; - } else { - // Otherwise it can be a job being resumed or a children job that was already been created - report.update(&library).await?; - } - - job.register_children(&library).await?; - - invalidate_queries(&library); - - let (report_watch_tx, report_watch_rx) = watch::channel(report.clone()); - let report_watch_tx = Arc::new(report_watch_tx); - let library_id = library.id; - - // spawn task to handle running the job - spawn(Self::do_work( - id, - JobWorkTable { - job, - manager: job_manager, - hash: job_hash, - report, - }, - Arc::clone(&report_watch_tx), - start_time, - (commands_tx.clone(), commands_rx), - library, - node, - )); - - Ok(Self { - library_id, - commands_tx, - report_watch_tx, - report_watch_rx, - paused: AtomicBool::new(false), - }) - } - - pub async fn pause(&self) { - if self.report_watch_rx.borrow().status == JobStatus::Running { - self.paused.store(true, Ordering::Relaxed); - if self - .commands_tx - .send(WorkerCommand::Pause(Instant::now())) - .await - .is_ok() - { - self.report_watch_tx - .send_modify(|report| report.status = JobStatus::Paused); - } - } - } - - pub async fn who_am_i(&self) -> Option { - let (tx, rx) = oneshot::channel(); - if self - .commands_tx - .send(WorkerCommand::IdentifyYourself(tx)) - .await - .is_err() - { - warn!("Failed to send identify yourself command to a job worker"); - return None; - } - - rx.await - .map_err(|_| warn!("Failed to receive identify yourself answer from a job worker")) - .ok() - } - - pub async fn resume(&self) { - if self.report_watch_rx.borrow().status == JobStatus::Paused { - self.paused.store(false, Ordering::Relaxed); - if self - .commands_tx - .send(WorkerCommand::Resume(Instant::now())) - .await - .is_ok() - { - self.report_watch_tx - .send_modify(|report| report.status = JobStatus::Running); - } - } - } - - pub async fn cancel(&self) { - if self.report_watch_rx.borrow().status != JobStatus::Canceled { - let (tx, rx) = oneshot::channel(); - if self - .commands_tx - .send(WorkerCommand::Cancel(Instant::now(), tx)) - .await - .is_ok() - { - self.report_watch_tx - .send_modify(|report| report.status = JobStatus::Canceled); - rx.await.ok(); - } - } - } - - pub async fn shutdown(&self) { - let (tx, rx) = oneshot::channel(); - if self - .commands_tx - .send(WorkerCommand::Shutdown(Instant::now(), tx)) - .await - .is_ok() - { - rx.await.ok(); - } - } - - pub fn report(&self) -> OldJobReport { - self.report_watch_rx.borrow().clone() - } - - pub fn is_paused(&self) -> bool { - self.paused.load(Ordering::Relaxed) - } - - #[instrument(skip_all, fields(job_id = %report.id, job_name = %report.name))] - fn track_progress( - report: &mut OldJobReport, - last_report_watch_update: &mut Instant, - report_watch_tx: &watch::Sender, - start_time: DateTime, - updates: Vec, - library: &Library, - ) { - // protect against updates if job is not running - if report.status != JobStatus::Running { - return; - }; - - for update in updates { - match update { - JobReportUpdate::TaskCount(task_count) => { - report.task_count = task_count as i32; - } - JobReportUpdate::CompletedTaskCount(completed_task_count) => { - report.completed_task_count = completed_task_count as i32; - } - - JobReportUpdate::Message(message) => { - trace!(?message); - report.message = message; - } - JobReportUpdate::Phase(phase) => { - trace!( - old_phase = %report.phase, - new_phase = %phase, - "Changing Job phase;", - ); - report.phase = phase; - } - JobReportUpdate::Info(info) => { - report.info = info; - } - } - } - - // Calculate elapsed time - let elapsed = Utc::now() - start_time; - - // Calculate remaining time - let task_count = report.task_count as usize; - let completed_task_count = report.completed_task_count as usize; - let remaining_task_count = task_count.saturating_sub(completed_task_count); - - // To avoid division by zero - let remaining_time_per_task = elapsed / (usize::max(completed_task_count, 1) as i32); - let remaining_time = remaining_time_per_task * remaining_task_count as i32; - - // Update the report with estimated remaining time - report.estimated_completion = Utc::now() - .checked_add_signed(remaining_time) - .unwrap_or(Utc::now()); - - // updated the report watcher - if last_report_watch_update.elapsed() > Duration::from_millis(500) { - report_watch_tx.send_modify(|old| { - old.task_count = report.task_count; - old.completed_task_count = report.completed_task_count; - old.estimated_completion = report.estimated_completion; - old.message.clone_from(&report.message); - }); - *last_report_watch_update = Instant::now(); - } - - // emit a CoreEvent - library.emit(CoreEvent::JobProgress(JobProgressEvent { - id: report.id, - library_id: library.id, - task_count: report.task_count, - completed_task_count: report.completed_task_count, - estimated_completion: report.estimated_completion, - info: report.info.clone(), - phase: report.phase.clone(), - message: report.message.clone(), - })); - } - - #[instrument( - skip_all, - fields( - job_id = %report.id, - job_name = %report.name, - worker_id = %worker_id, - %start_time, - library_id = %library.id, - ), - )] - async fn do_work( - worker_id: Uuid, - JobWorkTable { - mut job, - manager, - hash, - mut report, - }: JobWorkTable, - report_watch_tx: Arc>, - start_time: DateTime, - (commands_tx, commands_rx): (chan::Sender, chan::Receiver), - library: Arc, - node: Arc, - ) { - let (events_tx, events_rx) = chan::unbounded(); - - let mut timeout_checker = interval(FIVE_SECS); - timeout_checker.set_missed_tick_behavior(MissedTickBehavior::Skip); - - let mut last_update_received_at = Instant::now(); - - let mut last_reporter_watch_update = Instant::now(); - invalidate_query!(library, "jobs.reports"); - - let mut finalized_events_rx = pin!(events_rx.clone()); - - let mut is_paused = false; - - let mut run_task = { - let library = Arc::clone(&library); - spawn(async move { - let job_result = job - .run( - WorkerContext { - library, - node, - events_tx, - }, - commands_rx, - ) - .await; - - (job, job_result) - }) - }; - - type RunOutput = (Box, Result); - - enum StreamMessage { - JobResult(Result), - NewEvent(WorkerEvent), - Tick, - } - - let mut msg_stream = pin!(( - stream::once(&mut run_task).map(StreamMessage::JobResult), - events_rx.map(StreamMessage::NewEvent), - IntervalStream::new(timeout_checker).map(|_| StreamMessage::Tick), - ) - .merge()); - - let mut events_ended = false; - - while let Some(msg) = msg_stream.next().await { - match msg { - StreamMessage::JobResult(Err(join_error)) => { - error!(?join_error, "Critical error;"); - break; - } - StreamMessage::JobResult(Ok((job, job_result))) => { - if !events_ended { - finalized_events_rx.close(); - // There are still some progress events to be processed so we postpone the job result - while let Some(WorkerEvent::Progressed(updates)) = - finalized_events_rx.next().await - { - Self::track_progress( - &mut report, - &mut last_reporter_watch_update, - &report_watch_tx, - start_time, - updates, - &library, - ); - } - } - - let next_job = - Self::process_job_output(job, job_result, &mut report, &library).await; - - report_watch_tx.send(report.clone()).ok(); - - debug!("Worker completed Job"); - - return manager.complete(&library, worker_id, hash, next_job).await; - } - StreamMessage::NewEvent(WorkerEvent::Progressed(updates)) => { - is_paused = false; - last_update_received_at = Instant::now(); - Self::track_progress( - &mut report, - &mut last_reporter_watch_update, - &report_watch_tx, - start_time, - updates, - &library, - ); - } - StreamMessage::NewEvent(WorkerEvent::Paused) => { - is_paused = true; - } - StreamMessage::NewEvent(WorkerEvent::Stop) => { - events_ended = true; - } - StreamMessage::Tick => { - if !is_paused { - let elapsed = last_update_received_at.elapsed(); - if elapsed > FIVE_MINUTES { - error!( - time_without_updates = ?elapsed, - "Worker has not received any updates;", - ); - - let (tx, rx) = oneshot::channel(); - if commands_tx - .send(WorkerCommand::Timeout(elapsed, tx)) - .await - .is_err() - { - error!("Failed to send timeout step command to a running job"); - } else if timeout(FIVE_SECS, rx).await.is_err() { - error!("Failed to receive timeout step answer from a running job"); - } - - // As we already sent a timeout command, we can safely join as the job is over - let Ok((job, job_result)) = run_task - .await - .map_err(|join_error| error!(?join_error, "Had a critical error;")) - else { - break; - }; - - Self::process_job_output(job, job_result, &mut report, &library).await; - - report_watch_tx.send(report.clone()).ok(); - - error!("Timed out"); - - break; - } - } - } - } - } - - manager.complete(&library, worker_id, hash, None).await - } - - async fn process_job_output( - mut job: Box, - job_result: Result, - report: &mut OldJobReport, - library: &Library, - ) -> Option> { - // Run the job and handle the result - match job_result { - // -> Job completed successfully - Ok(JobRunOutput { - metadata, - errors: JobRunErrors(errors), - next_job, - }) if errors.is_empty() => { - report.status = JobStatus::Completed; - report.data = None; - report.metadata = match (report.metadata.take(), metadata) { - (Some(mut current_metadata), Some(new_metadata)) => { - current_metadata["output"] = new_metadata; - Some(current_metadata) - } - (None, Some(new_metadata)) => Some(json!({ "output": new_metadata })), - (Some(current_metadata), None) => Some(current_metadata), - _ => None, - }; - report.completed_at = Some(Utc::now()); - if let Err(e) = report.update(library).await { - error!(?e, "Failed to update job report;"); - } - - debug!(?report); - - invalidate_queries(library); - - return next_job; - } - // -> Job completed with errors - Ok(JobRunOutput { - metadata, - errors: JobRunErrors(errors), - next_job, - }) => { - warn!("Completed with errors"); - report.status = JobStatus::CompletedWithErrors; - report.errors_text = errors; - report.data = None; - report.metadata = match (report.metadata.take(), metadata) { - (Some(mut current_metadata), Some(new_metadata)) => { - current_metadata["output"] = new_metadata; - Some(current_metadata) - } - (None, Some(new_metadata)) => Some(json!({ "output": new_metadata })), - (Some(current_metadata), None) => Some(current_metadata), - _ => None, - }; - report.completed_at = Some(Utc::now()); - if let Err(e) = report.update(library).await { - error!(?e, "Failed to update job report;"); - } - - debug!(?report); - - invalidate_queries(library); - - return next_job; - } - // -> Job paused - Err(JobError::Paused(state, signal_tx)) => { - info!("Job paused, we will pause all children jobs"); - if let Err(e) = job.pause_children(library).await { - error!(?e, "Failed to pause children jobs;"); - } - - debug!("Setting worker status to paused"); - - report.status = JobStatus::Paused; - report.data = Some(state); - - if let Err(e) = report.update(library).await { - error!(?e, "Failed to update job report;"); - } - - debug!(?report); - - invalidate_queries(library); - - signal_tx.send(()).ok(); - } - // -> Job canceled - Err(JobError::Canceled(signal_tx)) => { - info!("Job canceled, we will cancel all children jobs"); - if let Err(e) = job.cancel_children(library).await { - error!(?e, "Failed to pause children jobs;"); - } - - debug!("Setting worker status to paused"); - - report.status = JobStatus::Canceled; - report.data = None; - - if let Err(e) = report.update(library).await { - error!(?e, "Failed to update job report;"); - } - - debug!(?report); - - invalidate_queries(library); - - signal_tx.send(()).ok(); - } - // -> Job failed - Err(e) => { - error!(?e, "Job failed with error;"); - if let Err(e) = job.cancel_children(library).await { - error!(?e, "Failed to cancel children jobs;"); - } - - report.status = JobStatus::Failed; - report.data = None; - if let Err(e) = report.update(library).await { - error!(?e, "Failed to update job report;"); - } - - warn!(?report); - - invalidate_queries(library); - } - } - - None - } -} - -struct JobWorkTable { - job: Box, - manager: Arc, - hash: u64, - report: OldJobReport, -} - -fn invalidate_queries(library: &Library) { - invalidate_query!(library, "jobs.isActive"); - invalidate_query!(library, "jobs.reports"); -} diff --git a/core/src/old_p2p/events.rs b/core/src/old_p2p/events.rs deleted file mode 100644 index 76694e858..000000000 --- a/core/src/old_p2p/events.rs +++ /dev/null @@ -1,159 +0,0 @@ -use std::{collections::HashSet, net::SocketAddr, sync::Arc}; - -use sd_old_p2p::{ - flume::bounded, hooks::QuicHandle, HookEvent, PeerConnectionCandidate, RemoteIdentity, P2P, -}; -use serde::Serialize; -use specta::Type; -use tokio::sync::broadcast; -use uuid::Uuid; - -use super::PeerMetadata; - -/// The method used for the connection with this peer. -/// *Technically* you can have multiple under the hood but this simplifies things for the UX. -#[derive(Debug, Clone, Serialize, Type)] -pub enum ConnectionMethod { - // Connected via the SD Relay - Relay, - // Connected directly via an IP address - Local, - // Not connected - Disconnected, -} - -/// The method used for the discovery of this peer. -/// *Technically* you can have multiple under the hood but this simplifies things for the UX. -#[derive(Debug, Clone, Serialize, Type)] -pub enum DiscoveryMethod { - // Found via the SD Relay - Relay, - // Found via mDNS or a manual IP - Local, - // Found via manual entry on either node - Manual, -} - -// This is used for synchronizing events between the backend and the frontend. -#[derive(Debug, Clone, Serialize, Type)] -#[serde(tag = "type")] -pub enum P2PEvent { - // An add or update event - PeerChange { - identity: RemoteIdentity, - connection: ConnectionMethod, - discovery: DiscoveryMethod, - metadata: PeerMetadata, - addrs: HashSet, - }, - // Delete a peer - PeerDelete { - identity: RemoteIdentity, - }, - SpacedropRequest { - id: Uuid, - identity: RemoteIdentity, - peer_name: String, - files: Vec, - }, - SpacedropProgress { - id: Uuid, - percent: u8, - }, - SpacedropTimedOut { - id: Uuid, - }, - SpacedropRejected { - id: Uuid, - }, -} - -/// A P2P hook which listens for events and sends them over a channel which can be connected to the frontend. -pub struct P2PEvents { - events: (broadcast::Sender, broadcast::Receiver), -} - -impl P2PEvents { - pub fn spawn(p2p: Arc, quic: Arc) -> Self { - let events = broadcast::channel(15); - let (tx, rx) = bounded(15); - let _ = p2p.register_hook("sd-frontend-events", tx); - - let events_tx = events.0.clone(); - tokio::spawn(async move { - while let Ok(event) = rx.recv_async().await { - let peer = match event { - HookEvent::PeerDisconnectedWith(_, identity) - | HookEvent::PeerExpiredBy(_, identity) => { - let peers = p2p.peers(); - let Some(peer) = peers.get(&identity) else { - let _ = events_tx.send(P2PEvent::PeerDelete { identity }); - continue; - }; - - peer.clone() - }, - // We use `HookEvent::PeerUnavailable`/`HookEvent::PeerAvailable` over `HookEvent::PeerExpiredBy`/`HookEvent::PeerDiscoveredBy` so that having an active connection is treated as "discovered". - // It's possible to have an active connection without mDNS data (which is what Peer*By` are for) - HookEvent::PeerConnectedWith(_, peer) - | HookEvent::PeerAvailable(peer) - // This will fire for updates to the mDNS metadata which are important for UX. - | HookEvent::PeerDiscoveredBy(_, peer) => peer, - HookEvent::PeerUnavailable(identity) => { - let _ = events_tx.send(P2PEvent::PeerDelete { identity }); - continue; - }, - HookEvent::Shutdown { _guard } => break, - _ => continue, - }; - - let Ok(metadata) = PeerMetadata::from_hashmap(&peer.metadata()).map_err(|err| { - println!("Invalid metadata for peer '{}': {err:?}", peer.identity()) - }) else { - continue; - }; - - let _ = events_tx.send(P2PEvent::PeerChange { - identity: peer.identity(), - connection: if peer.is_connected() { - if quic.is_relayed(peer.identity()) { - ConnectionMethod::Relay - } else { - ConnectionMethod::Local - } - } else { - ConnectionMethod::Disconnected - }, - discovery: if peer - .connection_candidates() - .iter() - .any(|c| matches!(c, PeerConnectionCandidate::Manual(_))) - { - DiscoveryMethod::Manual - } else if peer - .connection_candidates() - .iter() - .all(|c| *c == PeerConnectionCandidate::Relay) - { - DiscoveryMethod::Relay - } else { - DiscoveryMethod::Local - }, - metadata, - addrs: peer.addrs(), - }); - } - }); - - Self { events } - } - - pub fn subscribe(&self) -> broadcast::Receiver { - self.events.0.subscribe() - } - - #[allow(clippy::result_large_err)] - pub fn send(&self, event: P2PEvent) -> Result> { - self.events.0.send(event) - } -} diff --git a/core/src/old_p2p/libraries.rs b/core/src/old_p2p/libraries.rs deleted file mode 100644 index 43ebf7aed..000000000 --- a/core/src/old_p2p/libraries.rs +++ /dev/null @@ -1,139 +0,0 @@ -use std::{ - collections::HashMap, - sync::{Arc, Mutex, PoisonError}, -}; - -use sd_old_p2p::{hooks::QuicHandle, RemoteIdentity, P2P}; -use tracing::error; - -use crate::library::{Libraries, LibraryManagerEvent}; - -/// A P2P hook which integrates P2P into Spacedrive's library system. -/// -/// This hooks is responsible for: -/// - injecting library peers into the P2P system so we can connect to them over internet. -/// -pub fn libraries_hook(p2p: Arc, quic: Arc, libraries: Arc) { - let nodes_to_instance = Arc::new(Mutex::new(HashMap::new())); - - let handle = tokio::spawn({ - let quic = quic.clone(); - - async move { - if let Err(e) = libraries - .rx - .clone() - .subscribe(|msg| { - let p2p = p2p.clone(); - let nodes_to_instance = nodes_to_instance.clone(); - let quic = quic.clone(); - - async move { - match msg { - LibraryManagerEvent::InstancesModified(library) - | LibraryManagerEvent::Load(library) => { - p2p.metadata_mut().insert( - library.id.to_string(), - library.identity.to_remote_identity().to_string(), - ); - - let Ok(instances) = - library.db.instance().find_many(vec![]).exec().await - else { - return; - }; - - let mut nodes_to_instance = nodes_to_instance - .lock() - .unwrap_or_else(PoisonError::into_inner); - - for i in instances.iter() { - let identity = RemoteIdentity::from_bytes(&i.remote_identity) - .expect("invalid instance identity"); - let node_identity = RemoteIdentity::from_bytes( - i.node_remote_identity - .as_ref() - .expect("node remote identity is required"), - ) - .expect("invalid node remote identity"); - - // Skip self - if i.identity.is_some() { - continue; - } - - nodes_to_instance - .entry(identity) - .or_insert(vec![]) - .push(node_identity); - - quic.track_peer( - node_identity, - serde_json::from_slice( - i.metadata.as_ref().expect("this is a required field"), - ) - .expect("invalid metadata"), - ); - } - } - LibraryManagerEvent::Edit(_library) => { - // TODO: Send changes to all connected nodes or queue sending for when they are online! - } - LibraryManagerEvent::Delete(library) => { - p2p.metadata_mut().remove(&library.id.to_string()); - - let Ok(instances) = - library.db.instance().find_many(vec![]).exec().await - else { - return; - }; - - let mut nodes_to_instance = nodes_to_instance - .lock() - .unwrap_or_else(PoisonError::into_inner); - - for i in instances.iter() { - let identity = RemoteIdentity::from_bytes(&i.remote_identity) - .expect("invalid remote identity"); - let node_identity = RemoteIdentity::from_bytes( - i.node_remote_identity - .as_ref() - .expect("node remote identity is required"), - ) - .expect("invalid node remote identity"); - - // Skip self - if i.identity.is_some() { - continue; - } - - // Only remove if all instances pointing to this node are removed - let Some(identities) = nodes_to_instance.get_mut(&identity) - else { - continue; - }; - if let Some(i) = - identities.iter().position(|i| i == &node_identity) - { - identities.remove(i); - } - if identities.is_empty() { - quic.untrack_peer(node_identity); - } - } - } - } - } - }) - .await - { - error!(?e, "Core may become unstable! `LibraryServices::start` manager aborted with error;"); - } - } - }); - - tokio::spawn(async move { - quic.shutdown().await; - handle.abort(); - }); -} diff --git a/core/src/old_p2p/manager.rs b/core/src/old_p2p/manager.rs deleted file mode 100644 index a13616ce7..000000000 --- a/core/src/old_p2p/manager.rs +++ /dev/null @@ -1,466 +0,0 @@ -use crate::{ - node::{ - config::{self, P2PDiscoveryState}, - HardwareModel, - }, - old_p2p::{ - libraries::libraries_hook, operations, sync::SyncMessage, Header, OperatingSystem, - SPACEDRIVE_APP_ID, - }, - Node, -}; - -use axum::routing::IntoMakeService; - -use sd_old_p2p::{ - flume::{bounded, Receiver}, - hooks::{Libp2pPeerId, Mdns, QuicHandle, QuicTransport, RelayServerEntry}, - Peer, RemoteIdentity, UnicastStream, P2P, -}; -use sd_old_p2p_tunnel::Tunnel; -use serde::Serialize; -use serde_json::json; -use specta::Type; -use std::{ - collections::HashMap, - convert::Infallible, - sync::{atomic::AtomicBool, Arc, Mutex, PoisonError}, - time::Duration, -}; -use tower_service::Service; -use tracing::error; - -use tokio::sync::{oneshot, Notify}; -use tracing::info; -use uuid::Uuid; - -use super::{P2PEvents, PeerMetadata}; - -#[derive(Default, Clone, Serialize, Type)] -#[serde(tag = "type")] -pub enum ListenerState { - Listening, - Error { - error: String, - }, - #[default] - NotListening, -} - -#[derive(Default, Clone, Serialize, Type)] -pub struct Listeners { - ipv4: ListenerState, - ipv6: ListenerState, - relay: ListenerState, -} - -pub struct P2PManager { - pub(crate) p2p: Arc, - mdns: Mutex>, - quic_transport: QuicTransport, - pub quic: Arc, - // The `libp2p::PeerId`. This is for debugging only, use `RemoteIdentity` instead. - lp2p_peer_id: Libp2pPeerId, - pub(crate) events: P2PEvents, - pub(super) spacedrop_pairing_reqs: Arc>>>>, - pub(super) spacedrop_cancellations: Arc>>>, - pub(crate) node_config: Arc, - pub listeners: Mutex, - relay_config: Mutex>, - trigger_relay_config_update: Notify, -} - -impl P2PManager { - pub async fn new( - node_config: Arc, - libraries: Arc, - ) -> Result< - ( - Arc, - impl FnOnce(Arc, IntoMakeService>), - ), - String, - > { - let (tx, rx) = bounded(25); - let p2p = P2P::new(SPACEDRIVE_APP_ID, node_config.get().await.identity, tx); - let (quic, lp2p_peer_id) = QuicTransport::spawn(p2p.clone()).map_err(|e| e.to_string())?; - libraries_hook(p2p.clone(), quic.handle(), libraries); - let this = Arc::new(Self { - p2p: p2p.clone(), - lp2p_peer_id, - mdns: Mutex::new(None), - events: P2PEvents::spawn(p2p.clone(), quic.handle()), - quic: quic.handle(), - quic_transport: quic, - spacedrop_pairing_reqs: Default::default(), - spacedrop_cancellations: Default::default(), - node_config, - listeners: Default::default(), - relay_config: Default::default(), - trigger_relay_config_update: Default::default(), - }); - this.on_node_config_change().await; - - info!( - remote_identity = %this.p2p.remote_identity(), - peer_id = ?this.lp2p_peer_id, - addresses = ?this.p2p.listeners(), - "Node is now online listening;", - ); - - Ok((this.clone(), |node: Arc, router| { - tokio::spawn(start(this.clone(), node.clone(), rx, router)); - - // TODO: Cleanup this thread on p2p shutdown. - tokio::spawn(async move { - let client = reqwest::Client::new(); - loop { - match client - // FIXME(@fogodev): hardcoded URL for now as I'm moving stuff around - .get(format!("{}/api/p2p/relays", "https://app.spacedrive.com")) - .send() - .await - { - Ok(resp) => { - if resp.status() != 200 { - error!( - "Failed to pull p2p relay configuration: {} {:?}", - resp.status(), - resp.text().await - ); - } else { - match resp.json::>().await { - Ok(config) => { - node.p2p - .relay_config - .lock() - .unwrap_or_else(PoisonError::into_inner) - .clone_from(&config); - - let config = { - let node_config = node.config.get().await; - if !node_config.p2p.disabled - && !node_config.p2p.disable_relay - { - config - } else { - vec![] - } - }; - let no_relays = config.len(); - - this.listeners - .lock() - .unwrap_or_else(PoisonError::into_inner) - .relay = match this.quic_transport.set_relay_config(config).await { - Ok(_) => { - info!( - "Updated p2p relay configuration successfully." - ); - if no_relays == 0 { - this.quic.disable(); - - ListenerState::NotListening - } else { - this.quic.enable(); - - ListenerState::Listening - } - } - Err(err) => ListenerState::Error { - error: err.to_string(), - }, - }; - } - Err(e) => { - error!(?e, "Failed to parse p2p relay configuration;") - } - } - } - } - Err(e) => error!(?e, "Error pulling p2p relay configuration;"), - } - - tokio::select! { - _ = this.trigger_relay_config_update.notified() => {} - _ = tokio::time::sleep(Duration::from_secs(11 * 60)) => {} - } - } - }); - })) - } - - pub fn peer_metadata(&self) -> HashMap { - self.p2p.metadata().clone() - } - - // TODO: Remove this and add a subscription system to `config::Manager` - pub async fn on_node_config_change(&self) { - self.trigger_relay_config_update.notify_waiters(); - - let config = self.node_config.get().await; - - if config.p2p.discovery == P2PDiscoveryState::ContactsOnly { - PeerMetadata::remove(&mut self.p2p.metadata_mut()); - - // TODO: Hash Spacedrive account ID and put it in the metadata. - } else { - PeerMetadata { - name: config.name.clone(), - operating_system: Some(OperatingSystem::get_os()), - device_model: Some(HardwareModel::try_get().unwrap_or(HardwareModel::Other)), - version: Some(env!("CARGO_PKG_VERSION").to_string()), - } - .update(&mut self.p2p.metadata_mut()); - } - - let port = config.p2p.port.get(); - - let ipv4_port = (!config.p2p.disabled).then_some(port); - info!(?ipv4_port, "Setting quic ipv4 listener;"); - self.listeners - .lock() - .unwrap_or_else(PoisonError::into_inner) - .ipv4 = if let Err(e) = self.quic_transport.set_ipv4_enabled(ipv4_port).await { - error!(?e, "Failed to enabled quic ipv4 listener;"); - self.node_config - .write(|c| c.p2p.disabled = false) - .await - .ok(); - - ListenerState::Error { - error: e.to_string(), - } - } else { - match !config.p2p.disabled { - true => ListenerState::Listening, - false => ListenerState::NotListening, - } - }; - - let enable_ipv6 = !config.p2p.disabled && !config.p2p.disable_ipv6; - let ipv6_port = enable_ipv6.then_some(port); - info!(?ipv6_port, "Setting quic ipv6 listener;"); - self.listeners - .lock() - .unwrap_or_else(PoisonError::into_inner) - .ipv6 = if let Err(e) = self.quic_transport.set_ipv6_enabled(ipv6_port).await { - error!(?e, "Failed to enabled quic ipv6 listener;"); - self.node_config - .write(|c| c.p2p.disable_ipv6 = false) - .await - .ok(); - - ListenerState::Error { - error: e.to_string(), - } - } else { - match enable_ipv6 { - true => ListenerState::Listening, - false => ListenerState::NotListening, - } - }; - - self.quic_transport - .set_manual_peer_addrs(config.p2p.manual_peers); - - let should_revert = match (config.p2p.disabled, config.p2p.discovery) { - (true, _) | (_, P2PDiscoveryState::Disabled) => { - let mdns = { - let mut mdns = self.mdns.lock().unwrap_or_else(PoisonError::into_inner); - mdns.take() - }; - if let Some(mdns) = mdns { - mdns.shutdown().await; - info!("mDNS shutdown successfully."); - } - - false - } - (_, P2PDiscoveryState::Everyone | P2PDiscoveryState::ContactsOnly) => { - let mut mdns = self.mdns.lock().unwrap_or_else(PoisonError::into_inner); - if mdns.is_none() { - match Mdns::spawn(self.p2p.clone()) { - Ok(m) => { - info!("mDNS started successfully."); - *mdns = Some(m); - false - } - Err(e) => { - error!(?e, "Failed to start mDNS;"); - true - } - } - } else { - false - } - } - }; - - // The `should_revert` bit is weird but we need this future to stay `Send` as rspc requires. - // To make it send we have to drop `quic` (a `!Send` `MutexGuard`). - // Doing it within the above scope seems to not work (even when manually calling `drop`). - if should_revert { - let _ = self - .node_config - .write(|c| c.p2p.discovery = P2PDiscoveryState::Disabled) - .await; - } - } - - pub fn get_library_instances(&self, library: &Uuid) -> Vec<(RemoteIdentity, Arc)> { - let library_id = library.to_string(); - self.p2p - .peers() - .iter() - .filter(|(_, p)| p.metadata().contains_key(&library_id)) - .map(|(i, p)| (*i, p.clone())) - .collect() - } - - pub fn get_instance(&self, library: &Uuid, identity: RemoteIdentity) -> Option> { - let library_id = library.to_string(); - self.p2p - .peers() - .iter() - .find(|(i, p)| **i == identity && p.metadata().contains_key(&library_id)) - .map(|(_, p)| p.clone()) - } - - pub async fn state(&self) -> serde_json::Value { - let listeners = self.p2p.listeners(); - let node_config = self.node_config.get().await; - json!({ - "self_identity": self.p2p.remote_identity().to_string(), - "self_peer_id": format!("{:?}", self.lp2p_peer_id), - "metadata": self.p2p.metadata().clone(), - "peers": self.p2p.peers().iter().map(|(identity, p)| json!({ - "identity": identity.to_string(), - "metadata": p.metadata().clone(), - "can_connect": p.can_connect(), - "is_connected": p.is_connected(), - "active_connections": p.active_connections(), - "connection_methods": p.connection_methods().iter().map(|id| format!("{:?}", id)).collect::>(), - "discovered_by": p.discovered_by().iter().map(|id| format!("{:?}", id)).collect::>(), - "candidates": p.connection_candidates().iter().map(|a| format!("{a:?}")).collect::>(), - })).collect::>(), - "hooks": self.p2p.hooks().iter().map(|(id, name)| json!({ - "id": format!("{:?}", id), - "name": name, - "listener_addrs": listeners.iter().find(|l| l.is_hook_id(*id)).map(|l| l.addrs.clone()), - })).collect::>(), - "config": node_config.p2p, - "relay_config": self.quic_transport.get_relay_config(), - "listeners": self.listeners.lock().unwrap_or_else(PoisonError::into_inner).clone(), - }) - } - - pub async fn shutdown(&self) { - // `self.p2p` will automatically take care of shutting down all the hooks. Eg. `self.quic`, `self.mdns`, etc. - self.p2p.shutdown().await; - } -} - -async fn start( - this: Arc, - node: Arc, - rx: Receiver, - mut service: IntoMakeService>, -) -> Result<(), ()> { - while let Ok(mut stream) = rx.recv_async().await { - let this = this.clone(); - let node = node.clone(); - let mut service = unwrap_infallible(service.call(()).await); - - tokio::spawn(async move { - let Ok(header) = Header::from_stream(&mut stream).await.map_err(|e| { - error!(?e, "Failed to read header from stream;"); - }) else { - return; - }; - - match header { - Header::Ping => operations::ping::receiver(stream).await, - Header::Spacedrop(req) => { - let Err(()) = operations::spacedrop::receiver(&this, req, stream).await else { - return; - }; - - error!("Failed to handle Spacedrop request"); - } - Header::Sync => { - let Ok(mut tunnel) = Tunnel::responder(stream).await.map_err(|e| { - error!(?e, "Failed `Tunnel::responder`;"); - }) else { - return; - }; - - let Ok(msg) = SyncMessage::from_stream(&mut tunnel).await.map_err(|e| { - error!(?e, "Failed `SyncMessage::from_stream`"); - }) else { - return; - }; - - let Ok(library) = node - .libraries - .get_library_for_instance(&tunnel.library_remote_identity()) - .await - .ok_or_else(|| { - error!(remove_identity = %tunnel.library_remote_identity(), "Failed to get library;"); - - // TODO: Respond to remote client with warning! - }) - else { - return; - }; - - match msg { - SyncMessage::NewOperations => { - let Err(()) = super::sync::responder(&mut tunnel, library).await else { - return; - }; - - error!("Failed to handle sync responder request"); - } - }; - } - Header::RspcRemote => { - let remote = stream.remote_identity(); - let Err(e) = operations::rspc::receiver(stream, &mut service, &node).await - else { - return; - }; - - error!(%remote, ?e, "Failed to handling rspc request;"); - } - Header::LibraryFile { - file_path_id, - range, - } => { - let remote = stream.remote_identity(); - let Err(e) = - operations::library::receiver(stream, file_path_id, range, &node).await - else { - return; - }; - - error!( - ?remote, - %file_path_id, - ?e, - "Failed to handling library file request;", - ); - } - }; - }); - } - - Ok::<_, ()>(()) -} - -fn unwrap_infallible(result: Result) -> T { - match result { - Ok(value) => value, - Err(err) => match err {}, - } -} diff --git a/core/src/old_p2p/metadata.rs b/core/src/old_p2p/metadata.rs deleted file mode 100644 index 054eea0ca..000000000 --- a/core/src/old_p2p/metadata.rs +++ /dev/null @@ -1,132 +0,0 @@ -use crate::node::{HardwareModel, Platform}; - -use std::{collections::HashMap, env, fmt::Display, str::FromStr}; - -use serde::{Deserialize, Serialize}; -use specta::Type; - -#[derive(Debug, Clone, Type, Serialize, Deserialize)] -pub struct PeerMetadata { - pub name: String, - pub operating_system: Option, - pub device_model: Option, - pub version: Option, -} - -impl PeerMetadata { - pub fn remove(map: &mut HashMap) { - map.remove("name"); - map.remove("os"); - map.remove("device_model"); - map.remove("version"); - } - - pub fn update(self, map: &mut HashMap) { - map.insert("name".to_owned(), self.name.clone()); - if let Some(os) = self.operating_system { - map.insert("os".to_owned(), os.to_string()); - } - if let Some(version) = self.version { - map.insert("version".to_owned(), version); - } - if let Some(device_model) = self.device_model { - map.insert("device_model".to_owned(), device_model.to_string()); - } - } - - pub fn from_hashmap(data: &HashMap) -> Result { - Ok(Self { - name: data - .get("name") - .ok_or_else(|| { - "DNS record for field 'name' missing. Unable to decode 'PeerMetadata'!" - .to_owned() - })? - .to_owned(), - operating_system: data - .get("os") - .map(|os| os.parse().map_err(|_| "Unable to parse 'OperationSystem'!")) - .transpose()?, - device_model: Some(HardwareModel::from( - data.get("device_model") - .map(|s| s.as_str()) - .unwrap_or("Other"), - )), - version: data.get("version").map(|v| v.to_owned()), - }) - } -} - -/// Represents the operating system which the remote peer is running. -/// This is not used internally and predominantly is designed to be used for display purposes by the embedding application. -#[derive(Debug, Clone, Type, Serialize, Deserialize)] -pub enum OperatingSystem { - Windows, - Linux, - MacOS, - Ios, - Android, - Other(String), -} - -// TODO: Should `Platform` and `OperatingSystem` be merged into one? -impl From for OperatingSystem { - fn from(platform: Platform) -> Self { - match platform { - Platform::Unknown => OperatingSystem::Other("Unknown".into()), - Platform::Windows => OperatingSystem::Windows, - Platform::Linux => OperatingSystem::Linux, - Platform::MacOS => OperatingSystem::MacOS, - Platform::IOS => OperatingSystem::Ios, - Platform::Android => OperatingSystem::Android, - } - } -} - -impl OperatingSystem { - pub fn get_os() -> Self { - match env::consts::OS { - "windows" => OperatingSystem::Windows, - "macos" => OperatingSystem::MacOS, - "linux" => OperatingSystem::Linux, - "ios" => OperatingSystem::Ios, - "android" => OperatingSystem::Android, - platform => OperatingSystem::Other(platform.into()), - } - } -} - -impl Display for OperatingSystem { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let s = match self { - OperatingSystem::Windows => "Windows", - OperatingSystem::Linux => "Linux", - OperatingSystem::MacOS => "MacOS", - OperatingSystem::Ios => "IOS", - OperatingSystem::Android => "Android", - OperatingSystem::Other(s) => { - let mut chars = s.chars(); - chars.next(); - chars.as_str() - } - }; - - f.write_str(s) - } -} - -impl FromStr for OperatingSystem { - type Err = (); - - fn from_str(s: &str) -> Result { - let mut chars = s.chars(); - match chars.next() { - Some('W') => Ok(OperatingSystem::Windows), - Some('L') => Ok(OperatingSystem::Linux), - Some('M') => Ok(OperatingSystem::MacOS), - Some('I') => Ok(OperatingSystem::Ios), - Some('A') => Ok(OperatingSystem::Android), - _ => Ok(OperatingSystem::Other(s.to_owned())), - } - } -} diff --git a/core/src/old_p2p/mod.rs b/core/src/old_p2p/mod.rs deleted file mode 100644 index 5cff15717..000000000 --- a/core/src/old_p2p/mod.rs +++ /dev/null @@ -1,17 +0,0 @@ -#![warn(clippy::all, clippy::unwrap_used, clippy::panic)] -#![allow(clippy::unnecessary_cast)] // Yeah they aren't necessary on this arch, but they are on others - -mod events; -pub(super) mod libraries; -mod manager; -mod metadata; -pub mod operations; -mod protocol; -pub mod sync; - -pub use events::*; -pub use manager::*; -pub use metadata::*; -pub use protocol::*; - -pub(super) const SPACEDRIVE_APP_ID: &str = "sd"; diff --git a/core/src/old_p2p/operations/library.rs b/core/src/old_p2p/operations/library.rs deleted file mode 100644 index 5998407bd..000000000 --- a/core/src/old_p2p/operations/library.rs +++ /dev/null @@ -1,134 +0,0 @@ -use std::{ - error::Error, - path::Path, - sync::{atomic::AtomicBool, Arc}, -}; - -use sd_core_file_path_helper::IsolatedFilePathData; -use sd_core_prisma_helpers::file_path_to_handle_p2p_serve_file; -use sd_old_p2p::{Identity, RemoteIdentity, UnicastStream, P2P}; -use sd_old_p2p_block::{BlockSize, Range, SpaceblockRequest, SpaceblockRequests, Transfer}; -use sd_prisma::prisma::file_path; -use tokio::{ - fs::File, - io::{AsyncReadExt, AsyncWrite, AsyncWriteExt, BufReader}, -}; -use tracing::debug; -use uuid::Uuid; - -use crate::{old_p2p::Header, Node}; - -/// Request a file from a remote library -#[allow(unused)] -pub async fn request_file( - p2p: Arc, - identity: RemoteIdentity, - library_identity: &Identity, - file_path_id: Uuid, - range: Range, - output: impl AsyncWrite + Unpin, -) -> Result<(), Box> { - let peer = p2p.peers().get(&identity).ok_or("Peer offline")?.clone(); - let mut stream = peer.new_stream().await?; - - stream - .write_all( - &Header::LibraryFile { - file_path_id, - range: range.clone(), - } - .to_bytes(), - ) - .await?; - - let mut stream = sd_old_p2p_tunnel::Tunnel::initiator(stream, library_identity).await?; - - let block_size = BlockSize::from_stream(&mut stream).await?; - let size = stream.read_u64_le().await?; - - Transfer::new( - &SpaceblockRequests { - id: Uuid::new_v4(), - block_size, - requests: vec![SpaceblockRequest { - name: "_".to_string(), - size, - range, - }], - }, - |percent| debug!("P2P receiving file path {file_path_id:?} - progress {percent}%"), - &Arc::new(AtomicBool::new(false)), - ) - .receive(&mut stream, output) - .await; - - Ok(()) -} - -pub(crate) async fn receiver( - stream: UnicastStream, - file_path_id: Uuid, - range: Range, - node: &Arc, -) -> Result<(), Box> { - debug!( - "Received library request from peer '{}'", - stream.remote_identity() - ); - - // The tunnel takes care of authentication and encrypts all traffic to the library to be certain we are talking to a node with the library. - let mut stream = sd_old_p2p_tunnel::Tunnel::responder(stream).await?; - - let library = node - .libraries - .get_library_for_instance(&stream.library_remote_identity()) - .await - .ok_or_else(|| format!("Library not found: {:?}", stream.library_remote_identity()))?; - - let file_path = library - .db - .file_path() - .find_unique(file_path::pub_id::equals(file_path_id.as_bytes().to_vec())) - .select(file_path_to_handle_p2p_serve_file::select()) - .exec() - .await? - .ok_or_else(|| format!("File path {file_path_id:?} not found in {:?}", library.id))?; - - let location = file_path.location.as_ref().expect("included in query"); - let location_path = location.path.as_ref().expect("included in query"); - let path = - Path::new(location_path).join(IsolatedFilePathData::try_from((location.id, &file_path))?); - - debug!( - "Serving path {path:?} for library {:?} over P2P", - library.id - ); - - let file = File::open(&path).await?; - - let metadata = file.metadata().await?; - let block_size = BlockSize::from_file_size(metadata.len()); - - stream.write_all(&block_size.to_bytes()).await?; - stream.write_all(&metadata.len().to_le_bytes()).await?; - - let file = BufReader::new(file); - Transfer::new( - &SpaceblockRequests { - id: Uuid::new_v4(), - block_size, - requests: vec![SpaceblockRequest { - name: "_".into(), - size: metadata.len(), - range, - }], - }, - |percent| debug!("P2P loading file path {file_path_id:?} - progress {percent}%"), - // TODO: Properly handle cancellation with webview - &Arc::new(AtomicBool::new(false)), - ) - .send(&mut stream, file) - .await?; - - Ok(()) -} diff --git a/core/src/old_p2p/operations/mod.rs b/core/src/old_p2p/operations/mod.rs deleted file mode 100644 index 32ef65e29..000000000 --- a/core/src/old_p2p/operations/mod.rs +++ /dev/null @@ -1,8 +0,0 @@ -pub mod library; -pub mod ping; -pub mod rspc; -pub mod spacedrop; - -pub use library::request_file; -pub use rspc::remote_rspc; -pub use spacedrop::spacedrop; diff --git a/core/src/old_p2p/operations/ping.rs b/core/src/old_p2p/operations/ping.rs deleted file mode 100644 index e1c014b79..000000000 --- a/core/src/old_p2p/operations/ping.rs +++ /dev/null @@ -1,37 +0,0 @@ -use std::{error::Error, sync::Arc}; - -use sd_old_p2p::{RemoteIdentity, UnicastStream, P2P}; -use tokio::io::{AsyncReadExt, AsyncWriteExt}; -use tracing::debug; - -use crate::old_p2p::Header; - -/// Send a ping to all peers we are connected to -#[allow(unused)] -pub async fn ping(p2p: Arc, identity: RemoteIdentity) -> Result<(), Box> { - let peer = p2p - .peers() - .get(&identity) - .ok_or("Peer not found, has it been discovered?")? - .clone(); - let mut stream = peer.new_stream().await?; - - stream.write_all(&Header::Ping.to_bytes()).await?; - - let mut result = [0; 4]; - let _ = stream.read_exact(&mut result).await?; - if result != *b"PONG" { - return Err("Failed to receive pong".into()); - } - - Ok(()) -} - -pub(crate) async fn receiver(mut stream: UnicastStream) { - debug!(peer = %stream.remote_identity(), "Received ping from;"); - - stream - .write_all(b"PONG") - .await - .expect("Failed to send pong"); -} diff --git a/core/src/old_p2p/operations/rspc.rs b/core/src/old_p2p/operations/rspc.rs deleted file mode 100644 index 9c1b56c0e..000000000 --- a/core/src/old_p2p/operations/rspc.rs +++ /dev/null @@ -1,63 +0,0 @@ -use std::{error::Error, sync::Arc}; - -use axum::{extract::Request, http, Router}; -use hyper::{body::Incoming, client::conn::http1::handshake, server::conn::http1, Response}; -use hyper_util::rt::TokioIo; -use sd_old_p2p::{RemoteIdentity, UnicastStream, P2P}; -use tokio::io::AsyncWriteExt; -use tower_service::Service; -use tracing::debug; - -use crate::{old_p2p::Header, Node}; - -/// Transfer an rspc query to a remote node. -pub async fn remote_rspc( - p2p: Arc, - identity: RemoteIdentity, - request: http::Request, -) -> Result, Box> { - let peer = p2p - .peers() - .get(&identity) - .ok_or("Peer not found, has it been discovered?")? - .clone(); - let mut stream = peer.new_stream().await?; - - stream.write_all(&Header::RspcRemote.to_bytes()).await?; - - let (mut sender, conn) = handshake(TokioIo::new(stream)).await?; - tokio::task::spawn(async move { - if let Err(e) = conn.await { - println!("Connection error: {:?}", e); - } - }); - - sender.send_request(request).await.map_err(Into::into) -} - -pub(crate) async fn receiver( - stream: UnicastStream, - service: &mut Router, - node: &Node, -) -> Result<(), Box> { - debug!( - peer = %stream.remote_identity(), - "Received http request from;", - ); - - // TODO: Authentication - #[allow(clippy::todo)] - if !node.config.get().await.p2p.enable_remote_access { - todo!("No way buddy!"); - } - - let hyper_service = - hyper::service::service_fn(move |request: Request| service.clone().call(request)); - - http1::Builder::new() - .keep_alive(true) - .serve_connection(TokioIo::new(stream), hyper_service) - .with_upgrades() - .await - .map_err(Into::into) -} diff --git a/core/src/old_p2p/operations/spacedrop.rs b/core/src/old_p2p/operations/spacedrop.rs deleted file mode 100644 index a97aa2318..000000000 --- a/core/src/old_p2p/operations/spacedrop.rs +++ /dev/null @@ -1,382 +0,0 @@ -use std::{ - borrow::Cow, - path::PathBuf, - sync::{ - atomic::{AtomicBool, Ordering}, - Arc, PoisonError, - }, - time::Duration, -}; - -use crate::old_p2p::{Header, P2PEvent, P2PManager}; -use futures::future::join_all; -use sd_old_p2p::{RemoteIdentity, UnicastStream}; -use sd_old_p2p_block::{BlockSize, Range, SpaceblockRequest, SpaceblockRequests, Transfer}; -use thiserror::Error; -use tokio::{ - fs::{create_dir_all, File}, - io::{AsyncReadExt, AsyncWriteExt, BufReader, BufWriter}, - sync::oneshot, - time::{sleep, Instant}, -}; -use tracing::{debug, error, info, warn}; -use uuid::Uuid; - -/// The amount of time to wait for a Spacedrop request to be accepted or rejected before it's automatically rejected -pub(crate) const SPACEDROP_TIMEOUT: Duration = Duration::from_secs(60); - -#[derive(Debug, Error)] -pub enum SpacedropError { - #[error("paths argument is an empty vector")] - EmptyPath, - #[error("error connecting to peer")] - FailedPeerConnection, - #[error("error creating stream: {0}")] - FailedNewStream(#[from] sd_old_p2p::NewStreamError), - #[error("error opening file: {0}")] - FailedFileOpen(#[from] std::io::Error), -} - -pub async fn spacedrop( - p2p: Arc, - identity: RemoteIdentity, - paths: Vec, -) -> Result { - if paths.is_empty() { - return Err(SpacedropError::EmptyPath); - } - - let (files, requests): (Vec<_>, Vec<_>) = join_all(paths.into_iter().map(|path| async move { - let file = File::open(&path).await?; - let metadata = file.metadata().await?; - let name = path - .file_name() - .map(|v| v.to_string_lossy()) - .unwrap_or(Cow::Borrowed("")) - .to_string(); - - Ok(( - (path, file), - SpaceblockRequest { - name, - size: metadata.len(), - range: Range::Full, - }, - )) - })) - .await - .into_iter() - .collect::, std::io::Error>>() - .map_err(SpacedropError::FailedFileOpen)? - .into_iter() - .unzip(); - - let total_length: u64 = requests.iter().map(|req| req.size).sum(); - - let id = Uuid::new_v4(); - debug!(spacedrop_id = %id, peer = %identity, "Starting Spacedrop;"); - let peer = p2p - .p2p - .peers() - .get(&identity) - .ok_or_else(|| { - debug!(spacedrop_id = %id, peer = %identity, "Failed to find connection method;"); - SpacedropError::FailedPeerConnection - })? - .clone(); - - let mut stream = peer.new_stream().await.map_err(|e| { - debug!(spacedrop_id = %id, peer = %identity, ?e, "Failed to connect"); - SpacedropError::FailedNewStream(e) - })?; - - tokio::spawn(async move { - debug!(spacedrop_id = %id, "Connected, sending header"); - let header = Header::Spacedrop(SpaceblockRequests { - id, - block_size: BlockSize::from_file_size(total_length), - requests, - }); - if let Err(e) = stream.write_all(&header.to_bytes()).await { - debug!(spacedrop_id = %id, ?e, "Failed to send header"); - return; - } - let Header::Spacedrop(requests) = header else { - unreachable!(); - }; - - debug!(spacedrop_id = %id, "Waiting for response"); - let result = tokio::select! { - result = stream.read_u8() => result, - // Add 5 seconds incase the user responded on the deadline and slow network - _ = sleep(SPACEDROP_TIMEOUT + Duration::from_secs(5)) => { - debug!(spacedrop_id = %id, "Timed out, cancelling"); - p2p.events.send(P2PEvent::SpacedropTimedOut { id }).ok(); - return; - }, - }; - - match result { - Ok(0) => { - debug!(spacedrop_id = %id, peer = %identity, "Spacedrop was rejected from;"); - p2p.events.send(P2PEvent::SpacedropRejected { id }).ok(); - return; - } - Ok(1) => {} // Okay - Ok(_) => todo!(), // TODO: Proper error - Err(e) => todo!("{:?}", e), // TODO: Proper error - } - - let cancelled = Arc::new(AtomicBool::new(false)); - p2p.spacedrop_cancellations - .lock() - .unwrap_or_else(PoisonError::into_inner) - .insert(id, cancelled.clone()); - - debug!(spacedrop_id = %id, "Starting transfer"); - let i = Instant::now(); - - let mut transfer = Transfer::new( - &requests, - |percent| { - p2p.events - .send(P2PEvent::SpacedropProgress { id, percent }) - .ok(); - }, - &cancelled, - ); - - for (file_id, (path, file)) in files.into_iter().enumerate() { - debug!( - spacedrop_id = %id, - %file_id, - path = %path.display(), - "Transmitting;", - ); - - let file = BufReader::new(file); - if let Err(e) = transfer.send(&mut stream, file).await { - debug!( - spacedrop_id = %id, - %file_id, - ?e, - "Failed to send file;"); - // TODO: Error to frontend - // p2p.events - // .send(P2PEvent::SpacedropFailed { id, file_id }) - // .ok(); - return; - } - } - - debug!(spacedrop_id = %id, elapsed_time = ?i.elapsed(), "Finished;"); - }); - - Ok(id) -} - -// TODO: Move these off the manager -impl P2PManager { - pub async fn accept_spacedrop(&self, id: Uuid, path: String) { - if let Some(chan) = self - .spacedrop_pairing_reqs - .lock() - .unwrap_or_else(PoisonError::into_inner) - .remove(&id) - { - chan.send(Some(path)) - .map_err(|e| { - warn!(spacedrop_id = %id, ?e, "Error accepting Spacedrop;"); - }) - .ok(); - } - } - - pub async fn reject_spacedrop(&self, id: Uuid) { - if let Some(chan) = self - .spacedrop_pairing_reqs - .lock() - .unwrap_or_else(PoisonError::into_inner) - .remove(&id) - { - chan.send(None) - .map_err(|e| { - warn!(spacedrop_id = %id, ?e, "Error rejecting Spacedrop;"); - }) - .ok(); - } - } - - pub async fn cancel_spacedrop(&self, id: Uuid) { - if let Some(cancelled) = self - .spacedrop_cancellations - .lock() - .unwrap_or_else(PoisonError::into_inner) - .remove(&id) - { - cancelled.store(true, Ordering::Relaxed); - } - } -} - -pub(crate) async fn receiver( - this: &Arc, - req: SpaceblockRequests, - mut stream: UnicastStream, -) -> Result<(), ()> { - let id = req.id; - let (tx, rx) = oneshot::channel(); - - info!( - spacedrop_id = %id, - files_count = req.requests.len(), - peer = %stream.remote_identity(), - block_size = ?req.block_size, - "Receiving spacedrop files;", - ); - this.spacedrop_pairing_reqs - .lock() - .unwrap_or_else(PoisonError::into_inner) - .insert(id, tx); - - if this - .events - .send(P2PEvent::SpacedropRequest { - id, - identity: stream.remote_identity(), - peer_name: "Unknown".into(), - // TODO: A better solution to this - // manager - // .get_discovered_peers() - // .await - // .into_iter() - // .find(|p| p.peer_id == event.peer_id) - // .map(|p| p.metadata.name) - // .unwrap_or_else(|| "Unknown".to_string()), - files: req - .requests - .iter() - .map(|req| req.name.clone()) - .collect::>(), - }) - .is_err() - { - // No frontend's are active - - // TODO: Implement this - error!("TODO: Outright reject Spacedrop"); - } - - tokio::select! { - _ = sleep(SPACEDROP_TIMEOUT) => { - info!(spacedrop_id = %id, "Timeout, rejecting!;"); - - stream.write_all(&[0]).await.map_err(|e| { - error!(spacedrop_id = %id, ?e, "Error reject bit;"); - })?; - stream.flush().await.map_err(|e| { - error!(spacedrop_id = %id, ?e, "Error flushing reject bit;"); - })?; - } - file_path = rx => { - match file_path { - Ok(Some(file_path)) => { - info!(spacedrop_id = %id, saving_to = %file_path, "Accepted;"); - - let cancelled = Arc::new(AtomicBool::new(false)); - this.spacedrop_cancellations - .lock() - .unwrap_or_else(PoisonError::into_inner) - .insert(id, cancelled.clone()); - - stream.write_all(&[1]).await.map_err(|e| { - error!(spacedrop_id = %id, ?e, "Error sending continuation bit;"); - - // TODO: Send error to the frontend - - // TODO: make sure the other peer times out or we retry??? - })?; - - let names = req.requests.iter().map(|req| req.name.clone()).collect::>(); - let mut transfer = Transfer::new(&req, |percent| { - this.events.send(P2PEvent::SpacedropProgress { id, percent }).ok(); - }, &cancelled); - - let file_path = PathBuf::from(file_path); - let names_len = names.len(); - for file_name in names { - // When transferring more than 1 file we wanna join the incoming file name to the directory provided by the user - let mut path = file_path.clone(); - if names_len != 1 { - // We know the `file_path` will be a directory so we can just push the file name to it - path.push(&file_name); - } - - debug!( - spacedrop_id = %id, - %file_name, - saving_to = %path.display(), - "Accepting;", - ); - - if let Some(parent) = path.parent() { - create_dir_all(&parent).await.map_err(|e| { - error!( - spacedrop_id = %id, - parent = %parent.display(), - ?e, - "Error creating parent directory;"); - - // TODO: Send error to the frontend - - // TODO: Send error to remote peer - })?; - } - - let f = File::create(&path).await.map_err(|e| { - error!( - spacedrop_id = %id, - creating_file_at = %path.display(), - ?e, - "Error creating file;", - ); - - // TODO: Send error to the frontend - - // TODO: Send error to remote peer - })?; - let f = BufWriter::new(f); - if let Err(e) = transfer.receive(&mut stream, f).await { - error!( - spacedrop_id = %id, - %file_name, - ?e, - "Error receiving file;"); - - // TODO: Send error to frontend - - break; - } - } - - info!(spacedrop_id = %id, "Completed;"); - } - Ok(None) => { - info!(spacedrop_id = %id, "Rejected;"); - - stream.write_all(&[0]).await.map_err(|e| { - error!(spacedrop_id = %id, ?e, "Error sending rejection;"); - })?; - stream.flush().await.map_err(|e| { - error!(spacedrop_id = %id, ?e, "Error flushing rejection;"); - })?; - } - Err(_) => { - warn!(spacedrop_id = %id, "Error with Spacedrop pairing request receiver!;"); - } - } - } - }; - - Ok(()) -} diff --git a/core/src/old_p2p/protocol.rs b/core/src/old_p2p/protocol.rs deleted file mode 100644 index dac686a76..000000000 --- a/core/src/old_p2p/protocol.rs +++ /dev/null @@ -1,131 +0,0 @@ -use sd_old_p2p_block::{Range, SpaceblockRequests, SpaceblockRequestsError}; -use sd_old_p2p_proto::{decode, encode}; -use thiserror::Error; -use tokio::io::{AsyncRead, AsyncReadExt}; -use uuid::Uuid; - -/// TODO -#[derive(Debug, PartialEq, Eq)] -pub enum Header { - /// Basic pin protocol for demonstrating the P2P system - Ping, - /// Spacedrop file sending - Spacedrop(SpaceblockRequests), - /// Used for sending sync messages between nodes. - Sync, - // A HTTP server used for rspc requests and streaming files - RspcRemote, - // Request a file within a library - // We don't include a library ID here as it's taken care of by `sd_p2p_tunnel::Tunnel`. - LibraryFile { - file_path_id: Uuid, - range: Range, - }, -} - -#[derive(Debug, Error)] -pub enum HeaderError { - #[error("io error reading discriminator: {0}")] - DiscriminatorIo(std::io::Error), - #[error("invalid discriminator '{0}'")] - DiscriminatorInvalid(u8), - #[error("error reading spacedrop request: {0}")] - SpacedropRequest(#[from] SpaceblockRequestsError), - #[error("error with library file decode '{0}'")] - LibraryFileDecodeError(decode::Error), - #[error("error with library file io '{0}'")] - LibraryFileIoError(std::io::Error), - #[error("invalid range discriminator for library file req '{0}'")] - LibraryDiscriminatorInvalid(u8), -} - -impl Header { - pub async fn from_stream(stream: &mut (impl AsyncRead + Unpin)) -> Result { - let discriminator = stream - .read_u8() - .await - .map_err(HeaderError::DiscriminatorIo)?; - - match discriminator { - 0 => Ok(Self::Spacedrop( - SpaceblockRequests::from_stream(stream).await?, - )), - 1 => Ok(Self::Ping), - 3 => Ok(Self::Sync), - 5 => Ok(Self::RspcRemote), - 6 => Ok(Self::LibraryFile { - file_path_id: decode::uuid(stream) - .await - .map_err(HeaderError::LibraryFileDecodeError)?, - range: match stream - .read_u8() - .await - .map_err(HeaderError::LibraryFileIoError)? - { - 0 => Range::Full, - 1 => { - let start = stream - .read_u64_le() - .await - .map_err(HeaderError::LibraryFileIoError)?; - let end = stream - .read_u64_le() - .await - .map_err(HeaderError::LibraryFileIoError)?; - Range::Partial(start..end) - } - d => return Err(HeaderError::LibraryDiscriminatorInvalid(d)), - }, - }), - d => Err(HeaderError::DiscriminatorInvalid(d)), - } - } - - pub fn to_bytes(&self) -> Vec { - match self { - Self::Spacedrop(transfer_request) => { - let mut bytes = vec![0]; - bytes.extend_from_slice(&transfer_request.to_bytes()); - bytes - } - Self::Ping => vec![1], - Self::Sync => vec![3], - Self::RspcRemote => vec![5], - Self::LibraryFile { - file_path_id, - range, - } => { - let mut buf = vec![6]; - encode::uuid(&mut buf, file_path_id); - buf.extend_from_slice(&range.to_bytes()); - buf - } - } - } -} - -#[cfg(test)] -mod tests { - // use super::*; - - #[test] - fn test_header() { - // TODO: Finish this - - // assert_eq!( - // Header::from_bytes(&Header::Ping.to_bytes()), - // Ok(Header::Ping) - // ); - - // assert_eq!( - // Header::from_bytes(&Header::Spacedrop.to_bytes()), - // Ok(Header::Spacedrop) - // ); - - // let uuid = Uuid::new_v4(); - // assert_eq!( - // Header::from_bytes(&Header::Sync(uuid).to_bytes()), - // Ok(Header::Sync(uuid)) - // ); - } -} diff --git a/core/src/old_p2p/sync/mod.rs b/core/src/old_p2p/sync/mod.rs deleted file mode 100644 index 4832be93e..000000000 --- a/core/src/old_p2p/sync/mod.rs +++ /dev/null @@ -1,262 +0,0 @@ -#![allow(clippy::panic, clippy::unwrap_used)] // TODO: Finish this - -use crate::library::Library; - -// use sd_p2p_proto::{decode, encode}; -// use sd_sync::CompressedCRDTOperationsPerModelPerDevice; - -use std::sync::Arc; - -use tokio::io::{AsyncRead, AsyncWrite}; - -use super::P2PManager; - -mod proto; -pub use proto::*; - -pub use originator::run as originator; -mod originator { - // use crate::p2p::Header; - - use super::*; - // use responder::tx as rx; - use sd_core_sync::SyncManager; - // use sd_p2p_tunnel::Tunnel; - - pub mod tx { - - // use super::*; - - // #[derive(Debug)] - // pub struct Operations(pub CompressedCRDTOperationsPerModelPerDevice); - - // impl Operations { - // // TODO: Per field errors for better error handling - // pub async fn from_stream( - // stream: &mut (impl AsyncRead + Unpin), - // ) -> std::io::Result { - // Ok(Self( - // rmp_serde::from_slice(&decode::buf(stream).await.unwrap()).unwrap(), - // )) - // } - - // pub fn to_bytes(&self) -> Vec { - // let Self(args) = self; - // let mut buf = vec![]; - - // // TODO: Error handling - // encode::buf(&mut buf, &rmp_serde::to_vec_named(&args).unwrap()); - // buf - // } - // } - - // #[cfg(test)] - // #[tokio::test] - // async fn test() { - // use sd_sync::CRDTOperation; - // use uuid::Uuid; - - // { - // let original = Operations(CompressedCRDTOperationsPerModelPerDevice::new(vec![])); - - // let mut cursor = std::io::Cursor::new(original.to_bytes()); - // let result = Operations::from_stream(&mut cursor).await.unwrap(); - // assert_eq!(original, result); - // } - - // { - // let original = Operations(CompressedCRDTOperationsPerModelPerDevice::new(vec![ - // CRDTOperation { - // device_pub_id: Uuid::new_v4(), - // timestamp: sync::NTP64(0), - // record_id: rmpv::Value::Nil, - // model_id: 0, - // data: sd_sync::CRDTOperationData::create(), - // }, - // ])); - - // let mut cursor = std::io::Cursor::new(original.to_bytes()); - // let result = Operations::from_stream(&mut cursor).await.unwrap(); - // assert_eq!(original, result); - // } - // } - } - - // #[instrument(skip(sync, p2p))] - /// REMEMBER: This only syncs one direction! - pub async fn run(_library: Arc, _sync: &SyncManager, _p2p: &Arc) { - // for (remote_identity, peer) in p2p.get_library_instances(&library.id) { - // if !peer.is_connected() { - // continue; - // }; - - // let sync = sync.clone(); - - // let library = library.clone(); - // tokio::spawn(async move { - // debug!( - // ?remote_identity, - // %library.id, - // "Alerting peer of new sync events for library;" - // ); - - // let mut stream = peer.new_stream().await.unwrap(); - - // stream.write_all(&Header::Sync.to_bytes()).await.unwrap(); - - // let mut tunnel = Tunnel::initiator(stream, &library.identity).await.unwrap(); - - // tunnel - // .write_all(&SyncMessage::NewOperations.to_bytes()) - // .await - // .unwrap(); - // tunnel.flush().await.unwrap(); - - // while let Ok(rx::MainRequest::GetOperations(GetOpsArgs { - // timestamp_per_device, - // count, - // })) = rx::MainRequest::from_stream(&mut tunnel).await - // { - // tunnel - // .write_all( - // &tx::Operations(CompressedCRDTOperationsPerModelPerDevice::new( - // sync.get_ops(count, timestamp_per_device).await.unwrap(), - // )) - // .to_bytes(), - // ) - // .await - // .unwrap(); - // tunnel.flush().await.unwrap(); - // } - // }); - // } - } -} - -pub use responder::run as responder; -mod responder { - - use super::*; - // use futures::StreamExt; - - // pub mod tx { - // use serde::{Deserialize, Serialize}; - - // use super::*; - - // #[derive(Serialize, Deserialize, PartialEq, Debug)] - // pub enum MainRequest { - // GetOperations(GetOpsArgs), - // Done, - // } - - // impl MainRequest { - // // TODO: Per field errors for better error handling - // pub async fn from_stream( - // stream: &mut (impl AsyncRead + Unpin), - // ) -> std::io::Result { - // Ok( - // // TODO: Error handling - // rmp_serde::from_slice(&decode::buf(stream).await.unwrap()).unwrap(), - // ) - // } - - // pub fn to_bytes(&self) -> Vec { - // let mut buf = vec![]; - // // TODO: Error handling - // encode::buf(&mut buf, &rmp_serde::to_vec_named(&self).unwrap()); - // buf - // } - // } - - // #[cfg(test)] - // #[tokio::test] - // async fn test() { - // { - // let original = MainRequest::GetOperations(GetOpsArgs { - // timestamp_per_device: vec![], - // count: 0, - // }); - - // let mut cursor = std::io::Cursor::new(original.to_bytes()); - // let result = MainRequest::from_stream(&mut cursor).await.unwrap(); - // assert_eq!(original, result); - // } - - // { - // let original = MainRequest::Done; - - // let mut cursor = std::io::Cursor::new(original.to_bytes()); - // let result = MainRequest::from_stream(&mut cursor).await.unwrap(); - // assert_eq!(original, result); - // } - // } - // } - - pub async fn run( - _stream: &mut (impl AsyncRead + AsyncWrite + Unpin), - _library: Arc, - ) -> Result<(), ()> { - // use sync::ingest::*; - - // let ingest = &library.sync.ingest; - - // ingest.event_tx.send(Event::Notification).await.unwrap(); - - // let mut rx = pin!(ingest.req_rx.clone()); - - // while let Some(req) = rx.next().await { - // const OPS_PER_REQUEST: u32 = 1000; - - // let timestamps = match req { - // Request::FinishedIngesting => break, - // Request::Messages { timestamps, .. } => timestamps, - // }; - - // debug!(?timestamps, "Getting ops for timestamps;"); - - // stream - // .write_all( - // &tx::MainRequest::GetOperations(sync::GetOpsArgs { - // timestamp_per_device: timestamps, - // count: OPS_PER_REQUEST, - // }) - // .to_bytes(), - // ) - // .await - // .unwrap(); - // stream.flush().await.unwrap(); - - // let rx::Operations(ops) = rx::Operations::from_stream(stream).await.unwrap(); - - // let (wait_tx, wait_rx) = tokio::sync::oneshot::channel::<()>(); - - // // FIXME: If there are exactly a multiple of OPS_PER_REQUEST operations, - // // then this will bug, as we sent `has_more` as true, but we don't have - // // more operations to send. - - // ingest - // .event_tx - // .send(Event::Messages(MessagesEvent { - // device_pub_id: library.sync.device_pub_id.clone(), - // has_more: ops.len() == OPS_PER_REQUEST as usize, - // messages: ops, - // wait_tx: Some(wait_tx), - // })) - // .await - // .expect("TODO: Handle ingest channel closed, so we don't loose ops"); - - // wait_rx.await.unwrap() - // } - - // debug!("Sync responder done"); - - // stream - // .write_all(&tx::MainRequest::Done.to_bytes()) - // .await - // .unwrap(); - // stream.flush().await.unwrap(); - - Ok(()) - } -} diff --git a/core/src/old_p2p/sync/proto.rs b/core/src/old_p2p/sync/proto.rs deleted file mode 100644 index 4569e2a34..000000000 --- a/core/src/old_p2p/sync/proto.rs +++ /dev/null @@ -1,43 +0,0 @@ -use sd_old_p2p_proto::decode; -use tokio::io::{AsyncRead, AsyncReadExt}; - -// will probs have more variants in future -#[derive(Debug, PartialEq, Eq)] -pub enum SyncMessage { - NewOperations, -} - -impl SyncMessage { - // TODO: Per field errors for better error handling - pub async fn from_stream(stream: &mut (impl AsyncRead + Unpin)) -> Result { - match stream.read_u8().await? { - b'N' => Ok(Self::NewOperations), - header => Err(decode::Error::IoError(std::io::Error::new( - std::io::ErrorKind::InvalidData, - format!("Invalid sync message header: {}", (header as char)), - ))), - } - } - - // pub fn to_bytes(&self) -> Vec { - // match self { - // Self::NewOperations => vec![b'N'], - // } - // } -} - -// #[cfg(test)] -// mod tests { -// use super::*; - -// #[tokio::test] -// async fn test_types() { -// { -// let original = SyncMessage::NewOperations; - -// let mut cursor = std::io::Cursor::new(original.to_bytes()); -// let result = SyncMessage::from_stream(&mut cursor).await.unwrap(); -// assert_eq!(original, result); -// } -// } -// } diff --git a/core-new/src/operations/addressing.rs b/core/src/operations/addressing.rs similarity index 100% rename from core-new/src/operations/addressing.rs rename to core/src/operations/addressing.rs diff --git a/core-new/src/operations/content/action.rs b/core/src/operations/content/action.rs similarity index 100% rename from core-new/src/operations/content/action.rs rename to core/src/operations/content/action.rs diff --git a/core-new/src/operations/content/mod.rs b/core/src/operations/content/mod.rs similarity index 100% rename from core-new/src/operations/content/mod.rs rename to core/src/operations/content/mod.rs diff --git a/core-new/src/operations/devices/mod.rs b/core/src/operations/devices/mod.rs similarity index 100% rename from core-new/src/operations/devices/mod.rs rename to core/src/operations/devices/mod.rs diff --git a/core-new/src/operations/devices/revoke/action.rs b/core/src/operations/devices/revoke/action.rs similarity index 100% rename from core-new/src/operations/devices/revoke/action.rs rename to core/src/operations/devices/revoke/action.rs diff --git a/core-new/src/operations/devices/revoke/mod.rs b/core/src/operations/devices/revoke/mod.rs similarity index 100% rename from core-new/src/operations/devices/revoke/mod.rs rename to core/src/operations/devices/revoke/mod.rs diff --git a/core-new/src/operations/devices/revoke/output.rs b/core/src/operations/devices/revoke/output.rs similarity index 100% rename from core-new/src/operations/devices/revoke/output.rs rename to core/src/operations/devices/revoke/output.rs diff --git a/core-new/src/operations/entries/mod.rs b/core/src/operations/entries/mod.rs similarity index 100% rename from core-new/src/operations/entries/mod.rs rename to core/src/operations/entries/mod.rs diff --git a/core-new/src/operations/entries/state.rs b/core/src/operations/entries/state.rs similarity index 100% rename from core-new/src/operations/entries/state.rs rename to core/src/operations/entries/state.rs diff --git a/core-new/src/operations/files/copy/action.rs b/core/src/operations/files/copy/action.rs similarity index 100% rename from core-new/src/operations/files/copy/action.rs rename to core/src/operations/files/copy/action.rs diff --git a/core-new/src/operations/files/copy/database.rs b/core/src/operations/files/copy/database.rs similarity index 100% rename from core-new/src/operations/files/copy/database.rs rename to core/src/operations/files/copy/database.rs diff --git a/core-new/src/operations/files/copy/docs/ANALYSIS.md b/core/src/operations/files/copy/docs/ANALYSIS.md similarity index 100% rename from core-new/src/operations/files/copy/docs/ANALYSIS.md rename to core/src/operations/files/copy/docs/ANALYSIS.md diff --git a/core-new/src/operations/files/copy/docs/FILE_SYNC_OVERLAP.md b/core/src/operations/files/copy/docs/FILE_SYNC_OVERLAP.md similarity index 100% rename from core-new/src/operations/files/copy/docs/FILE_SYNC_OVERLAP.md rename to core/src/operations/files/copy/docs/FILE_SYNC_OVERLAP.md diff --git a/core-new/src/operations/files/copy/docs/PROGRESSIVE_COPY_DESIGN.md b/core/src/operations/files/copy/docs/PROGRESSIVE_COPY_DESIGN.md similarity index 100% rename from core-new/src/operations/files/copy/docs/PROGRESSIVE_COPY_DESIGN.md rename to core/src/operations/files/copy/docs/PROGRESSIVE_COPY_DESIGN.md diff --git a/core-new/src/operations/files/copy/docs/RESUME_VALIDATION_DESIGN.md b/core/src/operations/files/copy/docs/RESUME_VALIDATION_DESIGN.md similarity index 100% rename from core-new/src/operations/files/copy/docs/RESUME_VALIDATION_DESIGN.md rename to core/src/operations/files/copy/docs/RESUME_VALIDATION_DESIGN.md diff --git a/core-new/src/operations/files/copy/input.rs b/core/src/operations/files/copy/input.rs similarity index 100% rename from core-new/src/operations/files/copy/input.rs rename to core/src/operations/files/copy/input.rs diff --git a/core-new/src/operations/files/copy/job.rs b/core/src/operations/files/copy/job.rs similarity index 100% rename from core-new/src/operations/files/copy/job.rs rename to core/src/operations/files/copy/job.rs diff --git a/core-new/src/operations/files/copy/mod.rs b/core/src/operations/files/copy/mod.rs similarity index 100% rename from core-new/src/operations/files/copy/mod.rs rename to core/src/operations/files/copy/mod.rs diff --git a/core-new/src/operations/files/copy/output.rs b/core/src/operations/files/copy/output.rs similarity index 100% rename from core-new/src/operations/files/copy/output.rs rename to core/src/operations/files/copy/output.rs diff --git a/core-new/src/operations/files/copy/routing.rs b/core/src/operations/files/copy/routing.rs similarity index 100% rename from core-new/src/operations/files/copy/routing.rs rename to core/src/operations/files/copy/routing.rs diff --git a/core-new/src/operations/files/copy/strategy.rs b/core/src/operations/files/copy/strategy.rs similarity index 100% rename from core-new/src/operations/files/copy/strategy.rs rename to core/src/operations/files/copy/strategy.rs diff --git a/core-new/src/operations/files/delete/action.rs b/core/src/operations/files/delete/action.rs similarity index 100% rename from core-new/src/operations/files/delete/action.rs rename to core/src/operations/files/delete/action.rs diff --git a/core-new/src/operations/files/delete/job.rs b/core/src/operations/files/delete/job.rs similarity index 100% rename from core-new/src/operations/files/delete/job.rs rename to core/src/operations/files/delete/job.rs diff --git a/core-new/src/operations/files/delete/mod.rs b/core/src/operations/files/delete/mod.rs similarity index 100% rename from core-new/src/operations/files/delete/mod.rs rename to core/src/operations/files/delete/mod.rs diff --git a/core-new/src/operations/files/delete/output.rs b/core/src/operations/files/delete/output.rs similarity index 100% rename from core-new/src/operations/files/delete/output.rs rename to core/src/operations/files/delete/output.rs diff --git a/core-new/src/operations/files/duplicate_detection/action.rs b/core/src/operations/files/duplicate_detection/action.rs similarity index 100% rename from core-new/src/operations/files/duplicate_detection/action.rs rename to core/src/operations/files/duplicate_detection/action.rs diff --git a/core-new/src/operations/files/duplicate_detection/job.rs b/core/src/operations/files/duplicate_detection/job.rs similarity index 100% rename from core-new/src/operations/files/duplicate_detection/job.rs rename to core/src/operations/files/duplicate_detection/job.rs diff --git a/core-new/src/operations/files/duplicate_detection/mod.rs b/core/src/operations/files/duplicate_detection/mod.rs similarity index 100% rename from core-new/src/operations/files/duplicate_detection/mod.rs rename to core/src/operations/files/duplicate_detection/mod.rs diff --git a/core-new/src/operations/files/mod.rs b/core/src/operations/files/mod.rs similarity index 100% rename from core-new/src/operations/files/mod.rs rename to core/src/operations/files/mod.rs diff --git a/core-new/src/operations/files/validation/action.rs b/core/src/operations/files/validation/action.rs similarity index 100% rename from core-new/src/operations/files/validation/action.rs rename to core/src/operations/files/validation/action.rs diff --git a/core-new/src/operations/files/validation/job.rs b/core/src/operations/files/validation/job.rs similarity index 100% rename from core-new/src/operations/files/validation/job.rs rename to core/src/operations/files/validation/job.rs diff --git a/core-new/src/operations/files/validation/mod.rs b/core/src/operations/files/validation/mod.rs similarity index 100% rename from core-new/src/operations/files/validation/mod.rs rename to core/src/operations/files/validation/mod.rs diff --git a/core-new/src/operations/indexing/action.rs b/core/src/operations/indexing/action.rs similarity index 100% rename from core-new/src/operations/indexing/action.rs rename to core/src/operations/indexing/action.rs diff --git a/core-new/src/operations/indexing/change_detection/mod.rs b/core/src/operations/indexing/change_detection/mod.rs similarity index 100% rename from core-new/src/operations/indexing/change_detection/mod.rs rename to core/src/operations/indexing/change_detection/mod.rs diff --git a/core-new/src/operations/indexing/entry.rs b/core/src/operations/indexing/entry.rs similarity index 100% rename from core-new/src/operations/indexing/entry.rs rename to core/src/operations/indexing/entry.rs diff --git a/core-new/src/operations/indexing/hierarchy.rs b/core/src/operations/indexing/hierarchy.rs similarity index 100% rename from core-new/src/operations/indexing/hierarchy.rs rename to core/src/operations/indexing/hierarchy.rs diff --git a/core-new/src/operations/indexing/job.rs b/core/src/operations/indexing/job.rs similarity index 100% rename from core-new/src/operations/indexing/job.rs rename to core/src/operations/indexing/job.rs diff --git a/core-new/src/operations/indexing/metrics.rs b/core/src/operations/indexing/metrics.rs similarity index 100% rename from core-new/src/operations/indexing/metrics.rs rename to core/src/operations/indexing/metrics.rs diff --git a/core-new/src/operations/indexing/mod.rs b/core/src/operations/indexing/mod.rs similarity index 100% rename from core-new/src/operations/indexing/mod.rs rename to core/src/operations/indexing/mod.rs diff --git a/core-new/src/operations/indexing/path_resolver.rs b/core/src/operations/indexing/path_resolver.rs similarity index 100% rename from core-new/src/operations/indexing/path_resolver.rs rename to core/src/operations/indexing/path_resolver.rs diff --git a/core-new/src/operations/indexing/persistence.rs b/core/src/operations/indexing/persistence.rs similarity index 100% rename from core-new/src/operations/indexing/persistence.rs rename to core/src/operations/indexing/persistence.rs diff --git a/core-new/src/operations/indexing/phases/aggregation.rs b/core/src/operations/indexing/phases/aggregation.rs similarity index 100% rename from core-new/src/operations/indexing/phases/aggregation.rs rename to core/src/operations/indexing/phases/aggregation.rs diff --git a/core-new/src/operations/indexing/phases/content.rs b/core/src/operations/indexing/phases/content.rs similarity index 100% rename from core-new/src/operations/indexing/phases/content.rs rename to core/src/operations/indexing/phases/content.rs diff --git a/core-new/src/operations/indexing/phases/discovery.rs b/core/src/operations/indexing/phases/discovery.rs similarity index 100% rename from core-new/src/operations/indexing/phases/discovery.rs rename to core/src/operations/indexing/phases/discovery.rs diff --git a/core-new/src/operations/indexing/phases/mod.rs b/core/src/operations/indexing/phases/mod.rs similarity index 100% rename from core-new/src/operations/indexing/phases/mod.rs rename to core/src/operations/indexing/phases/mod.rs diff --git a/core-new/src/operations/indexing/phases/processing.rs b/core/src/operations/indexing/phases/processing.rs similarity index 100% rename from core-new/src/operations/indexing/phases/processing.rs rename to core/src/operations/indexing/phases/processing.rs diff --git a/core-new/src/operations/indexing/progress.rs b/core/src/operations/indexing/progress.rs similarity index 100% rename from core-new/src/operations/indexing/progress.rs rename to core/src/operations/indexing/progress.rs diff --git a/core-new/src/operations/indexing/rules.rs b/core/src/operations/indexing/rules.rs similarity index 100% rename from core-new/src/operations/indexing/rules.rs rename to core/src/operations/indexing/rules.rs diff --git a/core-new/src/operations/indexing/state.rs b/core/src/operations/indexing/state.rs similarity index 100% rename from core-new/src/operations/indexing/state.rs rename to core/src/operations/indexing/state.rs diff --git a/core-new/src/operations/indexing/tests/mod.rs b/core/src/operations/indexing/tests/mod.rs similarity index 100% rename from core-new/src/operations/indexing/tests/mod.rs rename to core/src/operations/indexing/tests/mod.rs diff --git a/core-new/src/operations/libraries/create/action.rs b/core/src/operations/libraries/create/action.rs similarity index 100% rename from core-new/src/operations/libraries/create/action.rs rename to core/src/operations/libraries/create/action.rs diff --git a/core-new/src/operations/libraries/create/mod.rs b/core/src/operations/libraries/create/mod.rs similarity index 100% rename from core-new/src/operations/libraries/create/mod.rs rename to core/src/operations/libraries/create/mod.rs diff --git a/core-new/src/operations/libraries/create/output.rs b/core/src/operations/libraries/create/output.rs similarity index 100% rename from core-new/src/operations/libraries/create/output.rs rename to core/src/operations/libraries/create/output.rs diff --git a/core-new/src/operations/libraries/delete/action.rs b/core/src/operations/libraries/delete/action.rs similarity index 100% rename from core-new/src/operations/libraries/delete/action.rs rename to core/src/operations/libraries/delete/action.rs diff --git a/core-new/src/operations/libraries/delete/mod.rs b/core/src/operations/libraries/delete/mod.rs similarity index 100% rename from core-new/src/operations/libraries/delete/mod.rs rename to core/src/operations/libraries/delete/mod.rs diff --git a/core-new/src/operations/libraries/delete/output.rs b/core/src/operations/libraries/delete/output.rs similarity index 100% rename from core-new/src/operations/libraries/delete/output.rs rename to core/src/operations/libraries/delete/output.rs diff --git a/core-new/src/operations/libraries/export/action.rs b/core/src/operations/libraries/export/action.rs similarity index 100% rename from core-new/src/operations/libraries/export/action.rs rename to core/src/operations/libraries/export/action.rs diff --git a/core-new/src/operations/libraries/export/mod.rs b/core/src/operations/libraries/export/mod.rs similarity index 100% rename from core-new/src/operations/libraries/export/mod.rs rename to core/src/operations/libraries/export/mod.rs diff --git a/core-new/src/operations/libraries/export/output.rs b/core/src/operations/libraries/export/output.rs similarity index 100% rename from core-new/src/operations/libraries/export/output.rs rename to core/src/operations/libraries/export/output.rs diff --git a/core-new/src/operations/libraries/mod.rs b/core/src/operations/libraries/mod.rs similarity index 100% rename from core-new/src/operations/libraries/mod.rs rename to core/src/operations/libraries/mod.rs diff --git a/core-new/src/operations/libraries/rename/action.rs b/core/src/operations/libraries/rename/action.rs similarity index 100% rename from core-new/src/operations/libraries/rename/action.rs rename to core/src/operations/libraries/rename/action.rs diff --git a/core-new/src/operations/libraries/rename/mod.rs b/core/src/operations/libraries/rename/mod.rs similarity index 100% rename from core-new/src/operations/libraries/rename/mod.rs rename to core/src/operations/libraries/rename/mod.rs diff --git a/core-new/src/operations/libraries/rename/output.rs b/core/src/operations/libraries/rename/output.rs similarity index 100% rename from core-new/src/operations/libraries/rename/output.rs rename to core/src/operations/libraries/rename/output.rs diff --git a/core-new/src/operations/locations/add/action.rs b/core/src/operations/locations/add/action.rs similarity index 100% rename from core-new/src/operations/locations/add/action.rs rename to core/src/operations/locations/add/action.rs diff --git a/core-new/src/operations/locations/add/mod.rs b/core/src/operations/locations/add/mod.rs similarity index 100% rename from core-new/src/operations/locations/add/mod.rs rename to core/src/operations/locations/add/mod.rs diff --git a/core-new/src/operations/locations/add/output.rs b/core/src/operations/locations/add/output.rs similarity index 100% rename from core-new/src/operations/locations/add/output.rs rename to core/src/operations/locations/add/output.rs diff --git a/core-new/src/operations/locations/index/action.rs b/core/src/operations/locations/index/action.rs similarity index 100% rename from core-new/src/operations/locations/index/action.rs rename to core/src/operations/locations/index/action.rs diff --git a/core-new/src/operations/locations/index/mod.rs b/core/src/operations/locations/index/mod.rs similarity index 100% rename from core-new/src/operations/locations/index/mod.rs rename to core/src/operations/locations/index/mod.rs diff --git a/core-new/src/operations/locations/mod.rs b/core/src/operations/locations/mod.rs similarity index 100% rename from core-new/src/operations/locations/mod.rs rename to core/src/operations/locations/mod.rs diff --git a/core-new/src/operations/locations/remove/action.rs b/core/src/operations/locations/remove/action.rs similarity index 100% rename from core-new/src/operations/locations/remove/action.rs rename to core/src/operations/locations/remove/action.rs diff --git a/core-new/src/operations/locations/remove/mod.rs b/core/src/operations/locations/remove/mod.rs similarity index 100% rename from core-new/src/operations/locations/remove/mod.rs rename to core/src/operations/locations/remove/mod.rs diff --git a/core-new/src/operations/locations/remove/output.rs b/core/src/operations/locations/remove/output.rs similarity index 100% rename from core-new/src/operations/locations/remove/output.rs rename to core/src/operations/locations/remove/output.rs diff --git a/core-new/src/operations/locations/rescan/action.rs b/core/src/operations/locations/rescan/action.rs similarity index 100% rename from core-new/src/operations/locations/rescan/action.rs rename to core/src/operations/locations/rescan/action.rs diff --git a/core-new/src/operations/locations/rescan/mod.rs b/core/src/operations/locations/rescan/mod.rs similarity index 100% rename from core-new/src/operations/locations/rescan/mod.rs rename to core/src/operations/locations/rescan/mod.rs diff --git a/core-new/src/operations/locations/rescan/output.rs b/core/src/operations/locations/rescan/output.rs similarity index 100% rename from core-new/src/operations/locations/rescan/output.rs rename to core/src/operations/locations/rescan/output.rs diff --git a/core-new/src/operations/media/live_photo.rs b/core/src/operations/media/live_photo.rs similarity index 100% rename from core-new/src/operations/media/live_photo.rs rename to core/src/operations/media/live_photo.rs diff --git a/core-new/src/operations/media/live_photo_query.rs b/core/src/operations/media/live_photo_query.rs similarity index 100% rename from core-new/src/operations/media/live_photo_query.rs rename to core/src/operations/media/live_photo_query.rs diff --git a/core-new/src/operations/media/mod.rs b/core/src/operations/media/mod.rs similarity index 100% rename from core-new/src/operations/media/mod.rs rename to core/src/operations/media/mod.rs diff --git a/core-new/src/operations/media/thumbnail/action.rs b/core/src/operations/media/thumbnail/action.rs similarity index 100% rename from core-new/src/operations/media/thumbnail/action.rs rename to core/src/operations/media/thumbnail/action.rs diff --git a/core-new/src/operations/media/thumbnail/error.rs b/core/src/operations/media/thumbnail/error.rs similarity index 100% rename from core-new/src/operations/media/thumbnail/error.rs rename to core/src/operations/media/thumbnail/error.rs diff --git a/core-new/src/operations/media/thumbnail/generator.rs b/core/src/operations/media/thumbnail/generator.rs similarity index 100% rename from core-new/src/operations/media/thumbnail/generator.rs rename to core/src/operations/media/thumbnail/generator.rs diff --git a/core-new/src/operations/media/thumbnail/job.rs b/core/src/operations/media/thumbnail/job.rs similarity index 100% rename from core-new/src/operations/media/thumbnail/job.rs rename to core/src/operations/media/thumbnail/job.rs diff --git a/core-new/src/operations/media/thumbnail/mod.rs b/core/src/operations/media/thumbnail/mod.rs similarity index 100% rename from core-new/src/operations/media/thumbnail/mod.rs rename to core/src/operations/media/thumbnail/mod.rs diff --git a/core-new/src/operations/media/thumbnail/state.rs b/core/src/operations/media/thumbnail/state.rs similarity index 100% rename from core-new/src/operations/media/thumbnail/state.rs rename to core/src/operations/media/thumbnail/state.rs diff --git a/core-new/src/operations/media/thumbnail/utils.rs b/core/src/operations/media/thumbnail/utils.rs similarity index 100% rename from core-new/src/operations/media/thumbnail/utils.rs rename to core/src/operations/media/thumbnail/utils.rs diff --git a/core-new/src/operations/metadata/action.rs b/core/src/operations/metadata/action.rs similarity index 100% rename from core-new/src/operations/metadata/action.rs rename to core/src/operations/metadata/action.rs diff --git a/core-new/src/operations/metadata/mod.rs b/core/src/operations/metadata/mod.rs similarity index 100% rename from core-new/src/operations/metadata/mod.rs rename to core/src/operations/metadata/mod.rs diff --git a/core-new/src/operations/mod.rs b/core/src/operations/mod.rs similarity index 100% rename from core-new/src/operations/mod.rs rename to core/src/operations/mod.rs diff --git a/core-new/src/operations/sidecar/mod.rs b/core/src/operations/sidecar/mod.rs similarity index 100% rename from core-new/src/operations/sidecar/mod.rs rename to core/src/operations/sidecar/mod.rs diff --git a/core-new/src/operations/sidecar/path.rs b/core/src/operations/sidecar/path.rs similarity index 100% rename from core-new/src/operations/sidecar/path.rs rename to core/src/operations/sidecar/path.rs diff --git a/core-new/src/operations/sidecar/types.rs b/core/src/operations/sidecar/types.rs similarity index 100% rename from core-new/src/operations/sidecar/types.rs rename to core/src/operations/sidecar/types.rs diff --git a/core-new/src/operations/volumes/mod.rs b/core/src/operations/volumes/mod.rs similarity index 100% rename from core-new/src/operations/volumes/mod.rs rename to core/src/operations/volumes/mod.rs diff --git a/core-new/src/operations/volumes/speed_test/action.rs b/core/src/operations/volumes/speed_test/action.rs similarity index 100% rename from core-new/src/operations/volumes/speed_test/action.rs rename to core/src/operations/volumes/speed_test/action.rs diff --git a/core-new/src/operations/volumes/speed_test/handler.rs b/core/src/operations/volumes/speed_test/handler.rs similarity index 100% rename from core-new/src/operations/volumes/speed_test/handler.rs rename to core/src/operations/volumes/speed_test/handler.rs diff --git a/core-new/src/operations/volumes/speed_test/mod.rs b/core/src/operations/volumes/speed_test/mod.rs similarity index 100% rename from core-new/src/operations/volumes/speed_test/mod.rs rename to core/src/operations/volumes/speed_test/mod.rs diff --git a/core-new/src/operations/volumes/track/action.rs b/core/src/operations/volumes/track/action.rs similarity index 100% rename from core-new/src/operations/volumes/track/action.rs rename to core/src/operations/volumes/track/action.rs diff --git a/core-new/src/operations/volumes/track/handler.rs b/core/src/operations/volumes/track/handler.rs similarity index 100% rename from core-new/src/operations/volumes/track/handler.rs rename to core/src/operations/volumes/track/handler.rs diff --git a/core-new/src/operations/volumes/track/mod.rs b/core/src/operations/volumes/track/mod.rs similarity index 100% rename from core-new/src/operations/volumes/track/mod.rs rename to core/src/operations/volumes/track/mod.rs diff --git a/core-new/src/operations/volumes/untrack/action.rs b/core/src/operations/volumes/untrack/action.rs similarity index 100% rename from core-new/src/operations/volumes/untrack/action.rs rename to core/src/operations/volumes/untrack/action.rs diff --git a/core-new/src/operations/volumes/untrack/handler.rs b/core/src/operations/volumes/untrack/handler.rs similarity index 100% rename from core-new/src/operations/volumes/untrack/handler.rs rename to core/src/operations/volumes/untrack/handler.rs diff --git a/core-new/src/operations/volumes/untrack/mod.rs b/core/src/operations/volumes/untrack/mod.rs similarity index 100% rename from core-new/src/operations/volumes/untrack/mod.rs rename to core/src/operations/volumes/untrack/mod.rs diff --git a/core/src/preferences/kv.rs b/core/src/preferences/kv.rs deleted file mode 100644 index bdfc25a73..000000000 --- a/core/src/preferences/kv.rs +++ /dev/null @@ -1,161 +0,0 @@ -use sd_prisma::prisma::{preference, PrismaClient}; - -use std::collections::BTreeMap; - -use itertools::Itertools; -use rmpv::Value; -use serde::{de::DeserializeOwned, Serialize}; - -use super::Preferences; - -#[derive(Debug)] -pub struct PreferenceKey(Vec); - -impl PreferenceKey { - pub fn new(value: impl Into) -> Self { - Self( - value - .into() - .split('.') - .map(ToString::to_string) - .collect_vec(), - ) - } - - pub fn prepend_path(&mut self, prefix: &str) { - self.0 = [prefix.to_string()] - .into_iter() - .chain(self.0.drain(..)) - .collect_vec(); - } -} - -impl std::fmt::Display for PreferenceKey { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{}", self.0.join(".")) - } -} - -#[derive(Debug)] -pub struct PreferenceValue(Vec); - -impl PreferenceValue { - pub fn new(value: impl Serialize) -> Self { - let mut bytes = vec![]; - - rmp_serde::encode::write_named(&mut bytes, &value) - .expect("Failed to serialize preference value"); - - // let value = rmpv::decode::read_value(&mut bytes.as_slice()).unwrap(); - - Self(bytes) - } - - pub fn from_value(value: Value) -> Self { - let mut bytes = vec![]; - - rmpv::encode::write_value(&mut bytes, &value) - .expect("Failed to serialize preference value"); - - Self(bytes) - } -} - -#[derive(Debug)] -pub struct PreferenceKVs(Vec<(PreferenceKey, PreferenceValue)>); - -impl IntoIterator for PreferenceKVs { - type Item = (PreferenceKey, PreferenceValue); - type IntoIter = std::vec::IntoIter; - - fn into_iter(self) -> Self::IntoIter { - self.0.into_iter() - } -} - -#[derive(Debug)] -pub enum Entry { - Value(Vec), - Nested(Entries), -} - -#[allow(clippy::unwrap_used, clippy::panic)] -impl Entry { - pub fn expect_value(self) -> T { - match self { - Self::Value(value) => rmp_serde::decode::from_read(value.as_slice()).unwrap(), - _ => panic!("Expected value"), - } - } - - pub fn expect_nested(self) -> Entries { - match self { - Self::Nested(entries) => entries, - _ => panic!("Expected nested entry"), - } - } -} - -pub type Entries = BTreeMap; - -impl PreferenceKVs { - pub fn new(values: Vec<(PreferenceKey, PreferenceValue)>) -> Self { - Self(values) - } - - pub fn with_prefix(mut self, prefix: &str) -> Self { - for (key, _) in &mut self.0 { - key.prepend_path(prefix); - } - - self - } - - pub fn into_upserts(self, db: &PrismaClient) -> Vec { - self.0 - .into_iter() - .map(|(key, value)| { - let value = vec![preference::value::set(Some(value.0))]; - - db.preference().upsert( - preference::key::equals(key.to_string()), - preference::create(key.to_string(), value.clone()), - value, - ) - }) - .collect() - } - - pub fn parse(self) -> T { - let entries = self - .0 - .into_iter() - .fold(BTreeMap::new(), |mut acc, (key, value)| { - let key_parts = key.0; - let key_parts_len = key_parts.len(); - - { - let mut curr_map: &mut BTreeMap = &mut acc; - - for (i, part) in key_parts.into_iter().enumerate() { - if i >= key_parts_len - 1 { - curr_map.insert(part, Entry::Value(value.0)); - break; - } else { - curr_map = match curr_map - .entry(part) - .or_insert(Entry::Nested(BTreeMap::new())) - { - Entry::Nested(map) => map, - _ => unreachable!(), - }; - } - } - } - - acc - }); - - T::from_entries(entries) - } -} diff --git a/core/src/preferences/library.rs b/core/src/preferences/library.rs deleted file mode 100644 index 2fadb8005..000000000 --- a/core/src/preferences/library.rs +++ /dev/null @@ -1,129 +0,0 @@ -use crate::api::search; - -use sd_prisma::prisma::PrismaClient; - -use std::collections::BTreeMap; - -use serde::{Deserialize, Serialize}; -use specta::Type; -use tracing::error; - -use super::*; - -#[derive(Clone, Serialize, Deserialize, Type, Debug)] -#[serde(rename_all = "camelCase")] -pub struct LibraryPreferences { - #[serde(default)] - #[specta(optional)] - location: HashMap>, - #[serde(default)] - #[specta(optional)] - tag: HashMap>, -} - -impl LibraryPreferences { - pub async fn write(self, db: &PrismaClient) -> prisma_client_rust::Result<()> { - let kvs = self.to_kvs(); - - db._batch(kvs.into_upserts(db)).await?; - - Ok(()) - } - - pub async fn read(db: &PrismaClient) -> prisma_client_rust::Result { - let kvs = db.preference().find_many(vec![]).exec().await?; - - let prefs = PreferenceKVs::new( - kvs.into_iter() - .filter_map(|data| { - rmpv::decode::read_value(&mut data.value?.as_slice()) - .map_err(|e| error!(?e)) - .ok() - .map(|value| { - ( - PreferenceKey::new(data.key), - PreferenceValue::from_value(value), - ) - }) - }) - .collect(), - ); - - Ok(prefs.parse()) - } -} - -#[derive(Clone, Serialize, Deserialize, Type, Debug)] -#[serde(rename_all = "camelCase")] -pub struct LocationSettings { - explorer: ExplorerSettings, -} - -#[derive(Clone, Serialize, Deserialize, Type, Debug)] -#[serde(rename_all = "camelCase")] -pub struct TagSettings { - explorer: ExplorerSettings, -} - -#[derive(Clone, Serialize, Deserialize, Type, Debug)] -#[serde(rename_all = "camelCase")] -pub struct ExplorerSettings { - layout_mode: Option, - grid_item_size: Option, - grid_gap: Option, - media_columns: Option, - media_aspect_square: Option, - media_view_with_descendants: Option, - open_on_double_click: Option, - show_bytes_in_grid_view: Option, - col_visibility: Option>, - col_sizes: Option>, - list_view_icon_size: Option, - list_view_text_size: Option, - // temporary - #[serde(skip_serializing_if = "Option::is_none")] - order: Option>, - #[serde(default)] - show_hidden_files: bool, -} - -#[derive(Clone, Serialize, Deserialize, Type, Debug)] -#[serde(rename_all = "camelCase")] -pub enum ExplorerLayout { - Grid, - List, - Media, -} - -#[derive(Clone, Serialize, Deserialize, Type, Debug)] -#[serde(rename_all = "camelCase")] -pub enum DoubleClickAction { - OpenFile, - QuickPreview, -} - -impl Preferences for LibraryPreferences { - fn to_kvs(self) -> PreferenceKVs { - let Self { location, tag } = self; - - let mut ret = vec![]; - - ret.extend(location.to_kvs().with_prefix("location")); - ret.extend(tag.to_kvs().with_prefix("tag")); - - PreferenceKVs::new(ret) - } - - fn from_entries(mut entries: Entries) -> Self { - Self { - location: entries - .remove("location") - .map(|value| HashMap::from_entries(value.expect_nested())) - .unwrap_or_default(), - tag: entries - .remove("tag") - .map(|value| HashMap::from_entries(value.expect_nested())) - .unwrap_or_default(), - } - } -} diff --git a/core/src/preferences/mod.rs b/core/src/preferences/mod.rs deleted file mode 100644 index 215798fb1..000000000 --- a/core/src/preferences/mod.rs +++ /dev/null @@ -1,56 +0,0 @@ -use std::collections::HashMap; - -use serde::{de::DeserializeOwned, Deserialize, Serialize}; -use specta::Type; -use tracing::error; -use uuid::Uuid; - -mod kv; -mod library; - -pub use kv::*; -pub use library::*; - -#[derive(Clone, Serialize, Deserialize, Type, Debug)] -#[specta(inline)] -pub struct Settings(V); - -impl Preferences for HashMap> -where - V: Serialize + DeserializeOwned, -{ - fn to_kvs(self) -> PreferenceKVs { - PreferenceKVs::new( - self.into_iter() - .map(|(id, value)| { - let mut buf = Uuid::encode_buffer(); - - let id = id.as_simple().encode_lower(&mut buf); - - (PreferenceKey::new(id), PreferenceValue::new(value)) - }) - .collect(), - ) - } - - fn from_entries(entries: Entries) -> Self { - entries - .into_iter() - .filter_map(|(key, entry)| { - Uuid::parse_str(&key) - .map_err(|e| error!(?e)) - .ok() - .map(|uuid| (uuid, entry.expect_value())) - }) - .collect() - } -} - -// Preferences are a set of types that are serialized as a list of key-value pairs, -// where nested type keys are serialized as a dot-separated path. -// They are serialized as a list because this allows preferences to be a synchronization boundary, -// whereas their values (referred to as settings) will be overwritten. -pub trait Preferences { - fn to_kvs(self) -> PreferenceKVs; - fn from_entries(entries: Entries) -> Self; -} diff --git a/core/src/search/mod.rs b/core/src/search/mod.rs deleted file mode 100644 index 8b1378917..000000000 --- a/core/src/search/mod.rs +++ /dev/null @@ -1 +0,0 @@ - diff --git a/core-new/src/services/device.rs b/core/src/services/device.rs similarity index 100% rename from core-new/src/services/device.rs rename to core/src/services/device.rs diff --git a/core-new/src/services/entry_state_service.rs b/core/src/services/entry_state_service.rs similarity index 100% rename from core-new/src/services/entry_state_service.rs rename to core/src/services/entry_state_service.rs diff --git a/core-new/src/services/file_sharing.rs b/core/src/services/file_sharing.rs similarity index 100% rename from core-new/src/services/file_sharing.rs rename to core/src/services/file_sharing.rs diff --git a/core-new/src/services/location_watcher/event_handler.rs b/core/src/services/location_watcher/event_handler.rs similarity index 100% rename from core-new/src/services/location_watcher/event_handler.rs rename to core/src/services/location_watcher/event_handler.rs diff --git a/core-new/src/services/location_watcher/mod.rs b/core/src/services/location_watcher/mod.rs similarity index 100% rename from core-new/src/services/location_watcher/mod.rs rename to core/src/services/location_watcher/mod.rs diff --git a/core-new/src/services/location_watcher/platform/linux.rs b/core/src/services/location_watcher/platform/linux.rs similarity index 100% rename from core-new/src/services/location_watcher/platform/linux.rs rename to core/src/services/location_watcher/platform/linux.rs diff --git a/core-new/src/services/location_watcher/platform/macos.rs b/core/src/services/location_watcher/platform/macos.rs similarity index 100% rename from core-new/src/services/location_watcher/platform/macos.rs rename to core/src/services/location_watcher/platform/macos.rs diff --git a/core-new/src/services/location_watcher/platform/mod.rs b/core/src/services/location_watcher/platform/mod.rs similarity index 100% rename from core-new/src/services/location_watcher/platform/mod.rs rename to core/src/services/location_watcher/platform/mod.rs diff --git a/core-new/src/services/location_watcher/platform/windows.rs b/core/src/services/location_watcher/platform/windows.rs similarity index 100% rename from core-new/src/services/location_watcher/platform/windows.rs rename to core/src/services/location_watcher/platform/windows.rs diff --git a/core-new/src/services/location_watcher/utils.rs b/core/src/services/location_watcher/utils.rs similarity index 100% rename from core-new/src/services/location_watcher/utils.rs rename to core/src/services/location_watcher/utils.rs diff --git a/core-new/src/services/mod.rs b/core/src/services/mod.rs similarity index 100% rename from core-new/src/services/mod.rs rename to core/src/services/mod.rs diff --git a/core-new/src/services/networking/core/event_loop.rs b/core/src/services/networking/core/event_loop.rs similarity index 100% rename from core-new/src/services/networking/core/event_loop.rs rename to core/src/services/networking/core/event_loop.rs diff --git a/core-new/src/services/networking/core/mod.rs b/core/src/services/networking/core/mod.rs similarity index 100% rename from core-new/src/services/networking/core/mod.rs rename to core/src/services/networking/core/mod.rs diff --git a/core-new/src/services/networking/device/connection.rs b/core/src/services/networking/device/connection.rs similarity index 100% rename from core-new/src/services/networking/device/connection.rs rename to core/src/services/networking/device/connection.rs diff --git a/core-new/src/services/networking/device/mod.rs b/core/src/services/networking/device/mod.rs similarity index 100% rename from core-new/src/services/networking/device/mod.rs rename to core/src/services/networking/device/mod.rs diff --git a/core-new/src/services/networking/device/persistence.rs b/core/src/services/networking/device/persistence.rs similarity index 100% rename from core-new/src/services/networking/device/persistence.rs rename to core/src/services/networking/device/persistence.rs diff --git a/core-new/src/services/networking/device/registry.rs b/core/src/services/networking/device/registry.rs similarity index 100% rename from core-new/src/services/networking/device/registry.rs rename to core/src/services/networking/device/registry.rs diff --git a/core-new/src/services/networking/mod.rs b/core/src/services/networking/mod.rs similarity index 100% rename from core-new/src/services/networking/mod.rs rename to core/src/services/networking/mod.rs diff --git a/core-new/src/services/networking/protocols/file_transfer.rs b/core/src/services/networking/protocols/file_transfer.rs similarity index 100% rename from core-new/src/services/networking/protocols/file_transfer.rs rename to core/src/services/networking/protocols/file_transfer.rs diff --git a/core-new/src/services/networking/protocols/messaging.rs b/core/src/services/networking/protocols/messaging.rs similarity index 100% rename from core-new/src/services/networking/protocols/messaging.rs rename to core/src/services/networking/protocols/messaging.rs diff --git a/core-new/src/services/networking/protocols/mod.rs b/core/src/services/networking/protocols/mod.rs similarity index 100% rename from core-new/src/services/networking/protocols/mod.rs rename to core/src/services/networking/protocols/mod.rs diff --git a/core-new/src/services/networking/protocols/pairing/initiator.rs b/core/src/services/networking/protocols/pairing/initiator.rs similarity index 100% rename from core-new/src/services/networking/protocols/pairing/initiator.rs rename to core/src/services/networking/protocols/pairing/initiator.rs diff --git a/core-new/src/services/networking/protocols/pairing/joiner.rs b/core/src/services/networking/protocols/pairing/joiner.rs similarity index 100% rename from core-new/src/services/networking/protocols/pairing/joiner.rs rename to core/src/services/networking/protocols/pairing/joiner.rs diff --git a/core-new/src/services/networking/protocols/pairing/messages.rs b/core/src/services/networking/protocols/pairing/messages.rs similarity index 100% rename from core-new/src/services/networking/protocols/pairing/messages.rs rename to core/src/services/networking/protocols/pairing/messages.rs diff --git a/core-new/src/services/networking/protocols/pairing/mod.rs b/core/src/services/networking/protocols/pairing/mod.rs similarity index 100% rename from core-new/src/services/networking/protocols/pairing/mod.rs rename to core/src/services/networking/protocols/pairing/mod.rs diff --git a/core-new/src/services/networking/protocols/pairing/persistence.rs b/core/src/services/networking/protocols/pairing/persistence.rs similarity index 100% rename from core-new/src/services/networking/protocols/pairing/persistence.rs rename to core/src/services/networking/protocols/pairing/persistence.rs diff --git a/core-new/src/services/networking/protocols/pairing/security.rs b/core/src/services/networking/protocols/pairing/security.rs similarity index 100% rename from core-new/src/services/networking/protocols/pairing/security.rs rename to core/src/services/networking/protocols/pairing/security.rs diff --git a/core-new/src/services/networking/protocols/pairing/types.rs b/core/src/services/networking/protocols/pairing/types.rs similarity index 100% rename from core-new/src/services/networking/protocols/pairing/types.rs rename to core/src/services/networking/protocols/pairing/types.rs diff --git a/core-new/src/services/networking/protocols/registry.rs b/core/src/services/networking/protocols/registry.rs similarity index 100% rename from core-new/src/services/networking/protocols/registry.rs rename to core/src/services/networking/protocols/registry.rs diff --git a/core-new/src/services/networking/utils/identity.rs b/core/src/services/networking/utils/identity.rs similarity index 100% rename from core-new/src/services/networking/utils/identity.rs rename to core/src/services/networking/utils/identity.rs diff --git a/core-new/src/services/networking/utils/logging.rs b/core/src/services/networking/utils/logging.rs similarity index 100% rename from core-new/src/services/networking/utils/logging.rs rename to core/src/services/networking/utils/logging.rs diff --git a/core-new/src/services/networking/utils/mod.rs b/core/src/services/networking/utils/mod.rs similarity index 100% rename from core-new/src/services/networking/utils/mod.rs rename to core/src/services/networking/utils/mod.rs diff --git a/core-new/src/services/sidecar_manager.rs b/core/src/services/sidecar_manager.rs similarity index 100% rename from core-new/src/services/sidecar_manager.rs rename to core/src/services/sidecar_manager.rs diff --git a/core-new/src/services/volume_monitor.rs b/core/src/services/volume_monitor.rs similarity index 100% rename from core-new/src/services/volume_monitor.rs rename to core/src/services/volume_monitor.rs diff --git a/core-new/src/shared/errors.rs b/core/src/shared/errors.rs similarity index 100% rename from core-new/src/shared/errors.rs rename to core/src/shared/errors.rs diff --git a/core-new/src/shared/mod.rs b/core/src/shared/mod.rs similarity index 100% rename from core-new/src/shared/mod.rs rename to core/src/shared/mod.rs diff --git a/core-new/src/shared/types.rs b/core/src/shared/types.rs similarity index 100% rename from core-new/src/shared/types.rs rename to core/src/shared/types.rs diff --git a/core-new/src/shared/utils.rs b/core/src/shared/utils.rs similarity index 100% rename from core-new/src/shared/utils.rs rename to core/src/shared/utils.rs diff --git a/core-new/src/test_framework/mod.rs b/core/src/test_framework/mod.rs similarity index 100% rename from core-new/src/test_framework/mod.rs rename to core/src/test_framework/mod.rs diff --git a/core-new/src/test_framework/runner.rs b/core/src/test_framework/runner.rs similarity index 100% rename from core-new/src/test_framework/runner.rs rename to core/src/test_framework/runner.rs diff --git a/core/src/util/abort_on_drop.rs b/core/src/util/abort_on_drop.rs deleted file mode 100644 index f001405f3..000000000 --- a/core/src/util/abort_on_drop.rs +++ /dev/null @@ -1,39 +0,0 @@ -use futures::{pin_mut, Future, Stream}; - -pub struct AbortOnDrop(pub tokio::task::JoinHandle); - -impl Drop for AbortOnDrop { - fn drop(&mut self) { - self.0.abort() - } -} - -impl Future for AbortOnDrop { - type Output = Result; - - fn poll( - mut self: std::pin::Pin<&mut Self>, - cx: &mut std::task::Context<'_>, - ) -> std::task::Poll { - let handle = &mut self.0; - - pin_mut!(handle); - - handle.poll(cx) - } -} - -impl Stream for AbortOnDrop { - type Item = (); - - fn poll_next( - mut self: std::pin::Pin<&mut Self>, - cx: &mut std::task::Context<'_>, - ) -> std::task::Poll> { - let handle = &mut self.0; - - pin_mut!(handle); - - handle.poll(cx).map(|_| None) - } -} diff --git a/core/src/util/batched_stream.rs b/core/src/util/batched_stream.rs deleted file mode 100644 index 1cd350840..000000000 --- a/core/src/util/batched_stream.rs +++ /dev/null @@ -1,67 +0,0 @@ -use std::{ - pin::Pin, - task::{Context, Poll}, -}; - -use futures::Stream; -use pin_project_lite::pin_project; - -// We limit the number of polls to prevent starvation of other tasks. -// This number is chosen arbitrarily but it is set smaller than `FuturesUnordered` or `StreamUnordered`. -const MAX_POLLS: usize = 15; - -pin_project! { - #[project = BatchedStreamProj] - pub enum BatchedStream where S: Stream { - Active { - #[pin] - stream: S, - batch: Vec, - }, - Complete - } -} - -impl BatchedStream { - pub fn new(stream: S) -> Self { - Self::Active { - stream, - batch: Vec::with_capacity(MAX_POLLS), - } - } -} - -impl Stream for BatchedStream { - type Item = Vec; - - fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - match self.as_mut().project() { - BatchedStreamProj::Active { mut stream, batch } => { - for _ in 0..MAX_POLLS { - match stream.as_mut().poll_next(cx) { - Poll::Ready(Some(item)) => batch.push(item), - Poll::Ready(None) => { - if batch.is_empty() { - return Poll::Ready(None); - } else { - let batch = std::mem::take(batch); - self.as_mut().set(BatchedStream::Complete); - return Poll::Ready(Some(batch)); - } - } - Poll::Pending => break, - } - } - - if batch.is_empty() { - cx.waker().wake_by_ref(); - Poll::Pending - } else { - let batch = std::mem::take(batch); - Poll::Ready(Some(batch)) - } - } - BatchedStreamProj::Complete => Poll::Ready(None), - } - } -} diff --git a/core/src/util/debug_initializer.rs b/core/src/util/debug_initializer.rs deleted file mode 100644 index 562ca7b07..000000000 --- a/core/src/util/debug_initializer.rs +++ /dev/null @@ -1,198 +0,0 @@ -// ! A system for loading a default set of data on startup. This is ONLY enabled in development builds. - -use crate::{ - library::{Libraries, LibraryManagerError, LibraryName}, - location::{ - delete_location, scan_location, LocationCreateArgs, LocationError, LocationManagerError, - ScanState, - }, - old_job::JobManagerError, - util::AbortOnDrop, - Node, -}; - -use sd_prisma::prisma::location; -use sd_utils::error::FileIOError; - -use std::{ - io, - path::{Path, PathBuf}, - sync::Arc, - time::Duration, -}; - -use prisma_client_rust::QueryError; -use serde::Deserialize; -use thiserror::Error; -use tokio::{ - fs::{self, metadata}, - time::sleep, -}; -use tracing::{info, instrument, warn}; -use uuid::Uuid; - -#[derive(Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct LocationInitConfig { - path: String, -} - -#[derive(Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct LibraryInitConfig { - id: Uuid, - name: LibraryName, - description: Option, - #[serde(default)] - reset_locations_on_startup: bool, - locations: Vec, -} - -#[derive(Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct InitConfig { - #[serde(default)] - reset_on_startup: bool, - libraries: Vec, - #[serde(skip, default)] - path: PathBuf, -} - -#[derive(Error, Debug)] -pub enum InitConfigError { - #[error("error parsing the init data: {0}")] - Json(#[from] serde_json::Error), - #[error("job manager: {0}")] - JobManager(#[from] JobManagerError), - #[error("location manager: {0}")] - LocationManager(#[from] LocationManagerError), - #[error("library manager: {0}")] - LibraryManager(#[from] LibraryManagerError), - #[error("query error: {0}")] - QueryError(#[from] QueryError), - #[error("location error: {0}")] - LocationError(#[from] LocationError), - #[error("failed to get current directory from environment: {0}")] - CurrentDir(io::Error), - - #[error(transparent)] - Processing(#[from] sd_core_heavy_lifting::Error), - #[error(transparent)] - FileIO(#[from] FileIOError), -} - -impl InitConfig { - pub async fn load(data_dir: &Path) -> Result, InitConfigError> { - let path = std::env::current_dir() - .map_err(InitConfigError::CurrentDir)? - .join(std::env::var("SD_INIT_DATA").unwrap_or("sd_init.json".to_string())); - - if metadata(&path).await.is_ok() { - let config = fs::read(&path) - .await - .map_err(|e| FileIOError::from((&path, e, "Failed to read init config file")))?; - - let mut config = serde_json::from_slice::(&config)?; - - config.path = path; - - if config.reset_on_startup && metadata(data_dir).await.is_ok() { - warn!("previous 'SD_DATA_DIR' was removed on startup!"); - fs::remove_dir_all(data_dir).await.map_err(|e| { - FileIOError::from((data_dir, e, "Failed to remove data directory")) - })?; - } - - return Ok(Some(config)); - } - - Ok(None) - } - - #[instrument(skip_all, fields(path = %self.path.display()), err)] - pub async fn apply( - self, - library_manager: &Arc, - node: &Arc, - ) -> Result<(), InitConfigError> { - info!("Initializing app from file"); - - for lib in self.libraries { - let name = lib.name.to_string(); - let _guard = AbortOnDrop(tokio::spawn(async move { - loop { - info!(library_name = %name, "Initializing library from 'sd_init.json'...;"); - sleep(Duration::from_secs(1)).await; - } - })); - - let library = if let Some(lib) = library_manager.get_library(&lib.id).await { - lib - } else { - let library = library_manager - .create_with_uuid(lib.id, lib.name, lib.description, true, None, node) - .await?; - - let Some(lib) = library_manager.get_library(&library.id).await else { - warn!( - "Debug init error: library '{}' was not found after being created!", - library.config().await.name.as_ref() - ); - return Ok(()); - }; - - lib - }; - - if lib.reset_locations_on_startup { - let locations = library.db.location().find_many(vec![]).exec().await?; - - for location in locations { - warn!(location_path = ?location.path, "deleting location;"); - delete_location(node, &library, location.id).await?; - } - } - - for loc in lib.locations { - if let Some(location) = library - .db - .location() - .find_first(vec![location::path::equals(Some(loc.path.clone()))]) - .exec() - .await? - { - warn!(location_path = ?location.path, "deleting location;"); - delete_location(node, &library, location.id).await?; - } - - let sd_file = PathBuf::from(&loc.path).join(".spacedrive"); - - if let Err(e) = fs::remove_file(sd_file).await { - if e.kind() != io::ErrorKind::NotFound { - warn!(?e, "failed to remove '.spacedrive' file;"); - } - } - - if let Some(location) = (LocationCreateArgs { - path: PathBuf::from(loc.path.clone()), - dry_run: false, - indexer_rules_ids: Vec::new(), - }) - .create(node, &library) - .await? - { - scan_location(node, &library, location, ScanState::Pending).await?; - } else { - warn!( - location_path = ?loc.path, - "Debug init error: location was not found after being created!", - ); - } - } - } - - info!("Initialized app from file"); - - Ok(()) - } -} diff --git a/core/src/util/infallible_request.rs b/core/src/util/infallible_request.rs deleted file mode 100644 index b5af9be46..000000000 --- a/core/src/util/infallible_request.rs +++ /dev/null @@ -1,28 +0,0 @@ -//! A HTTP response builder similar to the [http] crate but designed to be infallible. - -use axum::http::{ - self, header::IntoHeaderName, response::Parts, HeaderValue, Response, StatusCode, -}; - -#[derive(Debug)] -pub struct InfallibleResponse(Parts); - -impl InfallibleResponse { - pub fn builder() -> Self { - Self(Response::new(()).into_parts().0) - } - - pub fn status(mut self, status: StatusCode) -> Self { - self.0.status = status; - self - } - - pub fn header(mut self, key: K, val: HeaderValue) -> Self { - self.0.headers.insert(key, val); - self - } - - pub fn body(self, body: B) -> http::Response { - Response::from_parts(self.0, body) - } -} diff --git a/core/src/util/maybe_undefined.rs b/core/src/util/maybe_undefined.rs deleted file mode 100644 index 832b1d761..000000000 --- a/core/src/util/maybe_undefined.rs +++ /dev/null @@ -1,88 +0,0 @@ -//! Copied from: https://docs.rs/async-graphql/latest/async_graphql/types/enum.MaybeUndefined.html -#![allow(unused)] - -use serde::{Deserialize, Deserializer, Serialize, Serializer}; -use specta::Type; - -// This exports an incorrect Typescript type. https://github.com/oscartbeaumont/specta/issues/157 -#[derive(Debug, Clone, Type)] -#[specta(untagged)] -pub enum MaybeUndefined { - Undefined, - Null, - Value(T), -} - -impl MaybeUndefined { - // `Undefined` will return `true` else `false`. - pub fn is_undefined(&self) -> bool { - matches!(self, Self::Undefined) - } - - // `Null | Value(T)` will return `true` else `false`. - pub fn is_defined(&self) -> bool { - !matches!(self, Self::Undefined) - } - - pub fn unwrap_or(self, t: T) -> T { - match self { - Self::Value(v) => v, - _ => t, - } - } -} - -impl From> for Option> { - fn from(v: MaybeUndefined) -> Option> { - match v { - MaybeUndefined::Undefined => None, - MaybeUndefined::Null => Some(None), - MaybeUndefined::Value(v) => Some(Some(v)), - } - } -} - -impl MaybeUndefined> { - /// Transposes a `MaybeUndefined` of a [`Result`] into a [`Result`] of a - /// `MaybeUndefined`. - /// - /// [`MaybeUndefined::Undefined`] will be mapped to - /// [`Ok`]`(`[`MaybeUndefined::Undefined`]`)`. [`MaybeUndefined::Null`] - /// will be mapped to [`Ok`]`(`[`MaybeUndefined::Null`]`)`. - /// [`MaybeUndefined::Value`]`(`[`Ok`]`(_))` and - /// [`MaybeUndefined::Value`]`(`[`Err`]`(_))` will be mapped to - /// [`Ok`]`(`[`MaybeUndefined::Value`]`(_))` and [`Err`]`(_)`. - #[inline] - pub fn transpose(self) -> Result, E> { - match self { - MaybeUndefined::Undefined => Ok(MaybeUndefined::Undefined), - MaybeUndefined::Null => Ok(MaybeUndefined::Null), - MaybeUndefined::Value(Ok(v)) => Ok(MaybeUndefined::Value(v)), - MaybeUndefined::Value(Err(e)) => Err(e), - } - } -} - -impl Serialize for MaybeUndefined { - fn serialize(&self, serializer: S) -> Result { - match self { - MaybeUndefined::Value(value) => value.serialize(serializer), - _ => serializer.serialize_none(), - } - } -} - -impl<'de, T> Deserialize<'de> for MaybeUndefined -where - T: Deserialize<'de>, -{ - fn deserialize(deserializer: D) -> Result, D::Error> - where - D: Deserializer<'de>, - { - Option::::deserialize(deserializer).map(|value| match value { - Some(value) => MaybeUndefined::Value(value), - None => MaybeUndefined::Null, - }) - } -} diff --git a/core/src/util/mod.rs b/core/src/util/mod.rs deleted file mode 100644 index e8096c65b..000000000 --- a/core/src/util/mod.rs +++ /dev/null @@ -1,17 +0,0 @@ -mod abort_on_drop; -mod batched_stream; -#[cfg(debug_assertions)] -pub mod debug_initializer; -mod infallible_request; -mod maybe_undefined; -pub mod mpscrr; -mod observable; -mod unsafe_streamed_query; -pub mod version_manager; - -pub use abort_on_drop::*; -pub use batched_stream::*; -pub use infallible_request::*; -pub use maybe_undefined::*; -pub use observable::*; -pub use unsafe_streamed_query::*; diff --git a/core/src/util/mpscrr.rs b/core/src/util/mpscrr.rs deleted file mode 100644 index 72daf3441..000000000 --- a/core/src/util/mpscrr.rs +++ /dev/null @@ -1,326 +0,0 @@ -//! A multi-producer single-consumer channel (mpsc) with a strongly consistent emit method. -//! -//! What does this mean? Well, any call to [Sender::emit] will not resolve it's future until all active [Receiver]'s have received the value and returned from their callback. -//! -//! Why would you want this? U want to emit a message on a channel and ensure it has been processed by the subscribers before continuing. -//! -//! Things to be aware of: -//! - Receiver's are lazily registered. Eg. `let rx2 = rx.clone();` will only be required to receive values if [Receiver::subscribe_one] or [Receiver::subscribe] is called on it. -//! - Panic in a receiver will cause the sender to ignore that receiver. It will not infinitely block on it. -//! -//! ## Example -//! -//! ```rust -//! use sd_core::util::mpscrr; -//! -//! # tokio::runtime::Runtime::new().unwrap().block_on(async { -//! let (tx, mut rx) = mpscrr::unbounded_channel::(); -//! -//! tokio::spawn(async move { -//! rx.subscribe(|value| async move { -//! assert_eq!(value, 42); -//! -//! 1 -//! }) -//! .await -//! .unwrap(); -//! }); -//! -//! // Wait for Tokio to spawn the tasks -//! tokio::time::sleep(std::time::Duration::from_millis(200)).await; -//! -//! let result: Vec = tx.emit(42).await; -//! assert_eq!(result, vec![1]); -//! # }); -//! ``` -//! - -// We ignore Mutex poising as the code is written such that it will not break any invariants if it panics. -// Keep this in mind! - -use std::{ - fmt, - future::Future, - sync::{ - atomic::{AtomicBool, Ordering}, - Arc, PoisonError, RwLock, - }, -}; - -use futures::future::join_all; -use slotmap::{DefaultKey, SlotMap}; -use tokio::sync::{mpsc, oneshot}; - -pub type Pair = (Sender, Receiver); - -type MpscInnerTy = (T, oneshot::Sender); - -type Slots = - Arc>, Arc)>>>; - -enum SenderError { - /// Receiver was dropped, so remove it. - Finished(DefaultKey), - /// Receiver failed to respond but is still assumed active. - Ignored, -} - -/// Returned by a [Receiver] when the [Sender] is dropped while trying to receive a value. -pub struct RecvError {} - -impl fmt::Debug for RecvError { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.write_str("RecvError") - } -} - -#[derive(Debug)] -pub struct Sender(Slots); - -impl Sender { - pub async fn emit(&self, value: T) -> Vec { - // This is annoying AF but holding a mutex guard over await boundary is `!Sync` which will break code using this. - let map = self - .0 - .read() - .unwrap_or_else(PoisonError::into_inner) - .iter() - .map(|(k, v)| (k, v.clone())) - .collect::>(); - - join_all(map.into_iter().filter_map(|(key, (sender, active))| { - if !active.load(Ordering::Relaxed) { - // The receiver has no callback registered so we ignore it. - return None; - } - - let value = value.clone(); - Some(async move { - let (tx, rx) = oneshot::channel(); - if sender.send((value, tx)).is_err() { - // The receiver was dropped so we remove it from the map - Err(SenderError::Finished(key)) - } else { - // If oneshot was dropped we ignore this subscriber as something went wrong with it. - // It is assumed the mpsc is fine and if it's not it will be cleared up by it's `Drop` or the next `emit`. - rx.await.map_err(|_| SenderError::Ignored) - } - }) - })) - .await - .into_iter() - .filter_map(|x| { - x.map_err(|e| match e { - SenderError::Finished(key) => { - self.0 - .write() - .unwrap_or_else(PoisonError::into_inner) - .remove(key); - } - SenderError::Ignored => {} - }) - .ok() - }) - .collect::>() - } -} - -#[derive(Debug)] -pub struct Receiver { - slots: Slots, - entry: DefaultKey, - rx: mpsc::UnboundedReceiver>, - active: Arc, -} - -impl Receiver { - /// This method will call the callback for the next value sent to the channel. - /// - /// It will block until the next message than return. - /// - /// If the sender is dropped this will return an error else it will return itself. - /// This is to avoid using the subscription after the sender is dropped. - pub async fn subscribe_one<'a, Fu: Future + 'a>( - mut self, - func: impl FnOnce(T) -> Fu + 'a, - ) -> Result { - let _bomb = Bomb::new(&self.active); - - let (value, tx) = self.rx.recv().await.ok_or(RecvError {})?; - tx.send(func(value).await).map_err(|_| RecvError {})?; - - drop(_bomb); - Ok(self) - } - - /// This method will call the callback for every value sent to the channel. - /// - /// It will block the active task until the sender is dropped. - /// - /// If the sender is dropped this will return an error. - pub async fn subscribe<'a, Fu: Future + 'a>( - mut self, - mut func: impl FnMut(T) -> Fu + 'a, - ) -> Result<(), RecvError> { - let _bomb = Bomb::new(&self.active); - - loop { - let (value, tx) = self.rx.recv().await.ok_or(RecvError {})?; - tx.send(func(value).await).map_err(|_| RecvError {})?; - } - } -} - -impl Drop for Receiver { - fn drop(&mut self) { - self.slots - .write() - .unwrap_or_else(PoisonError::into_inner) - .remove(self.entry); - } -} - -/// Construct a new unbounded channel. -pub fn unbounded_channel() -> (Sender, Receiver) { - let mut map = SlotMap::new(); - - // Create first receiver - let (tx, rx) = mpsc::unbounded_channel(); - let active: Arc = Arc::default(); - let entry = map.insert((tx, active.clone())); - - let slots = Arc::new(RwLock::new(map)); - ( - Sender(slots.clone()), - Receiver { - slots, - entry, - rx, - active, - }, - ) -} - -impl Clone for Receiver { - fn clone(&self) -> Self { - let (tx, rx) = mpsc::unbounded_channel(); - let active: Arc = Arc::default(); - let entry = self - .slots - .write() - .unwrap_or_else(PoisonError::into_inner) - .insert((tx, active.clone())); - - Self { - slots: self.slots.clone(), - entry, - rx, - active, - } - } -} - -// Bomb exists so on panic the `active` flag is reset to false. -struct Bomb<'a>(&'a AtomicBool); - -impl<'a> Bomb<'a> { - pub fn new(b: &'a AtomicBool) -> Self { - b.store(true, Ordering::Relaxed); - Self(b) - } -} - -impl Drop for Bomb<'_> { - fn drop(&mut self) { - self.0.store(false, Ordering::Relaxed); - } -} - -#[cfg(test)] -mod tests { - use std::{sync::Arc, time::Duration}; - - use boxcar; - - // Not using super because `use super as mpscrr` doesn't work :( - use crate::util::mpscrr; - - #[derive(Debug, Clone, PartialEq, Eq)] - enum Step { - Send, - RecvA, - RecvB, - SendComplete, - } - - #[tokio::test] - async fn test_mpscrr() { - let stack = Arc::new(boxcar::Vec::new()); - - let (tx, rx) = mpscrr::unbounded_channel::(); - - tokio::spawn({ - let rx = rx.clone(); - let stack = stack.clone(); - - async move { - rx.subscribe(|value| { - let stack = stack.clone(); - - async move { - assert_eq!(value, 42); - stack.push(Step::RecvA); - 1 - } - }) - .await - .unwrap(); - - // assert!(true, "recv a closed"); - } - }); - - tokio::spawn({ - let rx = rx.clone(); - let stack = stack.clone(); - - async move { - rx.subscribe(|value| { - let stack = stack.clone(); - - async move { - assert_eq!(value, 42); - stack.push(Step::RecvB); - 2 - } - }) - .await - .unwrap(); - - // assert!(true, "recv b closed"); - } - }); - - // Test unsubscribed receiver doesn't cause `.emit` to hang - let rx3 = rx; - - tokio::time::sleep(Duration::from_millis(200)).await; // Wait for Tokio to spawn the tasks - - stack.push(Step::Send); - let result = tx.emit(42).await; - stack.push(Step::SendComplete); - drop(rx3); - - // Check responses -> U shouldn't should NEVER assume order but we do here for simplicity - assert_eq!(result, vec![1, 2]); - // Check the order of operations - assert_eq!( - &to_vec(&stack), - &[Step::Send, Step::RecvA, Step::RecvB, Step::SendComplete,] - ) - } - - fn to_vec(a: &boxcar::Vec) -> Vec { - a.iter().map(|(_, entry)| entry).cloned().collect() - } -} diff --git a/core/src/util/observable.rs b/core/src/util/observable.rs deleted file mode 100644 index 808d1e072..000000000 --- a/core/src/util/observable.rs +++ /dev/null @@ -1,101 +0,0 @@ -#![allow(dead_code)] - -use std::{ - collections::hash_map::DefaultHasher, - hash::{Hash, Hasher}, - ops::{Deref, DerefMut}, -}; - -use tokio::sync::{Notify, RwLock, RwLockReadGuard, RwLockWriteGuard}; - -/// A simple JS-style observable in Rust -pub struct Observable { - t: RwLock, - notify: Notify, -} - -impl Observable -where - T: Hash, -{ - pub fn new(t: T) -> Self { - Self { - t: RwLock::new(t), - notify: Notify::new(), - } - } - - pub async fn get_mut(&self) -> ObservableRef<'_, T> { - let t = self.t.write().await; - - ObservableRef { - start_hash: { - let mut s = DefaultHasher::new(); - t.hash(&mut s); - s.finish() - }, - t, - notify: &self.notify, - } - } - - pub async fn set(&self, t: T) { - *self.get_mut().await = t; - } - - pub async fn get(&self) -> RwLockReadGuard<'_, T> { - self.t.read().await - } - - /// Wait until the value changes, then return the new value - pub async fn wait(&self) -> T - where - T: Clone, - { - self.notify.notified().await; - self.t.read().await.clone() - } -} - -pub struct ObservableRef<'a, T> -where - T: Hash, -{ - t: RwLockWriteGuard<'a, T>, - notify: &'a Notify, - start_hash: u64, -} - -impl Deref for ObservableRef<'_, T> -where - T: Hash, -{ - type Target = T; - - fn deref(&self) -> &Self::Target { - &self.t - } -} - -impl DerefMut for ObservableRef<'_, T> -where - T: Hash, -{ - fn deref_mut(&mut self) -> &mut Self::Target { - &mut self.t - } -} - -impl Drop for ObservableRef<'_, T> -where - T: Hash, -{ - fn drop(&mut self) { - let mut s = DefaultHasher::new(); - self.t.hash(&mut s); - - if self.start_hash != s.finish() { - self.notify.notify_waiters(); - } - } -} diff --git a/core/src/util/unsafe_streamed_query.rs b/core/src/util/unsafe_streamed_query.rs deleted file mode 100644 index 9a316373a..000000000 --- a/core/src/util/unsafe_streamed_query.rs +++ /dev/null @@ -1,31 +0,0 @@ -use std::pin::pin; - -use async_stream::stream; -use futures::{Stream, StreamExt}; -use serde::Serialize; -use specta::{DataType, Generics, Type, TypeMap}; - -#[derive(Serialize)] -#[serde(untagged)] -pub enum Output { - Data(T), - Complete { __stream_complete: () }, -} - -impl Type for Output { - fn inline(type_map: &mut TypeMap, generics: Generics) -> DataType { - T::inline(type_map, generics) - } -} - -// Marked as unsafe as the types are a lie and this should always be used with `useUnsafeStreamedQuery` -pub fn unsafe_streamed_query(stream: S) -> impl Stream> { - stream! { - let mut stream = pin!(stream); - while let Some(v) = stream.next().await { - yield Output::Data(v); - } - - yield Output::Complete { __stream_complete: () }; - } -} diff --git a/core/src/util/version_manager.rs b/core/src/util/version_manager.rs deleted file mode 100644 index 62d42308d..000000000 --- a/core/src/util/version_manager.rs +++ /dev/null @@ -1,224 +0,0 @@ -use sd_utils::error::FileIOError; - -use std::{ - any::type_name, fmt::Display, future::Future, num::ParseIntError, path::Path, str::FromStr, -}; - -use int_enum::{IntEnum, IntEnumError}; -use itertools::Itertools; -use serde::{de::DeserializeOwned, Serialize}; -use serde_json::{json, Map, Value}; -use thiserror::Error; -use tokio::{fs, io}; -use tracing::{debug, info, warn}; - -#[derive(Error, Debug)] -pub enum VersionManagerError> { - #[error("version file does not exist")] - VersionFileDoesNotExist, - #[error("malformed version file, reason: {reason}")] - MalformedVersionFile { reason: &'static str }, - #[error("unexpected migration: {current_version} -> {next_version}")] - UnexpectedMigration { - current_version: u64, - next_version: u64, - }, - #[error("failed to convert version to config file")] - ConvertToConfig, - - #[error(transparent)] - FileIO(#[from] FileIOError), - #[error(transparent)] - ParseInt(#[from] ParseIntError), - #[error(transparent)] - SerdeJson(#[from] serde_json::Error), - #[error(transparent)] - IntConversion(#[from] IntEnumError), -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum Kind { - PlainText, - Json(&'static str), // Version field name! -} - -pub trait ManagedVersion + Display + Eq + Serialize + DeserializeOwned>: - Serialize + DeserializeOwned + 'static -{ - const LATEST_VERSION: Version; - - const KIND: Kind; - - type MigrationError: std::error::Error + Display + From> + 'static; - - fn from_latest_version() -> Option { - None - } -} - -/// An abstract system for saving a text file containing a version number. -/// The version number is an integer that can be converted to and from an enum. -/// The enum must implement the IntEnum trait. -pub struct VersionManager< - Config: ManagedVersion, - Version: IntEnum + Display + Eq + Serialize + DeserializeOwned, -> { - _marker: std::marker::PhantomData<(Config, Version)>, -} - -impl< - Config: ManagedVersion, - Version: IntEnum + Display + Eq + Serialize + DeserializeOwned, - > VersionManager -{ - async fn get_version( - &self, - version_file_path: impl AsRef, - ) -> Result> { - let version_file_path = version_file_path.as_ref(); - - match Config::KIND { - Kind::PlainText => match fs::read_to_string(version_file_path).await { - Ok(contents) => { - let version = u64::from_str(contents.trim())?; - Version::from_int(version).map_err(Into::into) - } - Err(e) if e.kind() == io::ErrorKind::NotFound => { - Err(VersionManagerError::VersionFileDoesNotExist) - } - Err(e) => Err(FileIOError::from((version_file_path, e)).into()), - }, - Kind::Json(field) => match fs::read(version_file_path).await { - Ok(bytes) => { - let Some(version) = serde_json::from_slice::>(&bytes)? - .get(field) - .and_then(|version| version.as_u64()) - else { - return Err(VersionManagerError::MalformedVersionFile { - reason: "missing version field", - }); - }; - - Version::from_int(version).map_err(Into::into) - } - Err(e) if e.kind() == io::ErrorKind::NotFound => { - Err(VersionManagerError::VersionFileDoesNotExist) - } - Err(e) => Err(FileIOError::from((version_file_path, e)).into()), - }, - } - } - - async fn set_version( - &self, - version_file_path: impl AsRef, - version: Version, - ) -> Result<(), VersionManagerError> { - let version_file_path = version_file_path.as_ref(); - - match Config::KIND { - Kind::PlainText => fs::write( - version_file_path, - version.int_value().to_string().as_bytes(), - ) - .await - .map_err(|e| FileIOError::from((version_file_path, e)).into()), - - Kind::Json(field) => { - let mut data_value = serde_json::from_slice::>( - &fs::read(version_file_path) - .await - .map_err(|e| FileIOError::from((version_file_path, e)))?, - )?; - - data_value.insert(String::from(field), json!(version.int_value())); - - fs::write(version_file_path, serde_json::to_vec(&data_value)?) - .await - .map_err(|e| FileIOError::from((version_file_path, e)).into()) - } - } - } - - pub async fn migrate_and_load( - version_file_path: impl AsRef, - migrate_fn: impl Fn(Version, Version) -> Fut, - ) -> Result - where - Fut: Future>, - { - let version_file_path = version_file_path.as_ref(); - - let this = VersionManager { - _marker: std::marker::PhantomData::<(Config, Version)>, - }; - - let current = match this.get_version(version_file_path).await { - Ok(version) => version, - Err(VersionManagerError::VersionFileDoesNotExist) => { - warn!( - config = %type_name::(), - latest_version = %Config::LATEST_VERSION, - "Config file for does not exist, trying to create a new one with latest version;", - ); - - let Some(latest_config) = Config::from_latest_version() else { - return Err(VersionManagerError::VersionFileDoesNotExist.into()); - }; - - fs::write( - version_file_path, - match Config::KIND { - Kind::PlainText => Config::LATEST_VERSION - .int_value() - .to_string() - .as_bytes() - .to_vec(), - Kind::Json(_) => serde_json::to_vec(&latest_config) - .map_err(|e| VersionManagerError::SerdeJson(e))?, - }, - ) - .await - .map_err(|e| { - VersionManagerError::FileIO(FileIOError::from((version_file_path, e))) - })?; - - return Ok(latest_config); - } - Err(e) => return Err(e.into()), - }; - - if current != Config::LATEST_VERSION { - for (current_version, next_version) in - (current.int_value()..=Config::LATEST_VERSION.int_value()).tuple_windows() - { - let (current, next) = ( - Version::from_int(current_version).map_err(VersionManagerError::from)?, - Version::from_int(next_version).map_err(VersionManagerError::from)?, - ); - - info!( - config = %type_name::(), - %current, - %next, - "Running migrator;", - ); - migrate_fn(current, next).await?; - } - - this.set_version(version_file_path, Config::LATEST_VERSION) - .await?; - } else { - debug!(config = %type_name::(), "No migration required;"); - } - - fs::read(version_file_path) - .await - .map_err(|e| { - VersionManagerError::FileIO(FileIOError::from((version_file_path, e))).into() - }) - .and_then(|bytes| { - serde_json::from_slice(&bytes).map_err(|e| VersionManagerError::SerdeJson(e).into()) - }) - } -} diff --git a/core/src/volume/actor.rs b/core/src/volume/actor.rs deleted file mode 100644 index d1fa062a1..000000000 --- a/core/src/volume/actor.rs +++ /dev/null @@ -1,521 +0,0 @@ -use super::{ - error::VolumeError, - speed::SpeedTest, - types::{Volume, VolumeEvent, VolumeOptions}, - volumes::Volumes, - watcher::VolumeWatcher, - VolumeManagerContext, VolumeManagerState, -}; -use crate::volume::types::VolumeFingerprint; -use crate::{ - library::{Library, LibraryManagerEvent}, - volume::MountType, -}; -use async_channel as chan; -use sd_core_sync::DevicePubId; -use sd_prisma::prisma::volume; -use std::{sync::Arc, time::Duration}; -use tokio::sync::{broadcast, oneshot, Mutex, RwLock}; -use tracing::{debug, error, info, trace, warn}; - -const DEFAULT_CHANNEL_SIZE: usize = 128; - -#[derive(Debug)] -pub enum VolumeManagerMessage { - TrackVolume { - fingerprint: VolumeFingerprint, - library: Arc, - ack: oneshot::Sender>, - }, - UntrackVolume { - fingerprint: VolumeFingerprint, - library: Arc, - ack: oneshot::Sender>, - }, - UpdateVolume { - volume: Volume, - ack: oneshot::Sender>, - }, - MountVolume { - fingerprint: VolumeFingerprint, - ack: oneshot::Sender>, - }, - UnmountVolume { - fingerprint: VolumeFingerprint, - ack: oneshot::Sender>, - }, - SpeedTest { - fingerprint: VolumeFingerprint, - library: Arc, - ack: oneshot::Sender>, - }, - ListSystemVolumes { - library: Arc, - ack: oneshot::Sender, VolumeError>>, - }, - ListLibraryVolumes { - library: Arc, - ack: oneshot::Sender, VolumeError>>, - }, -} - -#[derive(Clone)] -pub struct VolumeManagerActor { - state: Arc>, - message_rx: chan::Receiver, - event_tx: broadcast::Sender, - ctx: Arc, -} - -impl VolumeManagerActor { - pub async fn new(ctx: Arc) -> Result<(Volumes, Self), VolumeError> { - Self::new_with_config(ctx, VolumeOptions::default()).await - } - - // Creates a new VolumeManagerActor with custom configuration - pub async fn new_with_config( - ctx: Arc, - options: VolumeOptions, - ) -> Result<(Volumes, Self), VolumeError> { - let (message_tx, message_rx) = chan::bounded(DEFAULT_CHANNEL_SIZE); - let (event_tx, _) = broadcast::channel(DEFAULT_CHANNEL_SIZE); - - let manager = Volumes::new(message_tx, event_tx.clone()); - debug!("Volume manager event_tx initialized"); - let state = - VolumeManagerState::new(ctx.device_id.clone().into(), options, event_tx.clone()); - debug!("Volume manager state initialized"); - - let actor = VolumeManagerActor { - state: Arc::new(RwLock::new(state)), - message_rx, - event_tx, - ctx, - }; - - Ok((manager, actor)) - } - - /// Starts the VolumeManagerActor - /// It will scan volumes, start the watcher, start the maintenance task, and handle messages - pub async fn start(self, device_id: DevicePubId) { - info!("Volume manager actor started"); - let self_arc = Arc::new(Mutex::new(self)); - - // Start event monitoring - let actor = self_arc.lock().await; - let event_rx = actor.event_tx.subscribe(); - actor - .clone() - .start_event_monitoring(event_rx, device_id.clone()); - drop(actor); - - // Handle messages - let self_arc_msg = Arc::clone(&self_arc); - tokio::spawn(async move { - let message_rx = self_arc_msg.lock().await.message_rx.clone(); - while let Ok(msg) = message_rx.recv().await { - let self_arc_inner = Arc::clone(&self_arc_msg); - if let Err(e) = { - let mut actor = self_arc_inner.lock().await; - actor.handle_message(msg).await - } { - error!(?e, "Error handling volume manager message"); - } - } - }); - - // Scan for volumes on startup - // unlock registry rwlock - let _ = self_arc.lock().await.scan_volumes().await; - - // Subscribe to LibraryManagerEvent - let self_arc_subscribe = Arc::clone(&self_arc); - let rx = { - let actor = self_arc.lock().await; - actor.ctx.library_event_tx.clone() - }; - // This is a fire-and-forget subscription - tokio::spawn(async move { - if let Err(e) = rx - .subscribe(move |event| { - let self_arc_inner = Arc::clone(&self_arc_subscribe); - async move { - debug!("Received library event: {:?}", event); - match event { - LibraryManagerEvent::Load(library) => { - debug!("Initializing volume manager for library"); - if let Err(e) = { - let mut actor = self_arc_inner.lock().await; - - actor.initialize_for_library(library.clone()).await - } { - error!(?e, "Failed to initialize volume manager for library"); - } - } - LibraryManagerEvent::Delete(library) => { - if let Err(e) = { - let mut actor = self_arc_inner.lock().await; - actor.handle_library_deletion(library).await - } { - error!(?e, "Failed to cleanup library volumes"); - } - } - _ => {} - } - } - }) - .await - { - error!(?e, "VolumeManager's library subscription failed"); - } - }); - - let event_tx = self_arc.lock().await.event_tx.clone(); - - // Start the volume watcher - let self_arc_watcher = Arc::clone(&self_arc); - tokio::spawn(async move { - let watcher = VolumeWatcher::new(event_tx); - if let Err(e) = watcher - .start(device_id.clone(), self_arc_watcher.clone()) - .await - { - error!(?e, "Failed to start watcher for volumes"); - return; - } - }); - - // Start maintenance task - let self_arc_maintenance = Arc::clone(&self_arc); - tokio::spawn(async move { - loop { - tokio::time::sleep(Duration::from_secs(300)).await; - let mut actor = self_arc_maintenance.lock().await; - if let Err(e) = actor.perform_maintenance().await { - error!(?e, "Volume maintenance error"); - } - } - }); - - info!("Volume manager actor initialized"); - } - - fn start_event_monitoring( - self, - mut event_rx: broadcast::Receiver, - device_pub_id: DevicePubId, - ) { - tokio::spawn(async move { - debug!("Starting volume event monitoring"); - while let Ok(event) = event_rx.recv().await { - debug!("Volume event received: {:?}", event); - - match event { - VolumeEvent::VolumeSpeedTested { - fingerprint, - read_speed, - write_speed, - } => { - // Get read lock first to check volume existence - let volume_exists = { - let state = self.state.read().await; - state.get_volume(&fingerprint).await.is_some() - }; - - if volume_exists { - // Then get write lock to update speeds - let state = self.state.write().await; - let mut registry = state.registry.write().await; - if let Some(volume) = registry.get_volume_mut(&fingerprint) { - volume.read_speed_mbps = Some(read_speed); - volume.write_speed_mbps = Some(write_speed); - } - } - } - _ => { - // Handle other events with a single write lock - let state = self.state.write().await; - let mut registry = state.registry.write().await; - - match event { - VolumeEvent::VolumeAdded(volume) => { - registry.register_volume(volume); - } - VolumeEvent::VolumeRemoved(volume) => { - let fingerprint = VolumeFingerprint::new(&device_pub_id, &volume); - registry.remove_volume(&fingerprint); - } - VolumeEvent::VolumeUpdated { old: _, new } => { - registry.update_volume(new); - } - VolumeEvent::VolumeMountChanged { - fingerprint, - is_mounted, - } => { - if let Some(volume) = registry.get_volume_mut(&fingerprint) { - volume.is_mounted = is_mounted; - } - } - VolumeEvent::VolumeError { fingerprint, error } => { - if let Some(volume) = registry.get_volume_mut(&fingerprint) { - volume.error_status = Some(error); - } - } - _ => {} - } - } - } - } - warn!("Volume event monitoring ended"); - }); - } - - /// Syncs volume memory state with library database - pub async fn initialize_for_library( - &mut self, - library: Arc, - ) -> Result<(), VolumeError> { - use sd_prisma::prisma::device; - let device_id = DevicePubId::from(self.ctx.device_id.clone()); - let state = self.state.clone(); - let state = state.write().await; - - let db_device = library - .db - .device() - .find_unique(device::pub_id::equals(device_id.to_db())) - .exec() - .await? - .ok_or(VolumeError::DeviceNotFound(device_id.to_db()))?; - - // Get volumes from database - let db_volumes = library - .db - .volume() - .find_many(vec![volume::device_id::equals(Some(db_device.id))]) - .exec() - .await? - .into_iter() - .map(Volume::from) - .collect::>(); - - let registry_read = state.registry.read().await; - // Process each volume - for (fingerprint, volume) in registry_read.volumes() { - // Find matching database volume - if let Some(db_volume) = db_volumes - .iter() - .find(|db_vol| VolumeFingerprint::new(&device_id, db_vol) == *fingerprint) - { - // Update existing volume - let updated = Volume::merge_with_db(volume, db_volume); - let mut registry = state.registry.write().await; - registry.register_volume(updated.clone()); - } else if volume.mount_type == MountType::System { - // Create new system volume in database - let created = volume.create(&library.db, device_id.to_db()).await?; - } - } - - Ok(()) - } - - async fn perform_maintenance(&mut self) -> Result<(), VolumeError> { - let mut state = self.state.write().await; - - Ok(()) - } - - async fn scan_volumes(&mut self) -> Result<(), VolumeError> { - let mut state = self.state.write().await; - state.scan_volumes().await - } - - async fn handle_message(&mut self, msg: VolumeManagerMessage) -> Result<(), VolumeError> { - trace!("VolumeManagerActor received message: {:?}", msg); - match msg { - VolumeManagerMessage::ListSystemVolumes { ack, library } => { - let result = self.handle_list_system_volumes(library).await; - let _ = ack.send(result); - } - VolumeManagerMessage::ListLibraryVolumes { library, ack } => { - todo!(); - } - VolumeManagerMessage::TrackVolume { - fingerprint, - library, - ack, - } => { - let result = self.handle_track_volume(library, fingerprint).await; - let _ = ack.send(result); - } - VolumeManagerMessage::UntrackVolume { - fingerprint, - library, - ack, - } => todo!(), - VolumeManagerMessage::UpdateVolume { volume, ack } => todo!(), - VolumeManagerMessage::MountVolume { fingerprint, ack } => todo!(), - VolumeManagerMessage::UnmountVolume { fingerprint, ack } => { - let result = self - .handle_unmount_volume(fingerprint, self.ctx.device_id.clone().into()) - .await; - let _ = ack.send(result); - } - VolumeManagerMessage::SpeedTest { - fingerprint, - ack, - library, - } => todo!(), - } - Ok(()) - } - - /// Lists all volumes currently mounted on the system - async fn handle_list_system_volumes( - &self, - library: Arc, - ) -> Result, VolumeError> { - tracing::info!("Listing system volumes for library {}", library.id); - - self.state - .read() - .await - .get_volumes_for_library(library) - .await - } - - pub async fn get_volumes(&self) -> Vec { - self.state.read().await.list_volumes().await - } - - pub async fn volume_exists(&self, fingerprint: VolumeFingerprint) -> bool { - self.state.read().await.volume_exists(&fingerprint).await - } - - // async fn handle_list_library_volumes( - // &self, - // library: Arc, - // ) -> Result, VolumeError> { - // let device_pub_id = self.ctx.device_id.clone(); - // let mut result_volumes = Vec::new(); - - // // Get currently mounted volumes on this system - // let system_volumes = self.handle_list_system_volumes(library.clone()).await?; - - // // Get all volumes from the library database - // let db_volumes = library - // .db - // .volume() - // .find_many(vec![]) - // .exec() - // .await? - // .into_iter() - // .map(Volume::from) - // .collect::>(); - - // // Create fingerprint maps - create references to avoid moving - // let system_map: HashMap, &Volume> = system_volumes - // .iter() - // .map(|v| (v.generate_fingerprint(device_pub_id.clone().into()), v)) - // .collect(); - - // // First add all currently mounted volumes, merged with DB data if available - // for volume in &system_volumes { - // let fingerprint = volume.generate_fingerprint(device_pub_id.clone().into()); - // if let Some(db_volume) = db_volumes - // .iter() - // .find(|v| v.generate_fingerprint(device_pub_id.clone().into()) == fingerprint) - // { - // result_volumes.push(Volume::merge_with_db_volume(&volume, db_volume)); - // } else { - // result_volumes.push(volume.clone()); - // } - // } - - // // Then add any database volumes that aren't currently mounted - // for db_volume in db_volumes { - // let fingerprint = db_volume.generate_fingerprint(device_pub_id.clone().into()); - // if !system_map.contains_key(&fingerprint) { - // result_volumes.push(db_volume); - // } - // } - - // Ok(result_volumes) - // } - - /// When tracking a volume, we associate it with the current device in the database - async fn handle_track_volume( - &mut self, - library: Arc, - fingerprint: VolumeFingerprint, - ) -> Result<(), VolumeError> { - let state = self.state.write().await; - let device_pub_id = self.ctx.device_id.clone(); - - // Find the volume in our current system volumes - let mut registry = state.registry.write().await; - let mut volume = match registry.get_volume_mut(&fingerprint) { - Some(v) => v.clone(), - None => return Err(VolumeError::InvalidFingerprint(fingerprint.clone())), - }; - - // Create in database with current device association - volume.create(&library.db, device_pub_id.into()).await?; - - // Spawn a background task to perform the speed test - let event_tx = self.event_tx.clone(); - let mut volume = volume.clone(); - tokio::spawn(async move { - if let Err(e) = volume.speed_test(None, Some(&event_tx)).await { - error!(?e, "Failed to perform speed test for volume"); - } - }); - - Ok(()) - } - - async fn handle_unmount_volume( - &mut self, - fingerprint: VolumeFingerprint, - device_pub_id: DevicePubId, - ) -> Result<(), VolumeError> { - let state = self.state.read().await; - let volume = state - .get_volume(&fingerprint) - .await - .ok_or_else(|| VolumeError::NotFound(fingerprint.clone()))?; - - if !volume.is_mounted { - return Err(VolumeError::NotMounted(volume.mount_point.clone())); - } - - // Call platform-specific unmount - super::os::unmount_volume(&volume.mount_point).await?; - - let fingerprint = VolumeFingerprint::new(&device_pub_id, &volume); - - // Emit unmount event - if let Some(pub_id) = volume.pub_id.as_ref() { - let _ = self.event_tx.send(VolumeEvent::VolumeMountChanged { - fingerprint, - is_mounted: false, - }); - } - - Ok(()) - } - - async fn handle_library_deletion(&mut self, library: Arc) -> Result<(), VolumeError> { - // Clean up volumes associated with deleted library - let _state = self.state.write().await; - - // TODO: Implement library deletion cleanup - // This might involve: - // 1. Removing volumes only tracked by this library - // 2. Updating volumes tracked by multiple libraries - - Ok(()) - } -} diff --git a/core-new/src/volume/classification.rs b/core/src/volume/classification.rs similarity index 100% rename from core-new/src/volume/classification.rs rename to core/src/volume/classification.rs diff --git a/core/src/volume/error.rs b/core/src/volume/error.rs index ecc031322..cc2564ff0 100644 --- a/core/src/volume/error.rs +++ b/core/src/volume/error.rs @@ -1,309 +1,83 @@ -//! Error types for volume management operations -use std::fmt; -use std::path::{Path, PathBuf}; -use thiserror::Error; +//! Volume-related error types -use super::types::VolumeFingerprint; +use thiserror::Error; /// Errors that can occur during volume operations #[derive(Error, Debug)] pub enum VolumeError { - // Add context to all errors - #[error("{context}: {source}")] - WithContext { - context: String, - source: Box, - }, + /// IO error during volume operations + #[error("IO error: {0}")] + Io(#[from] std::io::Error), - // Add operation-specific errors - #[error("Operation {0} failed: {1}")] - OperationFailed(uuid::Uuid, String), + /// Platform-specific error + #[error("Platform error: {0}")] + Platform(String), - /// Failed to perform I/O operation - #[error("I/O error: {0}")] - Io(#[from] std::io::Error), + /// Volume not found + #[error("Volume not found: {0}")] + NotFound(String), - /// Operation timed out - #[error("Operation timed out after {0} seconds")] - Timeout(u64), + /// Volume is not mounted + #[error("Volume is not mounted: {0}")] + NotMounted(String), - /// No mount point found for volume - #[error("No mount point found for volume")] - NoMountPoint, + /// Volume is read-only + #[error("Volume is read-only: {0}")] + ReadOnly(String), - /// Volume is already mounted - #[error("Volume with fingerprint {} is not found", 0)] - NotFound(VolumeFingerprint), + /// Insufficient space on volume + #[error("Insufficient space on volume: required {required}, available {available}")] + InsufficientSpace { required: u64, available: u64 }, - /// Volume isn't in database yet - #[error("Volume not yet tracked in database")] - NotInDatabase, + /// Speed test was cancelled or failed + #[error("Speed test cancelled or failed")] + SpeedTestFailed, - /// Invalid volume ID - #[error("Invalid volume fingerprint: {0}")] - InvalidFingerprint(VolumeFingerprint), + /// Volume detection failed + #[error("Volume detection failed: {0}")] + DetectionFailed(String), - /// Directory operation failed - #[error("Directory operation failed: {0}")] - DirectoryError(String), + /// Permission denied + #[error("Permission denied: {0}")] + PermissionDenied(String), - /// Database operation failed - #[error("Database error: {0}")] - Database(#[from] prisma_client_rust::QueryError), + /// Operation timed out + #[error("Operation timed out")] + Timeout, - /// Device error - #[error("Device error: {0}")] - DeviceError(String), + /// Invalid volume data + #[error("Invalid volume data: {0}")] + InvalidData(String), - /// No device found - #[error("No device found in database")] - NoDeviceFound, + /// Database operation failed + #[error("Database error: {0}")] + Database(String), - /// Volume already exists - #[error("Volume already exists at path: {}", .0.display())] - VolumeExists(PathBuf), + /// Volume is already tracked + #[error("Volume is already tracked: {0}")] + AlreadyTracked(String), - /// Volume is not mounted - #[error("Volume is not mounted: {}", .0.display())] - NotMounted(PathBuf), - - /// Volume is read-only - #[error("Volume is read-only: {}", .0.display())] - ReadOnly(PathBuf), - - /// Device not found - #[error("Device not found: {:?}", .0)] - DeviceNotFound(Vec), - - /// Volume does not have enough space - #[error("Insufficient space on volume: {} (needed: {needed} bytes, available: {available} bytes)", .path.display())] - InsufficientSpace { - path: PathBuf, - needed: u64, - available: u64, - }, - - /// Speed test error - #[error("Speed test failed: {kind}: {message}")] - SpeedTest { - kind: SpeedTestErrorKind, - message: String, - }, - - /// Watcher error - #[error("Volume watcher error: {0}")] - Watcher(#[from] WatcherError), - - /// Platform-specific error - #[error("Platform error: {0}")] - Platform(String), - - /// Permission denied - #[error("Permission denied for path: {}", .0.display())] - PermissionDenied(PathBuf), - - /// Volume is busy - #[error("Volume is busy: {}", .0.display())] - VolumeBusy(PathBuf), - - /// Operation cancelled - #[error("Operation was cancelled")] - Cancelled, - - /// Invalid configuration - #[error("Invalid configuration: {0}")] - InvalidConfiguration(String), - - /// Resource exhausted - #[error("Resource exhausted: {0}")] - ResourceExhausted(String), -} - -/// Specific kinds of speed test errors -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum SpeedTestErrorKind { - /// Failed to create test file - CreateFile, - /// Failed to write test data - Write, - /// Failed to read test data - Read, - /// Failed to cleanup test files - Cleanup, - /// Test exceeded timeout - Timeout, -} - -/// Errors specific to volume watching -#[derive(Error, Debug)] -pub enum WatcherError { - /// Failed to initialize watcher - #[error("Failed to initialize watcher: {0}")] - InitializationError(String), - - /// Failed to add watch - #[error("Failed to add watch for path: {}", .0.display())] - AddWatchError(PathBuf), - - /// Failed to remove watch - #[error("Failed to remove watch for path: {}", .0.display())] - RemoveWatchError(PathBuf), - - /// Event stream error - #[error("Event stream error: {0}")] - EventStreamError(String), -} - -// Implement conversion from tokio::time::error::Elapsed -impl From for VolumeError { - fn from(error: tokio::time::error::Elapsed) -> Self { - VolumeError::Timeout(30) // Default timeout value - } + /// Volume is not tracked + #[error("Volume is not tracked: {0}")] + NotTracked(String), } impl VolumeError { - /// Creates a new speed test error - pub fn speed_test(kind: SpeedTestErrorKind, message: impl Into) -> Self { - VolumeError::SpeedTest { - kind, - message: message.into(), - } - } + /// Create a platform-specific error + pub fn platform(msg: impl Into) -> Self { + Self::Platform(msg.into()) + } - /// Checks if the error is a timeout - pub fn is_timeout(&self) -> bool { - matches!(self, VolumeError::Timeout(_)) - } + /// Create a detection failed error + pub fn detection_failed(msg: impl Into) -> Self { + Self::DetectionFailed(msg.into()) + } - /// Checks if the error is permission related - pub fn is_permission_denied(&self) -> bool { - matches!(self, VolumeError::PermissionDenied(_)) - } - - /// Checks if the error is space related - pub fn is_space_error(&self) -> bool { - matches!(self, VolumeError::InsufficientSpace { .. }) - } - - /// Checks if the operation can be retried - pub fn is_retriable(&self) -> bool { - matches!( - self, - VolumeError::Timeout(_) - | VolumeError::VolumeBusy(_) - | VolumeError::ResourceExhausted(_) - ) - } - - /// Gets the path associated with the error, if any - pub fn path(&self) -> Option<&Path> { - match self { - VolumeError::VolumeExists(path) - | VolumeError::NotMounted(path) - | VolumeError::ReadOnly(path) - | VolumeError::InsufficientSpace { path, .. } - | VolumeError::PermissionDenied(path) - | VolumeError::VolumeBusy(path) => Some(path), - _ => None, - } - } + /// Create an insufficient space error + pub fn insufficient_space(required: u64, available: u64) -> Self { + Self::InsufficientSpace { required, available } + } } -// Implement conversion from VolumeError to rspc::Error for API responses -impl From for rspc::Error { - fn from(err: VolumeError) -> Self { - // Map error types to appropriate HTTP status codes - let code = match &err { - VolumeError::NotInDatabase - | VolumeError::NoMountPoint - | VolumeError::InvalidFingerprint(_) => rspc::ErrorCode::NotFound, - - VolumeError::PermissionDenied(_) => rspc::ErrorCode::Forbidden, - - VolumeError::Timeout(_) | VolumeError::VolumeBusy(_) => rspc::ErrorCode::Timeout, - - VolumeError::InsufficientSpace { .. } => rspc::ErrorCode::PayloadTooLarge, - - VolumeError::InvalidConfiguration(_) => rspc::ErrorCode::BadRequest, - - _ => rspc::ErrorCode::InternalServerError, - }; - - rspc::Error::with_cause(code, err.to_string(), err) - } -} - -// Helper trait for Result extension methods -pub trait VolumeResultExt { - /// Adds context to an error - fn with_context(self, context: impl FnOnce() -> String) -> Result; - - /// Adds path context to an error - fn with_path(self, path: impl AsRef) -> Result; -} - -impl VolumeResultExt for Result { - fn with_context(self, context: impl FnOnce() -> String) -> Result { - self.map_err(|e| VolumeError::DirectoryError(format!("{}: {}", context(), e))) - } - - fn with_path(self, path: impl AsRef) -> Result { - self.map_err(|e| match e { - VolumeError::Io(io_err) => match io_err.kind() { - std::io::ErrorKind::PermissionDenied => { - VolumeError::PermissionDenied(path.as_ref().to_path_buf()) - } - _ => VolumeError::DirectoryError(format!( - "Operation failed on path '{}': {}", - path.as_ref().display(), - io_err - )), - }, - other => other, - }) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_error_conversion() { - let io_error = std::io::Error::new(std::io::ErrorKind::NotFound, "file not found"); - let volume_error = VolumeError::from(io_error); - assert!(matches!(volume_error, VolumeError::Io(_))); - } - - #[test] - fn test_error_context() { - let result: Result<(), VolumeError> = Err(VolumeError::NoMountPoint); - let with_context = result.with_context(|| "Failed to mount volume".to_string()); - assert!(with_context.is_err()); - } - - #[test] - fn test_error_helpers() { - let error = VolumeError::InsufficientSpace { - path: PathBuf::from("/test"), - needed: 1000, - available: 500, - }; - assert!(error.is_space_error()); - assert!(!error.is_timeout()); - assert!(error.path().is_some()); - } -} - -impl fmt::Display for SpeedTestErrorKind { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - let kind_str = match self { - SpeedTestErrorKind::CreateFile => "Failed to create test file", - SpeedTestErrorKind::Write => "Failed to write test data", - SpeedTestErrorKind::Read => "Failed to read test data", - SpeedTestErrorKind::Cleanup => "Failed to cleanup test files", - SpeedTestErrorKind::Timeout => "Test exceeded timeout", - }; - write!(f, "{}", kind_str) - } -} +/// Result type for volume operations +pub type VolumeResult = Result; \ No newline at end of file diff --git a/core-new/src/volume/manager.rs b/core/src/volume/manager.rs similarity index 100% rename from core-new/src/volume/manager.rs rename to core/src/volume/manager.rs diff --git a/core/src/volume/mod.rs b/core/src/volume/mod.rs index 7cfe3cc2e..5464bad8c 100644 --- a/core/src/volume/mod.rs +++ b/core/src/volume/mod.rs @@ -1,44 +1,36 @@ -//! Volume management for Spacedrive +//! Volume management for Spacedrive Core v2 //! //! This module provides functionality for detecting, monitoring, and managing storage volumes -//! across different platforms. -//! Volumes use a fingerprint to identify them as they sometimes are not persisted in the database -//! -pub(crate) mod actor; -mod error; -mod os; -mod speed; -mod state; -mod types; -mod volumes; -mod watcher; -use crate::library::LibraryManagerEvent; -use crate::util::mpscrr; +//! across different platforms. It's designed to integrate with the copy system for optimal +//! file operation routing. -pub use { - actor::VolumeManagerActor, - error::VolumeError, - state::VolumeManagerState, - types::{ - DiskType, FileSystem, MountType, Volume, VolumeEvent, VolumeFingerprint, VolumeOptions, - }, - volumes::Volumes, +pub mod classification; +pub mod error; +pub mod manager; +pub mod os_detection; +pub mod speed; +pub mod types; + +pub use error::VolumeError; +pub use manager::VolumeManager; +pub use types::{ + DiskType, FileSystem, MountType, Volume, VolumeDetectionConfig, VolumeEvent, VolumeFingerprint, + VolumeInfo, }; -#[derive(Clone)] -pub struct VolumeManagerContext { - // Used for device identification - pub device_id: Vec, - pub library_event_tx: mpscrr::Receiver, -} +// Re-export platform-specific detection +pub use os_detection::detect_volumes; -// Extension trait for Volume operations that don't require actor communication +/// Extension trait for Volume operations pub trait VolumeExt { /// Checks if volume is mounted and accessible async fn is_available(&self) -> bool; /// Checks if volume has enough free space fn has_space(&self, required_bytes: u64) -> bool; + + /// Check if path is on this volume + fn contains_path(&self, path: &std::path::Path) -> bool; } impl VolumeExt for Volume { @@ -49,37 +41,52 @@ impl VolumeExt for Volume { fn has_space(&self, required_bytes: u64) -> bool { self.total_bytes_available >= required_bytes } + + fn contains_path(&self, path: &std::path::Path) -> bool { + // Check primary mount point + if path.starts_with(&self.mount_point) { + return true; + } + + // Check additional mount points (for APFS volumes) + self.mount_points.iter().any(|mp| path.starts_with(mp)) + } } -// Re-export platform-specific volume detection -#[cfg(target_os = "linux")] -pub use os::linux::get_volumes; -#[cfg(target_os = "macos")] -pub use os::macos::get_volumes; -#[cfg(any(target_os = "ios", target_os = "android"))] -pub use os::mobile::get_volumes; -#[cfg(target_os = "windows")] -pub use os::windows::get_volumes; - -// Internal utilities -pub(crate) mod util { +/// Utilities for volume operations +pub mod util { use super::*; use std::path::Path; - pub(crate) fn is_path_on_volume(path: &Path, volume: &Volume) -> bool { - path.starts_with(&volume.mount_point) + /// Check if a path is on the specified volume + pub fn is_path_on_volume(path: &Path, volume: &Volume) -> bool { + volume.contains_path(&path.to_path_buf()) } - pub(crate) fn calculate_path_on_volume( - path: &Path, - volume: &Volume, - ) -> Option { - if is_path_on_volume(path, volume) { - path.strip_prefix(&volume.mount_point) - .ok() - .map(|p| p.to_path_buf()) - } else { - None + /// Calculate relative path from volume mount point + pub fn relative_path_on_volume(path: &Path, volume: &Volume) -> Option { + // Try primary mount point first + if let Ok(relative) = path.strip_prefix(&volume.mount_point) { + return Some(relative.to_path_buf()); } + + // Try additional mount points + for mount_point in &volume.mount_points { + if let Ok(relative) = path.strip_prefix(mount_point) { + return Some(relative.to_path_buf()); + } + } + + None + } + + /// Find the volume that contains the given path + pub fn find_volume_for_path<'a>( + path: &Path, + volumes: impl Iterator, + ) -> Option<&'a Volume> { + volumes + .filter(|vol| vol.contains_path(&path.to_path_buf())) + .max_by_key(|vol| vol.mount_point.as_os_str().len()) // Prefer most specific mount } } diff --git a/core/src/volume/os.rs b/core/src/volume/os.rs deleted file mode 100644 index 315d18fbb..000000000 --- a/core/src/volume/os.rs +++ /dev/null @@ -1,502 +0,0 @@ -use super::error::VolumeError; -use super::types::{DiskType, FileSystem, MountType, Volume}; -use tokio::task; - -// Re-export platform-specific get_volumes function -#[cfg(target_os = "linux")] -pub use self::linux::get_volumes; -#[cfg(target_os = "macos")] -pub use self::macos::get_volumes; -#[cfg(any(target_os = "ios", target_os = "android"))] -pub use self::mobile::get_volumes; -#[cfg(target_os = "windows")] -pub use self::windows::get_volumes; - -// Re-export platform-specific unmount_volume function -#[cfg(target_os = "linux")] -pub use self::linux::unmount_volume; -#[cfg(target_os = "macos")] -pub use self::macos::unmount_volume; -#[cfg(any(target_os = "ios", target_os = "android"))] -pub use self::mobile::unmount_volume; -#[cfg(target_os = "windows")] -pub use self::windows::unmount_volume; - -/// Common utilities for volume detection across platforms -mod common { - pub fn parse_size(size_str: &str) -> u64 { - size_str - .chars() - .filter(|c| c.is_ascii_digit()) - .collect::() - .parse() - .unwrap_or(0) - } - - pub fn is_virtual_filesystem(fs: &str) -> bool { - matches!( - fs.to_lowercase().as_str(), - "devfs" | "sysfs" | "proc" | "tmpfs" | "ramfs" | "devtmpfs" - ) - } -} -#[cfg(target_os = "macos")] -pub mod macos { - use super::*; - use std::{path::PathBuf, process::Command}; - use sysinfo::{DiskExt, System, SystemExt}; - - pub async fn get_volumes() -> Result, VolumeError> { - // First collect disk info in blocking context - let disk_info: Vec<(String, bool, PathBuf, Vec, u64, u64)> = - task::spawn_blocking(|| { - let mut sys = System::new_all(); - sys.refresh_disks_list(); - - sys.disks() - .iter() - .filter(|disk| { - !common::is_virtual_filesystem( - std::str::from_utf8(disk.file_system()).unwrap_or(""), - ) - }) - .map(|disk| { - ( - disk.name().to_string_lossy().to_string(), - disk.is_removable(), - disk.mount_point().to_path_buf(), - disk.file_system().to_vec(), - disk.total_space(), - disk.available_space(), - ) - }) - .collect::>() // Specify that the collection should be a Vec - }) - .await - .map_err(|e| VolumeError::Platform(format!("Task join error: {}", e)))?; - - // Then create volumes with the collected info - let mut volumes = Vec::new(); - for (name, is_removable, mount_point, file_system, total_space, available_space) in - disk_info - { - if !mount_point.exists() { - continue; - } - let read_only = is_volume_readonly(&mount_point)?; - // Skip adding the `/` mount point if it's both read-only and a system volume - if mount_point == PathBuf::from("/") && read_only { - continue; - } - let disk_type = detect_disk_type(&name)?; - let mut mount_points = vec![mount_point.clone()]; - - // For macOS APFS system volumes - if mount_point == PathBuf::from("/") { - let data_path = PathBuf::from("/System/Volumes/Data"); - if data_path.exists() { - mount_points.push(data_path); - } - } - - volumes.push(Volume::new( - name, - if is_removable { - MountType::External - } else { - MountType::System - }, - mount_point, - mount_points, - disk_type, - FileSystem::from_string(&String::from_utf8_lossy(&file_system)), - total_space, - available_space, - read_only, - )); - } - - Ok(volumes) - } - - fn detect_disk_type(device_name: &str) -> Result { - let output = Command::new("diskutil") - .args(["info", device_name]) - .output() - .map_err(|e| VolumeError::Platform(format!("Failed to run diskutil: {}", e)))?; - - let info = String::from_utf8_lossy(&output.stdout); - Ok(if info.contains("Solid State") { - DiskType::SSD - } else if info.contains("Rotational") { - DiskType::HDD - } else { - DiskType::Unknown - }) - } - - fn is_volume_readonly(mount_point: &std::path::Path) -> Result { - let output = Command::new("mount") - .output() - .map_err(|e| VolumeError::Platform(format!("Failed to run mount command: {}", e)))?; - - let mount_output = String::from_utf8_lossy(&output.stdout); - Ok(mount_output - .lines() - .find(|line| line.contains(&*mount_point.to_string_lossy())) - .map(|line| line.contains("read-only")) - .unwrap_or(false)) - } - pub async fn unmount_volume(path: &std::path::Path) -> Result<(), VolumeError> { - use std::process::Command; - use tokio::process::Command as TokioCommand; - - // First try diskutil - let result = TokioCommand::new("diskutil") - .arg("unmount") - .arg(path) - .output() - .await; - - match result { - Ok(output) => { - if output.status.success() { - return Ok(()); - } - // If diskutil fails, try umount as fallback - let fallback = Command::new("umount") - .arg(path) - .output() - .map_err(|e| VolumeError::Platform(format!("Unmount failed: {}", e)))?; - - if fallback.status.success() { - Ok(()) - } else { - Err(VolumeError::Platform(format!( - "Failed to unmount volume: {}", - String::from_utf8_lossy(&fallback.stderr) - ))) - } - } - Err(e) => Err(VolumeError::Platform(format!( - "Failed to execute unmount command: {}", - e - ))), - } - } -} - -#[cfg(target_os = "linux")] -pub mod linux { - use super::*; - use std::{path::PathBuf, process::Command}; - use sysinfo::{DiskExt, System, SystemExt}; - - pub async fn get_volumes() -> Result, VolumeError> { - let disk_info: Vec<(String, bool, PathBuf, Vec, u64, u64)> = - tokio::task::spawn_blocking(|| { - let mut sys = System::new_all(); - sys.refresh_disks_list(); - - sys.disks() - .iter() - .filter(|disk| { - !common::is_virtual_filesystem( - std::str::from_utf8(disk.file_system()).unwrap_or(""), - ) - }) - .map(|disk| { - ( - disk.name().to_string_lossy().to_string(), - disk.is_removable(), - disk.mount_point().to_path_buf(), - disk.file_system().to_vec(), - disk.total_space(), - disk.available_space(), - ) - }) - .collect() - }) - .await - .map_err(|e| VolumeError::Platform(format!("Task join error: {}", e)))?; - - let mut volumes = Vec::new(); - for (name, is_removable, mount_point, file_system, total_space, available_space) in - disk_info - { - if !mount_point.exists() { - continue; - } - - let read_only = is_volume_readonly(&mount_point)?; - let disk_type = detect_disk_type(&name)?; - - volumes.push(Volume::new( - name, - if is_removable { - MountType::External - } else { - MountType::System - }, - mount_point.clone(), - vec![mount_point], - disk_type, - FileSystem::from_string(&String::from_utf8_lossy(&file_system)), - total_space, - available_space, - read_only, - )); - } - - Ok(volumes) - } - - fn detect_disk_type(device_name: &str) -> Result { - let path = format!( - "/sys/block/{}/queue/rotational", - device_name.trim_start_matches("/dev/") - ); - match std::fs::read_to_string(path) { - Ok(contents) => match contents.trim() { - "0" => Ok(DiskType::SSD), - "1" => Ok(DiskType::HDD), - _ => Ok(DiskType::Unknown), - }, - Err(_) => Ok(DiskType::Unknown), - } - } - - fn is_volume_readonly(mount_point: &std::path::Path) -> Result { - let output = Command::new("findmnt") - .args([ - "--noheadings", - "--output", - "OPTIONS", - mount_point.to_str().unwrap(), - ]) - .output() - .map_err(|e| VolumeError::Platform(format!("Failed to run findmnt: {}", e)))?; - - let options = String::from_utf8_lossy(&output.stdout); - Ok(options.contains("ro,") || options.contains(",ro") || options.contains("ro ")) - } - - pub async fn unmount_volume(path: &std::path::Path) -> Result<(), VolumeError> { - // Try regular unmount first - let result = tokio::process::Command::new("umount") - .arg(path) - .output() - .await; - - match result { - Ok(output) if output.status.success() => Ok(()), - _ => { - // If regular unmount fails, try lazy unmount - let lazy_result = tokio::process::Command::new("umount") - .args(["-l", path.to_str().unwrap()]) - .output() - .await - .map_err(|e| VolumeError::Platform(format!("Lazy unmount failed: {}", e)))?; - - if lazy_result.status.success() { - Ok(()) - } else { - Err(VolumeError::Platform(format!( - "Failed to unmount volume: {}", - String::from_utf8_lossy(&lazy_result.stderr) - ))) - } - } - } - } -} - -#[cfg(target_os = "windows")] -pub mod windows { - use super::*; - use std::ffi::OsString; - use std::path::PathBuf; - - use ::windows::core::PCWSTR; - use ::windows::Win32::Storage::FileSystem::{ - GetDiskFreeSpaceExW, GetDriveTypeW, GetVolumeInformationW, - }; - use ::windows::Win32::System::WindowsProgramming::{ - DRIVE_FIXED, DRIVE_REMOTE, DRIVE_REMOVABLE, - }; - use std::os::windows::ffi::OsStrExt; - - pub async fn get_volumes() -> Vec { - task::spawn_blocking(|| { - let mut volumes = Vec::new(); - - // Get available drives - let drives = unsafe { ::windows::Win32::Storage::FileSystem::GetLogicalDrives() }; - - for i in 0..26 { - if (drives & (1 << i)) != 0 { - let drive_letter = (b'A' + i as u8) as char; - let path = format!("{}:\\", drive_letter); - let wide_path: Vec = OsString::from(&path) - .encode_wide() - .chain(std::iter::once(0)) - .collect(); - - let drive_type = unsafe { GetDriveTypeW(PCWSTR(wide_path.as_ptr())) }; - - // Skip CD-ROM drives and other unsupported types - if drive_type == DRIVE_FIXED - || drive_type == DRIVE_REMOVABLE - || drive_type == DRIVE_REMOTE - { - if let Some(volume) = get_volume_info(&path, drive_type) { - volumes.push(volume); - } - } - } - } - - volumes - }) - .await - .unwrap_or_default() - } - - fn detect_disk_type(path: &str) -> DiskType { - // We would need to use DeviceIoControl to get this information - // For brevity, returning Unknown, but you could implement the full detection - // using IOCTL_STORAGE_QUERY_PROPERTY - DiskType::Unknown - } - - fn get_volume_info(path: &str, drive_type: u32) -> Option { - let wide_path: Vec = OsString::from(path) - .encode_wide() - .chain(std::iter::once(0)) - .collect(); - - let mut name_buf = [0u16; 256]; - let mut fs_name_buf = [0u16; 256]; - let mut serial_number = 0; - let mut max_component_length = 0; - let mut flags = 0; - - unsafe { - let success = GetVolumeInformationW( - PCWSTR(wide_path.as_ptr()), - Some(name_buf.as_mut_slice()), - Some(&mut serial_number), - Some(&mut max_component_length), - Some(&mut flags), - Some(&mut fs_name_buf), - ); - - if let Ok(_) = success { - let mut total_bytes = 0; - let mut free_bytes = 0; - let mut available_bytes = 0; - - if let Ok(_) = GetDiskFreeSpaceExW( - PCWSTR(wide_path.as_ptr()), - Some(&mut available_bytes), - Some(&mut total_bytes), - Some(&mut free_bytes), - ) { - let mount_type = match drive_type { - DRIVE_FIXED => MountType::System, - DRIVE_REMOVABLE => MountType::External, - DRIVE_REMOTE => MountType::Network, - _ => MountType::System, - }; - - let volume_name = String::from_utf16_lossy(&name_buf) - .trim_matches(char::from(0)) - .to_string(); - - let fs_name = String::from_utf16_lossy(&fs_name_buf) - .trim_matches(char::from(0)) - .to_string(); - - Some(Volume::new( - if volume_name.is_empty() { - path.to_string() - } else { - volume_name - }, - mount_type, - PathBuf::from(path), - vec![PathBuf::from(path)], - detect_disk_type(path), - FileSystem::from_string(&fs_name), - total_bytes, - available_bytes, - false, - )) - } else { - None - } - } else { - None - } - } - } - - pub async fn unmount_volume(path: &std::path::Path) -> Result<(), VolumeError> { - use ::windows::core::PWSTR; - use ::windows::Win32::Storage::FileSystem::{ - DeleteVolumeMountPointW, GetVolumeNameForVolumeMountPointW, - }; - use std::ffi::OsStr; - use std::os::windows::ffi::OsStrExt; - - // Convert path to wide string for Windows API - let mut wide_path: Vec = OsStr::new(path) - .encode_wide() - .chain(std::iter::once(0)) - .collect(); - - let wide_path_ptr = PWSTR(wide_path.as_mut_ptr()); - - unsafe { - // Buffer for volume name - let mut volume_name = [0u16; 50]; - - // Get the volume name for the mount point - let result = - GetVolumeNameForVolumeMountPointW(wide_path_ptr, volume_name.as_mut_slice()); - - if result.is_err() { - return Err(VolumeError::Platform( - "Failed to get volume name".to_string(), - )); - } - - // Delete the mount point - let result = DeleteVolumeMountPointW(wide_path_ptr); - - if let Ok(_) = result { - Ok(()) - } else { - Err(VolumeError::Platform( - "Failed to unmount volume".to_string(), - )) - } - } - } -} - -#[cfg(any(target_os = "ios", target_os = "android"))] -pub mod mobile { - use super::*; - - pub async fn get_volumes() -> Result, VolumeError> { - // Mobile platforms don't have mountable volumes - Ok(Vec::new()) - } - - pub async fn unmount_volume(_path: &std::path::Path) -> Result<(), VolumeError> { - Err(VolumeError::Platform( - "Volumes not supported on mobile platforms".to_string(), - )) - } -} diff --git a/core-new/src/volume/os_detection.rs b/core/src/volume/os_detection.rs similarity index 100% rename from core-new/src/volume/os_detection.rs rename to core/src/volume/os_detection.rs diff --git a/core/src/volume/speed.rs b/core/src/volume/speed.rs index cd05baef6..670fc2ece 100644 --- a/core/src/volume/speed.rs +++ b/core/src/volume/speed.rs @@ -1,14 +1,16 @@ -use super::error::VolumeError; -use super::types::{MountType, Volume, VolumeEvent}; -use std::path::PathBuf; +//! Volume speed testing functionality + +use crate::volume::{ + error::{VolumeError, VolumeResult}, + types::{MountType, Volume, VolumeType}, +}; use std::time::Instant; use tokio::{ fs::{File, OpenOptions}, io::{AsyncReadExt, AsyncWriteExt}, - sync::broadcast::Sender, time::{timeout, Duration}, }; -use tracing::{debug, error, instrument, trace}; +use tracing::{debug, instrument, warn}; /// Configuration for speed tests #[derive(Debug, Clone)] @@ -17,8 +19,8 @@ pub struct SpeedTestConfig { pub file_size_mb: usize, /// Timeout for the test in seconds pub timeout_secs: u64, - /// Whether to emit events during the test - pub emit_events: bool, + /// Number of test iterations for averaging + pub iterations: usize, } impl Default for SpeedTestConfig { @@ -26,7 +28,7 @@ impl Default for SpeedTestConfig { Self { file_size_mb: 10, timeout_secs: 30, - emit_events: true, + iterations: 1, } } } @@ -35,54 +37,83 @@ impl Default for SpeedTestConfig { #[derive(Debug, Clone)] pub struct SpeedTestResult { /// Write speed in MB/s - pub write_speed: f64, + pub write_speed_mbps: f64, /// Read speed in MB/s - pub read_speed: f64, - /// Time taken for the test in seconds - pub duration: f64, + pub read_speed_mbps: f64, + /// Total time taken for the test + pub duration_secs: f64, } -/// Trait for performing speed tests on volumes -#[async_trait::async_trait] -pub trait SpeedTest { - /// Performs a speed test on the volume - async fn speed_test( - &mut self, - config: Option, - event_tx: Option<&Sender>, - ) -> Result; +/// Run a speed test on the given volume +#[instrument(skip(volume), fields(volume_name = %volume.name))] +pub async fn run_speed_test(volume: &Volume) -> VolumeResult<(u64, u64)> { + run_speed_test_with_config(volume, SpeedTestConfig::default()).await } -/// Helper for managing temporary test files and directories +/// Run a speed test with custom configuration +#[instrument(skip(volume, config), fields(volume_name = %volume.name))] +pub async fn run_speed_test_with_config( + volume: &Volume, + config: SpeedTestConfig, +) -> VolumeResult<(u64, u64)> { + if !volume.is_mounted { + return Err(VolumeError::NotMounted(volume.name.clone())); + } + + if volume.read_only { + return Err(VolumeError::ReadOnly(volume.name.clone())); + } + + debug!("Starting speed test with config: {:?}", config); + + let test_location = TestLocation::new(&volume.mount_point, &volume.mount_type).await?; + let result = perform_speed_test(&test_location, &config).await?; + + // Cleanup + test_location.cleanup().await?; + + debug!( + "Speed test completed: {:.2} MB/s write, {:.2} MB/s read", + result.write_speed_mbps, result.read_speed_mbps + ); + + Ok(( + result.read_speed_mbps as u64, + result.write_speed_mbps as u64, + )) +} + +/// Helper for managing test files and directories struct TestLocation { - dir: PathBuf, - file_path: PathBuf, - created_dir: bool, + test_file: std::path::PathBuf, + created_dir: Option, } impl TestLocation { - #[instrument(skip(volume_path, mount_type))] - async fn new(volume_path: &PathBuf, mount_type: &MountType) -> Result { + /// Create a new test location + async fn new(volume_path: &std::path::Path, mount_type: &MountType) -> VolumeResult { let (dir, created_dir) = get_writable_directory(volume_path, mount_type).await?; - let file_path = dir.join("sd_speed_test_file.tmp"); + let test_file = dir.join("spacedrive_speed_test.tmp"); Ok(Self { - dir, - file_path, + test_file, created_dir, }) } - async fn cleanup(&self) -> Result<(), VolumeError> { - trace!("Cleaning up test file: {:?}", self.file_path); - if let Err(e) = tokio::fs::remove_file(&self.file_path).await { - error!("Failed to remove test file: {}", e); + /// Clean up test files and directories + async fn cleanup(&self) -> VolumeResult<()> { + // Remove test file + if self.test_file.exists() { + if let Err(e) = tokio::fs::remove_file(&self.test_file).await { + warn!("Failed to remove test file: {}", e); + } } - if self.created_dir { - trace!("Removing created directory: {:?}", self.dir); - if let Err(e) = tokio::fs::remove_dir_all(&self.dir).await { - error!("Failed to remove directory: {}", e); + // Remove created directory if we created it + if let Some(ref dir) = self.created_dir { + if let Err(e) = tokio::fs::remove_dir_all(dir).await { + warn!("Failed to remove test directory: {}", e); } } @@ -90,185 +121,249 @@ impl TestLocation { } } -#[async_trait::async_trait] -impl SpeedTest for Volume { - #[instrument(skip(self, config, event_tx), fields(volume_name = %self.name))] - async fn speed_test( - &mut self, - config: Option, - event_tx: Option<&Sender>, - ) -> Result { - let config = config.unwrap_or_default(); +/// Perform the actual speed test +async fn perform_speed_test( + location: &TestLocation, + config: &SpeedTestConfig, +) -> VolumeResult { + let test_data = generate_test_data(config.file_size_mb); + let timeout_duration = Duration::from_secs(config.timeout_secs); - // if volume is not mounted or not writable, return an error - if !self.is_mounted || self.read_only { - return Err(VolumeError::Cancelled); - } + let mut write_speeds = Vec::new(); + let mut read_speeds = Vec::new(); + let overall_start = Instant::now(); - debug!("Starting speed test with config: {:?}", config); + for iteration in 0..config.iterations { + debug!( + "Speed test iteration {}/{}", + iteration + 1, + config.iterations + ); - let test_location = TestLocation::new(&self.mount_point, &self.mount_type).await?; - let data = vec![0u8; config.file_size_mb * 1024 * 1024]; - let timeout_duration = Duration::from_secs(config.timeout_secs); + // Write test + let write_speed = timeout( + timeout_duration, + perform_write_test(&location.test_file, &test_data), + ) + .await + .map_err(|_| VolumeError::Timeout)??; - // Perform write test - let write_speed = - perform_write_test(&test_location.file_path, &data, timeout_duration).await?; + write_speeds.push(write_speed); - // Perform read test - let read_speed = - perform_read_test(&test_location.file_path, data.len(), timeout_duration).await?; + // Read test + let read_speed = timeout( + timeout_duration, + perform_read_test(&location.test_file, test_data.len()), + ) + .await + .map_err(|_| VolumeError::Timeout)??; - let result = SpeedTestResult { - write_speed, - read_speed, - duration: timeout_duration.as_secs_f64(), - }; + read_speeds.push(read_speed); - // Update volume speeds - self.read_speed_mbps = Some(read_speed as u64); - self.write_speed_mbps = Some(write_speed as u64); - - // Emit event if requested - // if config.emit_events { - println!("emitting event for {:?}", self.fingerprint); - if let Some(fingerprint) = self.fingerprint.clone() { - if let Some(tx) = event_tx { - let _ = tx.send(VolumeEvent::VolumeSpeedTested { - fingerprint, - read_speed: read_speed as u64, - write_speed: write_speed as u64, - }); - } - } - // } - - // Cleanup - test_location.cleanup().await?; - - debug!("Speed test completed: {:?}", result); - Ok(result) - } -} - -/// Helper function to get a writable directory within a volume -#[instrument(skip(volume_path, mount_type))] -async fn get_writable_directory( - volume_path: &PathBuf, - mount_type: &MountType, -) -> Result<(PathBuf, bool), VolumeError> { - match mount_type { - MountType::System => { - trace!("Using system temp directory for system volume"); - Ok((std::env::temp_dir(), false)) - } - _ => { - let candidates = [ - volume_path.join("tmp"), - volume_path.join("var").join("tmp"), - volume_path.clone(), - ]; - - for dir in &candidates { - trace!("Checking directory: {:?}", dir); - - if let Ok(metadata) = tokio::fs::metadata(dir).await { - if metadata.is_dir() { - return Ok((dir.clone(), false)); - } - } - - trace!("Attempting to create directory: {:?}", dir); - if tokio::fs::create_dir_all(dir).await.is_ok() { - return Ok((dir.clone(), true)); - } - } - - Err(VolumeError::DirectoryError( - "No writable directory found".to_string(), - )) + // Clean up test file between iterations + if iteration < config.iterations - 1 { + let _ = tokio::fs::remove_file(&location.test_file).await; } } + + let avg_write_speed = write_speeds.iter().sum::() / write_speeds.len() as f64; + let avg_read_speed = read_speeds.iter().sum::() / read_speeds.len() as f64; + + Ok(SpeedTestResult { + write_speed_mbps: avg_write_speed, + read_speed_mbps: avg_read_speed, + duration_secs: overall_start.elapsed().as_secs_f64(), + }) } -/// Performs the write speed test -#[instrument(skip(path, data))] -async fn perform_write_test( - path: &PathBuf, - data: &[u8], - timeout_duration: Duration, -) -> Result { - trace!("Starting write test"); +/// Generate test data for speed testing +fn generate_test_data(size_mb: usize) -> Vec { + let size_bytes = size_mb * 1024 * 1024; + + // Use a pattern instead of zeros to avoid compression optimizations + let pattern = b"SpacedriveSpeedTest0123456789ABCDEF"; + let mut data = Vec::with_capacity(size_bytes); + + for i in 0..size_bytes { + data.push(pattern[i % pattern.len()]); + } + + data +} + +/// Perform write speed test +async fn perform_write_test(file_path: &std::path::Path, data: &[u8]) -> VolumeResult { let start = Instant::now(); let mut file = OpenOptions::new() .write(true) .create(true) - .open(path) + .truncate(true) + .open(file_path) .await?; - timeout(timeout_duration, file.write_all(data)).await??; + file.write_all(data).await?; + file.sync_all().await?; // Ensure data is written to disk let duration = start.elapsed(); - let speed = (data.len() as f64 / 1024.0 / 1024.0) / duration.as_secs_f64(); + let speed_mbps = (data.len() as f64 / 1024.0 / 1024.0) / duration.as_secs_f64(); - trace!("Write test completed: {} MB/s", speed); - Ok(speed) + Ok(speed_mbps) } -/// Performs the read speed test -#[instrument(skip(path))] -async fn perform_read_test( - path: &PathBuf, - size: usize, - timeout_duration: Duration, -) -> Result { - trace!("Starting read test"); +/// Perform read speed test +async fn perform_read_test(file_path: &std::path::Path, expected_size: usize) -> VolumeResult { let start = Instant::now(); - let mut file = File::open(path).await?; - let mut buffer = vec![0u8; size]; - - timeout(timeout_duration, file.read_exact(&mut buffer)).await??; + let mut file = File::open(file_path).await?; + let mut buffer = Vec::with_capacity(expected_size); + file.read_to_end(&mut buffer).await?; let duration = start.elapsed(); - let speed = (size as f64 / 1024.0 / 1024.0) / duration.as_secs_f64(); + let speed_mbps = (buffer.len() as f64 / 1024.0 / 1024.0) / duration.as_secs_f64(); - trace!("Read test completed: {} MB/s", speed); - Ok(speed) + Ok(speed_mbps) +} + +/// Get a writable directory within the volume +async fn get_writable_directory( + volume_path: &std::path::Path, + mount_type: &MountType, +) -> VolumeResult<(std::path::PathBuf, Option)> { + match mount_type { + MountType::System => { + // For system volumes, prefer using temp directory + let temp_dir = std::env::temp_dir(); + Ok((temp_dir, None)) + } + _ => { + // For external volumes, try to write in the root or create a temp directory + let candidates = [ + volume_path.join("tmp"), + volume_path.join(".spacedrive_temp"), + volume_path.to_path_buf(), + ]; + + for candidate in &candidates { + // Try to create the directory + if let Ok(()) = tokio::fs::create_dir_all(candidate).await { + // Test if we can write to it + let test_file = candidate.join("test_write_permissions"); + if tokio::fs::write(&test_file, b"test").await.is_ok() { + let _ = tokio::fs::remove_file(&test_file).await; + + // If we created a directory specifically for this test, mark it for cleanup + let created_dir = if candidate + .file_name() + .map_or(false, |name| name == "tmp" || name == ".spacedrive_temp") + { + Some(candidate.clone()) + } else { + None + }; + + return Ok((candidate.clone(), created_dir)); + } + } + } + + Err(VolumeError::PermissionDenied(format!( + "No writable directory found in volume: {}", + volume_path.display() + ))) + } + } } #[cfg(test)] mod tests { use super::*; - use tempfile::tempdir; + use crate::volume::{ + types::{DiskType, FileSystem}, + VolumeFingerprint, + }; + use tempfile::TempDir; #[tokio::test] - async fn test_speed_test() { - let temp_dir = tempdir().unwrap(); - let mut volume = Volume::new( - "test".to_string(), + async fn test_speed_test_config() { + let config = SpeedTestConfig::default(); + assert_eq!(config.file_size_mb, 10); + assert_eq!(config.timeout_secs, 30); + assert_eq!(config.iterations, 1); + } + + #[tokio::test] + async fn test_generate_test_data() { + let data = generate_test_data(1); // 1MB + assert_eq!(data.len(), 1024 * 1024); + + // Verify pattern is not all zeros + assert!(data.iter().any(|&b| b != 0)); + } + + #[tokio::test] + async fn test_writable_directory_external() { + let temp_dir = TempDir::new().unwrap(); + let volume_path = temp_dir.path(); + + let (writable_dir, created_dir) = get_writable_directory(volume_path, &MountType::External) + .await + .unwrap(); + + assert!(writable_dir.exists()); + + // Cleanup if we created a directory + if let Some(dir) = created_dir { + let _ = tokio::fs::remove_dir_all(dir).await; + } + } + + #[tokio::test] + async fn test_writable_directory_system() { + let (writable_dir, created_dir) = + get_writable_directory(&std::path::PathBuf::from("/"), &MountType::System) + .await + .unwrap(); + + assert!(writable_dir.exists()); + assert!(created_dir.is_none()); // Should use system temp, not create new dir + } + + #[tokio::test] + async fn test_full_speed_test() { + let temp_dir = TempDir::new().unwrap(); + + let volume = Volume::new( + uuid::Uuid::new_v4(), // Test device ID + "Test Volume".to_string(), MountType::External, + VolumeType::External, temp_dir.path().to_path_buf(), vec![], - super::super::types::DiskType::Unknown, - super::super::types::FileSystem::Other("test".to_string()), - 1000000, - 1000000, - false, + DiskType::Unknown, + FileSystem::Other("test".to_string()), + 1000000000, // 1GB capacity + 500000000, // 500MB available + false, // Not read-only + None, + VolumeFingerprint::new( + "Test Volume", + 1000000000, + "test", + ), ); let config = SpeedTestConfig { - file_size_mb: 1, - timeout_secs: 5, - emit_events: false, + file_size_mb: 1, // Small test file + timeout_secs: 10, + iterations: 1, }; - let result = volume.speed_test(Some(config), None).await.unwrap(); + let result = run_speed_test_with_config(&volume, config).await; + assert!(result.is_ok()); - assert!(result.read_speed > 0.0); - assert!(result.write_speed > 0.0); - assert_eq!(volume.read_speed_mbps, Some(result.read_speed as u64)); - assert_eq!(volume.write_speed_mbps, Some(result.write_speed as u64)); + let (read_speed, write_speed) = result.unwrap(); + assert!(read_speed > 0); + assert!(write_speed > 0); } } diff --git a/core/src/volume/state.rs b/core/src/volume/state.rs deleted file mode 100644 index 25d7a4c89..000000000 --- a/core/src/volume/state.rs +++ /dev/null @@ -1,237 +0,0 @@ -use crate::{ - library::Library, - volume::{ - speed::SpeedTest, - types::{Volume, VolumeEvent, VolumeFingerprint}, - }, -}; - -use sd_core_sync::DevicePubId; -use std::collections::HashSet; -use std::{collections::HashMap, sync::Arc, time::Instant}; -use tokio::sync::{broadcast, RwLock}; -use tracing::{debug, error}; - -use super::{MountType, VolumeError, VolumeOptions}; -// Core volume registry -pub struct VolumeRegistry { - volumes: HashMap, - device_id: DevicePubId, -} - -impl VolumeRegistry { - pub fn new(device_id: DevicePubId) -> Self { - Self { - volumes: HashMap::new(), - device_id, - } - } - - pub fn register_volume(&mut self, mut volume: Volume) -> (Volume, VolumeFingerprint) { - let fingerprint = VolumeFingerprint::new(&self.device_id, &volume); - debug!( - "Registering volume {} with fingerprint {}", - volume.name, fingerprint - ); - volume.fingerprint = Some(fingerprint.clone()); - self.volumes.insert(fingerprint.clone(), volume.clone()); - (volume, fingerprint) - } - - pub fn get_volume(&self, id: &VolumeFingerprint) -> Option<&Volume> { - self.volumes.get(id) - } - - pub fn volumes(&self) -> impl Iterator { - self.volumes.iter() - } - - pub fn remove_volume(&mut self, id: &VolumeFingerprint) -> Option { - self.volumes.remove(id) - } - - pub fn update_volume(&mut self, volume: Volume) -> VolumeFingerprint { - let fingerprint = VolumeFingerprint::new(&self.device_id, &volume); - self.volumes.insert(fingerprint.clone(), volume); - fingerprint - } - - pub fn get_volume_mut(&mut self, id: &VolumeFingerprint) -> Option<&mut Volume> { - self.volumes.get_mut(id) - } -} - -// Main state manager -pub struct VolumeManagerState { - pub registry: Arc>, - options: VolumeOptions, - event_tx: broadcast::Sender, - last_scan: Instant, -} - -impl VolumeManagerState { - pub fn new( - device_id: DevicePubId, - options: VolumeOptions, - event_tx: broadcast::Sender, - ) -> Self { - Self { - registry: Arc::new(RwLock::new(VolumeRegistry::new(device_id))), - options, - event_tx, - last_scan: Instant::now(), - } - } - - pub async fn scan_volumes(&mut self) -> Result<(), VolumeError> { - #[cfg(any(target_os = "linux", target_os = "macos"))] - let detected_volumes = super::os::get_volumes().await?; - - #[cfg(target_os = "windows")] - let detected_volumes = super::os::get_volumes().await; - - let mut registry = self.registry.write().await; - - // Track existing volumes for removal detection - let existing: HashSet<_> = registry.volumes().map(|(id, _)| id.clone()).collect(); - let mut seen = HashSet::new(); - - // Process detected volumes - for volume in detected_volumes { - let (volume, fingerprint) = registry.register_volume(volume.clone()); - seen.insert(fingerprint.clone()); - - // Emit event for new volumes - if !existing.contains(&fingerprint) { - let event_tx = self.event_tx.clone(); - let _ = event_tx.send(VolumeEvent::VolumeAdded(volume.clone())); - - let mut volume_clone = volume.clone(); - let event_tx = self.event_tx.clone(); - drop(registry); - - // Spawn a background task to perform the speed test only for system volumes - if volume.mount_type == MountType::System { - tokio::spawn(async move { - if let Err(e) = volume_clone.speed_test(None, Some(&event_tx)).await { - error!(?e, "Failed to perform speed test for volume"); - } - }); - } - - registry = self.registry.write().await; - } - } - - // Find and remove volumes that no longer exist - for fingerprint in existing.difference(&seen) { - if let Some(volume) = registry.remove_volume(fingerprint) { - let _ = self.event_tx.send(VolumeEvent::VolumeRemoved(volume)); - } - } - - self.last_scan = Instant::now(); - Ok(()) - } - - // pub async fn register_with_library( - // &self, - // library_id: LibraryId, - // volume: &Volume, - // library: Arc, - // ) -> Result<(), VolumeError> { - // let device_id = self.registry.read().await.device_id.clone(); - // let fingerprint = VolumeFingerprint::new(&device_id, volume); - - // // Create in database - // volume.create(&library.db, device_id.to_db()).await?; - - // // Track the relationship - // self.library_registry.write().await.track_volume( - // library_id, - // fingerprint, - // VolumePubId::from(volume.pub_id.clone().unwrap()), - // ); - - // Ok(()) - // } - - pub async fn get_volume(&self, fingerprint: &VolumeFingerprint) -> Option { - self.registry.read().await.get_volume(fingerprint).cloned() - } - - pub async fn list_volumes(&self) -> Vec { - self.registry - .read() - .await - .volumes() - .map(|(_, v)| v.clone()) - .collect() - } - - pub async fn get_volumes_for_library( - &self, - library: Arc, - ) -> Result, VolumeError> { - let registry = self.registry.read().await; - - let mut volumes = Vec::new(); - - let device_id = &self.registry.read().await.device_id; - - let db_volumes = library - .db - .volume() - .find_many(vec![]) - .exec() - .await? - .into_iter() - .map(Volume::from) - .map(|v| { - // TODO: maybe just store the fingerprint at this point? - let fingerprint = VolumeFingerprint::new(device_id, &v); - (fingerprint, v) - }) - .collect::>(); - - for (fingerprint, volume) in registry.volumes() { - let mut volume = volume.clone(); - - if let Some(db_volume) = db_volumes.get(fingerprint) { - volume = Volume::merge_with_db(&volume, db_volume); - } - - volumes.push(volume); - } - - Ok(volumes) - } - - pub async fn volume_exists(&self, fingerprint: &VolumeFingerprint) -> bool { - self.registry.read().await.get_volume(fingerprint).is_some() - } - - pub async fn update_mount_status( - &self, - fingerprint: &VolumeFingerprint, - is_mounted: bool, - ) -> Result<(), VolumeError> { - let volume = self - .get_volume(fingerprint) - .await - .ok_or_else(|| VolumeError::NotFound(fingerprint.clone()))?; - - let _ = self.event_tx.send(VolumeEvent::VolumeMountChanged { - fingerprint: fingerprint.clone(), - is_mounted, - }); - Ok(()) - } - // pub async fn get_statistics(&self) -> VolumeStats { - // VolumeStats { - // total_volumes: self.registry.read().await.volumes.len(), - // tracked_libraries: self.library_registry.read().await.mappings.len(), - // last_scan_age: self.last_scan.elapsed(), - // } - // } -} diff --git a/core/src/volume/types.rs b/core/src/volume/types.rs index 91d4cb743..f4b26fbbe 100644 --- a/core/src/volume/types.rs +++ b/core/src/volume/types.rs @@ -1,78 +1,199 @@ -use super::error::VolumeError; -use crate::volume::speed::SpeedTest; -use sd_core_sync::DevicePubId; -use sd_prisma::prisma::{ - device, - volume::{self}, - PrismaClient, -}; +//! Volume type definitions + use serde::{Deserialize, Serialize}; -use serde_with::{serde_as, DisplayFromStr}; -use specta::Type; use std::fmt; use std::path::PathBuf; -use std::{path::Path, sync::Arc}; -use strum_macros::Display; use uuid::Uuid; -/// A fingerprint of a volume, used to identify it when it is not persisted in the database -#[derive(Debug, Clone, Hash, Eq, PartialEq, Type)] -pub struct VolumeFingerprint(pub Vec); +/// Spacedrive volume identifier file content +/// This file is created in the root of writable volumes for persistent identification +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct SpacedriveVolumeId { + /// Unique identifier for this volume + pub id: Uuid, + /// When this identifier was created + pub created: chrono::DateTime, + /// Name of the device that created this identifier + pub device_name: Option, + /// Original volume name when identifier was created + pub volume_name: String, + /// Device ID that created this identifier + pub device_id: Uuid, +} + +/// Unique fingerprint for a storage volume +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, Hash)] +pub struct VolumeFingerprint(pub String); impl VolumeFingerprint { - pub fn new(device_id: &DevicePubId, volume: &Volume) -> Self { - // Hash the device ID, mount point, name, total bytes capacity, and file system + /// Create a new volume fingerprint from volume properties + /// Uses intrinsic volume characteristics for cross-device portable identification + pub fn new(name: &str, total_bytes: u64, file_system: &str) -> Self { let mut hasher = blake3::Hasher::new(); - hasher.update(&device_id.to_db()); - hasher.update(volume.mount_point.to_string_lossy().as_bytes()); - hasher.update(volume.name.as_bytes()); - hasher.update(&volume.total_bytes_capacity.to_be_bytes()); - hasher.update(volume.file_system.to_string().as_bytes()); - // These are all properties that are unique to a volume and unlikely to change - // If a .spacedrive file is found in the volume, and is fingerprint does not match, - // but the `pub_id` is the same, we can update the values and regenerate the fingerprint - // preserving the tracked instance of the volume - Self(hasher.finalize().as_bytes().to_vec()) + hasher.update(b"content_based:"); + hasher.update(name.as_bytes()); + hasher.update(&total_bytes.to_be_bytes()); + hasher.update(file_system.as_bytes()); + hasher.update(&(name.len() as u64).to_be_bytes()); + + Self(hasher.finalize().to_hex().to_string()) + } + + /// Create a fingerprint from a Spacedrive identifier UUID (preferred method) + /// This provides stable identification across devices, renames and remounts + pub fn from_spacedrive_id(spacedrive_id: Uuid) -> Self { + let mut hasher = blake3::Hasher::new(); + hasher.update(b"spacedrive_id:"); + hasher.update(spacedrive_id.as_bytes()); + + Self(hasher.finalize().to_hex().to_string()) + } + + /// Generate 8-character short ID for CLI display and commands + pub fn short_id(&self) -> String { + self.0.chars().take(8).collect() + } + + /// Generate 12-character medium ID for disambiguation + pub fn medium_id(&self) -> String { + self.0.chars().take(12).collect() + } + + /// Create fingerprint from hex string + pub fn from_hex(hex: impl Into) -> Self { + Self(hex.into()) + } + + /// Create fingerprint from string (alias for from_hex) + pub fn from_string(s: &str) -> Result { + Ok(Self(s.to_string())) + } + + /// Check if a string could be a short ID (8 chars, hex) + pub fn is_short_id(s: &str) -> bool { + s.len() == 8 && s.chars().all(|c| c.is_ascii_hexdigit()) + } + + /// Check if a string could be a medium ID (12 chars, hex) + pub fn is_medium_id(s: &str) -> bool { + s.len() == 12 && s.chars().all(|c| c.is_ascii_hexdigit()) + } + + /// Check if this fingerprint matches a short or medium ID + pub fn matches_short_id(&self, short_id: &str) -> bool { + if Self::is_short_id(short_id) { + self.short_id() == short_id + } else if Self::is_medium_id(short_id) { + self.medium_id() == short_id + } else { + false + } } } impl fmt::Display for VolumeFingerprint { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{}", hex::encode(&self.0)) + write!(f, "{}", self.0) } } -#[derive(Debug, Clone, Hash, Eq, PartialEq, Serialize, Deserialize)] -pub struct VolumePubId(pub Vec); +/// Classification of volume types for UX and auto-tracking decisions +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub enum VolumeType { + /// Primary system drive containing OS and user data + /// Examples: C:\ on Windows, / on Linux, Macintosh HD on macOS + Primary, -impl From> for VolumePubId { - fn from(v: Vec) -> Self { - Self(v) - } + /// Dedicated user data volumes (separate from OS) + /// Examples: /System/Volumes/Data on macOS, separate /home on Linux + UserData, + + /// External or removable storage devices + /// Examples: USB drives, external HDDs, /Volumes/* on macOS + External, + + /// Secondary internal storage (additional drives/partitions) + /// Examples: D:, E: drives on Windows, additional mounted drives + Secondary, + + /// System/OS internal volumes (hidden from normal view) + /// Examples: /System/Volumes/* on macOS, Recovery partitions + System, + + /// Network attached storage + /// Examples: SMB mounts, NFS, cloud storage + Network, + + /// Unknown or unclassified volumes + Unknown, } -impl Into> for VolumePubId { - fn into(self) -> Vec { - self.0 +impl VolumeType { + /// Should this volume type be auto-tracked by default? + pub fn auto_track_by_default(&self) -> bool { + match self { + // Only auto-track the primary system volume + // Users should explicitly choose to track other volumes + VolumeType::Primary => true, + VolumeType::UserData + | VolumeType::External + | VolumeType::Secondary + | VolumeType::Network + | VolumeType::System + | VolumeType::Unknown => false, + } + } + + /// Should this volume be shown in the default UI view? + pub fn show_by_default(&self) -> bool { + !matches!(self, VolumeType::System | VolumeType::Unknown) + } + + /// User-friendly display name for the volume type + pub fn display_name(&self) -> &'static str { + match self { + VolumeType::Primary => "Primary Drive", + VolumeType::UserData => "User Data", + VolumeType::External => "External Drive", + VolumeType::Secondary => "Secondary Drive", + VolumeType::System => "System Volume", + VolumeType::Network => "Network Drive", + VolumeType::Unknown => "Unknown", + } + } + + /// Icon/indicator for CLI display + pub fn icon(&self) -> &'static str { + match self { + VolumeType::Primary => "[PRI]", + VolumeType::UserData => "[USR]", + VolumeType::External => "[EXT]", + VolumeType::Secondary => "[SEC]", + VolumeType::System => "[SYS]", + VolumeType::Network => "[NET]", + VolumeType::Unknown => "[UNK]", + } } } -pub type LibraryId = Uuid; - /// Events emitted by the Volume Manager when volume state changes -#[derive(Debug, Clone, Type, Deserialize, Serialize)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub enum VolumeEvent { - /// Emitted when a new volume is discovered and added + /// Emitted when a new volume is discovered VolumeAdded(Volume), - /// Emitted when a volume is removed from the system - VolumeRemoved(Volume), + /// Emitted when a volume is removed/unmounted + VolumeRemoved { fingerprint: VolumeFingerprint }, /// Emitted when a volume's properties are updated - VolumeUpdated { old: Volume, new: Volume }, + VolumeUpdated { + fingerprint: VolumeFingerprint, + old: VolumeInfo, + new: VolumeInfo, + }, /// Emitted when a volume's speed test completes VolumeSpeedTested { fingerprint: VolumeFingerprint, - read_speed: u64, - write_speed: u64, + read_speed_mbps: u64, + write_speed_mbps: u64, }, /// Emitted when a volume's mount status changes VolumeMountChanged { @@ -87,272 +208,257 @@ pub enum VolumeEvent { } /// Represents a physical or virtual storage volume in the system -#[serde_as] -#[derive(Serialize, Deserialize, Debug, Clone, Type)] +#[derive(Serialize, Deserialize, Debug, Clone)] pub struct Volume { - /// Fingerprint of the volume as a hash of its properties, not persisted to the database - /// Used as the unique identifier for a volume in this module - pub fingerprint: Option, - /// Database ID (None if not yet committed to database) - pub id: Option, - /// Unique public identifier - pub pub_id: Option>, - /// Database ID of the device this volume is attached to, if any - pub device_id: Option, + /// Unique fingerprint for this volume + pub fingerprint: VolumeFingerprint, + + /// Device this volume belongs to + pub device_id: uuid::Uuid, /// Human-readable volume name pub name: String, /// Type of mount (system, external, etc) pub mount_type: MountType, - /// Path where the volume is mounted - #[specta(type = Vec)] + /// Classification of this volume for UX decisions + pub volume_type: VolumeType, + /// Primary path where the volume is mounted pub mount_point: PathBuf, - /// for APFS volumes like Macintosh HD, additional mount points are returned - #[specta(type = Vec)] + /// Additional mount points (for APFS volumes, etc.) pub mount_points: Vec, /// Whether the volume is currently mounted pub is_mounted: bool, + /// Type of storage device (SSD, HDD, etc) pub disk_type: DiskType, /// Filesystem type (NTFS, EXT4, etc) pub file_system: FileSystem, /// Whether the volume is mounted read-only pub read_only: bool, + + /// Hardware identifier (platform-specific) + pub hardware_id: Option, /// Current error status if any pub error_status: Option, - // Performance metrics + // Storage information + /// Total storage capacity in bytes + pub total_bytes_capacity: u64, + /// Available storage space in bytes + pub total_bytes_available: u64, + + // Performance metrics (populated by speed tests) /// Read speed in megabytes per second pub read_speed_mbps: Option, /// Write speed in megabytes per second pub write_speed_mbps: Option, - /// Total storage capacity in bytes - #[specta(type = String)] - #[serde_as(as = "DisplayFromStr")] - pub total_bytes_capacity: u64, - /// Available storage space in bytes - #[specta(type = String)] - #[serde_as(as = "DisplayFromStr")] - pub total_bytes_available: u64, + + /// Whether this volume should be visible in default views + pub is_user_visible: bool, + + /// Whether this volume should be auto-tracked + pub auto_track_eligible: bool, + + /// When this volume information was last updated + pub last_updated: chrono::DateTime, } -// We can use this to see if a volume has changed -impl PartialEq for Volume { - fn eq(&self, other: &Self) -> bool { - self.name == other.name - && self.disk_type == other.disk_type - && self.file_system == other.file_system - && self.mount_type == other.mount_type - && self.mount_point == other.mount_point - // Check if any mount points overlap - && (self.mount_points.iter().any(|mp| other.mount_points.contains(mp)) - || other.mount_points.iter().any(|mp| self.mount_points.contains(mp))) - && self.is_mounted == other.is_mounted - && self.read_only == other.read_only - && self.error_status == other.error_status - && self.total_bytes_capacity == other.total_bytes_capacity - && self.total_bytes_available == other.total_bytes_available +/// Summary information about a volume (for updates and caching) +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct VolumeInfo { + pub is_mounted: bool, + pub total_bytes_available: u64, + pub read_speed_mbps: Option, + pub write_speed_mbps: Option, + pub error_status: Option, +} + +/// Information about a tracked volume in the database +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct TrackedVolume { + pub id: i32, + pub uuid: uuid::Uuid, + pub device_id: uuid::Uuid, + pub fingerprint: VolumeFingerprint, + pub display_name: Option, + pub tracked_at: chrono::DateTime, + pub last_seen_at: chrono::DateTime, + pub is_online: bool, + pub total_capacity: Option, + pub available_capacity: Option, + pub read_speed_mbps: Option, + pub write_speed_mbps: Option, + pub last_speed_test_at: Option>, + pub file_system: Option, + pub mount_point: Option, + pub is_removable: Option, + pub is_network_drive: Option, + pub device_model: Option, + pub volume_type: String, + pub is_user_visible: Option, + pub auto_track_eligible: Option, +} + +impl From<&Volume> for VolumeInfo { + fn from(volume: &Volume) -> Self { + Self { + is_mounted: volume.is_mounted, + total_bytes_available: volume.total_bytes_available, + read_speed_mbps: volume.read_speed_mbps, + write_speed_mbps: volume.write_speed_mbps, + error_status: volume.error_status.clone(), + } } } -impl Eq for Volume {} +impl TrackedVolume { + /// Convert a TrackedVolume back to a Volume for display purposes + /// This is used for offline volumes that aren't currently detected + pub fn to_offline_volume(&self) -> Volume { + use std::path::PathBuf; -impl From for Volume { - fn from(vol: volume::Data) -> Self { Volume { - id: Some(vol.id), - pub_id: Some(vol.pub_id), - device_id: vol.device_id, - name: vol.name.unwrap_or_else(|| "Unknown".to_string()), - mount_type: vol - .mount_type - .as_deref() - .map(MountType::from_string) - .unwrap_or(MountType::System), - mount_point: PathBuf::from(vol.mount_point.unwrap_or_else(|| "/".to_string())), - mount_points: Vec::new(), - is_mounted: vol.is_mounted.unwrap_or(false), - disk_type: vol - .disk_type - .as_deref() - .map(DiskType::from_string) - .unwrap_or(DiskType::Unknown), - file_system: vol - .file_system - .as_deref() - .map(FileSystem::from_string) - .unwrap_or_else(|| FileSystem::Other("Unknown".to_string())), - read_only: vol.read_only.unwrap_or(false), - error_status: vol.error_status, - total_bytes_capacity: vol - .total_bytes_capacity - .and_then(|t| t.parse().ok()) - .unwrap_or(0), - total_bytes_available: vol - .total_bytes_available - .and_then(|a| a.parse().ok()) - .unwrap_or(0), - read_speed_mbps: vol.read_speed_mbps.map(|s| s as u64), - write_speed_mbps: vol.write_speed_mbps.map(|s| s as u64), - fingerprint: None, + fingerprint: self.fingerprint.clone(), + device_id: self.device_id, + name: self + .display_name + .clone() + .unwrap_or_else(|| "Unknown".to_string()), + mount_type: crate::volume::types::MountType::External, // Default for tracked volumes + volume_type: match self.volume_type.as_str() { + "Primary" => VolumeType::Primary, + "UserData" => VolumeType::UserData, + "External" => VolumeType::External, + "Secondary" => VolumeType::Secondary, + "System" => VolumeType::System, + "Network" => VolumeType::Network, + _ => VolumeType::Unknown, + }, + mount_point: PathBuf::from( + self.mount_point + .clone() + .unwrap_or_else(|| "Not connected".to_string()), + ), + mount_points: vec![], // Not available for offline volumes + disk_type: crate::volume::types::DiskType::Unknown, + file_system: crate::volume::types::FileSystem::from_string( + &self + .file_system + .clone() + .unwrap_or_else(|| "Unknown".to_string()), + ), + total_bytes_capacity: self.total_capacity.unwrap_or(0), + total_bytes_available: self.available_capacity.unwrap_or(0), + read_only: false, // Assume not read-only for tracked volumes + hardware_id: self.device_model.clone(), + is_mounted: false, // Offline volumes are not mounted + error_status: None, + read_speed_mbps: self.read_speed_mbps.map(|s| s as u64), + write_speed_mbps: self.write_speed_mbps.map(|s| s as u64), + last_updated: self.last_seen_at, + is_user_visible: self.is_user_visible.unwrap_or(true), + auto_track_eligible: self.auto_track_eligible.unwrap_or(false), } } } impl Volume { - /// Creates a new Volume instance from detected system volume information + /// Create a new Volume instance pub fn new( + device_id: uuid::Uuid, name: String, mount_type: MountType, + volume_type: VolumeType, mount_point: PathBuf, - mount_points: Vec, + additional_mount_points: Vec, disk_type: DiskType, file_system: FileSystem, total_bytes_capacity: u64, total_bytes_available: u64, read_only: bool, + hardware_id: Option, + fingerprint: VolumeFingerprint, // Accept pre-computed fingerprint ) -> Self { Self { - id: None, - pub_id: None, - device_id: None, + fingerprint, + device_id, name, mount_type, + volume_type, mount_point, - mount_points, + mount_points: additional_mount_points, is_mounted: true, disk_type, file_system, + total_bytes_capacity, + total_bytes_available, read_only, + hardware_id, error_status: None, read_speed_mbps: None, write_speed_mbps: None, - total_bytes_capacity, - total_bytes_available, - fingerprint: None, + auto_track_eligible: volume_type.auto_track_by_default(), + is_user_visible: volume_type.show_by_default(), + last_updated: chrono::Utc::now(), } } - /// Check if a path is under any of this volume's mount points - pub fn contains_path(&self, path: &Path) -> bool { - self.mount_points.iter().any(|mp| path.starts_with(mp)) + /// Update volume information + pub fn update_info(&mut self, info: VolumeInfo) { + self.is_mounted = info.is_mounted; + self.total_bytes_available = info.total_bytes_available; + self.read_speed_mbps = info.read_speed_mbps; + self.write_speed_mbps = info.write_speed_mbps; + self.error_status = info.error_status; + self.last_updated = chrono::Utc::now(); } - /// Merge system detected volume with database volume, preferring system values for hardware info - pub fn merge_with_db(system_volume: &Volume, db_volume: &Volume) -> Volume { - Volume { - // Keep system-detected hardware properties - mount_point: system_volume.mount_point.clone(), - mount_points: system_volume.mount_points.clone(), - total_bytes_capacity: system_volume.total_bytes_capacity, - total_bytes_available: system_volume.total_bytes_available, - disk_type: system_volume.disk_type.clone(), - file_system: system_volume.file_system.clone(), - mount_type: system_volume.mount_type.clone(), - is_mounted: system_volume.is_mounted, - fingerprint: system_volume.fingerprint.clone(), - name: system_volume.name.clone(), - read_only: system_volume.read_only, - error_status: system_volume.error_status.clone(), - read_speed_mbps: system_volume.read_speed_mbps, - write_speed_mbps: system_volume.write_speed_mbps, + /// Check if this volume supports fast copy operations (CoW) + pub fn supports_fast_copy(&self) -> bool { + matches!( + self.file_system, + FileSystem::APFS | FileSystem::Btrfs | FileSystem::ZFS | FileSystem::ReFS + ) + } - // Keep database-tracked properties and metadata - id: db_volume.id, - device_id: db_volume.device_id, - pub_id: db_volume.pub_id.clone(), + /// Get the optimal chunk size for copying to/from this volume + pub fn optimal_chunk_size(&self) -> usize { + match self.disk_type { + DiskType::SSD => 1024 * 1024, // 1MB for SSDs + DiskType::HDD => 256 * 1024, // 256KB for HDDs + DiskType::Unknown => 64 * 1024, // 64KB default } } - pub fn is_volume_tracked(&self) -> bool { - self.pub_id.is_some() + /// Estimate copy speed between this and another volume + pub fn estimate_copy_speed(&self, other: &Volume) -> Option { + let self_read = self.read_speed_mbps?; + let other_write = other.write_speed_mbps?; + + // Bottleneck is the slower of read or write speed + Some(self_read.min(other_write)) } - /// Creates a new volume record in the database - pub async fn create( - &self, - db: &Arc, - device_pub_id: Vec, - ) -> Result { - let pub_id = Uuid::now_v7().as_bytes().to_vec(); + /// Check if a path is contained within this volume + pub fn contains_path(&self, path: &PathBuf) -> bool { + // Check primary mount point + if path.starts_with(&self.mount_point) { + return true; + } - let device_id = db - .device() - .find_unique(device::pub_id::equals(device_pub_id.clone())) - .select(device::select!({ id })) - .exec() - .await? - .ok_or(VolumeError::DeviceNotFound(device_pub_id))? - .id; + // Check additional mount points + for mount_point in &self.mount_points { + if path.starts_with(mount_point) { + return true; + } + } - let volume = db - .volume() - .create( - pub_id, - vec![ - volume::name::set(Some(self.name.clone())), - volume::mount_type::set(Some(self.mount_type.to_string())), - volume::mount_point::set(Some(self.mount_point.to_str().unwrap().to_string())), - volume::is_mounted::set(Some(self.is_mounted)), - volume::disk_type::set(Some(self.disk_type.to_string())), - volume::file_system::set(Some(self.file_system.to_string())), - volume::read_only::set(Some(self.read_only)), - volume::error_status::set(self.error_status.clone()), - volume::total_bytes_capacity::set(Some(self.total_bytes_capacity.to_string())), - volume::total_bytes_available::set(Some( - self.total_bytes_available.to_string(), - )), - volume::read_speed_mbps::set( - self.read_speed_mbps.filter(|&v| v != 0).map(|v| v as i64), - ), - volume::write_speed_mbps::set( - self.write_speed_mbps.filter(|&v| v != 0).map(|v| v as i64), - ), - volume::device_id::set(Some(device_id)), - ], - ) - .exec() - .await?; - Ok(volume.into()) - } - - /// Updates an existing volume record in the database - pub async fn update(&self, db: &PrismaClient) -> Result<(), VolumeError> { - let id = self.id.ok_or(VolumeError::NotInDatabase)?; - - db.volume() - .update( - volume::id::equals(id), - vec![ - volume::name::set(Some(self.name.clone())), - volume::mount_type::set(Some(self.mount_type.to_string())), - volume::mount_point::set(Some(self.mount_point.to_str().unwrap().to_string())), - volume::is_mounted::set(Some(self.is_mounted)), - volume::disk_type::set(Some(self.disk_type.to_string())), - volume::file_system::set(Some(self.file_system.to_string())), - volume::read_only::set(Some(self.read_only)), - volume::error_status::set(self.error_status.clone()), - volume::total_bytes_capacity::set(Some(self.total_bytes_capacity.to_string())), - volume::total_bytes_available::set(Some( - self.total_bytes_available.to_string(), - )), - volume::read_speed_mbps::set( - self.read_speed_mbps.filter(|&v| v != 0).map(|v| v as i64), - ), - volume::write_speed_mbps::set( - self.write_speed_mbps.filter(|&v| v != 0).map(|v| v as i64), - ), - ], - ) - .exec() - .await?; - Ok(()) + false } } /// Represents the type of physical storage device -#[derive(Serialize, Deserialize, Debug, Clone, Type, Hash, PartialEq, Eq, Display)] -#[allow(clippy::upper_case_acronyms)] +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, Hash)] pub enum DiskType { /// Solid State Drive SSD, @@ -362,6 +468,16 @@ pub enum DiskType { Unknown, } +impl fmt::Display for DiskType { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + DiskType::SSD => write!(f, "SSD"), + DiskType::HDD => write!(f, "HDD"), + DiskType::Unknown => write!(f, "Unknown"), + } + } +} + impl DiskType { pub fn from_string(disk_type: &str) -> Self { match disk_type.to_uppercase().as_str() { @@ -373,7 +489,7 @@ impl DiskType { } /// Represents the filesystem type of the volume -#[derive(Serialize, Deserialize, Debug, Clone, Type, Hash, PartialEq, Eq, Display)] +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, Hash)] pub enum FileSystem { /// Windows NTFS filesystem NTFS, @@ -385,25 +501,60 @@ pub enum FileSystem { APFS, /// ExFAT filesystem ExFAT, + /// Btrfs filesystem (Linux) + Btrfs, + /// ZFS filesystem + ZFS, + /// Windows ReFS filesystem + ReFS, /// Other/unknown filesystem type Other(String), } +impl fmt::Display for FileSystem { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + FileSystem::NTFS => write!(f, "NTFS"), + FileSystem::FAT32 => write!(f, "FAT32"), + FileSystem::EXT4 => write!(f, "EXT4"), + FileSystem::APFS => write!(f, "APFS"), + FileSystem::ExFAT => write!(f, "ExFAT"), + FileSystem::Btrfs => write!(f, "Btrfs"), + FileSystem::ZFS => write!(f, "ZFS"), + FileSystem::ReFS => write!(f, "ReFS"), + FileSystem::Other(name) => write!(f, "{}", name), + } + } +} + impl FileSystem { pub fn from_string(fs: &str) -> Self { match fs.to_uppercase().as_str() { - "NTFS" => FileSystem::NTFS, - "FAT32" => FileSystem::FAT32, - "EXT4" => FileSystem::EXT4, - "APFS" => FileSystem::APFS, - "EXFAT" => FileSystem::ExFAT, - other => FileSystem::Other(other.to_string()), + "NTFS" => Self::NTFS, + "FAT32" => Self::FAT32, + "EXT4" => Self::EXT4, + "APFS" => Self::APFS, + "EXFAT" => Self::ExFAT, + "BTRFS" => Self::Btrfs, + "ZFS" => Self::ZFS, + "REFS" => Self::ReFS, + other => Self::Other(other.to_string()), } } + + /// Check if this filesystem supports reflinks/clones + pub fn supports_reflink(&self) -> bool { + matches!(self, Self::APFS | Self::Btrfs | Self::ZFS | Self::ReFS) + } + + /// Check if this filesystem supports sendfile optimization + pub fn supports_sendfile(&self) -> bool { + matches!(self, Self::EXT4 | Self::Btrfs | Self::ZFS | Self::NTFS) + } } /// Represents how the volume is mounted in the system -#[derive(Serialize, Deserialize, Debug, Clone, Type, Hash, PartialEq, Eq, Display)] +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, Hash)] pub enum MountType { /// System/boot volume System, @@ -415,6 +566,17 @@ pub enum MountType { Virtual, } +impl fmt::Display for MountType { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + MountType::System => write!(f, "System"), + MountType::External => write!(f, "External"), + MountType::Network => write!(f, "Network"), + MountType::Virtual => write!(f, "Virtual"), + } + } +} + impl MountType { pub fn from_string(mount_type: &str) -> Self { match mount_type.to_uppercase().as_str() { @@ -427,48 +589,97 @@ impl MountType { } } -/// Configuration options for volume operations +/// Configuration for volume detection and monitoring #[derive(Debug, Clone)] -pub struct VolumeOptions { +pub struct VolumeDetectionConfig { /// Whether to include system volumes pub include_system: bool, /// Whether to include virtual volumes pub include_virtual: bool, /// Whether to run speed tests on discovery pub run_speed_test: bool, - /// Maximum concurrent speed tests - pub max_concurrent_speed_tests: usize, + /// How often to refresh volume information (in seconds) + pub refresh_interval_secs: u64, } -impl Default for VolumeOptions { +impl Default for VolumeDetectionConfig { fn default() -> Self { Self { include_system: true, include_virtual: false, - run_speed_test: true, - max_concurrent_speed_tests: 2, + run_speed_test: false, // Expensive operation, off by default + refresh_interval_secs: 30, } } } -impl Serialize for VolumeFingerprint { - fn serialize(&self, serializer: S) -> Result - where - S: serde::Serializer, - { - // Convert to hex string when serializing - serializer.serialize_str(&hex::encode(&self.0)) - } -} +#[cfg(test)] +mod tests { + use super::*; -impl<'de> Deserialize<'de> for VolumeFingerprint { - fn deserialize(deserializer: D) -> Result - where - D: serde::Deserializer<'de>, - { - let s = String::deserialize(deserializer)?; - hex::decode(s) - .map(VolumeFingerprint) - .map_err(serde::de::Error::custom) + #[test] + fn test_volume_fingerprint() { + let volume = Volume::new( + uuid::Uuid::new_v4(), + "Test Volume".to_string(), + MountType::External, + VolumeType::External, + PathBuf::from("/mnt/test"), + vec![], + DiskType::SSD, + FileSystem::EXT4, + 1000000000, + 500000000, + false, + Some("test-hw-id".to_string()), + VolumeFingerprint::new("Test", 500000000, "ext4"), + ); + + // Test basic fingerprint creation + let fingerprint = VolumeFingerprint::new( + "Test Volume", + 1000000000, // 1GB + "ext4", + ); + assert!(!fingerprint.0.is_empty()); + + // Test Spacedrive ID fingerprint + let spacedrive_id = Uuid::new_v4(); + let spacedrive_fingerprint = VolumeFingerprint::from_spacedrive_id(spacedrive_id); + assert!(!spacedrive_fingerprint.0.is_empty()); + assert_ne!(fingerprint, spacedrive_fingerprint); + } + + #[test] + fn test_volume_contains_path() { + let volume = Volume::new( + uuid::Uuid::new_v4(), + "Test".to_string(), + MountType::System, + VolumeType::System, + PathBuf::from("/home"), + vec![PathBuf::from("/home"), PathBuf::from("/mnt/home")], + DiskType::SSD, + FileSystem::EXT4, + 1000000, + 500000, + false, + None, + VolumeFingerprint::new("Test", 1000000, "ext4"), + ); + + assert!(volume.contains_path(&PathBuf::from("/home/user/file.txt"))); + assert!(volume.contains_path(&PathBuf::from("/mnt/home/user/file.txt"))); + assert!(!volume.contains_path(&PathBuf::from("/var/log/file.txt"))); + } + + #[test] + fn test_filesystem_capabilities() { + assert!(FileSystem::APFS.supports_reflink()); + assert!(FileSystem::Btrfs.supports_reflink()); + assert!(!FileSystem::FAT32.supports_reflink()); + + assert!(FileSystem::EXT4.supports_sendfile()); + assert!(!FileSystem::FAT32.supports_sendfile()); } } diff --git a/core/src/volume/volumes.rs b/core/src/volume/volumes.rs deleted file mode 100644 index 20d9438b8..000000000 --- a/core/src/volume/volumes.rs +++ /dev/null @@ -1,130 +0,0 @@ -/// This module contains the public interface for volume management -use super::{ - actor::VolumeManagerMessage, - error::VolumeError, - types::{Volume, VolumeEvent, VolumeFingerprint}, -}; -use crate::library::Library; -use async_channel as chan; -use std::sync::Arc; -use tokio::sync::broadcast; -use tokio::sync::oneshot; -use tracing::instrument; - -/// The public interface for volume management -#[derive(Clone)] -pub struct Volumes { - pub message_tx: chan::Sender, - pub event_tx: broadcast::Sender, -} - -impl Volumes { - pub(crate) fn new( - message_tx: chan::Sender, - event_tx: broadcast::Sender, - ) -> Self { - Self { - message_tx, - event_tx, - } - } - - /// Creates a new subscription for volume events - pub fn subscribe(&self) -> broadcast::Receiver { - self.event_tx.subscribe() - } - - /// Lists all volumes, tracked and not tracked on the system - pub async fn list_system_volumes( - &self, - library: Arc, - ) -> Result, VolumeError> { - let (tx, rx) = oneshot::channel(); - let msg = VolumeManagerMessage::ListSystemVolumes { ack: tx, library }; - - self.message_tx - .send(msg) - .await - .map_err(|_| VolumeError::Cancelled)?; - - rx.await.map_err(|_| VolumeError::Cancelled)? - } - - /// Lists volumes for a specific library including system volumes - pub async fn list_library_volumes( - &self, - library: Arc, - ) -> Result, VolumeError> { - let (tx, rx) = oneshot::channel(); - let msg = VolumeManagerMessage::ListLibraryVolumes { library, ack: tx }; - - self.message_tx - .send(msg) - .await - .map_err(|_| VolumeError::Cancelled)?; - - rx.await.map_err(|_| VolumeError::Cancelled)? - } - - /// Track a volume in a specific library - #[instrument(skip(self))] - pub async fn track_volume( - &self, - fingerprint: VolumeFingerprint, - library: Arc, - ) -> Result<(), VolumeError> { - let (tx, rx) = oneshot::channel(); - let msg = VolumeManagerMessage::TrackVolume { - fingerprint, - library, - ack: tx, - }; - - self.message_tx - .send(msg) - .await - .map_err(|_| VolumeError::Cancelled)?; - - rx.await.map_err(|_| VolumeError::Cancelled)? - } - - /// Stop tracking a volume - #[instrument(skip(self))] - pub async fn untrack_volume( - &self, - fingerprint: VolumeFingerprint, - library: Arc, - ) -> Result<(), VolumeError> { - let (tx, rx) = oneshot::channel(); - let msg = VolumeManagerMessage::UntrackVolume { - fingerprint, - library, - ack: tx, - }; - - self.message_tx - .send(msg) - .await - .map_err(|_| VolumeError::Cancelled)?; - - rx.await.map_err(|_| VolumeError::Cancelled)? - } - - pub async fn unmount_volume(&self, fingerprint: VolumeFingerprint) -> Result<(), VolumeError> { - let (tx, rx) = oneshot::channel(); - let msg = VolumeManagerMessage::UnmountVolume { - fingerprint, - ack: tx, - }; - - self.message_tx - .send(msg) - .await - .map_err(|_| VolumeError::Cancelled)?; - - let _ = rx.await.map_err(|_| VolumeError::Cancelled)?; - Ok(()) - } - - // Other public methods... -} diff --git a/core/src/volume/watcher.rs b/core/src/volume/watcher.rs deleted file mode 100644 index 4a1e472f0..000000000 --- a/core/src/volume/watcher.rs +++ /dev/null @@ -1,262 +0,0 @@ -use crate::volume::types::VolumeFingerprint; - -use super::error::VolumeError; -use super::types::VolumeEvent; -use super::VolumeManagerActor; -use sd_core_sync::DevicePubId; -use std::{collections::HashSet, path::PathBuf, sync::Arc, time::Duration}; -use tokio::sync::Mutex; -use tokio::{ - sync::{broadcast, mpsc, RwLock}, - time::{sleep, Instant}, -}; -use tracing::{debug, error,warn}; - -const DEBOUNCE_MS: u64 = 100; - -#[derive(Debug)] -pub struct VolumeWatcher { - event_tx: broadcast::Sender, - ignored_paths: Arc>>, - running: Arc>, -} - -impl VolumeWatcher { - pub fn new(event_tx: broadcast::Sender) -> Self { - Self { - event_tx, - ignored_paths: Arc::new(RwLock::new(HashSet::new())), - running: Arc::new(RwLock::new(true)), - } - } - - pub async fn start( - &self, - device_id: DevicePubId, - actor: Arc>, - ) -> Result<(), VolumeError> { - debug!("Starting volume watcher"); - - let (check_tx, mut check_rx) = mpsc::channel(1); - - // Start OS-specific watcher - self.spawn_platform_watcher(check_tx.clone()).await?; - - // Handle volume checks when triggered by OS events - let event_tx = self.event_tx.clone(); - let running = self.running.clone(); - - tokio::spawn(async move { - let mut last_check = Instant::now(); - - while *running.read().await { - // Wait for check trigger from OS watcher - if check_rx.recv().await.is_some() { - // Debounce checks - if last_check.elapsed() < Duration::from_millis(DEBOUNCE_MS) { - continue; - } - last_check = Instant::now(); - - #[cfg(any(target_os = "linux", target_os = "macos"))] - let discovered_volumes = match super::os::get_volumes().await { - Ok(volumes) => volumes, - Err(e) => { - error!("Failed to get volumes: {}", e); - // Return empty volumes to avoid sending events - vec![] - } - }; - - #[cfg(target_os = "windows")] - let discovered_volumes = super::os::get_volumes().await; - - let actor = actor.lock().await; - - // Find new volumes - for volume in &discovered_volumes { - let fingerprint = VolumeFingerprint::new(&device_id, volume); - - let volume_exists = actor.volume_exists(fingerprint.clone()).await; - // if the volume doesn't exist in the actor state, we need to send an event - if !volume_exists { - let _ = event_tx.send(VolumeEvent::VolumeAdded(volume.clone())); - } - } - - // Find removed volumes and send an event - for volume in &actor.get_volumes().await { - let fingerprint = VolumeFingerprint::new(&device_id, volume); - if !discovered_volumes - .iter() - .any(|v| VolumeFingerprint::new(&device_id, v) == fingerprint) - { - let _ = event_tx.send(VolumeEvent::VolumeRemoved(volume.clone())); - } - } - } - } - }); - - Ok(()) - } - - async fn spawn_platform_watcher(&self, check_tx: mpsc::Sender<()>) -> Result<(), VolumeError> { - let running = self.running.clone(); - - #[cfg(target_os = "linux")] - { - use inotify::{Inotify, WatchMask}; - - let mut inotify = Inotify::init().map_err(|e| { - VolumeError::Platform(format!("Failed to initialize inotify: {}", e)) - })?; - - // Watch mount points and device changes - for path in ["/dev", "/media", "/mnt", "/run/media"] { - if let Err(e) = inotify.add_watch( - path, - WatchMask::CREATE | WatchMask::DELETE | WatchMask::MODIFY, - ) { - warn!("Failed to watch path {}: {}", path, e); - } - } - - let check_tx = check_tx.clone(); - tokio::spawn(async move { - let mut buffer = [0; 4096]; - while *running.read().await { - match inotify.read_events_blocking(&mut buffer) { - Ok(_) => { - if let Err(e) = check_tx.send(()).await { - error!("Failed to trigger volume check: {}", e); - } - } - Err(e) => error!("Inotify error: {}", e), - } - } - }); - } - - #[cfg(target_os = "macos")] - { - use fsevent::{self, StreamFlags}; - - let (fs_tx, fs_rx) = std::sync::mpsc::channel(); - let check_tx = check_tx.clone(); - - // Keep stream alive in the thread - std::thread::spawn(move || { - let mut stream = fsevent::FsEvent::new(vec![ - "/Volumes".to_string(), - "/System/Volumes".to_string(), - ]); - - match stream.observe_async(fs_tx) { - Ok(_) => { - // Block thread to keep stream alive - std::thread::park(); - } - Err(e) => { - error!("Failed to start FSEvent stream: {}", e); - } - } - }); - - tokio::spawn(async move { - while *running.read().await { - match fs_rx.recv() { - Ok(events) => { - // Only care about mount/unmount events - if events.flag.contains(StreamFlags::MOUNT) - || events.flag.contains(StreamFlags::UNMOUNT) - { - debug!("Received volume event: {:?}", events); - if let Err(e) = check_tx.send(()).await { - error!("Failed to trigger volume check: {}", e); - } - } - } - Err(e) => { - error!("FSEvent receive error: {}", e); - sleep(Duration::from_millis(100)).await; - } - } - } - }); - } - - #[cfg(target_os = "windows")] - { - use ::windows::Win32::Storage::FileSystem::{FindFirstVolumeW, FindVolumeClose}; - - let check_tx = check_tx.clone(); - tokio::spawn(async move { - while *running.read().await { - // Watch for volume arrival/removal - unsafe { - let mut volume_name = [0u16; 260]; - let mut volume_change_detected = false; - match FindFirstVolumeW(volume_name.as_mut_slice()) { - Ok(handle) => { - if !handle.is_invalid() { - volume_change_detected = true; - FindVolumeClose(handle); - } - } - Err(e) => { - error!("Failed to get a volume handle: {}", e); - } - } - if volume_change_detected { - // Volume change detected - if let Err(e) = check_tx.send(()).await { - error!("Failed to trigger volume check: {}", e); - } - } - } - sleep(Duration::from_millis(100)).await; - } - }); - } - - Ok(()) - } - - pub async fn stop(&self) { - debug!("Stopping volume watcher"); - *self.running.write().await = false; - } - - pub async fn ignore_path(&self, path: PathBuf) { - self.ignored_paths.write().await.insert(path); - } - - pub async fn unignore_path(&self, path: &PathBuf) { - self.ignored_paths.write().await.remove(path); - } -} - -// #[cfg(test)] -// mod tests { -// use super::*; -// use tokio::time::timeout; - -// #[tokio::test] -// async fn test_watcher() { -// let (tx, mut rx) = broadcast::channel(16); -// let watcher = VolumeWatcher::new(tx); - -// watcher.start().await.expect("Failed to start watcher"); - -// // Wait for potential volume events -// let result = timeout(Duration::from_secs(2), rx.recv()).await; - -// // Cleanup -// watcher.stop().await; - -// if let Ok(Ok(event)) = result { -// println!("Received volume event: {:?}", event); -// } -// } -// } diff --git a/core-new/tests/copy_action_test.rs b/core/tests/copy_action_test.rs similarity index 100% rename from core-new/tests/copy_action_test.rs rename to core/tests/copy_action_test.rs diff --git a/core-new/tests/copy_progress_test.rs b/core/tests/copy_progress_test.rs similarity index 100% rename from core-new/tests/copy_progress_test.rs rename to core/tests/copy_progress_test.rs diff --git a/core-new/tests/cross_device_copy_test.rs b/core/tests/cross_device_copy_test.rs similarity index 100% rename from core-new/tests/cross_device_copy_test.rs rename to core/tests/cross_device_copy_test.rs diff --git a/core-new/tests/database_migration_test.rs b/core/tests/database_migration_test.rs similarity index 100% rename from core-new/tests/database_migration_test.rs rename to core/tests/database_migration_test.rs diff --git a/core-new/tests/device_pairing_test.rs b/core/tests/device_pairing_test.rs similarity index 100% rename from core-new/tests/device_pairing_test.rs rename to core/tests/device_pairing_test.rs diff --git a/core-new/tests/device_persistence_test.rs b/core/tests/device_persistence_test.rs similarity index 100% rename from core-new/tests/device_persistence_test.rs rename to core/tests/device_persistence_test.rs diff --git a/core-new/tests/event_system_test.rs b/core/tests/event_system_test.rs similarity index 100% rename from core-new/tests/event_system_test.rs rename to core/tests/event_system_test.rs diff --git a/core-new/tests/file_transfer_test.rs b/core/tests/file_transfer_test.rs similarity index 100% rename from core-new/tests/file_transfer_test.rs rename to core/tests/file_transfer_test.rs diff --git a/core-new/tests/helpers/mod.rs b/core/tests/helpers/mod.rs similarity index 100% rename from core-new/tests/helpers/mod.rs rename to core/tests/helpers/mod.rs diff --git a/core-new/tests/helpers/test_volumes.rs b/core/tests/helpers/test_volumes.rs similarity index 100% rename from core-new/tests/helpers/test_volumes.rs rename to core/tests/helpers/test_volumes.rs diff --git a/core-new/tests/indexing_rules_test.rs b/core/tests/indexing_rules_test.rs similarity index 100% rename from core-new/tests/indexing_rules_test.rs rename to core/tests/indexing_rules_test.rs diff --git a/core-new/tests/indexing_test.rs b/core/tests/indexing_test.rs similarity index 100% rename from core-new/tests/indexing_test.rs rename to core/tests/indexing_test.rs diff --git a/core-new/tests/job_pause_resume_test.rs b/core/tests/job_pause_resume_test.rs similarity index 100% rename from core-new/tests/job_pause_resume_test.rs rename to core/tests/job_pause_resume_test.rs diff --git a/core-new/tests/job_registration_test.rs b/core/tests/job_registration_test.rs similarity index 100% rename from core-new/tests/job_registration_test.rs rename to core/tests/job_registration_test.rs diff --git a/core-new/tests/job_shutdown_test.rs b/core/tests/job_shutdown_test.rs similarity index 100% rename from core-new/tests/job_shutdown_test.rs rename to core/tests/job_shutdown_test.rs diff --git a/core-new/tests/library_test.rs b/core/tests/library_test.rs similarity index 100% rename from core-new/tests/library_test.rs rename to core/tests/library_test.rs diff --git a/core-new/tests/volume_tracking_test.rs b/core/tests/volume_tracking_test.rs similarity index 100% rename from core-new/tests/volume_tracking_test.rs rename to core/tests/volume_tracking_test.rs diff --git a/core-new/tests/volume_tracking_with_test_volumes.rs b/core/tests/volume_tracking_with_test_volumes.rs similarity index 100% rename from core-new/tests/volume_tracking_with_test_volumes.rs rename to core/tests/volume_tracking_with_test_volumes.rs diff --git a/core-new/whitepaper/architecture.png b/core/whitepaper/architecture.png similarity index 100% rename from core-new/whitepaper/architecture.png rename to core/whitepaper/architecture.png diff --git a/core-new/whitepaper/comment.cut b/core/whitepaper/comment.cut similarity index 100% rename from core-new/whitepaper/comment.cut rename to core/whitepaper/comment.cut diff --git a/core-new/whitepaper/grok-changes.md b/core/whitepaper/grok-changes.md similarity index 100% rename from core-new/whitepaper/grok-changes.md rename to core/whitepaper/grok-changes.md diff --git a/core-new/whitepaper/grok-critique.md b/core/whitepaper/grok-critique.md similarity index 100% rename from core-new/whitepaper/grok-critique.md rename to core/whitepaper/grok-critique.md diff --git a/core-new/whitepaper/grok-critique2.md b/core/whitepaper/grok-critique2.md similarity index 100% rename from core-new/whitepaper/grok-critique2.md rename to core/whitepaper/grok-critique2.md diff --git a/core-new/whitepaper/proposed-changes-v2.md b/core/whitepaper/proposed-changes-v2.md similarity index 100% rename from core-new/whitepaper/proposed-changes-v2.md rename to core/whitepaper/proposed-changes-v2.md diff --git a/core-new/whitepaper/proposed-changes.md b/core/whitepaper/proposed-changes.md similarity index 100% rename from core-new/whitepaper/proposed-changes.md rename to core/whitepaper/proposed-changes.md diff --git a/core-new/whitepaper/references.bib b/core/whitepaper/references.bib similarity index 100% rename from core-new/whitepaper/references.bib rename to core/whitepaper/references.bib diff --git a/core-new/whitepaper/spacedrive.bbl b/core/whitepaper/spacedrive.bbl similarity index 100% rename from core-new/whitepaper/spacedrive.bbl rename to core/whitepaper/spacedrive.bbl diff --git a/core-new/whitepaper/spacedrive.blg b/core/whitepaper/spacedrive.blg similarity index 100% rename from core-new/whitepaper/spacedrive.blg rename to core/whitepaper/spacedrive.blg diff --git a/core-new/whitepaper/spacedrive.out b/core/whitepaper/spacedrive.out similarity index 100% rename from core-new/whitepaper/spacedrive.out rename to core/whitepaper/spacedrive.out diff --git a/core-new/whitepaper/spacedrive.pdf b/core/whitepaper/spacedrive.pdf similarity index 100% rename from core-new/whitepaper/spacedrive.pdf rename to core/whitepaper/spacedrive.pdf diff --git a/core-new/whitepaper/spacedrive.tex b/core/whitepaper/spacedrive.tex similarity index 99% rename from core-new/whitepaper/spacedrive.tex rename to core/whitepaper/spacedrive.tex index dc14adc1d..3b321d9f2 100644 --- a/core-new/whitepaper/spacedrive.tex +++ b/core/whitepaper/spacedrive.tex @@ -2070,7 +2070,7 @@ Spacedrive's VDFS extends beyond local filesystems to natively integrate traditi The flexibility of the Spacedrive V2 architecture is best demonstrated by its application in creating a cloud service that natively integrates with the user's personal P2P network. Unlike traditional cloud backends that require custom APIs and treat the server as a privileged entity, our model treats the cloud instance as just another Spacedrive device. This approach leverages the core VDFS abstractions to provide cloud storage that feels native, secure, and seamlessly integrated into the user's existing ecosystem. \subsection{Core Principle: Managed Cores as First-Class Devices} -The foundational principle of the Spacedrive Cloud Service is that each user is provisioned a managed, containerized instance of the unmodified \texttt{sd-core-new} engine. This managed instance—which we refer to as a "Cloud Core" for convenience—is architecturally identical to any other Spacedrive core. It has its own unique device ID, participates in the same P2P network as the user's other devices, and exposes its storage as standard Spacedrive Locations. The term "Cloud Core" simply denotes its deployment context (managed hosting), not any special software or capabilities. +The foundational principle of the Spacedrive Cloud Service is that each user is provisioned a managed, containerized instance of the unmodified \texttt{sd-core} engine. This managed instance—which we refer to as a "Cloud Core" for convenience—is architecturally identical to any other Spacedrive core. It has its own unique device ID, participates in the same P2P network as the user's other devices, and exposes its storage as standard Spacedrive Locations. The term "Cloud Core" simply denotes its deployment context (managed hosting), not any special software or capabilities. This design offers profound architectural advantages: \begin{itemize}[noitemsep, topsep=0pt] diff --git a/crates/ai/Cargo.toml b/crates/ai/Cargo.toml deleted file mode 100644 index 35f9dc198..000000000 --- a/crates/ai/Cargo.toml +++ /dev/null @@ -1,79 +0,0 @@ -[package] -name = "sd-ai" -version = "0.1.0" - -authors = ["Ericson Soares "] -description = "A simple library to generate video thumbnails using ffmpeg with the webp format" -edition.workspace = true -license.workspace = true -readme = "README.md" -repository.workspace = true -rust-version.workspace = true - -[dependencies] -# Inner Core Sub-crates -sd-core-file-path-helper = { path = "../../core/crates/file-path-helper" } -sd-core-prisma-helpers = { path = "../../core/crates/prisma-helpers" } -sd-core-sync = { path = "../../core/crates/sync" } - -# Spacedrive Sub-crates -sd-prisma = { path = "../prisma" } -sd-sync = { path = "../sync" } -sd-utils = { path = "../utils" } - -async-channel = { workspace = true } -chrono = { workspace = true, features = ["serde"] } -futures = { workspace = true } -futures-concurrency = { workspace = true } -image = { workspace = true } -prisma-client-rust = { workspace = true } -reqwest = { workspace = true, features = ["native-tls-vendored", "stream"] } -rmp-serde = { workspace = true } -rmpv = { workspace = true } -serde = { workspace = true, features = ["derive"] } -thiserror = { workspace = true } -tokio = { workspace = true, features = ["fs"] } -tracing = { workspace = true } -uuid = { workspace = true, features = ["serde", "v4"] } - -# Specific AI dependencies -# Note: half and ndarray version must be the same as used in ort -half = { version = "2.4", features = ['num-traits'] } -ndarray = "0.15" -ort-sys = '=2.0.0-rc.0' # lock sys crate to the same version as ort -url = '2.5' - -# Microsoft does not provide a release for osx-gpu. See: https://github.com/microsoft/onnxruntime/releases -# "gpu" means CUDA or TensorRT EP. Thus, the ort crate cannot download them at build time. -# Ref: https://github.com/pykeio/ort/blob/d7defd1862969b4b44f7f3f4b9c72263690bd67b/build.rs#L148 -[target.'cfg(target_os = "windows")'.dependencies] -ort = { version = "=2.0.0-rc.0", default-features = false, features = [ - "directml", - "half", - "load-dynamic", - "ndarray" -] } -[target.'cfg(target_os = "linux")'.dependencies] -ort = { version = "=2.0.0-rc.0", default-features = false, features = [ - "half", - "ndarray", - "xnnpack" -] } -# [target.'cfg(target_os = "android")'.dependencies] -# ort = { version = "=2.0.0-rc.0", default-features = false, features = [ -# "half", -# "load-dynamic", -# "qnn", -# "nnapi", -# "xnnpack", -# "acl", -# "armnn", -# ] } -[target.'cfg(any(target_os = "macos", target_os = "ios"))'.dependencies] -ort = { version = "=2.0.0-rc.0", features = [ - "coreml", - "half", - "load-dynamic", - "ndarray", - "xnnpack" -] } diff --git a/crates/ai/README.md b/crates/ai/README.md deleted file mode 100644 index 514c3627d..000000000 --- a/crates/ai/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# Spacedrive AI - -A collection of AI baked features for Spacedrive. diff --git a/crates/ai/src/lib.rs b/crates/ai/src/lib.rs deleted file mode 100644 index 8e2806dae..000000000 --- a/crates/ai/src/lib.rs +++ /dev/null @@ -1,86 +0,0 @@ -use thiserror::Error; - -use ort::EnvironmentBuilder; -use tracing::{debug, error}; - -pub mod old_image_labeler; -mod utils; - -// This path must be relative to the running binary -#[cfg(target_os = "windows")] -const BINDING_LOCATION: &str = "."; - -#[cfg(target_os = "macos")] -const BINDING_LOCATION: &str = "../Frameworks/Spacedrive.framework/Libraries"; - -#[cfg(target_os = "windows")] -const LIB_NAME: &str = "onnxruntime.dll"; - -#[cfg(any(target_os = "macos", target_os = "ios"))] -const LIB_NAME: &str = "libonnxruntime.dylib"; - -pub fn init() -> Result<(), Error> { - #[cfg(any(target_os = "macos", target_os = "ios", target_os = "windows"))] - { - use std::path::Path; - let path = utils::get_path_relative_to_exe(Path::new(BINDING_LOCATION).join(LIB_NAME)); - std::env::set_var("ORT_DYLIB_PATH", path); - } - - // Initialize AI stuff - EnvironmentBuilder::default() - .with_name("spacedrive") - .with_execution_providers({ - #[cfg(any(target_os = "macos", target_os = "ios"))] - { - use ort::{CoreMLExecutionProvider, XNNPACKExecutionProvider}; - - [ - CoreMLExecutionProvider::default().build(), - XNNPACKExecutionProvider::default().build(), - ] - } - - #[cfg(target_os = "windows")] - { - use ort::DirectMLExecutionProvider; - - [DirectMLExecutionProvider::default().build()] - } - - #[cfg(target_os = "linux")] - { - use ort::XNNPACKExecutionProvider; - - [XNNPACKExecutionProvider::default().build()] - } - - // #[cfg(target_os = "android")] - // { - // use ort::{ - // ACLExecutionProvider, ArmNNExecutionProvider, NNAPIExecutionProvider, - // QNNExecutionProvider, XNNPACKExecutionProvider, - // }; - // [ - // QNNExecutionProvider::default().build(), - // NNAPIExecutionProvider::default().build(), - // XNNPACKExecutionProvider::default().build(), - // ACLExecutionProvider::default().build(), - // ArmNNExecutionProvider::default().build(), - // ] - // } - }) - .commit()?; - - debug!("Initialized AI environment"); - - Ok(()) -} - -#[derive(Error, Debug)] -pub enum Error { - #[error("failed to initialize AI environment: {0}")] - Init(#[from] ort::Error), - #[error(transparent)] - ImageLabeler(#[from] old_image_labeler::ImageLabelerError), -} diff --git a/crates/ai/src/old_image_labeler/mod.rs b/crates/ai/src/old_image_labeler/mod.rs deleted file mode 100644 index caad651ce..000000000 --- a/crates/ai/src/old_image_labeler/mod.rs +++ /dev/null @@ -1,56 +0,0 @@ -use sd_prisma::prisma::file_path; -use sd_utils::{db::MissingFieldError, error::FileIOError}; - -use std::path::Path; - -use thiserror::Error; -use tracing::error; -use uuid::Uuid; - -mod model; -mod old_actor; -mod process; - -pub use model::{DownloadModelError, Model, YoloV8, DEFAULT_MODEL_VERSION}; -pub use old_actor::OldImageLabeler; - -pub type BatchToken = Uuid; - -#[derive(Debug)] -pub struct LabelerOutput { - pub file_path_id: file_path::id::Type, - pub has_new_labels: bool, - pub result: Result<(), ImageLabelerError>, -} - -#[derive(Debug, Error)] -pub enum ImageLabelerError { - #[error("model executor failed: {0}")] - ModelExecutorFailed(#[from] ort::Error), - #[error("image load failed : {0}", path = .1.display())] - ImageLoadFailed(image::ImageError, Box), - #[error("failed to get isolated file path data: {0}")] - IsolateFilePathData(#[from] MissingFieldError), - #[error("file_path with unsupported extension: ")] - UnsupportedExtension(file_path::id::Type, String), - #[error("file_path too big: ")] - FileTooBig(file_path::id::Type, usize), - #[error("model file not found: {}", .0.display())] - ModelFileNotFound(Box), - #[error("no model available for inference")] - NoModelAvailable, - #[error("failed to decode pending batches: {0}")] - Decode(#[from] rmp_serde::decode::Error), - #[error("failed to encode pending batches: {0}")] - Encode(#[from] rmp_serde::encode::Error), - #[error("database error: {0}")] - Database(#[from] prisma_client_rust::QueryError), - #[error("resume token not found: {0}")] - TokenNotFound(BatchToken), - #[error(transparent)] - DownloadModel(#[from] DownloadModelError), - #[error(transparent)] - FileIO(#[from] FileIOError), - #[error(transparent)] - Sync(#[from] sd_core_sync::Error), -} diff --git a/crates/ai/src/old_image_labeler/model/mod.rs b/crates/ai/src/old_image_labeler/model/mod.rs deleted file mode 100644 index 64b30e858..000000000 --- a/crates/ai/src/old_image_labeler/model/mod.rs +++ /dev/null @@ -1,259 +0,0 @@ -use sd_utils::error::FileIOError; - -use std::{ - collections::HashSet, - path::{Path, PathBuf}, -}; - -use futures::prelude::stream::StreamExt; -use image::ImageFormat; -use ort::{Session, SessionBuilder, SessionInputs, SessionOutputs}; -use thiserror::Error; -use tokio::{ - fs, - io::{self, AsyncWriteExt}, -}; -use tracing::{error, info, trace}; -use url::Url; - -use super::ImageLabelerError; - -mod yolov8; - -pub use yolov8::YoloV8; -pub use yolov8::DEFAULT_MODEL_VERSION; - -pub enum ModelSource { - Url(Url), - Path(PathBuf), -} - -pub trait Model: Send + Sync + 'static { - fn name(&self) -> &'static str; - - fn origin(&self) -> &ModelSource; - - fn version(&self) -> &str; - - fn versions() -> Vec<&'static str> - where - Self: Sized; - - fn prepare_input<'image>( - &self, - image_path: &Path, - image: &'image [u8], - format: ImageFormat, - ) -> Result, ImageLabelerError>; - - fn process_output( - &self, - output: SessionOutputs<'_>, - ) -> Result, ImageLabelerError>; -} - -pub(super) struct ModelAndSession { - maybe_model: Option>, - maybe_session: Option, - model_data_dir: PathBuf, -} - -impl ModelAndSession { - pub async fn new( - model: Box, - data_dir: impl AsRef, - ) -> Result { - let data_dir = data_dir.as_ref().join(model.name()); - let model_path = download_model(model.origin(), &data_dir).await?; - - info!( - "Loading mode: {} from {}", - model.name(), - model_path.display() - ); - - let maybe_session = check_model_file(&model_path) - .await - .map_err(|e| error!("Failed to check model file before passing to Ort: {e:#?}")) - .ok() - .and_then(|()| { - load_model(&model_path) - .map(|session| { - info!("Loaded model: {}", model.name()); - trace!("{session:#?}"); - session - }) - .map_err(|e| error!("Failed to load model: {e:#?}")) - .ok() - }); - - Ok(Self { - maybe_model: maybe_session.is_some().then_some(model), - maybe_session, - model_data_dir: data_dir, - }) - } - - pub fn can_process(&self) -> bool { - self.maybe_session.is_some() && self.maybe_model.is_some() - } - - pub async fn update_model( - &mut self, - new_model: Box, - ) -> Result<(), ImageLabelerError> { - info!("Attempting to change image labeler models..."); - - let model_path = download_model(new_model.origin(), &self.model_data_dir).await?; - - info!( - "Change mode: {} to {}", - new_model.name(), - model_path.display() - ); - - check_model_file(&model_path).await.and_then(|()| { - load_model(&model_path) - .map(|session| { - info!( - "Changing models: {} -> {}", - self.maybe_model - .as_ref() - .map(|old_model| old_model.name()) - .unwrap_or("None"), - new_model.name() - ); - - self.maybe_model = Some(new_model); - self.maybe_session = Some(session); - }) - .inspect_err(|e| { - error!("Failed to load new model: {e:#?}"); - self.maybe_model = None; - self.maybe_session = None; - }) - }) - } - - pub fn process_single_image( - &self, - image_path: &Path, - image: Vec, - format: ImageFormat, - ) -> Result, ImageLabelerError> { - if let (Some(session), Some(model)) = (&self.maybe_session, self.maybe_model.as_deref()) { - let inputs = model.prepare_input(image_path, &image, format)?; - let outputs = session.run(inputs)?; - model.process_output(outputs) - } else { - error!("Tried to process image without a loaded model"); - Err(ImageLabelerError::NoModelAvailable) - } - } -} - -#[derive(Error, Debug)] -pub enum DownloadModelError { - #[error("Failed to download due to request error: {0}")] - RequestError(#[from] reqwest::Error), - #[error("Failed to download due to status code: {0}")] - HttpStatusError(reqwest::StatusCode), - #[error("Invalid file name for url: {0}")] - InvalidUrlFileName(Url), - #[error("Unknown model version to download: {0}")] - UnknownModelVersion(String), - - #[error(transparent)] - FileIO(#[from] FileIOError), -} - -fn load_model(model_path: impl AsRef) -> Result { - SessionBuilder::new()? - .with_parallel_execution(true)? - .with_memory_pattern(true)? - .with_model_from_file(model_path) - .map_err(Into::into) -} - -async fn download_model( - model_origin: &ModelSource, - data_dir: impl AsRef, -) -> Result { - let data_dir = data_dir.as_ref(); - - match model_origin { - ModelSource::Url(url) => { - let Some(file_name) = url.path_segments().and_then(|segments| segments.last()) else { - return Err(DownloadModelError::InvalidUrlFileName(url.to_owned())); - }; - - fs::create_dir_all(data_dir) - .await - .map_err(|e| FileIOError::from((data_dir, e, "Failed to create data directory")))?; - - let file_path = data_dir.join(file_name); - match fs::metadata(&file_path).await { - Ok(_) => return Ok(file_path), - Err(e) if e.kind() != io::ErrorKind::NotFound => { - return Err(DownloadModelError::FileIO(FileIOError::from(( - file_path, - e, - "Failed to get metadata for model file", - )))) - } - _ => { - info!("Downloading model from: {} to {}", url, file_path.display()); - let response = reqwest::get(url.as_str()).await?; - // Ensure the request was successful (status code 2xx) - if !response.status().is_success() { - return Err(DownloadModelError::HttpStatusError(response.status())); - } - - // Create or open a file at the specified path - let mut file = fs::File::create(&file_path).await.map_err(|e| { - FileIOError::from(( - &file_path, - e, - "Failed to create the model file on disk", - )) - })?; - // Stream the response body to the file - let mut body = response.bytes_stream(); - while let Some(chunk) = body.next().await { - let chunk = chunk?; - file.write_all(&chunk).await.map_err(|e| { - FileIOError::from(( - &file_path, - e, - "Failed to write chunk of data to the model file on disk", - )) - })?; - } - } - } - - Ok(file_path) - } - ModelSource::Path(file_path) => Ok(file_path.to_owned()), - } -} - -async fn check_model_file(model_path: impl AsRef) -> Result<(), ImageLabelerError> { - let model_path = model_path.as_ref(); - - match fs::metadata(model_path).await { - Ok(_) => Ok(()), - Err(e) if e.kind() == io::ErrorKind::NotFound => { - error!( - "Model file not found: '{}'. Image labeler will be disabled!", - model_path.display() - ); - Ok(()) - } - Err(e) => Err(ImageLabelerError::FileIO(FileIOError::from(( - model_path, - e, - "Failed to get metadata for model file", - )))), - } -} diff --git a/crates/ai/src/old_image_labeler/model/yolov8.rs b/crates/ai/src/old_image_labeler/model/yolov8.rs deleted file mode 100644 index 8440c9a7f..000000000 --- a/crates/ai/src/old_image_labeler/model/yolov8.rs +++ /dev/null @@ -1,168 +0,0 @@ -use crate::utils::get_path_relative_to_exe; - -use std::{ - collections::{HashMap, HashSet}, - fmt::Display, - path::Path, - sync::LazyLock, -}; - -use half::f16; -use image::{imageops::FilterType, load_from_memory_with_format, GenericImageView, ImageFormat}; -use ndarray::{s, Array, Axis}; -use ort::{inputs, SessionInputs, SessionOutputs}; -use url::Url; - -use super::{DownloadModelError, ImageLabelerError, Model, ModelSource}; - -pub struct YoloV8 { - model_origin: &'static ModelSource, - model_version: String, -} - -// This path must be relative to the running binary -#[cfg(windows)] -const MODEL_LOCATION: &str = "./models"; -#[cfg(unix)] -const MODEL_LOCATION: &str = if cfg!(target_os = "macos") { - "../Frameworks/Spacedrive.framework/Resources/Models" -} else { - "../share/spacedrive/models" -}; - -pub static DEFAULT_MODEL_VERSION: &str = "Yolo Small"; - -static MODEL_VERSIONS: LazyLock> = LazyLock::new(|| { - HashMap::from([ - ("Yolo Nano", ModelSource::Url(Url::parse("https://github.com/spacedriveapp/native-deps/releases/download/yolo-2023-12-05/yolov8n.onnx").expect("Must be a valid URL"))), - (DEFAULT_MODEL_VERSION, ModelSource::Path(get_path_relative_to_exe(Path::new(MODEL_LOCATION).join("yolov8s.onnx")))), - ("Yolo Medium", ModelSource::Url(Url::parse("https://github.com/spacedriveapp/native-deps/releases/download/yolo-2023-12-05/yolov8m.onnx").expect("Must be a valid URL"))), - ("Yolo Large", ModelSource::Url(Url::parse("https://github.com/spacedriveapp/native-deps/releases/download/yolo-2023-12-05/yolov8l.onnx").expect("Must be a valid URL"))), - ("Yolo Extra", ModelSource::Url(Url::parse("https://github.com/spacedriveapp/native-deps/releases/download/yolo-2023-12-05/yolov8x.onnx").expect("Must be a valid URL"))), - ]) -}); - -impl YoloV8 { - pub fn model(version: Option) -> Result, DownloadModelError> - where - T: AsRef + Display, - { - let (model_version, model_origin) = match version { - Some(version) => ( - version.to_string(), - MODEL_VERSIONS - .get(version.as_ref()) - .ok_or_else(|| DownloadModelError::UnknownModelVersion(version.to_string()))?, - ), - None => { - let version = DEFAULT_MODEL_VERSION; - ( - version.to_string(), - MODEL_VERSIONS - .get(version) - .expect("Default model version must be valid"), - ) - } - }; - - Ok(Box::new(Self { - model_origin, - model_version, - })) - } -} - -impl Model for YoloV8 { - fn name(&self) -> &'static str { - "YoloV8" - } - - fn origin(&self) -> &'static ModelSource { - self.model_origin - } - - fn version(&self) -> &str { - self.model_version.as_str() - } - - fn versions() -> Vec<&'static str> { - MODEL_VERSIONS.keys().copied().collect() - } - - fn prepare_input<'image>( - &self, - path: &Path, - image: &'image [u8], - format: ImageFormat, - ) -> Result, ImageLabelerError> { - let original_img = load_from_memory_with_format(image, format) - .map_err(|e| ImageLabelerError::ImageLoadFailed(e, path.into()))?; - - let img = original_img.resize_exact(640, 640, FilterType::CatmullRom); - let mut input = Array::::zeros((1, 3, 640, 640)); - for pixel in img.pixels() { - let x = pixel.0 as _; - let y = pixel.1 as _; - let [r, g, b, _] = pixel.2 .0; - input[[0, 0, y, x]] = f16::from_f32((r as f32) / 255.); - input[[0, 1, y, x]] = f16::from_f32((g as f32) / 255.); - input[[0, 2, y, x]] = f16::from_f32((b as f32) / 255.); - } - - inputs!["images" => input.view()] - .map(Into::into) - .map_err(Into::into) - } - - fn process_output( - &self, - output: SessionOutputs<'_>, - ) -> Result, ImageLabelerError> { - #[rustfmt::skip] - const YOLOV8_CLASS_LABELS: [&str; 80] = [ - "person", "bicycle", "car", "motorcycle", "airplane", "bus", "train", "truck", - "boat", "traffic light", "fire hydrant", "stop sign", "parking meter", "bench", - "bird", "cat", "dog", "horse", "sheep", "cow", "elephant", "bear", "zebra", - "giraffe", "backpack", "umbrella", "handbag", "tie", "suitcase", "frisbee", - "skis", "snowboard", "sports ball", "kite", "baseball bat", "baseball glove", - "skateboard", "surfboard", "tennis racket", "bottle", "wine glass", "cup", - "fork", "knife", "spoon", "bowl", "banana", "apple", "sandwich", "orange", - "broccoli", "carrot", "hot dog", "pizza", "donut", "cake", "chair", "couch", - "potted plant", "bed", "dining table", "toilet", "tv", "laptop", "mouse", - "remote", "keyboard", "cell phone", "microwave", "oven", "toaster", "sink", - "refrigerator", "book", "clock", "vase", "scissors", "teddy bear", - "hair drier", "toothbrush" - ]; - - let output0 = &output["output0"]; - - let output_tensor = output0.extract_tensor::()?; - - let output_view = output_tensor.view(); - - let output_tensor_transposed = output_view.t(); - - let output = output_tensor_transposed.slice(s![.., .., 0]); - - Ok(output - .axis_iter(Axis(0)) - .map(|row| { - row.iter() - // skip bounding box coordinates - .skip(4) - .enumerate() - .map(|(class_id, probability)| (class_id, *probability)) - .reduce(|accum, row| if row.1 > accum.1 { row } else { accum }) - .expect("not empty output") - }) - .filter(|(_, probability)| probability.to_f32() > 0.6) - .map(|(class_id, _)| YOLOV8_CLASS_LABELS[class_id]) - .fold(HashSet::default(), |mut set, label| { - if !set.contains(label) { - set.insert(label.to_string()); - } - - set - })) - } -} diff --git a/crates/ai/src/old_image_labeler/old_actor.rs b/crates/ai/src/old_image_labeler/old_actor.rs deleted file mode 100644 index 669d00ec6..000000000 --- a/crates/ai/src/old_image_labeler/old_actor.rs +++ /dev/null @@ -1,612 +0,0 @@ -use sd_core_prisma_helpers::file_path_for_media_processor; -use sd_core_sync::SyncManager; - -use sd_prisma::prisma::{device, location, PrismaClient}; -use sd_utils::error::FileIOError; - -use std::{ - cell::RefCell, - collections::{HashMap, VecDeque}, - ops::Deref, - path::{Path, PathBuf}, - pin::pin, - sync::Arc, - time::Duration, -}; - -use async_channel as chan; -use futures::stream::StreamExt; -use futures_concurrency::stream::Merge; -use serde::{Deserialize, Serialize}; -use tokio::{ - fs, io, spawn, - sync::{oneshot, RwLock}, - task::JoinHandle, - time::timeout, -}; -use tracing::{debug, error, info}; -use uuid::Uuid; - -use super::{ - model::{Model, ModelAndSession}, - process::{spawned_processing, FinishStatus}, - BatchToken, ImageLabelerError, LabelerOutput, -}; - -const ONE_SEC: Duration = Duration::from_secs(1); -const PENDING_BATCHES_FILE: &str = "pending_image_labeler_batches.bin"; - -type ResumeBatchRequest = ( - BatchToken, - Arc, - SyncManager, - oneshot::Sender, ImageLabelerError>>, -); - -type UpdateModelRequest = ( - Box, - oneshot::Sender>, -); - -pub(super) struct Batch { - pub(super) token: BatchToken, - pub(super) location_id: location::id::Type, - pub(super) location_path: PathBuf, - pub(super) device_id: device::id::Type, - pub(super) file_paths: Vec, - pub(super) output_tx: chan::Sender, - pub(super) is_resumable: bool, - pub(super) db: Arc, - pub(super) sync: SyncManager, -} - -#[derive(Serialize, Deserialize, Debug)] -struct ResumableBatch { - location_id: location::id::Type, - location_path: PathBuf, - device_id: device::id::Type, - file_paths: Vec, -} - -pub struct OldImageLabeler { - to_resume_batches_file_path: PathBuf, - new_batches_tx: chan::Sender, - resume_batch_tx: chan::Sender, - update_model_tx: chan::Sender, - shutdown_tx: chan::Sender>, - to_resume_batches: Arc>>, - handle: RefCell>>, -} - -impl OldImageLabeler { - pub async fn new( - model: Box, - data_directory: impl AsRef, - ) -> Result { - let to_resume_batches_file_path = data_directory.as_ref().join(PENDING_BATCHES_FILE); - - let model_and_session = Arc::new(RwLock::new( - ModelAndSession::new(model, data_directory.as_ref().join("models")).await?, - )); - - let to_resume_batches = Arc::new(RwLock::new( - match fs::read(&to_resume_batches_file_path).await { - Ok(bytes) => { - let pending_batches = - rmp_serde::from_slice::>(&bytes)?; - info!( - "Image labeler had {} pending batches to be resumed", - pending_batches.len() - ); - - if let Err(e) = fs::remove_file(&to_resume_batches_file_path).await { - error!( - "{:#?}", - ImageLabelerError::from(FileIOError::from(( - &to_resume_batches_file_path, - e, - "Failed to remove to resume batches file", - ))) - ); - } - - pending_batches - } - Err(e) if e.kind() == io::ErrorKind::NotFound => { - // If the file doesn't exist, we just start with an empty list - HashMap::new() - } - Err(e) => { - return Err(ImageLabelerError::FileIO(FileIOError::from(( - &to_resume_batches_file_path, - e, - "Failed to read to resume batches file", - )))) - } - }, - )); - - let (new_batches_tx, new_batches_rx) = chan::unbounded(); - let (resume_batch_tx, resume_batch_rx) = chan::bounded(4); - let (update_model_tx, update_model_rx) = chan::bounded(1); - let (shutdown_tx, shutdown_rx) = chan::bounded(1); - - let batch_supervisor_handle = tokio::spawn({ - let to_resume_batches = Arc::clone(&to_resume_batches); - async move { - loop { - let handle = tokio::spawn(actor_loop( - Arc::clone(&model_and_session), - new_batches_rx.clone(), - resume_batch_rx.clone(), - update_model_rx.clone(), - shutdown_rx.clone(), - Arc::clone(&to_resume_batches), - )); - - if let Err(e) = handle.await { - error!("Batch processor panicked: {e:#?}; restarting..."); - } else { - // process_batches exited normally, so we can exit as well - break; - } - } - } - }); - - Ok(Self { - to_resume_batches_file_path, - new_batches_tx, - resume_batch_tx, - update_model_tx, - shutdown_tx, - to_resume_batches, - handle: RefCell::new(Some(batch_supervisor_handle)), - }) - } - - #[allow(clippy::too_many_arguments)] - async fn new_batch_inner( - &self, - location_id: location::id::Type, - location_path: PathBuf, - device_id: device::id::Type, - file_paths: Vec, - db: Arc, - sync: SyncManager, - is_resumable: bool, - ) -> (BatchToken, chan::Receiver) { - let (tx, rx) = chan::bounded(usize::max(file_paths.len(), 1)); - let token = Uuid::new_v4(); - if !file_paths.is_empty() { - if self - .new_batches_tx - .send(Batch { - token, - location_id, - location_path, - device_id, - file_paths, - output_tx: tx, - is_resumable, - db, - sync, - }) - .await - .is_err() - { - error!("Failed to send batch to image labeller"); - } - } else { - // If there are no files to process, we close the channel immediately so the receiver - // side will never wait for a message - tx.close(); - } - - (token, rx) - } - - pub async fn new_batch( - &self, - location_id: location::id::Type, - device_id: device::id::Type, - location_path: PathBuf, - file_paths: Vec, - db: Arc, - sync: SyncManager, - ) -> chan::Receiver { - self.new_batch_inner( - location_id, - location_path, - device_id, - file_paths, - db, - sync, - false, - ) - .await - .1 - } - - /// Resumable batches have lower priority than normal batches - pub async fn new_resumable_batch( - &self, - location_id: location::id::Type, - location_path: PathBuf, - device_id: device::id::Type, - file_paths: Vec, - db: Arc, - sync: SyncManager, - ) -> (BatchToken, chan::Receiver) { - self.new_batch_inner( - location_id, - location_path, - device_id, - file_paths, - db, - sync, - true, - ) - .await - } - - pub async fn change_model(&self, model: Box) -> Result<(), ImageLabelerError> { - let (tx, rx) = oneshot::channel(); - - if self.update_model_tx.send((model, tx)).await.is_err() { - error!("Failed to send model update to image labeller"); - } - - rx.await - .expect("model update result channel unexpectedly closed") - } - - pub async fn shutdown(&self) { - debug!("Shutting down image labeller"); - - let (tx, rx) = oneshot::channel(); - - self.new_batches_tx.close(); - self.resume_batch_tx.close(); - self.update_model_tx.close(); - - if self.shutdown_tx.send(tx).await.is_err() { - error!("Failed to send stop signal to image labeller model executor"); - } - - self.shutdown_tx.close(); - - rx.await - .expect("critical error: image labeller shutdown result channel unexpectedly closed"); - - if let Some(handle) = self - .handle - .try_borrow_mut() - .ok() - .and_then(|mut maybe_handle| maybe_handle.take()) - { - if let Err(e) = handle.await { - error!("Failed to join image labeller supervisors: {e:#?}"); - } - } - - let to_resume_batches = self.to_resume_batches.read().await; - - if !to_resume_batches.is_empty() { - if let Ok(pending_batches) = rmp_serde::to_vec_named(to_resume_batches.deref()) - .map_err(|e| error!("{:#?}", ImageLabelerError::from(e))) - { - if let Err(e) = fs::write(&self.to_resume_batches_file_path, &pending_batches).await - { - error!( - "{:#?}", - ImageLabelerError::from(FileIOError::from(( - &self.to_resume_batches_file_path, - e, - "Failed to write to resume batches file", - ))) - ); - } - } - } - } - - pub async fn resume_batch( - &self, - token: BatchToken, - db: Arc, - sync: SyncManager, - ) -> Result, ImageLabelerError> { - let (tx, rx) = oneshot::channel(); - - self.resume_batch_tx - .send((token, db, sync, tx)) - .await - .expect("critical error: image labeler communication channel unexpectedly closed"); - - rx.await - .expect("critical error: image labeler resume batch result channel unexpectedly closed") - } -} - -/// SAFETY: Due to usage of refcell we lost `Sync` impl, but we only use it to have a shutdown method -/// receiving `&self` which is called once, and we also use `try_borrow_mut` so we never panic -unsafe impl Sync for OldImageLabeler {} - -async fn actor_loop( - model_and_session: Arc>, - new_batches_rx: chan::Receiver, - resume_batch_rx: chan::Receiver, - update_model_rx: chan::Receiver, - shutdown_rx: chan::Receiver>, - to_resume_batches: Arc>>, -) { - let (done_tx, done_rx) = chan::bounded(1); - let (stop_tx, stop_rx) = chan::bounded(1); - - let new_batches_rx_for_shutdown = new_batches_rx.clone(); - - // // TODO: Make this configurable! - // let available_parallelism = std::thread::available_parallelism().map_or_else( - // |e| { - // error!("Failed to get available parallelism: {e:#?}"); - // 1 - // }, - // // Using 25% of available parallelism - // |non_zero| usize::max(non_zero.get() / 4, 1), - // ); - - let available_parallelism = 1; - - info!( - "Image labeler available parallelism: {} cores", - available_parallelism - ); - - enum StreamMessage { - NewBatch(Batch), - ResumeBatch( - BatchToken, - Arc, - SyncManager, - oneshot::Sender, ImageLabelerError>>, - ), - UpdateModel( - Box, - oneshot::Sender>, - ), - BatchDone(FinishStatus), - Shutdown(oneshot::Sender<()>), - } - - let mut queue = VecDeque::with_capacity(16); - - let mut currently_processing = None; - - let mut msg_stream = pin!(( - new_batches_rx.map(StreamMessage::NewBatch), - resume_batch_rx - .map(|(token, db, sync, done_tx)| StreamMessage::ResumeBatch(token, db, sync, done_tx)), - update_model_rx.map(|(model, done_tx)| StreamMessage::UpdateModel(model, done_tx)), - done_rx.clone().map(StreamMessage::BatchDone), - shutdown_rx.map(StreamMessage::Shutdown) - ) - .merge()); - - while let Some(msg) = msg_stream.next().await { - match msg { - StreamMessage::NewBatch(batch @ Batch { is_resumable, .. }) => { - if currently_processing.is_none() { - currently_processing = Some(spawn(spawned_processing( - Arc::clone(&model_and_session), - batch, - available_parallelism, - stop_rx.clone(), - done_tx.clone(), - ))); - } else if !is_resumable { - // TODO: Maybe we should cancel the current batch and start this one instead? - queue.push_front(batch) - } else { - queue.push_back(batch) - } - } - - StreamMessage::ResumeBatch(token, db, sync, resume_done_tx) => { - let resume_result = if let Some((batch, output_rx)) = - to_resume_batches.write().await.remove(&token).map( - |ResumableBatch { - location_id, - device_id, - location_path, - file_paths, - }| { - let (output_tx, output_rx) = - chan::bounded(usize::max(file_paths.len(), 1)); - ( - Batch { - token, - db, - sync, - device_id, - output_tx, - location_id, - location_path, - file_paths, - is_resumable: true, - }, - output_rx, - ) - }, - ) { - if currently_processing.is_none() { - currently_processing = Some(spawn(spawned_processing( - Arc::clone(&model_and_session), - batch, - available_parallelism, - stop_rx.clone(), - done_tx.clone(), - ))); - } else { - queue.push_back(batch) - } - - Ok(output_rx) - } else { - Err(ImageLabelerError::TokenNotFound(token)) - }; - - if resume_done_tx.send(resume_result).is_err() { - error!("Failed to send batch resume result from image labeller"); - } - } - - StreamMessage::UpdateModel(new_model, update_done_tx) => { - if currently_processing.is_some() { - let (tx, rx) = oneshot::channel(); - - stop_tx.send(tx).await.expect("stop_tx unexpectedly closed"); - - if timeout(ONE_SEC, rx).await.is_err() { - error!("Failed to stop image labeller batch processor"); - if stop_rx.is_full() { - stop_rx.recv().await.ok(); - } - } - } - - if update_done_tx - .send( - model_and_session - .write() - .await - .update_model(new_model) - .await, - ) - .is_err() - { - error!("Failed to send model update result from image labeller"); - } - } - - StreamMessage::BatchDone(FinishStatus::Interrupted(batch)) => { - if currently_processing.is_none() { - currently_processing = Some(spawn(spawned_processing( - Arc::clone(&model_and_session), - batch, - 1, - stop_rx.clone(), - done_tx.clone(), - ))); - } else { - queue.push_front(batch); - } - } - - StreamMessage::BatchDone(FinishStatus::Done(token, output_tx)) => { - debug!("Batch done"); - - if let Some(handle) = currently_processing.take() { - if let Err(e) = handle.await { - error!("Failed to join image labeller batch processor: {e:#?}"); - } - } - - output_tx.close(); // So our listener can exit - - if let Some(next_batch) = queue.pop_front() { - currently_processing = Some(spawn(spawned_processing( - Arc::clone(&model_and_session), - next_batch, - 4, - stop_rx.clone(), - done_tx.clone(), - ))); - } - } - - StreamMessage::Shutdown(shutdown_done_tx) => { - debug!("Shutting down image labeller batch processor"); - - if let Some(handle) = currently_processing.take() { - let (tx, rx) = oneshot::channel(); - - stop_tx.send(tx).await.expect("stop_tx unexpectedly closed"); - - if timeout(ONE_SEC * 5, rx).await.is_err() { - error!("Failed to stop image labeller batch processor"); - if stop_rx.is_full() { - stop_rx.recv().await.ok(); - } - } - - if let Err(e) = handle.await { - error!("Failed to join image labeller batch processor: {e:#?}"); - } - - if let Ok(FinishStatus::Interrupted(batch)) = done_rx.recv().await { - queue.push_front(batch); - } - } - - let pending_batches = new_batches_rx_for_shutdown - .filter_map( - |Batch { - token, - location_id, - location_path, - device_id, - file_paths, - is_resumable, - .. - }| async move { - is_resumable.then_some(( - token, - ResumableBatch { - location_id, - location_path, - device_id, - file_paths, - }, - )) - }, - ) - .collect::>() - .await; - - to_resume_batches.write().await.extend( - queue - .into_iter() - .filter_map( - |Batch { - token, - location_id, - location_path, - device_id, - file_paths, - is_resumable, - .. - }| { - is_resumable.then_some(( - token, - ResumableBatch { - location_id, - location_path, - device_id, - file_paths, - }, - )) - }, - ) - .chain(pending_batches.into_iter()), - ); - - shutdown_done_tx - .send(()) - .expect("shutdown_done_tx unexpectedly closed"); - - break; - } - } - } -} diff --git a/crates/ai/src/old_image_labeler/process.rs b/crates/ai/src/old_image_labeler/process.rs deleted file mode 100644 index 0232afa79..000000000 --- a/crates/ai/src/old_image_labeler/process.rs +++ /dev/null @@ -1,507 +0,0 @@ -use sd_core_file_path_helper::IsolatedFilePathData; -use sd_core_prisma_helpers::file_path_for_media_processor; -use sd_core_sync::SyncManager; - -use sd_prisma::{ - prisma::{device, file_path, label, label_on_object, object, PrismaClient}, - prisma_sync, -}; -use sd_sync::OperationFactory; -use sd_utils::{db::MissingFieldError, error::FileIOError, msgpack}; - -use std::{ - collections::{BTreeMap, HashMap, HashSet, VecDeque}, - path::{Path, PathBuf}, - sync::Arc, -}; - -use async_channel as chan; -use chrono::{DateTime, FixedOffset, Utc}; -use futures_concurrency::future::{Join, Race}; -use image::ImageFormat; -use tokio::{ - fs, spawn, - sync::{oneshot, OwnedRwLockReadGuard, OwnedSemaphorePermit, RwLock, Semaphore}, -}; -use tracing::{error, warn}; - -use super::{ - model::ModelAndSession, old_actor::Batch, BatchToken, ImageLabelerError, LabelerOutput, -}; - -const MAX_FILE_SIZE: u64 = 100 * 1024 * 1024; // 100 MB - -async fn reject_all_no_model( - file_paths: Vec, - output_tx: &chan::Sender, -) { - file_paths - .into_iter() - .map( - |file_path_for_media_processor::Data { id, .. }| async move { - if output_tx - .send(LabelerOutput { - file_path_id: id, - has_new_labels: false, - result: Err(ImageLabelerError::NoModelAvailable), - }) - .await - .is_err() - { - error!( - "Failed to send batch output with no model error, " - ); - } - }, - ) - .collect::>() - .join() - .await; -} - -pub(super) enum FinishStatus { - Interrupted(Batch), - Done(BatchToken, chan::Sender), -} - -pub(super) async fn spawned_processing( - model_and_session: Arc>, - Batch { - token, - location_id, - location_path, - device_id, - file_paths, - output_tx, - db, - sync, - is_resumable, - }: Batch, - available_parallelism: usize, - stop_rx: chan::Receiver>, - done_tx: chan::Sender, -) { - let mut errors = Vec::new(); - - // We're already discarding failed ones, so we don't need to keep track of them - let mut queue = file_paths - .into_iter() - .filter_map(|file_path| { - if file_path.object.is_none() { - errors.push(( - file_path.id, - ImageLabelerError::IsolateFilePathData(MissingFieldError::new( - "file_path.object_id", - )), - )); - - return None; - } - - let file_path_id = file_path.id; - let Ok(iso_file_path) = IsolatedFilePathData::try_from((location_id, &file_path)) - .map_err(|e| { - errors.push((file_path_id, e.into())); - }) - else { - return None; - }; - - match ImageFormat::from_extension(iso_file_path.extension()) { - Some(format) => { - let path = location_path.join(&iso_file_path); - Some((file_path, path, format)) - } - None => { - errors.push(( - file_path_id, - ImageLabelerError::UnsupportedExtension( - file_path_id, - iso_file_path.extension().to_owned(), - ), - )); - - None - } - } - }) - .collect::>(); - - errors - .into_iter() - .map(|(file_path_id, error)| { - let output_tx = &output_tx; - async move { - if output_tx - .send(LabelerOutput { - file_path_id, - has_new_labels: false, - result: Err(error), - }) - .await - .is_err() - { - error!( - "Failed to send batch output with errors, " - ); - } - } - }) - .collect::>() - .join() - .await; - - if queue.is_empty() { - done_tx - .send(FinishStatus::Done(token, output_tx)) - .await - .expect("done_tx unexpectedly closed"); - return; - } - - let semaphore = Arc::new(Semaphore::new(available_parallelism)); - - // From this point onwards, we lock the model in read mode - let model_and_session = Arc::new(model_and_session.read_owned().await); - - if !model_and_session.can_process() { - reject_all_no_model( - queue - .into_iter() - .map(|(file_path, _, _)| file_path) - .collect(), - &output_tx, - ) - .await; - done_tx - .send(FinishStatus::Done(token, output_tx)) - .await - .expect("done_tx unexpectedly closed"); - return; - } - - enum RaceOutput { - Done, - Stop(oneshot::Sender<()>), - } - - let mut handles = Vec::with_capacity(queue.len()); - - let mut on_flight = HashMap::with_capacity(queue.len()); - - let (completed_tx, completed_rx) = chan::bounded(queue.len()); - - let (finish_status, maybe_interrupted_tx) = if let RaceOutput::Stop(tx) = ( - async { - while !queue.is_empty() { - let (file_path, path, format) = queue.pop_front().expect("queue is not empty"); - - let permit = Arc::clone(&semaphore) - .acquire_owned() - .await - .expect("semaphore unexpectedly closed"); - - let ids = ( - file_path.id, - file_path.object.as_ref().expect("already checked above").id, - device_id, - ); - - if output_tx.is_closed() { - warn!("Image labeler output channel was closed, dropping current batch..."); - queue.clear(); - on_flight.clear(); - - break; - } - - on_flight.insert(file_path.id, file_path); - - handles.push(spawn(spawned_process_single_file( - Arc::clone(&model_and_session), - ids, - path, - format, - (output_tx.clone(), completed_tx.clone()), - Arc::clone(&db), - sync.clone(), - permit, - ))); - } - - RaceOutput::Done - }, - async { RaceOutput::Stop(stop_rx.recv().await.expect("stop_rx unexpectedly closed")) }, - ) - .race() - .await - { - for handle in &handles { - handle.abort(); - } - - completed_tx.close(); - - while let Ok(file_path_id) = completed_rx.recv().await { - on_flight.remove(&file_path_id); - } - - let status = if queue.is_empty() && on_flight.is_empty() { - FinishStatus::Done(token, output_tx) - } else { - FinishStatus::Interrupted(Batch { - token, - location_id, - location_path, - device_id, - file_paths: on_flight - .into_values() - .chain(queue.into_iter().map(|(file_path, _, _)| file_path)) - .collect(), - output_tx, - db, - sync: sync.clone(), - is_resumable, - }) - }; - - (status, Some(tx)) - } else { - (FinishStatus::Done(token, output_tx), None) - }; - - if let Some(tx) = maybe_interrupted_tx { - if let Err(e) = tx.send(()) { - error!("Failed to send stop signal to image labeller batch processor: {e:#?}"); - } - } else { - handles - .into_iter() - .map(|handle| async move { - if let Err(e) = handle.await { - error!("Failed to join image labeller batch processor: {e:#?}"); - } - }) - .collect::>() - .join() - .await; - } - - done_tx - .send(finish_status) - .await - .expect("critical error: image labeller batch processor unexpectedly closed"); -} - -#[allow(clippy::too_many_arguments)] -async fn spawned_process_single_file( - model_and_session: Arc>, - (file_path_id, object_id, device_id): (file_path::id::Type, object::id::Type, device::id::Type), - path: PathBuf, - format: ImageFormat, - (output_tx, completed_tx): ( - chan::Sender, - chan::Sender, - ), - db: Arc, - sync: SyncManager, - _permit: OwnedSemaphorePermit, -) { - let image = - match extract_file_data(file_path_id, &path).await { - Ok(image) => image, - Err(e) => { - if output_tx - .send(LabelerOutput { - file_path_id, - has_new_labels: false, - result: Err(e), - }) - .await - .is_err() - { - error!("Failed to send batch output with I/O errors, "); - } - - if completed_tx.send(file_path_id).await.is_err() { - warn!("Failed to send completed file path id, ") - } - - return; - } - }; - - let labels = match model_and_session.process_single_image(path.as_path(), image, format) { - Ok(labels) => labels, - Err(e) => { - if output_tx - .send(LabelerOutput { - file_path_id, - has_new_labels: false, - result: Err(e), - }) - .await - .is_err() - { - error!("Failed to send batch output with model processing errors, "); - } - - if completed_tx.send(file_path_id).await.is_err() { - warn!("Failed to send completed file path id, ") - } - - return; - } - }; - - let (has_new_labels, result) = - match assign_labels(object_id, device_id, labels, &db, &sync).await { - Ok(has_new_labels) => (has_new_labels, Ok(())), - Err(e) => (false, Err(e)), - }; - - if output_tx - .send(LabelerOutput { - file_path_id, - has_new_labels, - result, - }) - .await - .is_err() - { - error!("Failed to send batch output with database assign label results, "); - } - - if completed_tx.send(file_path_id).await.is_err() { - warn!("Failed to send completed file path id, ") - } -} - -async fn extract_file_data( - file_path_id: file_path::id::Type, - path: impl AsRef, -) -> Result, ImageLabelerError> { - let path = path.as_ref(); - - let metadata = fs::metadata(path).await.map_err(|e| { - FileIOError::from((path, e, "Failed to get metadata for file to get labels")) - })?; - - if metadata.len() > MAX_FILE_SIZE { - return Err(ImageLabelerError::FileTooBig( - file_path_id, - metadata.len() as usize, - )); - } - - fs::read(path) - .await - .map_err(|e| FileIOError::from((path, e, "Failed to read file to get labels")).into()) -} - -pub async fn assign_labels( - object_id: object::id::Type, - device_id: device::id::Type, - mut labels: HashSet, - db: &PrismaClient, - sync: &SyncManager, -) -> Result { - let object = db - .object() - .find_unique(object::id::equals(object_id)) - .select(object::select!({ pub_id })) - .exec() - .await? - .unwrap(); - - let mut has_new_labels = false; - - let mut labels_ids = db - .label() - .find_many(vec![label::name::in_vec(labels.iter().cloned().collect())]) - .select(label::select!({ id name })) - .exec() - .await? - .into_iter() - .map(|label| { - labels.remove(&label.name); - - (label.id, label.name) - }) - .collect::>(); - - let date_created: DateTime = Utc::now().into(); - - if !labels.is_empty() { - let mut sync_params = Vec::with_capacity(labels.len() * 2); - - let db_params = labels - .into_iter() - .map(|name| { - sync_params.push(sync.shared_create( - prisma_sync::label::SyncId { name: name.clone() }, - [(label::date_created::NAME, msgpack!(&date_created))], - )); - - db.label() - .create(name, vec![label::date_created::set(Some(date_created))]) - .select(label::select!({ id name })) - }) - .collect::>(); - - labels_ids.extend( - sync.write_ops(db, (sync_params, db_params)) - .await? - .into_iter() - .map(|l| (l.id, l.name)), - ); - - has_new_labels = true; - } - - let mut sync_params = Vec::with_capacity(labels_ids.len() * 2); - - if !labels_ids.is_empty() { - let db_params: Vec<_> = labels_ids - .into_iter() - .map(|(label_id, name)| { - sync_params.push(sync.relation_create( - prisma_sync::label_on_object::SyncId { - label: prisma_sync::label::SyncId { name }, - object: prisma_sync::object::SyncId { - pub_id: object.pub_id.clone(), - }, - }, - [( - label_on_object::device::NAME, - msgpack!(prisma_sync::device::SyncId { - pub_id: sync.device_pub_id.to_db(), - }), - )], - )); - - label_on_object::create_unchecked( - label_id, - object_id, - vec![ - label_on_object::date_created::set(date_created), - label_on_object::device_id::set(Some(device_id)), - ], - ) - }) - .collect(); - - sync.write_ops( - db, - ( - sync_params, - db.label_on_object() - .create_many(db_params) - .skip_duplicates(), - ), - ) - .await?; - } - - Ok(has_new_labels) -} diff --git a/crates/ai/src/utils/mod.rs b/crates/ai/src/utils/mod.rs deleted file mode 100644 index 9cefe9d69..000000000 --- a/crates/ai/src/utils/mod.rs +++ /dev/null @@ -1,27 +0,0 @@ -use std::{ - env::{args_os, current_exe}, - path::{Path, PathBuf}, -}; -use tracing::error; - -pub(crate) fn get_path_relative_to_exe(path: impl AsRef) -> PathBuf { - current_exe() - .unwrap_or_else(|e| { - error!("Failed to get current exe path: {e:#?}"); - args_os() - .next() - .expect("there is always the first arg") - .into() - }) - .parent() - .map_or_else( - || path.as_ref().to_path_buf(), - |parent| { - let path = parent.join(path.as_ref()); - path.canonicalize().unwrap_or_else(|e| { - error!("Failed to canonicalize relative path to exe, return raw path and hope: {e:#?}"); - path - }) - }, - ) -} diff --git a/crates/file-ext/Cargo.toml b/crates/file-ext/Cargo.toml deleted file mode 100644 index b43b35907..000000000 --- a/crates/file-ext/Cargo.toml +++ /dev/null @@ -1,20 +0,0 @@ -[package] -name = "sd-file-ext" -version = "0.0.0" - -authors = ["Brendan Allen ", "Jamie Pine "] -edition.workspace = true -license.workspace = true -repository.workspace = true - -[dependencies] -# Workspace dependencies -serde = { workspace = true, features = ["derive"] } -serde_json = { workspace = true } -specta = { workspace = true } -strum = { workspace = true, features = ["derive"] } -strum_macros = { workspace = true } -tokio = { workspace = true, features = ["fs", "io-util", "rt"] } - -[dev-dependencies] -tokio = { workspace = true, features = ["fs", "macros", "rt"] } diff --git a/crates/file-ext/src/extensions.rs b/crates/file-ext/src/extensions.rs deleted file mode 100644 index 4365032bb..000000000 --- a/crates/file-ext/src/extensions.rs +++ /dev/null @@ -1,566 +0,0 @@ -/// -/// References: -/// https://www.garykessler.net/library/file_sigs.html -/// https://github.com/bojand/infer/ -/// https://github.com/features/copilot -/// -use crate::magic::{ - extension_category_enum, extension_enum, ExtensionPossibility, MagicBytes, MagicBytesMeta, -}; - -extension_enum! { - Extension { - Document(DocumentExtension), - Video(VideoExtension), - Image(ImageExtension), - Audio(AudioExtension), - Archive(ArchiveExtension), - Executable(ExecutableExtension), - Text(TextExtension), - Encrypted(EncryptedExtension), - Key(KeyExtension), - Font(FontExtension), - Mesh(MeshExtension), - Code(CodeExtension), - Database(DatabaseExtension), - Book(BookExtension), - Config(ConfigExtension), - } -} - -// video extensions -extension_category_enum! { - VideoExtension ALL_VIDEO_EXTENSIONS { - Avi = [0x52, 0x49, 0x46, 0x46, _, _, _, _, 0x41, 0x56, 0x49, 0x20], - Avifs = [], - Qt = [0x71, 0x74, 0x20, 0x20], - Mov = [0x66, 0x74, 0x79, 0x70, 0x71, 0x74, 0x20, 0x20] + 4, - Swf = [0x5A, 0x57, 0x53] | [0x46, 0x57, 0x53], - Mjpeg = [], - Ts = [0x47], - Mts = [0x47] | [_, _, _, 0x47], - Mpeg = [0x47] | [0x00, 0x00, 0x01, 0xBA] | [0x00, 0x00, 0x01, 0xB3], - Mxf = [0x06, 0x0E, 0x2B, 0x34, 0x02, 0x05, 0x01, 0x01, 0x0D, 0x01, 0x02, 0x01, 0x01, 0x02], - M2v = [0x00, 0x00, 0x01, 0xBA], - Mpg = [], - Mpe = [], - M2ts = [], - Flv = [0x46, 0x4C, 0x56], - Wm = [], - #[serde(rename = "3gp")] - #[strum(serialize = "3gp")] - _3gp = [], - M4v = [0x66, 0x74, 0x79, 0x70, 0x4D, 0x34, 0x56] + 4, - Wmv = [0x30, 0x26, 0xB2, 0x75, 0x8E, 0x66, 0xCF, 0x11, 0xA6, 0xD9, 0x00, 0xAA, 0x00, 0x62, 0xCE, 0x6C], - Asf = [0x30, 0x26, 0xB2, 0x75, 0x8E, 0x66, 0xCF, 0x11, 0xA6, 0xD9, 0x00, 0xAA, 0x00, 0x62, 0xCE, 0x6C], - Mp4 = [], - Webm = [0x1A, 0x45, 0xDF, 0xA3], - Mkv = [0x1A, 0x45, 0xDF, 0xA3], - Vob = [0x00, 0x00, 0x01, 0xBA], - Ogv = [0x4F, 0x67, 0x67, 0x53], - Wtv = [0xB7, 0xD8, 0x00], - Hevc = [], - F4v = [0x66, 0x74, 0x79, 0x70, 0x66, 0x72, 0x65, 0x65] + 4, - } -} - -// image extensions -extension_category_enum! { - ImageExtension ALL_IMAGE_EXTENSIONS { - Jpg = [0xFF, 0xD8], - Jpeg = [0xFF, 0xD8], - Png = [0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A], - Apng = [0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A, 0x00, 0x00, 0x00, 0x0D, 0x49, 0x48, 0x44, 0x52], - Gif = [0x47, 0x49, 0x46, 0x38, _, 0x61], - Bmp = [0x42, 0x4D], - Tiff = [0x49, 0x49, 0x2A, 0x00], - Webp = [0x52, 0x49, 0x46, 0x46, _, _, _, _, 0x57, 0x45, 0x42, 0x50], - Svg = [0x3C, 0x73, 0x76, 0x67], - Ico = [0x00, 0x00, 0x01, 0x00], - Heic = [0x00, 0x00, 0x00, 0x18, 0x66, 0x74, 0x79, 0x70, 0x68, 0x65, 0x69, 0x63], - Heics = [0x00, 0x00, 0x00, 0x18, 0x66, 0x74, 0x79, 0x70, 0x68, 0x65, 0x69, 0x63], - Heif = [], - Heifs = [], - Hif = [], - Avif = [], - Avci = [], - Avcs = [], - Raw = [], - Akw = [0x41, 0x4B, 0x57, 0x42], - Dng = [0x49, 0x49, 0x2A, 0x00, 0x08, 0x00, 0x00, 0x00, 0x44, 0x4E, 0x47, 0x00], - Cr2 = [0x49, 0x49, 0x2A, 0x00, 0x10, 0x00, 0x00, 0x00, 0x43, 0x52, 0x02, 0x00], - Dcr = [0x49, 0x49, 0x2A, 0x00, 0x10, 0x00, 0x00, 0x00, 0x44, 0x43, 0x52, 0x00], - Nwr = [0x49, 0x49, 0x2A, 0x00, 0x10, 0x00, 0x00, 0x00, 0x4E, 0x57, 0x52, 0x00], - Nef = [0x49, 0x49, 0x2A, 0x00, 0x08, 0x00, 0x00, 0x00, 0x4E, 0x45, 0x46, 0x00], - Arw = [0x49, 0x49, 0x2A, 0x00, 0x08], - Rw2 = [0x49, 0x49, 0x2A, 0x00, 0x18], - } -} - -// audio extensions -extension_category_enum! { - AudioExtension ALL_AUDIO_EXTENSIONS { - Mp3 = [0x49, 0x44, 0x33], - Mp2 = [0xFF, 0xFB] | [0xFF, 0xFD], - M4a = [0x66, 0x74, 0x79, 0x70, 0x4D, 0x34, 0x41, 0x20] + 4, - Wav = [0x52, 0x49, 0x46, 0x46, _, _, _, _, 0x57, 0x41, 0x56, 0x45], - Aiff = [0x46, 0x4F, 0x52, 0x4D, _, _, _, _, 0x41, 0x49, 0x46, 0x46], - Aif = [0x46, 0x4F, 0x52, 0x4D, _, _, _, _, 0x41, 0x49, 0x46, 0x46], - Flac = [0x66, 0x4C, 0x61, 0x43], - Ogg = [0x4F, 0x67, 0x67, 0x53], - Oga = [0x4F, 0x67, 0x67, 0x53], - Opus = [0x4F, 0x70, 0x75, 0x73, 0x48, 0x65, 0x61, 0x64] + 28, - Wma = [0x30, 0x26, 0xB2, 0x75, 0x8E, 0x66, 0xCF, 0x11, 0xA6, 0xD9, 0x00, 0xAA, 0x00, 0x62, 0xCE, 0x6C], - Amr = [0x23, 0x21, 0x41, 0x4D, 0x52], - Aac = [0xFF, 0xF1], - Wv = [0x77, 0x76, 0x70, 0x6B], - Voc = [0x43, 0x72, 0x65, 0x61, 0x74, 0x69, 0x76, 0x65, 0x20, 0x56, 0x6F, 0x69, 0x63, 0x65, 0x20, 0x46, 0x69, 0x6C, 0x65], - Tta = [0x54, 0x54, 0x41], - Loas = [0x56, 0xE0], - Caf = [0x63, 0x61, 0x66, 0x66], - Aptx = [0x4B, 0xBF, 0x4B, 0xBF], - Adts = [0xFF, 0xF1], - Ast = [0x53, 0x54, 0x52, 0x4D], - Mid = [0x4D, 0x54, 0x68, 0x64], - } -} - -// archive extensions -extension_category_enum! { - ArchiveExtension _ALL_ARCHIVE_EXTENSIONS { - Zip = [0x50, 0x4B, 0x03, 0x04], - Rar = [0x52, 0x61, 0x72, 0x21, 0x1A, 0x07, 0x00], - Tar = [0x75, 0x73, 0x74, 0x61, 0x72], - Gz = [0x1F, 0x8B, 0x08], - Bz2 = [0x42, 0x5A, 0x68], - #[serde(rename = "7z")] - #[strum(serialize = "7z")] - _7z = [0x37, 0x7A, 0xBC, 0xAF, 0x27, 0x1C], - Xz = [0xFD, 0x37, 0x7A, 0x58, 0x5A, 0x00], - } -} - -// executable extensions -extension_category_enum! { - ExecutableExtension _ALL_EXECUTABLE_EXTENSIONS { - Exe = [0x4D, 0x5A], - App = [0x4D, 0x5A], - Apk = [0x50, 0x4B, 0x03, 0x04], - Deb = [0x21, 0x3C, 0x61, 0x72, 0x63, 0x68, 0x3E, 0x0A, 0x64, 0x65, 0x62, 0x69, 0x61, 0x6E, 0x2D, 0x62, 0x69, 0x6E, 0x61, 0x72, 0x79], - Dmg = [0x78, 0x01, 0x73, 0x0D, 0x62, 0x62, 0x60], - Pkg = [0x4D, 0x5A], - Rpm = [0xED, 0xAB, 0xEE, 0xDB], - Msi = [0xD0, 0xCF, 0x11, 0xE0, 0xA1, 0xB1, 0x1A, 0xE1], - Jar = [0x50, 0x4B, 0x03, 0x04], - Bat = [], - } -} - -// document extensions -extension_category_enum! { - DocumentExtension ALL_DOCUMENT_EXTENSIONS { - Pdf = [0x25, 0x50, 0x44, 0x46, 0x2D], - Key = [0x50, 0x4B, 0x03, 0x04], - Pages = [0x50, 0x4B, 0x03, 0x04], - Numbers = [0x50, 0x4B, 0x03, 0x04], - Doc = [0xD0, 0xCF, 0x11, 0xE0, 0xA1, 0xB1, 0x1A, 0xE1], - Docx = [0x50, 0x4B, 0x03, 0x04], - Xls = [0xD0, 0xCF, 0x11, 0xE0, 0xA1, 0xB1, 0x1A, 0xE1], - Xlsx = [0x50, 0x4B, 0x03, 0x04], - Ppt = [0xD0, 0xCF, 0x11, 0xE0, 0xA1, 0xB1, 0x1A, 0xE1], - Pptx = [0x50, 0x4B, 0x03, 0x04], - Odt = [0x50, 0x4B, 0x03, 0x04], - Ods = [0x50, 0x4B, 0x03, 0x04], - Odp = [0x50, 0x4B, 0x03, 0x04], - Ics = [0x42, 0x45, 0x47, 0x49, 0x4E, 0x3A, 0x56, 0x43, 0x41, 0x52, 0x44], - Hwp = [0xD0, 0xCF, 0x11, 0xE0, 0xA1, 0xB1, 0x1A, 0xE1], - } -} - -// text file extensions -extension_category_enum! { - TextExtension _ALL_TEXT_EXTENSIONS { - Txt, - Rtf, - Md, - Markdown, - } -} -// config file extensions -extension_category_enum! { - ConfigExtension _ALL_CONFIG_EXTENSIONS { - Ini, - Json, - Yaml, - Yml, - Toml, - Xml, - Mathml, - Rss, - Csv, - Cfg, - Compose, - Tsconfig, - } -} - -// encrypted file extensions -extension_category_enum! { - EncryptedExtension _ALL_ENCRYPTED_EXTENSIONS { - // Spacedrive encrypted file - Bytes = [0x62, 0x61, 0x6C, 0x6C, 0x61, 0x70, 0x70], - // Spacedrive container - Container = [0x73, 0x64, 0x62, 0x6F, 0x78], - // Spacedrive block storage, - Block = [0x73, 0x64, 0x62, 0x6C, 0x6F, 0x63, 0x6B], - } -} - -// key extensions -extension_category_enum! { - KeyExtension _ALL_KEY_EXTENSIONS { - Pgp, - Pub, - Pem, - P12, - P8, - Keychain, - } -} - -// font extensions -extension_category_enum! { - FontExtension _ALL_FONT_EXTENSIONS { - Ttf = [0x00, 0x01, 0x00, 0x00, 0x00], - Otf = [0x4F, 0x54, 0x54, 0x4F, 0x00], - Woff = [0x77, 0x4F, 0x46, 0x46], - Woff2 = [0x77, 0x4F, 0x46, 0x32], - } -} - -// font extensions -extension_category_enum! { - MeshExtension _ALL_MESH_EXTENSIONS { - Fbx = [0x46, 0x42, 0x58, 0x20], - Obj = [0x6F, 0x62, 0x6A], - } -} - -// code extensions -extension_category_enum! { - CodeExtension _ALL_CODE_EXTENSIONS { - // AppleScript - Scpt, - Scptd, - Applescript, - // Shell script - Sh, - Zsh, - Fish, - Bash, - // C, C++ - C, - Cpp, - H, - Hpp, - // Ruby - Rb, - // Javascript - Js, - Mjs, - Jsx, - // Markup - Html, - // Stylesheet - Css, - Sass, - Scss, - Less, - // Crystal - Cr, - // C# - Cs, - Csx, - D, - Dart, - // Docker - Dockerfile, - Go, - // Haskell - Hs, - Java, - // Kotlin - Kt, - Kts, - Lua, - // Makefile - Make, - Nim, - Nims, - // Objective-C - M, - Mm, - // Ocaml - Ml, - Mli, - Mll, - Mly, - // Perl - Pl, - // PHP - Php, - Php1, - Php2, - Php3, - Php4, - Php5, - Php6, - Phps, - Phpt, - Phtml, - // Powershell - Ps1, - Psd1, - Psm1, - // Python - Py, - Qml, - R, - // Rust - Rs, - // Solidity - Sol, - Sql, - Swift, - // Typescript - Ts, - Tsx, - Vala, - Zig, - Vue, - Scala, - Mdx, - Astro, - Mts, - } -} - -// database extensions -extension_category_enum! { - DatabaseExtension _ALL_DATABASE_EXTENSIONS { - Sqlite = [0x53, 0x51, 0x4C, 0x69, 0x74, 0x65, 0x20, 0x66, 0x6F, 0x72, 0x6D, 0x61, 0x74, 0x20, 0x33, 0x00], - Db = [], - } -} - -// book extensions -extension_category_enum! { - BookExtension _ALL_BOOK_EXTENSIONS { - Azw = [0x52, 0x49, 0x46, 0x46], - Azw3 = [0x52, 0x49, 0x46, 0x46], - Epub = [0x50, 0x4B, 0x03, 0x04], - Mobi = [0x4D, 0x4F, 0x42, 0x49], - } -} - -#[cfg(test)] -mod test { - - use super::*; - - #[test] - fn extension_from_str() { - // single extension match - assert_eq!( - Extension::from_str("jpg"), - Some(ExtensionPossibility::Known(Extension::Image( - ImageExtension::Jpg - ))) - ); - // with conflicts - assert_eq!( - Extension::from_str("ts"), - Some(ExtensionPossibility::Conflicts(vec![ - Extension::Video(VideoExtension::Ts), - Extension::Code(CodeExtension::Ts) - ])) - ); - // invalid case - assert_eq!(Extension::from_str("jeff"), None); - } - - #[tokio::test] - async fn magic_bytes() { - async fn test_path(subpath: &str) -> Option { - println!("testing {subpath}..."); - Extension::resolve_conflicting(subpath.split('.').last().unwrap(), true).await - } - // Video extension tests - assert_eq!( - dbg!(test_path("video/video.ts").await), - Some(Extension::Video(VideoExtension::Ts)) - ); - assert_eq!( - dbg!(test_path("code/typescript.ts").await), - Some(Extension::Code(CodeExtension::Ts)) - ); - assert_eq!( - dbg!(test_path("video/video.3gp").await), - Some(Extension::Video(VideoExtension::_3gp)) - ); - assert_eq!( - dbg!(test_path("video/video.mov").await), - Some(Extension::Video(VideoExtension::Mov)) - ); - assert_eq!( - dbg!(test_path("video/video.asf").await), - Some(Extension::Video(VideoExtension::Asf)) - ); - assert_eq!( - dbg!(test_path("video/video.avi").await), - Some(Extension::Video(VideoExtension::Avi)) - ); - assert_eq!( - dbg!(test_path("video/video.flv").await), - Some(Extension::Video(VideoExtension::Flv)) - ); - assert_eq!( - dbg!(test_path("video/video.m4v").await), - Some(Extension::Video(VideoExtension::M4v)) - ); - assert_eq!( - dbg!(test_path("video/video.mkv").await), - Some(Extension::Video(VideoExtension::Mkv)) - ); - assert_eq!( - dbg!(test_path("video/video.mpg").await), - Some(Extension::Video(VideoExtension::Mpg)) - ); - assert_eq!( - dbg!(test_path("video/video.mpeg").await), - Some(Extension::Video(VideoExtension::Mpeg)) - ); - assert_eq!( - dbg!(test_path("video/video.mts").await), - Some(Extension::Video(VideoExtension::Mts)) - ); - assert_eq!( - dbg!(test_path("video/video.mxf").await), - Some(Extension::Video(VideoExtension::Mxf)) - ); - assert_eq!( - dbg!(test_path("video/video.ogv").await), - Some(Extension::Video(VideoExtension::Ogv)) - ); - assert_eq!( - dbg!(test_path("video/video.swf").await), - Some(Extension::Video(VideoExtension::Swf)) - ); - assert_eq!( - dbg!(test_path("video/video.ts").await), - Some(Extension::Video(VideoExtension::Ts)) - ); - assert_eq!( - dbg!(test_path("video/video.vob").await), - Some(Extension::Video(VideoExtension::Vob)) - ); - assert_eq!( - dbg!(test_path("video/video.ogv").await), - Some(Extension::Video(VideoExtension::Ogv)) - ); - assert_eq!( - dbg!(test_path("video/video.wmv").await), - Some(Extension::Video(VideoExtension::Wmv)) - ); - assert_eq!( - dbg!(test_path("video/video.wtv").await), - Some(Extension::Video(VideoExtension::Wtv)) - ); - - // Audio extension tests - assert_eq!( - dbg!(test_path("audio/audio.aac").await), - Some(Extension::Audio(AudioExtension::Aac)) - ); - assert_eq!( - dbg!(test_path("audio/audio.adts").await), - Some(Extension::Audio(AudioExtension::Adts)) - ); - assert_eq!( - dbg!(test_path("audio/audio.aif").await), - Some(Extension::Audio(AudioExtension::Aif)) - ); - assert_eq!( - dbg!(test_path("audio/audio.aiff").await), - Some(Extension::Audio(AudioExtension::Aiff)) - ); - assert_eq!( - dbg!(test_path("audio/audio.aptx").await), - Some(Extension::Audio(AudioExtension::Aptx)) - ); - assert_eq!( - dbg!(test_path("audio/audio.ast").await), - Some(Extension::Audio(AudioExtension::Ast)) - ); - assert_eq!( - dbg!(test_path("audio/audio.caf").await), - Some(Extension::Audio(AudioExtension::Caf)) - ); - assert_eq!( - dbg!(test_path("audio/audio.flac").await), - Some(Extension::Audio(AudioExtension::Flac)) - ); - assert_eq!( - dbg!(test_path("audio/audio.loas").await), - Some(Extension::Audio(AudioExtension::Loas)) - ); - assert_eq!( - dbg!(test_path("audio/audio.m4a").await), - Some(Extension::Audio(AudioExtension::M4a)) - ); - // assert_eq!( - // dbg!(test_path("audio/audio.m4b").await), - // Some(Extension::Audio(AudioExtension::M4b)) - // ); - assert_eq!( - dbg!(test_path("audio/audio.mp2").await), - Some(Extension::Audio(AudioExtension::Mp2)) - ); - assert_eq!( - dbg!(test_path("audio/audio.mp3").await), - Some(Extension::Audio(AudioExtension::Mp3)) - ); - assert_eq!( - dbg!(test_path("audio/audio.oga").await), - Some(Extension::Audio(AudioExtension::Oga)) - ); - assert_eq!( - dbg!(test_path("audio/audio.ogg").await), - Some(Extension::Audio(AudioExtension::Ogg)) - ); - assert_eq!( - dbg!(test_path("audio/audio.opus").await), - Some(Extension::Audio(AudioExtension::Opus)) - ); - assert_eq!( - dbg!(test_path("audio/audio.tta").await), - Some(Extension::Audio(AudioExtension::Tta)) - ); - assert_eq!( - dbg!(test_path("audio/audio.voc").await), - Some(Extension::Audio(AudioExtension::Voc)) - ); - assert_eq!( - dbg!(test_path("audio/audio.wav").await), - Some(Extension::Audio(AudioExtension::Wav)) - ); - assert_eq!( - dbg!(test_path("audio/audio.wma").await), - Some(Extension::Audio(AudioExtension::Wma)) - ); - assert_eq!( - dbg!(test_path("audio/audio.wv").await), - Some(Extension::Audio(AudioExtension::Wv)) - ); - } -} diff --git a/crates/file-ext/src/kind.rs b/crates/file-ext/src/kind.rs deleted file mode 100644 index 058e206a4..000000000 --- a/crates/file-ext/src/kind.rs +++ /dev/null @@ -1,62 +0,0 @@ -use serde::{Deserialize, Serialize}; -use specta::Type; -use strum_macros::{Display, EnumIter}; -// Note: The order of this enum should never change, and always be kept in sync with `packages/client/src/utils/objectKind.ts` -#[repr(i32)] -#[derive(Debug, Clone, Display, Copy, EnumIter, Type, Serialize, Deserialize, Eq, PartialEq)] -pub enum ObjectKind { - /// A file that can not be identified by the indexer - Unknown = 0, - /// A known filetype, but without specific support - Document = 1, - /// A virtual filesystem directory - Folder = 2, - /// A file that contains human-readable text - Text = 3, - /// A virtual directory int - Package = 4, - /// An image file - Image = 5, - /// An audio file - Audio = 6, - /// A video file - Video = 7, - /// A compressed archive of data - Archive = 8, - /// An executable, program or application - Executable = 9, - /// A link to another object - Alias = 10, - /// Raw bytes encrypted by Spacedrive with self contained metadata - Encrypted = 11, - /// A key or certificate file - Key = 12, - /// A link can open web pages, apps or Spaces - Link = 13, - /// A special filetype that represents a preserved webpage - WebPageArchive = 14, - /// A widget is a mini app that can be placed in a Space at various sizes, associated Widget struct required - Widget = 15, - /// Albums can only have one level of children, and are associated with the Album struct - Album = 16, - /// Its like a folder, but appears like a stack of files, designed for burst photos / associated groups of files - Collection = 17, - /// You know, text init - Font = 18, - /// 3D Object - Mesh = 19, - /// Editable source code file - Code = 20, - /// Database file - Database = 21, - /// E-book file - Book = 22, - /// Config file - Config = 23, - /// Dotfile - Dotfile = 24, - /// Screenshot - Screenshot = 25, - /// Label - Label = 26, -} diff --git a/crates/file-ext/src/lib.rs b/crates/file-ext/src/lib.rs deleted file mode 100644 index c7d6a2690..000000000 --- a/crates/file-ext/src/lib.rs +++ /dev/null @@ -1,4 +0,0 @@ -pub mod extensions; -pub mod kind; -pub mod magic; -pub mod text; diff --git a/crates/file-ext/src/magic.rs b/crates/file-ext/src/magic.rs deleted file mode 100644 index da858924b..000000000 --- a/crates/file-ext/src/magic.rs +++ /dev/null @@ -1,232 +0,0 @@ -#![allow(dead_code)] - -use crate::extensions::{CodeExtension, Extension, VideoExtension}; -use std::{ffi::OsStr, io::SeekFrom, path::Path}; - -use tokio::{ - fs::File, - io::{AsyncReadExt, AsyncSeekExt}, -}; - -#[derive(Debug, PartialEq, Eq)] -pub enum ExtensionPossibility { - Known(Extension), - Conflicts(Vec), -} - -#[derive(Debug)] -pub struct MagicBytesMeta { - pub offset: usize, - pub length: usize, -} - -pub trait MagicBytes: Sized + PartialEq { - fn has_magic_bytes(&self, buf: &[u8]) -> bool; - fn magic_bytes_meta(&self) -> Vec; -} - -#[macro_export] -macro_rules! magic_byte_value { - (_) => { - 0 as u8 - }; - ($val:literal) => {{ - $val as u8 - }}; -} -// pub(crate) use magic_byte_value; - -#[macro_export] -macro_rules! magic_byte_offset { - () => { - 0 - }; - ($val:literal) => { - $val - }; -} -// pub(crate) use magic_byte_offset; - -macro_rules! extension_enum { - ( - Extension { - $( $variant:ident($type:ident), )* - } - ) => { - // construct enum - #[derive(Debug, ::serde::Serialize, ::serde::Deserialize, PartialEq, Eq, Clone)] - pub enum Extension { - $( $variant($type), )* - } - impl Extension { - #[allow(clippy::should_implement_trait)] - pub fn from_str(s: &str) -> Option { - use std::str::FromStr; - let s = s.to_lowercase(); - - let mut exts = [$( - $type::from_str(&s).ok().map(Self::$variant) - ),*] - .into_iter() - .filter_map(|s| s) - .collect::>(); - - match exts { - _ if exts.len() == 0 => None, - _ if exts.len() == 1 => Some(ExtensionPossibility::Known(exts.swap_remove(0))), - _ => Some(ExtensionPossibility::Conflicts(exts)) - } - } - } - // convert Extension to ObjectKind - impl From for $crate::kind::ObjectKind { - fn from(ext: Extension) -> Self { - match ext { - $( Extension::$variant(_) => $crate::kind::ObjectKind::$variant, )* - } - } - } - // - impl std::fmt::Display for Extension { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - $( Extension::$variant(x) => write!(f, "{}", x), )* - } - } - } - }; -} -pub(crate) use extension_enum; - -/// Define a public enum with static array of all possible variants -/// including implementations to convert to/from string -macro_rules! extension_category_enum { - ( - $(#[$enum_attr:meta])* - $enum_name:ident $static_array_name:ident { - $($(#[$variant_attr:meta])* $variant:ident $(= $( [$($magic_bytes:tt),*] $(+ $offset:literal)? )|+ )? ,)* - } - ) => { - #[derive(Debug, ::serde::Serialize, ::serde::Deserialize, ::strum::Display, ::specta::Type, Clone, Copy, PartialEq, Eq)] - #[serde(rename_all = "snake_case")] - #[strum(serialize_all = "snake_case")] - $(#[$enum_attr])* - - // construct enum - pub enum $enum_name { - $( $(#[$variant_attr])* $variant, )* - } - // a static array of all variants - pub static $static_array_name: &[$enum_name] = &[ - $( $enum_name::$variant, )* - ]; - - $crate::magic::extension_category_enum!(@magic_bytes; $enum_name ( $($(#[$variant_attr])* $variant $(= $( [$($magic_bytes),*] $(+ $offset)? )|+ )? ),* )); - - // convert from string - impl std::str::FromStr for $enum_name { - type Err = serde_json::Error; - fn from_str(s: &str) -> Result { - serde_json::from_value(serde_json::Value::String(s.to_lowercase())) - } - } - }; - - (@magic_bytes; $enum_name:ident ($($(#[$variant_attr:meta])* $variant:ident = $( [$($magic_bytes:tt),*] $(+ $offset:literal)? )|+ ),*)) => { - impl MagicBytes for $enum_name { - fn has_magic_bytes(&self, buf: &[u8]) -> bool { - match (self, buf) { - $( $( ($enum_name::$variant, &[$($magic_bytes,)* ..]) => true, )+ )* - _ => false - } - } - // get offset and length of magic bytes - fn magic_bytes_meta(&self) -> Vec { - match self { - $( $enum_name::$variant => vec![ - $( MagicBytesMeta { - length: (&[$($crate::magic_byte_value!($magic_bytes)),*] as &[u8]).len(), - offset: $crate::magic_byte_offset!($($offset)?), - }, )+ - ] ),* - } - } - } - }; - (@magic_bytes; $enum_name:ident ($($(#[$variant_attr:meta])* $variant:ident),*)) => {}; -} -pub(crate) use extension_category_enum; - -pub async fn verify_magic_bytes(ext: T, file: &mut File) -> Option { - for magic in ext.magic_bytes_meta() { - let mut buf = vec![0; magic.length]; - - file.seek(SeekFrom::Start(magic.offset as u64)).await.ok()?; - file.read_exact(&mut buf).await.ok()?; - - if ext.has_magic_bytes(&buf) { - return Some(ext); - } - } - - None -} - -impl Extension { - pub async fn resolve_conflicting( - path: impl AsRef, - always_check_magic_bytes: bool, - ) -> Option { - let ext_str = path.as_ref().extension().and_then(OsStr::to_str)?; - - let ext = Self::from_str(ext_str)?; - - let Ok(ref mut file) = File::open(&path).await else { - return None; - }; - - match ext { - // we don't need to check the magic bytes unless there is conflict - // always_check_magic_bytes forces the check for tests - ExtensionPossibility::Known(e) => { - if always_check_magic_bytes { - match e { - Self::Image(x) => verify_magic_bytes(x, file).await.map(Self::Image), - Self::Audio(x) => verify_magic_bytes(x, file).await.map(Self::Audio), - Self::Video(x) => verify_magic_bytes(x, file).await.map(Self::Video), - Self::Archive(x) => verify_magic_bytes(x, file).await.map(Self::Archive), - Self::Executable(x) => { - verify_magic_bytes(x, file).await.map(Self::Executable) - } - Self::Font(x) => verify_magic_bytes(x, file).await.map(Self::Font), - Self::Encrypted(x) => { - verify_magic_bytes(x, file).await.map(Self::Encrypted) - } - Self::Mesh(x) => verify_magic_bytes(x, file).await.map(Self::Mesh), - Self::Database(x) => verify_magic_bytes(x, file).await.map(Self::Database), - _ => Some(e), - } - } else { - Some(e) - } - } - ExtensionPossibility::Conflicts(ext) => match ext_str { - "ts" if ext.iter().any(|e| matches!(e, Self::Video(_))) => { - verify_magic_bytes(VideoExtension::Ts, file) - .await - .map_or(Some(Self::Code(CodeExtension::Ts)), |video_ext| { - Some(Self::Video(video_ext)) - }) - } - "mts" if ext.iter().any(|e| matches!(e, Self::Video(_))) => { - verify_magic_bytes(VideoExtension::Mts, file) - .await - .map_or(Some(Self::Code(CodeExtension::Mts)), |video_ext| { - Some(Self::Video(video_ext)) - }) - } - _ => None, - }, - } - } -} diff --git a/crates/file-ext/src/text.rs b/crates/file-ext/src/text.rs deleted file mode 100644 index b555c5960..000000000 --- a/crates/file-ext/src/text.rs +++ /dev/null @@ -1,297 +0,0 @@ -/** - * Based on an excerpt from the File type identification utility by Ian F. Darwin and others - * https://github.com/file/file/blob/445f38730df6a2654eadcc180116035cc6788363/src/encoding.c - */ - -const F: u8 = 0; -const T: u8 = 1; -const I: u8 = 2; -const X: u8 = 3; - -static TEXT_CHARS: [u8; 256] = [ - /* BEL BS HT LF VT FF CR */ - F, F, F, F, F, F, F, T, T, T, T, T, T, T, F, F, /* 0x0X */ - /* ESC */ - F, F, F, F, F, F, F, F, F, F, F, T, F, F, F, F, /* 0x1X */ - T, T, T, T, T, T, T, T, T, T, T, T, T, T, T, T, /* 0x2X */ - T, T, T, T, T, T, T, T, T, T, T, T, T, T, T, T, /* 0x3X */ - T, T, T, T, T, T, T, T, T, T, T, T, T, T, T, T, /* 0x4X */ - T, T, T, T, T, T, T, T, T, T, T, T, T, T, T, T, /* 0x5X */ - T, T, T, T, T, T, T, T, T, T, T, T, T, T, T, T, /* 0x6X */ - T, T, T, T, T, T, T, T, T, T, T, T, T, T, T, F, /* 0x7X */ - /* NEL */ - X, X, X, X, X, T, X, X, X, X, X, X, X, X, X, X, /* 0x8X */ - X, X, X, X, X, X, X, X, X, X, X, X, X, X, X, X, /* 0x9X */ - I, I, I, I, I, I, I, I, I, I, I, I, I, I, I, I, /* 0xaX */ - I, I, I, I, I, I, I, I, I, I, I, I, I, I, I, I, /* 0xbX */ - I, I, I, I, I, I, I, I, I, I, I, I, I, I, I, I, /* 0xcX */ - I, I, I, I, I, I, I, I, I, I, I, I, I, I, I, I, /* 0xdX */ - I, I, I, I, I, I, I, I, I, I, I, I, I, I, I, I, /* 0xeX */ - I, I, I, I, I, I, I, I, I, I, I, I, I, I, I, I, /* 0xfX */ -]; - -fn looks_latin1(buf: &[u8]) -> bool { - buf.iter().all(|&byte| byte == T || byte == I) -} - -const XX: u8 = 0xF1; // invalid: size 1 -const AS: u8 = 0xF0; // ASCII: size 1 -const S1: u8 = 0x02; // accept 0, size 2 -const S2: u8 = 0x13; // accept 1, size 3 -const S3: u8 = 0x03; // accept 0, size 3 -const S4: u8 = 0x23; // accept 2, size 3 -const S5: u8 = 0x34; // accept 3, size 4 -const S6: u8 = 0x04; // accept 0, size 4 -const S7: u8 = 0x44; // accept 4, size 4 -const LOCB: u8 = 0x80; -const HICB: u8 = 0xBF; - -static FIRST: [u8; 256] = [ - // 1 2 3 4 5 6 7 8 9 A B C D E F - AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, // 0x00-0x0F - AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, // 0x10-0x1F - AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, // 0x20-0x2F - AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, // 0x30-0x3F - AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, // 0x40-0x4F - AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, // 0x50-0x5F - AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, // 0x60-0x6F - AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, // 0x70-0x7F - // 1 2 3 4 5 6 7 8 9 A B C D E F - XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, // 0x80-0x8F - XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, // 0x90-0x9F - XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, // 0xA0-0xAF - XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, // 0xB0-0xBF - XX, XX, S1, S1, S1, S1, S1, S1, S1, S1, S1, S1, S1, S1, S1, S1, // 0xC0-0xCF - S1, S1, S1, S1, S1, S1, S1, S1, S1, S1, S1, S1, S1, S1, S1, S1, // 0xD0-0xDF - S2, S3, S3, S3, S3, S3, S3, S3, S3, S3, S3, S3, S3, S4, S3, S3, // 0xE0-0xEF - S5, S6, S6, S6, S7, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, // 0xF0-0xFF -]; - -struct AcceptRange { - lo: u8, - hi: u8, -} - -static EMPTY_ACCEPT_RANGE: AcceptRange = AcceptRange { lo: 0, hi: 0 }; - -static ACCEPT_RANGES: [AcceptRange; 5] = [ - AcceptRange { lo: LOCB, hi: HICB }, - AcceptRange { lo: 0xA0, hi: HICB }, - AcceptRange { lo: LOCB, hi: 0x9F }, - AcceptRange { lo: 0x90, hi: HICB }, - AcceptRange { lo: LOCB, hi: 0x8F }, -]; - -fn looks_utf8(buf: &[u8], partial: bool) -> bool { - let mut ctrl = false; - - let mut it = buf.iter(); - 'outer: while let Some(byte) = it.next() { - /* 0xxxxxxx is plain ASCII */ - if (byte & 0x80) == 0 { - /* - * Even if the whole file is valid UTF-8 sequences, - * still reject it if it uses weird control characters. - */ - - if TEXT_CHARS[(*byte) as usize] != T { - ctrl = true; - } - /* 10xxxxxx never 1st byte */ - } else if (byte & 0x40) == 0 { - return false; - /* 11xxxxxx begins UTF-8 */ - } else { - let x = FIRST[(*byte) as usize]; - if x == XX { - return false; - } - - let following = if (byte & 0x20) == 0 { - /* 110xxxxx */ - 1 - } else if (byte & 0x10) == 0 { - /* 1110xxxx */ - 2 - } else if (byte & 0x08) == 0 { - /* 11110xxx */ - 3 - } else if (byte & 0x04) == 0 { - /* 111110xx */ - 4 - } else if (byte & 0x02) == 0 { - /* 1111110x */ - 5 - } else { - return false; - }; - - let accept_range = ACCEPT_RANGES - .get((x >> 4) as usize) - .unwrap_or(&EMPTY_ACCEPT_RANGE); - for n in 0..following { - let Some(&following_byte) = it.next() else { - break 'outer; - }; - - if n == 0 && (following_byte < accept_range.lo || following_byte > accept_range.hi) - { - return false; - } - - if (following_byte & 0x80) == 0 || (following_byte & 0x40) != 0 { - return false; - } - } - } - } - - partial || !ctrl -} - -fn looks_utf8_with_bom(buf: &[u8], partial: bool) -> bool { - if buf.len() > 3 && buf[0] == 0xef && buf[1] == 0xbb && buf[2] == 0xbf { - looks_utf8(&buf[3..], partial) - } else { - false - } -} - -enum UCS16 { - BigEnd, - LittleEnd, -} - -fn looks_ucs16(buf: &[u8]) -> Option { - if buf.len() % 2 == 0 { - return None; - } - - let bigend = if buf[0] == 0xff && buf[1] == 0xfe { - false - } else if buf[0] == 0xfe && buf[1] == 0xff { - true - } else { - return None; - }; - - let mut hi: u32 = 0; - for chunk in buf[2..].chunks_exact(2) { - let mut uc = (if bigend { - u32::from(chunk[1]) | u32::from(chunk[0]) << 8 - } else { - u32::from(chunk[0]) | u32::from(chunk[1]) << 8 - }) & 0xffff; - - match uc { - 0xfffe | 0xffff => return None, - // UCS16_NOCHAR - _ if (0xfdd0..=0xfdef).contains(&uc) => return None, - _ => (), - } - - if hi != 0 { - // UCS16_LOSURR - if (0xdc00..=0xdfff).contains(&uc) { - return None; - } - uc = 0x10000 + 0x400 * (hi - 1) + (uc - 0xdc00); - hi = 0; - } - - if uc < 128 && TEXT_CHARS[uc as usize] != T { - return None; - } - - // UCS16_HISURR - if (0xd800..=0xdbff).contains(&uc) { - hi = uc - 0xd800 + 1; - } - - // UCS16_LOSURR - if (0xdc00..=0xdfff).contains(&uc) { - return None; - } - } - - Some(if bigend { - UCS16::BigEnd - } else { - UCS16::LittleEnd - }) -} - -enum UCS32 { - BigEnd, - LittleEnd, -} - -fn looks_ucs32(buf: &[u8]) -> Option { - if buf.len() % 4 == 0 { - return None; - } - - let bigend = if buf[0] == 0xff && buf[1] == 0xfe && buf[2] == 0 && buf[3] == 0 { - false - } else if buf[0] == 0 && buf[1] == 0 && buf[2] == 0xfe && buf[3] == 0xff { - true - } else { - return None; - }; - - for chunk in buf[4..].chunks_exact(4) { - let uc: u32 = if bigend { - u32::from(chunk[3]) - | u32::from(chunk[2]) << 8 - | u32::from(chunk[1]) << 16 - | u32::from(chunk[0]) << 24 - } else { - u32::from(chunk[0]) - | u32::from(chunk[1]) << 8 - | u32::from(chunk[2]) << 16 - | u32::from(chunk[3]) << 24 - }; - - if uc == 0xfffe { - return None; - } - if uc < 128 && TEXT_CHARS[uc as usize] != T { - return None; - } - } - - Some(if bigend { - UCS32::BigEnd - } else { - UCS32::LittleEnd - }) -} - -#[must_use] -pub fn is_text(data: &[u8], partial: bool) -> Option<&'static str> { - if data.is_empty() { - return None; - } - - if looks_utf8_with_bom(data, partial) || looks_utf8(data, partial) { - return Some("utf-8"); - } - - match looks_ucs16(data) { - Some(UCS16::BigEnd) => return Some("utf-16be"), - Some(UCS16::LittleEnd) => return Some("utf-16le"), - None => (), - } - - match looks_ucs32(data) { - Some(UCS32::BigEnd) => return Some("utf-32be"), - Some(UCS32::LittleEnd) => return Some("utf-32le"), - None => (), - } - - if looks_latin1(data) { - Some("iso-8859-1") - } else { - None - } -} diff --git a/crates/old-p2p/Cargo.toml b/crates/old-p2p/Cargo.toml deleted file mode 100644 index cdfe95df7..000000000 --- a/crates/old-p2p/Cargo.toml +++ /dev/null @@ -1,51 +0,0 @@ -[package] -name = "sd-old-p2p" -version = "0.2.0" - -authors = ["Oscar Beaumont "] -description = "Rust Peer to Peer Networking Library" -edition.workspace = true -license.workspace = true -readme = "README.md" -repository.workspace = true - -# TODO: Remove features??? and dependencies - -[features] -default = [] -specta = [] - -[dependencies] -# Workspace dependencies -base64 = { workspace = true } -ed25519-dalek = { workspace = true } -flume = { workspace = true } -futures = { workspace = true } -rmp-serde = { workspace = true } -serde = { workspace = true, features = ["derive"] } -specta = { workspace = true } -thiserror = { workspace = true } -tokio = { workspace = true, features = ["fs", "io-util", "macros", "sync", "time"] } -tokio-util = { workspace = true, features = ["compat"] } -tracing = { workspace = true } -uuid = { workspace = true, features = ["serde"] } -zeroize = { workspace = true, features = ["derive"] } - -# Specific P2P dependencies -dns-lookup = "2.0" -hash_map_diff = "0.2.0" -if-watch = { version = "=3.2.0", features = ["tokio"] } # Override features used by libp2p-quic -libp2p-stream = "=0.2.0-alpha" # Update blocked due to custom patch -mdns-sd = "0.11.5" -rand_core = "0.6.4" -stable-vec = "0.4.1" -sync_wrapper = "1.0" - -[dependencies.libp2p] -features = ["autonat", "dcutr", "macros", "noise", "quic", "relay", "serde", "tokio", "yamux"] -version = "=0.54.1" # Update blocked due to custom patch - -[dev-dependencies] -tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } -tracing-subscriber = { workspace = true } -uuid = { workspace = true, features = ["v4"] } diff --git a/crates/old-p2p/README.md b/crates/old-p2p/README.md deleted file mode 100644 index 0f58c59bb..000000000 --- a/crates/old-p2p/README.md +++ /dev/null @@ -1,9 +0,0 @@ -# Spacedrive's P2P Crate - -## Functionality - -This crate handles low-level P2P networking and the transferring of data between machines across a network. - -## Credits - -We'd like to give credit towards both the `libp2p` developers/maintainers, as well as the [Syncthing BEP Protocol](https://docs.syncthing.net/specs/bep-v1.html)'s designers, developers and maintainers - our implementation was heavily inspired by the protocol and made our lives a lot easier. Thank you! diff --git a/crates/old-p2p/crates/block/Cargo.toml b/crates/old-p2p/crates/block/Cargo.toml deleted file mode 100644 index 31110722c..000000000 --- a/crates/old-p2p/crates/block/Cargo.toml +++ /dev/null @@ -1,18 +0,0 @@ -[package] -name = "sd-old-p2p-block" -version = "0.1.0" - -authors = ["Oscar Beaumont "] -edition.workspace = true -license.workspace = true -repository.workspace = true - -[dependencies] -# Spacedrive Sub-crates -sd-old-p2p-proto = { path = "../proto" } - -# Workspace dependencies -thiserror = { workspace = true } -tokio = { workspace = true } -tracing = { workspace = true } -uuid = { workspace = true } diff --git a/crates/old-p2p/crates/block/src/block.rs b/crates/old-p2p/crates/block/src/block.rs deleted file mode 100644 index 12268c822..000000000 --- a/crates/old-p2p/crates/block/src/block.rs +++ /dev/null @@ -1,97 +0,0 @@ -use std::io::{self, ErrorKind}; - -use tokio::io::AsyncReadExt; - -/// TODO -#[derive(Debug, PartialEq, Eq)] -pub struct Block<'a> { - // TODO: File content, checksum, source location so it can be resent! - pub offset: u64, - pub size: u64, - pub data: &'a [u8], - // TODO: Checksum? -} - -impl<'a> Block<'a> { - #[must_use] - pub fn to_bytes(&self) -> Vec { - let mut buf = Vec::new(); - buf.extend_from_slice(&self.offset.to_le_bytes()); - debug_assert_eq!(self.data.len(), self.size as usize); // TODO: Should `self.size` be inferred instead? - buf.extend_from_slice(&self.size.to_le_bytes()); - buf.extend_from_slice(self.data); - buf - } - - pub async fn from_stream( - stream: &mut (impl AsyncReadExt + Unpin), - data_buf: &mut [u8], - ) -> Result, io::Error> { - let mut offset = [0; 8]; - stream.read_exact(&mut offset).await?; - let offset = u64::from_le_bytes(offset); - - let mut size = [0; 8]; - stream.read_exact(&mut size).await?; - let size = u64::from_le_bytes(size); - - // TODO: Ensure `size` is `block_size` or smaller else buffer overflow - - if size as usize > data_buf.len() { - return Err(io::Error::new( - ErrorKind::Other, - "size and buffer length mismatch", - )); - } - - stream.read_exact(&mut data_buf[..size as usize]).await?; - - Ok(Self { - offset, - size, - data: &[], // TODO: This is super cringe. Data should be decoded here but lifetimes and extra allocations become a major concern. - }) - } -} - -#[cfg(test)] -mod tests { - use std::io::Cursor; - - use super::*; - - #[tokio::test] - async fn test_block() { - let mut req = Block { - offset: 420, - size: 10, // Matches length of string on next line - data: b"Spacedrive".as_ref(), - }; - let bytes = req.to_bytes(); - let mut data2 = vec![0; req.data.len()]; - let req2 = Block::from_stream(&mut Cursor::new(bytes), &mut data2) - .await - .unwrap(); - let data = std::mem::take(&mut req.data); - assert_eq!(req, req2); - assert_eq!(data, data2); - } - - #[tokio::test] - #[should_panic] // TODO: This currently panics but long term it should have proper error handling - async fn test_block_data_buf_overflow() { - let mut req = Block { - offset: 420, - size: 10, // Matches length of string on next line - data: b"Spacedrive".as_ref(), - }; - let bytes = req.to_bytes(); - let mut data2 = vec![0; 5]; // Length smaller than `req.data.len()` - let req2 = Block::from_stream(&mut Cursor::new(bytes), &mut data2) - .await - .unwrap(); - let data = std::mem::take(&mut req.data); - assert_eq!(req, req2); - assert_eq!(data, data2); - } -} diff --git a/crates/old-p2p/crates/block/src/block_size.rs b/crates/old-p2p/crates/block/src/block_size.rs deleted file mode 100644 index 0aac54de2..000000000 --- a/crates/old-p2p/crates/block/src/block_size.rs +++ /dev/null @@ -1,120 +0,0 @@ -#![allow(non_upper_case_globals)] - -use std::io; - -use tokio::io::{AsyncRead, AsyncReadExt}; - -const KiB: u32 = 1024; -const MiB: u32 = 1024 * KiB; -const GiB: u32 = 1024 * MiB; - -/// defines the size of each chunk of data that is sent -/// -/// We store this in an enum so it's super efficient. -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum BlockSize { - _128KiB, - _256KiB, - _512KiB, - _1MiB, - _2MiB, - _4MiB, - _8MiB, - _16MiB, -} - -impl BlockSize { - /// Determine the optimal block size for a given file size - #[must_use] - pub fn from_file_size(size: u64) -> Self { - // Values directly copied from https://docs.syncthing.net/specs/bep-v1.html#selection-of-block-size - if size < 250 * u64::from(MiB) { - return Self::_128KiB; - } else if size < 500 * u64::from(MiB) { - return Self::_256KiB; - } else if size < u64::from(GiB) { - return Self::_512KiB; - } else if size < 2 * u64::from(GiB) { - return Self::_1MiB; - } else if size < 4 * u64::from(GiB) { - return Self::_2MiB; - } else if size < 8 * u64::from(GiB) { - return Self::_4MiB; - } else if size < 16 * u64::from(GiB) { - return Self::_8MiB; - } - Self::_16MiB - } - - /// Get the size of the block in bytes - #[must_use] - pub fn size(&self) -> u32 { - match self { - Self::_128KiB => 128 * KiB, - Self::_256KiB => 256 * KiB, - Self::_512KiB => 512 * KiB, - Self::_1MiB => MiB, - Self::_2MiB => 2 * MiB, - Self::_4MiB => 4 * MiB, - Self::_8MiB => 8 * MiB, - Self::_16MiB => 16 * MiB, - } - } - - pub async fn from_stream(stream: &mut (impl AsyncRead + Unpin)) -> io::Result { - // WARNING: Be careful modifying this cause it may break backwards/forwards-compatibility - match stream.read_u8().await? { - 0 => Ok(Self::_128KiB), - 1 => Ok(Self::_256KiB), - 2 => Ok(Self::_512KiB), - 3 => Ok(Self::_1MiB), - 4 => Ok(Self::_2MiB), - 5 => Ok(Self::_4MiB), - 6 => Ok(Self::_8MiB), - 7 => Ok(Self::_16MiB), - _ => Err(io::Error::new( - io::ErrorKind::InvalidData, - "Invalid block size", - )), - } - } - - #[must_use] - pub fn to_bytes(&self) -> [u8; 1] { - // WARNING: Be careful modifying this cause it may break backwards/forwards-compatibility - [match self { - Self::_128KiB => 0, - Self::_256KiB => 1, - Self::_512KiB => 2, - Self::_1MiB => 3, - Self::_2MiB => 4, - Self::_4MiB => 5, - Self::_8MiB => 6, - Self::_16MiB => 7, - }] - } -} - -#[cfg(test)] -mod tests { - use std::io::Cursor; - - use super::*; - - #[tokio::test] - async fn test_block_size() { - let req = BlockSize::_128KiB; - let bytes = req.to_bytes(); - let req2 = BlockSize::from_stream(&mut Cursor::new(bytes)) - .await - .unwrap(); - assert_eq!(req, req2); - - let req = BlockSize::_16MiB; - let bytes = req.to_bytes(); - let req2 = BlockSize::from_stream(&mut Cursor::new(bytes)) - .await - .unwrap(); - assert_eq!(req, req2); - } -} diff --git a/crates/old-p2p/crates/block/src/lib.rs b/crates/old-p2p/crates/block/src/lib.rs deleted file mode 100644 index 7d6554dfd..000000000 --- a/crates/old-p2p/crates/block/src/lib.rs +++ /dev/null @@ -1,469 +0,0 @@ -//! A protocol for efficiently and securely transferring files between peers. -//! -//! Goals: -//! - Fast - Transfer files as quickly as possible -//! - Safe - Verify the files integrity on both ends -//! -//! This protocol was heavily inspired by SyncThing's Block Exchange Protocol protocol although it's not compatible. -//! You can read more about it here: -//! -#![warn(clippy::unwrap_used, clippy::panic)] - -use std::{ - io, - sync::atomic::{AtomicBool, Ordering}, -}; - -use tokio::io::{AsyncBufRead, AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt}; -use tracing::debug; - -mod block; -mod block_size; -mod sb_request; - -pub use block::*; -pub use block_size::*; -pub use sb_request::*; - -#[derive(Debug, PartialEq, Eq)] -pub enum Msg<'a> { - Block(Block<'a>), - Cancelled, -} - -impl<'a> Msg<'a> { - pub async fn from_stream<'b>( - stream: &mut (impl AsyncReadExt + Unpin), - data_buf: &'b mut [u8], - ) -> Result, io::Error> { - let discriminator = stream.read_u8().await?; - match discriminator { - 0 => Ok(Msg::Block(Block::from_stream(stream, data_buf).await?)), - 1 => Ok(Msg::Cancelled), - _ => Err(io::Error::new( - io::ErrorKind::Other, - "Invalid 'Msg' discriminator!", - )), - } - } - - #[must_use] - pub fn to_bytes(&self) -> Vec { - match self { - Msg::Block(block) => { - let mut bytes = Vec::new(); - bytes.push(0); - bytes.extend(block.to_bytes()); - bytes - } - Msg::Cancelled => vec![1], - } - } -} - -/// TODO -pub struct Transfer<'a, F> { - reqs: &'a SpaceblockRequests, - on_progress: F, - total_offset: u64, - total_bytes: u64, - // TODO: Remove `i` plz - i: usize, - cancelled: &'a AtomicBool, -} - -impl<'a, F> Transfer<'a, F> -where - F: Fn(u8) + 'a, -{ - // TODO: Handle `req.range` correctly in this code - - pub fn new(req: &'a SpaceblockRequests, on_progress: F, cancelled: &'a AtomicBool) -> Self { - Self { - reqs: req, - on_progress, - total_offset: 0, - total_bytes: req.requests.iter().map(|req| req.size).sum(), - i: 0, - cancelled, - } - } - - // TODO: Should `new` take in the streams too cause this means we `Stream` `SpaceblockRequest` could get outta sync. - pub async fn send( - &mut self, - stream: &mut (impl AsyncRead + AsyncWrite + Unpin), - mut file: (impl AsyncBufRead + Unpin), - ) -> Result<(), io::Error> { - // We manually implement what is basically a `BufReader` so we have more control - let mut buf = vec![0u8; self.reqs.block_size.size() as usize]; - let mut offset: u64 = 0; - - loop { - if self.cancelled.load(Ordering::Relaxed) { - stream.write_all(&Msg::Cancelled.to_bytes()).await?; - stream.flush().await?; - return Ok(()); - } - - let read = file.read(&mut buf[..]).await?; - self.total_offset += read as u64; - (self.on_progress)( - ((self.total_offset as f64 / self.total_bytes as f64) * 100.0) as u8, - ); // SAFETY: Percent must be between 0 and 100 - - if read == 0 { - // The file may have been modified during sender on the sender and we don't account for that. - // TODO: Error handling + send error to remote - assert!( - (offset + read as u64) == self.reqs.requests[self.i].size, - "File sending has stopped but it doesn't match the expected length!" - ); - - return Ok(()); - } - - let block = Block { - offset, - size: read as u64, - data: &buf[..read], - }; - debug!( - "Sending block at offset {} of size {}", - block.offset, block.size - ); - offset += read as u64; - - stream.write_all(&Msg::Block(block).to_bytes()).await?; - stream.flush().await?; - - match stream.read_u8().await? { - // Continue sending - 0 => {} - // Cancelled by user - 1 => { - debug!("Receiver cancelled Spacedrop transfer!"); - return Ok(()); - } - // Transfer complete - 2 => return Ok(()), - _ => todo!(), - } - } - } - - // TODO: Timeout on receiving/sending - pub async fn receive( - &mut self, - stream: &mut (impl AsyncRead + AsyncWrite + Unpin), - mut file: (impl AsyncWrite + Unpin), - // TODO: Proper error type - ) -> Result<(), io::Error> { - // We manually implement what is basically a `BufReader` so we have more control - let mut data_buf = vec![0u8; self.reqs.block_size.size() as usize]; - let mut offset: u64 = 0; - - if self.reqs.requests[self.i].size == 0 { - self.i += 1; - return Ok(()); - } - - // TODO: Prevent loop being a DOS vector - loop { - if self.cancelled.load(Ordering::Relaxed) { - stream.write_u8(1).await?; - stream.flush().await?; - return Ok(()); - } - - // TODO: Timeout if nothing is being received - let msg = Msg::from_stream(stream, &mut data_buf).await?; - match msg { - Msg::Block(block) => { - self.total_offset += block.size; - (self.on_progress)( - ((self.total_offset as f64 / self.total_bytes as f64) * 100.0) as u8, - ); // SAFETY: Percent must be between 0 and 100 - - debug!( - "Received block at offset {} of size {}", - block.offset, block.size - ); - offset += block.size; - - file.write_all(&data_buf[..block.size as usize]).await?; - - let req = self.reqs.requests.get(self.i).ok_or_else(|| { - debug!("Vector read out of bounds!"); - io::ErrorKind::Other - })?; - // TODO: Should this be `read == 0` - if offset == req.size { - break; - } - - stream - .write_u8(u8::from(self.cancelled.load(Ordering::Relaxed))) - .await?; - stream.flush().await?; - } - Msg::Cancelled => { - debug!("Sender cancelled Spacedrop transfer!"); - return Ok(()); - } - } - } - - stream.write_u8(2).await?; - stream.flush().await?; - file.flush().await?; - self.i += 1; - - Ok(()) - } -} - -#[cfg(test)] -mod tests { - use std::{io::Cursor, mem, sync::Arc}; - - use tokio::{io::BufReader, sync::oneshot}; - use uuid::Uuid; - - use super::*; - - #[tokio::test] - async fn test_spaceblock_single_block() { - let (mut client, mut server) = tokio::io::duplex(64); - - // This is sent out of band of Spaceblock - let data = b"Spacedrive".to_vec(); - let req = SpaceblockRequests { - id: Uuid::new_v4(), - block_size: BlockSize::from_file_size(data.len() as u64), - requests: vec![SpaceblockRequest { - name: "Demo".to_string(), - size: data.len() as u64, - range: Range::Full, - }], - }; - - let (tx, rx) = oneshot::channel(); - tokio::spawn({ - let req = req.clone(); - let data = data.clone(); - async move { - let file = BufReader::new(Cursor::new(data)); - tx.send(()).unwrap(); - Transfer::new(&req, |_| {}, &Default::default()) - .send(&mut client, file) - .await - } - }); - - rx.await.unwrap(); - - let mut result = Vec::new(); - Transfer::new(&req, |_| {}, &Default::default()) - .receive(&mut server, &mut result) - .await - .unwrap(); - assert_eq!(result, data); - } - - // https://github.com/spacedriveapp/spacedrive/pull/942 - #[tokio::test] - async fn test_spaceblock_multiple_blocks() { - let (mut client, mut server) = tokio::io::duplex(64); - - // This is sent out of band of Spaceblock - let block_size = BlockSize::_128KiB; - let data = vec![0u8; block_size.size() as usize * 4]; // Let's pacman some RAM - - let req = SpaceblockRequests { - id: Uuid::new_v4(), - block_size, - requests: vec![SpaceblockRequest { - name: "Demo".to_string(), - size: data.len() as u64, - range: Range::Full, - }], - }; - - let (tx, rx) = oneshot::channel(); - tokio::spawn({ - let req = req.clone(); - let data = data.clone(); - async move { - let file = BufReader::new(Cursor::new(data)); - tx.send(()).unwrap(); - Transfer::new(&req, |_| {}, &Default::default()) - .send(&mut client, file) - .await - } - }); - - rx.await.unwrap(); - - let mut result = Vec::new(); - Transfer::new(&req, |_| {}, &Default::default()) - .receive(&mut server, &mut result) - .await - .unwrap(); - - assert_eq!(result, data); - } - - #[tokio::test] - async fn test_transfer_receiver_cancelled() { - let (mut client, mut server) = tokio::io::duplex(64); - - // This is sent out of band of Spaceblock - let block_size = BlockSize::_128KiB; - let data = vec![0u8; block_size.size() as usize]; - - let req = SpaceblockRequests { - id: Uuid::new_v4(), - block_size, - requests: vec![SpaceblockRequest { - name: "Demo".to_string(), - size: data.len() as u64, - range: Range::Full, - }], - }; - - let (tx, rx) = oneshot::channel(); - tokio::spawn({ - let req = req.clone(); - let data = data.clone(); - async move { - let file = BufReader::new(Cursor::new(data)); - tx.send(()).unwrap(); - - Transfer::new(&req, |_| {}, &Arc::new(AtomicBool::new(true))) - .send(&mut client, file) - .await - } - }); - - rx.await.unwrap(); - - let mut result = Vec::new(); - let _ = Transfer::new(&req, |_| {}, &Default::default()) - .receive(&mut server, &mut result) - .await; - assert_eq!(result, Vec::::new()); // Cancelled by sender so no data - } - - #[tokio::test] - async fn test_transfer_sender_cancelled() { - let (mut client, mut server) = tokio::io::duplex(64); - - // This is sent out of band of Spaceblock - let block_size = BlockSize::_128KiB; - let data = vec![0u8; block_size.size() as usize]; - - let req = SpaceblockRequests { - id: Uuid::new_v4(), - block_size, - requests: vec![SpaceblockRequest { - name: "Demo".to_string(), - size: data.len() as u64, - range: Range::Full, - }], - }; - - let (tx, rx) = oneshot::channel(); - tokio::spawn({ - let req = req.clone(); - let data = data.clone(); - async move { - let file = BufReader::new(Cursor::new(data)); - tx.send(()).unwrap(); - - Transfer::new(&req, |_| {}, &Default::default()) - .send(&mut client, file) - .await - } - }); - - rx.await.unwrap(); - - let mut result = Vec::new(); - let _ = Transfer::new(&req, |_| {}, &Arc::new(AtomicBool::new(true))) - .receive(&mut server, &mut result) - .await; - assert_eq!(result, Vec::::new()); // Cancelled by sender so no data - } - - // https://linear.app/spacedriveapp/issue/ENG-1300/spaceblock-doesnt-like-zero-sized-files - #[tokio::test] - async fn test_spaceblock_zero_sized_file() { - let (mut client, mut server) = tokio::io::duplex(64); - - // This is sent out of band of Spaceblock - let block_size = BlockSize::_128KiB; - let data = vec![0u8; 0]; // Zero sized file - - let req = SpaceblockRequests { - id: Uuid::new_v4(), - block_size, - requests: vec![SpaceblockRequest { - name: "Demo".to_string(), - size: data.len() as u64, - range: Range::Full, - }], - }; - - let (tx, rx) = oneshot::channel(); - tokio::spawn({ - let req = req.clone(); - let data = data.clone(); - async move { - let file = BufReader::new(Cursor::new(data)); - tx.send(()).unwrap(); - - Transfer::new(&req, |_| {}, &Default::default()) - .send(&mut client, file) - .await - } - }); - - rx.await.unwrap(); - - let mut result = Vec::new(); - let _ = Transfer::new(&req, |_| {}, &Default::default()) - .receive(&mut server, &mut result) - .await; - assert_eq!(result, Vec::::new()); // Cancelled by sender so no data - } - - #[tokio::test] - async fn test_msg() { - let block = Block { - offset: 0, - size: 10, - data: b"Spacedrive".as_ref(), - }; - let data_len = block.data.len(); - let mut msg = Msg::Block(block); - let bytes = msg.to_bytes(); - let mut data2 = vec![0; data_len]; - let msg2 = Msg::from_stream(&mut Cursor::new(bytes), &mut data2) - .await - .unwrap(); - let data = mem::take(match &mut msg { - Msg::Block(block) => &mut block.data, - _ => unreachable!(), - }); // We decode the data into - assert_eq!(msg, msg2); - assert_eq!(data, data2); - - let msg = Msg::Cancelled; - let bytes = msg.to_bytes(); - let msg2 = Msg::from_stream(&mut Cursor::new(bytes), &mut [0u8; 64]) - .await - .unwrap(); - assert_eq!(msg, msg2); - } -} diff --git a/crates/old-p2p/crates/block/src/sb_request.rs b/crates/old-p2p/crates/block/src/sb_request.rs deleted file mode 100644 index f92aab1e2..000000000 --- a/crates/old-p2p/crates/block/src/sb_request.rs +++ /dev/null @@ -1,268 +0,0 @@ -use std::io; - -use thiserror::Error; -use tokio::io::{AsyncRead, AsyncReadExt}; -use uuid::Uuid; - -use sd_old_p2p_proto::{decode, encode}; - -use super::BlockSize; - -/// TODO -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum Range { - /// Request the entire file - Full, - /// Partial range - Partial(std::ops::Range), -} - -impl Range { - // TODO: Per field and proper error handling - pub async fn from_stream(stream: &mut (impl AsyncRead + Unpin)) -> std::io::Result { - match stream.read_u8().await? { - 0 => Ok(Self::Full), - 1 => { - let start = stream.read_u64_le().await?; - let end = stream.read_u64_le().await?; - Ok(Self::Partial(start..end)) - } - _ => Err(io::Error::new( - io::ErrorKind::Other, - "Invalid range discriminator", - )), - } - } - - #[must_use] - pub fn to_bytes(&self) -> Vec { - let mut buf = Vec::new(); - - match self { - Self::Full => buf.push(0), - Self::Partial(range) => { - buf.push(1); - buf.extend_from_slice(&range.start.to_le_bytes()); - buf.extend_from_slice(&range.end.to_le_bytes()); - } - } - buf - } -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct SpaceblockRequests { - pub id: Uuid, - pub block_size: BlockSize, - pub requests: Vec, -} - -#[derive(Debug, Error)] -pub enum SpaceblockRequestsError { - #[error("SpaceblockRequestsError::Id({0:?})")] - Id(#[from] decode::Error), - #[error("SpaceblockRequestsError::InvalidLen({0})")] - InvalidLen(std::io::Error), - #[error("SpaceblockRequestsError::SpaceblockRequest({0:?})")] - SpaceblockRequest(#[from] SpaceblockRequestError), - #[error("SpaceblockRequestsError::BlockSize({0:?})")] - BlockSize(std::io::Error), -} - -impl SpaceblockRequests { - pub async fn from_stream( - stream: &mut (impl AsyncRead + Unpin), - ) -> Result { - let id = decode::uuid(stream) - .await - .map_err(SpaceblockRequestsError::Id)?; - - let block_size = BlockSize::from_stream(stream) - .await - .map_err(SpaceblockRequestsError::BlockSize)?; - - let size = stream - // Max of 255 files in one request - .read_u8() - .await - .map_err(SpaceblockRequestsError::InvalidLen)?; - - let mut requests = Vec::new(); - for _i in 0..size { - requests.push(SpaceblockRequest::from_stream(stream).await?); - } - - Ok(Self { - id, - block_size, - requests, - }) - } - - #[must_use] - pub fn to_bytes(&self) -> Vec { - let Self { - id, - block_size, - requests, - } = self; - assert!( - requests.len() <= 255, - "Can't Spacedrop more than 255 files at once!" - ); - - let mut buf = vec![]; - encode::uuid(&mut buf, id); - buf.append(&mut block_size.to_bytes().to_vec()); - buf.push(requests.len() as u8); - for request in requests { - buf.extend_from_slice(&request.to_bytes()); - } - buf - } -} - -/// TODO -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct SpaceblockRequest { - pub name: String, - pub size: u64, - // TODO: Include file permissions - pub range: Range, -} - -#[derive(Debug, Error)] -pub enum SpaceblockRequestError { - #[error("SpaceblockRequestError::Name({0})")] - Name(decode::Error), - #[error("SpaceblockRequestError::Size({0})")] - Size(std::io::Error), - // TODO: From outside. Probs remove? - #[error("SpaceblockRequestError::RangeError({0:?})")] - RangeError(io::Error), -} - -impl SpaceblockRequest { - pub async fn from_stream( - stream: &mut (impl AsyncRead + Unpin), - ) -> Result { - let name = decode::string(stream) - .await - .map_err(SpaceblockRequestError::Name)?; - - let size = stream - .read_u64_le() - .await - .map_err(SpaceblockRequestError::Size)?; - - Ok(Self { - name, - size, - range: Range::from_stream(stream) - .await - .map_err(SpaceblockRequestError::Size)?, - }) - } - - #[must_use] - pub fn to_bytes(&self) -> Vec { - let mut buf = Vec::new(); - - encode::string(&mut buf, &self.name); - buf.extend_from_slice(&self.size.to_le_bytes()); - buf.extend_from_slice(&self.range.to_bytes()); - buf - } -} - -#[cfg(test)] -mod tests { - use std::io::Cursor; - - use super::*; - - #[tokio::test] - async fn test_range() { - let req = Range::Full; - let bytes = req.to_bytes(); - let req2 = Range::from_stream(&mut Cursor::new(bytes)).await.unwrap(); - assert_eq!(req, req2); - - let req = Range::Partial(0..10); - let bytes = req.to_bytes(); - let req2 = Range::from_stream(&mut Cursor::new(bytes)).await.unwrap(); - assert_eq!(req, req2); - } - - #[tokio::test] - async fn test_spaceblock_requests_empty() { - let req = SpaceblockRequests { - id: Uuid::new_v4(), - block_size: BlockSize::from_file_size(42069), - requests: vec![], - }; - - let bytes = req.to_bytes(); - let req2 = SpaceblockRequests::from_stream(&mut Cursor::new(bytes)) - .await - .unwrap(); - assert_eq!(req, req2); - } - - #[tokio::test] - async fn test_spaceblock_requests_one() { - let req = SpaceblockRequests { - id: Uuid::new_v4(), - block_size: BlockSize::from_file_size(42069), - requests: vec![SpaceblockRequest { - name: "Demo".to_string(), - size: 42069, - range: Range::Full, - }], - }; - - let bytes = req.to_bytes(); - let req2 = SpaceblockRequests::from_stream(&mut Cursor::new(bytes)) - .await - .unwrap(); - assert_eq!(req, req2); - - let req = SpaceblockRequest { - name: "Demo".to_string(), - size: 42069, - range: Range::Partial(0..420), - }; - - let bytes = req.to_bytes(); - let req2 = SpaceblockRequest::from_stream(&mut Cursor::new(bytes)) - .await - .unwrap(); - assert_eq!(req, req2); - } - - #[tokio::test] - async fn test_spaceblock_requests_many() { - let req = SpaceblockRequests { - id: Uuid::new_v4(), - block_size: BlockSize::from_file_size(42069), - requests: vec![ - SpaceblockRequest { - name: "Demo".to_string(), - size: 42069, - range: Range::Full, - }, - SpaceblockRequest { - name: "Demo2".to_string(), - size: 420, - range: Range::Full, - }, - ], - }; - - let bytes = req.to_bytes(); - let req2 = SpaceblockRequests::from_stream(&mut Cursor::new(bytes)) - .await - .unwrap(); - assert_eq!(req, req2); - } -} diff --git a/crates/old-p2p/crates/proto/Cargo.toml b/crates/old-p2p/crates/proto/Cargo.toml deleted file mode 100644 index 2198b1aa8..000000000 --- a/crates/old-p2p/crates/proto/Cargo.toml +++ /dev/null @@ -1,14 +0,0 @@ -[package] -name = "sd-old-p2p-proto" -version = "0.1.0" - -authors = ["Oscar Beaumont "] -edition.workspace = true -license.workspace = true -repository.workspace = true - -[dependencies] -ed25519-dalek = { workspace = true } -thiserror = { workspace = true } -tokio = { workspace = true, features = ["io-util"] } -uuid = { workspace = true } diff --git a/crates/old-p2p/crates/proto/src/lib.rs b/crates/old-p2p/crates/proto/src/lib.rs deleted file mode 100644 index ca4c6846f..000000000 --- a/crates/old-p2p/crates/proto/src/lib.rs +++ /dev/null @@ -1,88 +0,0 @@ -//! Temporary library for easier binary encoding/decoding. -//! -//! Eventually these will be deprecated by macros but I can't find one which supports large payloads (basically it needs to write to async stream not in-memory buffer) -> Binario is my own prototype of a Rust library to do this but it's not prod ready yet. -//! - -use thiserror::Error; -use uuid::Uuid; - -// TODO: Remove this from this crate cause it's a leak of responsibility. -#[derive(Debug, Error)] -#[error(transparent)] -pub enum SpaceTunnelIdentityErr { - #[error("{0}")] - Dalek(#[from] ed25519_dalek::ed25519::Error), - #[error("Invalid key length")] - InvalidKeyLength, -} - -pub mod decode { - use crate::SpaceTunnelIdentityErr; - - use super::{Error, Uuid}; - use tokio::io::{AsyncRead, AsyncReadExt}; - - #[derive(Error, Debug)] - pub enum Error { - #[error("IoError({0})")] - IoError(#[from] std::io::Error), - #[error("UuidFormatError({0})")] - UuidFormatError(#[from] uuid::Error), - #[error("NameFormatError({0})")] - NameFormatError(#[from] std::string::FromUtf8Error), - #[error("InvalidRemoteIdentity({0})")] - InvalidRemoteIdentity(#[from] SpaceTunnelIdentityErr), - } - - /// Deserialize uuid as it's fixed size data. - pub async fn uuid(stream: &mut (impl AsyncRead + Unpin)) -> Result { - let mut buf = vec![0u8; 16]; - stream.read_exact(&mut buf).await?; - Uuid::from_slice(&buf).map_err(Into::into) - } - - /// Deserialize string as it's u16 length and data. - pub async fn string(stream: &mut (impl AsyncRead + Unpin)) -> Result { - let len = stream.read_u16_le().await?; - - let mut buf = vec![0u8; len as usize]; - stream.read_exact(&mut buf).await?; - - String::from_utf8(buf).map_err(Into::into) - } - - /// Deserialize buf as it's u16 length and data. - pub async fn buf(stream: &mut (impl AsyncRead + Unpin)) -> Result, Error> { - let len = stream.read_u32_le().await?; - - let mut buf = vec![0u8; len as usize]; - stream.read_exact(&mut buf).await?; - - Ok(buf) - } -} - -pub mod encode { - use super::Uuid; - - /// Serialize uuid as it's fixed size data. - pub fn uuid(buf: &mut Vec, uuid: &Uuid) { - buf.extend(uuid.as_bytes()); - } - - /// Serialize string as it's u16 length and data. - pub fn string(buf: &mut Vec, s: &str) { - // TODO: Chunk this so it will never error - assert!(s.len() <= u16::MAX as usize, "String is too long!"); - buf.extend_from_slice(&(s.len() as u16).to_le_bytes()); - buf.extend(s.as_bytes()); - } - - /// Serialize buf as it's u16 length and data. - pub fn buf(buf: &mut Vec, b: &[u8]) { - // TODO: Chunk this so it will never error - assert!(b.len() <= u32::MAX as usize, "Buf is too long!"); - buf.extend_from_slice(&(b.len() as u32).to_le_bytes()); - buf.extend(b); - } -} diff --git a/crates/old-p2p/crates/tunnel/Cargo.toml b/crates/old-p2p/crates/tunnel/Cargo.toml deleted file mode 100644 index e9f1e0114..000000000 --- a/crates/old-p2p/crates/tunnel/Cargo.toml +++ /dev/null @@ -1,17 +0,0 @@ -[package] -name = "sd-old-p2p-tunnel" -version = "0.1.0" - -authors = ["Oscar Beaumont "] -edition.workspace = true -license.workspace = true -repository.workspace = true - -[dependencies] -# Spacedrive Sub-crates -sd-old-p2p = { path = "../../" } -sd-old-p2p-proto = { path = "../proto" } - -# Workspace dependencies -thiserror = { workspace = true } -tokio = { workspace = true, features = ["io-util"] } diff --git a/crates/old-p2p/crates/tunnel/src/lib.rs b/crates/old-p2p/crates/tunnel/src/lib.rs deleted file mode 100644 index 48ae2211a..000000000 --- a/crates/old-p2p/crates/tunnel/src/lib.rs +++ /dev/null @@ -1,143 +0,0 @@ -//! A system for creating encrypted tunnels between peers over untrusted connections. - -use std::{ - io, - pin::Pin, - task::{Context, Poll}, -}; - -use sd_old_p2p_proto::{decode, encode}; -use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt, ReadBuf}; - -use thiserror::Error; - -use sd_old_p2p::{Identity, IdentityErr, RemoteIdentity, UnicastStream}; - -#[derive(Debug, Error)] -pub enum TunnelError { - #[error("Error writing discriminator.")] - DiscriminatorWriteError, - #[error("Error reading discriminator. Is this stream actually a tunnel?")] - DiscriminatorReadError, - #[error("Invalid discriminator. Is this stream actually a tunnel?")] - InvalidDiscriminator, - #[error("Error sending library id: {0:?}")] - ErrorSendingLibraryId(io::Error), - #[error("Error receiving library identity: {0:?}")] - ErrorReceivingLibraryIdentity(decode::Error), - #[error("Error decoding library identity: {0:?}")] - ErrorDecodingLibraryIdentity(IdentityErr), -} - -/// An encrypted tunnel between two libraries. -/// -/// This sits on top of the existing node to node encryption provided by Quic. -/// -/// It's primarily designed to avoid an attack where traffic flows: -/// node <-> attacker node <-> node -/// The attackers node can't break TLS but if they get in the middle they can present their own node identity to each side and then intercept library related traffic. -/// To avoid that we use this tunnel to encrypt all library related traffic so it can only be decoded by another instance of the same library. -#[derive(Debug)] -pub struct Tunnel { - stream: UnicastStream, - library_remote_id: RemoteIdentity, -} - -impl Tunnel { - /// Create a new tunnel. - /// - /// This should be used by the node that initiated the request which this tunnel is used for. - pub async fn initiator( - mut stream: UnicastStream, - library_identity: &Identity, - ) -> Result { - stream - .write_all(b"T") - .await - .map_err(|_| TunnelError::DiscriminatorWriteError)?; - - let mut buf = vec![]; - encode::buf(&mut buf, &library_identity.to_remote_identity().get_bytes()); - stream - .write_all(&buf) - .await - .map_err(TunnelError::ErrorSendingLibraryId)?; - - // TODO: Do encryption things - - Ok(Self { - stream, - library_remote_id: library_identity.to_remote_identity(), - }) - } - - /// Create a new tunnel. - /// - /// This should be used by the node that responded to the request which this tunnel is used for. - pub async fn responder(mut stream: UnicastStream) -> Result { - let discriminator = stream - .read_u8() - .await - .map_err(|_| TunnelError::DiscriminatorReadError)?; - if discriminator != b'T' { - return Err(TunnelError::InvalidDiscriminator); - } - - // TODO: Blindly decoding this from the stream is not secure. We need a cryptographic handshake here to prove the peer on the other ends is holding the private key. - let library_remote_id = decode::buf(&mut stream) - .await - .map_err(TunnelError::ErrorReceivingLibraryIdentity)?; - - let library_remote_id = RemoteIdentity::from_bytes(&library_remote_id) - .map_err(TunnelError::ErrorDecodingLibraryIdentity)?; - - // TODO: Do encryption things - - Ok(Self { - library_remote_id, - stream, - }) - } - - /// Get the `RemoteIdentity` of the peer on the other end of the tunnel. - pub fn node_remote_identity(&self) -> RemoteIdentity { - self.stream.remote_identity() - } - - /// Get the `RemoteIdentity` of the library instance on the other end of the tunnel. - pub fn library_remote_identity(&self) -> RemoteIdentity { - self.library_remote_id - } -} - -impl AsyncRead for Tunnel { - fn poll_read( - self: Pin<&mut Self>, - cx: &mut Context<'_>, - buf: &mut ReadBuf<'_>, - ) -> Poll> { - // TODO: Do decryption - - Pin::new(&mut self.get_mut().stream).poll_read(cx, buf) - } -} - -impl AsyncWrite for Tunnel { - fn poll_write( - self: Pin<&mut Self>, - cx: &mut Context<'_>, - buf: &[u8], - ) -> Poll> { - // TODO: Do encryption - - Pin::new(&mut self.get_mut().stream).poll_write(cx, buf) - } - - fn poll_flush(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - Pin::new(&mut self.get_mut().stream).poll_flush(cx) - } - - fn poll_shutdown(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - Pin::new(&mut self.get_mut().stream).poll_shutdown(cx) - } -} diff --git a/crates/old-p2p/src/hook.rs b/crates/old-p2p/src/hook.rs deleted file mode 100644 index c3d9e0cbe..000000000 --- a/crates/old-p2p/src/hook.rs +++ /dev/null @@ -1,141 +0,0 @@ -use std::{ - collections::{BTreeSet, HashSet}, - fmt, - net::SocketAddr, - sync::Arc, -}; - -use flume::Sender; -use tokio::sync::oneshot; - -use crate::{Peer, PeerConnectionCandidate, RemoteIdentity}; - -#[derive(Debug, Clone)] -pub enum HookEvent { - /// `P2P::service` has changed - MetadataModified, - - /// A new listener was registered with the P2P system. - ListenerRegistered(ListenerId), - /// A listener's address was added. - ListenerAddrAdded(ListenerId, SocketAddr), - /// A listener's address was removed. - ListenerAddrRemoved(ListenerId, SocketAddr), - /// A listener was unregistered from the P2P system. - ListenerUnregistered(ListenerId), - - /// A peer was inserted into `P2P::peers` - /// This peer could have connected to or have been discovered by a hook. - PeerAvailable(Arc), - /// A peer was removed from `P2P::peers` - /// This is due to it no longer being discovered, containing no active connections or available connection methods. - PeerUnavailable(RemoteIdentity), - - /// A peer was discovered by a hook - /// This will fire for *every peer* per every *hook* that discovers it. - PeerDiscoveredBy(HookId, Arc), - /// A hook expired a peer - /// This will fire for *every peer* per every *hook* that discovers it. - PeerExpiredBy(HookId, RemoteIdentity), - - /// "Connections" are an internal concept to the P2P library but they will be automatically triggered by `Peer::new_stream`. - /// They are a concept users of the application may care about so they are exposed here. - - /// A new listener established a connection with a peer - PeerConnectedWith(ListenerId, Arc), - /// A connection closed with a peer. - PeerDisconnectedWith(ListenerId, RemoteIdentity), - - /// Your hook or the P2P system was told to shutdown. - Shutdown { - // We can detect when this guard is dropped, it doesn't need to be used. - _guard: ShutdownGuard, - }, -} - -#[derive(Debug)] -pub struct ShutdownGuard(pub(crate) Option>); - -impl ShutdownGuard { - pub(crate) fn new() -> (Self, oneshot::Receiver<()>) { - let (tx, rx) = oneshot::channel(); - (Self(Some(tx)), rx) - } -} - -impl Drop for ShutdownGuard { - fn drop(&mut self) { - if let Some(tx) = self.0.take() { - let _ = tx.send(()); - } - } -} - -impl Clone for ShutdownGuard { - fn clone(&self) -> Self { - Self(None) - } -} - -#[derive(Debug, Clone, Copy, Hash, Eq, PartialEq)] -pub struct HookId(pub(crate) usize); - -#[derive(Debug, Clone, Copy, Hash, Eq, PartialEq)] -pub struct ListenerId(pub(crate) usize); - -impl From for HookId { - fn from(value: ListenerId) -> Self { - Self(value.0) - } -} - -#[derive(Debug)] -pub(crate) struct Hook { - /// A name used for debugging purposes. - pub(crate) name: &'static str, - /// A channel to send events to the hook. - /// This hooks implementing will be responsible for subscribing to this channel. - pub(crate) tx: Sender, - /// If this hook is a listener this will be set. - pub(crate) listener: Option, -} - -impl Hook { - pub fn send(&self, event: HookEvent) { - let _ = self.tx.send(event); - } - - pub fn acceptor( - &self, - id: ListenerId, - peer: &Arc, - addrs: &BTreeSet, - ) { - if let Some(listener) = &self.listener { - (listener.acceptor.0)(id, peer, addrs); - } - } -} - -#[derive(Debug)] -pub(crate) struct ListenerData { - /// The address the listener is bound to. - /// These will be advertised by any discovery methods attached to the P2P system. - pub addrs: HashSet, - /// This is a function over a channel because we need to ensure the code runs prior to the peer being emitted to the application. - /// If not the peer would have no registered way to connect to it initially which would be confusing. - #[allow(clippy::type_complexity)] - pub acceptor: HandlerFn< - Arc, &BTreeSet) + Send + Sync>, - >, -} - -/// A little wrapper for functions to make them `Debug`. -#[derive(Clone)] -pub(crate) struct HandlerFn(pub(crate) F); - -impl fmt::Debug for HandlerFn { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "HandlerFn") - } -} diff --git a/crates/old-p2p/src/hooks.rs b/crates/old-p2p/src/hooks.rs deleted file mode 100644 index f8966e7eb..000000000 --- a/crates/old-p2p/src/hooks.rs +++ /dev/null @@ -1,9 +0,0 @@ -//! Components implemented as P2P hooks. -//! -//! Although these are included within `sd_p2p` you could be implemented in userspace. - -mod mdns; -mod quic; - -pub use mdns::Mdns; -pub use quic::{Libp2pPeerId, QuicHandle, QuicTransport, RelayServerEntry}; diff --git a/crates/old-p2p/src/hooks/mdns.rs b/crates/old-p2p/src/hooks/mdns.rs deleted file mode 100644 index 81511f402..000000000 --- a/crates/old-p2p/src/hooks/mdns.rs +++ /dev/null @@ -1,234 +0,0 @@ -//! mDNS-based service discovery. -//! -//! This uses [mdns-sd](https://docs.rs/mdns-sd) under the hood. - -use std::{ - collections::HashMap, net::SocketAddr, pin::Pin, str::FromStr, sync::Arc, time::Duration, -}; - -use flume::{bounded, Receiver}; -use mdns_sd::{ServiceDaemon, ServiceEvent, ServiceInfo}; -use tokio::time::{sleep_until, Instant, Sleep}; -use tracing::{error, trace, warn}; - -use crate::{HookEvent, HookId, PeerConnectionCandidate, RemoteIdentity, ShutdownGuard, P2P}; - -/// The time between re-advertising the mDNS service. -const MDNS_READVERTISEMENT_INTERVAL: Duration = Duration::from_secs(60); // Every minute re-advertise - -/// Multicast DNS (mDNS) is used for discovery of peers over local networks. -#[derive(Debug)] -pub struct Mdns { - p2p: Arc, - hook_id: HookId, -} - -impl Mdns { - pub fn spawn(p2p: Arc) -> Result { - let (tx, rx) = bounded(15); - let hook_id = p2p.register_hook("mdns", tx); - - start(p2p.clone(), hook_id, rx)?; - - Ok(Self { p2p, hook_id }) - } - - // pub fn is_discovered_by(&self, identity: &RemoteIdentity) -> bool { - // self.p2p - // .peers() - // .get(identity) - // .map(|p| p.discovered_by().contains(&self.hook_id)) - // .unwrap_or(false) - // } - - // pub fn is_connected_with(&self, identity: &RemoteIdentity) -> bool { - // self.p2p - // .peers() - // .get(identity) - // .map(|p| p.is_connected_with_hook(self.hook_id)) - // .unwrap_or(false) - // } - - pub async fn shutdown(self) { - self.p2p.unregister_hook(self.hook_id).await; - } -} - -struct State { - hook_id: HookId, - p2p: Arc, - service_domain: String, - service_name: String, - mdns_daemon: ServiceDaemon, - next_mdns_advertisement: Pin>, -} - -fn start(p2p: Arc, hook_id: HookId, rx: Receiver) -> Result<(), mdns_sd::Error> { - let service_domain = format!("_{}._udp.local.", p2p.app_name()); - let mut state = State { - hook_id, - service_name: format!("{}.{service_domain}", p2p.remote_identity()), - service_domain, - p2p, - mdns_daemon: ServiceDaemon::new()?, - next_mdns_advertisement: Box::pin(sleep_until( - Instant::now() + MDNS_READVERTISEMENT_INTERVAL, - )), - }; - let mdns_service = state.mdns_daemon.browse(&state.service_domain)?; - - tokio::spawn(async move { - loop { - tokio::select! { - Ok(event) = rx.recv_async() => match event { - HookEvent::MetadataModified | HookEvent::ListenerRegistered(_) | HookEvent::ListenerAddrAdded(_, _) | HookEvent::ListenerAddrRemoved(_, _) | HookEvent::ListenerUnregistered(_) => advertise(&mut state), - HookEvent::Shutdown { _guard } => { - shutdown(_guard, &mut state); - break; - }, - _ => continue, - }, - _ = &mut state.next_mdns_advertisement => advertise(&mut state), - Ok(event) = mdns_service.recv_async() => on_event(&state, event) - }; - } - }); - - Ok(()) -} - -fn advertise(state: &mut State) { - let mut ports_to_service = HashMap::new(); - for addr in state.p2p.listeners().iter().flat_map(|l| l.addrs.clone()) { - ports_to_service - .entry(addr.port()) - .or_insert_with(Vec::new) - .push(addr.ip()); - } - - let meta = state.p2p.metadata().clone(); - for (port, ips) in ports_to_service { - let service = ServiceInfo::new( - &state.service_domain, - &state.p2p.remote_identity().to_string(), - &state.service_name, - &*ips, - port, - // TODO: If a piece of metadata overflows a DNS record take care of splitting it across multiple. - Some(meta.clone()), - ) - .map(|s| s.enable_addr_auto()); - - let service = match service { - Ok(service) => service, - Err(err) => { - warn!("error creating mdns service info: {}", err); - continue; - } - }; - - trace!("advertising mdns service: {:?}", service); - match state.mdns_daemon.register(service) { - Ok(()) => {} - Err(err) => warn!("error registering mdns service: {}", err), - } - } - - state.next_mdns_advertisement = - Box::pin(sleep_until(Instant::now() + MDNS_READVERTISEMENT_INTERVAL)); -} - -fn on_event(state: &State, event: ServiceEvent) { - match event { - ServiceEvent::ServiceResolved(info) => { - let Some(identity) = fullname_to_identity(state, info.get_fullname()) else { - return; - }; - - state.p2p.clone().discover_peer( - state.hook_id, - identity, - info.get_properties() - .iter() - .map(|p| (p.key().to_string(), p.val_str().to_string())) - .collect(), - info.get_addresses() - .iter() - .map(|addr| { - PeerConnectionCandidate::SocketAddr(SocketAddr::new(*addr, info.get_port())) - }) - .collect(), - ); - } - ServiceEvent::ServiceRemoved(_, fullname) => { - let Some(identity) = fullname_to_identity(state, &fullname) else { - return; - }; - - if let Some(peer) = state.p2p.peers().get(&identity) { - peer.undiscover_peer(state.hook_id); - } - } - ServiceEvent::SearchStarted(_) - | ServiceEvent::SearchStopped(_) - | ServiceEvent::ServiceFound(_, _) => {} - } -} - -fn fullname_to_identity( - State { - p2p, - service_domain, - .. - }: &State, - fullname: &str, -) -> Option { - let Some(identity) = fullname - .strip_suffix(service_domain) - .map(|s| &s[0..s.len() - 1]) - else { - warn!( - "resolved peer advertising itself with an invalid fullname '{}'", - fullname - ); - return None; - }; - - let Ok(identity) = RemoteIdentity::from_str(identity) else { - warn!("resolved peer advertising itself with an invalid remote identity '{identity}'"); - return None; - }; - - // Prevent discovery of the current peer. - if identity == p2p.remote_identity() { - return None; - } - - Some(identity) -} - -fn shutdown(_guard: ShutdownGuard, state: &mut State) { - if let Ok(chan) = state - .mdns_daemon - .unregister(&state.service_name) - .map_err(|err| { - error!( - "error removing mdns service '{}': {err}", - state.service_name - ); - }) { - let _ = chan.recv(); - }; - - // TODO: Without this mDNS is not sending it goodbye packets without a timeout. Try and remove this cause it makes shutdown slow. - std::thread::sleep(Duration::from_millis(100)); - - match state.mdns_daemon.shutdown() { - Ok(chan) => { - let _ = chan.recv(); - } - Err(err) => { - error!("error shutting down mdns daemon: {err}"); - } - } -} diff --git a/crates/old-p2p/src/hooks/quic.rs b/crates/old-p2p/src/hooks/quic.rs deleted file mode 100644 index b81bae73a..000000000 --- a/crates/old-p2p/src/hooks/quic.rs +++ /dev/null @@ -1,10 +0,0 @@ -//! Quic-based transport. -//! -//! This uses [libp2p](https://docs.rs/libp2p) under the hood. - -pub(super) mod handle; -pub(super) mod transport; -pub(super) mod utils; - -pub use handle::QuicHandle; -pub use transport::{Libp2pPeerId, QuicTransport, RelayServerEntry}; diff --git a/crates/old-p2p/src/hooks/quic/handle.rs b/crates/old-p2p/src/hooks/quic/handle.rs deleted file mode 100644 index 15c667591..000000000 --- a/crates/old-p2p/src/hooks/quic/handle.rs +++ /dev/null @@ -1,122 +0,0 @@ -use std::{ - collections::{HashMap, HashSet}, - sync::{ - atomic::{AtomicBool, Ordering}, - Arc, Mutex, PoisonError, - }, -}; - -use tokio::sync::Notify; - -use crate::{HookId, PeerConnectionCandidate, RemoteIdentity, P2P}; - -/// A handle to the QUIC hook. -/// -/// This allows for manually registering peers, which is required so that we can ask the relay to connect to them. -#[derive(Debug)] -pub struct QuicHandle { - pub(super) shutdown: Notify, - pub(super) p2p: Arc, - pub(super) hook_id: HookId, - pub(super) nodes: Mutex>>, - pub(super) enabled: AtomicBool, - pub(super) connected_via_relay: Mutex>, -} - -impl QuicHandle { - /// A future that resolves when the QUIC hook is shut down. - pub async fn shutdown(&self) { - self.shutdown.notified().await - } - - /// add a new peer to be tracked. - /// - /// This will allow the relay to connect to it. - pub fn track_peer(&self, identity: RemoteIdentity, metadata: HashMap) { - self.nodes - .lock() - .unwrap_or_else(PoisonError::into_inner) - .insert(identity, metadata.clone()); - - if self.enabled.load(Ordering::Relaxed) { - self.p2p.clone().discover_peer( - self.hook_id, - identity, - metadata, - [PeerConnectionCandidate::Relay].into_iter().collect(), - ); - } - } - - /// remove a peer from being tracked. - /// - /// This will stop the relay from trying to connect to it. - pub fn untrack_peer(&self, identity: RemoteIdentity) { - self.nodes - .lock() - .unwrap_or_else(PoisonError::into_inner) - .remove(&identity); - - if self.enabled.load(Ordering::Relaxed) { - if let Some(peer) = self.p2p.peers().get(&identity) { - peer.undiscover_peer(self.hook_id) - } - } - } - - /// remove all peers from being tracked. - pub fn untrack_all(&self) { - let mut nodes = self.nodes.lock().unwrap_or_else(PoisonError::into_inner); - for (node, _) in nodes.drain() { - if let Some(peer) = self.p2p.peers().get(&node) { - peer.undiscover_peer(self.hook_id) - } - } - } - - /// enabled the track peers from being registered to the P2P system. - /// - /// This allows easily removing them when the relay is disabled. - pub fn enable(&self) { - self.enabled.store(true, Ordering::Relaxed); - - for (identity, metadata) in self - .nodes - .lock() - .unwrap_or_else(PoisonError::into_inner) - .iter() - { - self.p2p.clone().discover_peer( - self.hook_id, - *identity, - metadata.clone(), - [PeerConnectionCandidate::Relay].into_iter().collect(), - ); - } - } - - /// disabled tracking the peers from being registered to the P2P system. - pub fn disable(&self) { - self.enabled.store(false, Ordering::Relaxed); - - for (identity, _) in self - .nodes - .lock() - .unwrap_or_else(PoisonError::into_inner) - .iter() - { - if let Some(peer) = self.p2p.peers().get(identity) { - peer.undiscover_peer(self.hook_id) - } - } - } - - /// check if a peer is being relayed. - pub fn is_relayed(&self, identity: RemoteIdentity) -> bool { - self.connected_via_relay - .lock() - .unwrap_or_else(PoisonError::into_inner) - .get(&identity) - .is_some() - } -} diff --git a/crates/old-p2p/src/hooks/quic/transport.rs b/crates/old-p2p/src/hooks/quic/transport.rs deleted file mode 100644 index 262535b5f..000000000 --- a/crates/old-p2p/src/hooks/quic/transport.rs +++ /dev/null @@ -1,928 +0,0 @@ -use std::{ - collections::{BTreeSet, HashMap, HashSet}, - io::{self, ErrorKind}, - net::{Ipv4Addr, Ipv6Addr, SocketAddr}, - str::FromStr, - sync::{atomic::AtomicBool, Arc, Mutex, MutexGuard, PoisonError, RwLock}, - time::Duration, -}; - -use flume::{bounded, Receiver, Sender}; -use futures::future::join_all; -use libp2p::{ - autonat, dcutr, - futures::{AsyncReadExt, AsyncWriteExt, StreamExt}, - multiaddr::Protocol, - noise, relay, - swarm::{dial_opts::DialOpts, NetworkBehaviour, SwarmEvent}, - yamux, Multiaddr, PeerId, Stream, StreamProtocol, Swarm, SwarmBuilder, -}; -use serde::{Deserialize, Serialize}; -use thiserror::Error; -use tokio::{ - net::TcpListener, - sync::{mpsc, oneshot}, - time::timeout, -}; -use tokio_util::compat::FuturesAsyncReadCompatExt; -use tracing::{debug, error, info, warn}; -use uuid::Uuid; - -use super::{ - handle::QuicHandle, - utils::{ - identity_to_libp2p_keypair, remote_identity_to_libp2p_peerid, socketaddr_to_multiaddr, - }, -}; -use crate::{ - hooks::quic::utils::multiaddr_to_socketaddr, identity::REMOTE_IDENTITY_LEN, ConnectionRequest, - HookEvent, ListenerId, PeerConnectionCandidate, RemoteIdentity, UnicastStream, P2P, -}; - -const PROTOCOL: StreamProtocol = StreamProtocol::new("/sdp2p/1"); - -/// [libp2p::PeerId] for debugging purposes only. -#[derive(Debug)] -#[allow(dead_code)] -pub struct Libp2pPeerId(libp2p::PeerId); - -#[derive(Debug)] -enum InternalEvent { - RegisterListener { - id: ListenerId, - ipv4: bool, - addr: SocketAddr, - result: oneshot::Sender>, - }, - UnregisterListener { - id: ListenerId, - ipv4: bool, - result: oneshot::Sender>, - }, - RegisterRelays { - relays: Vec, - result: oneshot::Sender>, - }, - RegisterPeerAddr { - // These can be socket addr's or FQDN's - addrs: HashSet, - }, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct RelayServerEntry { - id: Uuid, - peer_id: String, - addrs: Vec, -} - -#[derive(NetworkBehaviour)] -struct MyBehaviour { - stream: libp2p_stream::Behaviour, - // TODO: Can this be optional? - relay: relay::client::Behaviour, - // TODO: Can this be optional? - autonat: autonat::Behaviour, - // TODO: Can this be optional? - dcutr: dcutr::Behaviour, -} - -#[derive(Debug, Error)] -pub enum QuicTransportError { - #[error("Failed to modify the SwarmBuilder: {0}")] - SwarmBuilderCreation(String), - #[error("Internal response channel closed: {0}")] - SendChannelClosed(String), - #[error("Internal response channel closed: {0}")] - ReceiveChannelClosed(#[from] oneshot::error::RecvError), - #[error("Failed internal event: {0}")] - InternalEvent(String), - #[error("Failed to create the Listener: {0}")] - ListenerSetup(std::io::Error), -} - -/// Transport using Quic to establish a connection between peers. -/// This uses `libp2p` internally. -#[derive(Debug)] -pub struct QuicTransport { - id: ListenerId, - p2p: Arc, - internal_tx: Sender, - relay_config: Mutex>, - ipv4_listener: Mutex, - ipv6_listener: Mutex, - handle: Arc, -} - -#[derive(Debug, Clone, Default)] -enum ListenerInfo { - /// The listener is disabled. - #[default] - Disabled, - /// The user requested a specific port. - Absolute(SocketAddr), - /// The user requested a random port. - /// The value contains the selected port. - Random(SocketAddr), -} - -impl QuicTransport { - /// Spawn the `QuicTransport` and register it with the P2P system. - /// Be aware spawning this does nothing unless you call `Self::set_ipv4_enabled`/`Self::set_ipv6_enabled` to enable the listeners. - pub fn spawn(p2p: Arc) -> Result<(Self, Libp2pPeerId), QuicTransportError> { - let keypair = identity_to_libp2p_keypair(p2p.identity()); - let libp2p_peer_id = Libp2pPeerId(keypair.public().to_peer_id()); - - let (tx, rx) = bounded(15); - let (internal_tx, internal_rx) = bounded(15); - let (connect_tx, connect_rx) = mpsc::channel(15); - let id = p2p.register_listener("libp2p-quic", tx, move |listener_id, peer, _addrs| { - // TODO: I don't love this always being registered. Really it should only show up if the other device is online (do a ping-type thing)??? - peer.clone() - .listener_available(listener_id, connect_tx.clone()); - }); - - let swarm = SwarmBuilder::with_existing_identity(keypair) - .with_tokio() - .with_quic() - .with_relay_client(noise::Config::new, yamux::Config::default) - .map_err(|err| QuicTransportError::SwarmBuilderCreation(err.to_string()))? - .with_behaviour(|keypair, relay_behaviour| MyBehaviour { - stream: libp2p_stream::Behaviour::new(), - relay: relay_behaviour, - autonat: autonat::Behaviour::new(keypair.public().to_peer_id(), Default::default()), - dcutr: dcutr::Behaviour::new(keypair.public().to_peer_id()), - }) - .map_err(|err| QuicTransportError::SwarmBuilderCreation(err.to_string()))? - .with_swarm_config(|c| c.with_idle_connection_timeout(Duration::from_secs(60))) - .build(); - - let handle = Arc::new(QuicHandle { - shutdown: Default::default(), - p2p: p2p.clone(), - hook_id: id.into(), - nodes: Default::default(), - enabled: AtomicBool::new(true), - connected_via_relay: Default::default(), - }); - - tokio::spawn(start( - p2p.clone(), - id, - swarm, - rx, - internal_rx, - connect_rx, - handle.clone(), - )); - - Ok(( - Self { - id, - p2p, - internal_tx, - relay_config: Mutex::new(Vec::new()), - ipv4_listener: Default::default(), - ipv6_listener: Default::default(), - handle, - }, - libp2p_peer_id, - )) - } - - /// Configure the relay servers to use. - /// This method will replace any existing relay servers. - pub async fn set_relay_config( - &self, - relays: Vec, - ) -> Result<(), QuicTransportError> { - let (tx, rx) = oneshot::channel(); - let event = InternalEvent::RegisterRelays { - relays: relays.clone(), - result: tx, - }; - - self.internal_tx - .send(event) - .map_err(|e| QuicTransportError::SendChannelClosed(e.to_string()))?; - - let result = rx - .await - .map_err(QuicTransportError::ReceiveChannelClosed) - .and_then(|r| r.map_err(QuicTransportError::InternalEvent)); - - if result.is_ok() { - *self - .relay_config - .lock() - .unwrap_or_else(PoisonError::into_inner) = relays; - } - - result - } - - pub fn get_relay_config(&self) -> Vec { - self.relay_config - .lock() - .unwrap_or_else(PoisonError::into_inner) - .clone() - } - - pub fn set_manual_peer_addrs(&self, addrs: HashSet) { - self.internal_tx - .send(InternalEvent::RegisterPeerAddr { addrs }) - .ok(); - } - - // `None` on the port means disabled. Use `0` for random port. - pub async fn set_ipv4_enabled(&self, port: Option) -> Result<(), QuicTransportError> { - self.setup_listener( - port.map(|p| SocketAddr::from((Ipv4Addr::UNSPECIFIED, p))), - true, - |this| { - this.ipv4_listener - .lock() - .unwrap_or_else(PoisonError::into_inner) - }, - ) - .await - } - - pub async fn set_ipv6_enabled(&self, port: Option) -> Result<(), QuicTransportError> { - self.setup_listener( - port.map(|p| SocketAddr::from((Ipv6Addr::UNSPECIFIED, p))), - false, - |this| { - this.ipv6_listener - .lock() - .unwrap_or_else(PoisonError::into_inner) - }, - ) - .await - } - - async fn setup_listener( - &self, - addr: Option, - ipv4: bool, - get_listener: impl Fn(&Self) -> MutexGuard, - ) -> Result<(), QuicTransportError> { - let mut desired = match addr { - Some(addr) if addr.port() == 0 => ListenerInfo::Random(addr), - Some(addr) => ListenerInfo::Absolute(addr), - None => ListenerInfo::Disabled, - }; - - let (tx, rx) = oneshot::channel(); - let event = { - let listener_state = get_listener(self).clone(); - - match (listener_state, &mut desired) { - // Desired state is the same as current state - // This is designed to preserve the random port that was determined earlier, making this operation idempotent. - (ListenerInfo::Disabled, ListenerInfo::Disabled) - | (ListenerInfo::Absolute(_), ListenerInfo::Absolute(_)) - | (ListenerInfo::Random(_), ListenerInfo::Random(_)) => return Ok(()), - - // We are enabled and want to be disabled - (_, ListenerInfo::Disabled) => InternalEvent::UnregisterListener { - id: self.id, - ipv4, - result: tx, - }, - - // We are any state (but not the same as the desired state) and want to be enabled - (_, ListenerInfo::Random(ref mut addr)) - | (_, ListenerInfo::Absolute(ref mut addr)) => { - // We mutable assign back to `desired` so it can be saved if this operation succeeds. - if addr.port() == 0 { - addr.set_port( - TcpListener::bind(*addr) - .await - .map_err(QuicTransportError::ListenerSetup)? - .local_addr() - .map_err(QuicTransportError::ListenerSetup)? - .port(), - ); - } - - InternalEvent::RegisterListener { - id: self.id, - ipv4, - addr: *addr, - result: tx, - } - } - } - }; - - self.internal_tx - .send(event) - .map_err(|e| QuicTransportError::SendChannelClosed(e.to_string()))?; - - rx.await - .map_err(QuicTransportError::ReceiveChannelClosed) - .and_then(|r| r.map_err(QuicTransportError::InternalEvent))?; - - *get_listener(self) = desired; - - Ok(()) - } - - pub fn handle(&self) -> Arc { - self.handle.clone() - } - - pub async fn shutdown(self) { - self.p2p.unregister_hook(self.id.into()).await; - } -} - -async fn start( - p2p: Arc, - id: ListenerId, - mut swarm: Swarm, - rx: Receiver, - internal_rx: Receiver, - mut connect_rx: mpsc::Receiver, - handle: Arc, -) { - let mut ipv4_listener = None; - let mut ipv6_listener = None; - - let mut control = swarm.behaviour().stream.new_control(); - #[allow(clippy::unwrap_used)] // TODO: Error handling - let mut incoming = control.accept(PROTOCOL).unwrap(); - let map = Arc::new(RwLock::new(HashMap::new())); - let mut relay_config = Vec::new(); - let mut registered_relays = HashMap::new(); - let mut manual_addrs = HashSet::new(); - let mut manual_addr_dial_attempts = HashMap::new(); - let (manual_peers_dial_tx, mut manual_peers_dial_rx) = mpsc::channel(15); - let mut interval = tokio::time::interval(Duration::from_secs(60)); - let mut peer_id_to_addrs: HashMap> = HashMap::new(); - - loop { - tokio::select! { - Ok(event) = rx.recv_async() => match event { - HookEvent::PeerExpiredBy(_, identity) => { - let Some(peer) = p2p.peers.read().unwrap_or_else(PoisonError::into_inner).get(&identity).cloned() else { - continue; - }; - - let peer_id = remote_identity_to_libp2p_peerid(&identity); - let addrs = { - let state = peer.state.read().unwrap_or_else(PoisonError::into_inner); - - get_addrs(peer_id, &relay_config, state.discovered.values().flatten()) - }; - - - let mut control = control.clone(); - tokio::spawn(async move { - match timeout(Duration::from_secs(5), control.open_stream_with_addrs( - peer_id, - PROTOCOL, - addrs - )).await { - Ok(Ok(_)) => {} - Err(_) | Ok(Err(_)) => peer.disconnected_from(id), - }; - }); - }, - HookEvent::Shutdown { _guard } => { - let connected_peers = swarm.connected_peers().cloned().collect::>(); - for peer_id in connected_peers { - let _ = swarm.disconnect_peer_id(peer_id); - } - - if let Some((id, _)) = ipv4_listener.take() { - let _ = swarm.remove_listener(id); - } - if let Some((id, _)) = ipv6_listener.take() { - let _ = swarm.remove_listener(id); - } - - // TODO: We don't break the event loop so libp2p can be polled to keep cleaning up. - // break; - }, - _ => {}, - }, - Some((peer_id, mut stream)) = incoming.next() => { - let p2p = p2p.clone(); - let map = map.clone(); - let peer_id_to_addrs = peer_id_to_addrs.clone(); - - tokio::spawn(async move { - let mut mode = [0; 1]; - match stream.read_exact(&mut mode).await { - Ok(_) => {}, - Err(e) => { - warn!("Failed to read mode with libp2p::PeerId({peer_id:?}): {e:?}"); - return; - } - } - - match mode[0] { - // This is the regular mode for relay or mDNS - 0 => {}, - // Used for manual peers to discover the peers identity. - 1 => { - match stream.write_all(&p2p.identity().to_remote_identity().get_bytes()).await { - Ok(_) => {}, - Err(e) => { - warn!("Failed to write remote identity in mode 1 with libp2p::PeerId({peer_id:?}): {e:?}"); - return; - } - } - } - mode => { - warn!("Peer libp2p::PeerId({peer_id:?}) attempted to use invalid mode '{mode}'"); - return; - } - } - - let mut actual = [0; REMOTE_IDENTITY_LEN]; - match stream.read_exact(&mut actual).await { - Ok(_) => {}, - Err(e) => { - warn!("Failed to read remote identity with libp2p::PeerId({peer_id:?}): {e:?}"); - return; - }, - } - let identity = match RemoteIdentity::from_bytes(&actual) { - Ok(i) => i, - Err(e) => { - warn!("Failed to parse remote identity with libp2p::PeerId({peer_id:?}): {e:?}"); - return; - }, - }; - - // We need to go `PeerId -> RemoteIdentity` but as `PeerId` is a hash that's impossible. - // So to make this work the connection initiator will send their remote identity. - // It is however untrusted as they could send anything, so we convert it to a PeerId and check it matches the PeerId for this connection. - // If it matches, we are certain they own the private key as libp2p takes care of ensuring the PeerId is trusted. - let remote_identity_peer_id = remote_identity_to_libp2p_peerid(&identity); - if peer_id != remote_identity_peer_id { - warn!("Derived remote identity '{remote_identity_peer_id:?}' does not match libp2p::PeerId({peer_id:?})"); - return; - } - map.write().unwrap_or_else(PoisonError::into_inner).insert(peer_id, identity); - - let remote_metadata = match read_metadata(&mut stream, &p2p).await { - Ok(metadata) => metadata, - Err(e) => { - warn!("Failed to read metadata from '{}': {e}", identity); - return; - }, - }; - - // For mode 1 the stream will be dropped now - if mode[0] != 1 { - let stream = UnicastStream::new(identity, stream.compat()); - p2p.connected_to_incoming( - id, - remote_metadata, - stream, - ); - } else { - p2p.discover_peer(id.into(), identity, remote_metadata, peer_id_to_addrs.get(&peer_id).into_iter().flatten().map(|v| PeerConnectionCandidate::Manual(*v)).collect()); - } - - debug!("established inbound stream with '{}'", identity); - }); - }, - event = swarm.select_next_some() => match event { - SwarmEvent::ConnectionEstablished { peer_id, endpoint, connection_id, .. } => { - if let Some(addr) = multiaddr_to_socketaddr(endpoint.get_remote_address()) { - peer_id_to_addrs.entry(peer_id).or_default().insert(addr); - } - - if let Some((addr, socket_addr)) = manual_addr_dial_attempts.remove(&connection_id) { - let mut control = control.clone(); - let map = map.clone(); - let p2p = p2p.clone(); - let self_remote_identity = p2p.identity().to_remote_identity(); - debug!("Successfully dialled manual peer '{addr}' found peer '{peer_id}'. Opening stream to get peer information..."); - - - tokio::spawn(async move { - match control.open_stream_with_addrs( - peer_id, - PROTOCOL, - vec![socketaddr_to_multiaddr(&socket_addr)] - ).await { - Ok(mut stream) => { - match stream.write_all(&[1]).await { - Ok(_) => {}, - Err(e) => { - warn!("Failed to write mode 1 to manual peer '{addr}': {e}"); - return; - }, - } - - let mut identity = [0; REMOTE_IDENTITY_LEN]; - match stream.read_exact(&mut identity).await { - Ok(_) => {}, - Err(e) => { - warn!("Failed to read remote identity from manual peer '{addr}': {e}"); - return; - }, - } - let identity = match RemoteIdentity::from_bytes(&identity) { - Ok(i) => i, - Err(e) => { - warn!("Failed to parse remote identity from manual peer '{addr}': {e}"); - return; - }, - }; - - info!("Successfully connected with manual peer '{addr}' and found peer '{identity}'"); - - map.write().unwrap_or_else(PoisonError::into_inner).insert(peer_id, identity); - - match stream.write_all(&self_remote_identity.get_bytes()).await { - Ok(_) => { - debug!("Established manual connection with '{identity}'"); - - let remote_metadata = match send_metadata(&mut stream, &p2p).await { - Ok(metadata) => metadata, - Err(e) => { - warn!("Failed to send metadata to manual peer '{identity}': {e}"); - return; - }, - }; - - p2p.discover_peer(id.into(), identity, remote_metadata, BTreeSet::from([PeerConnectionCandidate::Manual(socket_addr)])); - }, - Err(e) => { - warn!("Failed to write remote identity to manual peer '{identity}': {e}"); - return; - }, - } - - stream.close().await.ok(); - }, - Err(e) => { - warn!("Failed to open stream with manual peer '{addr}': {e}"); - }, - } - }); - } - - if endpoint.is_relayed() { - if let Some((remote_identity, _)) = handle.nodes.lock() - .unwrap_or_else(PoisonError::into_inner) - .iter() - .find(|(i, _)| remote_identity_to_libp2p_peerid(i) == peer_id) { - handle.connected_via_relay.lock() - .unwrap_or_else(PoisonError::into_inner) - .insert(*remote_identity); - } - } - } - SwarmEvent::ConnectionClosed { peer_id, num_established: 0, connection_id, endpoint, .. } => { - if let Some(addr) = multiaddr_to_socketaddr(endpoint.get_remote_address()) { - peer_id_to_addrs.entry(peer_id).or_default().remove(&addr); - } - - if let Some((addr, _)) = manual_addr_dial_attempts.remove(&connection_id) { - warn!("Failed to establish manual connection with '{addr}'"); - } - - let Some(identity) = map.write().unwrap_or_else(PoisonError::into_inner).remove(&peer_id) else { - warn!("Tried to remove a peer that wasn't in the map."); - continue; - }; - - let peers = p2p.peers.read().unwrap_or_else(PoisonError::into_inner); - let Some(peer) = peers.get(&identity) else { - warn!("Tried to remove a peer that wasn't in the P2P system."); - continue; - }; - - peer.disconnected_from(id); - }, - _ => {} - }, - Ok(event) = internal_rx.recv_async() => match event { - InternalEvent::RegisterListener { id, ipv4, addr, result } => { - match swarm.listen_on(socketaddr_to_multiaddr(&addr)) { - Ok(libp2p_listener_id) => { - let this = match ipv4 { - true => &mut ipv4_listener, - false => &mut ipv6_listener, - }; - // TODO: Diff the `addr` & if it's changed actually update it - if this.is_none() { - *this = Some((libp2p_listener_id, addr)); - p2p.register_listener_addr(id, addr); - } - - let _ = result.send(Ok(())); - }, - Err(e) => { - let _ = result.send(Err(e.to_string())); - }, - } - }, - InternalEvent::UnregisterListener { id, ipv4, result } => { - let this = match ipv4 { - true => &mut ipv4_listener, - false => &mut ipv6_listener, - }; - if let Some((addr_id, addr)) = this.take() { - if swarm.remove_listener(addr_id) { - p2p.unregister_listener_addr(id, addr); - } - } - let _ = result.send(Ok(())); - }, - InternalEvent::RegisterRelays { relays, result } => { - // TODO: We should only add some of the relays - This is discussion in P2P documentation about the Relay - let mut err = None; - for relay in &relays { - let peer_id = match PeerId::from_str(&relay.peer_id) { - Ok(peer_id) => peer_id, - Err(err) => { - error!("Failed to parse Relay peer ID '{}': {err:?}", relay.peer_id); - continue; - }, - }; - - let addrs = relay - .addrs - .iter() - .map(socketaddr_to_multiaddr) - .collect::>(); - - for addr in addrs { - swarm - .behaviour_mut() - .autonat - .add_server(peer_id, Some(addr.clone())); - swarm.add_peer_address(peer_id, addr); - } - - match swarm.listen_on( - Multiaddr::empty() - .with(Protocol::Memory(40)) - .with(Protocol::P2p(peer_id)) - .with(Protocol::P2pCircuit) - ) { - Ok(listener_id) => { - for addr in &relay.addrs { - registered_relays.insert(*addr, listener_id); - } - }, - Err(e) => { - err = Some(format!("Failed to listen on relay server '{}': {e}", relay.id)); - break; - }, - } - } - - if let Some(err) = err { - let _ = result.send(Err(err)); - continue; - } - - // Cleanup connections to relays that are no longer in the config - // We intentionally do this after establishing new connections so we don't have a gap in connectivity - for (addr, listener_id) in ®istered_relays { - if relays.iter().any(|e| e.addrs.contains(addr)) { - continue; - } - - swarm.remove_listener(*listener_id); - } - - relay_config = relays; - - result.send(Ok(())).ok(); - }, - InternalEvent::RegisterPeerAddr { addrs } => { - manual_addrs = addrs; - interval.reset_immediately(); - } - }, - Some(req) = connect_rx.recv() => { - let mut control = control.clone(); - let self_remote_identity = p2p.identity().to_remote_identity(); - let map = map.clone(); - let p2p = p2p.clone(); - let peer_id = remote_identity_to_libp2p_peerid(&req.to); - let addrs = get_addrs(peer_id, &relay_config, req.addrs.iter()); - - tokio::spawn(async move { - match control.open_stream_with_addrs( - peer_id, - PROTOCOL, - addrs, - ).await { - Ok(mut stream) => { - map.write().unwrap_or_else(PoisonError::into_inner).insert(peer_id, req.to); - - // We are in mode `0` so we send a 0 before the remote identity. - let mut buf = [0; REMOTE_IDENTITY_LEN + 1]; - buf[1..].copy_from_slice(&self_remote_identity.get_bytes()); - - match stream.write_all(&buf).await { - Ok(_) => { - debug!("Established outbound stream with '{}'", req.to); - - let remote_metadata = match send_metadata(&mut stream, &p2p).await { - Ok(metadata) => metadata, - Err(e) => { - let _ = req.tx.send(Err(e)); - return; - }, - }; - - p2p.connected_to_outgoing(id, remote_metadata, req.to); - - let _ = req.tx.send(Ok(UnicastStream::new(req.to, stream.compat()))); - }, - Err(e) => { - let _ = req.tx.send(Err(e.to_string())); - }, - } - }, - Err(e) => { - let _ = req.tx.send(Err(e.to_string())); - }, - } - }); - } - Some((addr, socket_addr)) = manual_peers_dial_rx.recv() => { - let opts = DialOpts::unknown_peer_id() - .address(socketaddr_to_multiaddr(&socket_addr)) - .build(); - - manual_addr_dial_attempts.insert(opts.connection_id(), (addr, socket_addr)); - match swarm.dial(opts) { - Ok(_) => debug!("Dialling manual peer '{socket_addr}'"), - Err(err) => warn!("Failed to dial manual peer '{socket_addr}': {err}"), - } - } - _ = interval.tick() => { - let addrs = manual_addrs.clone(); - let manual_peers_dial_tx = manual_peers_dial_tx.clone(); - - // Off loop we resolve the IP's and message them back to the main loop, for it to dial as the `swarm` can't be moved. - tokio::spawn(async move { - join_all(addrs.into_iter().map(|addr| { - let manual_peers_dial_tx = manual_peers_dial_tx.clone(); - async move { - // TODO: We should probs track these errors for the UI - let Ok(socket_addr) = parse_manual_addr(addr.clone()) - .map_err(|err| { - warn!("Failed to parse manual peer address '{addr}': {err}"); - }) else { - return; - }; - - manual_peers_dial_tx.send((addr, socket_addr)).await.ok(); - } - })).await; - }); - - } - } - } -} - -async fn send_metadata( - stream: &mut Stream, - p2p: &Arc, -) -> Result, String> { - { - let metadata = p2p.metadata().clone(); - let result = rmp_serde::encode::to_vec_named(&metadata) - .map_err(|err| format!("Error encoding metadata: {err:?}"))?; - stream - .write_all(&(result.len() as u64).to_le_bytes()) - .await - .map_err(|err| format!("Error writing metadata length: {err:?}"))?; - stream - .write_all(&result) - .await - .map_err(|err| format!("Error writing metadata: {err:?}"))?; - } - - let mut len = [0; 8]; - stream - .read_exact(&mut len) - .await - .map_err(|err| format!("Error reading metadata length: {err:?}"))?; - let len = u64::from_le_bytes(len); - if len > 1000 { - return Err("Error metadata too large".into()); - } - let mut buf = vec![0; len as usize]; - stream - .read_exact(&mut buf) - .await - .map_err(|err| format!("Error reading metadata length: {err:?}"))?; - rmp_serde::decode::from_slice(&buf).map_err(|err| format!("Error decoding metadata: {err:?}")) -} - -async fn read_metadata( - stream: &mut Stream, - p2p: &Arc, -) -> Result, String> { - let metadata = { - let mut len = [0; 8]; - stream - .read_exact(&mut len) - .await - .map_err(|err| format!("Error reading metadata length: {err:?}"))?; - let len = u64::from_le_bytes(len); - if len > 1000 { - return Err("Error metadata too large".into()); - } - let mut buf = vec![0; len as usize]; - stream - .read_exact(&mut buf) - .await - .map_err(|err| format!("Error reading metadata length: {err:?}"))?; - rmp_serde::decode::from_slice(&buf) - .map_err(|err| format!("Error decoding metadata: {err:?}"))? - }; - - { - let metadata = p2p.metadata().clone(); - let result = rmp_serde::encode::to_vec_named(&metadata) - .map_err(|err| format!("Error encoding metadata: {err:?}"))?; - stream - .write_all(&(result.len() as u64).to_le_bytes()) - .await - .map_err(|err| format!("Error writing metadata length: {err:?}"))?; - stream - .write_all(&result) - .await - .map_err(|err| format!("Error writing metadata: {err:?}"))?; - } - - Ok(metadata) -} - -fn get_addrs<'a>( - peer_id: PeerId, - relay_config: &[RelayServerEntry], - addrs: impl Iterator + 'a, -) -> Vec { - addrs - .flat_map(|v| match v { - PeerConnectionCandidate::SocketAddr(addr) => vec![socketaddr_to_multiaddr(addr)], - PeerConnectionCandidate::Manual(addr) => vec![socketaddr_to_multiaddr(addr)], - PeerConnectionCandidate::Relay => relay_config - .iter() - .filter_map(|e| match PeerId::from_str(&e.peer_id) { - Ok(peer_id) => Some(e.addrs.iter().map(move |addr| (peer_id, addr))), - Err(err) => { - error!("Failed to parse peer ID '{}': {err:?}", e.peer_id); - None - } - }) - .flatten() - .map(|(relay_peer_id, addr)| { - let mut addr = socketaddr_to_multiaddr(addr); - addr.push(Protocol::P2p(relay_peer_id)); - addr.push(Protocol::P2pCircuit); - addr.push(Protocol::P2p(peer_id)); - addr - }) - .collect::>(), - }) - .collect::>() -} - -/// Parse the user's input into and do DNS resolution if required. -/// -/// `dns_lookup::lookup_host` does allow IP addresses but not socket addresses (ports) so we split them out and handle them separately. -/// -fn parse_manual_addr(addr: String) -> io::Result { - let mut addr = addr.split(':').peekable(); - - match (addr.next(), addr.next(), addr.peek()) { - (Some(host), None, _) => dns_lookup::lookup_host(host).and_then(|addr| { - addr.into_iter() - .next() - .map(|ip| SocketAddr::new(ip, 7373)) - .ok_or(io::Error::new(ErrorKind::Other, "Invalid address")) - }), - (Some(host), Some(port), None) => { - let port = port - .parse::() - .map_err(|_| io::Error::new(ErrorKind::Other, "Invalid port number"))?; - dns_lookup::lookup_host(host).and_then(|addr| { - addr.into_iter() - .next() - .map(|ip| SocketAddr::new(ip, port)) - .ok_or(io::Error::new(ErrorKind::Other, "Invalid address")) - }) - } - (_, _, _) => Err(io::Error::new(ErrorKind::Other, "Invalid address")), - } -} diff --git a/crates/old-p2p/src/hooks/quic/utils.rs b/crates/old-p2p/src/hooks/quic/utils.rs deleted file mode 100644 index 7d77c0638..000000000 --- a/crates/old-p2p/src/hooks/quic/utils.rs +++ /dev/null @@ -1,53 +0,0 @@ -//! This file contains some fairly meaningless glue code for integrating with libp2p. - -use std::net::SocketAddr; - -use libp2p::{identity::Keypair, multiaddr::Protocol, Multiaddr, PeerId}; - -use crate::{Identity, RemoteIdentity}; - -#[must_use] -pub(crate) fn socketaddr_to_multiaddr(m: &SocketAddr) -> Multiaddr { - let mut addr = Multiaddr::empty(); - match m { - SocketAddr::V4(ip) => addr.push(Protocol::Ip4(*ip.ip())), - SocketAddr::V6(ip) => addr.push(Protocol::Ip6(*ip.ip())), - } - addr.push(Protocol::Udp(m.port())); - addr.push(Protocol::QuicV1); - addr -} - -#[must_use] -pub(crate) fn multiaddr_to_socketaddr(m: &Multiaddr) -> Option { - let mut iter = m.iter(); - let ip = match iter.next()? { - Protocol::Ip4(ip) => ip.into(), - Protocol::Ip6(ip) => ip.into(), - _ => return None, - }; - let port = match iter.next()? { - Protocol::Tcp(port) | Protocol::Udp(port) => port, - _ => return None, - }; - Some(SocketAddr::new(ip, port)) -} - -// This is sketchy, but it makes the whole system a lot easier to work with -// We are assuming the libp2p `PublicKey` is the same format as our `RemoteIdentity` type. -// This is *acktually* true but they reserve the right to change it at any point. -#[must_use] -pub fn remote_identity_to_libp2p_peerid(identity: &RemoteIdentity) -> PeerId { - let public_key = libp2p::identity::ed25519::PublicKey::try_from_bytes(&identity.get_bytes()) - .expect("should be the same format"); - PeerId::from_public_key(&public_key.into()) -} - -// This is sketchy, but it makes the whole system a lot easier to work with -// We are assuming the libp2p `Keypair` is the same format as our `Identity` type. -// This is *acktually* true but they reserve the right to change it at any point. -#[must_use] -pub fn identity_to_libp2p_keypair(identity: &Identity) -> Keypair { - libp2p::identity::Keypair::ed25519_from_bytes(identity.to_bytes()) - .expect("should be the same format") -} diff --git a/crates/old-p2p/src/identity.rs b/crates/old-p2p/src/identity.rs deleted file mode 100644 index 58478392d..000000000 --- a/crates/old-p2p/src/identity.rs +++ /dev/null @@ -1,172 +0,0 @@ -// TODO: Document all types in this file - -use std::{ - hash::{Hash, Hasher}, - str::FromStr, -}; - -use base64::{engine::general_purpose, Engine}; -use ed25519_dalek::{VerifyingKey, SECRET_KEY_LENGTH}; -use rand_core::OsRng; -use serde::{Deserialize, Serialize}; -use specta::Type; -use thiserror::Error; -use zeroize::ZeroizeOnDrop; - -pub const REMOTE_IDENTITY_LEN: usize = 32; - -#[derive(Debug, Error)] -#[error(transparent)] -pub enum IdentityErr { - #[error("{0}")] - Dalek(#[from] ed25519_dalek::ed25519::Error), - #[error("Invalid key length")] - InvalidKeyLength, -} - -/// TODO -#[derive(Debug, Clone, ZeroizeOnDrop)] -pub struct Identity(ed25519_dalek::SigningKey); - -impl PartialEq for Identity { - fn eq(&self, other: &Self) -> bool { - self.0.verifying_key().eq(&other.0.verifying_key()) - } -} - -impl Default for Identity { - fn default() -> Self { - Self(ed25519_dalek::SigningKey::generate(&mut OsRng)) - } -} - -impl Identity { - #[must_use] - pub fn new() -> Self { - Self::default() - } - - pub fn from_bytes(bytes: &[u8]) -> Result { - Ok(Self(ed25519_dalek::SigningKey::from_bytes( - bytes[..SECRET_KEY_LENGTH] - .try_into() - .map_err(|_| IdentityErr::InvalidKeyLength)?, - ))) - } - - #[must_use] - pub fn to_bytes(&self) -> Vec { - self.0.to_bytes().to_vec() - } - - #[must_use] - pub fn to_remote_identity(&self) -> RemoteIdentity { - RemoteIdentity(self.0.verifying_key()) - } -} - -#[derive(Copy, Clone, PartialEq, Eq, Type)] -#[specta(transparent)] -pub struct RemoteIdentity(#[specta(type = String)] ed25519_dalek::VerifyingKey); - -impl Hash for RemoteIdentity { - fn hash(&self, state: &mut H) { - self.0.as_bytes().hash(state); - } -} - -impl std::fmt::Debug for RemoteIdentity { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_tuple("RemoteIdentity") - .field(&general_purpose::STANDARD_NO_PAD.encode(self.0.as_bytes())) - .finish() - } -} - -impl std::fmt::Display for RemoteIdentity { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.write_str(&general_purpose::STANDARD_NO_PAD.encode(self.0.as_bytes())) - } -} - -impl Serialize for RemoteIdentity { - fn serialize(&self, serializer: S) -> Result { - serializer.serialize_str(&general_purpose::STANDARD_NO_PAD.encode(self.0.as_bytes())) - } -} - -impl<'de> Deserialize<'de> for RemoteIdentity { - fn deserialize(deserializer: D) -> Result - where - D: serde::Deserializer<'de>, - { - let s = String::deserialize(deserializer)?; - let bytes = general_purpose::STANDARD_NO_PAD - .decode(s) - .map_err(serde::de::Error::custom)?; - Ok(Self( - ed25519_dalek::VerifyingKey::from_bytes( - bytes[..SECRET_KEY_LENGTH] - .try_into() - .map_err(|_| serde::de::Error::custom("Invalid key length"))?, - ) - .map_err(serde::de::Error::custom)?, - )) - } -} - -impl TryFrom for RemoteIdentity { - type Error = IdentityErr; - - fn try_from(value: String) -> Result { - let bytes = general_purpose::STANDARD_NO_PAD - .decode(value) - .map_err(|_| IdentityErr::InvalidKeyLength)?; - Ok(Self(ed25519_dalek::VerifyingKey::from_bytes( - bytes[..SECRET_KEY_LENGTH] - .try_into() - .map_err(|_| IdentityErr::InvalidKeyLength)?, - )?)) - } -} - -impl FromStr for RemoteIdentity { - type Err = IdentityErr; - - fn from_str(s: &str) -> Result { - let bytes = general_purpose::STANDARD_NO_PAD - .decode(s) - .map_err(|_| IdentityErr::InvalidKeyLength)?; - Ok(Self(ed25519_dalek::VerifyingKey::from_bytes( - bytes[..SECRET_KEY_LENGTH] - .try_into() - .map_err(|_| IdentityErr::InvalidKeyLength)?, - )?)) - } -} - -impl RemoteIdentity { - pub fn from_bytes(bytes: &[u8]) -> Result { - Ok(Self(ed25519_dalek::VerifyingKey::from_bytes( - bytes[..SECRET_KEY_LENGTH] - .try_into() - .map_err(|_| IdentityErr::InvalidKeyLength)?, - )?)) - } - - #[must_use] - pub fn get_bytes(&self) -> [u8; REMOTE_IDENTITY_LEN] { - self.0.to_bytes() - } - - #[must_use] - pub fn verifying_key(&self) -> VerifyingKey { - self.0 - } -} - -impl From for Identity { - fn from(value: ed25519_dalek::SigningKey) -> Self { - Self(value) - } -} diff --git a/crates/old-p2p/src/lib.rs b/crates/old-p2p/src/lib.rs deleted file mode 100644 index db7c5afa6..000000000 --- a/crates/old-p2p/src/lib.rs +++ /dev/null @@ -1,34 +0,0 @@ -//! Rust Peer to Peer Networking Library -#![warn(clippy::all, clippy::unwrap_used, clippy::panic)] - -pub(crate) mod hook; -pub mod hooks; -mod identity; -mod p2p; -mod peer; -mod smart_guards; -mod stream; - -pub use hook::{HookEvent, HookId, ListenerId, ShutdownGuard}; -pub use identity::{Identity, IdentityErr, RemoteIdentity}; -pub use p2p::{Listener, P2P}; -pub use peer::{ConnectionRequest, Peer, PeerConnectionCandidate}; -pub use smart_guards::SmartWriteGuard; -pub use stream::UnicastStream; - -pub use flume; - -use thiserror::Error; -use tokio::sync::{mpsc, oneshot}; - -#[derive(Debug, Error)] -pub enum NewStreamError { - #[error("No connection methods available for peer")] - NoConnectionMethodsAvailable, - #[error("The event loop is offline")] - EventLoopOffline(mpsc::error::SendError), - #[error("Failed to establish the connection w/ error: {0}")] - ConnectionNeverEstablished(oneshot::error::RecvError), - #[error("error connecting to peer: {0}")] - Connecting(String), -} diff --git a/crates/old-p2p/src/p2p.rs b/crates/old-p2p/src/p2p.rs deleted file mode 100644 index 65d502b9e..000000000 --- a/crates/old-p2p/src/p2p.rs +++ /dev/null @@ -1,407 +0,0 @@ -use std::{ - collections::{hash_map::Entry, BTreeSet, HashMap, HashSet}, - net::SocketAddr, - sync::{Arc, PoisonError, RwLock, RwLockReadGuard}, - time::Duration, -}; - -use flume::Sender; -use hash_map_diff::hash_map_diff; -use libp2p::futures::future::join_all; -use stable_vec::StableVec; -use tokio::time::timeout; -use tracing::info; - -use crate::{ - hook::{HandlerFn, Hook, HookEvent, ListenerData, ListenerId, ShutdownGuard}, - smart_guards::SmartWriteGuard, - HookId, Identity, Peer, PeerConnectionCandidate, RemoteIdentity, UnicastStream, -}; - -/// Manager for the entire P2P system. -#[derive(Debug)] -pub struct P2P { - /// A unique identifier for the application. - /// This will differentiate between different applications using this same P2P library. - app_name: &'static str, - /// The identity of the local node. - /// This is the public/private keypair used to uniquely identify the node. - identity: Identity, - /// The channel is used by the application to handle incoming connections. - /// Connection's are automatically closed when dropped so if user forgets to subscribe to this that will just happen as expected. - handler_tx: Sender, - /// Metadata is shared from the local node to the remote nodes. - /// This will contain information such as the node's name, version, and services we provide. - metadata: RwLock>, - /// A list of all peers known to the P2P system. Be aware a peer could be connected and/or discovered at any time. - pub(crate) peers: RwLock>>, - /// Hooks can be registered to react to state changes in the P2P system. - pub(crate) hooks: RwLock>, -} - -impl P2P { - /// Construct a new P2P system. - pub fn new( - app_name: &'static str, - identity: Identity, - handler_tx: Sender, - ) -> Arc { - app_name - .chars() - .all(|c| char::is_alphanumeric(c) || c == '-') - .then_some(()) - .expect("'P2P::new': invalid app_name. Must be alphanumeric or '-' only."); - #[allow(clippy::panic)] - if app_name.len() > 12 { - panic!("'P2P::new': app_name too long. Must be 12 characters or less."); - } - - Arc::new(P2P { - app_name, - identity, - metadata: Default::default(), - peers: Default::default(), - handler_tx, - hooks: Default::default(), - }) - } - - /// The unique identifier for this application. - pub fn app_name(&self) -> &'static str { - self.app_name - } - - /// The identifier of this node that can *MUST* be kept secret. - /// This is a private key in crypto terms. - pub fn identity(&self) -> &Identity { - &self.identity - } - - /// The identifier of this node that can be shared. - /// This is a public key in crypto terms. - pub fn remote_identity(&self) -> RemoteIdentity { - self.identity.to_remote_identity() - } - - /// Metadata is shared from the local node to the remote nodes. - /// This will contain information such as the node's name, version, and services we provide. - pub fn metadata(&self) -> RwLockReadGuard> { - self.metadata.read().unwrap_or_else(PoisonError::into_inner) - } - - pub fn metadata_mut(&self) -> SmartWriteGuard> { - let lock = self - .metadata - .write() - .unwrap_or_else(PoisonError::into_inner); - - SmartWriteGuard::new(self, lock, |p2p, before, after| { - let diff = hash_map_diff(&before, after); - if diff.updated.is_empty() && diff.removed.is_empty() { - return; - } - - p2p.hooks - .read() - .unwrap_or_else(PoisonError::into_inner) - .iter() - .for_each(|(_, hook)| { - hook.send(HookEvent::MetadataModified); - }); - }) - } - - /// A list of all peers known to the P2P system. Be aware a peer could be connected and/or discovered at any time. - pub fn peers(&self) -> RwLockReadGuard>> { - self.peers.read().unwrap_or_else(PoisonError::into_inner) - } - - // TODO: Should this take `addrs`???, A connection through the Relay probs doesn't have one in the same form. - pub fn discover_peer( - self: Arc, - hook_id: HookId, - identity: RemoteIdentity, - metadata: HashMap, - addrs: BTreeSet, - ) -> Arc { - let mut peers = self.peers.write().unwrap_or_else(PoisonError::into_inner); - let peer = peers.entry(identity); - let was_peer_inserted = matches!(peer, Entry::Vacant(_)); - let peer = peer - .or_insert_with({ - let p2p = self.clone(); - || Peer::new(identity, p2p) - }) - .clone(); - - let addrs = { - let mut state = peer.state.write().unwrap_or_else(PoisonError::into_inner); - let a = state.discovered.entry(hook_id).or_default(); - a.extend(addrs); - a.clone() - }; - - peer.metadata_mut().extend(metadata); - - { - let hooks = self.hooks.read().unwrap_or_else(PoisonError::into_inner); - hooks - .iter() - .for_each(|(id, hook)| hook.acceptor(ListenerId(id), &peer, &addrs)); - - if was_peer_inserted { - hooks - .iter() - .for_each(|(_, hook)| hook.send(HookEvent::PeerAvailable(peer.clone()))); - } - - hooks.iter().for_each(|(_, hook)| { - hook.send(HookEvent::PeerDiscoveredBy(hook_id, peer.clone())) - }); - } - - peer - } - - pub fn connected_to_incoming( - self: Arc, - listener: ListenerId, - metadata: HashMap, - stream: UnicastStream, - ) -> Arc { - let peer = self - .clone() - .connected_to_outgoing(listener, metadata, stream.remote_identity()); - let _ = self.handler_tx.send(stream); - peer - } - - pub fn connected_to_outgoing( - self: Arc, - listener: ListenerId, - metadata: HashMap, - identity: RemoteIdentity, - ) -> Arc { - let mut peers = self.peers.write().unwrap_or_else(PoisonError::into_inner); - let peer = peers.entry(identity); - let was_peer_inserted = matches!(peer, Entry::Vacant(_)); - let peer = peer - .or_insert_with({ - let p2p = self.clone(); - move || Peer::new(identity, p2p) - }) - .clone(); - - { - let mut state = peer.state.write().unwrap_or_else(PoisonError::into_inner); - state.active_connections.insert(listener); - } - - peer.metadata_mut().extend(metadata); - - { - let hooks = self.hooks.read().unwrap_or_else(PoisonError::into_inner); - - if was_peer_inserted { - hooks - .iter() - .for_each(|(_, hook)| hook.send(HookEvent::PeerAvailable(peer.clone()))); - } - - hooks.iter().for_each(|(_, hook)| { - hook.send(HookEvent::PeerConnectedWith(listener, peer.clone())) - }); - } - - peer - } - - /// All active listeners registered with the P2P system. - pub fn listeners(&self) -> Vec { - self.hooks - .read() - .unwrap_or_else(PoisonError::into_inner) - .iter() - .filter_map(|(id, hook)| { - hook.listener.as_ref().map(|listener| Listener { - id: ListenerId(id), - name: hook.name, - addrs: listener.addrs.clone(), - }) - }) - .collect() - } - - /// A listener is a special type of hook which is responsible for accepting incoming connections. - /// - /// It is expected you call `Self::register_listener_addr` after this to register the addresses you are listening on. - /// - /// `acceptor` is called when a peer is discovered, but before it is emitted to the application. - /// This lets you register a connection method if you have one. - pub fn register_listener( - &self, - name: &'static str, - tx: Sender, - acceptor: impl Fn(ListenerId, &Arc, &BTreeSet) - + Send - + Sync - + 'static, - ) -> ListenerId { - let mut hooks = self.hooks.write().unwrap_or_else(PoisonError::into_inner); - let hook_id = hooks.push(Hook { - name, - tx, - listener: Some(ListenerData { - addrs: Default::default(), - acceptor: HandlerFn(Arc::new(acceptor)), - }), - }); - - hooks.iter().for_each(|(id, hook)| { - if id == hook_id { - return; - } - - hook.send(HookEvent::ListenerRegistered(ListenerId(hook_id))); - }); - - ListenerId(hook_id) - } - - pub fn register_listener_addr(&self, listener_id: ListenerId, addr: SocketAddr) { - let mut hooks = self.hooks.write().unwrap_or_else(PoisonError::into_inner); - if let Some(listener) = hooks - .get_mut(listener_id.0) - .and_then(|l| l.listener.as_mut()) - { - listener.addrs.insert(addr); - } - - info!("HookEvent::ListenerAddrAdded({listener_id:?}, {addr})"); - hooks.iter().for_each(|(_, hook)| { - hook.send(HookEvent::ListenerAddrAdded(listener_id, addr)); - }); - } - - pub fn unregister_listener_addr(&self, listener_id: ListenerId, addr: SocketAddr) { - let mut hooks = self.hooks.write().unwrap_or_else(PoisonError::into_inner); - if let Some(listener) = hooks - .get_mut(listener_id.0) - .and_then(|l| l.listener.as_mut()) - { - listener.addrs.remove(&addr); - } - - info!("HookEvent::ListenerAddrRemoved({listener_id:?}, {addr})"); - hooks.iter().for_each(|(_, hook)| { - hook.send(HookEvent::ListenerAddrRemoved(listener_id, addr)); - }); - } - - // TODO: Probs cleanup return type - pub fn hooks(&self) -> Vec<(HookId, &'static str)> { - self.hooks - .read() - .unwrap_or_else(PoisonError::into_inner) - .iter() - .map(|(id, hook)| (HookId(id), hook.name)) - .collect() - } - - /// Register a new hook which can be used to react to state changes in the P2P system. - pub fn register_hook(&self, name: &'static str, tx: Sender) -> HookId { - HookId( - self.hooks - .write() - .unwrap_or_else(PoisonError::into_inner) - .push(Hook { - name, - tx, - listener: None, - }), - ) - } - - /// Unregister a hook. This will also call `HookEvent::Shutdown` on the hook. - pub async fn unregister_hook(&self, id: HookId) { - let mut shutdown_rxs = Vec::new(); - { - let mut hooks = self.hooks.write().unwrap_or_else(PoisonError::into_inner); - if let Some(hook) = hooks.remove(id.0) { - let (_guard, rx) = ShutdownGuard::new(); - shutdown_rxs.push(rx); - hook.send(HookEvent::Shutdown { _guard }); - - if hook.listener.is_some() { - hooks.iter().for_each(|(_, hook)| { - hook.send(HookEvent::ListenerUnregistered(ListenerId(id.0))); - }); - } - - let mut peers = self.peers.write().unwrap_or_else(PoisonError::into_inner); - let mut peers_to_remove = HashSet::new(); // We are mutate while iterating - for (identity, peer) in peers.iter_mut() { - let mut state = peer.state.write().unwrap_or_else(PoisonError::into_inner); - if state.active_connections.remove(&ListenerId(id.0)) { - hooks.iter().for_each(|(_, hook)| { - hook.send(HookEvent::PeerDisconnectedWith( - ListenerId(id.0), - peer.identity(), - )); - }); - } - state.connection_methods.remove(&ListenerId(id.0)); - state.discovered.remove(&id); - - hooks.iter().for_each(|(_, hook)| { - hook.send(HookEvent::PeerExpiredBy(id, peer.identity())); - }); - - if state.connection_methods.is_empty() && state.discovered.is_empty() { - peers_to_remove.insert(*identity); - } - } - - for identity in peers_to_remove { - peers.remove(&identity); - } - } - } - - // We rely on the fact that when the oneshot is dropped this will return an error as opposed to hanging. - // So we can detect when the hooks shutdown code has completed. - let _ = timeout(Duration::from_secs(2), join_all(shutdown_rxs)).await; - } - - /// Shutdown the whole P2P system. - /// This will close all connections and remove all hooks. - pub async fn shutdown(&self) { - let hooks = { - self.hooks - .write() - .unwrap_or_else(PoisonError::into_inner) - .iter() - .map(|i| i.0) - .collect::>() - .clone() - }; - - for hook_id in hooks { - self.unregister_hook(HookId(hook_id)).await; - } - } -} - -#[derive(Debug)] -#[non_exhaustive] -pub struct Listener { - pub id: ListenerId, - pub name: &'static str, - pub addrs: HashSet, -} - -impl Listener { - pub fn is_hook_id(&self, id: HookId) -> bool { - self.id.0 == id.0 - } -} diff --git a/crates/old-p2p/src/peer.rs b/crates/old-p2p/src/peer.rs deleted file mode 100644 index 13c827cfc..000000000 --- a/crates/old-p2p/src/peer.rs +++ /dev/null @@ -1,331 +0,0 @@ -use std::{ - collections::{BTreeSet, HashMap, HashSet}, - net::SocketAddr, - sync::{Arc, PoisonError, RwLock, RwLockReadGuard, RwLockWriteGuard, Weak}, -}; - -use crate::NewStreamError; -use tokio::sync::{mpsc, oneshot}; -use tracing::warn; - -use crate::{HookEvent, HookId, ListenerId, RemoteIdentity, UnicastStream, P2P}; - -#[derive(Debug)] -pub struct Peer { - /// RemoteIdentity of the peer. - pub(crate) identity: RemoteIdentity, - /// Information from `P2P::service` on the remote node. - pub(crate) metadata: RwLock>, - /// We want these states to locked by the same lock so we can ensure they are consistent. - pub(crate) state: RwLock, - /// A reference back to the P2P system. - /// This is weak so we don't have recursive `Arc`'s that can never be dropped. - pub(crate) p2p: Weak, -} - -// The order of this enum is the preference of the connection type. -#[derive(Debug, Clone, Hash, Eq, PartialEq, PartialOrd, Ord)] -pub enum PeerConnectionCandidate { - SocketAddr(SocketAddr), - Relay, - Manual(SocketAddr), - // Custom(String), -} - -#[derive(Debug, Default)] -pub(crate) struct State { - /// Active connections with the remote - pub(crate) active_connections: HashSet, - /// Methods for establishing an active connections with the remote - /// These should be inject by `Listener::acceptor` which is called when a new peer is discovered. - pub(crate) connection_methods: HashMap>, - /// Methods that have discovered this peer. - pub(crate) discovered: HashMap>, -} - -/// A request to connect to a client. -/// This will be handled by a configured listener hook. -#[derive(Debug)] -#[non_exhaustive] -pub struct ConnectionRequest { - pub to: RemoteIdentity, - pub addrs: BTreeSet, - pub tx: oneshot::Sender>, -} - -// TODO: Maybe use this? -// impl State { -// pub(crate) fn needs_removal(&self) -> bool { -// self.discovered.is_empty() -// && self.connection_methods.is_empty() -// && self.active_connections.is_empty() -// } -// } - -impl Eq for Peer {} -impl PartialEq for Peer { - fn eq(&self, other: &Self) -> bool { - self.identity == other.identity - } -} - -// Internal methods -impl Peer { - // TODO: Make this private - pub fn new(identity: RemoteIdentity, p2p: Arc) -> Arc { - Arc::new(Self { - identity, - metadata: Default::default(), - state: Default::default(), - p2p: Arc::downgrade(&p2p), - }) - } -} - -// User-facing methods -impl Peer { - pub fn identity(&self) -> RemoteIdentity { - self.identity - } - - pub fn metadata(&self) -> RwLockReadGuard> { - self.metadata.read().unwrap_or_else(PoisonError::into_inner) - } - - pub fn metadata_mut(&self) -> RwLockWriteGuard> { - self.metadata - .write() - .unwrap_or_else(PoisonError::into_inner) - } - - pub fn can_connect(&self) -> bool { - !self - .state - .read() - .unwrap_or_else(PoisonError::into_inner) - .connection_methods - .is_empty() - } - - pub fn can_connect_with(&self, hook_id: HookId) -> bool { - self.state - .read() - .unwrap_or_else(PoisonError::into_inner) - .discovered - .contains_key(&hook_id) - } - - pub fn connection_candidates(&self) -> BTreeSet { - self.state - .read() - .unwrap_or_else(PoisonError::into_inner) - .discovered - .values() - .flatten() - .cloned() - .collect() - } - - pub fn is_connected(&self) -> bool { - !self - .state - .read() - .unwrap_or_else(PoisonError::into_inner) - .active_connections - .is_empty() - } - - pub fn active_connections(&self) -> usize { - self.state - .read() - .unwrap_or_else(PoisonError::into_inner) - .active_connections - .len() - } - - // TODO: Possibly remove this, it's not great??? - pub fn is_connected_with_hook(&self, hook_id: HookId) -> bool { - self.state - .read() - .unwrap_or_else(PoisonError::into_inner) - .active_connections - .contains(&ListenerId(hook_id.0)) - } - - pub fn is_connected_with(&self, listener_id: ListenerId) -> bool { - self.state - .read() - .unwrap_or_else(PoisonError::into_inner) - .active_connections - .contains(&listener_id) - } - - pub fn connection_methods(&self) -> HashSet { - self.state - .read() - .unwrap_or_else(PoisonError::into_inner) - .connection_methods - .keys() - .copied() - .collect() - } - - pub fn discovered_by(&self) -> HashSet { - self.state - .read() - .unwrap_or_else(PoisonError::into_inner) - .discovered - .keys() - .copied() - .collect() - } - - pub fn addrs(&self) -> HashSet { - self.state - .read() - .unwrap_or_else(PoisonError::into_inner) - .discovered - .values() - .flatten() - .filter_map(|addr| match addr { - PeerConnectionCandidate::SocketAddr(addr) => Some(*addr), - _ => None, - }) - .collect() - } - - /// Construct a new Quic stream to the peer. - pub async fn new_stream(&self) -> Result { - let (addrs, connect_tx) = { - let state = self.state.read().unwrap_or_else(PoisonError::into_inner); - - let addrs = state - .discovered - .values() - .flatten() - .cloned() - .collect::>(); - - let Some((_id, connect_tx)) = state - .connection_methods - .iter() - .map(|(id, tx)| (*id, tx.clone())) - .next() - else { - return Err(NewStreamError::NoConnectionMethodsAvailable); - }; - - (addrs, connect_tx) - }; - - let (tx, rx) = oneshot::channel(); - connect_tx - .send(ConnectionRequest { - to: self.identity, - addrs, - tx, - }) - .await - .map_err(|err| { - warn!("Failed to send connect request to peer: {}", err); - NewStreamError::EventLoopOffline(err) - })?; - rx.await - .map_err(|err| { - warn!("Failed to receive connect response from peer: {err}"); - NewStreamError::ConnectionNeverEstablished(err) - })? - .map_err(|err| { - warn!("Failed to do the thing: {err}"); - NewStreamError::Connecting(err) - }) - } -} - -// Hook-facing methods -impl Peer { - pub fn hook_discovered(&self, hook: HookId, addrs: BTreeSet) { - // TODO: Emit event maybe??? - - self.state - .write() - .unwrap_or_else(PoisonError::into_inner) - .discovered - .insert(hook, addrs); - } - - pub fn listener_available( - self: Arc, - listener: ListenerId, - tx: mpsc::Sender, - ) { - self.state - .write() - .unwrap_or_else(PoisonError::into_inner) - .connection_methods - .insert(listener, tx); - - let Some(p2p) = self.p2p.upgrade() else { - return; - }; - - p2p.hooks - .read() - .unwrap_or_else(PoisonError::into_inner) - .iter() - .for_each(|(_, hook)| { - hook.send(HookEvent::PeerDiscoveredBy(listener.into(), self.clone())); - }); - } - - pub fn undiscover_peer(&self, hook_id: HookId) { - let Some(p2p) = self.p2p.upgrade() else { - return; - }; - - let mut state = self.state.write().unwrap_or_else(PoisonError::into_inner); - state.discovered.remove(&hook_id); - - let hooks = p2p.hooks.read().unwrap_or_else(PoisonError::into_inner); - hooks.iter().for_each(|(_, hook)| { - hook.send(HookEvent::PeerExpiredBy(hook_id, self.identity)); - }); - - if state.connection_methods.is_empty() && state.discovered.is_empty() { - p2p.peers - .write() - .unwrap_or_else(PoisonError::into_inner) - .remove(&self.identity); - - hooks.iter().for_each(|(_, hook)| { - hook.send(HookEvent::PeerUnavailable(self.identity)); - }); - } - } - - pub fn disconnected_from(&self, listener_id: ListenerId) { - let Some(p2p) = self.p2p.upgrade() else { - return; - }; - - let mut state = self.state.write().unwrap_or_else(PoisonError::into_inner); - state.connection_methods.remove(&listener_id); - state.active_connections.remove(&listener_id); - - let hooks = p2p.hooks.read().unwrap_or_else(PoisonError::into_inner); - hooks.iter().for_each(|(_, hook)| { - hook.send(HookEvent::PeerDisconnectedWith(listener_id, self.identity)); - }); - - if state.connection_methods.is_empty() && state.discovered.is_empty() { - p2p.peers - .write() - .unwrap_or_else(PoisonError::into_inner) - .remove(&self.identity); - - hooks.iter().for_each(|(_, hook)| { - hook.send(HookEvent::PeerUnavailable(self.identity)); - }); - } - } -} diff --git a/crates/old-p2p/src/smart_guards.rs b/crates/old-p2p/src/smart_guards.rs deleted file mode 100644 index a920508aa..000000000 --- a/crates/old-p2p/src/smart_guards.rs +++ /dev/null @@ -1,55 +0,0 @@ -use std::{ - ops::{Deref, DerefMut}, - sync::RwLockWriteGuard, -}; - -use crate::P2P; - -type SaveFn = fn(&P2P, /* before */ T, /* after */ &T); - -/// A special guard for `RwLockWriteGuard` that will call a `save` function when it's dropped. -/// This allows changes to the value to automatically trigger `HookEvents` to be emitted. -#[derive(Debug)] -pub struct SmartWriteGuard<'a, T> { - p2p: &'a P2P, - lock: RwLockWriteGuard<'a, T>, - before: Option, - save: SaveFn, -} - -impl<'a, T: Clone> SmartWriteGuard<'a, T> { - pub(crate) fn new(p2p: &'a P2P, lock: RwLockWriteGuard<'a, T>, save: SaveFn) -> Self { - Self { - p2p, - before: Some(lock.clone()), - lock, - save, - } - } -} - -impl Deref for SmartWriteGuard<'_, T> { - type Target = T; - - fn deref(&self) -> &Self::Target { - &self.lock - } -} - -impl DerefMut for SmartWriteGuard<'_, T> { - fn deref_mut(&mut self) -> &mut Self::Target { - &mut self.lock - } -} - -impl Drop for SmartWriteGuard<'_, T> { - fn drop(&mut self) { - (self.save)( - self.p2p, - self.before - .take() - .expect("'SmartWriteGuard::drop' called more than once!"), - &self.lock, - ); - } -} diff --git a/crates/old-p2p/src/stream.rs b/crates/old-p2p/src/stream.rs deleted file mode 100644 index 6fd106f5e..000000000 --- a/crates/old-p2p/src/stream.rs +++ /dev/null @@ -1,81 +0,0 @@ -use std::{ - fmt, io, - pin::Pin, - task::{Context, Poll}, -}; - -use sync_wrapper::SyncWrapper; -use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt, ReadBuf}; - -use crate::RemoteIdentity; - -trait IoStream: AsyncRead + AsyncWrite {} -impl IoStream for S {} - -/// A unicast stream is a direct stream to a specific peer. -pub struct UnicastStream { - io: SyncWrapper>>, - remote: RemoteIdentity, -} - -impl fmt::Debug for UnicastStream { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("UnicastStream") - .field("remote", &self.remote) - .finish() - } -} - -impl UnicastStream { - pub fn new(remote: RemoteIdentity, io: S) -> Self { - Self { - io: SyncWrapper::new(Box::pin(io)), - remote, - } - } - - #[must_use] - pub fn remote_identity(&self) -> RemoteIdentity { - self.remote - } - - pub async fn close(self) -> Result<(), io::Error> { - self.io.into_inner().shutdown().await - } -} - -impl AsyncRead for UnicastStream { - fn poll_read( - self: Pin<&mut Self>, - cx: &mut Context<'_>, - buf: &mut ReadBuf<'_>, - ) -> Poll> { - Pin::new(&mut self.get_mut().io) - .get_pin_mut() - .poll_read(cx, buf) - } -} - -impl AsyncWrite for UnicastStream { - fn poll_write( - self: Pin<&mut Self>, - cx: &mut Context<'_>, - buf: &[u8], - ) -> Poll> { - Pin::new(&mut self.get_mut().io) - .get_pin_mut() - .poll_write(cx, buf) - } - - fn poll_flush(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - Pin::new(&mut self.get_mut().io) - .get_pin_mut() - .poll_flush(cx) - } - - fn poll_shutdown(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - Pin::new(&mut self.get_mut().io) - .get_pin_mut() - .poll_shutdown(cx) - } -} diff --git a/crates/prisma-cli/Cargo.toml b/crates/prisma-cli/Cargo.toml deleted file mode 100644 index c5c0fb669..000000000 --- a/crates/prisma-cli/Cargo.toml +++ /dev/null @@ -1,18 +0,0 @@ -[package] -name = "prisma-cli" -version = "0.1.0" - -edition.workspace = true -license.workspace = true -repository.workspace = true - -[dependencies] -# Spacedrive Sub-crates -sd-sync-generator = { path = "../sync-generator" } - -# Specific prisma-cli dependencies -[dependencies.prisma-client-rust-generator] -default-features = false -features = ["migrations", "specta", "sqlite", "sqlite-create-many"] -git = "https://github.com/spacedriveapp/prisma-client-rust" -rev = "b22ad7dc7d" diff --git a/crates/prisma-cli/src/bin/prisma.rs b/crates/prisma-cli/src/bin/prisma.rs deleted file mode 100644 index 70c2c3620..000000000 --- a/crates/prisma-cli/src/bin/prisma.rs +++ /dev/null @@ -1,3 +0,0 @@ -fn main() { - prisma_client_rust_generator::run(); -} diff --git a/crates/prisma-cli/src/bin/sync.rs b/crates/prisma-cli/src/bin/sync.rs deleted file mode 100644 index 58b173755..000000000 --- a/crates/prisma-cli/src/bin/sync.rs +++ /dev/null @@ -1,3 +0,0 @@ -fn main() { - sd_sync_generator::run(); -} diff --git a/crates/prisma/.gitignore b/crates/prisma/.gitignore deleted file mode 100644 index 4d20cf2e7..000000000 --- a/crates/prisma/.gitignore +++ /dev/null @@ -1 +0,0 @@ -src/*/ diff --git a/crates/prisma/Cargo.toml b/crates/prisma/Cargo.toml deleted file mode 100644 index b61e4e863..000000000 --- a/crates/prisma/Cargo.toml +++ /dev/null @@ -1,16 +0,0 @@ -[package] -name = "sd-prisma" -version = "0.1.0" - -edition = "2021" - -[dependencies] -# Spacedrive Sub-crates -sd-sync = { path = "../sync" } - -# Workspace dependencies -prisma-client-rust = { workspace = true } -rmp-serde = { workspace = true } -rmpv = { workspace = true } -serde = { workspace = true } -uuid = { workspace = true } diff --git a/crates/prisma/src/lib.rs b/crates/prisma/src/lib.rs deleted file mode 100644 index d5116b855..000000000 --- a/crates/prisma/src/lib.rs +++ /dev/null @@ -1,15 +0,0 @@ -#![recursion_limit = "256"] -#[allow(warnings, unused)] -pub mod prisma; -#[allow(warnings, unused)] -pub mod prisma_sync; - -pub async fn test_db() -> std::sync::Arc { - std::sync::Arc::new( - prisma::PrismaClient::_builder() - .with_url(format!("file:/tmp/test-db-{}", uuid::Uuid::new_v4())) - .build() - .await - .unwrap(), - ) -} diff --git a/crates/sync-generator/Cargo.toml b/crates/sync-generator/Cargo.toml deleted file mode 100644 index 20fde252e..000000000 --- a/crates/sync-generator/Cargo.toml +++ /dev/null @@ -1,16 +0,0 @@ -[package] -name = "sd-sync-generator" -version = "0.1.0" - -edition.workspace = true -license.workspace = true -repository.workspace = true - -[dependencies] -# Workspace dependencies -prisma-client-rust-sdk = { workspace = true } -serde = { workspace = true, features = ["derive"] } -thiserror = { workspace = true } - -# Specific Sync Generator dependencies -nom = "7.1" diff --git a/crates/sync-generator/src/attribute/mod.rs b/crates/sync-generator/src/attribute/mod.rs deleted file mode 100644 index e836fa795..000000000 --- a/crates/sync-generator/src/attribute/mod.rs +++ /dev/null @@ -1,55 +0,0 @@ -use prisma_client_rust_sdk::prisma::prisma_models::{ast::WithDocumentation, walkers::ModelWalker}; - -mod parser; - -#[derive(Debug)] -pub enum AttributeFieldValue<'a> { - Single(&'a str), - List(Vec<&'a str>), -} - -#[allow(unused)] -impl AttributeFieldValue<'_> { - pub const fn as_single(&self) -> Option<&str> { - if let AttributeFieldValue::Single(field) = self { - Some(field) - } else { - None - } - } - - pub const fn as_list(&self) -> Option<&Vec<&str>> { - if let AttributeFieldValue::List(fields) = self { - Some(fields) - } else { - None - } - } -} - -#[derive(Debug)] -pub struct Attribute<'a> { - pub name: &'a str, - pub fields: Vec<(&'a str, AttributeFieldValue<'a>)>, -} - -impl<'a> Attribute<'a> { - pub fn parse(input: &'a str) -> Result { - parser::parse(input).map(|(_, a)| a).map_err(|_| ()) - } - - pub fn field(&self, name: &str) -> Option<&AttributeFieldValue<'_>> { - self.fields - .iter() - .find_map(|(n, v)| (*n == name).then_some(v)) - } -} - -pub fn model_attributes(model: ModelWalker<'_>) -> Vec> { - model - .ast_model() - .documentation() - .as_ref() - .map(|docs| docs.lines().flat_map(Attribute::parse).collect()) - .unwrap_or_default() -} diff --git a/crates/sync-generator/src/attribute/parser.rs b/crates/sync-generator/src/attribute/parser.rs deleted file mode 100644 index 384be1ff2..000000000 --- a/crates/sync-generator/src/attribute/parser.rs +++ /dev/null @@ -1,155 +0,0 @@ -use nom::{ - branch::alt, - bytes::complete::{is_not, tag}, - character::complete::{alpha1, char, multispace0}, - combinator::{map, opt}, - error::{ErrorKind, ParseError}, - multi::separated_list1, - sequence::{delimited, separated_pair}, - AsChar, IResult, InputTakeAtPosition, -}; - -use super::{Attribute, AttributeFieldValue}; - -fn remove_ws<'a, O, E: ParseError<&'a str>, F>( - wrapped: F, -) -> impl FnMut(&'a str) -> IResult<&'a str, O, E> -where - F: FnMut(&'a str) -> IResult<&'a str, O, E>, -{ - delimited(multispace0, wrapped, multispace0) -} - -fn parens(input: &str) -> IResult<&str, &str> { - delimited(char('('), is_not(")"), char(')'))(input) -} - -fn single_value>(i: &T) -> IResult -where - T: InputTakeAtPosition, - ::Item: AsChar, -{ - i.split_at_position1_complete( - |item| { - let char_item = item.as_char(); - char_item != '_' && !char_item.is_alphanum() - }, - ErrorKind::AlphaNumeric, - ) -} - -fn list_value(input: &str) -> IResult<&str, Vec<&str>> { - delimited( - char('['), - separated_list1(char(','), remove_ws(|a| single_value(&a))), - char(']'), - )(input) -} - -fn attribute_field_value(input: &str) -> IResult<&str, AttributeFieldValue<'_>> { - remove_ws(alt(( - map(|a| list_value(a), AttributeFieldValue::List), - map(|a| single_value(&a), AttributeFieldValue::Single), - )))(input) -} - -fn attribute_field(input: &str) -> IResult<&str, (&str, AttributeFieldValue<'_>)> { - remove_ws(separated_pair( - remove_ws(is_not(":")), - char(':'), - remove_ws(attribute_field_value), - ))(input) -} - -fn attribute_fields(input: &str) -> IResult<&str, Vec<(&str, AttributeFieldValue<'_>)>> { - separated_list1(char(','), attribute_field)(input) -} - -pub fn parse(input: &str) -> IResult<&str, Attribute<'_>> { - let (input, _) = remove_ws(tag("@"))(input)?; - let (input, name) = alpha1(input)?; - let (input, values_str) = opt(remove_ws(parens))(input)?; - - let fields = match values_str { - Some(values_str) => attribute_fields(values_str)?.1, - None => vec![], - }; - - Ok((input, Attribute { name, fields })) -} - -#[cfg(test)] -mod test { - use super::*; - - #[test] - fn marker() { - let s = "@local"; - - let (remaining, attribute) = parse(s).unwrap(); - - assert_eq!(remaining, ""); - assert_eq!(attribute.name, "local"); - assert_eq!(attribute.fields.len(), 0); - } - - #[test] - fn single() { - let s = "@local(foo: bar)"; - - let (remaining, attribute) = parse(s).unwrap(); - - assert_eq!(remaining, ""); - assert_eq!(attribute.name, "local"); - assert_eq!(attribute.fields.len(), 1); - assert_eq!(attribute.fields[0].0, "foo"); - assert!(matches!( - attribute.fields[0].1, - AttributeFieldValue::Single("bar") - )); - } - - #[test] - fn list() { - let s = "@local(foo: [bar, baz])"; - - let (remaining, attribute) = match parse(s) { - Ok(v) => v, - Err(e) => panic!("{}", e), - }; - - assert_eq!(remaining, ""); - assert_eq!(attribute.name, "local"); - assert_eq!(attribute.fields.len(), 1); - assert_eq!(attribute.fields[0].0, "foo"); - - if let AttributeFieldValue::List(list) = &attribute.fields[0].1 { - assert_eq!(list.len(), 2); - assert_eq!(list[0], "bar"); - assert_eq!(list[1], "baz"); - } else { - panic!("Expected list, got {:?}", attribute.fields[0].1); - } - } - - #[test] - fn multiple() { - let s = "@local(foo: bar, baz: qux)"; - - let (remaining, attribute) = parse(s).unwrap(); - - assert_eq!(remaining, ""); - assert_eq!(attribute.name, "local"); - assert_eq!(attribute.fields.len(), 2); - assert_eq!(attribute.fields[0].0, "foo"); - assert!(matches!( - attribute.fields[0].1, - AttributeFieldValue::Single("bar") - )); - assert_eq!(attribute.fields[1].0, "baz"); - assert!(matches!( - attribute.fields[1].1, - AttributeFieldValue::Single("qux") - )); - } -} diff --git a/crates/sync-generator/src/lib.rs b/crates/sync-generator/src/lib.rs deleted file mode 100644 index 1fb8279b9..000000000 --- a/crates/sync-generator/src/lib.rs +++ /dev/null @@ -1,195 +0,0 @@ -#![warn( - clippy::all, - clippy::pedantic, - clippy::correctness, - clippy::perf, - clippy::style, - clippy::suspicious, - clippy::complexity, - clippy::nursery, - clippy::unwrap_used, - unused_qualifications, - rust_2018_idioms, - trivial_casts, - trivial_numeric_casts, - unused_allocation, - clippy::unnecessary_cast, - clippy::cast_lossless, - clippy::cast_possible_truncation, - clippy::cast_possible_wrap, - clippy::cast_precision_loss, - clippy::cast_sign_loss, - clippy::dbg_macro, - clippy::deprecated_cfg_attr, - clippy::separated_literal_suffix, - deprecated -)] -#![forbid(deprecated_in_future)] -#![allow(clippy::missing_errors_doc, clippy::module_name_repetitions)] - -use prisma_client_rust_sdk::{ - prelude::*, - prisma::prisma_models::walkers::{ - FieldWalker, ModelWalker, RefinedFieldWalker, RelationFieldWalker, - }, -}; - -mod attribute; -mod model; -mod sync_data; - -use attribute::{model_attributes, Attribute, AttributeFieldValue}; - -#[derive(Debug, serde::Serialize, thiserror::Error)] -enum Error {} - -#[derive(serde::Deserialize)] -struct SDSyncGenerator {} - -#[derive(Clone)] -pub enum ModelSyncType<'a> { - Local { - id: FieldWalker<'a>, - }, - // Owned { - // id: FieldVec<'a>, - // }, - Shared { - id: FieldWalker<'a>, - // model ids help reduce storage cost of sync messages - model_id: u16, - }, - Relation { - group: RelationFieldWalker<'a>, - item: RelationFieldWalker<'a>, - model_id: u16, - }, -} - -impl<'a> ModelSyncType<'a> { - fn from_attribute(attr: &Attribute<'_>, model: ModelWalker<'a>) -> Option { - Some(match attr.name { - "local" | "shared" => { - let id = attr - .field("id") - .and_then(|field| match field { - AttributeFieldValue::Single(s) => Some(s), - AttributeFieldValue::List(_) => None, - }) - .and_then(|name| model.fields().find(|f| f.name() == *name))?; - - match attr.name { - "local" => Self::Local { id }, - "shared" => Self::Shared { - id, - model_id: attr - .field("modelId") - .and_then(|a| a.as_single()) - .and_then(|s| s.parse().ok())?, - }, - _ => return None, - } - } - "relation" => { - let get_field = |name| { - attr.field(name) - .and_then(|field| match field { - AttributeFieldValue::Single(s) => Some(*s), - AttributeFieldValue::List(_) => None, - }) - .and_then(|name| { - if let RefinedFieldWalker::Relation(r) = model - .fields() - .find(|f| f.name() == name) - .unwrap_or_else(|| panic!("'{name}' field not found")) - .refine() - { - Some(r) - } else { - None - } - }) - .unwrap_or_else(|| panic!("'{name}' must be a relation field")) - }; - - Self::Relation { - item: get_field("item"), - group: get_field("group"), - model_id: attr - .field("modelId") - .and_then(|a| a.as_single()) - .and_then(|s| s.parse().ok())?, - } - } - // "owned" => Self::Owned { id }, - _ => return None, - }) - } - - fn sync_id(&self) -> Vec> { - match self { - // Self::Owned { id } => id.clone(), - Self::Local { id, .. } | Self::Shared { id, .. } => vec![*id], - Self::Relation { group, item, .. } => vec![(*group).into(), (*item).into()], - } - } -} - -impl ToTokens for ModelSyncType<'_> { - fn to_tokens(&self, tokens: &mut TokenStream) { - let variant = match self { - Self::Local { .. } => "Local", - // Self::Owned { .. } => "Owned", - Self::Shared { .. } => "Shared", - Self::Relation { .. } => "Relation", - }; - - tokens.append(format_ident!("{variant}SyncType")); - } -} - -pub type ModelWithSyncType<'a> = (ModelWalker<'a>, Option>); - -impl PrismaGenerator for SDSyncGenerator { - const NAME: &'static str = "SD Sync Generator"; - const DEFAULT_OUTPUT: &'static str = "prisma-sync.rs"; - - type Error = Error; - - fn generate(self, args: GenerateArgs<'_>) -> Result { - let db = &args.schema.db; - - let models_with_sync_types = db - .walk_models() - .map(|model| (model, model_attributes(model))) - .map(|(model, attributes)| { - let sync_type = attributes - .into_iter() - .find_map(|a| ModelSyncType::from_attribute(&a, model)); - - (model, sync_type) - }) - .collect::>(); - - let model_sync_data = sync_data::enumerate(&models_with_sync_types); - - let mut module = Module::new( - "root", - quote! { - use crate::prisma; - - #model_sync_data - }, - ); - models_with_sync_types - .into_iter() - .map(model::module) - .for_each(|model| module.add_submodule(model)); - - Ok(module) - } -} - -pub fn run() { - SDSyncGenerator::run(); -} diff --git a/crates/sync-generator/src/model.rs b/crates/sync-generator/src/model.rs deleted file mode 100644 index e171634b8..000000000 --- a/crates/sync-generator/src/model.rs +++ /dev/null @@ -1,214 +0,0 @@ -use prisma_client_rust_sdk::{prelude::*, prisma::prisma_models::walkers::RefinedFieldWalker}; -use prisma_models::{ast::ModelId, walkers::Walker}; - -use crate::{ModelSyncType, ModelWithSyncType}; - -pub fn module((model, sync_type): ModelWithSyncType<'_>) -> Module { - let model_name_snake = snake_ident(model.name()); - - let sync_id = sync_type.as_ref().map(|sync_type| { - let fields = sync_type.sync_id(); - let fields = fields.iter().map(|field| { - let name_snake = snake_ident(field.name()); - - let typ = match field.refine() { - RefinedFieldWalker::Scalar(_) => field.type_tokens("e!(self)), - RefinedFieldWalker::Relation(relation) => { - let relation_model_name_snake = snake_ident(relation.related_model().name()); - Some(quote!(super::#relation_model_name_snake::SyncId)) - } - }; - - quote!(pub #name_snake: #typ) - }); - - let model_stuff = parse_model(sync_type, &model_name_snake); - - quote! { - #[derive(serde::Serialize, serde::Deserialize, Clone, Debug)] - pub struct SyncId { - #(#fields),* - } - - impl sd_sync::SyncId for SyncId { - type Model = #model_name_snake::Types; - } - - #model_stuff - } - }); - - let set_param_impl = { - let field_matches = model.fields().filter_map(|field| { - let field_name_snake = snake_ident(field.name()); - - match field.refine() { - RefinedFieldWalker::Scalar(scalar_field) => { - (!scalar_field.is_in_required_relation()).then(|| { - quote! { - #model_name_snake::#field_name_snake::set(::rmpv::ext::from_value(val)?), - } - }) - } - RefinedFieldWalker::Relation(relation_field) => { - let relation_model_name_snake = - snake_ident(relation_field.related_model().name()); - - relation_field.referenced_fields().map_or_else( - || None, - |i| { - if i.count() == 1 { - Some(quote! {{ - - let (field, value) = ::rmpv - ::ext - ::from_value::>(val)? - .into_iter() - .next() - .ok_or(Error::MissingRelationData { - field: field.to_string(), - model: #relation_model_name_snake::NAME.to_string() - })?; - - #model_name_snake::#field_name_snake::connect( - #relation_model_name_snake::UniqueWhereParam::deserialize(&field, value)? - ) - }}) - } else { - None - } - }, - ) - } - } - .map(|body| quote!(#model_name_snake::#field_name_snake::NAME => #body)) - }); - - if field_matches.clone().count() == 0 { - quote!() - } else { - quote! { - impl #model_name_snake::SetParam { - pub fn deserialize(field: &str, val: ::rmpv::Value) -> Result { - Ok(match field { - #(#field_matches)* - _ => return Err(Error::FieldNotFound { - field: field.to_string(), - model: #model_name_snake::NAME.to_string(), - }), - }) - } - } - } - } - }; - - let unique_param_impl = process_unique_params(model, &model_name_snake); - - Module::new( - model.name(), - quote! { - use super::Error; - - use prisma_client_rust::scalar_types::*; - - use super::prisma::*; - - #sync_id - - #set_param_impl - - #unique_param_impl - }, - ) -} - -#[inline] -fn parse_model(sync_type: &ModelSyncType<'_>, model_name_snake: &Ident) -> Option { - match sync_type { - ModelSyncType::Relation { - item, - group, - model_id, - } => { - let item_name_snake = snake_ident(item.name()); - let item_model_name_snake = snake_ident(item.related_model().name()); - - let group_name_snake = snake_ident(group.name()); - let group_model_name_snake = snake_ident(group.related_model().name()); - - Some(quote! { - impl sd_sync::RelationSyncId for SyncId { - type ItemSyncId = super::#item_model_name_snake::SyncId; - type GroupSyncId = super::#group_model_name_snake::SyncId; - - fn split(&self) -> (&Self::ItemSyncId, &Self::GroupSyncId) { - ( - &self.#item_name_snake, - &self.#group_name_snake - ) - } - } - - pub const MODEL_ID: u16 = #model_id; - - impl sd_sync::SyncModel for #model_name_snake::Types { - const MODEL_ID: u16 = MODEL_ID; - } - - impl sd_sync::RelationSyncModel for #model_name_snake::Types { - type SyncId = SyncId; - } - }) - } - ModelSyncType::Shared { model_id, .. } => Some(quote! { - pub const MODEL_ID: u16 = #model_id; - - impl sd_sync::SyncModel for #model_name_snake::Types { - const MODEL_ID: u16 = MODEL_ID; - } - - impl sd_sync::SharedSyncModel for #model_name_snake::Types { - type SyncId = SyncId; - } - }), - ModelSyncType::Local { .. } => None, - } -} - -#[inline] -fn process_unique_params(model: Walker<'_, ModelId>, model_name_snake: &Ident) -> TokenStream { - let field_matches = model - .unique_criterias() - .filter_map(|criteria| match &criteria.fields().next() { - Some(field) if criteria.fields().len() == 1 => { - let field_name_snake = snake_ident(field.name()); - - Some(quote!(#model_name_snake::#field_name_snake::NAME => - #model_name_snake::#field_name_snake::equals( - ::rmpv::ext::from_value(val)? - ), - )) - } - _ => None, - }) - .collect::>(); - - if field_matches.is_empty() { - quote!() - } else { - quote! { - impl #model_name_snake::UniqueWhereParam { - pub fn deserialize(field: &str, val: ::rmpv::Value) -> Result { - Ok(match field { - #(#field_matches)* - _ => return Err(Error::FieldNotFound { - field: field.to_string(), - model: #model_name_snake::NAME.to_string(), - }) - }) - } - } - } - } -} diff --git a/crates/sync-generator/src/sync_data.rs b/crates/sync-generator/src/sync_data.rs deleted file mode 100644 index e8ee713e6..000000000 --- a/crates/sync-generator/src/sync_data.rs +++ /dev/null @@ -1,351 +0,0 @@ -use prisma_client_rust_sdk::{ - prelude::*, - prisma::prisma_models::walkers::{RefinedFieldWalker, RelationFieldWalker}, -}; -use prisma_models::walkers::{FieldWalker, ScalarFieldWalker}; - -use crate::{ModelSyncType, ModelWithSyncType}; - -pub fn enumerate(models: &[ModelWithSyncType<'_>]) -> TokenStream { - let (variants, matches) = models - .iter() - .filter_map(|(model, sync_type)| { - let model_name_snake = snake_ident(model.name()); - let model_name_pascal = pascal_ident(model.name()); - - sync_type - .as_ref() - .filter(|s| { - matches!( - s, - ModelSyncType::Shared { .. } | ModelSyncType::Relation { .. } - ) - }) - .map(|_| { - ( - quote!(#model_name_pascal(#model_name_snake::SyncId, sd_sync::CRDTOperationData)), - quote! { - #model_name_snake::MODEL_ID => - Self::#model_name_pascal(rmpv::ext::from_value(op.record_id)?, op.data) - }, - ) - }) - }) - .unzip::<_, _, Vec<_>, Vec<_>>(); - - let exec_matches = models.iter().filter_map(|(model, sync_type)| { - let model_name_pascal = pascal_ident(model.name()); - let model_name_snake = snake_ident(model.name()); - - let match_arms = match sync_type.as_ref()? { - ModelSyncType::Shared { id, model_id } => { - handle_crdt_ops_shared(id, *model_id, &model_name_snake) - } - ModelSyncType::Relation { item, group, .. } => { - handle_crdt_ops_relation(models, item, group, &model_name_snake) - } - ModelSyncType::Local { .. } => return None, - }; - - Some(quote! { - Self::#model_name_pascal(id, data) => { - #match_arms - } - }) - }); - - let error_enum = declare_error_enum(); - - quote! { - pub enum ModelSyncData { - #(#variants),* - } - - impl ModelSyncData { - pub fn from_op(op: sd_sync::CRDTOperation) -> Result { - Ok(match op.model_id { - #(#matches),*, - _ => return Err(Error::InvalidModelId(op.model_id)), - }) - } - - pub async fn exec(self, db: &prisma::PrismaClient) -> Result<(), Error> { - match self { - #(#exec_matches),* - } - - Ok(()) - } - } - - #error_enum - } -} - -fn declare_error_enum() -> TokenStream { - quote! { - #[derive(Debug)] - pub enum Error { - Rmpv(rmpv::ext::Error), - RmpSerialize(rmp_serde::encode::Error), - Prisma(prisma_client_rust::QueryError), - InvalidModelId(sd_sync::ModelId), - FieldNotFound { field: String, model: String }, - MissingRelationData { field: String, model: String }, - RelatedEntryNotFound { field: String, model: String }, - } - - impl From for Error { - fn from(e: rmpv::ext::Error) -> Self { - Self::Rmpv(e) - } - } - - impl From for Error { - fn from(e: rmp_serde::encode::Error) -> Self { - Self::RmpSerialize(e) - } - } - - impl From for Error { - fn from(e: prisma_client_rust::QueryError) -> Self { - Self::Prisma(e) - } - } - - impl std::fmt::Display for Error { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - Self::Rmpv(e) => write!(f, "Failed to serialize or deserialize rmpv data: {e}"), - Self::RmpSerialize(e) => write!(f, "Failed to serialize rmp data: {e}"), - Self::Prisma(e) => write!(f, "Prisma error: {e}"), - Self::InvalidModelId(id) => write!(f, "Invalid model id: {id}"), - Self::FieldNotFound { field, model } => { - write!(f, "Field '{field}' not found in model '{model}'") - } - Self::MissingRelationData { field, model } => { - write!( - f, - "Field '{field}' missing relation data in model '{model}'" - ) - } - Self::RelatedEntryNotFound { field, model } => { - write!( - f, - "Related entry for field '{field}' not found in table '{model}'" - ) - } - } - } - } - - impl std::error::Error for Error {} - } -} - -fn handle_crdt_ops_relation( - models: &[ModelWithSyncType<'_>], - item: &RelationFieldWalker<'_>, - group: &RelationFieldWalker<'_>, - model_name_snake: &Ident, -) -> TokenStream { - let compound_id = format_ident!( - "{}", - group - .fields() - .expect("missing group fields") - .chain(item.fields().expect("missing item fields")) - .map(ScalarFieldWalker::name) - .collect::>() - .join("_") - ); - - let db_batch_items = { - let batch_item = |item: &RelationFieldWalker<'_>| { - let item_model_sync_id_field_name_snake = models - .iter() - .find(|m| m.0.name() == item.related_model().name()) - .and_then(|(_m, sync)| sync.as_ref()) - .map(|sync| snake_ident(sync.sync_id()[0].name())) - .expect("missing sync id field name for relation"); - - let item_model_name_snake = snake_ident(item.related_model().name()); - let item_field_name_snake = snake_ident(item.name()); - - quote! { - db.#item_model_name_snake() - .find_unique( - prisma::#item_model_name_snake::#item_model_sync_id_field_name_snake::equals( - id.#item_field_name_snake.#item_model_sync_id_field_name_snake.clone() - ) - ) - .select(prisma::#item_model_name_snake::select!({ id })) - } - }; - - [batch_item(group), batch_item(item)] - }; - - let create_items = { - let create_item = |item: &RelationFieldWalker<'_>, var: TokenStream| { - let item_model_name_snake = snake_ident(item.related_model().name()); - - quote!( - prisma::#item_model_name_snake::id::equals(#var.id) - ) - }; - - [ - create_item(item, quote!(item)), - create_item(group, quote!(group)), - ] - }; - - quote! { - let (Some(group), Some(item)) = - (#(#db_batch_items.exec().await?),*) else { - panic!("item and group not found!"); - }; - - let id = prisma::#model_name_snake::#compound_id(group.id, item.id); - - match data { - sd_sync::CRDTOperationData::Create(_) => { - db.#model_name_snake() - .upsert( - id, - prisma::#model_name_snake::create( - #(#create_items),*, - vec![] - ), - vec![], - ) - .exec() - .await?; - }, - - sd_sync::CRDTOperationData::Update(data) => { - let data = data.into_iter() - .map(|(field, value)| { - prisma::#model_name_snake::SetParam::deserialize(&field, value) - }) - .collect::, _>>()?; - - db.#model_name_snake() - .upsert( - id, - prisma::#model_name_snake::create( - #(#create_items),*, - data.clone(), - ), - data, - ) - .exec() - .await?; - }, - - sd_sync::CRDTOperationData::Delete => { - db.#model_name_snake() - .delete(id) - .exec() - .await?; - }, - } - } -} - -#[inline] -fn handle_crdt_ops_shared( - id: &FieldWalker<'_>, - model_id: u16, - model_name_snake: &Ident, -) -> TokenStream { - let (get_id, equals_value, id_name_snake, create_id) = match id.refine() { - RefinedFieldWalker::Relation(rel) => { - let scalar_field = rel - .fields() - .expect("missing fields") - .next() - .expect("empty fields"); - - let id_name_snake = snake_ident(scalar_field.name()); - let field_name_snake = snake_ident(rel.name()); - - let opposite_model_name_snake = snake_ident( - rel.opposite_relation_field() - .expect("missing opposite relation field") - .model() - .name(), - ); - - let relation_equals_condition = quote!(prisma::#opposite_model_name_snake::pub_id::equals( - id.#field_name_snake.pub_id.clone() - )); - - let pub_id_field = format!("{field_name_snake}::pub_id"); - - let rel_fetch = quote! { - let rel = db.#opposite_model_name_snake() - .find_unique(#relation_equals_condition) - .exec() - .await?.ok_or_else(|| Error::RelatedEntryNotFound { - field: #pub_id_field.to_string(), - model: prisma::#opposite_model_name_snake::NAME.to_string(), - })?; - }; - - ( - Some(rel_fetch), - quote!(rel.id), - id_name_snake, - relation_equals_condition, - ) - } - - RefinedFieldWalker::Scalar(s) => { - let field_name_snake = snake_ident(s.name()); - let thing = quote!(id.#field_name_snake.clone()); - - (None, thing.clone(), field_name_snake, thing) - } - }; - - quote! { - #get_id - - match data { - sd_sync::CRDTOperationData::Create(data) | sd_sync::CRDTOperationData::Update(data) => { - let data = data.into_iter() - .map(|(field, value)| { - prisma::#model_name_snake::SetParam::deserialize(&field, value) - }) - .collect::, _>>()?; - - db.#model_name_snake() - .upsert( - prisma::#model_name_snake::#id_name_snake::equals(#equals_value), - prisma::#model_name_snake::create(#create_id, data.clone()), - data, - ) - .exec() - .await?; - }, - - sd_sync::CRDTOperationData::Delete => { - db.#model_name_snake() - .delete(prisma::#model_name_snake::#id_name_snake::equals(#equals_value)) - .exec() - .await?; - - db.crdt_operation() - .delete_many(vec![ - prisma::crdt_operation::model::equals(#model_id as i32), - prisma::crdt_operation::record_id::equals(rmp_serde::to_vec(&id)?), - prisma::crdt_operation::kind::equals(sd_sync::OperationKind::Create.to_string()), - ]) - .exec() - .await?; - }, - } - } -} diff --git a/crates/sync/Cargo.toml b/crates/sync/Cargo.toml deleted file mode 100644 index 302b37a53..000000000 --- a/crates/sync/Cargo.toml +++ /dev/null @@ -1,16 +0,0 @@ -[package] -name = "sd-sync" -version = "0.1.0" - -edition.workspace = true -license.workspace = true -repository.workspace = true - -[dependencies] -prisma-client-rust = { workspace = true } -rmp = { workspace = true } -rmp-serde = { workspace = true } -rmpv = { workspace = true } -serde = { workspace = true } -uhlc = { workspace = true } -uuid = { workspace = true, features = ["serde", "v7"] } diff --git a/crates/sync/src/compressed.rs b/crates/sync/src/compressed.rs deleted file mode 100644 index a2e3a147d..000000000 --- a/crates/sync/src/compressed.rs +++ /dev/null @@ -1,359 +0,0 @@ -use crate::{CRDTOperation, CRDTOperationData, DevicePubId, ModelId, RecordId}; - -use std::collections::{hash_map::Entry, BTreeMap, HashMap}; - -use serde::{Deserialize, Serialize}; -use uhlc::NTP64; - -#[derive(Serialize, Deserialize, Debug, Clone)] -pub struct CompressedCRDTOperationsPerModel(pub Vec<(ModelId, CompressedCRDTOperationsPerRecord)>); - -pub type CompressedCRDTOperationsPerRecord = Vec<(RecordId, Vec)>; - -/// Stores a bunch of [`CRDTOperation`]s in a more memory-efficient form for sending to the cloud. -#[derive(Serialize, Deserialize, Debug)] -pub struct CompressedCRDTOperationsPerModelPerDevice( - pub Vec<(DevicePubId, CompressedCRDTOperationsPerModel)>, -); - -impl CompressedCRDTOperationsPerModelPerDevice { - /// Creates a new [`CompressedCRDTOperationsPerModelPerDevice`] from a vector of [`CRDTOperation`]s. - /// - /// # Panics - /// - /// Will panic if for some reason `rmp_serde::to_vec` fails to serialize a `rmpv::Value` to bytes. - #[must_use] - pub fn new(ops: Vec) -> Self { - let mut compressed_map = BTreeMap::< - DevicePubId, - BTreeMap, (RecordId, Vec)>>, - >::new(); - - for CRDTOperation { - device_pub_id, - timestamp, - model_id, - record_id, - data, - } in ops - { - let records = compressed_map - .entry(device_pub_id) - .or_default() - .entry(model_id) - .or_default(); - - // Can't use RecordId as a key because rmpv::Value doesn't implement Hash + Eq. - // So we use it's serialized bytes as a key. - let record_id_bytes = - rmp_serde::to_vec_named(&record_id).expect("already serialized to Value"); - - match records.entry(record_id_bytes) { - Entry::Occupied(mut entry) => { - entry - .get_mut() - .1 - .push(CompressedCRDTOperation { timestamp, data }); - } - Entry::Vacant(entry) => { - entry.insert((record_id, vec![CompressedCRDTOperation { timestamp, data }])); - } - } - } - - Self( - compressed_map - .into_iter() - .map(|(device_pub_id, model_map)| { - ( - device_pub_id, - CompressedCRDTOperationsPerModel( - model_map - .into_iter() - .map(|(model_id, ops_per_record_map)| { - (model_id, ops_per_record_map.into_values().collect()) - }) - .collect(), - ), - ) - }) - .collect(), - ) - } - - /// Creates a new [`CompressedCRDTOperationsPerModel`] from crdt operation of a single device. - /// - /// # Panics - /// Will panic if there are more than one device. - #[must_use] - pub fn new_single_device( - ops: Vec, - ) -> (DevicePubId, CompressedCRDTOperationsPerModel) { - let Self(mut compressed) = Self::new(ops); - - assert_eq!(compressed.len(), 1, "Expected a single device"); - - compressed.remove(0) - } - - #[must_use] - pub fn first(&self) -> Option<(DevicePubId, ModelId, &RecordId, &CompressedCRDTOperation)> { - self.0.first().and_then(|(instance, data)| { - data.0.first().and_then(|(model, data)| { - data.first() - .and_then(|(record, ops)| ops.first().map(|op| (*instance, *model, record, op))) - }) - }) - } - - #[must_use] - pub fn last(&self) -> Option<(DevicePubId, ModelId, &RecordId, &CompressedCRDTOperation)> { - self.0.last().and_then(|(instance, data)| { - data.0.last().and_then(|(model, data)| { - data.last() - .and_then(|(record, ops)| ops.last().map(|op| (*instance, *model, record, op))) - }) - }) - } - - #[must_use] - pub fn len(&self) -> usize { - self.0 - .iter() - .map(|(_, data)| { - data.0 - .iter() - .map(|(_, data)| data.iter().map(|(_, ops)| ops.len()).sum::()) - .sum::() - }) - .sum() - } - - #[must_use] - pub fn is_empty(&self) -> bool { - self.len() == 0 - } - - #[must_use] - pub fn into_ops(self) -> Vec { - let mut ops = Vec::with_capacity(self.len()); - - for (device_pub_id, device_messages) in self.0 { - for (model_id, model_messages) in device_messages.0 { - for (record_id, record) in model_messages { - for op in record { - ops.push(CRDTOperation { - device_pub_id, - model_id, - record_id: record_id.clone(), - timestamp: op.timestamp, - data: op.data, - }); - } - } - } - } - - ops - } -} - -impl CompressedCRDTOperationsPerModel { - #[must_use] - pub fn first(&self) -> Option<(ModelId, &RecordId, &CompressedCRDTOperation)> { - self.0.first().and_then(|(model_id, data)| { - data.first() - .and_then(|(record_id, ops)| ops.first().map(|op| (*model_id, record_id, op))) - }) - } - - #[must_use] - pub fn last(&self) -> Option<(ModelId, &RecordId, &CompressedCRDTOperation)> { - self.0.last().and_then(|(model_id, data)| { - data.last() - .and_then(|(record_id, ops)| ops.last().map(|op| (*model_id, record_id, op))) - }) - } - - #[must_use] - pub fn len(&self) -> usize { - self.0 - .iter() - .map(|(_, data)| data.iter().map(|(_, ops)| ops.len()).sum::()) - .sum() - } - - #[must_use] - pub fn is_empty(&self) -> bool { - self.len() == 0 - } - - #[must_use] - pub fn into_ops(self, device_pub_id: DevicePubId) -> Vec { - let mut ops = Vec::with_capacity(self.len()); - - for (model_id, model_messages) in self.0 { - for (record_id, record) in model_messages { - for op in record { - ops.push(CRDTOperation { - device_pub_id, - model_id, - record_id: record_id.clone(), - timestamp: op.timestamp, - data: op.data, - }); - } - } - } - - ops - } -} - -#[derive(PartialEq, Serialize, Deserialize, Clone, Debug)] -pub struct CompressedCRDTOperation { - pub timestamp: NTP64, - pub data: CRDTOperationData, -} - -impl From for CompressedCRDTOperation { - fn from( - CRDTOperation { - timestamp, data, .. - }: CRDTOperation, - ) -> Self { - Self { timestamp, data } - } -} - -#[cfg(test)] -mod test { - use super::*; - use uuid::Uuid; - - #[test] - fn compress() { - let device_pub_id = Uuid::now_v7(); - - let uncompressed = vec![ - CRDTOperation { - device_pub_id, - timestamp: NTP64(0), - model_id: 0, - record_id: rmpv::Value::Nil, - data: CRDTOperationData::create(), - }, - CRDTOperation { - device_pub_id, - timestamp: NTP64(0), - model_id: 0, - record_id: rmpv::Value::Nil, - data: CRDTOperationData::create(), - }, - CRDTOperation { - device_pub_id, - timestamp: NTP64(0), - model_id: 0, - record_id: rmpv::Value::Nil, - data: CRDTOperationData::create(), - }, - CRDTOperation { - device_pub_id, - timestamp: NTP64(0), - model_id: 1, - record_id: rmpv::Value::Nil, - data: CRDTOperationData::create(), - }, - CRDTOperation { - device_pub_id, - timestamp: NTP64(0), - model_id: 1, - record_id: rmpv::Value::Nil, - data: CRDTOperationData::create(), - }, - CRDTOperation { - device_pub_id, - timestamp: NTP64(0), - model_id: 0, - record_id: rmpv::Value::Nil, - data: CRDTOperationData::create(), - }, - CRDTOperation { - device_pub_id, - timestamp: NTP64(0), - model_id: 0, - record_id: rmpv::Value::Nil, - data: CRDTOperationData::create(), - }, - ]; - - let CompressedCRDTOperationsPerModelPerDevice(compressed) = - CompressedCRDTOperationsPerModelPerDevice::new(uncompressed); - - assert_eq!(compressed[0].1 .0[0].0, 0); - assert_eq!(compressed[0].1 .0[1].0, 1); - - assert_eq!(compressed[0].1 .0[0].1[0].1.len(), 5); - assert_eq!(compressed[0].1 .0[1].1[0].1.len(), 2); - } - - #[test] - fn into_ops() { - let compressed = CompressedCRDTOperationsPerModelPerDevice(vec![( - Uuid::new_v4(), - CompressedCRDTOperationsPerModel(vec![ - ( - 0, - vec![( - rmpv::Value::Nil, - vec![ - CompressedCRDTOperation { - timestamp: NTP64(0), - data: CRDTOperationData::create(), - }, - CompressedCRDTOperation { - timestamp: NTP64(0), - data: CRDTOperationData::create(), - }, - CompressedCRDTOperation { - timestamp: NTP64(0), - data: CRDTOperationData::create(), - }, - CompressedCRDTOperation { - timestamp: NTP64(0), - data: CRDTOperationData::create(), - }, - CompressedCRDTOperation { - timestamp: NTP64(0), - data: CRDTOperationData::create(), - }, - ], - )], - ), - ( - 1, - vec![( - rmpv::Value::Nil, - vec![ - CompressedCRDTOperation { - timestamp: NTP64(0), - data: CRDTOperationData::create(), - }, - CompressedCRDTOperation { - timestamp: NTP64(0), - data: CRDTOperationData::create(), - }, - ], - )], - ), - ]), - )]); - - let uncompressed = compressed.into_ops(); - - assert_eq!(uncompressed.len(), 7); - assert_eq!(uncompressed[2].model_id, 0); - assert_eq!(uncompressed[4].model_id, 0); - assert_eq!(uncompressed[6].model_id, 1); - } -} diff --git a/crates/sync/src/crdt.rs b/crates/sync/src/crdt.rs deleted file mode 100644 index 3cbdf23d2..000000000 --- a/crates/sync/src/crdt.rs +++ /dev/null @@ -1,76 +0,0 @@ -use crate::{DevicePubId, ModelId}; - -use std::{collections::BTreeMap, fmt}; - -use serde::{Deserialize, Serialize}; -use uhlc::NTP64; - -pub enum OperationKind<'a> { - Create, - Update(Vec<&'a str>), - Delete, -} - -impl fmt::Display for OperationKind<'_> { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - OperationKind::Create => write!(f, "c"), - OperationKind::Update(fields) => write!(f, "u:{}:", fields.join(":")), - OperationKind::Delete => write!(f, "d"), - } - } -} - -#[derive(PartialEq, Serialize, Deserialize, Clone, Debug)] -pub enum CRDTOperationData { - #[serde(rename = "c")] - Create(BTreeMap), - #[serde(rename = "u")] - Update(BTreeMap), - #[serde(rename = "d")] - Delete, -} - -impl CRDTOperationData { - #[must_use] - pub fn create() -> Self { - Self::Create(BTreeMap::default()) - } - - #[must_use] - pub fn as_kind(&self) -> OperationKind<'_> { - match self { - Self::Create(_) => OperationKind::Create, - Self::Update(fields_and_values) => { - OperationKind::Update(fields_and_values.keys().map(String::as_str).collect()) - } - Self::Delete => OperationKind::Delete, - } - } -} - -#[derive(PartialEq, Serialize, Deserialize, Clone)] -pub struct CRDTOperation { - pub device_pub_id: DevicePubId, - pub timestamp: NTP64, - pub model_id: ModelId, - pub record_id: rmpv::Value, - pub data: CRDTOperationData, -} - -impl CRDTOperation { - #[must_use] - pub fn kind(&self) -> OperationKind<'_> { - self.data.as_kind() - } -} - -impl fmt::Debug for CRDTOperation { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("CRDTOperation") - .field("data", &self.data) - .field("model", &self.model_id) - .field("record_id", &self.record_id.to_string()) - .finish_non_exhaustive() - } -} diff --git a/crates/sync/src/factory.rs b/crates/sync/src/factory.rs deleted file mode 100644 index 7c73f8b5f..000000000 --- a/crates/sync/src/factory.rs +++ /dev/null @@ -1,163 +0,0 @@ -use crate::{ - CRDTOperation, CRDTOperationData, DevicePubId, RelationSyncId, RelationSyncModel, - SharedSyncModel, SyncId, SyncModel, -}; - -use uhlc::HLC; - -pub trait OperationFactory { - fn get_clock(&self) -> &HLC; - - fn get_device_pub_id(&self) -> DevicePubId; - - fn new_op>( - &self, - id: &SId, - data: CRDTOperationData, - ) -> CRDTOperation { - CRDTOperation { - device_pub_id: self.get_device_pub_id(), - timestamp: *self.get_clock().new_timestamp().get_time(), - model_id: ::MODEL_ID, - record_id: rmp_serde::from_slice::( - &rmp_serde::to_vec_named(id).expect("failed to serialize record id to msgpack"), - ) - .expect("failed to deserialize record id to msgpack value"), - data, - } - } - - fn shared_create( - &self, - id: impl SyncId, - values: impl IntoIterator + 'static, - ) -> CRDTOperation { - self.new_op( - &id, - CRDTOperationData::Create( - values - .into_iter() - .map(|(name, value)| (name.to_string(), value)) - .collect(), - ), - ) - } - - fn shared_update( - &self, - id: impl SyncId, - values: impl IntoIterator + 'static, - ) -> CRDTOperation { - self.new_op( - &id, - CRDTOperationData::Update( - values - .into_iter() - .map(|(name, value)| (name.to_string(), value)) - .collect(), - ), - ) - } - - fn shared_delete(&self, id: impl SyncId) -> CRDTOperation { - self.new_op(&id, CRDTOperationData::Delete) - } - - fn relation_create( - &self, - id: impl RelationSyncId, - values: impl IntoIterator + 'static, - ) -> CRDTOperation { - self.new_op( - &id, - CRDTOperationData::Create( - values - .into_iter() - .map(|(name, value)| (name.to_string(), value)) - .collect(), - ), - ) - } - - fn relation_update( - &self, - id: impl RelationSyncId, - values: impl IntoIterator + 'static, - ) -> CRDTOperation { - self.new_op( - &id, - CRDTOperationData::Update( - values - .into_iter() - .map(|(name, value)| (name.to_string(), value)) - .collect(), - ), - ) - } - - fn relation_delete( - &self, - id: impl RelationSyncId, - ) -> CRDTOperation { - self.new_op(&id, CRDTOperationData::Delete) - } -} - -#[macro_export] -macro_rules! sync_entry { - (nil, $($prisma_column_module:tt)+) => { - ($($prisma_column_module)+::NAME, ::sd_utils::msgpack!(nil)) - }; - - ($value:expr, $($prisma_column_module:tt)+) => { - ($($prisma_column_module)+::NAME, ::sd_utils::msgpack!($value)) - }; - -} - -#[macro_export] -macro_rules! option_sync_entry { - ($value:expr, $($prisma_column_module:tt)+) => { - $value.map(|value| $crate::sync_entry!(value, $($prisma_column_module)+)) - } -} - -#[macro_export] -macro_rules! sync_db_entry { - ($value:expr, $($prisma_column_module:tt)+) => {{ - let value = $value.into(); - ( - $crate::sync_entry!(&value, $($prisma_column_module)+), - $($prisma_column_module)+::set(Some(value)) - ) - }} -} - -#[macro_export] -macro_rules! sync_db_nullable_entry { - ($value:expr, $($prisma_column_module:tt)+) => {{ - let value = $value.into(); - ( - $crate::sync_entry!(&value, $($prisma_column_module)+), - $($prisma_column_module)+::set(value) - ) - }} -} - -#[macro_export] -macro_rules! sync_db_not_null_entry { - ($value:expr, $($prisma_column_module:tt)+) => {{ - let value = $value.into(); - ( - $crate::sync_entry!(&value, $($prisma_column_module)+), - $($prisma_column_module)+::set(value) - ) - }} -} - -#[macro_export] -macro_rules! option_sync_db_entry { - ($value:expr, $($prisma_column_module:tt)+) => { - $value.map(|value| $crate::sync_db_entry!(value, $($prisma_column_module)+)) - }; -} diff --git a/crates/sync/src/lib.rs b/crates/sync/src/lib.rs deleted file mode 100644 index 239a1298d..000000000 --- a/crates/sync/src/lib.rs +++ /dev/null @@ -1,44 +0,0 @@ -#![warn( - clippy::all, - clippy::pedantic, - clippy::correctness, - clippy::perf, - clippy::style, - clippy::suspicious, - clippy::complexity, - clippy::nursery, - clippy::unwrap_used, - unused_qualifications, - rust_2018_idioms, - trivial_casts, - trivial_numeric_casts, - unused_allocation, - clippy::unnecessary_cast, - clippy::cast_lossless, - clippy::cast_possible_truncation, - clippy::cast_possible_wrap, - clippy::cast_precision_loss, - clippy::cast_sign_loss, - clippy::dbg_macro, - clippy::deprecated_cfg_attr, - clippy::separated_literal_suffix, - deprecated -)] -#![forbid(deprecated_in_future)] -#![allow(clippy::missing_errors_doc, clippy::module_name_repetitions)] - -mod compressed; -mod crdt; -mod factory; -mod model_traits; - -pub use compressed::*; -pub use crdt::*; -pub use factory::*; -pub use model_traits::*; - -pub use uhlc::NTP64; - -pub type DevicePubId = uuid::Uuid; -pub type ModelId = u16; -pub type RecordId = rmpv::Value; diff --git a/crates/sync/src/model_traits.rs b/crates/sync/src/model_traits.rs deleted file mode 100644 index 48a4efacd..000000000 --- a/crates/sync/src/model_traits.rs +++ /dev/null @@ -1,27 +0,0 @@ -use crate::ModelId; - -use prisma_client_rust::ModelTypes; -use serde::{de::DeserializeOwned, Serialize}; - -pub trait SyncId: Serialize + DeserializeOwned { - type Model; -} - -pub trait SyncModel: ModelTypes { - const MODEL_ID: ModelId; -} - -pub trait SharedSyncModel: SyncModel { - type SyncId: SyncId; -} - -pub trait RelationSyncId: SyncId { - type ItemSyncId: SyncId; - type GroupSyncId: SyncId; - - fn split(&self) -> (&Self::ItemSyncId, &Self::GroupSyncId); -} - -pub trait RelationSyncModel: SyncModel { - type SyncId: RelationSyncId; -} diff --git a/core-new/task-validator/Cargo.toml b/crates/task-validator/Cargo.toml similarity index 100% rename from core-new/task-validator/Cargo.toml rename to crates/task-validator/Cargo.toml diff --git a/core-new/task-validator/src/main.rs b/crates/task-validator/src/main.rs similarity index 100% rename from core-new/task-validator/src/main.rs rename to crates/task-validator/src/main.rs diff --git a/crates/utils/Cargo.toml b/crates/utils/Cargo.toml deleted file mode 100644 index 2960a21c4..000000000 --- a/crates/utils/Cargo.toml +++ /dev/null @@ -1,23 +0,0 @@ -[package] -name = "sd-utils" -version = "0.1.0" - -edition.workspace = true -license.workspace = true -repository.workspace = true -rust-version.workspace = true - -[dependencies] -# Spacedrive Sub-crates -sd-prisma = { path = "../prisma" } - -# Workspace dependencies -chrono = { workspace = true } -prisma-client-rust = { workspace = true } -rmp-serde = { workspace = true } -rmpv = { workspace = true } -rspc = { workspace = true, features = ["unstable"] } -thiserror = { workspace = true } -tracing = { workspace = true } -uhlc = { workspace = true } -uuid = { workspace = true } diff --git a/crates/utils/src/db.rs b/crates/utils/src/db.rs deleted file mode 100644 index e27a000a6..000000000 --- a/crates/utils/src/db.rs +++ /dev/null @@ -1,163 +0,0 @@ -use prisma_client_rust::{ - migrations::{DbPushError, MigrateDeployError}, - NewClientError, -}; -use sd_prisma::prisma::PrismaClient; -use thiserror::Error; - -/// `[MigrationError]` represents an error that occurring while opening a initialising and running migrations on the database. -#[derive(Error, Debug)] -pub enum MigrationError { - #[error("An error occurred while initialising a new database connection: {0}")] - NewClient(#[from] Box), - #[error("An error occurred during migration: {0}")] - MigrateFailed(#[from] MigrateDeployError), - #[cfg(debug_assertions)] - #[error("An error occurred during migration: {0}")] - DbPushFailed(#[from] DbPushError), -} - -/// `[load_and_migrate]` will load the database from the given path and migrate it to the latest version of the schema. -pub async fn load_and_migrate(db_url: &str) -> Result { - let client = PrismaClient::_builder() - .with_url(db_url.to_string()) - .build() - .await - .map_err(Box::new)?; - - client._migrate_deploy().await?; - - #[cfg(debug_assertions)] - { - let mut builder = client._db_push(); - - if std::env::var("SD_ACCEPT_DATA_LOSS") - .map(|v| v == "true") - .unwrap_or(false) - { - builder = builder.accept_data_loss(); - } - - if std::env::var("SD_FORCE_RESET_DB") - .map(|v| v == "true") - .unwrap_or(false) - { - builder = builder.force_reset(); - } - - let res = builder.await; - - match res { - Ok(_) => {} - Err(e @ DbPushError::PossibleDataLoss(_)) => { - eprintln!("Pushing Prisma schema may result in data loss. Use `SD_ACCEPT_DATA_LOSS=true` to force it."); - Err(e)?; - } - Err(e) => Err(e)?, - } - } - - Ok(client) -} - -/// Construct back an inode after storing it in database -#[must_use] -pub const fn inode_from_db(db_inode: &[u8]) -> u64 { - u64::from_le_bytes([ - db_inode[0], - db_inode[1], - db_inode[2], - db_inode[3], - db_inode[4], - db_inode[5], - db_inode[6], - db_inode[7], - ]) -} - -/// Constructs a database representation of an inode -#[must_use] -pub fn inode_to_db(inode: u64) -> Vec { - inode.to_le_bytes().to_vec() -} - -#[must_use] -pub fn ffmpeg_data_field_to_db(field: i64) -> Vec { - field.to_be_bytes().to_vec() -} - -#[must_use] -pub const fn ffmpeg_data_field_from_db(field: &[u8]) -> i64 { - i64::from_be_bytes([ - field[0], field[1], field[2], field[3], field[4], field[5], field[6], field[7], - ]) -} - -#[must_use] -pub const fn size_in_bytes_from_db(db_size_in_bytes: &[u8]) -> u64 { - u64::from_be_bytes([ - db_size_in_bytes[0], - db_size_in_bytes[1], - db_size_in_bytes[2], - db_size_in_bytes[3], - db_size_in_bytes[4], - db_size_in_bytes[5], - db_size_in_bytes[6], - db_size_in_bytes[7], - ]) -} - -#[must_use] -pub fn size_in_bytes_to_db(size: u64) -> Vec { - size.to_be_bytes().to_vec() -} - -#[derive(Error, Debug)] -#[error("Missing field {0}")] -pub struct MissingFieldError(&'static str); - -impl MissingFieldError { - #[must_use] - pub const fn new(value: &'static str) -> Self { - Self(value) - } -} - -impl From for rspc::Error { - fn from(value: MissingFieldError) -> Self { - Self::with_cause( - rspc::ErrorCode::InternalServerError, - "Missing crucial data in the database".to_string(), - value, - ) - } -} - -pub trait OptionalField: Sized { - type Out; - - fn transform(self) -> Option; -} - -impl OptionalField for Option { - type Out = T; - - fn transform(self) -> Self { - self - } -} -impl<'a, T> OptionalField for &'a Option { - type Out = &'a T; - - fn transform(self) -> Option { - self.as_ref() - } -} - -/// If `data` is `Some(t)` returns `Ok(t)`, otherwise returns a `MissingFieldError(field)` -pub fn maybe_missing( - data: T, - field: &'static str, -) -> Result { - data.transform().ok_or(MissingFieldError(field)) -} diff --git a/crates/utils/src/error.rs b/crates/utils/src/error.rs deleted file mode 100644 index 6bd3c08f6..000000000 --- a/crates/utils/src/error.rs +++ /dev/null @@ -1,59 +0,0 @@ -use std::{io, path::Path}; - -use thiserror::Error; -use tracing::error; - -pub fn report_error( - message: &'static str, -) -> impl Fn(E) -> E { - move |e| { - error!(?e, "{message}"); - e - } -} - -#[derive(Debug, Error)] -#[error("error accessing path: '{}'", .path.display())] -pub struct FileIOError { - pub path: Box, - #[source] - pub source: io::Error, - pub maybe_context: Option<&'static str>, -} - -impl> From<(P, io::Error)> for FileIOError { - fn from((path, source): (P, io::Error)) -> Self { - Self { - path: path.as_ref().into(), - source, - maybe_context: None, - } - } -} - -impl> From<(P, io::Error, &'static str)> for FileIOError { - fn from((path, source, context): (P, io::Error, &'static str)) -> Self { - Self { - path: path.as_ref().into(), - source, - maybe_context: Some(context), - } - } -} - -impl From for rspc::Error { - fn from(value: FileIOError) -> Self { - Self::with_cause( - rspc::ErrorCode::InternalServerError, - value - .maybe_context - .unwrap_or("Error accessing file system") - .to_string(), - value, - ) - } -} - -#[derive(Debug, Error)] -#[error("received a non UTF-8 path: ", .0.to_string_lossy())] -pub struct NonUtf8PathError(pub Box); diff --git a/crates/utils/src/lib.rs b/crates/utils/src/lib.rs deleted file mode 100644 index 09f45d609..000000000 --- a/crates/utils/src/lib.rs +++ /dev/null @@ -1,135 +0,0 @@ -#![warn( - clippy::all, - clippy::pedantic, - clippy::correctness, - clippy::perf, - clippy::style, - clippy::suspicious, - clippy::complexity, - clippy::nursery, - clippy::unwrap_used, - unused_qualifications, - rust_2018_idioms, - trivial_casts, - trivial_numeric_casts, - unused_allocation, - clippy::unnecessary_cast, - clippy::cast_lossless, - clippy::cast_possible_truncation, - clippy::cast_possible_wrap, - clippy::cast_precision_loss, - clippy::cast_sign_loss, - clippy::dbg_macro, - clippy::deprecated_cfg_attr, - clippy::separated_literal_suffix, - deprecated -)] -#![forbid(deprecated_in_future)] -#![allow(clippy::missing_errors_doc, clippy::module_name_repetitions)] - -use std::time::{SystemTime, UNIX_EPOCH}; - -use chrono::{DateTime, Utc}; -use uhlc::NTP64; -use uuid::Uuid; - -pub mod db; -pub mod error; - -/// Combines an iterator of `T` and an iterator of `Option`, -/// removing any `None` values in the process -pub fn chain_optional_iter( - required: impl IntoIterator, - optional: impl IntoIterator>, -) -> Vec { - required - .into_iter() - .map(Some) - .chain(optional) - .flatten() - .collect() -} - -/// A splitted version of `u64`, divided into `(u32, u32)` -/// -/// rspc/specta doesn't support `BigInt`, so we need this hack -pub type U64Front = (u32, u32); - -#[inline] -#[must_use] -pub const fn u64_to_frontend(num: u64) -> U64Front { - #[allow(clippy::cast_possible_truncation)] - { - // SAFETY: We're splitting in (high, low) parts, so we're not going to lose data on truncation - ((num >> 32) as u32, num as u32) - } -} - -/// A splitted version of `i64`, divided into `(i32, u32)` -/// -/// rspc/specta doesn't support `BigInt`, so we need this hack -pub type I64Front = (i32, u32); - -#[inline] -#[must_use] -pub const fn i64_to_frontend(num: i64) -> I64Front { - #[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)] - { - // SAFETY: We're splitting in (high, low) parts, so we're not going to lose data on truncation - ((num >> 32) as i32, num as u32) - } -} - -#[inline] -#[must_use] -pub fn uuid_to_bytes(uuid: &Uuid) -> Vec { - uuid.as_bytes().to_vec() -} - -/// Converts a byte slice to a `Uuid` -/// # Panics -/// Panics if the byte slice is not a valid `Uuid` which means we have a corrupted database -#[inline] -#[must_use] -pub fn from_bytes_to_uuid(bytes: &[u8]) -> Uuid { - Uuid::from_slice(bytes).expect("corrupted uuid in database") -} - -#[macro_export] -macro_rules! msgpack { - (nil) => { - ::rmpv::Value::Nil - }; - ($e:expr) => {{ - let bytes = rmp_serde::to_vec_named(&$e).expect("failed to serialize msgpack"); - let value: rmpv::Value = rmp_serde::from_slice(&bytes).expect("failed to deserialize msgpack"); - - value - }} -} - -/// Helper function to convert a [`chrono::DateTime`] to a [`uhlc::NTP64`] -#[allow(clippy::missing_panics_doc)] // Doesn't actually panic -#[must_use] -pub fn datetime_to_timestamp(latest_time: DateTime) -> NTP64 { - NTP64::from( - SystemTime::from(latest_time) - .duration_since(UNIX_EPOCH) - .expect("hardcoded earlier time, nothing is earlier than UNIX_EPOCH"), - ) -} - -/// Helper function to convert a [`uhlc::NTP64`] to a [`chrono::DateTime`] -#[must_use] -pub fn timestamp_to_datetime(timestamp: NTP64) -> DateTime { - DateTime::from(timestamp.to_system_time()) -} - -// Only used for testing purposes. Do not use in production code. -use std::any::type_name; - -#[inline] -#[must_use] -pub fn test_type_of(_: T) -> &'static str { - type_name::() -} diff --git a/core-new/docs/README.md b/docs/README.md similarity index 100% rename from core-new/docs/README.md rename to docs/README.md diff --git a/core-new/docs/benchmarks.md b/docs/benchmarks.md similarity index 100% rename from core-new/docs/benchmarks.md rename to docs/benchmarks.md diff --git a/core-new/docs/architecture.md b/docs/core/architecture.md similarity index 100% rename from core-new/docs/architecture.md rename to docs/core/architecture.md diff --git a/core-new/docs/cli-multi-instance.md b/docs/core/cli-multi-instance.md similarity index 100% rename from core-new/docs/cli-multi-instance.md rename to docs/core/cli-multi-instance.md diff --git a/core-new/docs/cli.md b/docs/core/cli.md similarity index 100% rename from core-new/docs/cli.md rename to docs/core/cli.md diff --git a/core-new/docs/database.md b/docs/core/database.md similarity index 100% rename from core-new/docs/database.md rename to docs/core/database.md diff --git a/core-new/docs/domain-models.md b/docs/core/domain-models.md similarity index 100% rename from core-new/docs/domain-models.md rename to docs/core/domain-models.md diff --git a/core-new/docs/examples.md b/docs/core/examples.md similarity index 100% rename from core-new/docs/examples.md rename to docs/core/examples.md diff --git a/core-new/docs/indexing.md b/docs/core/indexing.md similarity index 100% rename from core-new/docs/indexing.md rename to docs/core/indexing.md diff --git a/core-new/docs/job-system.md b/docs/core/job-system.md similarity index 100% rename from core-new/docs/job-system.md rename to docs/core/job-system.md diff --git a/core-new/docs/library.md b/docs/core/library.md similarity index 100% rename from core-new/docs/library.md rename to docs/core/library.md diff --git a/core-new/docs/locations.md b/docs/core/locations.md similarity index 100% rename from core-new/docs/locations.md rename to docs/core/locations.md diff --git a/core-new/docs/networking.md b/docs/core/networking.md similarity index 100% rename from core-new/docs/networking.md rename to docs/core/networking.md diff --git a/core-new/docs/pairing.md b/docs/core/pairing.md similarity index 100% rename from core-new/docs/pairing.md rename to docs/core/pairing.md diff --git a/core-new/docs/task-tracking.md b/docs/core/task-tracking.md similarity index 100% rename from core-new/docs/task-tracking.md rename to docs/core/task-tracking.md diff --git a/core-new/docs/testing.md b/docs/core/testing.md similarity index 100% rename from core-new/docs/testing.md rename to docs/core/testing.md diff --git a/core-new/docs/virtual_sidecars.md b/docs/core/virtual_sidecars.md similarity index 100% rename from core-new/docs/virtual_sidecars.md rename to docs/core/virtual_sidecars.md diff --git a/core-new/docs/volume-system.md b/docs/core/volume-system.md similarity index 100% rename from core-new/docs/volume-system.md rename to docs/core/volume-system.md diff --git a/core-new/docs/design/ACTIONS_REFACTOR.md b/docs/design/ACTIONS_REFACTOR.md similarity index 100% rename from core-new/docs/design/ACTIONS_REFACTOR.md rename to docs/design/ACTIONS_REFACTOR.md diff --git a/core-new/docs/design/ACTION_SYSTEM_DESIGN.md b/docs/design/ACTION_SYSTEM_DESIGN.md similarity index 100% rename from core-new/docs/design/ACTION_SYSTEM_DESIGN.md rename to docs/design/ACTION_SYSTEM_DESIGN.md diff --git a/core-new/docs/design/AGENT_MANAGER_DESIGN.md b/docs/design/AGENT_MANAGER_DESIGN.md similarity index 100% rename from core-new/docs/design/AGENT_MANAGER_DESIGN.md rename to docs/design/AGENT_MANAGER_DESIGN.md diff --git a/core-new/docs/design/API_COMPARISON.md b/docs/design/API_COMPARISON.md similarity index 100% rename from core-new/docs/design/API_COMPARISON.md rename to docs/design/API_COMPARISON.md diff --git a/core-new/docs/design/ARCHITECTURE_DECISIONS.md b/docs/design/ARCHITECTURE_DECISIONS.md similarity index 100% rename from core-new/docs/design/ARCHITECTURE_DECISIONS.md rename to docs/design/ARCHITECTURE_DECISIONS.md diff --git a/core-new/docs/design/AT_REST_LIBRARY_ENCRYPTION.md b/docs/design/AT_REST_LIBRARY_ENCRYPTION.md similarity index 100% rename from core-new/docs/design/AT_REST_LIBRARY_ENCRYPTION.md rename to docs/design/AT_REST_LIBRARY_ENCRYPTION.md diff --git a/core-new/docs/design/BENCHMARKING_SUITE_DESIGN.md b/docs/design/BENCHMARKING_SUITE_DESIGN.md similarity index 100% rename from core-new/docs/design/BENCHMARKING_SUITE_DESIGN.md rename to docs/design/BENCHMARKING_SUITE_DESIGN.md diff --git a/core-new/docs/design/CLOSURE_TABLE_INDEXING_PROPOSAL.md b/docs/design/CLOSURE_TABLE_INDEXING_PROPOSAL.md similarity index 100% rename from core-new/docs/design/CLOSURE_TABLE_INDEXING_PROPOSAL.md rename to docs/design/CLOSURE_TABLE_INDEXING_PROPOSAL.md diff --git a/core-new/docs/design/CROSS_DEVICE_FILE_TRANSFER_IMPLEMENTATION.md b/docs/design/CROSS_DEVICE_FILE_TRANSFER_IMPLEMENTATION.md similarity index 100% rename from core-new/docs/design/CROSS_DEVICE_FILE_TRANSFER_IMPLEMENTATION.md rename to docs/design/CROSS_DEVICE_FILE_TRANSFER_IMPLEMENTATION.md diff --git a/core-new/docs/design/CROSS_PLATFORM_COPY_AND_VOLUME_AWARENESS.md b/docs/design/CROSS_PLATFORM_COPY_AND_VOLUME_AWARENESS.md similarity index 100% rename from core-new/docs/design/CROSS_PLATFORM_COPY_AND_VOLUME_AWARENESS.md rename to docs/design/CROSS_PLATFORM_COPY_AND_VOLUME_AWARENESS.md diff --git a/core-new/docs/design/DAEMON_REFACTOR.md b/docs/design/DAEMON_REFACTOR.md similarity index 100% rename from core-new/docs/design/DAEMON_REFACTOR.md rename to docs/design/DAEMON_REFACTOR.md diff --git a/core-new/docs/design/DESIGN_CORE_LIFECYCLE.md b/docs/design/DESIGN_CORE_LIFECYCLE.md similarity index 100% rename from core-new/docs/design/DESIGN_CORE_LIFECYCLE.md rename to docs/design/DESIGN_CORE_LIFECYCLE.md diff --git a/core-new/docs/design/DESIGN_DEVICE_MANAGEMENT.md b/docs/design/DESIGN_DEVICE_MANAGEMENT.md similarity index 100% rename from core-new/docs/design/DESIGN_DEVICE_MANAGEMENT.md rename to docs/design/DESIGN_DEVICE_MANAGEMENT.md diff --git a/core-new/docs/design/DESIGN_FILE_DATA_MODEL.md b/docs/design/DESIGN_FILE_DATA_MODEL.md similarity index 100% rename from core-new/docs/design/DESIGN_FILE_DATA_MODEL.md rename to docs/design/DESIGN_FILE_DATA_MODEL.md diff --git a/core-new/docs/design/DESIGN_FILE_DATA_MODEL_VISUAL.md b/docs/design/DESIGN_FILE_DATA_MODEL_VISUAL.md similarity index 100% rename from core-new/docs/design/DESIGN_FILE_DATA_MODEL_VISUAL.md rename to docs/design/DESIGN_FILE_DATA_MODEL_VISUAL.md diff --git a/core-new/docs/design/DESIGN_FILE_TYPE_SYSTEM.md b/docs/design/DESIGN_FILE_TYPE_SYSTEM.md similarity index 100% rename from core-new/docs/design/DESIGN_FILE_TYPE_SYSTEM.md rename to docs/design/DESIGN_FILE_TYPE_SYSTEM.md diff --git a/core-new/docs/design/DESIGN_LIBRARY_IMPLEMENTATION.md b/docs/design/DESIGN_LIBRARY_IMPLEMENTATION.md similarity index 100% rename from core-new/docs/design/DESIGN_LIBRARY_IMPLEMENTATION.md rename to docs/design/DESIGN_LIBRARY_IMPLEMENTATION.md diff --git a/core-new/docs/design/DESIGN_LIBRARY_ORGANIZATION.md b/docs/design/DESIGN_LIBRARY_ORGANIZATION.md similarity index 100% rename from core-new/docs/design/DESIGN_LIBRARY_ORGANIZATION.md rename to docs/design/DESIGN_LIBRARY_ORGANIZATION.md diff --git a/core-new/docs/design/DESIGN_LIBRARY_ORGANIZATION_VISUAL.md b/docs/design/DESIGN_LIBRARY_ORGANIZATION_VISUAL.md similarity index 100% rename from core-new/docs/design/DESIGN_LIBRARY_ORGANIZATION_VISUAL.md rename to docs/design/DESIGN_LIBRARY_ORGANIZATION_VISUAL.md diff --git a/core-new/docs/design/DESIGN_OPTIMIZED_STORAGE.md b/docs/design/DESIGN_OPTIMIZED_STORAGE.md similarity index 100% rename from core-new/docs/design/DESIGN_OPTIMIZED_STORAGE.md rename to docs/design/DESIGN_OPTIMIZED_STORAGE.md diff --git a/core-new/docs/design/DESIGN_VDFS_INTEGRATION.md b/docs/design/DESIGN_VDFS_INTEGRATION.md similarity index 100% rename from core-new/docs/design/DESIGN_VDFS_INTEGRATION.md rename to docs/design/DESIGN_VDFS_INTEGRATION.md diff --git a/core-new/docs/design/DEVICE_PAIRING_PROTOCOL.md b/docs/design/DEVICE_PAIRING_PROTOCOL.md similarity index 100% rename from core-new/docs/design/DEVICE_PAIRING_PROTOCOL.md rename to docs/design/DEVICE_PAIRING_PROTOCOL.md diff --git a/core-new/docs/design/DOMAIN_MODELS_README.md b/docs/design/DOMAIN_MODELS_README.md similarity index 100% rename from core-new/docs/design/DOMAIN_MODELS_README.md rename to docs/design/DOMAIN_MODELS_README.md diff --git a/core-new/docs/design/ENTITY_REFACTOR_DESIGN.md b/docs/design/ENTITY_REFACTOR_DESIGN.md similarity index 100% rename from core-new/docs/design/ENTITY_REFACTOR_DESIGN.md rename to docs/design/ENTITY_REFACTOR_DESIGN.md diff --git a/core-new/docs/design/FFMPEG_BUNDLING_DESIGN.md b/docs/design/FFMPEG_BUNDLING_DESIGN.md similarity index 94% rename from core-new/docs/design/FFMPEG_BUNDLING_DESIGN.md rename to docs/design/FFMPEG_BUNDLING_DESIGN.md index 757fe0a4a..863ddd7a5 100644 --- a/core-new/docs/design/FFMPEG_BUNDLING_DESIGN.md +++ b/docs/design/FFMPEG_BUNDLING_DESIGN.md @@ -1,12 +1,13 @@ -# FFmpeg Bundling Design for Core-New +# FFmpeg Bundling Design for core ## Executive Summary -This document outlines the design for bundling FFmpeg with Spacedrive core-new, based on the original implementation's approach. FFmpeg is essential for video thumbnail generation, media metadata extraction, and future transcoding capabilities. The system must handle cross-platform bundling while maintaining a reasonable binary size and ensuring all required codecs are available. +This document outlines the design for bundling FFmpeg with Spacedrive core, based on the original implementation's approach. FFmpeg is essential for video thumbnail generation, media metadata extraction, and future transcoding capabilities. The system must handle cross-platform bundling while maintaining a reasonable binary size and ensuring all required codecs are available. ## Background The original Spacedrive core uses FFmpeg for: + - Video thumbnail generation via `sd-ffmpeg` crate - Media metadata extraction (duration, codec info, bitrate, etc.) - Future planned features: video transcoding, format conversion @@ -28,26 +29,30 @@ The implementation uses `ffmpeg-sys-next` (v7.0) which requires FFmpeg libraries ### Platform-Specific Bundling #### macOS + - Bundle FFmpeg as a framework in `.deps/Spacedrive.framework` - Include in Tauri config: `"frameworks": ["../../.deps/Spacedrive.framework"]` - Use `install_name_tool` to fix library paths for distribution - Symlink shared libraries during build process (as seen in preprep.mjs) #### Windows + - Bundle FFmpeg DLLs alongside the executable - Use static linking where possible to reduce DLL dependencies - Handle both 64-bit builds (32-bit and ARM not supported per setup.ps1) - Requires LLVM 15.0.7 for building ffmpeg-sys-next #### Linux + - Dynamic linking with system FFmpeg where available - Bundle as fallback in AppImage/Flatpak distributions - Debian package dependencies: include FFmpeg libraries -#### iOS *(Implemented)* +#### iOS _(Implemented)_ + - **Architecture Support**: Full support for arm64, x86_64 simulator, and arm64 simulator - **Static Linking**: Uses `CARGO_FEATURE_STATIC=1` for all builds -- **Build Process**: +- **Build Process**: - Separate FFmpeg builds for each architecture stored in `.deps/`: - `aarch64-apple-ios` (device) - `aarch64-apple-ios-sim` (M1 simulator) @@ -56,13 +61,14 @@ The implementation uses `ffmpeg-sys-next` (v7.0) which requires FFmpeg libraries - Sets `FFMPEG_DIR` dynamically based on target architecture - Creates universal binaries using `lipo` - Symlinks FFmpeg libraries to target directory -- **Pod Configuration**: +- **Pod Configuration**: - Extensive codec support including: mp3lame, opus, vorbis, x264, x265, vpx, av1 - Links against iOS frameworks: AudioToolbox, VideoToolbox, AVFoundation - Libraries linked: libsd_mobile_ios (device) or libsd_mobile_iossim (simulator) - **Feature Flags**: FFmpeg explicitly enabled in `sd-mobile-core` for iOS targets -#### Android *(Not Currently Implemented)* +#### Android _(Not Currently Implemented)_ + - **Current Status**: FFmpeg is NOT enabled for Android builds - **Build System**: Uses `cargo ndk` with platform API 34 - **Target Architectures**: Primarily arm64-v8a (with optional armeabi-v7a and x86_64) @@ -75,38 +81,41 @@ The implementation uses `ffmpeg-sys-next` (v7.0) which requires FFmpeg libraries ### Build Process Integration 1. **Dependency Download Phase** + ```bash # Add to scripts/preprep.mjs or similar async function downloadFFmpeg() { const platform = process.platform; const arch = process.arch; - + // Download pre-built FFmpeg binaries const ffmpegVersion = "6.1"; // or latest stable const downloadUrl = getFFmpegUrl(platform, arch, ffmpegVersion); - + // Extract to .deps directory await downloadAndExtract(downloadUrl, ".deps/ffmpeg"); } ``` 2. **Cargo Build Configuration** + ```toml # In Cargo.toml or .cargo/config.toml [env] FFMPEG_DIR = { value = ".deps/ffmpeg", relative = true } - + [target.'cfg(target_os = "macos")'] rustflags = ["-C", "link-arg=-Wl,-rpath,@loader_path/../Frameworks"] ``` 3. **Feature Flag Management** + ```toml - # In core-new/Cargo.toml + # In core/Cargo.toml [features] default = ["ffmpeg"] ffmpeg = ["dep:sd-ffmpeg", "sd-media-processor/ffmpeg"] - + # Allow building without FFmpeg for testing no-ffmpeg = [] ``` @@ -114,6 +123,7 @@ The implementation uses `ffmpeg-sys-next` (v7.0) which requires FFmpeg libraries ### FFmpeg Configuration #### Desktop Minimal Configuration + Minimal FFmpeg build configuration to reduce size: ```bash @@ -141,6 +151,7 @@ Minimal FFmpeg build configuration to reduce size: ``` #### iOS Extended Configuration + iOS build includes extensive codec support for maximum compatibility: - **Audio Codecs**: MP3 (lame), Opus, Vorbis, AAC @@ -158,7 +169,7 @@ iOS build includes extensive codec support for maximum compatibility: mod bundled { use std::env; use std::path::PathBuf; - + pub fn setup_ffmpeg_paths() { #[cfg(target_os = "macos")] { @@ -169,10 +180,10 @@ mod bundled { .parent() .unwrap() .join("Frameworks/Spacedrive.framework/Libraries"); - + env::set_var("DYLD_LIBRARY_PATH", framework_path); } - + #[cfg(target_os = "windows")] { // FFmpeg DLLs should be in same directory as exe @@ -180,16 +191,16 @@ mod bundled { .unwrap() .parent() .unwrap(); - + env::set_var("PATH", format!("{};{}", exe_dir.display(), env::var("PATH").unwrap_or_default())); } - + #[cfg(target_os = "ios")] { // iOS uses static linking, no runtime path setup needed // Libraries are linked at compile time via build.rs } - + #[cfg(target_os = "android")] { // Android will load libraries via System.loadLibrary() in JNI @@ -201,12 +212,12 @@ mod bundled { pub fn initialize() -> Result<(), Error> { #[cfg(feature = "bundled")] bundled::setup_ffmpeg_paths(); - + // Initialize FFmpeg unsafe { ffmpeg_sys_next::av_log_set_level(ffmpeg_sys_next::AV_LOG_ERROR); } - + Ok(()) } ``` @@ -221,11 +232,12 @@ pub fn initialize() -> Result<(), Error> { ### Testing Strategy 1. **Binary Validation** + ```rust #[test] fn test_ffmpeg_available() { assert!(sd_ffmpeg::initialize().is_ok()); - + // Test basic probe functionality let test_file = include_bytes!("../test_data/sample.mp4"); let metadata = sd_ffmpeg::probe_bytes(test_file).unwrap(); @@ -248,6 +260,7 @@ pub fn initialize() -> Result<(), Error> { ## Implementation Checklist ### Desktop Platforms + - [ ] Create `.deps` directory structure - [ ] Add FFmpeg download script to build process - [ ] Update Cargo build configuration @@ -260,6 +273,7 @@ pub fn initialize() -> Result<(), Error> { - [ ] Implement size monitoring in CI ### iOS (Completed in Original Core) + - [x] FFmpeg libraries for all iOS architectures - [x] Build script (`build-rust.sh`) with architecture detection - [x] Pod configuration with codec libraries @@ -267,6 +281,7 @@ pub fn initialize() -> Result<(), Error> { - [x] Framework linking (AudioToolbox, VideoToolbox, etc.) ### Android (To Be Implemented) + - [ ] Add FFmpeg feature flag to Android build - [ ] Download/build FFmpeg for Android ABIs - [ ] Update `build.sh` for FFmpeg paths @@ -286,4 +301,4 @@ pub fn initialize() -> Result<(), Error> { - Original implementation: `spacedrive/crates/ffmpeg/` - ffmpeg-sys-next: https://github.com/zmwangx/rust-ffmpeg-sys - FFmpeg licensing: https://ffmpeg.org/legal.html -- Tauri bundling: https://tauri.app/v1/guides/building/resources \ No newline at end of file +- Tauri bundling: https://tauri.app/v1/guides/building/resources diff --git a/core-new/docs/design/FILE_SHARING_DESIGN.md b/docs/design/FILE_SHARING_DESIGN.md similarity index 100% rename from core-new/docs/design/FILE_SHARING_DESIGN.md rename to docs/design/FILE_SHARING_DESIGN.md diff --git a/core-new/docs/design/FUNCTION_BASED_TEST_FRAMEWORK_DESIGN.md b/docs/design/FUNCTION_BASED_TEST_FRAMEWORK_DESIGN.md similarity index 100% rename from core-new/docs/design/FUNCTION_BASED_TEST_FRAMEWORK_DESIGN.md rename to docs/design/FUNCTION_BASED_TEST_FRAMEWORK_DESIGN.md diff --git a/core-new/docs/design/IDEA_FOR_BETTER_INDEXING.md b/docs/design/IDEA_FOR_BETTER_INDEXING.md similarity index 100% rename from core-new/docs/design/IDEA_FOR_BETTER_INDEXING.md rename to docs/design/IDEA_FOR_BETTER_INDEXING.md diff --git a/core-new/docs/design/IMPLEMENTATION_STATUS.md b/docs/design/IMPLEMENTATION_STATUS.md similarity index 100% rename from core-new/docs/design/IMPLEMENTATION_STATUS.md rename to docs/design/IMPLEMENTATION_STATUS.md diff --git a/core-new/docs/design/INDEXER_ANALYSIS.md b/docs/design/INDEXER_ANALYSIS.md similarity index 100% rename from core-new/docs/design/INDEXER_ANALYSIS.md rename to docs/design/INDEXER_ANALYSIS.md diff --git a/core-new/docs/design/INDEXER_ANALYSIS_2.md b/docs/design/INDEXER_ANALYSIS_2.md similarity index 100% rename from core-new/docs/design/INDEXER_ANALYSIS_2.md rename to docs/design/INDEXER_ANALYSIS_2.md diff --git a/core-new/docs/design/INDEXER_JOB_EXAMPLE.md b/docs/design/INDEXER_JOB_EXAMPLE.md similarity index 100% rename from core-new/docs/design/INDEXER_JOB_EXAMPLE.md rename to docs/design/INDEXER_JOB_EXAMPLE.md diff --git a/core-new/docs/design/INDEXER_PROGRESS.md b/docs/design/INDEXER_PROGRESS.md similarity index 97% rename from core-new/docs/design/INDEXER_PROGRESS.md rename to docs/design/INDEXER_PROGRESS.md index a271aa64d..ad406ba09 100644 --- a/core-new/docs/design/INDEXER_PROGRESS.md +++ b/docs/design/INDEXER_PROGRESS.md @@ -146,13 +146,13 @@ The new indexer uses a clean phase-based pipeline: - Add text extraction - Implement media metadata extraction -4. **Memory Management** - Production readiness +3. **Memory Management** - Production readiness - Implement actual memory tracking - Add adaptive batch sizing - Enforce memory limits -5. **Testing & Documentation** +4. **Testing & Documentation** - Add comprehensive test coverage - Document public APIs - Create integration examples @@ -162,7 +162,7 @@ The new indexer uses a clean phase-based pipeline: - The new indexer prioritizes correctness and maintainability over complex optimizations - CRDT sync support is intentionally deferred to a later phase - The phase-based architecture makes it easier to add new processing steps -- Real-time file system monitoring is handled by the separate `location_watcher` service (see `/core-new/src/services/location_watcher/` and `/core-new/docs/design/WATCHER_VDFS_INTEGRATION.md`) +- Real-time file system monitoring is handled by the separate `location_watcher` service (see `/core/src/services/location_watcher/` and `/core/docs/design/WATCHER_VDFS_INTEGRATION.md`) - Directory sizes are calculated in a dedicated aggregation phase, making them more accurate and efficient than the old materialized path approach - Parent-child relationships use explicit parent_id references instead of materialized paths, enabling more flexible hierarchical queries - Current implementation provides a solid foundation for future enhancements diff --git a/core-new/docs/design/INDEXER_RULES_SYSTEM.md b/docs/design/INDEXER_RULES_SYSTEM.md similarity index 90% rename from core-new/docs/design/INDEXER_RULES_SYSTEM.md rename to docs/design/INDEXER_RULES_SYSTEM.md index 40c1c4d91..bd5b23a28 100644 --- a/core-new/docs/design/INDEXER_RULES_SYSTEM.md +++ b/docs/design/INDEXER_RULES_SYSTEM.md @@ -18,7 +18,7 @@ This document outlines the design for implementing an indexer rules system in Sp ### Domain Model ```rust -// core-new/src/domain/indexer_rule.rs +// core/src/domain/indexer_rule.rs pub struct IndexerRule { pub id: Uuid, pub name: String, @@ -37,17 +37,17 @@ pub enum IndexerRuleType { RejectGlob { patterns: Vec }, AcceptRegex { patterns: Vec }, RejectRegex { patterns: Vec }, - + // Directory rules AcceptIfChildExists { children: Vec }, RejectIfChildExists { children: Vec }, - + // File attribute rules RejectLargerThan { size_bytes: u64 }, RejectOlderThan { days: u32 }, AcceptExtensions { extensions: Vec }, RejectExtensions { extensions: Vec }, - + // Integration rules RespectGitignore, RejectSystemFiles, @@ -100,7 +100,7 @@ CREATE TABLE indexer_rule_applications ( ### Rule Evaluation Engine ```rust -// core-new/src/services/indexer_rules/engine.rs +// core/src/services/indexer_rules/engine.rs pub struct IndexerRuleEngine { compiled_rules: Vec, gitignore_cache: Option, @@ -125,7 +125,7 @@ impl IndexerRuleEngine { // Sort by priority // Initialize gitignore if needed } - + pub fn should_index(&self, path: &Path, metadata: &EntryMetadata) -> RuleDecision { // Evaluate rules in priority order // Short-circuit on first definitive decision @@ -145,14 +145,14 @@ pub struct RuleDecision { #### 1. Indexer Job Integration ```rust -// Modify core-new/src/operations/indexing/indexer_job.rs +// Modify core/src/operations/indexing/indexer_job.rs impl IndexerJob { async fn setup_rule_engine(&self, location: &Location) -> Result { // Load rules for location from database // Merge with system rules if enabled // Compile and cache rule engine } - + async fn read_directory(&self, path: &Path, rule_engine: &IndexerRuleEngine) -> Result> { // Apply rules during directory traversal // Skip rejected paths early @@ -164,7 +164,7 @@ impl IndexerJob { #### 2. Location Manager Integration ```rust -// Extend core-new/src/location/manager.rs +// Extend core/src/location/manager.rs impl LocationManager { pub async fn create_location_with_rules( &self, @@ -175,7 +175,7 @@ impl LocationManager { // Attach rules // Validate rule compatibility } - + pub async fn update_location_rules( &self, location_id: Uuid, @@ -190,7 +190,7 @@ impl LocationManager { #### 3. File Watcher Integration ```rust -// Extend core-new/src/services/location_watcher/event_handler.rs +// Extend core/src/services/location_watcher/event_handler.rs impl EventHandler { async fn should_process_event(&self, path: &Path) -> bool { // Get cached rule engine for location @@ -220,7 +220,7 @@ pub fn create_system_rules() -> Vec { is_system: true, ..Default::default() }, - + // Hidden files IndexerRule { name: "Ignore Hidden Files".to_string(), @@ -229,7 +229,7 @@ pub fn create_system_rules() -> Vec { is_system: true, ..Default::default() }, - + // Development artifacts IndexerRule { name: "Ignore Development Folders".to_string(), @@ -262,60 +262,65 @@ pub fn create_system_rules() -> Vec { ```graphql type IndexerRule { - id: ID! - name: String! - description: String - isSystem: Boolean! - isEnabled: Boolean! - priority: Int! - ruleType: IndexerRuleType! - createdAt: DateTime! - updatedAt: DateTime! + id: ID! + name: String! + description: String + isSystem: Boolean! + isEnabled: Boolean! + priority: Int! + ruleType: IndexerRuleType! + createdAt: DateTime! + updatedAt: DateTime! } type IndexerRuleType { - type: String! - config: JSON! + type: String! + config: JSON! } type Query { - indexerRules(locationId: ID): [IndexerRule!]! - systemRules: [IndexerRule!]! + indexerRules(locationId: ID): [IndexerRule!]! + systemRules: [IndexerRule!]! } type Mutation { - createIndexerRule(input: CreateIndexerRuleInput!): IndexerRule! - updateIndexerRule(id: ID!, input: UpdateIndexerRuleInput!): IndexerRule! - deleteIndexerRule(id: ID!): Boolean! - - attachRuleToLocation(locationId: ID!, ruleId: ID!): Location! - detachRuleFromLocation(locationId: ID!, ruleId: ID!): Location! + createIndexerRule(input: CreateIndexerRuleInput!): IndexerRule! + updateIndexerRule(id: ID!, input: UpdateIndexerRuleInput!): IndexerRule! + deleteIndexerRule(id: ID!): Boolean! + + attachRuleToLocation(locationId: ID!, ruleId: ID!): Location! + detachRuleFromLocation(locationId: ID!, ruleId: ID!): Location! } ``` ## Implementation Plan ### Phase 1: Core Infrastructure + 1. Create domain models and database schema 2. Implement rule compilation and matching logic 3. Create system default rules ### Phase 2: Indexer Integration + 1. Integrate rule engine into indexer job 2. Add rule evaluation during directory traversal 3. Update database entities to track excluded paths ### Phase 3: Location Integration + 1. Add rule management to location manager 2. Update location creation to support rules 3. Implement rule inheritance logic ### Phase 4: API and UI + 1. Add GraphQL types and resolvers 2. Create rule management UI 3. Add rule testing/preview functionality ### Phase 5: Advanced Features + 1. Git integration (.gitignore support) 2. Rule templates and presets 3. Rule import/export @@ -340,4 +345,4 @@ type Mutation { 2. **Cloud Rule Sharing**: Share rule sets between users 3. **Rule Analytics**: Track which rules are most effective 4. **Dynamic Rules**: Rules that adapt based on system resources -5. **Content-Based Rules**: Rules based on file content, not just metadata \ No newline at end of file +5. **Content-Based Rules**: Rules based on file content, not just metadata diff --git a/core-new/docs/design/INDEXING_DISCOVERY_OPTIMIZATION_PLAN.md b/docs/design/INDEXING_DISCOVERY_OPTIMIZATION_PLAN.md similarity index 100% rename from core-new/docs/design/INDEXING_DISCOVERY_OPTIMIZATION_PLAN.md rename to docs/design/INDEXING_DISCOVERY_OPTIMIZATION_PLAN.md diff --git a/core-new/docs/design/INTEGRATION_SYSTEM_DESIGN.md b/docs/design/INTEGRATION_SYSTEM_DESIGN.md similarity index 100% rename from core-new/docs/design/INTEGRATION_SYSTEM_DESIGN.md rename to docs/design/INTEGRATION_SYSTEM_DESIGN.md diff --git a/core-new/docs/design/INTEGRATION_SYSTEM_DESIGN_GEMINI.md b/docs/design/INTEGRATION_SYSTEM_DESIGN_GEMINI.md similarity index 100% rename from core-new/docs/design/INTEGRATION_SYSTEM_DESIGN_GEMINI.md rename to docs/design/INTEGRATION_SYSTEM_DESIGN_GEMINI.md diff --git a/core-new/docs/design/IPHONE_AS_VOLUME_DESIGN.md b/docs/design/IPHONE_AS_VOLUME_DESIGN.md similarity index 100% rename from core-new/docs/design/IPHONE_AS_VOLUME_DESIGN.md rename to docs/design/IPHONE_AS_VOLUME_DESIGN.md diff --git a/core-new/docs/design/IROH_MIGRATION_DESIGN.md b/docs/design/IROH_MIGRATION_DESIGN.md similarity index 100% rename from core-new/docs/design/IROH_MIGRATION_DESIGN.md rename to docs/design/IROH_MIGRATION_DESIGN.md diff --git a/core-new/docs/design/JOB_SYSTEM_DESIGN.md b/docs/design/JOB_SYSTEM_DESIGN.md similarity index 100% rename from core-new/docs/design/JOB_SYSTEM_DESIGN.md rename to docs/design/JOB_SYSTEM_DESIGN.md diff --git a/core-new/docs/design/JOB_SYSTEM_MACRO_EXAMPLE.md b/docs/design/JOB_SYSTEM_MACRO_EXAMPLE.md similarity index 100% rename from core-new/docs/design/JOB_SYSTEM_MACRO_EXAMPLE.md rename to docs/design/JOB_SYSTEM_MACRO_EXAMPLE.md diff --git a/core-new/docs/design/JOB_SYSTEM_README.md b/docs/design/JOB_SYSTEM_README.md similarity index 100% rename from core-new/docs/design/JOB_SYSTEM_README.md rename to docs/design/JOB_SYSTEM_README.md diff --git a/core-new/docs/design/LIBP2P_INTEGRATION_DESIGN.md b/docs/design/LIBP2P_INTEGRATION_DESIGN.md similarity index 100% rename from core-new/docs/design/LIBP2P_INTEGRATION_DESIGN.md rename to docs/design/LIBP2P_INTEGRATION_DESIGN.md diff --git a/core-new/docs/design/LIBRARY_LEADERSHIP.md b/docs/design/LIBRARY_LEADERSHIP.md similarity index 100% rename from core-new/docs/design/LIBRARY_LEADERSHIP.md rename to docs/design/LIBRARY_LEADERSHIP.md diff --git a/core-new/docs/design/NETWORKING_SYSTEM_DESIGN.md b/docs/design/NETWORKING_SYSTEM_DESIGN.md similarity index 87% rename from core-new/docs/design/NETWORKING_SYSTEM_DESIGN.md rename to docs/design/NETWORKING_SYSTEM_DESIGN.md index d9aecb474..459b26915 100644 --- a/core-new/docs/design/NETWORKING_SYSTEM_DESIGN.md +++ b/docs/design/NETWORKING_SYSTEM_DESIGN.md @@ -62,13 +62,14 @@ This document outlines a flexible networking system for Spacedrive that supports **CRITICAL: Integration with Existing Device Identity** -The networking module MUST integrate with Spacedrive's existing persistent device identity system (see `core-new/src/device/`). The current device system provides: +The networking module MUST integrate with Spacedrive's existing persistent device identity system (see `core/src/device/`). The current device system provides: - **Persistent Device UUID**: Stored in `device.json`, survives restarts - **Device Configuration**: Name, OS, hardware model, creation time - **Cross-Instance Consistency**: Multiple Spacedrive instances on same device share identity **Problem with Original Design:** + - NetworkingDeviceId derived from public key changes each restart - No persistence of cryptographic keys - Multiple instances would have different network identities @@ -81,16 +82,16 @@ The networking module MUST integrate with Spacedrive's existing persistent devic pub struct NetworkIdentity { /// MUST match the persistent device UUID from DeviceManager pub device_id: Uuid, // From existing device system - + /// Device's public key (Ed25519) - STORED PERSISTENTLY pub public_key: PublicKey, - - /// Device's private key (encrypted at rest) - STORED PERSISTENTLY + + /// Device's private key (encrypted at rest) - STORED PERSISTENTLY private_key: EncryptedPrivateKey, - + /// Human-readable device name (from DeviceConfig) pub device_name: String, - + /// Network-specific identifier (derived from device_id + public_key) pub network_fingerprint: NetworkFingerprint, } @@ -118,10 +119,10 @@ pub struct ExtendedDeviceConfig { /// Base device configuration #[serde(flatten)] pub device: DeviceConfig, - + /// Network cryptographic keys (encrypted) pub network_keys: Option, - + /// When network identity was created pub network_identity_created_at: Option>, } @@ -130,13 +131,13 @@ pub struct ExtendedDeviceConfig { pub struct EncryptedNetworkKeys { /// Ed25519 private key encrypted with user password pub encrypted_private_key: EncryptedPrivateKey, - + /// Public key (not encrypted) pub public_key: PublicKey, - + /// Salt for key derivation pub salt: [u8; 32], - + /// Key derivation parameters pub kdf_params: KeyDerivationParams, } @@ -149,7 +150,7 @@ impl NetworkIdentity { password: &str, ) -> Result { let device_config = device_manager.config()?; - + // Try to load existing network keys if let Some(keys) = Self::load_network_keys(&device_config.id, password)? { return Ok(Self { @@ -158,22 +159,22 @@ impl NetworkIdentity { private_key: keys.encrypted_private_key, device_name: device_config.name, network_fingerprint: NetworkFingerprint::from_device( - device_config.id, + device_config.id, &keys.public_key ), }); } - + // Generate new network keys if none exist let (public_key, private_key) = Self::generate_keys(password)?; let network_fingerprint = NetworkFingerprint::from_device( - device_config.id, + device_config.id, &public_key ); - + // Save keys persistently Self::save_network_keys(&device_config.id, &public_key, &private_key, password)?; - + Ok(Self { device_id: device_config.id, public_key, @@ -182,18 +183,18 @@ impl NetworkIdentity { network_fingerprint, }) } - + /// Load network keys from device-specific storage fn load_network_keys( - device_id: &Uuid, + device_id: &Uuid, password: &str ) -> Result, NetworkError> { // Keys stored in device-specific file: /network_keys.json // This ensures multiple Spacedrive instances share the same keys todo!("Load from persistent storage") } - - /// Save network keys to device-specific storage + + /// Save network keys to device-specific storage fn save_network_keys( device_id: &Uuid, public_key: &PublicKey, @@ -208,7 +209,7 @@ impl NetworkIdentity { pub struct MasterKey { /// User's master password derives this key_encryption_key: [u8; 32], - + /// Encrypted with key_encryption_key - NOW USES PERSISTENT DEVICE IDs device_private_keys: HashMap, // UUID not derived ID } @@ -217,10 +218,10 @@ pub struct MasterKey { pub struct PairingCode { /// Temporary shared secret secret: [u8; 32], - + /// Expires after 5 minutes expires_at: DateTime, - + /// Visual representation (6 words from BIP39 wordlist) words: [String; 6], } @@ -234,10 +235,10 @@ impl Core { pub async fn init_networking(&mut self, password: &str) -> Result<()> { // Use existing device manager - NO separate identity creation let network_identity = NetworkIdentity::from_device_manager( - &self.device, + &self.device, password ).await?; - + let network = Network::new(network_identity, config).await?; self.network = Some(Arc::new(network)); Ok(()) @@ -261,13 +262,13 @@ impl Core { pub struct NetworkMessage { /// Persistent device UUID (for logs, correlation) pub device_id: Uuid, - + /// Network fingerprint (for wire protocol security) pub network_fingerprint: NetworkFingerprint, - + /// Message payload pub payload: MessagePayload, - + /// Cryptographic signature pub signature: Signature, } @@ -282,19 +283,19 @@ Abstract connection interface: pub trait NetworkConnection: Send + Sync { /// Send data reliably async fn send(&mut self, data: &[u8]) -> Result<()>; - + /// Receive data async fn receive(&mut self) -> Result>; - + /// Stream a file efficiently async fn send_file(&mut self, path: &Path) -> Result<()>; - + /// Receive file stream async fn receive_file(&mut self, path: &Path) -> Result<()>; - + /// Get remote device info fn remote_device(&self) -> &DeviceInfo; - + /// Check if connection is alive fn is_connected(&self) -> bool; } @@ -303,10 +304,10 @@ pub trait NetworkConnection: Send + Sync { pub struct ConnectionManager { /// Our device identity identity: Arc, - + /// Active connections connections: Arc>>>, - + /// Available transports transports: Vec>, } @@ -323,7 +324,7 @@ Using existing libraries: pub struct LocalTransport { /// mDNS for discovery (using mdns crate) mdns: ServiceDiscovery, - + /// QUIC for connections (using quinn) quinn_endpoint: quinn::Endpoint, } @@ -335,17 +336,17 @@ impl LocalTransport { "_spacedrive._tcp.local", identity.device_id.to_string(), )?; - + // Setup QUIC endpoint let config = quinn::ServerConfig::with_crypto( Arc::new(noise_crypto_config(identity)) ); - + let endpoint = quinn::Endpoint::server( config, "0.0.0.0:0".parse()? // Random port )?; - + Ok(Self { mdns, quinn_endpoint: endpoint }) } } @@ -360,7 +361,7 @@ For NAT traversal and internet connectivity: pub struct RelayTransport { /// WebSocket or QUIC connection to relay relay_client: RelayClient, - + /// Our registration with relay registration: RelayRegistration, } @@ -368,18 +369,18 @@ pub struct RelayTransport { /// Relay protocol messages pub enum RelayMessage { /// Register device with relay - Register { + Register { device_id: DeviceId, public_key: PublicKey, auth_token: String, // From Spacedrive account }, - + /// Request connection to another device - Connect { + Connect { target_device_id: DeviceId, offer: SessionOffer, // Crypto handshake }, - + /// Relay data between devices Data { session_id: SessionId, @@ -397,7 +398,7 @@ Using Noise Protocol or similar: pub struct NoiseSession { /// Handshake state handshake: snow::HandshakeState, - + /// Transport state (after handshake) transport: Option, } @@ -410,14 +411,14 @@ impl NoiseSession { ) -> Result { let params = "Noise_XX_25519_ChaChaPoly_BLAKE2s"; let builder = snow::Builder::new(params.parse()?); - + let handshake = builder .local_private_key(&local_key.to_bytes()) .build_initiator()?; - + Ok(Self { handshake, transport: None }) } - + /// Complete handshake and establish encrypted transport pub fn complete_handshake(&mut self) -> Result<()> { if self.handshake.is_handshake_finished() { @@ -433,11 +434,13 @@ impl NoiseSession { ### Core Networking 1. **quinn** - QUIC implementation in Rust + - Pros: Built-in encryption, multiplexing, modern protocol - Cons: Requires UDP, might have firewall issues - Use for: Local P2P, future direct internet 2. **tokio-tungstenite** - WebSocket for relay + - Pros: Works everywhere, HTTP-based - Cons: TCP head-of-line blocking - Use for: Relay connections, fallback @@ -450,6 +453,7 @@ impl NoiseSession { ### Discovery 1. **mdns** - mDNS/DNS-SD implementation + - Pros: Simple, works on all platforms - Use for: Local device discovery @@ -460,6 +464,7 @@ impl NoiseSession { ### Security 1. **snow** - Noise Protocol Framework + - Pros: Modern, simple, well-tested - Use for: Transport encryption @@ -470,6 +475,7 @@ impl NoiseSession { ### Utilities 1. **async-stream** - File streaming + - Use for: Efficient file transfer 2. **backoff** - Retry logic @@ -487,19 +493,19 @@ async fn connect_local(target: DeviceId) -> Result { .iter() .find(|s| s.device_id == target) .ok_or("Device not found")?; - + // 2. Connect via QUIC let connection = quinn_endpoint .connect(target_service.addr, &target_service.name)? .await?; - + // 3. Noise handshake let noise = NoiseSession::initiate(&identity.private_key, None)?; perform_handshake(&mut connection, noise).await?; - + // 4. Verify device identity verify_remote_device(&connection, target)?; - + Ok(Connection::Local(connection)) } ``` @@ -510,17 +516,17 @@ async fn connect_local(target: DeviceId) -> Result { async fn connect_relay(target: DeviceId) -> Result { // 1. Connect to relay server let relay = RelayClient::connect("relay.spacedrive.com").await?; - + // 2. Authenticate with relay relay.authenticate(&identity, auth_token).await?; - + // 3. Request connection to target let session = relay.connect_to(target).await?; - + // 4. Noise handshake through relay let noise = NoiseSession::initiate(&identity.private_key, None)?; perform_relayed_handshake(&relay, session, noise).await?; - + Ok(Connection::Relay(relay, session)) } ``` @@ -534,13 +540,13 @@ Efficient file streaming over any transport: pub struct FileHeader { /// File name pub name: String, - + /// Total size in bytes pub size: u64, - + /// Blake3 hash for verification pub hash: [u8; 32], - + /// Optional: Resume from offset pub resume_offset: Option, } @@ -552,7 +558,7 @@ async fn stream_file( ) -> Result<()> { let file = tokio::fs::File::open(path).await?; let metadata = file.metadata().await?; - + // Send header let header = FileHeader { name: path.file_name().unwrap().to_string(), @@ -560,20 +566,20 @@ async fn stream_file( hash: calculate_hash(path).await?, resume_offset: None, }; - + conn.send(&serialize(&header)?).await?; - + // Stream chunks let mut reader = BufReader::new(file); let mut buffer = vec![0u8; 1024 * 1024]; // 1MB chunks - + loop { let n = reader.read(&mut buffer).await?; if n == 0 { break; } - + conn.send(&buffer[..n]).await?; } - + Ok(()) } ``` @@ -593,11 +599,11 @@ impl NetworkConnection { // Send request let request = PullRequest { from_seq, limit }; self.send(&serialize(&request)?).await?; - + // Receive response let response_data = self.receive().await?; let response: PullResponse = deserialize(&response_data)?; - + Ok(response) } } @@ -620,12 +626,12 @@ impl Network { if let Ok(conn) = self.manager.connect_local(device_id).await { return Ok(DeviceConnection::new(conn)); } - + // Fall back to relay self.manager.connect_relay(device_id).await .map(DeviceConnection::new) } - + /// Share file with device pub async fn share_file( &self, @@ -635,7 +641,7 @@ impl Network { let mut conn = self.connect(device_id).await?; conn.send_file(file_path).await } - + /// Sync with device pub async fn sync_with( &self, @@ -660,37 +666,37 @@ impl Network { async fn initiate_pairing() -> Result { let secret = generate_random_bytes(32); let code = PairingCode::from_secret(&secret); - + // Display code.words to user println!("Pairing code: {}", code.words.join(" ")); - + // Listen for pairing requests pairing_listener.register(code.clone()).await; - + Ok(code) } /// On device B (new device) async fn complete_pairing(words: Vec) -> Result<()> { let code = PairingCode::from_words(&words)?; - + // Connect to device A let conn = discover_and_connect_pairing_device().await?; - + // Exchange keys using pairing secret let shared_key = derive_key_from_secret(&code.secret); - + // Send our public key encrypted let encrypted_key = encrypt(&identity.public_key, &shared_key); conn.send(&encrypted_key).await?; - + // Receive encrypted master key let encrypted_master = conn.receive().await?; let master_key = decrypt(&encrypted_master, &shared_key)?; - + // Save master key locally save_master_key(master_key).await?; - + Ok(()) } ``` @@ -722,7 +728,7 @@ impl ConnectionManager { return Ok(conn.clone()); } } - + // Create new connection let conn = self.connect_new(device_id).await?; self.connections.write().await.insert(device_id, conn.clone()); @@ -740,12 +746,12 @@ async fn select_transport(target: DeviceId) -> Transport { if is_same_network(target).await { return Transport::Local; } - + // Has public IP? Try direct if has_public_ip(target).await { return Transport::Direct; } - + // Otherwise use relay Transport::Relay } @@ -754,16 +760,19 @@ async fn select_transport(target: DeviceId) -> Transport { ## Future Enhancements ### WebRTC DataChannels + - For browser support - Better NAT traversal - Built-in STUN/TURN ### Bluetooth Support + - For mobile devices - Low power scenarios - Offline sync ### Tor Integration + - Anonymous connections - Privacy-focused users - Hidden service support @@ -780,48 +789,49 @@ async fn select_transport(target: DeviceId) -> Transport { ### Full Stack Solutions -| Library | Pros | Cons | Best For | -|---------|------|------|----------| -| **libp2p** | • Complete P2P stack
• Multiple transports
• DHT, gossip, etc
• Battle-tested | • Large & complex
• Opinionated design
• Learning curve
• Heavy dependencies | Full decentralized P2P | -| **iroh** | • Built for sync
• QUIC-based
• Content addressing
• Modern Rust | • Young project
• Limited docs
• Specific use case | Content-addressed sync | -| **Magic Wormhole** | • Simple pairing
• E2E encrypted
• No account needed | • One-time transfers
• Not persistent
• Limited protocol | Simple file sharing | +| Library | Pros | Cons | Best For | +| ------------------ | -------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------- | ---------------------- | +| **libp2p** | • Complete P2P stack
• Multiple transports
• DHT, gossip, etc
• Battle-tested | • Large & complex
• Opinionated design
• Learning curve
• Heavy dependencies | Full decentralized P2P | +| **iroh** | • Built for sync
• QUIC-based
• Content addressing
• Modern Rust | • Young project
• Limited docs
• Specific use case | Content-addressed sync | +| **Magic Wormhole** | • Simple pairing
• E2E encrypted
• No account needed | • One-time transfers
• Not persistent
• Limited protocol | Simple file sharing | ### Transport Libraries -| Library | Pros | Cons | Best For | -|---------|------|------|----------| -| **quinn** | • Pure Rust QUIC
• Fast & modern
• Multiplexing
• Built-in crypto | • UDP only
• Firewall issues
• Newer protocol | Local network, future | -| **tokio-tungstenite** | • WebSocket
• Works everywhere
• Simple API
• HTTP-based | • TCP limitations
• No multiplexing
• Text/binary only | Relay connections | -| **tarpc** | • RPC framework
• Multiple transports
• Type-safe | • RPC-focused
• Not streaming
• Overhead | Control protocol | +| Library | Pros | Cons | Best For | +| --------------------- | -------------------------------------------------------------------------- | ------------------------------------------------------------ | --------------------- | +| **quinn** | • Pure Rust QUIC
• Fast & modern
• Multiplexing
• Built-in crypto | • UDP only
• Firewall issues
• Newer protocol | Local network, future | +| **tokio-tungstenite** | • WebSocket
• Works everywhere
• Simple API
• HTTP-based | • TCP limitations
• No multiplexing
• Text/binary only | Relay connections | +| **tarpc** | • RPC framework
• Multiple transports
• Type-safe | • RPC-focused
• Not streaming
• Overhead | Control protocol | ### Discovery Libraries -| Library | Pros | Cons | Best For | -|---------|------|------|----------| -| **mdns** | • Simple mDNS
• Cross-platform
• Lightweight | • Local only
• Basic features | Local discovery | -| **libp2p-mdns** | • Part of libp2p
• More features | • Requires libp2p
• Heavier | If using libp2p | -| **bonjour** | • Full Bonjour
• Apple native | • Platform specific
• Complex | macOS/iOS native | +| Library | Pros | Cons | Best For | +| --------------- | -------------------------------------------------- | -------------------------------- | ---------------- | +| **mdns** | • Simple mDNS
• Cross-platform
• Lightweight | • Local only
• Basic features | Local discovery | +| **libp2p-mdns** | • Part of libp2p
• More features | • Requires libp2p
• Heavier | If using libp2p | +| **bonjour** | • Full Bonjour
• Apple native | • Platform specific
• Complex | macOS/iOS native | ### Security Libraries -| Library | Pros | Cons | Best For | -|---------|------|------|----------| -| **snow** | • Noise Protocol
• Simple API
• Well-tested
• Modern crypto | • Just crypto
• No networking | Our choice ✓ | -| **rustls** | • TLS in Rust
• Fast
• Audited | • Certificate based
• Complex setup | HTTPS/TLS needs | -| **sodiumoxide** | • libsodium wrapper
• Many primitives | • C dependency
• Lower level | Crypto primitives | +| Library | Pros | Cons | Best For | +| --------------- | -------------------------------------------------------------------- | -------------------------------------- | ----------------- | +| **snow** | • Noise Protocol
• Simple API
• Well-tested
• Modern crypto | • Just crypto
• No networking | Our choice ✓ | +| **rustls** | • TLS in Rust
• Fast
• Audited | • Certificate based
• Complex setup | HTTPS/TLS needs | +| **sodiumoxide** | • libsodium wrapper
• Many primitives | • C dependency
• Lower level | Crypto primitives | ## Recommended Stack Based on the analysis, here's the recommended combination: ### Core Stack + ```toml [dependencies] # Transport quinn = "0.10" # QUIC for local/direct connections tokio-tungstenite = "0.20" # WebSocket for relay fallback -# Discovery +# Discovery mdns = "3.0" # Local network discovery if-watch = "3.0" # Network monitoring @@ -841,15 +851,18 @@ bincode = "1.5" # Efficient encoding ### Why This Stack? 1. **quinn + tokio-tungstenite** + - Covers all transport needs - QUIC for performance, WebSocket for compatibility - Both well-maintained 2. **mdns** + - Simple and sufficient for local discovery - No need for complex libp2p stack 3. **snow** + - Perfect fit for our security needs - Simpler than TLS - Better than rolling our own @@ -879,12 +892,14 @@ libp2p = { version = "0.53", features = [ ``` Pros: + - Everything included - Proven P2P patterns - DHT for device discovery - NAT traversal built-in Cons: + - Much larger dependency - Harder to customize - More complex to debug @@ -893,6 +908,7 @@ Cons: ## Implementation Complexity ### Minimal Viable Implementation (2-3 weeks) + ```rust // Just local network support struct SimpleNetwork { @@ -909,6 +925,7 @@ impl SimpleNetwork { ``` ### Full Implementation (6-8 weeks) + - Local P2P ✓ - Relay service ✓ - Encryption ✓ @@ -918,6 +935,7 @@ impl SimpleNetwork { - Auto-reconnect ✓ ### With libp2p (4-6 weeks) + - Faster initial development - But more time debugging/customizing - Less control over protocol @@ -926,4 +944,4 @@ impl SimpleNetwork { This design provides a flexible, secure networking layer that abstracts transport details from the application. By leveraging existing libraries like quinn, mdns, and snow, we minimize implementation complexity while maintaining full control over the protocol design. The transport-agnostic API ensures we can add new connection methods without changing application code. -The recommended stack balances simplicity with capability, avoiding the complexity of full P2P frameworks while still providing all needed functionality. This approach lets us ship a working solution quickly and iterate based on real usage. \ No newline at end of file +The recommended stack balances simplicity with capability, avoiding the complexity of full P2P frameworks while still providing all needed functionality. This approach lets us ship a working solution quickly and iterate based on real usage. diff --git a/core-new/docs/design/OLD_SPACEDRIVE_ANALYSIS.md b/docs/design/OLD_SPACEDRIVE_ANALYSIS.md similarity index 100% rename from core-new/docs/design/OLD_SPACEDRIVE_ANALYSIS.md rename to docs/design/OLD_SPACEDRIVE_ANALYSIS.md diff --git a/core-new/docs/design/OPERATIONS_REFACTOR_PLAN.md b/docs/design/OPERATIONS_REFACTOR_PLAN.md similarity index 100% rename from core-new/docs/design/OPERATIONS_REFACTOR_PLAN.md rename to docs/design/OPERATIONS_REFACTOR_PLAN.md diff --git a/core-new/docs/design/PERSISTENT_DEVICE_CONNECTIONS_DESIGN.md b/docs/design/PERSISTENT_DEVICE_CONNECTIONS_DESIGN.md similarity index 100% rename from core-new/docs/design/PERSISTENT_DEVICE_CONNECTIONS_DESIGN.md rename to docs/design/PERSISTENT_DEVICE_CONNECTIONS_DESIGN.md diff --git a/core-new/docs/design/REFERENCE_SIDECARS.md b/docs/design/REFERENCE_SIDECARS.md similarity index 100% rename from core-new/docs/design/REFERENCE_SIDECARS.md rename to docs/design/REFERENCE_SIDECARS.md diff --git a/core-new/docs/design/REWRITE_PLAN.MD b/docs/design/REWRITE_PLAN.MD similarity index 99% rename from core-new/docs/design/REWRITE_PLAN.MD rename to docs/design/REWRITE_PLAN.MD index aa069c7db..eda353896 100644 --- a/core-new/docs/design/REWRITE_PLAN.MD +++ b/docs/design/REWRITE_PLAN.MD @@ -227,7 +227,7 @@ The original Spacedrive captured imagination but was crippled by architectural d ## Next Steps -1. Complete core-new implementation alongside existing core +1. Complete core implementation alongside existing core 2. Migrate frontend to use new APIs gradually 3. Launch with basic feature set that works reliably 4. Build monetization through cloud sync and pro features diff --git a/core-new/docs/design/SDPATH_REFACTOR.md b/docs/design/SDPATH_REFACTOR.md similarity index 100% rename from core-new/docs/design/SDPATH_REFACTOR.md rename to docs/design/SDPATH_REFACTOR.md diff --git a/core-new/docs/design/SDPATH_REFACTOR_COVERAGE.md b/docs/design/SDPATH_REFACTOR_COVERAGE.md similarity index 100% rename from core-new/docs/design/SDPATH_REFACTOR_COVERAGE.md rename to docs/design/SDPATH_REFACTOR_COVERAGE.md diff --git a/core-new/docs/design/SEARCH_DESIGN.md b/docs/design/SEARCH_DESIGN.md similarity index 93% rename from core-new/docs/design/SEARCH_DESIGN.md rename to docs/design/SEARCH_DESIGN.md index da09a62a2..96607a2cd 100644 --- a/core-new/docs/design/SEARCH_DESIGN.md +++ b/docs/design/SEARCH_DESIGN.md @@ -505,7 +505,7 @@ pub struct MethodConfig { **PDF Documents:** ```toml -# core-new/src/file_type/definitions/documents.toml +# core/src/file_type/definitions/documents.toml [[file_types]] id = "application/pdf" name = "PDF Document" @@ -532,7 +532,7 @@ settings = { page = 1, dpi = 150, format = "webp" } **Source Code Files:** ```toml -# core-new/src/file_type/definitions/code.toml +# core/src/file_type/definitions/code.toml [[file_types]] id = "text/rust" name = "Rust Source Code" @@ -557,7 +557,7 @@ settings = { model = "microsoft/codebert-base", chunk_size = 512 } **Image Files:** ```toml -# core-new/src/file_type/definitions/images.toml +# core/src/file_type/definitions/images.toml [[file_types]] id = "image/jpeg" name = "JPEG Image" @@ -661,6 +661,7 @@ The `LightningSearchEngine` acts as a lightweight orchestrator. Its primary role A single user query triggers a multi-stage job that can progress through several power levels, emitting updates as more relevant results are found. #### The `SearchMode` Enum + The `SearchMode` now represents the internal power level or stage of a search. ```rust @@ -680,21 +681,23 @@ pub enum SearchMode { 1. **Dispatch:** A `SearchRequest` is received. The `LightningSearchEngine` creates and dispatches a `SearchJob`. 2. **Phase 1: `Fast` Search (Instant Results)** - * The job immediately runs the `Fast` search (FTS5 on metadata). - * Within ~50ms, the initial results are cached and a `SearchResultsReady(result_id)` event is sent to the UI. - * **The user sees instant results for any matching filenames.** + + - The job immediately runs the `Fast` search (FTS5 on metadata). + - Within ~50ms, the initial results are cached and a `SearchResultsReady(result_id)` event is sent to the UI. + - **The user sees instant results for any matching filenames.** 3. **Phase 2: `Normal` Search (Background Enhancement)** - * After Phase 1 completes, the job analyzes the query and initial results. - * If the query appears semantic or the `Fast` results are ambiguous, the job automatically promotes itself to the `Normal` stage. - * It re-ranks the results using VSS embedding sidecars. - * When complete, it **updates the existing cached result set** for the same `result_id` and sends a `SearchResultsUpdated(result_id)` event. - * **The UI seamlessly re-sorts the results list, bringing more relevant files to the top.** + + - After Phase 1 completes, the job analyzes the query and initial results. + - If the query appears semantic or the `Fast` results are ambiguous, the job automatically promotes itself to the `Normal` stage. + - It re-ranks the results using VSS embedding sidecars. + - When complete, it **updates the existing cached result set** for the same `result_id` and sends a `SearchResultsUpdated(result_id)` event. + - **The UI seamlessly re-sorts the results list, bringing more relevant files to the top.** 4. **Phase 3: `Full` Search (Optional Deep Dive)** - * This phase can be triggered by explicit user action (e.g., a "search deeper" button) or by an AI agent. - * It may perform more expensive operations, like expanding the candidate pool for semantic search. - * Like Phase 2, it updates the cached results when complete. + - This phase can be triggered by explicit user action (e.g., a "search deeper" button) or by an AI agent. + - It may perform more expensive operations, like expanding the candidate pool for semantic search. + - Like Phase 2, it updates the cached results when complete. ### 6. Search Result Caching: A Device-Local Filesystem Approach @@ -703,27 +706,29 @@ To ensure a fast experience and avoid re-computing searches, Spacedrive uses a s This solution has three components: 1. **Cache Directory (Non-Syncing)** - * The cache lives outside the portable `.sdlibrary` directory, in a standard system cache location, ensuring it is never synced or backed up. - * Example: `~/.cache/spacedrive/libraries/{library_id}/search/` + + - The cache lives outside the portable `.sdlibrary` directory, in a standard system cache location, ensuring it is never synced or backed up. + - Example: `~/.cache/spacedrive/libraries/{library_id}/search/` 2. **Result Files (Binary)** - * The ordered list of `entry_id`s for a search is stored in a compact binary file (e.g., a raw array of `i64`s). - * The filename is the unique `query_hash` of the search request (e.g., `.../search/a1b2c3d4.../results.bin`). - * This scales to millions of results and allows for extremely efficient pagination by seeking to the required offset in the file without loading the entire list into memory. + + - The ordered list of `entry_id`s for a search is stored in a compact binary file (e.g., a raw array of `i64`s). + - The filename is the unique `query_hash` of the search request (e.g., `.../search/a1b2c3d4.../results.bin`). + - This scales to millions of results and allows for extremely efficient pagination by seeking to the required offset in the file without loading the entire list into memory. 3. **Cache Index (Local Database)** - * A tiny, separate SQLite database (`cache_index.db`) is kept in the cache directory to manage the result files. - * This database is also local and never synced. It contains a single table to provide fast lookups for cached results. - * **Schema:** - ```sql - -- In: cache_index.db - CREATE TABLE cached_searches ( - query_hash TEXT PRIMARY KEY, - result_count INTEGER NOT NULL, - created_at TEXT NOT NULL DEFAULT (datetime('now')), - expires_at TEXT NOT NULL - ); - ``` + - A tiny, separate SQLite database (`cache_index.db`) is kept in the cache directory to manage the result files. + - This database is also local and never synced. It contains a single table to provide fast lookups for cached results. + - **Schema:** + ```sql + -- In: cache_index.db + CREATE TABLE cached_searches ( + query_hash TEXT PRIMARY KEY, + result_count INTEGER NOT NULL, + created_at TEXT NOT NULL DEFAULT (datetime('now')), + expires_at TEXT NOT NULL + ); + ``` This architecture strictly separates durable, syncable library data from ephemeral, device-local cache data, providing a robust and scalable caching solution. @@ -1147,8 +1152,6 @@ WHERE extension IN ('rs', 'js', 'py', 'cpp', 'java', 'go'); ## API Design - - ### GraphQL Integration ```graphql @@ -1177,12 +1180,18 @@ extend type Query { """ Get available facets for a query """ - searchFacets(query: String!, filters: [SearchFilterInput!] = []): SearchFacetsResponse! + searchFacets( + query: String! + filters: [SearchFilterInput!] = [] + ): SearchFacetsResponse! """ Search analytics and insights """ - searchAnalytics(timeRange: TimeRangeInput, groupBy: AnalyticsGroupBy): SearchAnalyticsResponse! + searchAnalytics( + timeRange: TimeRangeInput + groupBy: AnalyticsGroupBy + ): SearchAnalyticsResponse! } enum SearchMode { @@ -1415,17 +1424,19 @@ impl LightningSearchEngine { ## Updated Search Workflow 1. **Indexing & Intelligence Queueing** - * A file is discovered and its `content_uuid` is determined. - * The indexer dispatches jobs to generate sidecars: `OcrJob`, `TextExtractionJob`, `EmbeddingJob`, etc. + + - A file is discovered and its `content_uuid` is determined. + - The indexer dispatches jobs to generate sidecars: `OcrJob`, `TextExtractionJob`, `EmbeddingJob`, etc. 2. **Sidecar Generation** - * Jobs run asynchronously, creating text and embedding sidecars and populating the necessary database tables (`sidecars`, `sidecar_availability`). + + - Jobs run asynchronously, creating text and embedding sidecars and populating the necessary database tables (`sidecars`, `sidecar_availability`). 3. **Progressive Search Execution (`SearchJob`)** - * The job starts with a `Fast` search (FTS5) and immediately returns results to the UI. - * It then automatically enhances the results in the background by progressing to a `Normal` search (semantic re-ranking), issuing updates to the UI as better results are found. - * An optional `Full` search can be triggered for the most comprehensive results. - * All results are managed in the device-local cache. + - The job starts with a `Fast` search (FTS5) and immediately returns results to the UI. + - It then automatically enhances the results in the background by progressing to a `Normal` search (semantic re-ranking), issuing updates to the UI as better results are found. + - An optional `Full` search can be triggered for the most comprehensive results. + - All results are managed in the device-local cache. ## Conclusion diff --git a/core-new/docs/design/SIMULATION_ENGINE_DESIGN.md b/docs/design/SIMULATION_ENGINE_DESIGN.md similarity index 100% rename from core-new/docs/design/SIMULATION_ENGINE_DESIGN.md rename to docs/design/SIMULATION_ENGINE_DESIGN.md diff --git a/SPACEDRIVE_COMPLETE_OVERVIEW.md b/docs/design/SPACEDRIVE_COMPLETE_OVERVIEW.md similarity index 89% rename from SPACEDRIVE_COMPLETE_OVERVIEW.md rename to docs/design/SPACEDRIVE_COMPLETE_OVERVIEW.md index 1b30a2a93..a79034f73 100644 --- a/SPACEDRIVE_COMPLETE_OVERVIEW.md +++ b/docs/design/SPACEDRIVE_COMPLETE_OVERVIEW.md @@ -1,11 +1,11 @@ # Spacedrive: Complete Technical Overview -*A comprehensive analysis of the Spacedrive ecosystem, covering the core-new rewrite, cloud infrastructure, and the path to production.* +_A comprehensive analysis of the Spacedrive ecosystem, covering the core rewrite, cloud infrastructure, and the path to production._ ## Table of Contents 1. [Project Overview](#project-overview) -2. [Core-New: The Foundation Rewrite](#core-new-the-foundation-rewrite) +2. [core: The Foundation Rewrite](#core-the-foundation-rewrite) 3. [Spacedrive Cloud: Infrastructure & Business Model](#spacedrive-cloud-infrastructure--business-model) 4. [The Complete Technical Stack](#the-complete-technical-stack) 5. [Implementation Status & Roadmap](#implementation-status--roadmap) @@ -27,34 +27,35 @@ ### Market Problems Solved - Files scattered across multiple devices with no unified view -- No way to search or organize files across device boundaries +- No way to search or organize files across device boundaries - Platform lock-in with iCloud, Google Drive, OneDrive - Privacy concerns with cloud-based solutions - Duplicate files wasting storage across devices --- -## Core-New: The Foundation Rewrite +## core: The Foundation Rewrite -The **core-new** directory contains a complete architectural reimplementation with **111,052 lines** of Rust code that addresses fundamental flaws in the original codebase while establishing a modern foundation for the VDFS vision. +The **core** directory contains a complete architectural reimplementation with **111,052 lines** of Rust code that addresses fundamental flaws in the original codebase while establishing a modern foundation for the VDFS vision. ### Why The Rewrite Was Necessary The original implementation had fatal architectural flaws that would have eventually forced a rewrite: -| **Original Problems** | **Rewrite Solutions** | -|--------------------|-------------------| -| **Dual file systems** (indexed/ephemeral) | Single unified system with SdPath | -| **Impossible operations** (can't copy between systems) | All operations work everywhere | -| **Backend-frontend coupling** (`invalidate_query!`) | Event-driven decoupling | -| **Abandoned dependencies** (Prisma fork) | Modern SeaORM | -| **1000-line job boilerplate** | 50-line jobs with derive macros | -| **No real search** (just SQL LIKE) | SQLite FTS5 foundation ready | -| **Identity confusion** (Node/Device/Instance) | Single Device concept | +| **Original Problems** | **Rewrite Solutions** | +| ------------------------------------------------------ | --------------------------------- | +| **Dual file systems** (indexed/ephemeral) | Single unified system with SdPath | +| **Impossible operations** (can't copy between systems) | All operations work everywhere | +| **Backend-frontend coupling** (`invalidate_query!`) | Event-driven decoupling | +| **Abandoned dependencies** (Prisma fork) | Modern SeaORM | +| **1000-line job boilerplate** | 50-line jobs with derive macros | +| **No real search** (just SQL LIKE) | SQLite FTS5 foundation ready | +| **Identity confusion** (Node/Device/Instance) | Single Device concept | ### Core Architectural Innovations #### 1. **SdPath: Universal File Addressing** + The breakthrough innovation that makes device boundaries disappear: ```rust @@ -72,6 +73,7 @@ copy_files(sources: Vec, destination: SdPath) **Impact**: Prepares for true VDFS while working locally today. Enables features impossible in traditional file managers. #### 2. **Unified Entry Model** + Every file gets immediate metadata capabilities: ```rust @@ -84,11 +86,13 @@ pub struct Entry { ``` **Benefits**: + - Tag and organize files instantly without waiting for indexing - Progressive enhancement as analysis completes - Unified operations for files and directories #### 3. **Multi-Phase Indexing System** + Production-ready indexer with sophisticated capabilities: - **Scope control**: Current (single-level, <500ms) vs Recursive (full tree) @@ -97,12 +101,13 @@ Production-ready indexer with sophisticated capabilities: - **Resume capability**: Checkpointing allows resuming interrupted operations #### 4. **Self-Contained Libraries** + Revolutionary approach to data portability: ``` My Photos.sdlibrary/ ├── library.json # Configuration -├── database.db # All metadata +├── database.db # All metadata ├── thumbnails/ # All thumbnails ├── indexes/ # Search indexes └── .lock # Concurrency control @@ -113,6 +118,7 @@ My Photos.sdlibrary/ ### Production-Ready Features #### ✅ **Working CLI Interface** + Complete command-line tool demonstrating all features: ```bash @@ -124,13 +130,16 @@ spacedrive network pair generate ``` #### ✅ **Modern Database Layer** + Built on SeaORM replacing abandoned Prisma: + - Type-safe queries and migrations - Optimized schema with materialized paths - 70%+ space savings for large collections - Proper relationship mapping #### ✅ **Advanced Job System** + Dramatic improvement from original (50 lines vs 500+ lines): ```rust @@ -143,12 +152,14 @@ pub struct FileCopyJob { ``` Features: + - Automatic registration with derive macros - MessagePack serialization - Database persistence with resumption - Type-safe progress reporting #### ✅ **Production Networking (99% Complete)** + LibP2P-based networking stack: - **Device pairing**: BIP39 12-word codes with cryptographic verification @@ -158,6 +169,7 @@ LibP2P-based networking stack: - **Trust management**: Configurable device trust levels and session keys #### ✅ **Event-Driven Architecture** + Replaces the problematic `invalidate_query!` pattern: ```rust @@ -171,6 +183,7 @@ pub enum Event { ### Domain Model Excellence #### **Entry-Centric Design** + ```rust pub struct Entry { pub metadata_id: i32, // Always present - immediate tagging @@ -181,6 +194,7 @@ pub struct Entry { ``` #### **Content Deduplication** + ```rust pub struct ContentIdentity { pub cas_id: String, // Blake3 content hash @@ -190,6 +204,7 @@ pub struct ContentIdentity { ``` #### **Flexible Organization** + - **Tags**: Many-to-many relationships with colors and icons - **Labels**: Hierarchical organization system - **User metadata**: Immediate notes and favorites @@ -198,26 +213,29 @@ pub struct ContentIdentity { ### Advanced Indexing Capabilities #### **Flexible Scoping & Persistence** + ```rust // UI Navigation - Fast current directory scan let config = IndexerJobConfig::ui_navigation(location_id, path); // <500ms UI // External Path Browsing - Memory-only, no database pollution -let config = IndexerJobConfig::ephemeral_browse(path, scope); +let config = IndexerJobConfig::ephemeral_browse(path, scope); // Full Analysis - Complete coverage with content hashing -let config = IndexerJobConfig::new(location_id, path, IndexMode::Deep); +let config = IndexerJobConfig::new(location_id, path, IndexMode::Deep); ``` ### Networking Architecture #### **Device Pairing Protocol** + - **BIP39 codes**: 12-word pairing with ~128 bits entropy - **Challenge-response**: Cryptographic authentication - **Session persistence**: Automatic reconnection across restarts - **Trust levels**: Configurable device authentication #### **Universal Message Protocol** + ```rust pub enum DeviceMessage { FileTransferRequest { transfer_id: Uuid, file_path: String, file_size: u64 }, @@ -230,11 +248,13 @@ pub enum DeviceMessage { ### Implementation Status #### **68/76 Tests Passing** (89% pass rate) + The core functionality is comprehensively tested with working examples. #### **What's Production-Ready** + - ✅ Library and location management -- ✅ Multi-phase indexing with progress tracking +- ✅ Multi-phase indexing with progress tracking - ✅ Modern database layer with migrations - ✅ Event-driven architecture - ✅ Device networking and pairing (99% complete) @@ -243,6 +263,7 @@ The core functionality is comprehensively tested with working examples. - ✅ CLI interface demonstrating all features #### **What's Framework-Ready** + - 🚧 File operations (infrastructure complete, handlers need implementation) - 🚧 Search system (FTS5 integration planned) - 🚧 Advanced networking protocols (message system complete) @@ -260,7 +281,7 @@ The **spacedrive-cloud** project provides Spacedrive-as-a-Service by running man ### Core Concepts - **Cloud Core as Device**: Each user gets a managed Spacedrive core that appears as a regular device -- **Native Networking**: Users connect via built-in P2P pairing and networking protocols +- **Native Networking**: Users connect via built-in P2P pairing and networking protocols - **Location-Based Storage**: Cloud storage exposed through Spacedrive's native location system - **Device Semantics**: No custom APIs - cloud cores are indistinguishable from local devices - **Seamless Integration**: Users pair with cloud cores just like any other Spacedrive device @@ -268,6 +289,7 @@ The **spacedrive-cloud** project provides Spacedrive-as-a-Service by running man ### Technical Architecture #### **System Components** + ``` ┌─────────────────────────────────────────────────────────────┐ │ User's Local Spacedrive │ @@ -291,6 +313,7 @@ The **spacedrive-cloud** project provides Spacedrive-as-a-Service by running man ``` #### **Cloud Core Implementation** + ```rust pub struct CloudCoreManager { user_id: UserId, @@ -303,20 +326,21 @@ impl CloudCoreManager { pub async fn start_core(&self) -> Result { // Start a regular Spacedrive core let core = Core::new_with_config(&self.device_config.data_directory).await?; - + // Enable networking for P2P pairing core.init_networking("cloud-device-password").await?; core.start_networking().await?; - + // Create default cloud storage locations self.setup_cloud_locations(&core).await?; - + Ok(core) } } ``` #### **User Connection Flow** + ```rust // User's local Spacedrive generates pairing code let pairing_session = local_core.networking @@ -341,33 +365,35 @@ core.networking ### Kubernetes Deployment #### **Cloud Core Pod Template** + ```yaml apiVersion: v1 kind: Pod spec: containers: - - name: spacedrive-cloud-device - image: spacedrive/core-new:latest - env: - - name: USER_ID - value: "user-123" - - name: DEVICE_NAME - value: "user-123's Cloud Device" - ports: - - containerPort: 37520 # P2P networking port - resources: - requests: - memory: "1Gi" - cpu: "500m" - limits: - memory: "4Gi" - cpu: "2" - volumeMounts: - - name: user-device-data - mountPath: /data + - name: spacedrive-cloud-device + image: spacedrive/core:latest + env: + - name: USER_ID + value: "user-123" + - name: DEVICE_NAME + value: "user-123's Cloud Device" + ports: + - containerPort: 37520 # P2P networking port + resources: + requests: + memory: "1Gi" + cpu: "500m" + limits: + memory: "4Gi" + cpu: "2" + volumeMounts: + - name: user-device-data + mountPath: /data ``` #### **Storage Management** + ``` /data/ ├── spacedrive/ # Standard Spacedrive data directory @@ -387,11 +413,13 @@ spec: ### Business Model Integration #### **Service Tiers** + - **Starter**: 1 cloud device, 25GB storage, 1 vCPU, 2GB RAM - **Professional**: 1 cloud device, 250GB storage, 2 vCPU, 4GB RAM, priority locations - **Enterprise**: Multiple cloud devices, 1TB+ storage, 4+ vCPU, 8GB+ RAM, custom locations #### **User Experience Benefits** + - **Seamless Integration**: Cloud device appears like any other Spacedrive device - **Native File Operations**: Copy, move, sync using standard Spacedrive operations - **Cross-Device Access**: Access cloud files from any paired device @@ -399,6 +427,7 @@ spec: - **Always Available**: 24/7 device availability without leaving local devices on #### **SLA Commitments** + - **Device Uptime**: 99.9% availability (8.77 hours downtime/year) - **P2P Connection**: <2 second device discovery and connection - **Data Durability**: 99.999999999% (11 9's) with automated backup @@ -411,24 +440,28 @@ spec: ### Core Technologies #### **Runtime & Language** + - **Rust**: Memory-safe systems programming for core components - **TypeScript**: Type-safe frontend development - **React**: Modern UI framework with cross-platform support - **Tauri**: Native desktop app framework #### **Database & Storage** + - **SQLite**: Per-device database with SeaORM - **PostgreSQL**: Cloud service metadata - **MessagePack**: Efficient binary serialization - **Blake3**: Fast cryptographic hashing #### **Networking** + - **LibP2P**: Production-grade P2P networking stack - **Noise Protocol**: Transport-layer encryption - **BIP39**: Human-readable pairing codes - **Kademlia DHT**: Global peer discovery #### **Infrastructure** + - **Kubernetes**: Container orchestration - **Docker**: Containerization - **Prometheus**: Metrics and monitoring @@ -437,21 +470,24 @@ spec: ### Architecture Patterns #### **Clean Architecture** + ``` src/ ├── domain/ # Core business entities -├── operations/ # User-facing functionality +├── operations/ # User-facing functionality ├── infrastructure/ # External interfaces └── shared/ # Common types and utilities ``` #### **Event-Driven Design** + - Loose coupling between components - Real-time UI updates - Plugin-ready architecture - Comprehensive audit trail #### **Domain-Driven Development** + - Business logic in domain layer - Rich domain models - Ubiquitous language @@ -459,13 +495,15 @@ src/ ### Performance Characteristics -#### **Core-New Performance** +#### **core Performance** + - **Indexing**: <500ms for current scope, batched processing for recursive - **Database**: 70%+ space savings with materialized paths - **Memory**: Streaming operations, bounded queues - **Networking**: 1000+ messages/second per connection #### **Cloud Performance** + - **Device Startup**: ~2-3 seconds for full networking initialization - **Memory Usage**: ~10-50MB depending on number of paired devices - **Storage**: ~1-5KB per paired device (encrypted) @@ -477,7 +515,8 @@ src/ ### Current Status -#### **Core-New: 89% Complete** +#### **core: 89% Complete** + - ✅ **Foundation**: Library and location management - ✅ **Indexing**: Multi-phase indexer with scope and persistence control - ✅ **Database**: Modern SeaORM layer with migrations @@ -489,6 +528,7 @@ src/ - 🚧 **UI Integration**: Ready to replace original core as backend #### **Spacedrive Cloud: Architecture Complete** + - ✅ **Technical Design**: Complete cloud-native architecture - ✅ **Kubernetes**: Production-ready deployment templates - ✅ **Security**: Device isolation and network policies @@ -498,24 +538,28 @@ src/ ### Roadmap #### **Phase 1: Core Completion (Weeks 1-4)** + - Complete file operations implementation - Integrate SQLite FTS5 search - Finish networking message routing - Desktop app integration #### **Phase 2: Cloud MVP (Weeks 5-8)** + - Implement CloudDeviceOrchestrator - Deploy basic Kubernetes infrastructure - User device provisioning and pairing - Basic monitoring and health checks #### **Phase 3: Production Ready (Weeks 9-12)** + - Advanced storage management - Security hardening and compliance - Performance optimization - Customer support tools #### **Phase 4: Scale & Features (Weeks 13-16)** + - Multi-region deployment - Advanced search capabilities - Enhanced networking protocols @@ -531,13 +575,14 @@ src/ 1. **Solves Real Problems**: Addresses every architectural flaw from the original 2. **Working Today**: 89% test pass rate with comprehensive CLI demos -3. **Future-Ready**: SdPath enables features impossible in traditional file managers +3. **Future-Ready**: SdPath enables features impossible in traditional file managers 4. **Maintainable**: Modern patterns and comprehensive documentation 5. **Performance**: Optimized for real-world usage patterns #### **Innovation Impact** The **SdPath abstraction** is the key innovation that enables the VDFS vision: + - Makes device boundaries transparent - Enables cross-device operations as first-class features - Prepares for distributed file systems while working locally today @@ -556,6 +601,7 @@ The **SdPath abstraction** is the key innovation that enables the VDFS vision: #### **Business Model Strength** The cloud offering provides a sustainable business model: + - **Recurring Revenue**: Subscription-based cloud device services - **Natural Upselling**: Users start free, upgrade for cloud features - **Sticky Product**: File management is essential daily workflow @@ -566,6 +612,7 @@ The cloud offering provides a sustainable business model: #### **Technical Debt Resolution** The rewrite eliminates technical debt that was blocking progress: + - Modern dependencies (SeaORM vs abandoned Prisma fork) - Clean architecture enabling rapid feature development - Comprehensive testing preventing regressions @@ -596,12 +643,14 @@ The rewrite eliminates technical debt that was blocking progress: ### Success Metrics #### **Technical KPIs** + - Test coverage > 90% - API response times < 100ms - P2P connection establishment < 2 seconds - Cross-device file operation success rate > 99% #### **Business KPIs** + - Monthly active users (target: 1M within 12 months) - Cloud service conversion rate (target: 15% of free users) - Average revenue per user (target: $10/month for paid tiers) @@ -611,7 +660,7 @@ The rewrite eliminates technical debt that was blocking progress: ## Conclusion -Spacedrive represents a fundamental reimagining of file management for the modern multi-device world. The **core-new rewrite** provides a solid technical foundation that resolves the architectural issues of the original while establishing clean patterns for future development. The **cloud infrastructure** design enables a sustainable business model through native device semantics. +Spacedrive represents a fundamental reimagining of file management for the modern multi-device world. The **core rewrite** provides a solid technical foundation that resolves the architectural issues of the original while establishing clean patterns for future development. The **cloud infrastructure** design enables a sustainable business model through native device semantics. ### Key Achievements @@ -634,4 +683,4 @@ With the foundation complete, Spacedrive is positioned to: The 34,000 GitHub stars demonstrate clear market demand. The rewrite ensures the project can finally deliver on its ambitious vision of making file management truly device-agnostic while maintaining user privacy and control. -**Spacedrive is ready to transform how people interact with their files across all their devices.** \ No newline at end of file +**Spacedrive is ready to transform how people interact with their files across all their devices.** diff --git a/core-new/docs/design/SPACEDROP_DESIGN.md b/docs/design/SPACEDROP_DESIGN.md similarity index 100% rename from core-new/docs/design/SPACEDROP_DESIGN.md rename to docs/design/SPACEDROP_DESIGN.md diff --git a/core-new/docs/design/SPACEDROP_IMPLEMENTATION_PLAN.md b/docs/design/SPACEDROP_IMPLEMENTATION_PLAN.md similarity index 98% rename from core-new/docs/design/SPACEDROP_IMPLEMENTATION_PLAN.md rename to docs/design/SPACEDROP_IMPLEMENTATION_PLAN.md index d93430421..0467de5d1 100644 --- a/core-new/docs/design/SPACEDROP_IMPLEMENTATION_PLAN.md +++ b/docs/design/SPACEDROP_IMPLEMENTATION_PLAN.md @@ -7,7 +7,7 @@ This document outlines the step-by-step implementation of Spacedrop on top of Sp ## File Structure ``` -core-new/src/networking/ +core/src/networking/ ├── mod.rs # Add spacedrop exports ├── behavior.rs # Extend with spacedrop protocol ├── spacedrop/ # New spacedrop module @@ -29,7 +29,7 @@ core-new/src/networking/ ### 1.1 Create Module Structure -**File**: `core-new/src/networking/spacedrop/mod.rs` +**File**: `core/src/networking/spacedrop/mod.rs` ```rust //! Spacedrop: Cross-platform file sharing protocol @@ -65,7 +65,7 @@ pub type DeviceId = Uuid; ### 1.2 Define Message Types -**File**: `core-new/src/networking/spacedrop/messages.rs` +**File**: `core/src/networking/spacedrop/messages.rs` ```rust use serde::{Serialize, Deserialize}; @@ -159,7 +159,7 @@ pub enum ChunkStatus { ### 1.3 Implement LibP2P Codec -**File**: `core-new/src/networking/spacedrop/codec.rs` +**File**: `core/src/networking/spacedrop/codec.rs` ```rust use async_trait::async_trait; @@ -240,7 +240,7 @@ impl Codec for SpacedropCodec { ### 2.1 Extend NetworkBehaviour -**File**: `core-new/src/networking/behavior.rs` (modify existing) +**File**: `core/src/networking/behavior.rs` (modify existing) ```rust use libp2p::request_response::{ @@ -284,7 +284,7 @@ impl SpacedriveBehaviour { ### 2.2 Implement Discovery -**File**: `core-new/src/networking/spacedrop/discovery.rs` +**File**: `core/src/networking/spacedrop/discovery.rs` ```rust use libp2p::{PeerId, Multiaddr}; @@ -372,7 +372,7 @@ impl SpacedropDiscovery { ### 3.1 Encryption Utilities -**File**: `core-new/src/networking/spacedrop/encryption.rs` +**File**: `core/src/networking/spacedrop/encryption.rs` ```rust use ring::{ @@ -477,7 +477,7 @@ pub fn calculate_chunk_checksum(data: &[u8]) -> [u8; 32] { ### 3.2 File Transfer Engine -**File**: `core-new/src/networking/spacedrop/transfer.rs` +**File**: `core/src/networking/spacedrop/transfer.rs` ```rust use std::path::Path; @@ -686,7 +686,7 @@ impl FileTransfer { ### 4.1 Session Manager -**File**: `core-new/src/networking/spacedrop/manager.rs` +**File**: `core/src/networking/spacedrop/manager.rs` ```rust use std::collections::HashMap; @@ -875,7 +875,7 @@ impl SpacedropManager { ### 5.1 UI Abstraction -**File**: `core-new/src/networking/spacedrop/ui.rs` +**File**: `core/src/networking/spacedrop/ui.rs` ```rust use async_trait::async_trait; @@ -958,7 +958,7 @@ impl SpacedropUserInterface for ConsoleSpacedropUI { ### 5.2 Demo Application -**File**: `core-new/examples/spacedrop_demo.rs` +**File**: `core/examples/spacedrop_demo.rs` ```rust use tokio; @@ -1025,7 +1025,7 @@ async fn main() -> Result<(), Box> { ### Update Main Module -**File**: `core-new/src/networking/mod.rs` (modify) +**File**: `core/src/networking/mod.rs` (modify) ```rust // Add spacedrop module diff --git a/core-new/docs/design/STRUCTURE.md b/docs/design/STRUCTURE.md similarity index 97% rename from core-new/docs/design/STRUCTURE.md rename to docs/design/STRUCTURE.md index 7b9218400..ff3a92606 100644 --- a/core-new/docs/design/STRUCTURE.md +++ b/docs/design/STRUCTURE.md @@ -1,7 +1,7 @@ -# Core-New Structure +# core Structure ``` -core-new/ +core/ ├── Cargo.toml # Dependencies (SeaORM, axum, etc.) ├── README.md # Overview and strategy ├── MIGRATION.md # How to migrate from old core @@ -57,4 +57,4 @@ core-new/ 2. Port file operations one at a time 3. Build GraphQL API incrementally 4. Create integration tests for each operation -5. Develop migration tooling \ No newline at end of file +5. Develop migration tooling diff --git a/core-new/docs/design/SYNC_DESIGN.md b/docs/design/SYNC_DESIGN.md similarity index 100% rename from core-new/docs/design/SYNC_DESIGN.md rename to docs/design/SYNC_DESIGN.md diff --git a/core-new/docs/design/SYNC_DESIGN_2025_08_19.md b/docs/design/SYNC_DESIGN_2025_08_19.md similarity index 100% rename from core-new/docs/design/SYNC_DESIGN_2025_08_19.md rename to docs/design/SYNC_DESIGN_2025_08_19.md diff --git a/core-new/docs/design/SYNC_FIRST_DRAFT_DESIGN.md b/docs/design/SYNC_FIRST_DRAFT_DESIGN.md similarity index 100% rename from core-new/docs/design/SYNC_FIRST_DRAFT_DESIGN.md rename to docs/design/SYNC_FIRST_DRAFT_DESIGN.md diff --git a/core-new/docs/design/SYNC_INTEGRATION_NOTES.md b/docs/design/SYNC_INTEGRATION_NOTES.md similarity index 100% rename from core-new/docs/design/SYNC_INTEGRATION_NOTES.md rename to docs/design/SYNC_INTEGRATION_NOTES.md diff --git a/core-new/docs/design/THUMBNAIL_SYSTEM_DESIGN.md b/docs/design/THUMBNAIL_SYSTEM_DESIGN.md similarity index 95% rename from core-new/docs/design/THUMBNAIL_SYSTEM_DESIGN.md rename to docs/design/THUMBNAIL_SYSTEM_DESIGN.md index 4751ede1e..f48ca6ec0 100644 --- a/core-new/docs/design/THUMBNAIL_SYSTEM_DESIGN.md +++ b/docs/design/THUMBNAIL_SYSTEM_DESIGN.md @@ -1,13 +1,13 @@ -# Thumbnail System Design for Core-New +# Thumbnail System Design for core ## Executive Summary -This document outlines the design for a modern thumbnail generation system for Spacedrive core-new, learning from the original implementation while leveraging core-new's improved job system architecture. The system will run as a separate job alongside indexing operations, providing efficient, scalable thumbnail generation with support for a wide variety of media formats. +This document outlines the design for a modern thumbnail generation system for Spacedrive core, learning from the original implementation while leveraging core's improved job system architecture. The system will run as a separate job alongside indexing operations, providing efficient, scalable thumbnail generation with support for a wide variety of media formats. ## Design Principles 1. **Separation of Concerns**: Thumbnail generation is independent from indexing, allowing for flexible scheduling and processing -2. **Job-Based Architecture**: Leverages core-new's simplified job system with minimal boilerplate +2. **Job-Based Architecture**: Leverages core's simplified job system with minimal boilerplate 3. **Content-Addressable Storage**: Uses CAS IDs from indexing for efficient deduplication and storage 4. **Library-Scoped Storage**: Thumbnails are stored within each library directory for portability 5. **Progressive Enhancement**: Thumbnails can be generated after initial indexing completes @@ -57,7 +57,7 @@ This document outlines the design for a modern thumbnail generation system for S ### ThumbnailJob Structure -Building on core-new's job system, the thumbnail job follows the established patterns: +Building on core's job system, the thumbnail job follows the established patterns: ```rust use crate::infrastructure::jobs::prelude::*; @@ -596,7 +596,7 @@ impl LibraryConfig { ## Benefits Over Original System 1. **Cleaner Architecture**: Separated from indexing, follows job system patterns -2. **Better Resumability**: Leverages core-new's checkpoint system +2. **Better Resumability**: Leverages core's checkpoint system 3. **Improved Performance**: Batch processing and better concurrency control 4. **Enhanced Error Handling**: Non-critical errors don't stop the entire job 5. **Greater Flexibility**: Multiple trigger mechanisms and processing modes @@ -605,7 +605,7 @@ impl LibraryConfig { ## Conclusion -This thumbnail system design provides a robust, scalable solution for thumbnail generation in core-new. By leveraging the improved job system architecture and maintaining compatibility with the original storage approach, it offers the best of both worlds: modern implementation patterns with proven storage efficiency. +This thumbnail system design provides a robust, scalable solution for thumbnail generation in core. By leveraging the improved job system architecture and maintaining compatibility with the original storage approach, it offers the best of both worlds: modern implementation patterns with proven storage efficiency. The system is designed to be: @@ -614,4 +614,4 @@ The system is designed to be: - **Reliable**: Comprehensive error handling and resumable operations - **Extensible**: Easy to add new formats and processing options -This design positions the thumbnail system as a first-class citizen in the core-new architecture while maintaining the performance and reliability expectations established by the original implementation. +This design positions the thumbnail system as a first-class citizen in the core architecture while maintaining the performance and reliability expectations established by the original implementation. diff --git a/core-new/docs/design/UI_DESIGN.md b/docs/design/UI_DESIGN.md similarity index 100% rename from core-new/docs/design/UI_DESIGN.md rename to docs/design/UI_DESIGN.md diff --git a/core-new/docs/design/VDFS_MODEL_VISUAL.md b/docs/design/VDFS_MODEL_VISUAL.md similarity index 100% rename from core-new/docs/design/VDFS_MODEL_VISUAL.md rename to docs/design/VDFS_MODEL_VISUAL.md diff --git a/core-new/docs/design/VIRTUAL_LOCATIONS_DESIGN.md b/docs/design/VIRTUAL_LOCATIONS_DESIGN.md similarity index 100% rename from core-new/docs/design/VIRTUAL_LOCATIONS_DESIGN.md rename to docs/design/VIRTUAL_LOCATIONS_DESIGN.md diff --git a/core-new/docs/design/VIRTUAL_SIDECAR_SYSTEM.md b/docs/design/VIRTUAL_SIDECAR_SYSTEM.md similarity index 100% rename from core-new/docs/design/VIRTUAL_SIDECAR_SYSTEM.md rename to docs/design/VIRTUAL_SIDECAR_SYSTEM.md diff --git a/core-new/docs/design/VOLUME_CLASSIFICATION_DESIGN.md b/docs/design/VOLUME_CLASSIFICATION_DESIGN.md similarity index 100% rename from core-new/docs/design/VOLUME_CLASSIFICATION_DESIGN.md rename to docs/design/VOLUME_CLASSIFICATION_DESIGN.md diff --git a/core-new/docs/design/VOLUME_TRACKING_IMPLEMENTATION_PLAN.md b/docs/design/VOLUME_TRACKING_IMPLEMENTATION_PLAN.md similarity index 100% rename from core-new/docs/design/VOLUME_TRACKING_IMPLEMENTATION_PLAN.md rename to docs/design/VOLUME_TRACKING_IMPLEMENTATION_PLAN.md diff --git a/core-new/docs/design/WATCHER_VDFS_INTEGRATION.md b/docs/design/WATCHER_VDFS_INTEGRATION.md similarity index 93% rename from core-new/docs/design/WATCHER_VDFS_INTEGRATION.md rename to docs/design/WATCHER_VDFS_INTEGRATION.md index f2052cd76..2fb476be3 100644 --- a/core-new/docs/design/WATCHER_VDFS_INTEGRATION.md +++ b/docs/design/WATCHER_VDFS_INTEGRATION.md @@ -2,17 +2,19 @@ ## Overview -This document outlines how the cross-platform file watcher integrates with the core-new Virtual Distributed File System (VDFS), leveraging the new Entry-centric data model and SdPath addressing system. +This document outlines how the cross-platform file watcher integrates with the core Virtual Distributed File System (VDFS), leveraging the new Entry-centric data model and SdPath addressing system. ## Key Differences from Original Implementation ### Original Spacedrive Architecture + - **FilePath-centric**: Files were primarily `file_path` records with optional `object` links -- **Content-first**: Required content hashing for full functionality +- **Content-first**: Required content hashing for full functionality - **Prisma ORM**: Complex query patterns with extensive invalidation - **Immediate indexing**: Heavy operations triggered on every file event -### Core-New Architecture +### core Architecture + - **Entry-centric**: Every file/directory is an `Entry` with mandatory `UserMetadata` - **Metadata-first**: User metadata (tags, notes) available immediately - **SeaORM**: Modern Rust ORM with better performance patterns @@ -27,6 +29,7 @@ File System Event → Platform Handler → Direct Database Operations → Event ``` **Detailed Flow:** + 1. **File system events** detected by platform-specific handlers (FSEvents, inotify, etc.) 2. **Platform handler** filters and processes events (debouncing, rename correlation) 3. **Direct database operations** immediately create/update Entry and UserMetadata records @@ -41,22 +44,22 @@ File System Event → Platform Handler → Direct Database Operations → Event ```rust async fn handle_file_created( - sd_path: SdPath, + sd_path: SdPath, library_id: Uuid, db: &DatabaseConnection ) -> Result { // 1. Get filesystem metadata let metadata = tokio::fs::metadata(sd_path.as_local_path()?).await?; - + // 2. Check for existing Entry (handle duplicates/race conditions) if let Some(existing) = find_entry_by_sdpath(&sd_path, db).await? { return Ok(existing); } - + // 3. Create Entry record let entry_id = Uuid::new_v7(); let metadata_id = Uuid::new_v7(); - + let entry = entry::ActiveModel { id: Set(entry_id), uuid: Set(Uuid::new_v4()), // Public UUID for API @@ -64,12 +67,12 @@ async fn handle_file_created( path: Set(sd_path.path().to_string()), library_id: Set(Some(library_id)), name: Set(sd_path.file_name().unwrap_or_default()), - kind: Set(if metadata.is_dir() { - EntryKind::Directory - } else { - EntryKind::File { - extension: sd_path.extension().map(|s| s.to_string()) - } + kind: Set(if metadata.is_dir() { + EntryKind::Directory + } else { + EntryKind::File { + extension: sd_path.extension().map(|s| s.to_string()) + } }), size: Set(if metadata.is_dir() { None } else { Some(metadata.len()) }), created_at: Set(metadata.created().ok().map(|t| t.into())), @@ -78,7 +81,7 @@ async fn handle_file_created( content_id: Set(None), // Will be set during indexing // ... other fields }; - + // 4. Create UserMetadata record let user_metadata = user_metadata::ActiveModel { id: Set(metadata_id), @@ -89,22 +92,22 @@ async fn handle_file_created( hidden: Set(false), // ... other fields }; - + // 5. Insert both in transaction let txn = db.begin().await?; let entry = entry.insert(&txn).await?; user_metadata.insert(&txn).await?; txn.commit().await?; - + // 6. Generate content identity immediately (following original pattern) if should_index_content(&sd_path) { if let Ok(cas_id) = generate_cas_id(&sd_path).await { let content_identity = find_or_create_content_identity(cas_id, &txn).await?; - + // Link entry to content entry.content_id = Set(Some(content_identity.id)); entry.save(&txn).await?; - + // Spawn background task for heavy operations (thumbnails, media extraction) let sd_path_clone = sd_path.clone(); let entry_id = entry.id.clone(); @@ -118,7 +121,7 @@ async fn handle_file_created( }); } } - + Ok(entry) } ``` @@ -139,14 +142,14 @@ async fn handle_file_modified( return handle_file_created(sd_path, library_id, db).await.map(Some); } }; - + // 2. Update basic metadata let metadata = tokio::fs::metadata(sd_path.as_local_path()?).await?; - + let mut active_entry: entry::ActiveModel = entry.into(); active_entry.size = Set(if metadata.is_dir() { None } else { Some(metadata.len()) }); active_entry.modified_at = Set(metadata.modified().ok().map(|t| t.into())); - + // 3. Handle content changes immediately if let Some(content_id) = entry.content_id { // File had content identity - check if content actually changed @@ -156,11 +159,11 @@ async fn handle_file_modified( // Content changed - create or link to new content identity let new_content = find_or_create_content_identity(new_cas_id, db).await?; active_entry.content_id = Set(Some(new_content.id)); - + // Update reference counts decrease_content_reference_count(content_id, db).await?; increase_content_reference_count(new_content.id, db).await?; - + // Spawn background task for re-generating thumbnails/media data let sd_path_clone = sd_path.clone(); let entry_id = entry.id; @@ -176,10 +179,10 @@ async fn handle_file_modified( active_entry.content_id = Set(Some(content_identity.id)); } } - + // 4. Update Entry let updated_entry = active_entry.update(db).await?; - + Ok(Some(updated_entry)) } ``` @@ -189,12 +192,12 @@ async fn handle_file_modified( ```rust async fn handle_file_moved( old_path: SdPath, - new_path: SdPath, + new_path: SdPath, db: &DatabaseConnection ) -> Result> { // 1. Find existing Entry by old path let entry = find_entry_by_sdpath(&old_path, db).await?; - + let entry = match entry { Some(entry) => entry, None => { @@ -202,13 +205,13 @@ async fn handle_file_moved( return handle_file_created(new_path, library_id, db).await.map(Some); } }; - + // 2. Update path information let mut active_entry: entry::ActiveModel = entry.into(); active_entry.device_id = Set(new_path.device_id()); active_entry.path = Set(new_path.path().to_string()); active_entry.name = Set(new_path.file_name().unwrap_or_default()); - + // Update extension if it changed if let EntryKind::File { extension } = &entry.kind { let new_extension = new_path.extension().map(|s| s.to_string()); @@ -216,25 +219,25 @@ async fn handle_file_moved( active_entry.kind = Set(EntryKind::File { extension: new_extension }); } } - + // 3. Handle directory moves (update all children) if matches!(entry.kind, EntryKind::Directory) { update_child_paths_recursively(entry.id, &old_path, &new_path, db).await?; } - + // 4. Update parent relationship if let Some(parent_path) = new_path.parent() { if let Some(parent_entry) = find_entry_by_sdpath(&parent_path, db).await? { active_entry.parent_id = Set(Some(parent_entry.id)); } } - + // 5. Update Entry let updated_entry = active_entry.update(db).await?; - + // Note: UserMetadata and ContentIdentity remain unchanged during moves // This preserves tags, notes, and deduplication relationships - + Ok(Some(updated_entry)) } ``` @@ -251,20 +254,20 @@ async fn handle_file_deleted( Some(entry) => entry, None => return Ok(()), // Already deleted or never existed }; - + // 2. Handle directory deletion (recursive) if matches!(entry.kind, EntryKind::Directory) { delete_children_recursively(entry.id, db).await?; } - + // 3. Check ContentIdentity reference count if let Some(content_id) = entry.content_id { decrease_content_reference_count(content_id, db).await?; } - + // 4. Delete Entry (UserMetadata is deleted via cascade) entry::Entity::delete_by_id(entry.id).execute(db).await?; - + Ok(()) } @@ -277,24 +280,24 @@ async fn decrease_content_reference_count( .filter(entry::Column::ContentId.eq(content_id)) .count(db) .await? as u32; - + // 2. Update ContentIdentity if remaining_count == 0 { // No more entries reference this content - delete it content_identity::Entity::delete_by_id(content_id).execute(db).await?; } else { // Update reference count - let mut active_content: content_identity::ActiveModel = + let mut active_content: content_identity::ActiveModel = content_identity::Entity::find_by_id(content_id) .one(db) .await? .unwrap() .into(); - + active_content.entry_count = Set(remaining_count); active_content.update(db).await?; } - + Ok(()) } ``` @@ -307,29 +310,29 @@ Following the original approach, heavy operations are handled via spawned tasks, /// Generate thumbnails in background (original pattern) async fn generate_thumbnails(sd_path: &SdPath, entry_id: Uuid) -> Result<()> { let file_path = sd_path.as_local_path()?; - + // Check if file is a supported media type if !is_thumbnail_supported(&file_path) { return Ok(()); } - + // Generate thumbnail (this can be slow) let thumbnail_data = create_thumbnail(&file_path).await?; - + // Save thumbnail to storage let thumbnail_path = get_thumbnail_path(entry_id); save_thumbnail(thumbnail_path, thumbnail_data).await?; - + // Update entry with thumbnail info update_entry_thumbnail_info(entry_id, true).await?; - + Ok(()) } /// Extract media metadata in background (original pattern) async fn extract_media_metadata(sd_path: &SdPath, entry_id: Uuid) -> Result<()> { let file_path = sd_path.as_local_path()?; - + // Extract metadata based on file type let media_data = match get_file_type(&file_path) { FileType::Image => extract_exif_data(&file_path).await?, @@ -337,10 +340,10 @@ async fn extract_media_metadata(sd_path: &SdPath, entry_id: Uuid) -> Result<()> FileType::Audio => extract_audio_metadata(&file_path).await?, _ => return Ok(()), // Not a media file }; - + // Update content identity with media data update_content_media_data(entry_id, media_data).await?; - + Ok(()) } @@ -348,7 +351,7 @@ async fn extract_media_metadata(sd_path: &SdPath, entry_id: Uuid) -> Result<()> async fn spawn_directory_scan(location_id: Uuid, path: SdPath) { // Wait 1 second like original to avoid scanning rapidly changing directories tokio::time::sleep(Duration::from_secs(1)).await; - + // Trigger location sub-path scan job (this part uses job system) if let Err(e) = trigger_location_scan_job(location_id, path).await { tracing::error!("Failed to trigger directory scan job: {}", e); @@ -364,7 +367,7 @@ File watchers operate within the context of indexed Locations: impl LocationWatcher { async fn add_location_to_watcher(&self, location: &Location) -> Result<()> { let sd_path = SdPath::from_serialized(&location.device_id, &location.path)?; - + let watched_location = WatchedLocation { id: location.id, library_id: location.library_id, @@ -372,15 +375,15 @@ impl LocationWatcher { enabled: location.watch_enabled, index_mode: location.index_mode, }; - + self.add_location(watched_location).await?; - + // Emit event self.events.emit(Event::LocationWatchingStarted { library_id: location.library_id, location_id: location.id, }); - + Ok(()) } } @@ -394,47 +397,47 @@ The watcher emits detailed events for real-time UI updates: #[derive(Debug, Clone, Serialize, Deserialize)] pub enum Event { // Existing events... - + // Enhanced file system events - EntryCreated { - library_id: Uuid, + EntryCreated { + library_id: Uuid, entry_id: Uuid, entry_uuid: Uuid, // Public UUID for frontend sd_path: String, // Serialized SdPath kind: EntryKind, }, - EntryModified { - library_id: Uuid, + EntryModified { + library_id: Uuid, entry_id: Uuid, entry_uuid: Uuid, changes: EntryChanges, // What specifically changed }, - EntryDeleted { - library_id: Uuid, + EntryDeleted { + library_id: Uuid, entry_id: Uuid, entry_uuid: Uuid, sd_path: String, // Path before deletion }, - EntryMoved { - library_id: Uuid, + EntryMoved { + library_id: Uuid, entry_id: Uuid, entry_uuid: Uuid, - old_path: String, + old_path: String, new_path: String, }, - + // Content indexing events ContentIndexingStarted { entry_id: Uuid }, - ContentIndexingCompleted { - entry_id: Uuid, + ContentIndexingCompleted { + entry_id: Uuid, content_id: Option, // None if no unique content found is_duplicate: bool, }, - ContentIndexingFailed { - entry_id: Uuid, - error: String + ContentIndexingFailed { + entry_id: Uuid, + error: String }, - + // Location watching events LocationWatchingStarted { library_id: Uuid, location_id: Uuid }, LocationWatchingPaused { library_id: Uuid, location_id: Uuid }, @@ -481,40 +484,42 @@ impl WatcherDatabaseOperations { ### 7. Performance Optimizations #### Batch Operations + ```rust impl WatcherDatabaseOperations { async fn flush_pending_operations(&self) -> Result<()> { let pending = self.pending_operations.lock().await; - + if pending.is_empty() { return Ok(()); } - + // Group operations by type for efficient batch processing let creates: Vec<_> = pending.iter().filter_map(|op| { if let PendingOperation::Create(path) = op { Some(path) } else { None } }).collect(); - + let updates: Vec<_> = pending.iter().filter_map(|op| { if let PendingOperation::Update(id, changes) = op { Some((id, changes)) } else { None } }).collect(); - + // Batch insert entries if !creates.is_empty() { self.batch_create_entries(creates).await?; } - + // Batch update entries if !updates.is_empty() { self.batch_update_entries(updates).await?; } - + Ok(()) } } ``` #### Debouncing Strategy + ```rust struct WatcherDebouncer { pending_events: HashMap, @@ -525,7 +530,7 @@ impl WatcherDebouncer { async fn process_event(&mut self, event: WatcherEvent) -> Option { let path = event.primary_path()?.clone(); let now = Instant::now(); - + // Check if we have a recent event for this path if let Some((_, last_time)) = self.pending_events.get(&path) { if now.duration_since(*last_time) < self.debounce_duration { @@ -534,7 +539,7 @@ impl WatcherDebouncer { return None; // Event is debounced } } - + // Event should be processed self.pending_events.insert(path, (event.clone(), now)); Some(event) @@ -542,33 +547,38 @@ impl WatcherDebouncer { } ``` -## Benefits of Core-New Integration +## Benefits of core Integration + +### 1. **Immediate Database Consistency** -### 1. **Immediate Database Consistency** - File system changes immediately reflected in database (like original) - Entry + UserMetadata records created synchronously - Content identity generated on-the-fly when possible - Real-time consistency between file system and database state ### 2. **True VDFS Support** + - SdPath enables cross-device file operations - UserMetadata survives file moves/renames - ContentIdentity provides global deduplication - Cross-device operations work seamlessly ### 3. **Separated Concerns** + - Core database operations happen immediately (critical path) - Heavy operations (thumbnails, media extraction) spawn in background - Directory scanning uses job system for complex indexing operations - Performance-critical path remains fast and responsive ### 4. **Enhanced Reliability** + - Follows proven original architecture patterns - Atomic database transactions prevent partial states - Platform-specific optimizations for edge cases - Graceful degradation when background tasks fail ### 5. **Better Performance** + - Direct database operations are faster than job overhead - Smart debouncing prevents duplicate work - Background tasks don't block file system event processing @@ -577,18 +587,20 @@ impl WatcherDebouncer { ## Future Enhancements ### 1. **Conflict Resolution** + When the same file is modified on multiple devices: + ```rust async fn resolve_content_conflict( - entry_a: &Entry, + entry_a: &Entry, entry_b: &Entry ) -> ConflictResolution { if entry_a.content_id == entry_b.content_id { return ConflictResolution::NoConflict; } - + // User choice, timestamp-based, or content-aware resolution - ConflictResolution::UserChoice { + ConflictResolution::UserChoice { options: vec![entry_a.clone(), entry_b.clone()], suggested: suggest_resolution(entry_a, entry_b).await, } @@ -596,7 +608,9 @@ async fn resolve_content_conflict( ``` ### 2. **Smart Indexing** + Machine learning to predict which files should be indexed: + ```rust async fn should_index_content_ml(entry: &Entry) -> bool { let features = extract_features(entry); @@ -605,7 +619,9 @@ async fn should_index_content_ml(entry: &Entry) -> bool { ``` ### 3. **Version History** + Track file content changes over time: + ```rust struct ContentVersion { id: Uuid, @@ -616,4 +632,4 @@ struct ContentVersion { } ``` -This design provides a robust foundation for real-time file system monitoring while maintaining the flexibility and performance characteristics of the core-new architecture. \ No newline at end of file +This design provides a robust foundation for real-time file system monitoring while maintaining the flexibility and performance characteristics of the core architecture. diff --git a/core-new/docs/design/WHITEPAPER_IMPL_ROADMAP.md b/docs/design/WHITEPAPER_IMPL_ROADMAP.md similarity index 100% rename from core-new/docs/design/WHITEPAPER_IMPL_ROADMAP.md rename to docs/design/WHITEPAPER_IMPL_ROADMAP.md diff --git a/core-new/docs/design/cli-output-refactor.md b/docs/design/cli-output-refactor.md similarity index 100% rename from core-new/docs/design/cli-output-refactor.md rename to docs/design/cli-output-refactor.md diff --git a/core-new/docs/design/frontend_graphql_usage.tsx b/docs/design/frontend_graphql_usage.tsx similarity index 100% rename from core-new/docs/design/frontend_graphql_usage.tsx rename to docs/design/frontend_graphql_usage.tsx diff --git a/core-new/docs/design/indexer-scope-upgrade.md b/docs/design/indexer-scope-upgrade.md similarity index 100% rename from core-new/docs/design/indexer-scope-upgrade.md rename to docs/design/indexer-scope-upgrade.md diff --git a/core-new/docs/design/landing-page-idea.md b/docs/design/landing-page-idea.md similarity index 100% rename from core-new/docs/design/landing-page-idea.md rename to docs/design/landing-page-idea.md diff --git a/core-new/docs/design/networking_implementation_summary.md b/docs/design/networking_implementation_summary.md similarity index 100% rename from core-new/docs/design/networking_implementation_summary.md rename to docs/design/networking_implementation_summary.md diff --git a/core-new/docs/history.md b/docs/history.md similarity index 100% rename from core-new/docs/history.md rename to docs/history.md diff --git a/core-new/docs/philosophy.md b/docs/philosophy.md similarity index 100% rename from core-new/docs/philosophy.md rename to docs/philosophy.md diff --git a/core-new/docs/roadmap.md b/docs/roadmap.md similarity index 100% rename from core-new/docs/roadmap.md rename to docs/roadmap.md diff --git a/core-new/docs/whitepaper.md b/docs/whitepaper.md similarity index 100% rename from core-new/docs/whitepaper.md rename to docs/whitepaper.md diff --git a/error.txt b/error.txt deleted file mode 100644 index 03bf81d67..000000000 --- a/error.txt +++ /dev/null @@ -1,1024 +0,0 @@ - -> @sd/monorepo@ tauri /home/rocky/spacedrive -> pnpm desktop tauri "dev" - - -> @sd/monorepo@ desktop /home/rocky/spacedrive -> pnpm --filter @sd/desktop -- "tauri" "dev" - - -> @sd/desktop@ tauri /home/rocky/spacedrive/apps/desktop -> pnpm --filter @sd/scripts -- tauri "dev" - - -> @sd/scripts@ tauri /home/rocky/spacedrive/scripts -> node tauri.mjs "dev" - - -> @sd/desktop@ dev /home/rocky/spacedrive/apps/desktop -> vite dev - - - VITE v5.4.9 ready in 4536 ms - - ➜ Local: http://localhost:8001/ - ➜ Network: use --host to expose - -> @sd/monorepo@ tauri /home/rocky/spacedrive -> pnpm desktop tauri "dev" - - -> @sd/monorepo@ desktop /home/rocky/spacedrive -> pnpm --filter @sd/desktop -- "tauri" "dev" - - -> @sd/desktop@ tauri /home/rocky/spacedrive/apps/desktop -> pnpm --filter @sd/scripts -- tauri "dev" - - -> @sd/scripts@ tauri /home/rocky/spacedrive/scripts -> node tauri.mjs "dev" - - -> @sd/desktop@ dev /home/rocky/spacedrive/apps/desktop -> vite dev - - - VITE v5.4.9 ready in 3029 ms - - ➜ Local: http://localhost:8001/ - ➜ Network: use --host to expose - 2025-02-21T01:14:58.603532Z  INFO sd_core: Starting core;, data_directory: /home/rocky/.local/share/spacedrive/dev - at core/src/lib.rs:93 - - 2025-02-21T01:14:58.605281Z DEBUG sd_core::util::version_manager: No migration required;, config: sd_core::node::config::NodeConfig - at core/src/util/version_manager.rs:212 - - 2025-02-21T01:14:58.605795Z DEBUG sd_core::location::manager: Starting location manager actor - at core/src/location/manager/mod.rs:216 - - 2025-02-21T01:14:58.606617Z  INFO sd_task_system::system: Task system online!, workers_count: 4 - at crates/task-system/src/system.rs:99 - - 2025-02-21T01:14:58.608432Z  INFO libp2p_swarm: local_peer_id: 12D3KooWSeemqchtKezrRf8txhkw12cst6Rsj9rDwQ2QoCLra4WA - at /home/rocky/.cargo/git/checkouts/rust-libp2p-5bf6f5083e224a9a/1024411/swarm/src/lib.rs:372 - - 2025-02-21T01:14:58.608843Z  INFO sd_core::p2p::manager: Setting quic ipv4 listener;, ipv4_port: None - at core/src/p2p/manager.rs:220 - - 2025-02-21T01:14:58.608922Z  INFO sd_core::p2p::manager: Setting quic ipv6 listener;, ipv6_port: None - at core/src/p2p/manager.rs:243 - - 2025-02-21T01:14:58.609027Z  INFO sd_core::p2p::manager: Node is now online listening;, remote_identity: +hmlI/BRfRgbkcT/wUR6eMJU1NuaZgefAEVPvwf7TN8, peer_id: Libp2pPeerId(PeerId("12D3KooWSeemqchtKezrRf8txhkw12cst6Rsj9rDwQ2QoCLra4WA")), addresses: [Listener { id: ListenerId(0), name: "libp2p-quic", addrs: {} }] - at core/src/p2p/manager.rs:104 - - 2025-02-21T01:14:58.609228Z DEBUG sd_core::volume::actor: Volume manager event_tx initialized - at core/src/volume/actor.rs:84 - - 2025-02-21T01:14:58.609273Z DEBUG sd_core::volume::actor: Volume manager state initialized - at core/src/volume/actor.rs:87 - - 2025-02-21T01:14:58.696125Z  INFO quaint::pooled: Starting a sqlite pool with 1 connections. - at /home/rocky/.cargo/git/checkouts/prisma-engines-27ace036a11c2dbf/9a8a66a/quaint/src/pooled.rs:342 - in sd_core::library::manager::load with library_id=01951a96-a285-7a62-8412-2f0b6860915c db_path=/home/rocky/.local/share/spacedrive/dev/libraries/01951a96-a285-7a62-8412-2f0b6860915c.db config_path=/home/rocky/.local/share/spacedrive/dev/libraries/01951a96-a285-7a62-8412-2f0b6860915c.sdlibrary should_seed=true - - 2025-02-21T01:14:58.704182Z  INFO schema_core::commands::apply_migrations: Analysis run in 3ms, analysis_duration_ms: 3 - at /home/rocky/.cargo/git/checkouts/prisma-engines-27ace036a11c2dbf/9a8a66a/schema-engine/core/src/commands/apply_migrations.rs:44 - - 2025-02-21T01:14:58.732697Z  INFO sd_core::library::manager::pragmas: Set journal mode to WAL - at core/src/library/manager/pragmas.rs:41 - in sd_core::library::manager::load with library_id=01951a96-a285-7a62-8412-2f0b6860915c db_path=/home/rocky/.local/share/spacedrive/dev/libraries/01951a96-a285-7a62-8412-2f0b6860915c.db config_path=/home/rocky/.local/share/spacedrive/dev/libraries/01951a96-a285-7a62-8412-2f0b6860915c.sdlibrary should_seed=true - - 2025-02-21T01:14:58.732947Z  INFO sd_core::library::manager::pragmas: Set synchronous to NORMAL - at core/src/library/manager/pragmas.rs:41 - in sd_core::library::manager::load with library_id=01951a96-a285-7a62-8412-2f0b6860915c db_path=/home/rocky/.local/share/spacedrive/dev/libraries/01951a96-a285-7a62-8412-2f0b6860915c.db config_path=/home/rocky/.local/share/spacedrive/dev/libraries/01951a96-a285-7a62-8412-2f0b6860915c.sdlibrary should_seed=true - - 2025-02-21T01:14:58.733200Z  INFO sd_core::library::manager::pragmas: Set mmap_size to 512MB - at core/src/library/manager/pragmas.rs:41 - in sd_core::library::manager::load with library_id=01951a96-a285-7a62-8412-2f0b6860915c db_path=/home/rocky/.local/share/spacedrive/dev/libraries/01951a96-a285-7a62-8412-2f0b6860915c.db config_path=/home/rocky/.local/share/spacedrive/dev/libraries/01951a96-a285-7a62-8412-2f0b6860915c.sdlibrary should_seed=true - - 2025-02-21T01:14:58.733431Z  INFO sd_core::library::manager::pragmas: Set journal size limit to 64MB - at core/src/library/manager/pragmas.rs:41 - in sd_core::library::manager::load with library_id=01951a96-a285-7a62-8412-2f0b6860915c db_path=/home/rocky/.local/share/spacedrive/dev/libraries/01951a96-a285-7a62-8412-2f0b6860915c.db config_path=/home/rocky/.local/share/spacedrive/dev/libraries/01951a96-a285-7a62-8412-2f0b6860915c.sdlibrary should_seed=true - - 2025-02-21T01:14:58.733595Z  INFO sd_core::library::manager::pragmas: Set cache size to 10k pages - at core/src/library/manager/pragmas.rs:41 - in sd_core::library::manager::load with library_id=01951a96-a285-7a62-8412-2f0b6860915c db_path=/home/rocky/.local/share/spacedrive/dev/libraries/01951a96-a285-7a62-8412-2f0b6860915c.db config_path=/home/rocky/.local/share/spacedrive/dev/libraries/01951a96-a285-7a62-8412-2f0b6860915c.sdlibrary should_seed=true - - 2025-02-21T01:14:58.733808Z DEBUG sd_core::library::manager: Created sync partial index, model_name: "Device" - at core/src/library/manager/mod.rs:666 - in sd_core::library::manager::load with library_id=01951a96-a285-7a62-8412-2f0b6860915c db_path=/home/rocky/.local/share/spacedrive/dev/libraries/01951a96-a285-7a62-8412-2f0b6860915c.db config_path=/home/rocky/.local/share/spacedrive/dev/libraries/01951a96-a285-7a62-8412-2f0b6860915c.sdlibrary should_seed=true - - 2025-02-21T01:14:58.733951Z DEBUG sd_core::library::manager: Created sync partial index, model_name: "Volume" - at core/src/library/manager/mod.rs:666 - in sd_core::library::manager::load with library_id=01951a96-a285-7a62-8412-2f0b6860915c db_path=/home/rocky/.local/share/spacedrive/dev/libraries/01951a96-a285-7a62-8412-2f0b6860915c.db config_path=/home/rocky/.local/share/spacedrive/dev/libraries/01951a96-a285-7a62-8412-2f0b6860915c.sdlibrary should_seed=true - - 2025-02-21T01:14:58.734095Z DEBUG sd_core::library::manager: Created sync partial index, model_name: "Tag" - at core/src/library/manager/mod.rs:666 - in sd_core::library::manager::load with library_id=01951a96-a285-7a62-8412-2f0b6860915c db_path=/home/rocky/.local/share/spacedrive/dev/libraries/01951a96-a285-7a62-8412-2f0b6860915c.db config_path=/home/rocky/.local/share/spacedrive/dev/libraries/01951a96-a285-7a62-8412-2f0b6860915c.sdlibrary should_seed=true - - 2025-02-21T01:14:58.734238Z DEBUG sd_core::library::manager: Created sync partial index, model_name: "Location" - at core/src/library/manager/mod.rs:666 - in sd_core::library::manager::load with library_id=01951a96-a285-7a62-8412-2f0b6860915c db_path=/home/rocky/.local/share/spacedrive/dev/libraries/01951a96-a285-7a62-8412-2f0b6860915c.db config_path=/home/rocky/.local/share/spacedrive/dev/libraries/01951a96-a285-7a62-8412-2f0b6860915c.sdlibrary should_seed=true - - 2025-02-21T01:14:58.734357Z DEBUG sd_core::library::manager: Created sync partial index, model_name: "Object" - at core/src/library/manager/mod.rs:666 - in sd_core::library::manager::load with library_id=01951a96-a285-7a62-8412-2f0b6860915c db_path=/home/rocky/.local/share/spacedrive/dev/libraries/01951a96-a285-7a62-8412-2f0b6860915c.db config_path=/home/rocky/.local/share/spacedrive/dev/libraries/01951a96-a285-7a62-8412-2f0b6860915c.sdlibrary should_seed=true - - 2025-02-21T01:14:58.734472Z DEBUG sd_core::library::manager: Created sync partial index, model_name: "Label" - at core/src/library/manager/mod.rs:666 - in sd_core::library::manager::load with library_id=01951a96-a285-7a62-8412-2f0b6860915c db_path=/home/rocky/.local/share/spacedrive/dev/libraries/01951a96-a285-7a62-8412-2f0b6860915c.db config_path=/home/rocky/.local/share/spacedrive/dev/libraries/01951a96-a285-7a62-8412-2f0b6860915c.sdlibrary should_seed=true - - 2025-02-21T01:14:58.734582Z DEBUG sd_core::library::manager: Created sync partial index, model_name: "ExifData" - at core/src/library/manager/mod.rs:666 - in sd_core::library::manager::load with library_id=01951a96-a285-7a62-8412-2f0b6860915c db_path=/home/rocky/.local/share/spacedrive/dev/libraries/01951a96-a285-7a62-8412-2f0b6860915c.db config_path=/home/rocky/.local/share/spacedrive/dev/libraries/01951a96-a285-7a62-8412-2f0b6860915c.sdlibrary should_seed=true - - 2025-02-21T01:14:58.734690Z DEBUG sd_core::library::manager: Created sync partial index, model_name: "FilePath" - at core/src/library/manager/mod.rs:666 - in sd_core::library::manager::load with library_id=01951a96-a285-7a62-8412-2f0b6860915c db_path=/home/rocky/.local/share/spacedrive/dev/libraries/01951a96-a285-7a62-8412-2f0b6860915c.db config_path=/home/rocky/.local/share/spacedrive/dev/libraries/01951a96-a285-7a62-8412-2f0b6860915c.sdlibrary should_seed=true - - 2025-02-21T01:14:58.734801Z DEBUG sd_core::library::manager: Created sync partial index, model_name: "TagOnObject" - at core/src/library/manager/mod.rs:666 - in sd_core::library::manager::load with library_id=01951a96-a285-7a62-8412-2f0b6860915c db_path=/home/rocky/.local/share/spacedrive/dev/libraries/01951a96-a285-7a62-8412-2f0b6860915c.db config_path=/home/rocky/.local/share/spacedrive/dev/libraries/01951a96-a285-7a62-8412-2f0b6860915c.sdlibrary should_seed=true - - 2025-02-21T01:14:58.734914Z DEBUG sd_core::library::manager: Created sync partial index, model_name: "LabelOnObject" - at core/src/library/manager/mod.rs:666 - in sd_core::library::manager::load with library_id=01951a96-a285-7a62-8412-2f0b6860915c db_path=/home/rocky/.local/share/spacedrive/dev/libraries/01951a96-a285-7a62-8412-2f0b6860915c.db config_path=/home/rocky/.local/share/spacedrive/dev/libraries/01951a96-a285-7a62-8412-2f0b6860915c.sdlibrary should_seed=true - - 2025-02-21T01:14:58.735549Z DEBUG sd_core::util::version_manager: No migration required;, config: sd_core::library::config::LibraryConfig - at core/src/util/version_manager.rs:212 - in sd_core::library::manager::load with library_id=01951a96-a285-7a62-8412-2f0b6860915c db_path=/home/rocky/.local/share/spacedrive/dev/libraries/01951a96-a285-7a62-8412-2f0b6860915c.db config_path=/home/rocky/.local/share/spacedrive/dev/libraries/01951a96-a285-7a62-8412-2f0b6860915c.sdlibrary should_seed=true - - 2025-02-21T01:14:58.751536Z DEBUG sd_core::location::manager::watcher: Stopping Location Manager event handler for location - at core/src/location/manager/watcher/mod.rs:280 - in sd_core::location::manager::watcher::location_watcher with location_id=1 library_id=01951a96-a285-7a62-8412-2f0b6860915c location_path=Some("/home/rocky/Documents") - - 2025-02-21T01:14:58.751650Z  INFO sd_core::location::manager::watcher: Location watcher gracefully shutdown - at core/src/location/manager/watcher/mod.rs:177 - in sd_core::location::manager::watcher::location_watcher with location_id=1 library_id=01951a96-a285-7a62-8412-2f0b6860915c location_path=Some("/home/rocky/Documents") - - 2025-02-21T01:14:58.752587Z DEBUG sd_core::location::manager::watcher: Stopping Location Manager event handler for location - at core/src/location/manager/watcher/mod.rs:280 - in sd_core::location::manager::watcher::location_watcher with location_id=2 library_id=01951a96-a285-7a62-8412-2f0b6860915c location_path=Some("/home/rocky/Desktop") - - 2025-02-21T01:14:58.752686Z  INFO sd_core::location::manager::watcher: Location watcher gracefully shutdown - at core/src/location/manager/watcher/mod.rs:177 - in sd_core::location::manager::watcher::location_watcher with location_id=2 library_id=01951a96-a285-7a62-8412-2f0b6860915c location_path=Some("/home/rocky/Desktop") - - 2025-02-21T01:14:58.753516Z DEBUG sd_core::location::manager::watcher: Stopping Location Manager event handler for location - at core/src/location/manager/watcher/mod.rs:280 - in sd_core::location::manager::watcher::location_watcher with location_id=3 library_id=01951a96-a285-7a62-8412-2f0b6860915c location_path=Some("/home/rocky/Downloads") - - 2025-02-21T01:14:58.753608Z  INFO sd_core::location::manager::watcher: Location watcher gracefully shutdown - at core/src/location/manager/watcher/mod.rs:177 - in sd_core::location::manager::watcher::location_watcher with location_id=3 library_id=01951a96-a285-7a62-8412-2f0b6860915c location_path=Some("/home/rocky/Downloads") - - 2025-02-21T01:14:58.754227Z  INFO sd_core::volume::actor: Volume manager actor started - at core/src/volume/actor.rs:102 - - 2025-02-21T01:14:58.754323Z DEBUG sd_core::volume::actor: Starting volume event monitoring - at core/src/volume/actor.rs:210 - - 2025-02-21T01:14:58.881797Z DEBUG sd_core::volume::state: Registering volume /dev/vda1 with fingerprint 1b2c2481f863e49762ab0f05d523a621d03aabc99c1fbc68d4087294e02f0b6e - at core/src/volume/state.rs:32 - - 2025-02-21T01:14:58.881939Z DEBUG sd_core::volume::state: Registering volume /dev/vda13 with fingerprint f127b20af31ffa9fcaf13ce5e285c651316b9657fb49b301e89370f952fa4fc8 - at core/src/volume/state.rs:32 - - 2025-02-21T01:14:58.881986Z DEBUG sd_core::volume::state: Registering volume /dev/vda15 with fingerprint 750c383c61c7879de9b391a5cd39e1bd64ed6777939fe17bd94a0f35f445c4e4 - at core/src/volume/state.rs:32 - - 2025-02-21T01:14:58.882059Z DEBUG sd_core::volume::actor: Volume event received: VolumeAdded(Volume { fingerprint: Some(VolumeFingerprint([27, 44, 36, 129, 248, 99, 228, 151, 98, 171, 15, 5, 213, 35, 166, 33, 208, 58, 171, 201, 156, 31, 188, 104, 212, 8, 114, 148, 224, 47, 11, 110])), id: None, pub_id: None, device_id: None, name: "/dev/vda1", mount_type: System, mount_point: "/", mount_points: ["/"], is_mounted: true, disk_type: Unknown, file_system: EXT4, read_only: true, error_status: None, read_speed_mbps: None, write_speed_mbps: None, total_bytes_capacity: 102915489792, total_bytes_available: 70432550912 }) - at core/src/volume/actor.rs:212 - - 2025-02-21T01:14:58.882100Z ERROR sd_core::volume::state: Failed to perform speed test for volume, e: Cancelled - at core/src/volume/state.rs:117 - - 2025-02-21T01:14:58.882118Z DEBUG sd_core::volume::state: Registering volume /dev/vda1 with fingerprint 1b2c2481f863e49762ab0f05d523a621d03aabc99c1fbc68d4087294e02f0b6e - at core/src/volume/state.rs:32 - - 2025-02-21T01:14:58.882102Z DEBUG sd_core::volume::speed: Starting speed test with config: SpeedTestConfig { file_size_mb: 10, timeout_secs: 30, emit_events: true } - at core/src/volume/speed.rs:108 - in sd_core::volume::speed::speed_test with volume_name=/dev/vda13 - - 2025-02-21T01:14:58.882129Z DEBUG sd_core::volume::speed: Starting speed test with config: SpeedTestConfig { file_size_mb: 10, timeout_secs: 30, emit_events: true } - at core/src/volume/speed.rs:108 - in sd_core::volume::speed::speed_test with volume_name=/dev/vda15 - - 2025-02-21T01:14:58.882159Z DEBUG sd_core::volume::actor: Volume event received: VolumeAdded(Volume { fingerprint: Some(VolumeFingerprint([241, 39, 178, 10, 243, 31, 250, 159, 202, 241, 60, 229, 226, 133, 198, 81, 49, 107, 150, 87, 251, 73, 179, 1, 232, 147, 112, 249, 82, 250, 79, 200])), id: None, pub_id: None, device_id: None, name: "/dev/vda13", mount_type: System, mount_point: "/boot", mount_points: ["/boot"], is_mounted: true, disk_type: Unknown, file_system: EXT4, read_only: false, error_status: None, read_speed_mbps: None, write_speed_mbps: None, total_bytes_capacity: 1036431360, total_bytes_available: 847003648 }) - at core/src/volume/actor.rs:212 - - 2025-02-21T01:14:58.882182Z DEBUG sd_core::volume::state: Registering volume /dev/vda13 with fingerprint f127b20af31ffa9fcaf13ce5e285c651316b9657fb49b301e89370f952fa4fc8 - at core/src/volume/state.rs:32 - - 2025-02-21T01:14:58.882205Z DEBUG sd_core::volume::actor: Volume event received: VolumeAdded(Volume { fingerprint: Some(VolumeFingerprint([117, 12, 56, 60, 97, 199, 135, 157, 233, 179, 145, 165, 205, 57, 225, 189, 100, 237, 103, 119, 147, 159, 225, 123, 217, 74, 15, 53, 244, 69, 196, 228])), id: None, pub_id: None, device_id: None, name: "/dev/vda15", mount_type: System, mount_point: "/boot/efi", mount_points: ["/boot/efi"], is_mounted: true, disk_type: Unknown, file_system: Other("VFAT"), read_only: false, error_status: None, read_speed_mbps: None, write_speed_mbps: None, total_bytes_capacity: 109395456, total_bytes_available: 103012352 }) - at core/src/volume/actor.rs:212 - - 2025-02-21T01:14:58.882233Z DEBUG sd_core::volume::state: Registering volume /dev/vda15 with fingerprint 750c383c61c7879de9b391a5cd39e1bd64ed6777939fe17bd94a0f35f445c4e4 - at core/src/volume/state.rs:32 - - 2025-02-21T01:14:58.883714Z  INFO sd_core::volume::actor: Volume manager actor initialized - at core/src/volume/actor.rs:201 - - 2025-02-21T01:14:58.883746Z DEBUG sd_core::volume::watcher: Starting volume watcher - at core/src/volume/watcher.rs:38 - - 2025-02-21T01:14:58.883868Z  WARN sd_core::volume::watcher: Failed to watch path /run/media: No such file or directory (os error 2) - at core/src/volume/watcher.rs:121 - - 2025-02-21T01:14:58.883914Z DEBUG sd_core_heavy_lifting::job_system: No pending jobs found on disk - at core/crates/heavy-lifting/src/job_system/mod.rs:304 - - 2025-02-21T01:14:58.884450Z  INFO sd_core: Spacedrive online! - at core/src/lib.rs:264 - - 2025-02-21T01:14:58.884876Z  INFO sd_desktop::tauri_plugins: Internal server listening on: http://127.0.0.1:38507 - at apps/desktop/src-tauri/src/tauri_plugins.rs:71 - -emitting event for Some(VolumeFingerprint([117, 12, 56, 60, 97, 199, 135, 157, 233, 179, 145, 165, 205, 57, 225, 189, 100, 237, 103, 119, 147, 159, 225, 123, 217, 74, 15, 53, 244, 69, 196, 228])) - 2025-02-21T01:14:58.920055Z DEBUG sd_core::volume::actor: Volume event received: VolumeSpeedTested { fingerprint: VolumeFingerprint([117, 12, 56, 60, 97, 199, 135, 157, 233, 179, 145, 165, 205, 57, 225, 189, 100, 237, 103, 119, 147, 159, 225, 123, 217, 74, 15, 53, 244, 69, 196, 228]), read_speed: 613, write_speed: 844 } - at core/src/volume/actor.rs:212 - - 2025-02-21T01:14:58.920200Z DEBUG sd_core::volume::speed: Speed test completed: SpeedTestResult { write_speed: 844.1648832980036, read_speed: 613.000176727951, duration: 30.0 } - at core/src/volume/speed.rs:149 - in sd_core::volume::speed::speed_test with volume_name=/dev/vda15 - -emitting event for Some(VolumeFingerprint([241, 39, 178, 10, 243, 31, 250, 159, 202, 241, 60, 229, 226, 133, 198, 81, 49, 107, 150, 87, 251, 73, 179, 1, 232, 147, 112, 249, 82, 250, 79, 200])) - 2025-02-21T01:14:58.923780Z DEBUG sd_core::volume::actor: Volume event received: VolumeSpeedTested { fingerprint: VolumeFingerprint([241, 39, 178, 10, 243, 31, 250, 159, 202, 241, 60, 229, 226, 133, 198, 81, 49, 107, 150, 87, 251, 73, 179, 1, 232, 147, 112, 249, 82, 250, 79, 200]), read_speed: 562, write_speed: 825 } - at core/src/volume/actor.rs:212 - - 2025-02-21T01:14:58.923888Z ERROR sd_core::volume::speed: Failed to remove test file: No such file or directory (os error 2) - at core/src/volume/speed.rs:79 - in sd_core::volume::speed::speed_test with volume_name=/dev/vda13 - - 2025-02-21T01:14:58.923949Z DEBUG sd_core::volume::speed: Speed test completed: SpeedTestResult { write_speed: 825.351360327605, read_speed: 562.1557323160564, duration: 30.0 } - at core/src/volume/speed.rs:149 - in sd_core::volume::speed::speed_test with volume_name=/dev/vda13 - - 2025-02-21T01:14:59.006635Z  INFO sd_core::p2p::manager: Updated p2p relay configuration successfully. - at core/src/p2p/manager.rs:157 - - - ⚡ Million.js 2.6.4 - - Tip: enable { mute: true } to disable info logs - - Hotline: https://million.dev/hotline - - ⚡ now renders ~17% faster - ⚡ now renders ~25% faster - ⚡ now renders ~43% faster - ⚡ now renders ~11% faster - ⚡ now renders ~11% faster - ⚡ now renders ~43% faster - ⚡ now renders ~25% faster -Window did not emit `app_ready` event fast enough. Showing window... - ⚡

- -

- {t('app_crashed')} -

-

- {t('app_crashed_description')} -

- 
-				{message}
-
- {submessage && 
{submessage}
} -
- {reloadBtn && ( - - )} - {telemetryLevelPreference !== 'none' && ( - - )} - {platform.openLogsDir && ( - - )} - - {(errorsThatRequireACoreReset.includes(message) || - message.startsWith('NodeError::FailedToInitializeConfig') || - message.startsWith('failed to initialize library manager')) && ( -
-

- {t('reset_to_continue')} -

-

{t('reset_warning')}

- -
- )} -
-
- ); -} - -export const BetterErrorBoundary = ({ - children, - FallbackComponent, - ...props -}: PropsWithChildren) => { - useEffect(() => { - const id = setTimeout( - () => localStorage.removeItem(RENDERING_ERROR_LOCAL_STORAGE_KEY), - 1000 - ); - - return () => clearTimeout(id); - }, []); - - return ( - - {children} - - ); -}; diff --git a/interface/RoutingContext.tsx b/interface/RoutingContext.tsx deleted file mode 100644 index 54547ef58..000000000 --- a/interface/RoutingContext.tsx +++ /dev/null @@ -1,30 +0,0 @@ -import { type Router } from '@remix-run/router'; -import { createContext, useContext } from 'react'; - -import { createRoutes } from './app'; - -export const RoutingContext = createContext<{ - visible: boolean; - currentIndex: number; - tabId: string; - maxIndex: number; - routes: ReturnType; -} | null>(null); - -// We split this into a different context because we don't want to trigger the hook unnecessarily -export const RouterContext = createContext(null); - -export function useRoutingContext() { - const ctx = useContext(RoutingContext); - - if (!ctx) throw new Error('useRoutingContext must be used within a RoutingContext.Provider'); - - return ctx; -} - -export function useRouter() { - const ctx = useContext(RouterContext); - if (!ctx) throw new Error('useRouter must be used within a RouterContext.Provider'); - - return ctx; -} diff --git a/interface/TabsContext.tsx b/interface/TabsContext.tsx deleted file mode 100644 index 0e0ac68a5..000000000 --- a/interface/TabsContext.tsx +++ /dev/null @@ -1,18 +0,0 @@ -import { createContext, useContext } from 'react'; - -import { Router } from './'; - -export const TabsContext = createContext<{ - tabIndex: number; - setTabIndex: (i: number) => void; - tabs: { router: Router; title: string }[]; - createTab(redirect?: { pathname: string; search: string | undefined }): void; - removeTab(index: number): void; - duplicateTab(): void; -} | null>(null); - -export function useTabsContext() { - const ctx = useContext(TabsContext); - - return ctx; -} diff --git a/interface/app/$libraryId/404.tsx b/interface/app/$libraryId/404.tsx deleted file mode 100644 index 6ac16cc5c..000000000 --- a/interface/app/$libraryId/404.tsx +++ /dev/null @@ -1,29 +0,0 @@ -import { useNavigate } from 'react-router'; -import { Button } from '@sd/ui'; -import { useLocale } from '~/hooks'; - -export const Component = () => { - const navigate = useNavigate(); - - const { t } = useLocale(); - - return ( -
-
-

Error: 404

-

There's nothing here.

-

- Its likely that this page has not been built yet, if so we're on it! -

-
- -
-
-
- ); -}; diff --git a/interface/app/$libraryId/Explorer/Context.tsx b/interface/app/$libraryId/Explorer/Context.tsx deleted file mode 100644 index a7bcb5484..000000000 --- a/interface/app/$libraryId/Explorer/Context.tsx +++ /dev/null @@ -1,29 +0,0 @@ -import { ContextType, createContext, PropsWithChildren, useContext } from 'react'; -import { type Ordering } from '@sd/client'; - -import { UseExplorer } from './useExplorer'; - -/** - * Context that must wrap anything to do with the explorer. - * This includes explorer views, the inspector, and top bar items. - */ -const ExplorerContext = createContext | null>(null); - -type ExplorerContext = NonNullable>; - -export const useExplorerContext = ( - { suspense }: { suspense?: T } = { suspense: true as T } -) => { - const ctx = useContext(ExplorerContext); - - if (suspense && ctx === null) throw new Error('ExplorerContext.Provider not found!'); - - return ctx as T extends true ? ExplorerContext : ExplorerContext | undefined; -}; - -export const ExplorerContextProvider = >({ - explorer, - children -}: PropsWithChildren<{ - explorer: TExplorer; -}>) => {children}; diff --git a/interface/app/$libraryId/Explorer/ContextMenu/AssignTagMenuItems.tsx b/interface/app/$libraryId/Explorer/ContextMenu/AssignTagMenuItems.tsx deleted file mode 100644 index c4a826436..000000000 --- a/interface/app/$libraryId/Explorer/ContextMenu/AssignTagMenuItems.tsx +++ /dev/null @@ -1,258 +0,0 @@ -import { Plus } from '@phosphor-icons/react'; -import { useQueryClient } from '@tanstack/react-query'; -import { useVirtualizer } from '@tanstack/react-virtual'; -import clsx from 'clsx'; -import { RefObject, useMemo, useRef } from 'react'; -import { ErrorBoundary } from 'react-error-boundary'; -import { ExplorerItem, useLibraryQuery } from '@sd/client'; -import { Button, dialogManager, ModifierKeys, tw } from '@sd/ui'; -import CreateDialog, { - AssignTagItems, - useAssignItemsToTag -} from '~/app/$libraryId/settings/library/tags/CreateDialog'; -import { Menu } from '~/components/Menu'; -import { useLocale, useOperatingSystem } from '~/hooks'; -import { useScrolled } from '~/hooks/useScrolled'; -import { keybindForOs } from '~/util/keybinds'; - -const EmptyContainer = tw.div`py-1 text-center text-xs text-ink-faint`; - -interface Props { - items: Array>; -} - -function useData({ items }: Props) { - const tags = useLibraryQuery(['tags.list'], { suspense: true }); - - // Map> - const tagsWithObjects = useLibraryQuery( - [ - 'tags.getWithObjects', - items - .map((item) => { - if (item.type === 'Path') return item.item.object?.id; - else if (item.type === 'Object') return item.item.id; - }) - .filter((item): item is number => item !== undefined) - ], - { suspense: true } - ); - - return { - tags: { - ...tags, - data: tags.data - }, - tagsWithObjects - }; -} - -export default (props: Props) => { - const ref = useRef(null); - const { isScrolled } = useScrolled(ref, 10); - - const { t } = useLocale(); - - const os = useOperatingSystem(); - const keybind = keybindForOs(os); - - const queryClient = useQueryClient(); - - return ( - <> - { - dialogManager.create((dp) => ); - }} - /> - - queryClient.invalidateQueries()} - fallbackRender={(props) => ( - - {t('failed_to_load_tags')} - - - )} - > - - - - ); -}; - -const Tags = ({ items, parentRef }: Props & { parentRef: RefObject }) => { - const { tags, tagsWithObjects } = useData({ items }); - - const { t } = useLocale(); - - // tags are sorted by assignment, and assigned tags are sorted by most recently assigned - const sortedTags = useMemo(() => { - if (!tags.data) return []; - - const assigned = []; - const unassigned = []; - - for (const tag of tags.data) { - if (tagsWithObjects.data?.[tag.id] === undefined) unassigned.push(tag); - else assigned.push(tag); - } - - if (tagsWithObjects.data) { - assigned.sort((a, b) => { - const aObjs = tagsWithObjects.data[a.id], - bObjs = tagsWithObjects.data[b.id]; - - function getMaxDate(data: typeof aObjs) { - if (!data) return null; - let max = null; - - for (const { date_created } of data) { - if (!date_created) continue; - - const date = new Date(date_created); - - if (!max) max = date; - else if (date > max) max = date; - } - - return max; - } - - const aMaxDate = getMaxDate(aObjs), - bMaxDate = getMaxDate(bObjs); - - if (!aMaxDate || !bMaxDate) { - if (aMaxDate && !bMaxDate) return 1; - else if (!aMaxDate && bMaxDate) return -1; - else return 0; - } else { - return Number(bMaxDate) - Number(aMaxDate); - } - }); - } - - return [...assigned, ...unassigned]; - }, [tags.data, tagsWithObjects.data]); - - const rowVirtualizer = useVirtualizer({ - count: sortedTags.length, - getScrollElement: () => parentRef.current, - estimateSize: () => 30, - paddingStart: 2 - }); - - const assignItemsToTag = useAssignItemsToTag(); - - return ( - <> - {sortedTags.length > 0 ? ( -
-
- {rowVirtualizer.getVirtualItems().map((virtualRow) => { - const tag = sortedTags[virtualRow.index]; - if (!tag) return null; - - const objectsWithTag = new Set( - tagsWithObjects.data?.[tag?.id]?.map((d) => d.object.id) - ); - - // only unassign if all objects have tag - // this is the same functionality as finder - const unassign = items.every((item) => { - if (item.type === 'Object') { - return objectsWithTag.has(item.item.id); - } else { - const { object } = item.item; - - if (!object) return false; - return objectsWithTag.has(object.id); - } - }); - - // TODO: UI to differentiate tag assigning when some objects have tag when no objects have tag - ENG-965 - - return ( - { - e.preventDefault(); - - await assignItemsToTag( - tag.id, - unassign - ? // use objects that already have tag - items.flatMap((item) => { - if ( - item.type === 'Object' || - item.type === 'Path' - ) { - return [item]; - } - - return []; - }) - : // use objects that don't have tag - items.flatMap( - (item) => { - if (item.type === 'Object') { - if ( - !objectsWithTag.has( - item.item.id - ) - ) - return [item]; - } else if (item.type === 'Path') { - return [item]; - } - - return []; - } - ), - unassign - ); - - tagsWithObjects.refetch(); - }} - > -
0 && - tag.color - ? tag.color - : 'transparent', - borderColor: tag.color || '#efefef' - }} - /> - {tag.name} - - ); - })} -
-
- ) : ( - {t('no_tags')} - )} - - ); -}; diff --git a/interface/app/$libraryId/Explorer/ContextMenu/ConditionalItem.tsx b/interface/app/$libraryId/Explorer/ContextMenu/ConditionalItem.tsx deleted file mode 100644 index 5074088d7..000000000 --- a/interface/app/$libraryId/Explorer/ContextMenu/ConditionalItem.tsx +++ /dev/null @@ -1,45 +0,0 @@ -import { ReactNode } from 'react'; - -type UseCondition = () => TProps | null; - -export class ConditionalItem { - // Named like a hook to please eslint - useCondition: UseCondition; - // Capital 'C' to please eslint + make rendering after destructuring easier - Component: React.FC; - - constructor(public args: { useCondition: UseCondition; Component: React.FC }) { - this.useCondition = args.useCondition; - this.Component = args.Component; - } -} - -export interface ConditionalGroupProps { - items: ConditionalItem[]; - children?: (children: ReactNode) => ReactNode; -} - -/** - * Takes an array of `ConditionalItem` and attempts to render them all, - * returning `null` if all conditions are `null`. - * - * @param items An array of `ConditionalItem` to render. - * @param children An optional render function that can be used to wrap the rendered items. - */ -export const Conditional = ({ items, children }: ConditionalGroupProps) => { - const itemConditions = items.map((item) => item.useCondition()); - - if (itemConditions.every((c) => c === null)) return null; - - const renderedItems = ( - <> - {itemConditions.map((props, i) => { - if (props === null) return null; - const { Component } = items[i]!; - return ; - })} - - ); - - return <>{children ? children(renderedItems) : renderedItems}; -}; diff --git a/interface/app/$libraryId/Explorer/ContextMenu/FilePath/CutCopyItems.tsx b/interface/app/$libraryId/Explorer/ContextMenu/FilePath/CutCopyItems.tsx deleted file mode 100644 index c7411c856..000000000 --- a/interface/app/$libraryId/Explorer/ContextMenu/FilePath/CutCopyItems.tsx +++ /dev/null @@ -1,56 +0,0 @@ -import { Copy, Scissors } from '@phosphor-icons/react'; -import { ContextMenu, ModifierKeys } from '@sd/ui'; -import { useLocale } from '~/hooks'; -import { useKeybindFactory } from '~/hooks/useKeybindFactory'; -import { isNonEmpty } from '~/util'; - -import { useExplorerContext } from '../../Context'; -import { useExplorerCopyPaste } from '../../hooks/useExplorerCopyPaste'; -import { ConditionalItem } from '../ConditionalItem'; -import { useContextMenuContext } from '../context'; - -import type {} from '@sd/client'; - -export const CutCopyItems = new ConditionalItem({ - useCondition: () => { - const { parent } = useExplorerContext(); - const { selectedFilePaths, selectedEphemeralPaths } = useContextMenuContext(); - - if ( - (parent?.type !== 'Location' && parent?.type !== 'Ephemeral') || - (!isNonEmpty(selectedFilePaths) && !isNonEmpty(selectedEphemeralPaths)) - ) - return null; - - return { parent, selectedFilePaths, selectedEphemeralPaths }; - }, - Component: () => { - const { t } = useLocale(); - const keybind = useKeybindFactory(); - const { copy, cut, duplicate } = useExplorerCopyPaste(); - - return ( - <> - - - - - - - ); - } -}); diff --git a/interface/app/$libraryId/Explorer/ContextMenu/FilePath/Items.tsx b/interface/app/$libraryId/Explorer/ContextMenu/FilePath/Items.tsx deleted file mode 100644 index 497c1d381..000000000 --- a/interface/app/$libraryId/Explorer/ContextMenu/FilePath/Items.tsx +++ /dev/null @@ -1,299 +0,0 @@ -import { Hash, Image, Package, Trash, TrashSimple } from '@phosphor-icons/react'; -import { libraryClient, useLibraryMutation } from '@sd/client'; -import { ContextMenu, dialogManager, ModifierKeys, toast } from '@sd/ui'; -import { Menu } from '~/components/Menu'; -import { useKeysMatcher, useLocale, useOperatingSystem } from '~/hooks'; -import { useKeybindFactory } from '~/hooks/useKeybindFactory'; -import { useQuickRescan } from '~/hooks/useQuickRescan'; -import { isNonEmpty } from '~/util'; - -import { useExplorerContext } from '../../Context'; -import { CopyAsPathBase } from '../../CopyAsPath'; -import DeleteDialog from '../../FilePath/DeleteDialog'; -// import EraseDialog from '../../FilePath/EraseDialog'; -import { ConditionalItem } from '../ConditionalItem'; -import { useContextMenuContext } from '../context'; - -export * from './CutCopyItems'; - -export const Delete = new ConditionalItem({ - useCondition: () => { - const { selectedFilePaths, selectedEphemeralPaths } = useContextMenuContext(); - - if (!isNonEmpty(selectedFilePaths) && !isNonEmpty(selectedEphemeralPaths)) return null; - - return { selectedFilePaths, selectedEphemeralPaths }; - }, - Component: ({ selectedFilePaths, selectedEphemeralPaths }) => { - const { t } = useLocale(); - const rescan = useQuickRescan(); - const os = useOperatingSystem(); - const dirCount = - selectedFilePaths.filter((p) => p.is_dir).length + - selectedEphemeralPaths.filter((p) => p.is_dir).length; - const fileCount = - selectedFilePaths.filter((p) => !p.is_dir).length + - selectedEphemeralPaths.filter((p) => !p.is_dir).length; - - const indexedArgs = - isNonEmpty(selectedFilePaths) && selectedFilePaths[0].location_id - ? { - locationId: selectedFilePaths[0].location_id, - rescan, - pathIds: selectedFilePaths.map((p) => p.id) - } - : undefined; - - const ephemeralArgs = isNonEmpty(selectedEphemeralPaths) - ? { - paths: selectedEphemeralPaths.map((p) => p.path) - } - : undefined; - const deleteKeybind = useKeysMatcher(['Meta', 'Backspace']); - - return ( - - dialogManager.create((dp) => ( - - )) - } - /> - ); - } -}); - -export const CopyAsPath = new ConditionalItem({ - useCondition: () => { - const { selectedFilePaths, selectedEphemeralPaths } = useContextMenuContext(); - if ( - !isNonEmpty(selectedFilePaths) || - selectedFilePaths.length > 1 || - !isNonEmpty(selectedEphemeralPaths) || - selectedEphemeralPaths.length > 1 || - (selectedFilePaths.length === 1 && selectedEphemeralPaths.length === 1) // should never happen - ) - return null; - - return { selectedFilePaths, selectedEphemeralPaths }; - }, - Component: ({ selectedFilePaths, selectedEphemeralPaths }) => { - if (selectedFilePaths.length === 1) { - return ( - libraryClient.query(['files.getPath', selectedFilePaths[0].id])} - /> - ); - } else if (selectedEphemeralPaths.length === 1) { - return selectedEphemeralPaths[0].path} />; - } - } -}); - -export const Compress = new ConditionalItem({ - useCondition: () => { - const { selectedFilePaths } = useContextMenuContext(); - if (!isNonEmpty(selectedFilePaths)) return null; - - return { selectedFilePaths }; - }, - Component: ({ selectedFilePaths: _ }) => { - const keybind = useKeybindFactory(); - const { t } = useLocale(); - - return ( - - ); - } -}); - -export const Crypto = new ConditionalItem({ - useCondition: () => { - const { selectedFilePaths } = useContextMenuContext(); - if (!isNonEmpty(selectedFilePaths)) return null; - - return { selectedFilePaths }; - }, - Component: ({ selectedFilePaths: _ }) => { - return ( - <> - {/* { - if (keyManagerUnlocked && hasMountedKeys) { - dialogManager.create((dp) => ( - - )); - } else if (!keyManagerUnlocked) { - showAlertDialog({ - title: 'Key manager locked', - value: 'The key manager is currently locked. Please unlock it and try again.' - }); - } else if (!hasMountedKeys) { - showAlertDialog({ - title: 'No mounted keys', - value: 'No mounted keys were found. Please mount a key and try again.' - }); - } - }} - /> */} - {/* should only be shown if the file is a valid spacedrive-encrypted file (preferably going from the magic bytes) */} - {/* { - if (keyManagerUnlocked) { - dialogManager.create((dp) => ( - - )); - } else { - showAlertDialog({ - title: 'Key manager locked', - value: 'The key manager is currently locked. Please unlock it and try again.' - }); - } - }} - /> */} - - ); - } -}); - -// export const SecureDelete = new ConditionalItem({ -// useCondition: () => { -// const { selectedFilePaths } = useContextMenuContext(); -// if (!isNonEmpty(selectedFilePaths)) return null; - -// const locationId = selectedFilePaths[0].location_id; -// if (locationId === null) return null; - -// return { locationId, selectedFilePaths }; -// }, -// Component: ({ locationId, selectedFilePaths }) => { -// const { t } = useLocale(); -// return ( -// -// dialogManager.create((dp) => ( -// -// )) -// } -// disabled -// /> -// ); -// } -// }); - -export const ParentFolderActions = new ConditionalItem({ - useCondition: () => { - const { parent } = useExplorerContext(); - - if (parent?.type !== 'Location') return null; - - return { parent }; - }, - Component: ({ parent }) => { - const { selectedFilePaths } = useContextMenuContext(); - - const fullRescan = useLibraryMutation('locations.fullRescan'); - const generateThumbnails = useLibraryMutation('jobs.generateThumbsForLocation'); - // const generateLabels = useLibraryMutation('jobs.generateLabelsForLocation'); - - const { t } = useLocale(); - - return ( - <> - { - try { - await fullRescan.mutateAsync({ - location_id: parent.location.id, - reidentify_objects: false - }); - } catch (error) { - toast.error({ - title: t('failed_to_rescan_location'), - body: t('error_message', { error }) - }); - } - }} - label={t('rescan_directory')} - icon={Package} - /> - { - try { - await generateThumbnails.mutateAsync({ - id: parent.location.id, - path: selectedFilePaths[0]?.materialized_path ?? '/', - regenerate: true - }); - } catch (error) { - toast.error({ - title: t('failed_to_generate_thumbnails'), - body: `Error: ${error}.` - }); - } - }} - label={t('regen_thumbnails')} - icon={Image} - /> - {/* { - try { - await generateLabels.mutateAsync({ - id: parent.location.id, - path: selectedFilePaths[0]?.materialized_path ?? '/', - regenerate: true - }); - } catch (error) { - toast.error({ - title: t('failed_to_generate_labels'), - body: `Error: ${error}.` - }); - } - }} - label={t('regen_labels')} - icon={Hash} - /> */} - - ); - } -}); diff --git a/interface/app/$libraryId/Explorer/ContextMenu/Object/Items.tsx b/interface/app/$libraryId/Explorer/ContextMenu/Object/Items.tsx deleted file mode 100644 index 502b806a1..000000000 --- a/interface/app/$libraryId/Explorer/ContextMenu/Object/Items.tsx +++ /dev/null @@ -1,112 +0,0 @@ -import { ArrowBendUpRight, TagSimple } from '@phosphor-icons/react'; -import { useMemo } from 'react'; -import { ExplorerItem, ObjectKind, useLibraryMutation, type ObjectKindEnum } from '@sd/client'; -import { ContextMenu, toast } from '@sd/ui'; -import { Menu } from '~/components/Menu'; -import { useLocale } from '~/hooks'; -import { isNonEmpty } from '~/util'; - -import AssignTagMenuItems from '../AssignTagMenuItems'; -import { ConditionalItem } from '../ConditionalItem'; -import { useContextMenuContext } from '../context'; - -export const RemoveFromRecents = new ConditionalItem({ - useCondition: () => { - const { selectedObjects } = useContextMenuContext(); - - if (!isNonEmpty(selectedObjects)) return null; - - return { selectedObjects }; - }, - - Component: ({ selectedObjects }) => { - const removeFromRecents = useLibraryMutation('files.removeAccessTime'); - - const { t } = useLocale(); - - return ( - { - try { - await removeFromRecents.mutateAsync( - selectedObjects.map((object) => object.id) - ); - } catch (error) { - toast.error({ - title: t('failed_to_remove_file_from_recents'), - body: t('error_message', { error }) - }); - } - }} - /> - ); - } -}); - -export const AssignTag = new ConditionalItem({ - useCondition: () => { - const { selectedItems } = useContextMenuContext(); - - const items = selectedItems - .map((item) => { - if (item.type === 'Object' || item.type === 'Path') return item; - }) - .filter( - (item): item is Extract => - item !== undefined - ); - - if (!isNonEmpty(items)) return null; - - return { items }; - }, - Component: ({ items }) => { - const { t } = useLocale(); - return ( - - - - ); - } -}); - -const ObjectConversions: Record = { - [ObjectKind.Image]: ['PNG', 'WebP', 'Gif'], - [ObjectKind.Video]: ['MP4', 'MOV', 'AVI'] -}; - -const ConvertableKinds = [ObjectKind.Image, ObjectKind.Video]; - -export const ConvertObject = new ConditionalItem({ - useCondition: () => { - const { selectedObjects } = useContextMenuContext(); - - const kinds = useMemo(() => { - const set = new Set(); - - for (const o of selectedObjects) { - if (o.kind === null || !ConvertableKinds.includes(o.kind)) break; - set.add(o.kind); - } - - return [...set]; - }, [selectedObjects]); - - if (!isNonEmpty(kinds) || kinds.length > 1) return null; - - const [kind] = kinds; - - return { kind }; - }, - Component: ({ kind }) => { - const { t } = useLocale(); - return ( - - {ObjectConversions[kind]?.map((ext) => ( - - ))} - - ); - } -}); diff --git a/interface/app/$libraryId/Explorer/ContextMenu/OpenWith.tsx b/interface/app/$libraryId/Explorer/ContextMenu/OpenWith.tsx deleted file mode 100644 index d11ba36bc..000000000 --- a/interface/app/$libraryId/Explorer/ContextMenu/OpenWith.tsx +++ /dev/null @@ -1,143 +0,0 @@ -import { useQuery } from '@tanstack/react-query'; -import { Suspense } from 'react'; -import { useLibraryContext } from '@sd/client'; -import { toast } from '@sd/ui'; -import { Menu } from '~/components/Menu'; -import { useLocale } from '~/hooks'; -import { OpenWithApplication, Platform, Result, usePlatform } from '~/util/Platform'; - -import { ConditionalItem } from './ConditionalItem'; -import { useContextMenuContext } from './context'; - -export default new ConditionalItem({ - useCondition: () => { - const { selectedFilePaths, selectedEphemeralPaths } = useContextMenuContext(); - const { - getFilePathOpenWithApps, - openFilePathWith, - getEphemeralFilesOpenWithApps, - openEphemeralFileWith - } = usePlatform(); - - if ( - !getFilePathOpenWithApps || - !openFilePathWith || - !getEphemeralFilesOpenWithApps || - !openEphemeralFileWith - ) - return null; - if (selectedFilePaths.some((p) => p.is_dir) || selectedEphemeralPaths.some((p) => p.is_dir)) - return null; - - return { - getFilePathOpenWithApps, - openFilePathWith, - getEphemeralFilesOpenWithApps, - openEphemeralFileWith - }; - }, - Component: ({ - getFilePathOpenWithApps, - openFilePathWith, - getEphemeralFilesOpenWithApps, - openEphemeralFileWith - }) => { - const { t } = useLocale(); - return ( - - - - - - ); - } -}); - -const Items = ({ - actions -}: { - actions: Required< - Pick< - Platform, - | 'getFilePathOpenWithApps' - | 'openFilePathWith' - | 'getEphemeralFilesOpenWithApps' - | 'openEphemeralFileWith' - > - >; -}) => { - const { selectedFilePaths, selectedEphemeralPaths } = useContextMenuContext(); - - const { library } = useLibraryContext(); - - const ids = selectedFilePaths.map((obj) => obj.id); - const paths = selectedEphemeralPaths.map((obj) => obj.path); - const { t } = useLocale(); - - const { data: apps } = useQuery({ - queryKey: ['openWith', ids, paths], - queryFn: async () => { - const handleError = (res: Result) => { - if (res?.status === 'error') { - toast.error('Failed to get applications capable to open file'); - if (res.error) console.error(res.error); - return []; - } - return res?.data; - }; - - return Promise.all([ - ids.length > 0 - ? actions.getFilePathOpenWithApps(library.uuid, ids).then(handleError) - : Promise.resolve([]), - paths.length > 0 - ? actions.getEphemeralFilesOpenWithApps(paths).then(handleError) - : Promise.resolve([]) - ]) - .then((res) => res.flat()) - .then((res) => res.sort((a, b) => a.name.localeCompare(b.name))); - }, - initialData: [] - }); - - return ( - <> - {apps.length > 0 ? ( - apps.map((data, index) => ( - { - try { - if (ids.length > 0) { - await actions.openFilePathWith( - library.uuid, - ids.map((id) => [id, data.url]) - ); - } - - if (paths.length > 0) { - await actions.openEphemeralFileWith( - paths.map((path) => [path, data.url]) - ); - } - } catch (e) { - toast.error(t('failed_to_open_file_with', { data: data.url })); - } - }} - > - {data.name} - - )) - ) : ( -

{t('no_apps_available')}

- )} - - ); -}; diff --git a/interface/app/$libraryId/Explorer/ContextMenu/SharedItems.tsx b/interface/app/$libraryId/Explorer/ContextMenu/SharedItems.tsx deleted file mode 100644 index 41a882e0e..000000000 --- a/interface/app/$libraryId/Explorer/ContextMenu/SharedItems.tsx +++ /dev/null @@ -1,236 +0,0 @@ -import { FileX, Share as ShareIcon } from '@phosphor-icons/react'; -import { useMemo } from 'react'; -import { useBridgeMutation, useDiscoveredPeers, useSelector } from '@sd/client'; -import { ContextMenu, ModifierKeys } from '@sd/ui'; -import { Menu } from '~/components/Menu'; -import { useLocale, useOperatingSystem } from '~/hooks'; -import { useKeybindFactory } from '~/hooks/useKeybindFactory'; -import { isNonEmpty } from '~/util'; -import { usePlatform, type Platform } from '~/util/Platform'; - -import { useExplorerContext } from '../Context'; -import { getQuickPreviewStore } from '../QuickPreview/store'; -import { RevealInNativeExplorerBase } from '../RevealInNativeExplorer'; -import { explorerStore } from '../store'; -import { getPaths } from '../useExplorerDnd'; -import { useViewItemDoubleClick } from '../View/ViewItem'; -import { Conditional, ConditionalItem } from './ConditionalItem'; -import { useContextMenuContext } from './context'; -import OpenWith from './OpenWith'; - -export const OpenOrDownload = new ConditionalItem({ - useCondition: () => { - const { selectedFilePaths, selectedEphemeralPaths } = useContextMenuContext(); - const { openFilePaths, openEphemeralFiles } = usePlatform(); - - if ( - !openFilePaths || - !openEphemeralFiles || - (!isNonEmpty(selectedFilePaths) && !isNonEmpty(selectedEphemeralPaths)) - ) - return null; - - return { openFilePaths, openEphemeralFiles, selectedFilePaths, selectedEphemeralPaths }; - }, - Component: () => { - const keybind = useKeybindFactory(); - const { platform } = usePlatform(); - const { doubleClick } = useViewItemDoubleClick(); - const os = useOperatingSystem(true); - - const { t } = useLocale(); - - if (platform === 'web') return ; - else - return ( - <> - doubleClick()} - /> - - - ); - } -}); - -export const OpenQuickView = () => { - const keybind = useKeybindFactory(); - const { t } = useLocale(); - - return ( - (getQuickPreviewStore().open = true)} - /> - ); -}; - -export const Details = new ConditionalItem({ - useCondition: () => { - const showInspector = useSelector(explorerStore, (s) => s.showInspector); - if (showInspector) return null; - - return {}; - }, - Component: () => { - const keybind = useKeybindFactory(); - const { t } = useLocale(); - - return ( - (explorerStore.showInspector = true)} - /> - ); - } -}); - -export const Rename = new ConditionalItem({ - useCondition: () => { - const { selectedItems } = useContextMenuContext(); - - const settings = useExplorerContext().useSettingsSnapshot(); - - if (settings.layoutMode === 'media' || selectedItems.length > 1) return null; - - return {}; - }, - Component: () => { - const keybind = useKeybindFactory(); - const os = useOperatingSystem(true); - const { t } = useLocale(); - - return ( - (explorerStore.isRenaming = true)} - /> - ); - } -}); - -export const RevealInNativeExplorer = new ConditionalItem({ - useCondition: () => { - const { selectedItems } = useContextMenuContext(); - - const items = useMemo(() => { - const array: Parameters>[1] = []; - - for (const item of selectedItems) { - switch (item.type) { - case 'Path': { - array.push({ - FilePath: { id: item.item.id } - }); - break; - } - case 'Object': { - // this isn't good but it's the current behaviour - const filePath = item.item.file_paths[0]; - if (filePath) - array.push({ - FilePath: { - id: filePath.id - } - }); - else return []; - break; - } - case 'Location': { - array.push({ - Location: { - id: item.item.id - } - }); - break; - } - case 'NonIndexedPath': { - array.push({ - Ephemeral: { - path: item.item.path - } - }); - break; - } - } - } - - return array; - }, [selectedItems]); - - if (!isNonEmpty(items)) return null; - - return { items }; - }, - Component: ({ items }) => -}); - -export const Deselect = new ConditionalItem({ - useCondition: () => { - const cutCopyState = useSelector(explorerStore, (s) => s.cutCopyState); - - if (cutCopyState.type === 'Idle') return null; - - return {}; - }, - Component: () => { - const { t } = useLocale(); - return ( - { - explorerStore.cutCopyState = { - type: 'Idle' - }; - }} - /> - ); - } -}); - -export const Share = () => { - const { t } = useLocale(); - - return ( - - - - - - ); -}; - -const SpacedropNodes = () => { - const { t } = useLocale(); - const explorer = useExplorerContext(); - const discoveredPeers = useDiscoveredPeers(); - - const spacedrop = useBridgeMutation('p2p.spacedrop'); - - if (discoveredPeers.size === 0) { - return

{t('no_nodes_found')}

; - } - - return Array.from(discoveredPeers).map(([id, peer]) => ( - { - spacedrop.mutateAsync({ - identity: id, - file_path: await getPaths([...explorer.selectedItems]) - }); - }} - /> - )); -}; diff --git a/interface/app/$libraryId/Explorer/ContextMenu/context.tsx b/interface/app/$libraryId/Explorer/ContextMenu/context.tsx deleted file mode 100644 index 9d8b5e39c..000000000 --- a/interface/app/$libraryId/Explorer/ContextMenu/context.tsx +++ /dev/null @@ -1,43 +0,0 @@ -import { createContext, PropsWithChildren, useContext } from 'react'; -import { - ExplorerItem, - FilePath, - NonIndexedPathItem, - Object, - useItemsAsEphemeralPaths, - useItemsAsFilePaths, - useItemsAsObjects -} from '@sd/client'; -import { NonEmptyArray } from '~/util'; - -const ContextMenuContext = createContext<{ - selectedItems: NonEmptyArray; - selectedFilePaths: FilePath[]; - selectedObjects: Object[]; - selectedEphemeralPaths: NonIndexedPathItem[]; -} | null>(null); - -export const ContextMenuContextProvider = ({ - selectedItems, - children -}: PropsWithChildren<{ - selectedItems: NonEmptyArray; -}>) => { - const selectedFilePaths = useItemsAsFilePaths(selectedItems); - const selectedObjects = useItemsAsObjects(selectedItems); - const selectedEphemeralPaths = useItemsAsEphemeralPaths(selectedItems); - - return ( - - {children} - - ); -}; - -export const useContextMenuContext = () => { - const context = useContext(ContextMenuContext); - if (!context) throw new Error('ContextMenuContext.Provider not found'); - return context; -}; diff --git a/interface/app/$libraryId/Explorer/ContextMenu/index.tsx b/interface/app/$libraryId/Explorer/ContextMenu/index.tsx deleted file mode 100644 index 1cc1ea8c1..000000000 --- a/interface/app/$libraryId/Explorer/ContextMenu/index.tsx +++ /dev/null @@ -1,95 +0,0 @@ -import { Plus } from '@phosphor-icons/react'; -import { useMemo, type PropsWithChildren } from 'react'; -import { ExplorerItem } from '@sd/client'; -import { ContextMenu } from '@sd/ui'; -import { useLocale } from '~/hooks'; -import { isNonEmpty } from '~/util'; - -import { useExplorerContext } from '../Context'; -import { Conditional, type ConditionalGroupProps } from './ConditionalItem'; -import { ContextMenuContextProvider } from './context'; -import * as FilePathItems from './FilePath/Items'; -import * as ObjectItems from './Object/Items'; -import * as SharedItems from './SharedItems'; - -export * as FilePathItems from './FilePath/Items'; -export * as ObjectItems from './Object/Items'; -export * as SharedItems from './SharedItems'; - -const Items = ({ children }: PropsWithChildren) => { - const { t } = useLocale(); - return ( - <> - - - - - - - - {children} - - - - - - - - {(items) => ( - - {items} - - )} - - - - - ); -}; - -export default (props: PropsWithChildren<{ items?: ExplorerItem[]; custom?: boolean }>) => { - const explorer = useExplorerContext(); - - const selectedItems = useMemo( - () => props.items || [...explorer.selectedItems], - [explorer.selectedItems, props.items] - ); - - if (!isNonEmpty(selectedItems)) return null; - - return ( - - {props.custom ? <>{props.children} : {props.children}} - - ); -}; - -/** - * A `Conditional` that inserts a `` above its items. - */ -export const SeparatedConditional = ({ items, children }: ConditionalGroupProps) => ( - - {(c) => ( - <> - - {children ? children(c) : c} - - )} - -); diff --git a/interface/app/$libraryId/Explorer/CopyAsPath.tsx b/interface/app/$libraryId/Explorer/CopyAsPath.tsx deleted file mode 100644 index 9b15b6e08..000000000 --- a/interface/app/$libraryId/Explorer/CopyAsPath.tsx +++ /dev/null @@ -1,36 +0,0 @@ -import { ClipboardText } from '@phosphor-icons/react'; -import { toast } from '@sd/ui'; -import { Menu } from '~/components/Menu'; -import { useLocale } from '~/hooks'; - -export const CopyAsPathBase = ( - props: { path: string } | { getPath: () => Promise } -) => { - const { t } = useLocale(); - - return ( - { - try { - const path = 'path' in props ? props.path : await props.getPath(); - { - /* 'path' in props - ? props.path - : await libraryClient.query(['files.getPath', props.filePath.id]); */ - } - - if (path == null) throw new Error('No file path available'); - - navigator.clipboard.writeText(path); - } catch (error) { - toast.error({ - title: t('failed_to_copy_file_path'), - body: t('error_message', { error }) - }); - } - }} - /> - ); -}; diff --git a/interface/app/$libraryId/Explorer/DismissibleNotice.tsx b/interface/app/$libraryId/Explorer/DismissibleNotice.tsx deleted file mode 100644 index 7e93accfa..000000000 --- a/interface/app/$libraryId/Explorer/DismissibleNotice.tsx +++ /dev/null @@ -1,79 +0,0 @@ -import { ReactNode } from 'react'; -import { ExplorerLayout } from '@sd/client'; -import i18n from '~/app/I18n'; -import { Icon } from '~/components'; -import DismissibleNotice from '~/components/DismissibleNotice'; -import { useLocale } from '~/hooks'; -import { dismissibleNoticeStore } from '~/hooks/useDismissibleNoticeStore'; - -import { useExplorerContext } from './Context'; - -const MediaViewIcon = () => { - return ( -
- - -
- ); -}; - -const CollectionIcon = () => { - return ( -
- -
- ); -}; - -interface Notice { - key: keyof typeof dismissibleNoticeStore; - title: string; - description: string; - icon: ReactNode; -} - -const notices = { - grid: { - key: 'gridView', - title: i18n.t('grid_view'), - description: i18n.t('grid_view_notice_description'), - icon: - }, - list: { - key: 'listView', - title: i18n.t('list_view'), - description: i18n.t('list_view_notice_description'), - icon: - }, - media: { - key: 'mediaView', - title: i18n.t('media_view'), - description: i18n.t('media_view_notice_description'), - icon: - } - // columns: undefined -} satisfies Record; - -export default () => { - const { t } = useLocale(); - - const settings = useExplorerContext().useSettingsSnapshot(); - - const notice = notices[settings.layoutMode]; - - if (!notice) return null; - - return ( - {t('meet_title', { title: notice.title })}} - icon={notice.icon} - description={notice.description} - className="m-5" - storageKey={notice.key} - onContextMenu={(e) => e.preventDefault()} - /> - ); -}; diff --git a/interface/app/$libraryId/Explorer/DragOverlay.tsx b/interface/app/$libraryId/Explorer/DragOverlay.tsx deleted file mode 100644 index 4d5c087b8..000000000 --- a/interface/app/$libraryId/Explorer/DragOverlay.tsx +++ /dev/null @@ -1,96 +0,0 @@ -import { DragOverlay as DragOverlayPrimitive, type ClientRect, type Modifier } from '@dnd-kit/core'; -import { getEventCoordinates } from '@dnd-kit/utilities'; -import clsx from 'clsx'; -import { memo, useEffect, useRef } from 'react'; -import { ExplorerItem, useSelector } from '@sd/client'; -import { useIsDark } from '~/hooks'; - -import { FileThumb } from './FilePath/Thumb'; -import { explorerStore } from './store'; -import { RenamableItemText } from './View/RenamableItemText'; - -const useSnapToCursorModifier = () => { - const drag = useSelector(explorerStore, (s) => s.drag); - - const initialRect = useRef(null); - - const modifier: Modifier = ({ activatorEvent, activeNodeRect, transform }) => { - if (!activeNodeRect || !activatorEvent) return transform; - - const activatorCoordinates = getEventCoordinates(activatorEvent); - if (!activatorCoordinates) return transform; - - const rect = initialRect.current ?? activeNodeRect; - - if (!initialRect.current) initialRect.current = activeNodeRect; - - // Default offset so during drag the cursor doesn't overlap the overlay - // which can cause issues with mouse events on other elements - const offset = 12; - - const offsetX = activatorCoordinates.x - rect.left; - const offsetY = activatorCoordinates.y - rect.top; - - return { - ...transform, - x: transform.x + offsetX + offset, - y: transform.y + offsetY + offset - }; - }; - - useEffect(() => { - if (!drag) initialRect.current = null; - }, [drag]); - - return modifier; -}; - -export const DragOverlay = memo(() => { - const isDark = useIsDark(); - - const modifier = useSnapToCursorModifier(); - const drag = useSelector(explorerStore, (s) => s.drag); - - return ( - - {!drag || drag.type === 'touched' ? null : ( -
- {drag.items.length > 1 && ( -
- {drag.items.length} -
- )} - - {(drag.items.slice(0, 8) as ExplorerItem[]).map((item, i, items) => ( -
7 && [ - i + 1 === items.length && 'opacity-10', - i + 2 === items.length && 'opacity-50', - i + 3 === items.length && 'opacity-90' - ] - )} - > - - -
- ))} -
- )} - - ); -}); diff --git a/interface/app/$libraryId/Explorer/ExplorerDraggable.tsx b/interface/app/$libraryId/Explorer/ExplorerDraggable.tsx deleted file mode 100644 index f6172b0f9..000000000 --- a/interface/app/$libraryId/Explorer/ExplorerDraggable.tsx +++ /dev/null @@ -1,28 +0,0 @@ -import { HTMLAttributes } from 'react'; - -import { useExplorerDraggable, UseExplorerDraggableProps } from './useExplorerDraggable'; - -/** - * Wrapper for explorer draggable items until dnd-kit solvers their re-rendering issues - * https://github.com/clauderic/dnd-kit/issues/1194#issuecomment-1696704815 - */ -export const ExplorerDraggable = ({ - draggable, - ...props -}: Omit, 'draggable'> & { - draggable: UseExplorerDraggableProps; -}) => { - const { attributes, listeners, style, setDraggableRef } = useExplorerDraggable(draggable); - - return ( -
- {props.children} -
- ); -}; diff --git a/interface/app/$libraryId/Explorer/ExplorerDroppable.tsx b/interface/app/$libraryId/Explorer/ExplorerDroppable.tsx deleted file mode 100644 index f08459c02..000000000 --- a/interface/app/$libraryId/Explorer/ExplorerDroppable.tsx +++ /dev/null @@ -1,36 +0,0 @@ -import clsx from 'clsx'; -import { createContext, HTMLAttributes, useContext, useMemo } from 'react'; - -import { useExplorerDroppable, UseExplorerDroppableProps } from './useExplorerDroppable'; - -const ExplorerDroppableContext = createContext<{ isDroppable: boolean } | null>(null); - -export const useExplorerDroppableContext = () => { - const ctx = useContext(ExplorerDroppableContext); - - if (ctx === null) throw new Error('ExplorerDroppableContext.Provider not found!'); - - return ctx; -}; - -/** - * Wrapper for explorer droppable items until dnd-kit solvers their re-rendering issues - * https://github.com/clauderic/dnd-kit/issues/1194#issuecomment-1696704815 - */ -export const ExplorerDroppable = ({ - droppable, - children, - ...props -}: HTMLAttributes & { droppable: UseExplorerDroppableProps }) => { - const { isDroppable, className, setDroppableRef } = useExplorerDroppable(droppable); - - const context = useMemo(() => ({ isDroppable }), [isDroppable]); - - return ( - -
- {children} -
- - ); -}; diff --git a/interface/app/$libraryId/Explorer/ExplorerPathBar.tsx b/interface/app/$libraryId/Explorer/ExplorerPathBar.tsx deleted file mode 100644 index 7d06c9288..000000000 --- a/interface/app/$libraryId/Explorer/ExplorerPathBar.tsx +++ /dev/null @@ -1,257 +0,0 @@ -import { AppWindow, ArrowSquareOut, CaretRight, ClipboardText } from '@phosphor-icons/react'; -import clsx from 'clsx'; -import { memo, useMemo, useState } from 'react'; -import { useNavigate } from 'react-router'; -import { createSearchParams } from 'react-router-dom'; -import { - getExplorerItemData, - getIndexedItemFilePath, - useLibraryContext, - useLibraryQuery -} from '@sd/client'; -import { ContextMenu } from '@sd/ui'; -import { Icon } from '~/components'; -import { useIsDark, useLocale, useOperatingSystem } from '~/hooks'; -import { useTabsContext } from '~/TabsContext'; -import { usePlatform } from '~/util/Platform'; - -import { useExplorerContext } from './Context'; -import { FileThumb } from './FilePath/Thumb'; -import { lookup } from './RevealInNativeExplorer'; -import { useExplorerDroppable } from './useExplorerDroppable'; -import { useExplorerSearchParams } from './util'; - -// todo: ENTIRELY replace with computed combined pathbar+tagbar height -export const PATH_BAR_HEIGHT = 32; - -export const ExplorerPathBar = memo(() => { - const os = useOperatingSystem(true); - const navigate = useNavigate(); - const [{ path: searchPath }] = useExplorerSearchParams(); - const { parent: explorerParent, selectedItems } = useExplorerContext(); - - const location = explorerParent?.type === 'Location' ? explorerParent.location : undefined; - - const selectedItem = useMemo( - () => (selectedItems.size === 1 ? [...selectedItems][0] : undefined), - [selectedItems] - ); - - const indexedFilePath = selectedItem && getIndexedItemFilePath(selectedItem); - - const queryPath = !!indexedFilePath && (!searchPath || !location); - - const { data: filePathname } = useLibraryQuery(['files.getPath', indexedFilePath?.id ?? -1], { - enabled: queryPath - }); - - const paths = useMemo(() => { - const pathSlash = os === 'windows' ? '\\' : '/'; - - // Replace all slashes with native slashes - // TODO: Fix returned path from query on windows as the location part of the path - // uses "/" instead of "\" -> C:\Users\sd-user\Documents\spacedrive\packages/assets/deps - let _filePathname = filePathname?.replaceAll(/[\\/]/g, pathSlash); - - // Remove file name from the path - _filePathname = _filePathname?.slice(0, _filePathname.lastIndexOf(pathSlash) + 1); - - const pathname = _filePathname ?? [location?.path, searchPath].filter(Boolean).join(''); - - const paths = [...(pathname.match(/[^\\/]+/g) ?? [])]; - - let locationPath = location?.path; - - if (!locationPath && indexedFilePath?.materialized_path) { - if (indexedFilePath.materialized_path === '/') locationPath = pathname; - else { - let materializedPath = indexedFilePath.materialized_path; - - // Replace all slashes with native slashes - if (os === 'windows') materializedPath = materializedPath.replaceAll('/', '\\'); - - // Extract location path from pathname - locationPath = pathname.slice(0, pathname.lastIndexOf(materializedPath)); - } - } - - const locationIndex = (locationPath ?? '').split(pathSlash).filter(Boolean).length - 1; - - return paths.map((path, i) => { - const isLocation = locationIndex !== -1 && i >= locationIndex; - - const _paths = [ - ...paths.slice(!isLocation ? 0 : locationIndex + 1, i), - i === locationIndex ? '' : path - ]; - - let pathname = _paths.join(isLocation ? '/' : pathSlash); - - // Wrap pathname in slashes if it's a location - if (isLocation) pathname = pathname ? `/${pathname}/` : '/'; - // Add slash to the end of the pathname if it's the root of a drive on windows (C: -> C:\) - else if (os === 'windows' && _paths.length === 1) pathname += pathSlash; - // Add slash to the beginning of the ephemeral pathname (Users -> /Users) - else if (os !== 'windows') pathname = `/${pathname}`; - - return { - name: path, - pathname, - locationId: isLocation ? (indexedFilePath?.location_id ?? location?.id) : undefined - }; - }); - }, [location, indexedFilePath, filePathname, searchPath, os]); - - const handleOnClick = ({ pathname, locationId }: (typeof paths)[number]) => { - if (locationId === undefined) { - // TODO: Handle ephemeral volumes - navigate({ - pathname: '../ephemeral/0-0', - search: `${createSearchParams({ path: pathname })}` - }); - } else { - navigate({ - pathname: `../location/${locationId}`, - search: pathname === '/' ? undefined : `${createSearchParams({ path: pathname })}` - }); - } - }; - - return ( -
- {paths.map((path, idx) => ( - handleOnClick(path)} - disabled={path.pathname === (searchPath ?? (location && '/'))} - /> - ))} - - {selectedItem && (!queryPath || filePathname) && ( -
- - - - {getExplorerItemData(selectedItem).fullName} - -
- )} -
- ); -}); - -interface PathProps { - path: { name: string; pathname: string; locationId?: number }; - onClick: () => void; - disabled: boolean; - locationPath: string; - isLast: boolean; -} - -const Path = ({ path, onClick, disabled, locationPath, isLast }: PathProps) => { - const isDark = useIsDark(); - const { revealItems } = usePlatform(); - const { library } = useLibraryContext(); - const { t } = useLocale(); - const os = useOperatingSystem(); - const tabs = useTabsContext(); - const [contextMenuOpen, setContextMenuOpen] = useState(false); - - const osFileBrowserName = lookup[os] ?? 'file manager'; - const pathValue = path.pathname.endsWith('/') - ? locationPath + path.pathname.substring(0, path.pathname.length - 1) - : path.pathname; - const osPath = os === 'windows' ? pathValue?.replace(/\//g, '\\') : pathValue; - - // "Open in new tab" redirect - const basePath = path.locationId ? `location/${path.locationId}` : `ephemeral/0-0`; - const searchParam = - path.pathname === '/' ? undefined : createSearchParams({ path: path.pathname }); - const redirect = { - pathname: `${library.uuid}/${basePath}`, - search: searchParam ? `${searchParam}` : undefined - }; - - const { setDroppableRef, className, isDroppable } = useExplorerDroppable({ - data: { - type: 'location', - path: path.pathname, - data: path.locationId ? { id: path.locationId, path: path.pathname } : undefined - }, - allow: ['Path', 'NonIndexedPath', 'Object'], - navigateTo: onClick, - disabled - }); - - return ( - - - {path.name} - {!isLast && } - - } - > - { - if (!tabs) return null; - tabs.createTab(redirect); - }} - label={t('open_in_new_tab')} - icon={ArrowSquareOut} - /> - { - if (!revealItems) return null; - revealItems(library.uuid, [ - path.locationId - ? { - Location: { id: path.locationId } - } - : { - Ephemeral: { path: path.pathname } - } - ]); - }} - label={t('revel_in_browser', { browser: osFileBrowserName })} - icon={AppWindow} - /> - navigator.clipboard.writeText(osPath)} - icon={ClipboardText} - label={t('copy_as_path')} - /> - - ); -}; diff --git a/interface/app/$libraryId/Explorer/ExplorerTagBar.tsx b/interface/app/$libraryId/Explorer/ExplorerTagBar.tsx deleted file mode 100644 index 2dd8ef46e..000000000 --- a/interface/app/$libraryId/Explorer/ExplorerTagBar.tsx +++ /dev/null @@ -1,369 +0,0 @@ -import { Circle } from '@phosphor-icons/react'; -import clsx from 'clsx'; -import { useEffect, useRef, useState } from 'react'; -import { - ExplorerItem, - getItemObject, - Tag, - Target, - useLibraryMutation, - useLibraryQuery, - useRspcContext, - useSelector -} from '@sd/client'; -import { Shortcut, toast } from '@sd/ui'; -import { useIsDark, useKeybind, useLocale, useOperatingSystem } from '~/hooks'; -import { keybindForOs } from '~/util/keybinds'; - -import { useExplorerContext } from './Context'; -import { explorerStore } from './store'; - -export const TAG_BAR_HEIGHT = 54; -const NUMBER_KEYCODES: string[][] = [ - ['Key1'], - ['Key2'], - ['Key3'], - ['Key4'], - ['Key5'], - ['Key6'], - ['Key7'], - ['Key8'], - ['Key9'] -]; - -// TODO: hoist this to somewhere higher as a utility function -const toTarget = (data: ExplorerItem): Target => { - if (!data || !('id' in data.item)) - throw new Error('Tried to convert an invalid object to Target.'); - - return ( - data.type === 'Object' - ? { - Object: data.item.id - } - : { - FilePath: data.item.id - } - ) satisfies Target; -}; - -type TagBulkAssignHotkeys = typeof explorerStore.tagBulkAssignHotkeys; -function getHotkeysWithNewAssignment( - hotkeys: TagBulkAssignHotkeys, - options: - | { - unassign?: false; - tagId: number; - hotkey: string; - } - | { - unassign: true; - tagId: number; - hotkey?: string; - } -): TagBulkAssignHotkeys { - const hotkeysWithoutCurrentTag = hotkeys.filter( - ({ hotkey, tagId }) => !(tagId === options.tagId || hotkey === options.hotkey) - ); - - if (options.unassign) { - return hotkeysWithoutCurrentTag; - } - - return hotkeysWithoutCurrentTag.concat({ - hotkey: options.hotkey, - tagId: options.tagId - }); -} - -// million-ignore -export const ExplorerTagBar = () => { - const [tagBulkAssignHotkeys] = useSelector(explorerStore, (s) => [s.tagBulkAssignHotkeys]); - const explorer = useExplorerContext(); - const rspc = useRspcContext(); - const tagsRef = useRef(null); - const [isTagsOverflowing, setIsTagsOverflowing] = useState(false); - - const updateOverflowState = () => { - const element = tagsRef.current; - if (element) { - setIsTagsOverflowing( - element.scrollHeight > element.clientHeight || - element.scrollWidth > element.clientWidth - ); - } - }; - - useEffect(() => { - const element = tagsRef.current; - if (!element) return; - //handles initial render when not resizing - setIsTagsOverflowing( - element.scrollHeight > element.clientHeight || element.scrollWidth > element.clientWidth - ); - //make sure state updates when window resizing - window.addEventListener('resize', () => { - updateOverflowState(); - }); - //remove listeners on unmount - return () => { - window.removeEventListener('resize', () => { - updateOverflowState(); - }); - }; - }, [tagsRef]); - - const [tagListeningForKeyPress, setTagListeningForKeyPress] = useState(); - - const { data: allTags = [] } = useLibraryQuery(['tags.list']); - const mutation = useLibraryMutation(['tags.assign'], { - onSuccess: () => rspc.queryClient.invalidateQueries({ queryKey: ['search.paths'] }) - }); - - const { t } = useLocale(); - - // This will automagically listen for any keypress 1-9 while the tag bar is visible. - // These listeners will unmount when ExplorerTagBar is unmounted. - useKeybind( - NUMBER_KEYCODES, - async (e) => { - const targets = Array.from(explorer.selectedItems.entries()).map((item) => - toTarget(item[0]) - ); - - // Silent fail if no files are selected - if (targets.length < 1) return; - - const keyPressed = e.key; - - let tag: Tag | undefined; - - findTag: { - const tagId = tagBulkAssignHotkeys.find( - ({ hotkey }) => hotkey === keyPressed - )?.tagId; - const foundTag = allTags.find((t) => t.id === tagId); - - if (!foundTag) break findTag; - - tag = foundTag; - } - - if (!tag) return; - - // extract the list of tags from each object in the selected items - const targetsTagList = Array.from(explorer.selectedItems.entries()).map( - // issues with type here. unsure as to why, and not causing any noticeable errors, so ignoring for now with as any - (item) => (item[0] as any).item.object.tags - ); - - // iterate through each tag in the selected items and check if the tag we want to assign is already assigned - const areAllAssigned = targetsTagList.every((tags) => { - return tags.some((t: { tag_id: any }) => t.tag_id === tag.id); - }); - - try { - if (areAllAssigned) { - await mutation.mutateAsync({ - targets, - tag_id: tag.id, - unassign: true - }); - - toast( - t('tags_bulk_unassigned', { - tag_name: tag.name, - file_count: targets.length - }), - { - type: 'success' - } - ); - } else { - await mutation.mutateAsync({ - targets, - tag_id: tag.id, - unassign: false - }); - - toast( - t('tags_bulk_assigned', { - tag_name: tag.name, - file_count: targets.length - }), - { - type: 'success' - } - ); - } - } catch (err) { - let msg: string = t('error_unknown'); - - if (err instanceof Error || (typeof err === 'object' && err && 'message' in err)) { - msg = err.message as string; - } else if (typeof err === 'string') { - msg = err; - } - - console.error('Tag assignment failed with error', err); - - let failedToastMessage: string = t('tags_bulk_failed_without_tag', { - file_count: targets.length, - error_message: msg - }); - - if (tag) - failedToastMessage = t('tags_bulk_failed_with_tag', { - tag_name: tag.name, - file_count: targets.length, - error_message: msg - }); - - toast(failedToastMessage, { - type: 'error' - }); - } - }, - { - enabled: typeof tagListeningForKeyPress !== 'number' - } - ); - - return ( -
- {t('tags_bulk_instructions')} - -
    - {/* Did not want to write a .toSorted() predicate for this so lazy spreading things with hotkeys first then the rest after */} - {allTags - .toSorted((tagA, tagB) => { - // Sort this array by hotkeys 1-9 first, then unasssigned tags. I know, it's terrible. - // This 998/999 bit is likely terrible for sorting. I'm bad at writing sort predicates. - // Improvements (probably much simpler than this anyway) are much welcome <3 - // -- iLynxcat 3/jun/2024 - - const hotkeyA = +( - tagBulkAssignHotkeys.find((k) => k.tagId === tagA.id)?.hotkey ?? 998 - ); - const hotkeyB = +( - tagBulkAssignHotkeys.find((k) => k.tagId === tagB.id)?.hotkey ?? 999 - ); - - return hotkeyA - hotkeyB; - }) - .map((tag) => ( -
  • - tagId === tag.id) - ?.hotkey - } - isAwaitingKeyPress={tagListeningForKeyPress === tag.id} - onClick={() => { - setTagListeningForKeyPress(tag.id); - }} - onKeyPress={(e) => { - if (e.key === 'Escape') { - explorerStore.tagBulkAssignHotkeys = - getHotkeysWithNewAssignment(tagBulkAssignHotkeys, { - unassign: true, - tagId: tag.id - }); - - setTagListeningForKeyPress(undefined); - - return; - } - - explorerStore.tagBulkAssignHotkeys = - getHotkeysWithNewAssignment(tagBulkAssignHotkeys, { - tagId: tag.id, - hotkey: e.key - }); - setTagListeningForKeyPress(undefined); - }} - /> -
    • - ))} -
-
- ); -}; - -interface TagItemProps { - tag: Tag; - assignKey?: string; - isAwaitingKeyPress: boolean; - onKeyPress: (e: KeyboardEvent) => void; - onClick: () => void; -} - -const TagItem = ({ - tag, - assignKey, - isAwaitingKeyPress = false, - onKeyPress, - onClick -}: TagItemProps) => { - const buttonRef = useRef(null); - const isDark = useIsDark(); - - const os = useOperatingSystem(true); - const keybind = keybindForOs(os); - - useKeybind( - [...NUMBER_KEYCODES, ['Escape']], - (e) => { - buttonRef.current?.blur(); // Hides the focus ring after Escape is pressed to cancel assignment - return onKeyPress(e); - }, - { - enabled: isAwaitingKeyPress - } - ); - - return ( - - ); -}; diff --git a/interface/app/$libraryId/Explorer/FilePath/DeleteDialog.tsx b/interface/app/$libraryId/Explorer/FilePath/DeleteDialog.tsx deleted file mode 100644 index 8ce5a5184..000000000 --- a/interface/app/$libraryId/Explorer/FilePath/DeleteDialog.tsx +++ /dev/null @@ -1,130 +0,0 @@ -import { useLibraryMutation, useZodForm } from '@sd/client'; -import { Dialog, useDialog, UseDialogProps } from '@sd/ui'; -import i18n from '~/app/I18n'; -import { Icon } from '~/components'; -import { useLocale } from '~/hooks'; - -interface Props extends UseDialogProps { - indexedArgs?: { - locationId: number; - rescan?: () => void; - pathIds: number[]; - }; - ephemeralArgs?: { - paths: string[]; - }; - dirCount?: number; - fileCount?: number; -} - -function getWording(dirCount: number, fileCount: number) { - let type = 'file'; - let translatedType = i18n.t('file', { count: 1 }); - let prefix = i18n.t('prefix_a'); - - if (dirCount == 1 && fileCount == 0) { - type = 'directory'; - translatedType = i18n.t('directory', { count: dirCount }); - prefix = i18n.t('prefix_a'); - } - - if (dirCount > 1 && fileCount == 0) { - type = 'directories'; - translatedType = i18n.t('directory', { count: dirCount }); - prefix = dirCount.toString(); - } - - if (fileCount > 1 && dirCount == 0) { - type = 'files'; - translatedType = i18n.t('file', { count: fileCount }); - prefix = fileCount.toString(); - } - - if (fileCount > 0 && dirCount > 0) { - type = 'items'; - translatedType = i18n.t('item', { count: fileCount + dirCount }); - prefix = (fileCount + dirCount).toString(); - } - - return { type, prefix, translatedType }; -} - -export default (props: Props) => { - const { t } = useLocale(); - const deleteFile = useLibraryMutation('files.deleteFiles'); - const deleteEphemeralFile = useLibraryMutation('ephemeralFiles.deleteFiles'); - const moveToTrashFile = useLibraryMutation('files.moveToTrash'); - const moveToTrashEphemeralFile = useLibraryMutation('ephemeralFiles.moveToTrash'); - - const form = useZodForm(); - const { dirCount = 0, fileCount = 0, indexedArgs, ephemeralArgs } = props; - - const { type, prefix, translatedType } = getWording(dirCount, fileCount); - - const icon = type === 'file' || type === 'files' ? 'Document' : 'Folder'; - - const description = t('delete_warning', { type: translatedType }); - - return ( - { - if (indexedArgs != undefined) { - const { locationId, rescan, pathIds } = indexedArgs; - await deleteFile.mutateAsync({ - location_id: locationId, - file_path_ids: pathIds - }); - - rescan?.(); - } - - if (ephemeralArgs != undefined) { - const { paths } = ephemeralArgs; - await deleteEphemeralFile.mutateAsync(paths); - } - })} - onSubmitSecond={form.handleSubmit(async () => { - if (indexedArgs != undefined) { - console.log( - 'DEBUG: DeleteDialog.tsx: onSubmitSecond (Move to Trash) -> Indexed Files' - ); - const { locationId, rescan, pathIds } = indexedArgs; - await moveToTrashFile.mutateAsync({ - location_id: locationId, - file_path_ids: pathIds - }); - - rescan?.(); - } - - if (ephemeralArgs != undefined) { - console.log( - 'DEBUG: DeleteDialog.tsx: onSubmitSecond (Move to Trash) -> Ephemeral Files' - ); - const { paths } = ephemeralArgs; - await moveToTrashEphemeralFile.mutateAsync(paths); - } - })} - icon={} - dialog={useDialog(props)} - title={t('delete_dialog_title', { prefix, type: translatedType })} - description={description} - loading={deleteFile.isPending} - ctaLabel={t('delete_forever')} - ctaSecondLabel={t('move_to_trash')} - closeLabel={t('close')} - ctaDanger - className="w-[200px]" - > - {/* -
- -

- Delete all matching {type.endsWith('s') ? type : type + 's'} -

-
- */} - - ); -}; diff --git a/interface/app/$libraryId/Explorer/FilePath/ErrorBarrier.tsx b/interface/app/$libraryId/Explorer/FilePath/ErrorBarrier.tsx deleted file mode 100644 index 9290ca589..000000000 --- a/interface/app/$libraryId/Explorer/FilePath/ErrorBarrier.tsx +++ /dev/null @@ -1,40 +0,0 @@ -import React, { Component, ReactNode } from 'react'; - -interface ErrorBarrierProps { - onError: (error: Error, info: React.ErrorInfo) => void; - children: ReactNode; -} - -interface ErrorBarrierState { - hasError: boolean; -} - -export class ErrorBarrier extends Component { - constructor(props: ErrorBarrierProps) { - super(props); - this.state = { hasError: false }; - } - - static getDerivedStateFromError(error: Error) { - // Update state so the next render will show the fallback UI. - return { hasError: true }; - } - - componentDidCatch(error: Error, info: React.ErrorInfo) { - // Call the onError function passed as a prop - this.props.onError(error, info); - // Reset the error state after calling onError - Promise.resolve().then(() => this.setState({ hasError: false })); - } - - render() { - if (this.state.hasError) { - // Render nothing since the parent component will handle the error - return null; - } - - return this.props.children; - } -} - -export default ErrorBarrier; diff --git a/interface/app/$libraryId/Explorer/FilePath/Image.tsx b/interface/app/$libraryId/Explorer/FilePath/Image.tsx deleted file mode 100644 index e7036595b..000000000 --- a/interface/app/$libraryId/Explorer/FilePath/Image.tsx +++ /dev/null @@ -1,14 +0,0 @@ -import { ComponentProps, forwardRef } from 'react'; - -export const Image = forwardRef>( - ({ crossOrigin, ...props }, ref) => ( - - ) -); diff --git a/interface/app/$libraryId/Explorer/FilePath/LayeredFileIcon.tsx b/interface/app/$libraryId/Explorer/FilePath/LayeredFileIcon.tsx deleted file mode 100644 index c945756cf..000000000 --- a/interface/app/$libraryId/Explorer/FilePath/LayeredFileIcon.tsx +++ /dev/null @@ -1,62 +0,0 @@ -import { getIcon, getIconByName, getLayeredIcon, IconTypes } from '@sd/assets/util'; -import clsx from 'clsx'; -import { forwardRef, Suspense, useMemo, type ImgHTMLAttributes } from 'react'; -import { type ObjectKindKey } from '@sd/client'; -import { useIsDark } from '~/hooks'; - -interface LayeredFileIconProps extends Omit, 'src'> { - kind: ObjectKindKey; - isDir: boolean; - extension: string | null; - customIcon: IconTypes | null; -} - -const SUPPORTED_ICONS = ['Document', 'Code', 'Text', 'Config']; - -const positionConfig: Record = { - Text: 'flex h-full w-full items-center justify-center', - Code: 'flex h-full w-full items-center justify-center pt-[18px]', - Config: 'flex h-full w-full items-center justify-center pt-[18px]' -}; - -const LayeredFileIcon = forwardRef( - ({ kind, isDir, extension, customIcon, ...props }, ref) => { - const isDark = useIsDark(); - - const src = useMemo( - () => - customIcon - ? getIconByName(customIcon, isDark) - : getIcon(kind, isDark, extension, isDir), - [customIcon, isDark, kind, extension, isDir] - ); - - const iconImg = {`${kind}; - - if (SUPPORTED_ICONS.includes(kind) === false) { - return iconImg; - } - - const IconComponent = extension ? getLayeredIcon(kind, extension) : null; - - const positionClass = - positionConfig[kind] || 'flex h-full w-full items-end justify-end pb-4 pr-2'; - - return IconComponent == null ? ( - iconImg - ) : ( -
- {iconImg} -
- - - -
-
- ); - } -); - -export default LayeredFileIcon; diff --git a/interface/app/$libraryId/Explorer/FilePath/Original.tsx b/interface/app/$libraryId/Explorer/FilePath/Original.tsx deleted file mode 100644 index eb1496828..000000000 --- a/interface/app/$libraryId/Explorer/FilePath/Original.tsx +++ /dev/null @@ -1,263 +0,0 @@ -import { getIcon, iconNames } from '@sd/assets/util'; -import clsx from 'clsx'; -import { - SyntheticEvent, - useEffect, - useMemo, - useRef, - useState, - type VideoHTMLAttributes -} from 'react'; -import { ObjectKindKey, useLibraryContext } from '@sd/client'; -import i18n from '~/app/I18n'; -import { PDFViewer, TextViewer } from '~/components'; -import { useIsDark, useLocale } from '~/hooks'; -import { pdfViewerEnabled } from '~/util/pdfViewer'; -import { usePlatform } from '~/util/Platform'; - -import { useExplorerContext } from '../Context'; -import { explorerStore } from '../store'; -import { Image } from './Image'; -import { useBlackBars, useSize } from './utils'; - -interface OriginalRendererProps { - src: string; - fileId: number | null; - locationId: number | null; - path: string | null; - className?: string; - frameClassName?: string; - kind: ObjectKindKey; - extension: string | null; - childClassName?: string; - magnification?: number; - mediaControls?: boolean; - frame?: boolean; - isSidebarPreview?: boolean; - pauseVideo?: boolean; - blackBars?: boolean; - blackBarsSize?: number; - onLoad?(): void; -} - -export function Original({ - path, - fileId, - locationId, - ...props -}: Omit) { - const [error, setError] = useState(null); - if (error != null) throw error; - - const Renderer = useMemo(() => { - const kind = originalRendererKind(props.kind, props.extension); - return ORIGINAL_RENDERERS[kind]; - }, [props.kind, props.extension]); - - if (!Renderer) throw new Error('no renderer!'); - - const platform = usePlatform(); - const { library } = useLibraryContext(); - const { parent } = useExplorerContext(); - locationId = locationId ?? (parent?.type === 'Location' ? parent.location.id : null); - - const src = useMemo(() => { - if (props.extension !== 'pdf' || pdfViewerEnabled()) { - if (fileId != null && locationId) - return platform.getFileUrl(library.uuid, locationId, fileId); - else if (path) return platform.getFileUrlByPath(path); - } - }, [props.extension, fileId, locationId, platform, library.uuid, path]); - - if (src === undefined) throw new Error('no src!'); - - return ( - - setError( - ('error' in event && event.error instanceof Error && event.error) || - new Error( - ('message' in event && event.message) || 'Filetype is not supported yet' - ) - ) - } - {...props} - /> - ); -} - -const TEXT_RENDERER: OriginalRenderer = (props) => ( - -); - -type OriginalRenderer = ( - props: Omit & { - onError?(e: ErrorEvent | SyntheticEvent): void; - } -) => JSX.Element; - -function originalRendererKind(kind: ObjectKindKey, extension: string | null) { - return extension === 'pdf' ? 'PDF' : kind; -} - -type OriginalRendererKind = ReturnType; - -const ORIGINAL_RENDERERS: { - [K in OriginalRendererKind]?: OriginalRenderer; -} = { - PDF: (props) => ( - - ), - Text: TEXT_RENDERER, - Code: TEXT_RENDERER, - Config: TEXT_RENDERER, - Video: (props) => ( -