From 8c41d64cc3fe2dca6f79300ae90a2d7be21e4d9a Mon Sep 17 00:00:00 2001 From: Ericson Soares Date: Wed, 16 Oct 2024 13:36:09 -0300 Subject: [PATCH 01/21] Fix tag assign rspc route --- core/src/api/tags.rs | 124 ++++++++++++------------------------------- 1 file changed, 33 insertions(+), 91 deletions(-) diff --git a/core/src/api/tags.rs b/core/src/api/tags.rs index 451ac624e..0d71b848c 100644 --- a/core/src/api/tags.rs +++ b/core/src/api/tags.rs @@ -5,7 +5,6 @@ use sd_prisma::{ prisma_sync, }; use sd_sync::{option_sync_db_entry, sync_entry, OperationFactory}; -use sd_utils::{msgpack, uuid_to_bytes}; use std::collections::BTreeMap; @@ -14,7 +13,6 @@ use itertools::{Either, Itertools}; use rspc::{alpha::AlphaRouter, ErrorCode}; use serde::{Deserialize, Serialize}; use specta::Type; -use uuid::Uuid; use super::{utils::library, Ctx, R}; @@ -185,17 +183,6 @@ pub(crate) fn mount() -> AlphaRouter { }) .await?; - macro_rules! sync_id { - ($pub_id:expr) => { - prisma_sync::tag_on_object::SyncId { - tag: prisma_sync::tag::SyncId { - pub_id: tag.pub_id.clone(), - }, - object: prisma_sync::object::SyncId { pub_id: $pub_id }, - } - }; - } - if args.unassign { let query = db.tag_on_object().delete_many(vec![ tag_on_object::tag_id::equals(args.tag_id), @@ -220,63 +207,20 @@ pub(crate) fn mount() -> AlphaRouter { .into_iter() .filter_map(|fp| fp.object.map(|o| o.pub_id)), ) - .map(|pub_id| sync.relation_delete(sync_id!(pub_id))) + .map(|pub_id| { + sync.relation_delete(prisma_sync::tag_on_object::SyncId { + tag: prisma_sync::tag::SyncId { + pub_id: tag.pub_id.clone(), + }, + object: prisma_sync::object::SyncId { pub_id }, + }) + }) .collect::>(); if !ops.is_empty() { sync.write_ops(db, (ops, query)).await?; } } else { - let mut ops = vec![]; - - let db_params: (Vec<_>, Vec<_>) = file_paths - .iter() - .filter(|fp| fp.is_dir.unwrap_or_default() && fp.object.is_none()) - .map(|fp| { - let id = uuid_to_bytes(&Uuid::now_v7()); - let device_pub_id = sync.device_pub_id.to_db(); - - ops.push(sync.shared_create( - prisma_sync::object::SyncId { pub_id: id.clone() }, - [sync_entry!( - prisma_sync::device::SyncId { - pub_id: device_pub_id.clone(), - }, - object::device - )], - )); - - ops.push(sync.shared_update( - prisma_sync::file_path::SyncId { - pub_id: fp.pub_id.clone(), - }, - file_path::object::NAME, - msgpack!(id), - )); - - ( - db.object().create( - id.clone(), - vec![object::device::connect(device::pub_id::equals( - device_pub_id, - ))], - ), - db.file_path().update( - file_path::id::equals(fp.id), - vec![file_path::object::connect(object::pub_id::equals( - id, - ))], - ), - ) - }) - .unzip(); - - if ops.is_empty() { - return Ok(()); - } - - let (new_objects, _) = sync.write_ops(db, (ops, db_params)).await?; - let (sync_ops, db_creates) = objects .into_iter() .map(|o| (o.id, o.pub_id)) @@ -285,22 +229,23 @@ pub(crate) fn mount() -> AlphaRouter { .into_iter() .filter_map(|fp| fp.object.map(|o| (o.id, o.pub_id))), ) - .chain(new_objects.into_iter().map(|o| (o.id, o.pub_id))) - .fold( - (vec![], vec![]), - |(mut sync_ops, mut db_creates), (id, pub_id)| { - let device_pub_id = sync.device_pub_id.to_db(); - sync_ops.push(sync.relation_create( - sync_id!(pub_id), + .map(|(id, pub_id)| { + ( + sync.relation_create( + prisma_sync::tag_on_object::SyncId { + tag: prisma_sync::tag::SyncId { + pub_id: tag.pub_id.clone(), + }, + object: prisma_sync::object::SyncId { pub_id }, + }, [sync_entry!( prisma_sync::device::SyncId { - pub_id: device_pub_id.clone(), + pub_id: sync.device_pub_id.to_db(), }, tag_on_object::device )], - )); - - db_creates.push(tag_on_object::CreateUnchecked { + ), + tag_on_object::CreateUnchecked { tag_id: args.tag_id, object_id: id, _params: vec![ @@ -309,24 +254,21 @@ pub(crate) fn mount() -> AlphaRouter { )), tag_on_object::device_id::set(Some(device_id)), ], - }); + }, + ) + }) + .unzip::<_, _, Vec<_>, Vec<_>>(); - (sync_ops, db_creates) - }, - ); - - if sync_ops.is_empty() && db_creates.is_empty() { - return Ok(()); + if !sync_ops.is_empty() && !db_creates.is_empty() { + sync.write_ops( + db, + ( + sync_ops, + db.tag_on_object().create_many(db_creates).skip_duplicates(), + ), + ) + .await?; } - - sync.write_ops( - db, - ( - sync_ops, - db.tag_on_object().create_many(db_creates).skip_duplicates(), - ), - ) - .await?; } invalidate_query!(library, "tags.getForObject"); From 0bbf7aa0ac9e4ace545893f770bdd24fa05a12c5 Mon Sep 17 00:00:00 2001 From: wany-oh <98004396+wany-oh@users.noreply.github.com> Date: Thu, 17 Oct 2024 05:46:46 +0900 Subject: [PATCH 02/21] Improved Japanese translation (#2773) --- interface/locales/ja/common.json | 115 ++++++++++++++++--------------- 1 file changed, 60 insertions(+), 55 deletions(-) diff --git a/interface/locales/ja/common.json b/interface/locales/ja/common.json index 89548338a..0b967084f 100644 --- a/interface/locales/ja/common.json +++ b/interface/locales/ja/common.json @@ -3,7 +3,7 @@ "about_vision_text": "私達は通常、複数のクラウドのアカウントを持ち、バックアップのないドライブを利用し、データを失う危険にさらされています。またGoogle PhotosやiCloudのようなクラウドサービスに依存していますが、それらは容量に制限があり、サービスやOS間に互換性はほとんどありません。フォトアルバムは、デバイスのエコシステムに縛られたり広告データとして利用されたりすべきではなく、OSにとらわれず、永続的で、個人所有のものであるべきです。私達が作成したデータは私達の遺産であり、私達よりもずっと長生きします。オープンソース・テクノロジーは、無制限のスケールで、私達の生活を定義するデータの絶対的なコントロールを確実に保持する唯一の方法なのです。", "about_vision_title": "ビジョン", "accept": "Accept", - "accept_files": "Accept files", + "accept_files": "ファイルを含める", "accessed": "最終アクセス", "account": "アカウント", "actions": "操作", @@ -20,7 +20,7 @@ "add_tag": "タグを追加", "added_location": "ロケーション {{name}} を追加しました", "adding_location": "ロケーション {{name}} を追加中", - "advanced": "高度な", + "advanced": "高度", "advanced_settings": "高度な設定", "album": "アルバム", "alias": "エイリアス", @@ -47,6 +47,7 @@ "backfill_sync_description": "バックフィルが完了するまでライブラリは一時停止されます", "backups": "バックアップ", "backups_description": "Spacedriveデータベースのバックアップの設定を行います。", + "bar_graph_info": "各バーにカーソルを合わせると、ファイルの種類が表示されます。 ダブルクリックで移動します。", "bitrate": "ビットレート", "blur_effects": "ぼかし効果", "blur_effects_description": "いくつかのUI要素にぼかし効果を適用します。", @@ -56,23 +57,23 @@ "canceled": "キャンセル", "celcius": "摂氏", "change": "変更", - "change_view_setting_description": "デフォルトのエクスプローラー ビューを変更する", + "change_view_setting_description": "エクスプローラーのデフォルトの表示形式を変更します。", "changelog": "変更履歴", "changelog_page_description": "Spacedriveの魅力ある新機能をご確認ください。", "changelog_page_title": "変更履歴", "checksum": "チェックサム", "clear_finished_jobs": "完了ジョブを削除", - "click_to_hide": "非表示にするにはクリック", - "click_to_lock": "ロックするにはクリック", + "click_to_hide": "クリックで非表示", + "click_to_lock": "クリックでロック", "client": "クライアント", "close": "閉じる", "close_command_palette": "コマンドパレットを閉じる", "close_current_tab": "タブを閉じる", "cloud": "クラウド", - "cloud_connect_description": "ライブラリをクラウドに接続しますか?", + "cloud_connect_description": "ライブラリをクラウドに接続しますか？", "cloud_drives": "クラウドドライブ", "cloud_sync": "クラウド同期", - "cloud_sync_description": "ライブラリを Spacedrive Cloud と同期するプロセスを管理する", + "cloud_sync_description": "ライブラリをSpacedrive Cloudと同期するプロセスを管理します。", "clouds": "クラウド", "code": "コード", "collection": "コレクション", @@ -88,13 +89,13 @@ "confirm": "Confirm", "connect": "接続する", "connect_cloud": "クラウドに接続する", - "connect_cloud_description": "クラウドアカウントをSpacedriveに接続する。", + "connect_cloud_description": "クラウドアカウントをSpacedriveに接続", "connect_device": "デバイスを接続する", "connect_device_description": "Spacedriveはすべてのデバイスで最適に機能します。", "connect_library_to_cloud": "ライブラリをSpacedrive Cloudに接続する", "connected": "接続中", "connecting": "接続中", - "connecting_library_to_cloud": "ライブラリを Spacedrive Cloud に接続しています...", + "connecting_library_to_cloud": "ライブラリをSpacedrive Cloudに接続しています……", "contacts": "連絡先", "contacts_description": "Spacedriveで連絡先を管理。", "contains": "が次を含む", @@ -141,7 +142,7 @@ "date_modified": "更新日時", "date_taken": "取得日時", "date_time_format": "日付と時刻のフォーマット", - "date_time_format_description": "Spacedriveに表示される日付フォーマットを選択します", + "date_time_format_description": "Spacedriveに表示される日付フォーマットを選択します。", "debug_mode": "デバッグモード", "debug_mode_description": "アプリ内で追加のデバッグ機能を有効にします。", "default": "デフォルト", @@ -197,7 +198,7 @@ "enable_networking_description": "あなたのノードが周囲の他のSpacedriveノードと通信できるようにします。", "enable_networking_description_required": "ライブラリの同期やSpacedropに必要です。", "enable_relay": "リレーを有効にする", - "enable_relay_description": "リレー サーバーを有効にして、デバイスがパブリック インターネット経由で通信できるようにします。", + "enable_relay_description": "リレー・サーバーを有効にして、デバイスがパブリック・インターネット経由で通信できるようにします。", "enable_sync": "同期を有効にする", "enable_sync_description": "このライブラリ内のすべての既存データに対して同期操作を生成し、将来何かが発生したときに同期操作を生成するように Spacedrive を構成します。", "enabled": "有効", @@ -221,7 +222,7 @@ "explorer": "エクスプローラー", "explorer_settings": "エクスプローラーの設定", "explorer_shortcut_description": "ファイルシステムの移動・操作を設定します。", - "explorer_view": "エクスプローラービュー", + "explorer_view": "エクスプローラーの表示形式", "export": "エクスポート", "export_library": "ライブラリのエクスポート", "export_library_coming_soon": "ライブラリのエクスポート機能は今後実装予定です", @@ -230,7 +231,7 @@ "extensions": "拡張機能", "extensions_description": "このクライアントの機能を拡張するための拡張機能をインストールします。", "fahrenheit": "華氏", - "failed": "失敗した", + "failed": "失敗しました", "failed_to_add_location": "ロケーションの追加に失敗", "failed_to_cancel_job": "ジョブの中止に失敗", "failed_to_clear_all_jobs": "全てのジョブの削除に失敗", @@ -263,8 +264,8 @@ "feedback_placeholder": "フィードバックを入力...", "feedback_toast_error_message": "フィードバックの送信中にエラーが発生しました。もう一度お試しください。", "file_already_exist_in_this_location": "このファイルは既にこのロケーションに存在します", - "file_directory_name": "ファイル/ディレクトリ名", - "file_extension_description": "ファイル拡張子 (例: .mp4、.jpg、.txt)", + "file_directory_name": "ファイル／ディレクトリ名", + "file_extension_description": "拡張子（例: .mp4、.jpg、.txt）", "file_from": "File {{file}} from {{name}}", "file_indexing_rules": "ファイルのインデックス化ルール", "file_other": "ファイル", @@ -289,8 +290,8 @@ "general_shortcut_description": "一般に使用されるショートカットキー。", "generate_checksums": "チェックサムを作成", "generate_preview_media_label": "このロケーションのプレビューメディアを作成する", - "gitignore": "Git 無視", - "glob_description": "グロブ (例: **/.git)", + "gitignore": "Git Ignore", + "glob_description": "グロブ（例: **/.git）", "go_back": "戻る", "go_to_labels": "ラベルに移動", "go_to_location": "ロケーションに移動", @@ -324,11 +325,11 @@ "indexer_rule_reject_allow_label": "デフォルトでは、インデックス化ルールはブラックリストとして機能し、その基準に一致する全てのファイルを除外します。このオプションを有効にすると、ホワイトリストに変換され、指定されたルールに一致するファイルのみをインデックス化するようになります。", "indexer_rules": "インデックス化のルール", "indexer_rules_error": "インデクス化ルールの取得エラー", - "indexer_rules_info": "globを使用して無視するパスを指定できます。", + "indexer_rules_info": "グロブを使用して無視するパスを指定できます。", "indexer_rules_not_available": "利用可能なインデックス化ルールがありません", "ingester": "インジェスター", "ingester_description": "このプロセスは、受信したクラウド操作を取得し、メインの同期インジェスターに送信します。", - "injester_description": "このプロセスは、P2P 接続と Spacedrive Cloud から同期操作を取得し、ライブラリに適用します。", + "injester_description": "このプロセスは、P2P接続とSpacedrive Cloudから同期操作を取得し、ライブラリに適用します。", "install": "インストール", "install_update": "アップデートをインストールする", "installed": "インストール完了", @@ -339,7 +340,7 @@ "ipv4_ipv6_listeners_error": "IPv4・IPv6リスナーの作成エラー。ファイアウォールの設定を確認してください！", "ipv4_listeners_error": "IPv4リスナーの作成エラー。ファイアウォールの設定を確認してください！", "ipv6": "IPv6ネットワーキング", - "ipv6_description": "IPv6 ネットワークを使用したピアツーピア通信を許可する", + "ipv6_description": "IPv6ネットワークを使用したピアツーピア通信を許可します。", "ipv6_listeners_error": "IPv6リスナーの作成エラー。ファイアウォールの設定を確認してください！", "is": "が", "is_not": "が次と異なる", @@ -347,7 +348,7 @@ "item_other": "項目", "item_size": "アイテムの表示サイズ", "items": "項目", - "job_error_description": "ジョブはエラーで完了しました。\n詳細については、以下のエラー ログを参照してください。\nサポートが必要な場合は、サポートに連絡してこのエラーを伝えてください。", + "job_error_description": "ジョブはエラーで完了しました。\n詳細については、以下のエラーログを参照してください。\nサポートが必要な場合は、サポートに連絡してこのエラーを伝えてください。", "job_has_been_canceled": "ジョブが中止されました。", "job_has_been_paused": "ジョブが一時停止されました。", "job_has_been_removed": "ジョブが削除されました。", @@ -376,7 +377,7 @@ "libraries_description": "データベースには、すべてのライブラリデータとファイルのメタデータが含まれています。", "library": "ライブラリ", "library_bytes": "ライブラリのサイズ", - "library_bytes_description": "ライブラリ内のすべての場所の合計サイズ。", + "library_bytes_description": "ライブラリ内のすべてのロケーションの合計サイズ。", "library_db_size": "インデックスサイズ", "library_db_size_description": "ライブラリのデータベースのサイズ。", "library_name": "ライブラリの名前", @@ -392,9 +393,9 @@ "local_locations": "ローカルロケーション", "local_node": "ローカルノード", "location": "ロケーション", - "location_added_successfully": "場所が正常に追加されました。", + "location_added_successfully": "ロケーションが正常に追加されました。", "location_connected_tooltip": "ロケーションの変化が監視されています", - "location_deleted_successfully": "場所は正常に削除されました。", + "location_deleted_successfully": "ロケーションは正常に削除されました。", "location_disconnected_tooltip": "ロケーションの変更は監視されていません", "location_display_name_info": "サイドバーに表示されるロケーションの名前を設定します。ディスク上の実際のフォルダの名前は変更されません。", "location_empty_notice_message": "ファイルが見つかりません", @@ -408,7 +409,7 @@ "locations": "ロケーション", "locations_description": "ローケーションを管理します。", "lock": "ロック", - "lock_sidebar": "サイドバーをロックする", + "lock_sidebar": "サイドバーをロック", "log_in": "ログイン", "log_in_with_browser": "ブラウザでログイン", "log_out": "ログアウト", @@ -416,9 +417,9 @@ "logging_in": "ログイン中...", "logout": "ログアウト", "manage_library": "ライブラリの設定", - "managed": "マネージド", + "managed": "Managed", "manual_peers": "ピアを手動で追加する", - "manual_peers_description": "IP アドレスとポートを入力してピアを手動で追加します。\nこれは、自動検出が不可能な場合に役立ちます。", + "manual_peers_description": "IPアドレスとポートを入力してピアを手動で追加します。\nこれは、自動検出が不可能な場合に役立ちます。", "media": "メディア", "media_view": "メディアビュー", "media_view_context": "メディア ビュー", @@ -453,12 +454,12 @@ "network_settings": "ネットワーク設定", "network_settings_advanced": "高度なネットワークの概要", "network_settings_advanced_description": "現在のネットワーク設定に関する詳細情報。", - "network_settings_description": "ネットワークと接続に関する設定。", + "network_settings_description": "ネットワークと接続に関する設定を行います。", "networking": "ネットワーク", "networking_error": "ネットワークの起動エラー！", "networking_port": "ネットワークポート", "networking_port_description": "SpacedriveのP2Pネットワークが使用するポートを設定します。ファイアウォールによる制限がない限り、無効のままにしておくことを推奨します。インターネット上に公開しないでください！", - "new": "新しい", + "new": "新規", "new_folder": "新しいフォルダー", "new_library": "新しいライブラリ", "new_location": "新しいロケーション", @@ -468,14 +469,14 @@ "new_update_available": "アップデートが利用可能です！", "no_apps_available": "利用可能なアプリはありません", "no_favorite_items": "お気に入りのアイテムはありません", - "no_git_files": "Git ファイルがありません", - "no_hidden_files": "隠しファイルはありません", + "no_git_files": "Gitファイルを除外", + "no_hidden_files": "隠しファイルを除外", "no_items_found": "アイテムが見つかりませんでした", "no_jobs": "ジョブがありません。", "no_labels": "ラベルなし", - "no_nodes_found": "Spacedriveノードが見つかりませんでした。", + "no_nodes_found": "Spacedriveノードが見つかりません。", "no_search_selected": "検索が選択されていません", - "no_system_files": "システムファイルがありません", + "no_system_files": "システムファイルを除外", "no_tag_selected": "タグが選択されていません。", "no_tags": "タグがありません。", "no_tags_description": "タグが作成されていません", @@ -486,7 +487,7 @@ "normal": "Normal", "not_you": "あなたではありませんか？", "note": "Note", - "nothing_selected": "何も選択されていない", + "nothing_selected": "何も選択されていません", "number_of_passes": "# パス数", "object": "対象", "object_id": "オブジェクトID", @@ -507,12 +508,13 @@ "open_with": "プログラムから開く", "opening_trash": "ごみ箱を開く", "or": "OR", + "other": "その他", "overview": "概要", "p2p_visibility": "P2Pの可視性", "p2p_visibility_contacts_only": "連絡先のみ", "p2p_visibility_description": "Spacedrive インストールを閲覧できるユーザーを設定します。", "p2p_visibility_disabled": "無効", - "p2p_visibility_everyone": "みんな", + "p2p_visibility_everyone": "全員", "package": "パッケージ", "page": "ページ", "page_shortcut_description": "アプリ内の各ページへの移動のショートカット", @@ -542,10 +544,10 @@ "quick_rescan_started": "簡易再スキャンを開始", "quick_view": "クイック プレビュー", "quickpreview_thumbnail_error_message": "フル解像度の画像を読み込めません", - "quickpreview_thumbnail_error_tip": "画像が見つかりませんでした。そのため、サムネイルが表示されます。", + "quickpreview_thumbnail_error_tip": "画像が見つかりませんでした。代わりにサムネイルを表示します。", "random": "ランダム", "receiver": "受信機", - "receiver_description": "このプロセスは、Spacedrive Cloud から操作を受信して​​保存します。", + "receiver_description": "このプロセスは、Spacedrive Cloudから操作を受信して保存します。", "recent_jobs": "最近のジョブ", "recents": "最近のアクセス", "recents_notice_message": "ファイルを開くと最近のアクセスが表示されます。", @@ -554,8 +556,8 @@ "regenerate_thumbs": "サムネイルを再作成", "reindex": "再インデックス化", "reject": "拒否", - "reject_files": "Reject files", - "relay_listeners_error": "リレー リスナーの作成中にエラーが発生しました。ファイアウォールの設定を確認してください。", + "reject_files": "ファイルを除外", + "relay_listeners_error": "リレー・リスナーの作成中にエラーが発生しました。ファイアウォールの設定を確認してください。", "reload": "更新", "remote_access": "リモートアクセスを有効にする", "remote_access_description": "他のノードがこのノードに直接接続できるようにします。", @@ -600,7 +602,7 @@ "send": "送信", "send_report": "レポートを送信", "sender": "送信者", - "sender_description": "このプロセスでは、同期操作が Spacedrive Cloud に送信されます。", + "sender_description": "このプロセスでは、同期操作がSpacedrive Cloudに送信されます。", "settings": "設定", "setup": "セットアップ", "share": "共有", @@ -630,7 +632,7 @@ "spacedrive_account": "Spacedriveアカウント", "spacedrive_cloud": "Spacedriveクラウド", "spacedrive_cloud_description": "Spacedriveは常にローカルでの利用を優先しますが、将来的には独自オプションのクラウドサービスを提供する予定です。現在、アカウント認証はフィードバック機能のみに使用されており、それ以外では必要ありません。", - "spacedrop": "Spacedropの対照", + "spacedrop": "Spacedropの可視性", "spacedrop_a_file": "ファイルをSpacedropへ", "spacedrop_already_progress": "Spacedropは既に実行中です", "spacedrop_contacts_only": "連絡先のみ", @@ -641,10 +643,10 @@ "square_thumbnails": "正方形のサムネイル", "star_on_github": "Star on GitHub", "start": "始める", - "starting": "起動...", + "starting": "起動中……", "starts_with": "が次で始まる", "stop": "中止", - "stopping": "停止中...", + "stopping": "停止中……", "success": "成功", "support": "サポート", "switch_to_grid_view": "グリッド ビューに切り替え", @@ -663,17 +665,17 @@ "tags": "タグ", "tags_description": "タグを管理します。", "tags_notice_message": "このタグに割り当てられたアイテムはありません。", - "task": "task", - "task_other": "tasks", + "task": "タスク", + "task_other": "タスク", "telemetry_description": "有効にすると、アプリを改善するための詳細なテレメトリ・利用状況データが開発者に提供されます。無効にすると、基本的なデータ（実行状況、アプリバージョン、コアバージョン、プラットフォーム［モバイル／ウェブ／デスクトップなど］）のみが送信されます。", - "telemetry_share_anonymous": "利用状況を送信する", - "telemetry_share_anonymous_description": "アプリの改善のために、完全に匿名のテレメトリデータを送信します", - "telemetry_share_anonymous_short": "匿名での使用", - "telemetry_share_minimal": "最小限のデータのみを送信する", - "telemetry_share_minimal_description": "自分がSpacedriveのアクティブユーザーであることと、多少の技術的データのみを送信します", + "telemetry_share_anonymous": "匿名で利用状況を送信", + "telemetry_share_anonymous_description": "アプリの改善のために、完全に匿名のテレメトリデータを送信します。", + "telemetry_share_anonymous_short": "匿名で送信", + "telemetry_share_minimal": "最小限のデータのみを送信", + "telemetry_share_minimal_description": "自分がSpacedriveのアクティブユーザーであることと、多少の技術的データのみを送信します。", "telemetry_share_minimal_short": "最小限", - "telemetry_share_none": "共有しないでください", - "telemetry_share_none_description": "Spacedrive アプリから分析データをまったく送信しません。", + "telemetry_share_none": "送信しない", + "telemetry_share_none_description": "Spacedriveアプリから分析データをまったく送信しません。", "telemetry_share_none_short": "なし", "telemetry_title": "テレメトリ・利用状況データを送信する", "temperature": "温度", @@ -701,10 +703,13 @@ "total_bytes_free_description": "ライブラリに接続されているすべてのノードで利用可能な空き容量。", "total_bytes_used": "総使用量", "total_bytes_used_description": "ライブラリに接続されているすべてのノードで使用されているスペースの合計。", + "total_files": "総ファイル数", "trash": "ごみ箱", "type": "種類", "ui_animations": "UIアニメーション", "ui_animations_description": "ダイアログやその他のUI要素を開いたり閉じたりするときにアニメーションを有効にします。", + "unidentified_files": "不明ファイル数", + "unidentified_files_info": "Spacedriveが識別できなかったファイル。", "unknown": "不明", "unnamed_location": "名前の無いロケーション", "update": "アップデート", @@ -714,10 +719,10 @@ "usage": "利用状況", "usage_description": "ライブラリの利用状況とハードウェア情報", "vacuum": "バキューム", - "vacuum_library": "バキュームライブラリ", + "vacuum_library": "ライブラリをバキューム", "vacuum_library_description": "データベースを再パックして、不要なスペースを解放します。", "value": "値", - "value_required": "必要な値", + "value_required": "必須項目", "version": "バージョン {{version}}", "video": "ビデオ", "video_preview_not_supported": "ビデオのプレビューには対応していません。", @@ -734,4 +739,4 @@ "zoom": "ズーム", "zoom_in": "拡大する", "zoom_out": "縮小する" -} +} \ No newline at end of file From 12fdfb8b782add6060feadb9058b79bc96e9ea2c Mon Sep 17 00:00:00 2001 From: Ericson Soares Date: Sat, 19 Oct 2024 02:48:27 -0300 Subject: [PATCH 03/21] Update sync messages push and pull Also fix scalability issues on sync design --- Cargo.lock | Bin 345122 -> 345145 bytes Cargo.toml | 2 +- .../crates/cloud-services/src/sync/receive.rs | 185 +------- core/crates/cloud-services/src/sync/send.rs | 419 ++--------------- .../src/isolated_file_path_data.rs | 5 +- .../heavy-lifting/src/file_identifier/job.rs | 32 +- .../src/file_identifier/tasks/identifier.rs | 14 +- .../src/file_identifier/tasks/mod.rs | 12 +- core/crates/heavy-lifting/src/indexer/job.rs | 30 +- core/crates/heavy-lifting/src/indexer/mod.rs | 91 ++-- .../heavy-lifting/src/indexer/shallow.rs | 2 +- .../heavy-lifting/src/indexer/tasks/saver.rs | 17 +- .../src/indexer/tasks/updater.rs | 26 +- .../heavy-lifting/src/job_system/report.rs | 2 + .../heavy-lifting/src/media_processor/job.rs | 30 +- core/crates/prisma-helpers/src/lib.rs | 14 + core/crates/sync/src/ingest_utils.rs | 154 +++---- core/crates/sync/src/lib.rs | 7 +- core/crates/sync/tests/lib.rs | 234 ---------- core/crates/sync/tests/mock_instance.rs | 143 ------ core/src/api/files.rs | 83 ++-- core/src/api/labels.rs | 31 +- core/src/api/search/saved.rs | 51 +-- core/src/api/tags.rs | 118 +++-- core/src/library/manager/mod.rs | 1 + core/src/location/manager/watcher/android.rs | 13 +- core/src/location/manager/watcher/ios.rs | 10 +- core/src/location/manager/watcher/linux.rs | 13 +- core/src/location/manager/watcher/macos.rs | 10 +- core/src/location/manager/watcher/mod.rs | 16 +- core/src/location/manager/watcher/utils.rs | 427 ++++++++---------- core/src/location/manager/watcher/windows.rs | 10 +- core/src/location/mod.rs | 295 +++++------- core/src/object/fs/old_copy.rs | 4 +- core/src/object/tag/mod.rs | 4 +- .../object/validation/old_validator_job.rs | 19 +- core/src/old_job/manager.rs | 1 + core/src/old_job/report.rs | 1 + core/src/volume/mod.rs | 95 ++-- crates/crypto/src/cloud/decrypt.rs | 2 +- crates/crypto/src/cloud/secret_key.rs | 2 +- crates/crypto/src/primitives.rs | 4 +- crates/sync-generator/src/model.rs | 41 +- crates/sync-generator/src/sync_data.rs | 139 ++++-- crates/sync/src/crdt.rs | 10 +- crates/sync/src/factory.rs | 57 ++- crates/utils/Cargo.toml | 2 + 47 files changed, 1118 insertions(+), 1760 deletions(-) delete mode 100644 core/crates/sync/tests/lib.rs delete mode 100644 core/crates/sync/tests/mock_instance.rs diff --git a/Cargo.lock b/Cargo.lock index 674aab99ecb23a43ded9898271510f0734fa0183..bbff9ba73223261d4949c704e553fa9abbeae1a0 100644 GIT binary patch delta 105 zcmZ3~A-c0ew4sHug=q`(saXce=E;T@CdnqLhRO(Lnn{Y8S(2fFiGiu5d6JQtiIJ&6 zVp_7fsj2z&fO{+w(|;~z;@&Pho4H1D`uZKrV$%&)G4o9KUct=WzJ3q$_Vs&MirWDN C03%)i delta 90 zcmdnlA-bqTw4sHug=q`(sab|;NhXGg#>pnBNy-SOWumcZijk?2iMf%biKUsPNvfq~ mVzQ-yQHo{aWJe*1>Aw~;ac!5K&0M3{zF`mZ_6>VjI@ Result<(devices::PubId, DateTime), Error> { // FIXME(@fogodev): If we don't have the key hash, we need to fetch it from another device in the group if possible let Some(secret_key) = key_manager.get_key(sync_group_pub_id, &key_hash).await else { return Err(Error::MissingKeyHash); }; - let response = http_client - .get(signed_download_link) - .send() - .await - .map_err(Error::DownloadSyncMessages)? - .error_for_status() - .map_err(Error::ErrorResponseDownloadSyncMessages)?; + debug!( + size = encrypted_messages.len(), + "Received encrypted sync messages collection" + ); - let crdt_ops = if let Some(size) = response.content_length() { - debug!(size, "Received encrypted sync messages collection"); - extract_messages_known_size(response, size, secret_key, original_device_pub_id).await - } else { - debug!("Received encrypted sync messages collection of unknown size"); - extract_messages_unknown_size(response, secret_key, original_device_pub_id).await - }?; + let crdt_ops = decrypt_messages(encrypted_messages, secret_key, original_device_pub_id).await?; assert_eq!( crdt_ops.len(), @@ -285,44 +264,28 @@ async fn handle_single_message( Ok((original_device_pub_id, end_time)) } -#[instrument(skip(response, size, secret_key), err)] -async fn extract_messages_known_size( - response: Response, - size: u64, +#[instrument(skip(encrypted_messages, secret_key), fields(messages_size = %encrypted_messages.len()), err)] +async fn decrypt_messages( + encrypted_messages: Vec, secret_key: SecretKey, devices::PubId(device_pub_id): devices::PubId, ) -> Result, Error> { - let plain_text = if size <= EncryptedBlock::CIPHER_TEXT_SIZE as u64 { - OneShotDecryption::decrypt( - &secret_key, - response - .bytes() - .await - .map_err(Error::ErrorResponseDownloadReadBytesSyncMessages)? - .as_ref() - .into(), - ) - .map_err(Error::Decrypt)? + let plain_text = if encrypted_messages.len() <= EncryptedBlock::CIPHER_TEXT_SIZE { + OneShotDecryption::decrypt(&secret_key, encrypted_messages.as_slice().into()) + .map_err(Error::Decrypt)? } else { - let mut reader = StreamReader::new(response.bytes_stream().map_err(|e| { - error!(?e, "Failed to read sync messages bytes stream"); - io::Error::new(io::ErrorKind::Other, e) - })); + let (nonce, cipher_text) = encrypted_messages.split_at(size_of::()); - let mut nonce = StreamNonce::default(); + let mut plain_text = Vec::with_capacity(cipher_text.len()); - reader - .read_exact(&mut nonce) - .await - .map_err(Error::ReadNonceStreamDecryption)?; - - // TODO: Reimplement using async streaming with serde if it ever gets implemented - - let mut plain_text = vec![]; - - StreamDecryption::decrypt(&secret_key, &nonce, reader, &mut plain_text) - .await - .map_err(Error::Decrypt)?; + StreamDecryption::decrypt( + &secret_key, + nonce.try_into().expect("we split the correct amount"), + cipher_text, + &mut plain_text, + ) + .await + .map_err(Error::Decrypt)?; plain_text }; @@ -332,34 +295,6 @@ async fn extract_messages_known_size( .map_err(Error::DeserializationFailureToPullSyncMessages) } -#[instrument(skip_all, err)] -async fn extract_messages_unknown_size( - response: Response, - secret_key: SecretKey, - devices::PubId(device_pub_id): devices::PubId, -) -> Result, Error> { - let plain_text = match UnknownDownloadKind::new(response).await? { - UnknownDownloadKind::OneShot(buffer) => { - OneShotDecryption::decrypt(&secret_key, buffer.as_slice().into()) - .map_err(Error::Decrypt)? - } - - UnknownDownloadKind::Stream((nonce, reader)) => { - let mut plain_text = vec![]; - - StreamDecryption::decrypt(&secret_key, &nonce, reader, &mut plain_text) - .await - .map_err(Error::Decrypt)?; - - plain_text - } - }; - - rmp_serde::from_slice::(&plain_text) - .map(|compressed_ops| compressed_ops.into_ops(device_pub_id)) - .map_err(Error::DeserializationFailureToPullSyncMessages) -} - #[instrument(skip_all, err)] pub async fn write_cloud_ops_to_db( ops: Vec, @@ -411,73 +346,3 @@ impl LastTimestampKeeper { .map_err(Error::FailedToWriteLastTimestampKeeper) } } - -struct UnknownDownloadSizeStreamer { - stream_reader: Box, - buffer: Vec, - was_read: usize, -} - -enum UnknownDownloadKind { - OneShot(Vec), - Stream((StreamNonce, UnknownDownloadSizeStreamer)), -} - -impl UnknownDownloadKind { - async fn new(response: Response) -> Result { - let mut buffer = Vec::with_capacity(EncryptedBlock::CIPHER_TEXT_SIZE * 2); - - let mut stream = response.bytes_stream(); - - while let Some(res) = stream.next().await { - buffer.extend(res.map_err(Error::ErrorResponseDownloadReadBytesSyncMessages)?); - if buffer.len() > EncryptedBlock::CIPHER_TEXT_SIZE { - break; - } - } - - if buffer.len() < size_of::() { - return Err(Error::IncompleteDownloadBytesSyncMessages); - } - - if buffer.len() <= EncryptedBlock::CIPHER_TEXT_SIZE { - Ok(Self::OneShot(buffer)) - } else { - let nonce_size = size_of::(); - - Ok(Self::Stream(( - StreamNonce::try_from(&buffer[..nonce_size]).expect("passing the right nonce size"), - UnknownDownloadSizeStreamer { - stream_reader: Box::new(StreamReader::new(stream.map_err(|e| { - error!(?e, "Failed to read sync messages bytes stream"); - io::Error::new(io::ErrorKind::Other, e) - }))), - buffer, - was_read: nonce_size, - }, - ))) - } - } -} - -impl AsyncRead for UnknownDownloadSizeStreamer { - fn poll_read( - mut self: Pin<&mut Self>, - cx: &mut Context<'_>, - buf: &mut ReadBuf<'_>, - ) -> Poll> { - if buf.remaining() == 0 { - return Poll::Ready(Ok(())); - } - - if self.was_read == self.buffer.len() { - Pin::new(&mut self.stream_reader).poll_read(cx, buf) - } else { - let len = std::cmp::min(self.buffer.len() - self.was_read, buf.remaining()); - buf.put_slice(&self.buffer[self.was_read..(self.was_read + len)]); - self.was_read += len; - - Poll::Ready(Ok(())) - } - } -} diff --git a/core/crates/cloud-services/src/sync/send.rs b/core/crates/cloud-services/src/sync/send.rs index 2e36b8118..4fd3842da 100644 --- a/core/crates/cloud-services/src/sync/send.rs +++ b/core/crates/cloud-services/src/sync/send.rs @@ -6,7 +6,7 @@ use sd_actors::{Actor, Stopper}; use sd_cloud_schema::{ devices, error::{ClientSideError, NotFoundError}, - sync::{self, groups, messages}, + sync::{groups, messages}, Client, Service, }; use sd_crypto::{ @@ -18,8 +18,7 @@ use sd_utils::{datetime_to_timestamp, timestamp_to_datetime}; use std::{ future::IntoFuture, - num::NonZero, - pin::{pin, Pin}, + pin::pin, sync::{ atomic::{AtomicBool, Ordering}, Arc, @@ -27,24 +26,20 @@ use std::{ time::{Duration, UNIX_EPOCH}, }; -use async_stream::try_stream; use chrono::{DateTime, Utc}; -use futures::{FutureExt, SinkExt, Stream, StreamExt, TryStream, TryStreamExt}; +use futures::{FutureExt, StreamExt, TryStreamExt}; use futures_concurrency::future::{Race, TryJoin}; -use quic_rpc::{client::UpdateSink, pattern::bidi_streaming, transport::quinn::QuinnConnection}; -use reqwest_middleware::reqwest::{header, Body}; +use quic_rpc::transport::quinn::QuinnConnection; use tokio::{ - spawn, - sync::{broadcast, oneshot, Notify, Semaphore}, + sync::{broadcast, Notify}, time::sleep, }; -use tracing::{debug, error}; +use tracing::error; use uuid::Uuid; use super::{SyncActors, ONE_MINUTE}; const TEN_SECONDS: Duration = Duration::from_secs(10); -const THIRTY_SECONDS: Duration = Duration::from_secs(30); const MESSAGES_COLLECTION_SIZE: u32 = 100_000; @@ -60,18 +55,6 @@ enum LoopStatus { type LatestTimestamp = NTP64; -type PushResponsesStream = Pin< - Box< - dyn Stream< - Item = Result< - Result, - bidi_streaming::ItemError>, - >, - > + Send - + Sync, - >, ->; - #[derive(Debug)] pub struct Sender { sync_group_pub_id: groups::PubId, @@ -205,17 +188,7 @@ impl Sender { let messages_bytes = rmp_serde::to_vec_named(&compressed_ops) .map_err(Error::SerializationFailureToPushSyncMessages)?; - let plain_text_size = messages_bytes.len(); - let expected_blob_size = if plain_text_size <= EncryptedBlock::PLAIN_TEXT_SIZE { - OneShotEncryption::cipher_text_size(&secret_key, plain_text_size) - } else { - StreamEncryption::cipher_text_size(&secret_key, plain_text_size) - } as u64; - - debug!(?expected_blob_size, ?key_hash, "Preparing sync message"); - - let (mut push_updates, mut push_responses) = self - .cloud_client + self.cloud_client .sync() .messages() .push(messages::push::Request { @@ -228,54 +201,15 @@ impl Sender { device_pub_id: current_device_pub_id, key_hash: key_hash.clone(), operations_count, - start_time, - end_time, - expected_blob_size, + time_range: (start_time, end_time), + encrypted_messages: encrypt_messages( + &secret_key, + &mut self.rng, + messages_bytes, + ) + .await?, }) - .await?; - - let Some(response) = push_responses.next().await else { - return Err(Error::EmptyResponse("push initial response")); - }; - - let messages::push::Response(response_kind) = response??; - - match response_kind { - messages::push::ResponseKind::SinglePresignedUrl(url) => { - upload_to_single_url( - url, - secret_key.clone(), - self.cloud_services.http_client(), - messages_bytes, - &mut self.rng, - ) - .await?; - } - messages::push::ResponseKind::ManyPresignedUrls(urls) => { - upload_to_many_urls( - urls, - secret_key.clone(), - self.cloud_services.http_client().clone(), - messages_bytes, - &mut self.rng, - &mut push_updates, - &mut push_responses, - ) - .await?; - } - messages::push::ResponseKind::Pong => { - return Err(Error::UnexpectedResponse( - "Pong on first messages push request", - )) - } - messages::push::ResponseKind::End => { - return Err(Error::UnexpectedResponse( - "End on first messages push request", - )) - } - } - - finalize_protocol(&mut push_updates, &mut push_responses).await?; + .await??; status = LoopStatus::SentMessages; } @@ -303,8 +237,7 @@ impl Sender { .get_access_token() .await?, group_pub_id: self.sync_group_pub_id, - current_device_pub_id, - kind: messages::get_latest_time::Kind::ForCurrentDevice, + kind: messages::get_latest_time::Kind::ForCurrentDevice(current_device_pub_id), }) .await? { @@ -328,320 +261,44 @@ impl Sender { } } -async fn finalize_protocol( - push_updates: &mut UpdateSink< - Service, - QuinnConnection, - messages::push::RequestUpdate, - sync::Service, - >, - push_responses: &mut PushResponsesStream, -) -> Result<(), Error> { - push_updates - .send(messages::push::RequestUpdate( - messages::push::UpdateKind::End, - )) - .await - .map_err(Error::EndUpdatePushSyncMessages)?; - - let Some(response) = push_responses.next().await else { - return Err(Error::EmptyResponse("push initial response")); - }; - - let messages::push::Response(response_kind) = response??; - - match response_kind { - messages::push::ResponseKind::SinglePresignedUrl(_) - | messages::push::ResponseKind::ManyPresignedUrls(_) => { - return Err(Error::UnexpectedResponse( - "Urls responses on final messages push response", - )) - } - messages::push::ResponseKind::Pong => { - return Err(Error::UnexpectedResponse( - "Pong on final message push response", - )) - } - messages::push::ResponseKind::End => { - /* - Everything is awesome! - */ - } - } - - Ok(()) -} - -async fn upload_to_many_urls( - urls: Vec, - secret_key: SecretKey, - http_client: reqwest_middleware::ClientWithMiddleware, - messages_bytes: Vec, +async fn encrypt_messages( + secret_key: &SecretKey, rng: &mut CryptoRng, - push_updates: &mut UpdateSink< - Service, - QuinnConnection, - messages::push::RequestUpdate, - sync::Service, - >, - push_responses: &mut PushResponsesStream, -) -> Result<(), Error> { - let stop_ping_pong = Arc::new(AtomicBool::new(false)); - let (out_tx, mut out_rx) = oneshot::channel(); - let rng = CryptoRng::from_seed(rng.generate_fixed()); - - let handle = spawn(handle_multipart_upload( - urls, - secret_key, - http_client, - messages_bytes, - rng, - Arc::clone(&stop_ping_pong), - out_tx, - )); - - loop { - if stop_ping_pong.load(Ordering::Acquire) { - break; - } - - if let Err(e) = push_updates - .send(messages::push::RequestUpdate( - messages::push::UpdateKind::Ping, - )) - .await - { - error!(?e, "Failed to send push ping update"); - sleep(TEN_SECONDS).await; - continue; - } - - let Some(response) = push_responses.next().await else { - error!("Empty response from push ping response"); - continue; - }; - - match response { - Ok(Ok(messages::push::Response( - messages::push::ResponseKind::SinglePresignedUrl(_) - | messages::push::ResponseKind::ManyPresignedUrls(_), - ))) => { - unreachable!("can't receive url if we didn't send an initial request") - } - - Ok(Ok(messages::push::Response(messages::push::ResponseKind::Pong))) => { - /* - Everything is awesome! - */ - } - Ok(Ok(messages::push::Response(messages::push::ResponseKind::End))) => { - unreachable!("Can't receive an End if we didn't send an End first"); - } - - Ok(Err(e)) => { - error!(?e, "Error from push ping response"); - sleep(TEN_SECONDS).await; - continue; - } - - Err(e) => { - error!(?e, "Error from push ping response"); - sleep(TEN_SECONDS).await; - continue; - } - } - - if stop_ping_pong.load(Ordering::Acquire) { - break; - } - - sleep(THIRTY_SECONDS).await; - } - - let Ok(out) = out_rx.try_recv() else { - // SAFETY: This try_recv error can only happen if the upload task panicked - // so we're good to unwrap the error. - let e = handle.await.expect_err("upload task panicked"); - error!(?e, "Critical error while uploading sync messages"); - return Err(Error::CriticalErrorWhileUploadingSyncMessages); - }; - - out -} - -async fn handle_multipart_upload( - urls: Vec, - secret_key: SecretKey, - http_client: reqwest_middleware::ClientWithMiddleware, messages_bytes: Vec, - rng: CryptoRng, - stop_ping_pong: Arc, - out_tx: oneshot::Sender>, -) { - async fn inner( - urls: Vec, - secret_key: SecretKey, - http_client: reqwest_middleware::ClientWithMiddleware, - messages_bytes: Vec, - mut rng: CryptoRng, - ) -> Result<(), Error> { - let urls_count = urls.len(); - let message_size = messages_bytes.len(); - let blocks_per_url = message_size / urls_count / EncryptedBlock::PLAIN_TEXT_SIZE; - let cipher_text_size = StreamEncryption::cipher_text_size(&secret_key, message_size); - - let parallel_upload_semaphore = Arc::new(Semaphore::new( - std::thread::available_parallelism() - .map(NonZero::get) - .unwrap_or(1), +) -> Result, Error> { + if messages_bytes.len() <= EncryptedBlock::PLAIN_TEXT_SIZE { + let mut nonce_and_cipher_text = Vec::with_capacity(OneShotEncryption::cipher_text_size( + secret_key, + messages_bytes.len(), )); - // If we're uploading to many URLs, it implies that the message size is bigger than a single - // encryption block, so we always use stream encryption. - - let mut buffers = vec![Vec::with_capacity(cipher_text_size / urls_count); urls_count]; - let (nonce, cipher_stream) = - StreamEncryption::encrypt(&secret_key, messages_bytes.as_slice(), &mut rng); - - buffers[0].extend_from_slice(&nonce); - - let mut cipher_stream = pin!(cipher_stream); - - let mut handles = Vec::with_capacity(urls_count); - - for (idx, (mut buffer, url)) in buffers.into_iter().zip(urls).enumerate() { - for _ in 0..blocks_per_url { - if let Some(cipher_res) = cipher_stream.next().await { - buffer.extend(cipher_res.map_err(Error::Encrypt)?); - } else { - return Err(Error::UnexpectedEndOfStream); - } - } - - handles.push(spawn(upload_part( - idx, - url, - http_client.clone(), - buffer, - Arc::clone(¶llel_upload_semaphore), - ))); - } - - assert!( - cipher_stream.next().await.is_none(), - "Unexpected ciphered bytes still on stream" - ); - - handles.try_join().await.map_err(|e| { - error!(?e, "Error while uploading sync messages"); - Error::CriticalErrorWhileUploadingSyncMessages - })?; - - Ok(()) - } - - let res = inner(urls, secret_key, http_client, messages_bytes, rng).await; - stop_ping_pong.store(true, Ordering::Release); - out_tx - .send(res) - .expect("upload output channel never closes"); -} - -async fn upload_part( - idx: usize, - url: reqwest::Url, - http_client: reqwest_middleware::ClientWithMiddleware, - buffer: Vec, - parallel_upload_semaphore: Arc, -) -> Result<(), Error> { - let _permit = parallel_upload_semaphore - .acquire() - .await - .expect("Semaphore never closes"); - - let response = http_client - .put(url) - .header(header::CONTENT_LENGTH, buffer.len()) - .body(buffer) - .send() - .await - .map_err(Error::UploadSyncMessages)? - .error_for_status() - .map_err(Error::ErrorResponseUploadSyncMessages)?; - - debug!(?response, idx, "Uploaded sync messages part"); - - Ok(()) -} - -async fn upload_to_single_url( - url: reqwest::Url, - secret_key: SecretKey, - http_client: &reqwest_middleware::ClientWithMiddleware, - messages_bytes: Vec, - rng: &mut CryptoRng, -) -> Result<(), Error> { - let (cipher_text_size, body) = if messages_bytes.len() <= EncryptedBlock::PLAIN_TEXT_SIZE { let EncryptedBlock { nonce, cipher_text } = - OneShotEncryption::encrypt(&secret_key, messages_bytes.as_slice(), rng) + OneShotEncryption::encrypt(secret_key, messages_bytes.as_slice(), rng) .map_err(Error::Encrypt)?; - let cipher_text_size = nonce.len() + cipher_text.len(); + nonce_and_cipher_text.extend_from_slice(nonce.as_slice()); + nonce_and_cipher_text.extend(&cipher_text); - let mut body_bytes = Vec::with_capacity(cipher_text_size); - body_bytes.extend_from_slice(nonce.as_slice()); - body_bytes.extend(&cipher_text); - - (cipher_text_size, Body::from(body_bytes)) + Ok(nonce_and_cipher_text) } else { let mut rng = CryptoRng::from_seed(rng.generate_fixed()); - let cipher_text_size = - StreamEncryption::cipher_text_size(&secret_key, messages_bytes.len()); + let mut nonce_and_cipher_text = Vec::with_capacity(StreamEncryption::cipher_text_size( + secret_key, + messages_bytes.len(), + )); - let body_bytes = stream_encryption(secret_key, messages_bytes, &mut rng) - .try_fold( - Vec::with_capacity(cipher_text_size), - |mut body_bytes, ciphered_chunk| async move { - body_bytes.extend(ciphered_chunk); - Ok(body_bytes) - }, - ) - .await?; - - (cipher_text_size, Body::from(body_bytes)) - }; - - http_client - .put(url) - .header(header::CONTENT_LENGTH, cipher_text_size) - .body(body) - .send() - .await - .map_err(Error::UploadSyncMessages)? - .error_for_status() - .map_err(Error::ErrorResponseUploadSyncMessages)?; - - Ok(()) -} - -fn stream_encryption( - secret_key: SecretKey, - messages_bytes: Vec, - rng: &mut CryptoRng, -) -> impl TryStream, Error = Error> + Send + 'static { - let mut rng = CryptoRng::from_seed(rng.generate_fixed()); - - try_stream! { let (nonce, cipher_stream) = - StreamEncryption::encrypt(&secret_key, messages_bytes.as_slice(), &mut rng); + StreamEncryption::encrypt(secret_key, messages_bytes.as_slice(), &mut rng); + + nonce_and_cipher_text.extend_from_slice(nonce.as_slice()); let mut cipher_stream = pin!(cipher_stream); - yield nonce.to_vec(); - - while let Some(res) = cipher_stream.next().await { - yield res.map_err(Error::Encrypt)?; + while let Some(ciphered_chunk) = cipher_stream.try_next().await.map_err(Error::Encrypt)? { + nonce_and_cipher_text.extend(ciphered_chunk); } + + Ok(nonce_and_cipher_text) } } diff --git a/core/crates/file-path-helper/src/isolated_file_path_data.rs b/core/crates/file-path-helper/src/isolated_file_path_data.rs index 3e89cce0f..fe83bbee9 100644 --- a/core/crates/file-path-helper/src/isolated_file_path_data.rs +++ b/core/crates/file-path-helper/src/isolated_file_path_data.rs @@ -2,7 +2,7 @@ use sd_core_prisma_helpers::{ file_path_for_file_identifier, file_path_for_media_processor, file_path_for_object_validator, file_path_to_full_path, file_path_to_handle_custom_uri, file_path_to_handle_p2p_serve_file, file_path_to_isolate, file_path_to_isolate_with_id, file_path_to_isolate_with_pub_id, - file_path_walker, file_path_with_object, + file_path_walker, file_path_watcher_remove, file_path_with_object, }; use sd_prisma::prisma::{file_path, location}; @@ -506,7 +506,8 @@ impl_from_db!( file_path_to_isolate_with_pub_id, file_path_walker, file_path_to_isolate_with_id, - file_path_with_object + file_path_with_object, + file_path_watcher_remove ); impl_from_db_without_location_id!( diff --git a/core/crates/heavy-lifting/src/file_identifier/job.rs b/core/crates/heavy-lifting/src/file_identifier/job.rs index a75bbb2ca..249ea57f2 100644 --- a/core/crates/heavy-lifting/src/file_identifier/job.rs +++ b/core/crates/heavy-lifting/src/file_identifier/job.rs @@ -14,7 +14,11 @@ use crate::{ use sd_core_file_path_helper::IsolatedFilePathData; use sd_core_prisma_helpers::{file_path_for_file_identifier, CasId}; -use sd_prisma::prisma::{device, file_path, location, SortOrder}; +use sd_prisma::{ + prisma::{device, file_path, location, SortOrder}, + prisma_sync, +}; +use sd_sync::{sync_db_not_null_entry, OperationFactory}; use sd_task_system::{ AnyTaskOutput, IntoTask, SerializableTask, Task, TaskDispatcher, TaskHandle, TaskId, TaskOutput, TaskStatus, @@ -267,15 +271,25 @@ impl Job for FileIdentifier { .. } = self; - ctx.db() - .location() - .update( - location::id::equals(location.id), - vec![location::scan_state::set( - LocationScanState::FilesIdentified as i32, - )], + let (sync_param, db_param) = sync_db_not_null_entry!( + LocationScanState::FilesIdentified as i32, + location::scan_state + ); + + ctx.sync() + .write_op( + ctx.db(), + ctx.sync().shared_update( + prisma_sync::location::SyncId { + pub_id: location.pub_id.clone(), + }, + [sync_param], + ), + ctx.db() + .location() + .update(location::id::equals(location.id), vec![db_param]) + .select(location::select!({ id })), ) - .exec() .await .map_err(file_identifier::Error::from)?; diff --git a/core/crates/heavy-lifting/src/file_identifier/tasks/identifier.rs b/core/crates/heavy-lifting/src/file_identifier/tasks/identifier.rs index 22d244cac..125a72713 100644 --- a/core/crates/heavy-lifting/src/file_identifier/tasks/identifier.rs +++ b/core/crates/heavy-lifting/src/file_identifier/tasks/identifier.rs @@ -12,11 +12,11 @@ use sd_prisma::{ prisma::{device, file_path, location, PrismaClient}, prisma_sync, }; -use sd_sync::OperationFactory; +use sd_sync::{sync_db_entry, OperationFactory}; use sd_task_system::{ ExecStatus, Interrupter, InterruptionKind, IntoAnyTaskOutput, SerializableTask, Task, TaskId, }; -use sd_utils::{error::FileIOError, msgpack}; +use sd_utils::error::FileIOError; use std::{ collections::HashMap, convert::identity, future::IntoFuture, mem, path::PathBuf, pin::pin, @@ -403,19 +403,17 @@ async fn assign_cas_id_to_file_paths( let (ops, queries) = identified_files .iter() .map(|(pub_id, IdentifiedFile { cas_id, .. })| { + let (sync_param, db_param) = sync_db_entry!(cas_id, file_path::cas_id); + ( sync.shared_update( prisma_sync::file_path::SyncId { pub_id: pub_id.to_db(), }, - file_path::cas_id::NAME, - msgpack!(cas_id), + [sync_param], ), db.file_path() - .update( - file_path::pub_id::equals(pub_id.to_db()), - vec![file_path::cas_id::set(cas_id.into())], - ) + .update(file_path::pub_id::equals(pub_id.to_db()), vec![db_param]) // We don't need any data here, just the id avoids receiving the entire object // as we can't pass an empty select macro call .select(file_path::select!({ id })), diff --git a/core/crates/heavy-lifting/src/file_identifier/tasks/mod.rs b/core/crates/heavy-lifting/src/file_identifier/tasks/mod.rs index 13e4d7d9f..59f75d0a9 100644 --- a/core/crates/heavy-lifting/src/file_identifier/tasks/mod.rs +++ b/core/crates/heavy-lifting/src/file_identifier/tasks/mod.rs @@ -9,7 +9,7 @@ use sd_prisma::{ prisma_sync, }; use sd_sync::{option_sync_db_entry, sync_db_entry, sync_entry, CRDTOperation, OperationFactory}; -use sd_utils::{chain_optional_iter, msgpack}; +use sd_utils::chain_optional_iter; use std::collections::{HashMap, HashSet}; @@ -47,10 +47,12 @@ fn connect_file_path_to_object<'db>( prisma_sync::file_path::SyncId { pub_id: file_path_pub_id.to_db(), }, - file_path::object::NAME, - msgpack!(prisma_sync::object::SyncId { - pub_id: object_pub_id.to_db(), - }), + [sync_entry!( + prisma_sync::object::SyncId { + pub_id: object_pub_id.to_db(), + }, + file_path::object + )], ), db.file_path() .update( diff --git a/core/crates/heavy-lifting/src/indexer/job.rs b/core/crates/heavy-lifting/src/indexer/job.rs index d910fec81..cf19fbb90 100644 --- a/core/crates/heavy-lifting/src/indexer/job.rs +++ b/core/crates/heavy-lifting/src/indexer/job.rs @@ -16,7 +16,11 @@ use sd_core_file_path_helper::IsolatedFilePathData; use sd_core_indexer_rules::{IndexerRule, IndexerRuler}; use sd_core_prisma_helpers::location_with_indexer_rules; -use sd_prisma::prisma::{device, location}; +use sd_prisma::{ + prisma::{device, location}, + prisma_sync, +}; +use sd_sync::{sync_db_not_null_entry, OperationFactory}; use sd_task_system::{ AnyTaskOutput, IntoTask, SerializableTask, Task, TaskDispatcher, TaskHandle, TaskId, TaskOutput, TaskStatus, @@ -269,7 +273,7 @@ impl Job for Indexer { .await?; } - update_location_size(location.id, ctx.db(), &ctx).await?; + update_location_size(location.id, location.pub_id.clone(), &ctx).await?; metadata.mean_db_write_time += start_size_update_time.elapsed(); } @@ -287,13 +291,23 @@ impl Job for Indexer { "all tasks must be completed here" ); - ctx.db() - .location() - .update( - location::id::equals(location.id), - vec![location::scan_state::set(LocationScanState::Indexed as i32)], + let (sync_param, db_param) = + sync_db_not_null_entry!(LocationScanState::Indexed as i32, location::scan_state); + + ctx.sync() + .write_op( + ctx.db(), + ctx.sync().shared_update( + prisma_sync::location::SyncId { + pub_id: location.pub_id.clone(), + }, + [sync_param], + ), + ctx.db() + .location() + .update(location::id::equals(location.id), vec![db_param]) + .select(location::select!({ id })), ) - .exec() .await .map_err(indexer::Error::from)?; diff --git a/core/crates/heavy-lifting/src/indexer/mod.rs b/core/crates/heavy-lifting/src/indexer/mod.rs index 1ad78902b..6880e6d91 100644 --- a/core/crates/heavy-lifting/src/indexer/mod.rs +++ b/core/crates/heavy-lifting/src/indexer/mod.rs @@ -10,11 +10,11 @@ use sd_prisma::{ prisma::{file_path, indexer_rule, location, PrismaClient, SortOrder}, prisma_sync, }; -use sd_sync::OperationFactory; +use sd_sync::{sync_db_entry, OperationFactory}; use sd_utils::{ db::{size_in_bytes_from_db, size_in_bytes_to_db, MissingFieldError}, error::{FileIOError, NonUtf8PathError}, - from_bytes_to_uuid, msgpack, + from_bytes_to_uuid, }; use std::{ @@ -146,22 +146,20 @@ async fn update_directory_sizes( .map(|file_path| { let size_bytes = iso_paths_and_sizes .get(&IsolatedFilePathData::try_from(&file_path)?) - .map(|size| size.to_be_bytes().to_vec()) + .map(|size| size_in_bytes_to_db(*size)) .expect("must be here"); + let (sync_param, db_param) = sync_db_entry!(size_bytes, file_path::size_in_bytes_bytes); + Ok(( sync.shared_update( prisma_sync::file_path::SyncId { pub_id: file_path.pub_id.clone(), }, - file_path::size_in_bytes_bytes::NAME, - msgpack!(size_bytes), + [sync_param], ), db.file_path() - .update( - file_path::pub_id::equals(file_path.pub_id), - vec![file_path::size_in_bytes_bytes::set(Some(size_bytes))], - ) + .update(file_path::pub_id::equals(file_path.pub_id), vec![db_param]) .select(file_path::select!({ id })), )) }) @@ -178,35 +176,45 @@ async fn update_directory_sizes( async fn update_location_size( location_id: location::id::Type, - db: &PrismaClient, + location_pub_id: location::pub_id::Type, ctx: &impl OuterContext, ) -> Result<(), Error> { - let total_size = db - .file_path() - .find_many(vec![ - file_path::location_id::equals(Some(location_id)), - file_path::materialized_path::equals(Some("/".to_string())), - ]) - .select(file_path::select!({ size_in_bytes_bytes })) - .exec() - .await? - .into_iter() - .filter_map(|file_path| { - file_path - .size_in_bytes_bytes - .map(|size_in_bytes_bytes| size_in_bytes_from_db(&size_in_bytes_bytes)) - }) - .sum::(); + let db = ctx.db(); + let sync = ctx.sync(); - db.location() - .update( - location::id::equals(location_id), - vec![location::size_in_bytes::set(Some( - total_size.to_be_bytes().to_vec(), - ))], - ) - .exec() - .await?; + let total_size = size_in_bytes_to_db( + db.file_path() + .find_many(vec![ + file_path::location_id::equals(Some(location_id)), + file_path::materialized_path::equals(Some("/".to_string())), + ]) + .select(file_path::select!({ size_in_bytes_bytes })) + .exec() + .await? + .into_iter() + .filter_map(|file_path| { + file_path + .size_in_bytes_bytes + .map(|size_in_bytes_bytes| size_in_bytes_from_db(&size_in_bytes_bytes)) + }) + .sum::(), + ); + + let (sync_param, db_param) = sync_db_entry!(total_size, location::size_in_bytes); + + sync.write_op( + db, + sync.shared_update( + prisma_sync::location::SyncId { + pub_id: location_pub_id, + }, + [sync_param], + ), + db.location() + .update(location::id::equals(location_id), vec![db_param]) + .select(location::select!({ id })), + ) + .await?; ctx.invalidate_query("locations.list"); ctx.invalidate_query("locations.get"); @@ -334,18 +342,19 @@ pub async fn reverse_update_directories_sizes( { let size_bytes = size_in_bytes_to_db(size); + let (sync_param, db_param) = + sync_db_entry!(size_bytes, file_path::size_in_bytes_bytes); + Some(( sync.shared_update( prisma_sync::file_path::SyncId { pub_id: pub_id.clone(), }, - file_path::size_in_bytes_bytes::NAME, - msgpack!(size_bytes), - ), - db.file_path().update( - file_path::pub_id::equals(pub_id), - vec![file_path::size_in_bytes_bytes::set(Some(size_bytes))], + [sync_param], ), + db.file_path() + .update(file_path::pub_id::equals(pub_id), vec![db_param]) + .select(file_path::select!({ id })), )) } else { warn!("Got a missing ancestor for a file_path in the database, ignoring..."); diff --git a/core/crates/heavy-lifting/src/indexer/shallow.rs b/core/crates/heavy-lifting/src/indexer/shallow.rs index 90a22eead..1bc55b556 100644 --- a/core/crates/heavy-lifting/src/indexer/shallow.rs +++ b/core/crates/heavy-lifting/src/indexer/shallow.rs @@ -136,7 +136,7 @@ pub async fn shallow( .await?; } - update_location_size(location.id, db, ctx).await?; + update_location_size(location.id, location.pub_id, ctx).await?; } if indexed_count > 0 || removed_count > 0 { diff --git a/core/crates/heavy-lifting/src/indexer/tasks/saver.rs b/core/crates/heavy-lifting/src/indexer/tasks/saver.rs index 9fbe24554..c5d0951d0 100644 --- a/core/crates/heavy-lifting/src/indexer/tasks/saver.rs +++ b/core/crates/heavy-lifting/src/indexer/tasks/saver.rs @@ -9,10 +9,7 @@ use sd_prisma::{ }; use sd_sync::{sync_db_entry, sync_entry, OperationFactory}; use sd_task_system::{ExecStatus, Interrupter, IntoAnyTaskOutput, SerializableTask, Task, TaskId}; -use sd_utils::{ - db::{inode_to_db, size_in_bytes_to_db}, - msgpack, -}; +use sd_utils::db::{inode_to_db, size_in_bytes_to_db}; use std::{sync::Arc, time::Duration}; @@ -121,13 +118,13 @@ impl Task for Saver { new file_paths and they were not identified yet" ); - let (sync_params, db_params): (Vec<_>, Vec<_>) = [ + let (sync_params, db_params) = [ ( - ( - location::NAME, - msgpack!(prisma_sync::location::SyncId { + sync_entry!( + prisma_sync::location::SyncId { pub_id: location_pub_id.clone() - }), + }, + location ), location_id::set(Some(*location_id)), ), @@ -152,7 +149,7 @@ impl Task for Saver { ), ] .into_iter() - .unzip(); + .unzip::<_, _, Vec<_>, Vec<_>>(); ( sync.shared_create( diff --git a/core/crates/heavy-lifting/src/indexer/tasks/updater.rs b/core/crates/heavy-lifting/src/indexer/tasks/updater.rs index 91eb72899..80cf3d6f4 100644 --- a/core/crates/heavy-lifting/src/indexer/tasks/updater.rs +++ b/core/crates/heavy-lifting/src/indexer/tasks/updater.rs @@ -93,7 +93,7 @@ impl Task for Updater { check_interruption!(interrupter); - let (sync_stuff, paths_to_update) = walked_entries + let (crdt_ops, paths_to_update) = walked_entries .drain(..) .map( |WalkedEntry { @@ -138,18 +138,12 @@ impl Task for Updater { .unzip::<_, _, Vec<_>, Vec<_>>(); ( - sync_params - .into_iter() - .map(|(field, value)| { - sync.shared_update( - prisma_sync::file_path::SyncId { - pub_id: pub_id.to_db(), - }, - field, - value, - ) - }) - .collect::>(), + sync.shared_update( + prisma_sync::file_path::SyncId { + pub_id: pub_id.to_db(), + }, + sync_params, + ), db.file_path() .update(file_path::pub_id::equals(pub_id.into()), db_params) // selecting id to avoid fetching whole object from database @@ -159,9 +153,7 @@ impl Task for Updater { ) .unzip::<_, _, Vec<_>, Vec<_>>(); - let ops = sync_stuff.into_iter().flatten().collect::>(); - - if ops.is_empty() && paths_to_update.is_empty() { + if crdt_ops.is_empty() && paths_to_update.is_empty() { return Ok(ExecStatus::Done( Output { updated_count: 0, @@ -172,7 +164,7 @@ impl Task for Updater { } let updated = sync - .write_ops(db, (ops, paths_to_update)) + .write_ops(db, (crdt_ops, paths_to_update)) .await .map_err(indexer::Error::from)?; diff --git a/core/crates/heavy-lifting/src/job_system/report.rs b/core/crates/heavy-lifting/src/job_system/report.rs index 3d536e7dd..b747b8195 100644 --- a/core/crates/heavy-lifting/src/job_system/report.rs +++ b/core/crates/heavy-lifting/src/job_system/report.rs @@ -290,6 +290,7 @@ impl Report { .map(|id| job::parent::connect(job::id::equals(id.as_bytes().to_vec())))], ), ) + .select(job::select!({ id })) .exec() .await .map_err(ReportError::Create)?; @@ -318,6 +319,7 @@ impl Report { job::date_completed::set(self.completed_at.map(Into::into)), ], ) + .select(job::select!({ id })) .exec() .await .map_err(ReportError::Update)?; diff --git a/core/crates/heavy-lifting/src/media_processor/job.rs b/core/crates/heavy-lifting/src/media_processor/job.rs index cadeb5f03..fb622e162 100644 --- a/core/crates/heavy-lifting/src/media_processor/job.rs +++ b/core/crates/heavy-lifting/src/media_processor/job.rs @@ -14,7 +14,11 @@ use sd_core_file_path_helper::IsolatedFilePathData; use sd_core_prisma_helpers::file_path_for_media_processor; use sd_file_ext::extensions::Extension; -use sd_prisma::prisma::{location, PrismaClient}; +use sd_prisma::{ + prisma::{location, PrismaClient}, + prisma_sync, +}; +use sd_sync::{sync_db_not_null_entry, OperationFactory}; use sd_task_system::{ AnyTaskOutput, IntoTask, SerializableTask, Task, TaskDispatcher, TaskHandle, TaskId, TaskOutput, TaskStatus, TaskSystemError, @@ -214,15 +218,23 @@ impl Job for MediaProcessor { .. } = self; - ctx.db() - .location() - .update( - location::id::equals(location.id), - vec![location::scan_state::set( - LocationScanState::Completed as i32, - )], + let (sync_param, db_param) = + sync_db_not_null_entry!(LocationScanState::Completed as i32, location::scan_state); + + ctx.sync() + .write_op( + ctx.db(), + ctx.sync().shared_update( + prisma_sync::location::SyncId { + pub_id: location.pub_id.clone(), + }, + [sync_param], + ), + ctx.db() + .location() + .update(location::id::equals(location.id), vec![db_param]) + .select(location::select!({ id })), ) - .exec() .await .map_err(media_processor::Error::from)?; diff --git a/core/crates/prisma-helpers/src/lib.rs b/core/crates/prisma-helpers/src/lib.rs index 48a400e65..ee8f11bb9 100644 --- a/core/crates/prisma-helpers/src/lib.rs +++ b/core/crates/prisma-helpers/src/lib.rs @@ -74,6 +74,20 @@ file_path::select!(file_path_for_media_processor { pub_id } }); +file_path::select!(file_path_watcher_remove { + id + pub_id + location_id + materialized_path + is_dir + name + extension + object: select { + id + pub_id + } + +}); file_path::select!(file_path_to_isolate { location_id materialized_path diff --git a/core/crates/sync/src/ingest_utils.rs b/core/crates/sync/src/ingest_utils.rs index 3cc4c8a68..6c77a96b7 100644 --- a/core/crates/sync/src/ingest_utils.rs +++ b/core/crates/sync/src/ingest_utils.rs @@ -1,7 +1,7 @@ use sd_core_prisma_helpers::DevicePubId; use sd_prisma::{ - prisma::{crdt_operation, PrismaClient, SortOrder}, + prisma::{crdt_operation, PrismaClient}, prisma_sync::ModelSyncData, }; use sd_sync::{ @@ -17,6 +17,8 @@ use uuid::Uuid; use super::{db_operation::write_crdt_op_to_db, Error, TimestampPerDevice}; +crdt_operation::select!(crdt_operation_id { id }); + // where the magic happens #[instrument(skip(clock, ops), fields(operations_count = %ops.len()), err)] pub async fn process_crdt_operations( @@ -24,7 +26,7 @@ pub async fn process_crdt_operations( timestamp_per_device: &TimestampPerDevice, db: &PrismaClient, device_pub_id: DevicePubId, - model: ModelId, + model_id: ModelId, record_id: RecordId, mut ops: Vec, ) -> Result<(), Error> { @@ -50,7 +52,7 @@ pub async fn process_crdt_operations( .find(|op| matches!(op.data, CRDTOperationData::Delete)) { trace!("Deleting operation"); - handle_crdt_deletion(db, &device_pub_id, model, record_id, delete_op).await?; + handle_crdt_deletion(db, &device_pub_id, model_id, record_id, delete_op).await?; } // Create + > 0 Update - overwrites the create's data with the updates else if let Some(timestamp) = ops @@ -61,23 +63,22 @@ pub async fn process_crdt_operations( trace!("Create + Updates operations"); // conflict resolution - let delete = db + let delete_count = db .crdt_operation() - .find_first(vec![ - crdt_operation::model::equals(i32::from(model)), + .count(vec![ + crdt_operation::model::equals(i32::from(model_id)), crdt_operation::record_id::equals(rmp_serde::to_vec(&record_id)?), crdt_operation::kind::equals(OperationKind::Delete.to_string()), ]) - .order_by(crdt_operation::timestamp::order(SortOrder::Desc)) .exec() .await?; - if delete.is_some() { + if delete_count > 0 { debug!("Found a previous delete operation with the same SyncId, will ignore these operations"); return Ok(()); } - handle_crdt_create_and_updates(db, &device_pub_id, model, record_id, ops, timestamp) + handle_crdt_create_and_updates(db, &device_pub_id, model_id, record_id, ops, timestamp) .await?; } // > 0 Update - batches updates with a fake Create op @@ -87,51 +88,57 @@ pub async fn process_crdt_operations( let mut data = BTreeMap::new(); for op in ops.into_iter().rev() { - let CRDTOperationData::Update { field, value } = op.data else { + let CRDTOperationData::Update(fields_and_values) = op.data else { unreachable!("Create + Delete should be filtered out!"); }; - data.insert(field, (value, op.timestamp)); + for (field, value) in fields_and_values { + data.insert(field, (value, op.timestamp)); + } } // conflict resolution - let (create, updates) = db + let (create, newer_updates_count) = db ._batch(( - db.crdt_operation() - .find_first(vec![ - crdt_operation::model::equals(i32::from(model)), - crdt_operation::record_id::equals(rmp_serde::to_vec(&record_id)?), - crdt_operation::kind::equals(OperationKind::Create.to_string()), - ]) - .order_by(crdt_operation::timestamp::order(SortOrder::Desc)), + db.crdt_operation().count(vec![ + crdt_operation::model::equals(i32::from(model_id)), + crdt_operation::record_id::equals(rmp_serde::to_vec(&record_id)?), + crdt_operation::kind::equals(OperationKind::Create.to_string()), + ]), data.iter() .map(|(k, (_, timestamp))| { - Ok(db - .crdt_operation() - .find_first(vec![ - crdt_operation::timestamp::gt({ - #[allow(clippy::cast_possible_wrap)] - // SAFETY: we had to store using i64 due to SQLite limitations - { - timestamp.as_u64() as i64 - } - }), - crdt_operation::model::equals(i32::from(model)), - crdt_operation::record_id::equals(rmp_serde::to_vec(&record_id)?), - crdt_operation::kind::equals(OperationKind::Update(k).to_string()), - ]) - .order_by(crdt_operation::timestamp::order(SortOrder::Desc))) + Ok(db.crdt_operation().count(vec![ + crdt_operation::timestamp::gt({ + #[allow(clippy::cast_possible_wrap)] + // SAFETY: we had to store using i64 due to SQLite limitations + { + timestamp.as_u64() as i64 + } + }), + crdt_operation::model::equals(i32::from(model_id)), + crdt_operation::record_id::equals(rmp_serde::to_vec(&record_id)?), + crdt_operation::kind::contains(format!(":{k}:")), + ])) }) .collect::, Error>>()?, )) .await?; - if create.is_none() { + if create == 0 { warn!("Failed to find a previous create operation with the same SyncId"); return Ok(()); } - handle_crdt_updates(db, &device_pub_id, model, record_id, data, updates).await?; + let keys = data.keys().cloned().collect::>(); + + // remove entries if we possess locally more recent updates for this field + for (update, key) in newer_updates_count.into_iter().zip(keys) { + if update > 0 { + data.remove(&key); + } + } + + handle_crdt_updates(db, &device_pub_id, model_id, record_id, data).await?; } // read the timestamp for the operation's device, or insert one if it doesn't exist @@ -157,24 +164,15 @@ async fn handle_crdt_updates( device_pub_id: &DevicePubId, model_id: ModelId, record_id: rmpv::Value, - mut data: BTreeMap, - updates: Vec>, + data: BTreeMap, ) -> Result<(), Error> { - let keys = data.keys().cloned().collect::>(); let device_pub_id = sd_sync::DevicePubId::from(device_pub_id); - // does the same thing as processing ops one-by-one and returning early if a newer op was found - for (update, key) in updates.into_iter().zip(keys) { - if update.is_some() { - data.remove(&key); - } - } - db._transaction() .with_timeout(30 * 10000) .with_max_wait(30 * 10000) .run(|db| async move { - // fake operation to batch them all at once + // fake operation to batch them all at once, inserting the latest data on appropriate table ModelSyncData::from_op(CRDTOperation { device_pub_id, model_id, @@ -185,35 +183,32 @@ async fn handle_crdt_updates( .map(|(k, (data, _))| (k.clone(), data.clone())) .collect(), ), - }) - .ok_or(Error::InvalidModelId(model_id))? + })? .exec(&db) .await?; - // need to only apply ops that haven't been filtered out - data.into_iter() - .map(|(field, (value, timestamp))| { - let record_id = record_id.clone(); - let db = &db; - - async move { - write_crdt_op_to_db( - &CRDTOperation { - device_pub_id, - model_id, - record_id, - timestamp, - data: CRDTOperationData::Update { field, value }, - }, - db, - ) - .await + let (fields_and_values, latest_timestamp) = data.into_iter().fold( + (BTreeMap::new(), NTP64::default()), + |(mut fields_and_values, mut latest_time_stamp), (field, (value, timestamp))| { + fields_and_values.insert(field, value); + if timestamp > latest_time_stamp { + latest_time_stamp = timestamp; } - }) - .collect::>() - .try_join() - .await - .map(|_| ()) + (fields_and_values, latest_time_stamp) + }, + ); + + write_crdt_op_to_db( + &CRDTOperation { + device_pub_id, + model_id, + record_id, + timestamp: latest_timestamp, + data: CRDTOperationData::Update(fields_and_values), + }, + &db, + ) + .await }) .await } @@ -244,8 +239,11 @@ async fn handle_crdt_create_and_updates( break; } - CRDTOperationData::Update { field, value } => { - data.insert(field.clone(), value.clone()); + CRDTOperationData::Update(fields_and_values) => { + for (field, value) in fields_and_values { + data.insert(field.clone(), value.clone()); + } + applied_ops.push(op); } } @@ -262,8 +260,7 @@ async fn handle_crdt_create_and_updates( record_id: record_id.clone(), timestamp, data: CRDTOperationData::Create(data), - }) - .ok_or(Error::InvalidModelId(model_id))? + })? .exec(&db) .await?; @@ -314,10 +311,7 @@ async fn handle_crdt_deletion( .with_timeout(30 * 10000) .with_max_wait(30 * 10000) .run(|db| async move { - ModelSyncData::from_op(op.clone()) - .ok_or(Error::InvalidModelId(model))? - .exec(&db) - .await?; + ModelSyncData::from_op(op.clone())?.exec(&db).await?; write_crdt_op_to_db(&op, &db).await }) diff --git a/core/crates/sync/src/lib.rs b/core/crates/sync/src/lib.rs index 903fb812b..56822509c 100644 --- a/core/crates/sync/src/lib.rs +++ b/core/crates/sync/src/lib.rs @@ -27,7 +27,10 @@ #![forbid(deprecated_in_future)] #![allow(clippy::missing_errors_doc, clippy::module_name_repetitions)] -use sd_prisma::prisma::{cloud_crdt_operation, crdt_operation}; +use sd_prisma::{ + prisma::{cloud_crdt_operation, crdt_operation}, + prisma_sync, +}; use sd_utils::uuid_to_bytes; use std::{collections::HashMap, sync::Arc}; @@ -66,6 +69,8 @@ pub enum Error { Deserialization(#[from] rmp_serde::decode::Error), #[error("database error: {0}")] Database(#[from] prisma_client_rust::QueryError), + #[error("PrismaSync error: {0}")] + PrismaSync(#[from] prisma_sync::Error), #[error("invalid model id: {0}")] InvalidModelId(ModelId), #[error("tried to write an empty operations list")] diff --git a/core/crates/sync/tests/lib.rs b/core/crates/sync/tests/lib.rs deleted file mode 100644 index 5c9dbf584..000000000 --- a/core/crates/sync/tests/lib.rs +++ /dev/null @@ -1,234 +0,0 @@ -// mod mock_instance; - -// use sd_core_sync::*; - -// use sd_prisma::{prisma::location, prisma_sync}; -// use sd_sync::*; -// use sd_utils::{msgpack, uuid_to_bytes}; - -// use mock_instance::Device; -// use tracing::info; -// use tracing_test::traced_test; -// use uuid::Uuid; - -// const MOCK_LOCATION_NAME: &str = "Location 0"; -// const MOCK_LOCATION_PATH: &str = "/User/Anon/Documents"; - -// async fn write_test_location(instance: &Device) -> location::Data { -// let location_pub_id = Uuid::new_v4(); - -// let (sync_ops, db_ops): (Vec<_>, Vec<_>) = [ -// sync_db_entry!(MOCK_LOCATION_NAME, location::name), -// sync_db_entry!(MOCK_LOCATION_PATH, location::path), -// ] -// .into_iter() -// .unzip(); - -// let location = instance -// .sync -// .write_op( -// &instance.db, -// instance.sync.shared_create( -// prisma_sync::location::SyncId { -// pub_id: uuid_to_bytes(&location_pub_id), -// }, -// sync_ops, -// ), -// instance -// .db -// .location() -// .create(uuid_to_bytes(&location_pub_id), db_ops), -// ) -// .await -// .expect("failed to create mock location"); - -// instance -// .sync -// .write_ops(&instance.db, { -// let (sync_ops, db_ops): (Vec<_>, Vec<_>) = [ -// sync_db_entry!(1024, location::total_capacity), -// sync_db_entry!(512, location::available_capacity), -// ] -// .into_iter() -// .unzip(); - -// ( -// sync_ops -// .into_iter() -// .map(|(k, v)| { -// instance.sync.shared_update( -// prisma_sync::location::SyncId { -// pub_id: uuid_to_bytes(&location_pub_id), -// }, -// k, -// v, -// ) -// }) -// .collect::>(), -// instance -// .db -// .location() -// .update(location::id::equals(location.id), db_ops), -// ) -// }) -// .await -// .expect("failed to create mock location"); - -// location -// } - -// #[tokio::test] -// #[traced_test] -// async fn writes_operations_and_rows_together() -> Result<(), Box> { -// let instance = Device::new(Uuid::new_v4()).await; - -// write_test_location(&instance).await; - -// let operations = instance -// .db -// .crdt_operation() -// .find_many(vec![]) -// .exec() -// .await?; - -// // 1 create, 2 update -// assert_eq!(operations.len(), 3); -// assert_eq!(operations[0].model, prisma_sync::location::MODEL_ID as i32); - -// let out = instance.sync.get_ops(100, vec![]).await?; - -// assert_eq!(out.len(), 3); - -// let locations = instance.db.location().find_many(vec![]).exec().await?; - -// assert_eq!(locations.len(), 1); -// let location = locations.first().unwrap(); -// assert_eq!(location.name.as_deref(), Some(MOCK_LOCATION_NAME)); -// assert_eq!(location.path.as_deref(), Some(MOCK_LOCATION_PATH)); - -// Ok(()) -// } - -// #[tokio::test] -// #[traced_test] -// async fn operations_send_and_ingest() -> Result<(), Box> { -// let instance1 = Device::new(Uuid::new_v4()).await; -// let instance2 = Device::new(Uuid::new_v4()).await; - -// let mut instance2_sync_rx = instance2.sync_rx.resubscribe(); - -// info!("Created instances!"); - -// Device::pair(&instance1, &instance2).await; - -// info!("Paired instances!"); - -// write_test_location(&instance1).await; - -// info!("Created mock location!"); - -// assert!(matches!( -// instance2_sync_rx.recv().await?, -// SyncEvent::Ingested -// )); - -// let out = instance2.sync.get_ops(100, vec![]).await?; - -// assert_locations_equality( -// &instance1.db.location().find_many(vec![]).exec().await?[0], -// &instance2.db.location().find_many(vec![]).exec().await?[0], -// ); - -// assert_eq!(out.len(), 3); - -// instance1.teardown().await; -// instance2.teardown().await; - -// Ok(()) -// } - -// #[tokio::test] -// async fn no_update_after_delete() -> Result<(), Box> { -// let instance1 = Device::new(Uuid::new_v4()).await; -// let instance2 = Device::new(Uuid::new_v4()).await; - -// let mut instance2_sync_rx = instance2.sync_rx.resubscribe(); - -// Device::pair(&instance1, &instance2).await; - -// let location = write_test_location(&instance1).await; - -// assert!(matches!( -// instance2_sync_rx.recv().await?, -// SyncEvent::Ingested -// )); - -// instance2 -// .sync -// .write_op( -// &instance2.db, -// instance2.sync.shared_delete(prisma_sync::location::SyncId { -// pub_id: location.pub_id.clone(), -// }), -// instance2.db.location().delete_many(vec![]), -// ) -// .await?; - -// assert!(matches!( -// instance1.sync_rx.resubscribe().recv().await?, -// SyncEvent::Ingested -// )); - -// instance1 -// .sync -// .write_op( -// &instance1.db, -// instance1.sync.shared_update( -// prisma_sync::location::SyncId { -// pub_id: location.pub_id.clone(), -// }, -// "name", -// msgpack!("New Location"), -// ), -// instance1.db.location().find_many(vec![]), -// ) -// .await?; - -// // one spare update operation that actually gets ignored by instance 2 -// assert_eq!(instance1.db.crdt_operation().count(vec![]).exec().await?, 5); -// assert_eq!(instance2.db.crdt_operation().count(vec![]).exec().await?, 4); - -// assert_eq!(instance1.db.location().count(vec![]).exec().await?, 0); -// // the whole point of the test - the update (which is ingested as an upsert) should be ignored -// assert_eq!(instance2.db.location().count(vec![]).exec().await?, 0); - -// instance1.teardown().await; -// instance2.teardown().await; - -// Ok(()) -// } - -// fn assert_locations_equality(l1: &location::Data, l2: &location::Data) { -// assert_eq!(l1.pub_id, l2.pub_id, "pub id"); -// assert_eq!(l1.name, l2.name, "name"); -// assert_eq!(l1.path, l2.path, "path"); -// assert_eq!(l1.total_capacity, l2.total_capacity, "total capacity"); -// assert_eq!( -// l1.available_capacity, l2.available_capacity, -// "available capacity" -// ); -// assert_eq!(l1.size_in_bytes, l2.size_in_bytes, "size in bytes"); -// assert_eq!(l1.is_archived, l2.is_archived, "is archived"); -// assert_eq!( -// l1.generate_preview_media, l2.generate_preview_media, -// "generate preview media" -// ); -// assert_eq!( -// l1.sync_preview_media, l2.sync_preview_media, -// "sync preview media" -// ); -// assert_eq!(l1.hidden, l2.hidden, "hidden"); -// assert_eq!(l1.date_created, l2.date_created, "date created"); -// assert_eq!(l1.scan_state, l2.scan_state, "scan state"); -// assert_eq!(l1.instance_id, l2.instance_id, "instance id"); -// } diff --git a/core/crates/sync/tests/mock_instance.rs b/core/crates/sync/tests/mock_instance.rs deleted file mode 100644 index 9dd5f1aff..000000000 --- a/core/crates/sync/tests/mock_instance.rs +++ /dev/null @@ -1,143 +0,0 @@ -// use sd_core_sync::*; - -// use sd_prisma::prisma; -// use sd_sync::CompressedCRDTOperationsPerModelPerDevice; - -// use std::sync::{atomic::AtomicBool, Arc}; - -// use tokio::{fs, spawn, sync::broadcast}; -// use tracing::{info, instrument, warn, Instrument}; -// use uuid::Uuid; - -// fn db_path(id: Uuid) -> String { -// format!("/tmp/test-{id}.db") -// } - -// #[derive(Clone)] -// pub struct Device { -// pub pub_id: DevicePubId, -// pub db: Arc, -// pub sync: Arc, -// pub sync_rx: Arc>, -// } - -// impl Device { -// pub async fn new(id: Uuid) -> Arc { -// let url = format!("file:{}", db_path(id)); -// let device_pub_id = DevicePubId::from(id); - -// let db = Arc::new( -// prisma::PrismaClient::_builder() -// .with_url(url.to_string()) -// .build() -// .await -// .unwrap(), -// ); - -// db._db_push().await.unwrap(); - -// db.device() -// .create(device_pub_id.to_db(), vec![]) -// .exec() -// .await -// .unwrap(); - -// // let (sync, sync_rx) = sd_core_sync::SyncManager::new( -// // Arc::clone(&db), -// // &device_pub_id, -// // Arc::new(AtomicBool::new(true)), -// // Default::default(), -// // ) -// // .await -// // .expect("failed to create sync manager"); - -// // Arc::new(Self { -// // pub_id: device_pub_id, -// // db, -// // sync: Arc::new(sync), -// // sync_rx: Arc::new(sync_rx), -// // }) -// } - -// pub async fn teardown(&self) { -// fs::remove_file(db_path(Uuid::from(&self.pub_id))) -// .await -// .unwrap(); -// } - -// pub async fn pair(instance1: &Arc, instance2: &Arc) { -// #[instrument(skip(left, right))] -// async fn half(left: &Arc, right: &Arc, context: &'static str) { -// left.db -// .device() -// .create(right.pub_id.to_db(), vec![]) -// .exec() -// .await -// .unwrap(); - -// spawn({ -// let mut sync_rx_left = left.sync_rx.resubscribe(); -// let right = Arc::clone(right); - -// async move { -// while let Ok(msg) = sync_rx_left.recv().await { -// info!(?msg, "sync_rx_left received message"); -// if matches!(msg, SyncEvent::Created) { -// right -// .sync -// .ingest -// .event_tx -// .send(ingest::Event::Notification) -// .await -// .unwrap(); -// info!("sent notification to instance 2"); -// } -// } -// } -// .in_current_span() -// }); - -// spawn({ -// let left = Arc::clone(left); -// let right = Arc::clone(right); - -// async move { -// while let Ok(msg) = right.sync.ingest.req_rx.recv().await { -// info!(?msg, "right instance received request"); -// match msg { -// ingest::Request::Messages { timestamps, tx } => { -// let messages = left.sync.get_ops(100, timestamps).await.unwrap(); - -// let ingest = &right.sync.ingest; - -// ingest -// .event_tx -// .send(ingest::Event::Messages(ingest::MessagesEvent { -// messages: CompressedCRDTOperationsPerModelPerDevice::new( -// messages, -// ), -// has_more: false, -// device_pub_id: left.pub_id.clone(), -// wait_tx: None, -// })) -// .await -// .unwrap(); - -// if tx.send(()).is_err() { -// warn!("failed to send ack to instance 1"); -// } -// } -// ingest::Request::FinishedIngesting => { -// right.sync.tx.send(SyncEvent::Ingested).unwrap(); -// } -// } -// } -// } -// .in_current_span() -// }); -// } - -// half(instance1, instance2, "instance1 -> instance2").await; -// half(instance2, instance1, "instance2 -> instance1").await; -// } -// } diff --git a/core/src/api/files.rs b/core/src/api/files.rs index 8e0f29992..155fd2884 100644 --- a/core/src/api/files.rs +++ b/core/src/api/files.rs @@ -28,8 +28,8 @@ use sd_prisma::{ prisma::{file_path, location, object}, prisma_sync, }; -use sd_sync::OperationFactory; -use sd_utils::{db::maybe_missing, error::FileIOError, msgpack}; +use sd_sync::{sync_db_entry, sync_db_nullable_entry, sync_entry, OperationFactory}; +use sd_utils::{db::maybe_missing, error::FileIOError}; use std::{ ffi::OsString, @@ -195,19 +195,19 @@ pub(crate) fn mount() -> AlphaRouter { ) })?; + let (sync_param, db_param) = sync_db_nullable_entry!(args.note, object::note); + sync.write_op( db, sync.shared_update( prisma_sync::object::SyncId { pub_id: object.pub_id, }, - object::note::NAME, - msgpack!(&args.note), - ), - db.object().update( - object::id::equals(args.id), - vec![object::note::set(args.note)], + [sync_param], ), + db.object() + .update(object::id::equals(args.id), vec![db_param]) + .select(object::select!({ id })), ) .await?; @@ -241,19 +241,19 @@ pub(crate) fn mount() -> AlphaRouter { ) })?; + let (sync_param, db_param) = sync_db_entry!(args.favorite, object::favorite); + sync.write_op( db, sync.shared_update( prisma_sync::object::SyncId { pub_id: object.pub_id, }, - object::favorite::NAME, - msgpack!(&args.favorite), - ), - db.object().update( - object::id::equals(args.id), - vec![object::favorite::set(Some(args.favorite))], + [sync_param], ), + db.object() + .update(object::id::equals(args.id), vec![db_param]) + .select(object::select!({ id })), ) .await?; @@ -346,19 +346,20 @@ pub(crate) fn mount() -> AlphaRouter { let date_accessed = Utc::now().into(); - let (ops, object_ids): (Vec<_>, Vec<_>) = objects + let (ops, object_ids) = objects .into_iter() - .map(|d| { + .map(|object| { ( sync.shared_update( - prisma_sync::object::SyncId { pub_id: d.pub_id }, - object::date_accessed::NAME, - msgpack!(date_accessed), + prisma_sync::object::SyncId { + pub_id: object.pub_id, + }, + [sync_entry!(date_accessed, object::date_accessed)], ), - d.id, + object.id, ) }) - .unzip(); + .unzip::<_, _, Vec<_>, Vec<_>>(); if !ops.is_empty() && !object_ids.is_empty() { sync.write_ops( @@ -392,19 +393,20 @@ pub(crate) fn mount() -> AlphaRouter { .exec() .await?; - let (ops, object_ids): (Vec<_>, Vec<_>) = objects + let (ops, object_ids) = objects .into_iter() - .map(|d| { + .map(|object| { ( sync.shared_update( - prisma_sync::object::SyncId { pub_id: d.pub_id }, - object::date_accessed::NAME, - msgpack!(nil), + prisma_sync::object::SyncId { + pub_id: object.pub_id, + }, + [sync_entry!(nil, object::date_accessed)], ), - d.id, + object.id, ) }) - .unzip(); + .unzip::<_, _, Vec<_>, Vec<_>>(); if !ops.is_empty() && !object_ids.is_empty() { sync.write_ops( @@ -487,11 +489,32 @@ pub(crate) fn mount() -> AlphaRouter { path = %full_path.display(), "File not found in the file system, will remove from database;", ); - library + + let file_path_pub_id = library .db .file_path() - .delete(file_path::id::equals(args.file_path_ids[0])) + .find_unique(file_path::id::equals(args.file_path_ids[0])) + .select(file_path::select!({ pub_id })) .exec() + .await? + .ok_or(LocationError::FilePath(FilePathError::IdNotFound( + args.file_path_ids[0], + )))? + .pub_id; + + library + .sync + .write_op( + &library.db, + library.sync.shared_delete( + prisma_sync::file_path::SyncId { + pub_id: file_path_pub_id, + }, + ), + library.db.file_path().delete(file_path::id::equals( + args.file_path_ids[0], + )), + ) .await .map_err(LocationError::from)?; diff --git a/core/src/api/labels.rs b/core/src/api/labels.rs index 9aaaf30e3..eed08d8d3 100644 --- a/core/src/api/labels.rs +++ b/core/src/api/labels.rs @@ -116,7 +116,7 @@ pub(crate) fn mount() -> AlphaRouter { .procedure( "delete", R.with2(library()) - .mutation(|(_, library), label_id: i32| async move { + .mutation(|(_, library), label_id: label::id::Type| async move { let Library { db, sync, .. } = library.as_ref(); let label = db @@ -131,6 +131,35 @@ pub(crate) fn mount() -> AlphaRouter { ) })?; + let delete_ops = db + .label_on_object() + .find_many(vec![label_on_object::label_id::equals(label_id)]) + .select(label_on_object::select!({ object: select { pub_id } })) + .exec() + .await? + .into_iter() + .map(|label_on_object| { + sync.relation_delete(prisma_sync::label_on_object::SyncId { + label: prisma_sync::label::SyncId { + name: label.name.clone(), + }, + object: prisma_sync::object::SyncId { + pub_id: label_on_object.object.pub_id, + }, + }) + }) + .collect::>(); + + sync.write_ops( + db, + ( + delete_ops, + db.label_on_object() + .delete_many(vec![label_on_object::label_id::equals(label_id)]), + ), + ) + .await?; + sync.write_op( db, sync.shared_delete(prisma_sync::label::SyncId { name: label.name }), diff --git a/core/src/api/search/saved.rs b/core/src/api/search/saved.rs index 37dec602e..957474c49 100644 --- a/core/src/api/search/saved.rs +++ b/core/src/api/search/saved.rs @@ -69,7 +69,7 @@ pub(crate) fn mount() -> AlphaRouter { let pub_id = Uuid::now_v7().as_bytes().to_vec(); let date_created: DateTime = Utc::now().into(); - let (sync_params, db_params): (Vec<_>, Vec<_>) = chain_optional_iter( + let (sync_params, db_params) = chain_optional_iter( [ sync_db_entry!(date_created, saved_search::date_created), sync_db_entry!(args.name, saved_search::name), @@ -96,7 +96,7 @@ pub(crate) fn mount() -> AlphaRouter { ], ) .into_iter() - .unzip(); + .unzip::<_, _, Vec<_>, Vec<_>>(); sync.write_op( db, @@ -106,7 +106,9 @@ pub(crate) fn mount() -> AlphaRouter { }, sync_params, ), - db.saved_search().create(pub_id, db_params), + db.saved_search() + .create(pub_id, db_params) + .select(saved_search::select!({ id })), ) .await?; @@ -162,7 +164,7 @@ pub(crate) fn mount() -> AlphaRouter { rspc::Error::new(rspc::ErrorCode::NotFound, "search not found".into()) })?; - let (ops, db_params): (Vec<_>, Vec<_>) = chain_optional_iter( + let (sync_params, db_params) = chain_optional_iter( [sync_db_entry!(updated_at, saved_search::date_modified)], [ option_sync_db_entry!(args.name.flatten(), saved_search::name), @@ -173,34 +175,23 @@ pub(crate) fn mount() -> AlphaRouter { ], ) .into_iter() - .map(|((k, v), p)| { - ( - sync.shared_update( - prisma_sync::saved_search::SyncId { - pub_id: search.pub_id.clone(), - }, - k, - v, - ), - p, - ) - }) - .unzip(); + .unzip::<_, _, Vec<_>, Vec<_>>(); - if !ops.is_empty() && !db_params.is_empty() { - sync.write_ops( - db, - ( - ops, - db.saved_search() - .update_unchecked(saved_search::id::equals(id), db_params), - ), - ) - .await?; + sync.write_op( + db, + sync.shared_update( + prisma_sync::saved_search::SyncId { + pub_id: search.pub_id.clone(), + }, + sync_params, + ), + db.saved_search() + .update_unchecked(saved_search::id::equals(id), db_params), + ) + .await?; - invalidate_query!(library, "search.saved.list"); - invalidate_query!(library, "search.saved.get"); - } + invalidate_query!(library, "search.saved.list"); + invalidate_query!(library, "search.saved.get"); Ok(()) } diff --git a/core/src/api/tags.rs b/core/src/api/tags.rs index 0d71b848c..0035ea592 100644 --- a/core/src/api/tags.rs +++ b/core/src/api/tags.rs @@ -4,7 +4,7 @@ use sd_prisma::{ prisma::{device, file_path, object, tag, tag_on_object}, prisma_sync, }; -use sd_sync::{option_sync_db_entry, sync_entry, OperationFactory}; +use sd_sync::{option_sync_db_entry, sync_db_entry, sync_entry, OperationFactory}; use std::collections::BTreeMap; @@ -286,13 +286,17 @@ pub(crate) fn mount() -> AlphaRouter { pub color: Option, } - R.with2(library()) - .mutation(|(_, library), args: TagUpdateArgs| async move { + R.with2(library()).mutation( + |(_, library), TagUpdateArgs { id, name, color }: TagUpdateArgs| async move { + if name.is_none() && color.is_none() { + return Ok(()); + } + let Library { sync, db, .. } = library.as_ref(); let tag = db .tag() - .find_unique(tag::id::equals(args.id)) + .find_unique(tag::id::equals(id)) .select(tag::select!({ pub_id })) .exec() .await? @@ -301,68 +305,88 @@ pub(crate) fn mount() -> AlphaRouter { "Error finding tag in db".into(), ))?; - db.tag() - .update( - tag::id::equals(args.id), - vec![tag::date_modified::set(Some(Utc::now().into()))], - ) - .exec() - .await?; - - let (sync_params, db_params): (Vec<_>, Vec<_>) = [ - option_sync_db_entry!(args.name, tag::name), - option_sync_db_entry!(args.color, tag::color), + let (sync_params, db_params) = [ + option_sync_db_entry!(name, tag::name), + option_sync_db_entry!(color, tag::color), + Some(sync_db_entry!(Utc::now(), tag::date_modified)), ] .into_iter() .flatten() - .unzip(); + .unzip::<_, _, Vec<_>, Vec<_>>(); - if sync_params.is_empty() && db_params.is_empty() { - return Ok(()); - } - - sync.write_ops( + sync.write_op( db, - ( - sync_params - .into_iter() - .map(|(k, v)| { - sync.shared_update( - prisma_sync::tag::SyncId { - pub_id: tag.pub_id.clone(), - }, - k, - v, - ) - }) - .collect(), - db.tag().update(tag::id::equals(args.id), db_params), + sync.shared_update( + prisma_sync::tag::SyncId { + pub_id: tag.pub_id.clone(), + }, + sync_params, ), + db.tag() + .update(tag::id::equals(id), db_params) + .select(tag::select!({ id })), ) .await?; invalidate_query!(library, "tags.list"); Ok(()) - }) + }, + ) }) .procedure( "delete", R.with2(library()) - .mutation(|(_, library), tag_id: i32| async move { - library - .db - .tag_on_object() - .delete_many(vec![tag_on_object::tag_id::equals(tag_id)]) - .exec() - .await?; + .mutation(|(_, library), tag_id: tag::id::Type| async move { + let Library { sync, db, .. } = &*library; - library - .db + let tag_pub_id = db .tag() - .delete(tag::id::equals(tag_id)) + .find_unique(tag::id::equals(tag_id)) + .select(tag::select!({ pub_id })) .exec() - .await?; + .await? + .ok_or(rspc::Error::new( + rspc::ErrorCode::NotFound, + "Tag not found".to_string(), + ))? + .pub_id; + + let delete_ops = db + .tag_on_object() + .find_many(vec![tag_on_object::tag_id::equals(tag_id)]) + .select(tag_on_object::select!({ object: select { pub_id } })) + .exec() + .await? + .into_iter() + .map(|tag_on_object| { + sync.relation_delete(prisma_sync::tag_on_object::SyncId { + tag: prisma_sync::tag::SyncId { + pub_id: tag_pub_id.clone(), + }, + object: prisma_sync::object::SyncId { + pub_id: tag_on_object.object.pub_id, + }, + }) + }) + .collect::>(); + + sync.write_ops( + db, + ( + delete_ops, + db.tag_on_object() + .delete_many(vec![tag_on_object::tag_id::equals(tag_id)]), + ), + ) + .await?; + + sync.write_op( + db, + sync.shared_delete(prisma_sync::tag::SyncId { pub_id: tag_pub_id }), + db.tag().delete(tag::id::equals(tag_id)), + ) + .await?; invalidate_query!(library, "tags.list"); diff --git a/core/src/library/manager/mod.rs b/core/src/library/manager/mod.rs index 7649576e6..06aa0b8c9 100644 --- a/core/src/library/manager/mod.rs +++ b/core/src/library/manager/mod.rs @@ -537,6 +537,7 @@ impl Libraries { )), ], ) + .select(instance::select!({ id })) .exec() .await?; } diff --git a/core/src/location/manager/watcher/android.rs b/core/src/location/manager/watcher/android.rs index 01bd8a2a1..723f2e076 100644 --- a/core/src/location/manager/watcher/android.rs +++ b/core/src/location/manager/watcher/android.rs @@ -27,6 +27,7 @@ use super::{ #[derive(Debug)] pub(super) struct EventHandler { location_id: location::id::Type, + location_pub_id: location::pub_id::Type, library: Arc, node: Arc, last_events_eviction_check: Instant, @@ -40,9 +41,18 @@ pub(super) struct EventHandler { } impl super::EventHandler for EventHandler { - fn new(location_id: location::id::Type, library: Arc, node: Arc) -> Self { + fn new( + location_id: location::id::Type, + location_pub_id: location::pub_id::Type, + library: Arc, + node: Arc, + ) -> Self + where + Self: Sized, + { Self { location_id, + location_pub_id, library, node, last_events_eviction_check: Instant::now(), @@ -182,6 +192,7 @@ impl super::EventHandler for EventHandler { &mut self.to_recalculate_size, &mut self.path_and_instant_buffer, self.location_id, + self.location_pub_id.clone(), &self.library, ) .await diff --git a/core/src/location/manager/watcher/ios.rs b/core/src/location/manager/watcher/ios.rs index 3a9c91500..25f0a49fd 100644 --- a/core/src/location/manager/watcher/ios.rs +++ b/core/src/location/manager/watcher/ios.rs @@ -33,6 +33,7 @@ use super::{ #[derive(Debug)] pub(super) struct EventHandler { location_id: location::id::Type, + location_pub_id: location::pub_id::Type, library: Arc, node: Arc, last_events_eviction_check: Instant, @@ -48,12 +49,18 @@ pub(super) struct EventHandler { } impl super::EventHandler for EventHandler { - fn new(location_id: location::id::Type, library: Arc, node: Arc) -> Self + fn new( + location_id: location::id::Type, + location_pub_id: location::pub_id::Type, + library: Arc, + node: Arc, + ) -> Self where Self: Sized, { Self { location_id, + location_pub_id, library, node, last_events_eviction_check: Instant::now(), @@ -183,6 +190,7 @@ impl super::EventHandler for EventHandler { &mut self.to_recalculate_size, &mut self.path_and_instant_buffer, self.location_id, + self.location_pub_id.clone(), &self.library, ) .await diff --git a/core/src/location/manager/watcher/linux.rs b/core/src/location/manager/watcher/linux.rs index 0ec459a3c..34d37ed15 100644 --- a/core/src/location/manager/watcher/linux.rs +++ b/core/src/location/manager/watcher/linux.rs @@ -32,6 +32,7 @@ use super::{ #[derive(Debug)] pub(super) struct EventHandler { location_id: location::id::Type, + location_pub_id: location::pub_id::Type, library: Arc, node: Arc, last_events_eviction_check: Instant, @@ -45,9 +46,18 @@ pub(super) struct EventHandler { } impl super::EventHandler for EventHandler { - fn new(location_id: location::id::Type, library: Arc, node: Arc) -> Self { + fn new( + location_id: location::id::Type, + location_pub_id: location::pub_id::Type, + library: Arc, + node: Arc, + ) -> Self + where + Self: Sized, + { Self { location_id, + location_pub_id, library, node, last_events_eviction_check: Instant::now(), @@ -187,6 +197,7 @@ impl super::EventHandler for EventHandler { &mut self.to_recalculate_size, &mut self.path_and_instant_buffer, self.location_id, + self.location_pub_id.clone(), &self.library, ) .await diff --git a/core/src/location/manager/watcher/macos.rs b/core/src/location/manager/watcher/macos.rs index 11486cd20..4d3b1ffec 100644 --- a/core/src/location/manager/watcher/macos.rs +++ b/core/src/location/manager/watcher/macos.rs @@ -42,6 +42,7 @@ use super::{ #[derive(Debug)] pub(super) struct EventHandler { location_id: location::id::Type, + location_pub_id: location::pub_id::Type, library: Arc, node: Arc, last_events_eviction_check: Instant, @@ -57,12 +58,18 @@ pub(super) struct EventHandler { } impl super::EventHandler for EventHandler { - fn new(location_id: location::id::Type, library: Arc, node: Arc) -> Self + fn new( + location_id: location::id::Type, + location_pub_id: location::pub_id::Type, + library: Arc, + node: Arc, + ) -> Self where Self: Sized, { Self { location_id, + location_pub_id, library, node, last_events_eviction_check: Instant::now(), @@ -206,6 +213,7 @@ impl super::EventHandler for EventHandler { &mut self.to_recalculate_size, &mut self.path_and_instant_buffer, self.location_id, + self.location_pub_id.clone(), &self.library, ) .await diff --git a/core/src/location/manager/watcher/mod.rs b/core/src/location/manager/watcher/mod.rs index 81b70ef87..d63709740 100644 --- a/core/src/location/manager/watcher/mod.rs +++ b/core/src/location/manager/watcher/mod.rs @@ -4,7 +4,7 @@ use sd_core_indexer_rules::{IndexerRule, IndexerRuler}; use sd_core_prisma_helpers::{location_ids_and_path, location_with_indexer_rules}; use sd_prisma::prisma::{location, PrismaClient}; -use sd_utils::db::maybe_missing; +use sd_utils::{db::maybe_missing, uuid_to_bytes}; use std::{ collections::HashSet, @@ -76,7 +76,12 @@ const THIRTY_SECONDS: Duration = Duration::from_secs(30); const HUNDRED_MILLIS: Duration = Duration::from_millis(100); trait EventHandler: 'static { - fn new(location_id: location::id::Type, library: Arc, node: Arc) -> Self + fn new( + location_id: location::id::Type, + location_pub_id: location::pub_id::Type, + library: Arc, + node: Arc, + ) -> Self where Self: Sized; @@ -200,7 +205,12 @@ impl LocationWatcher { Stop, } - let mut event_handler = Handler::new(location_id, Arc::clone(&library), Arc::clone(&node)); + let mut event_handler = Handler::new( + location_id, + uuid_to_bytes(&location_pub_id), + Arc::clone(&library), + Arc::clone(&node), + ); let mut last_event_at = Instant::now(); diff --git a/core/src/location/manager/watcher/utils.rs b/core/src/location/manager/watcher/utils.rs index 0adaf9f8c..88b065810 100644 --- a/core/src/location/manager/watcher/utils.rs +++ b/core/src/location/manager/watcher/utils.rs @@ -27,7 +27,9 @@ use sd_core_indexer_rules::{ seed::{GitIgnoreRules, GITIGNORE}, IndexerRuler, RulerDecision, }; -use sd_core_prisma_helpers::{file_path_with_object, object_ids, CasId, ObjectPubId}; +use sd_core_prisma_helpers::{ + file_path_watcher_remove, file_path_with_object, object_ids, CasId, ObjectPubId, +}; use sd_file_ext::{ extensions::{AudioExtension, ImageExtension, VideoExtension}, @@ -37,11 +39,11 @@ use sd_prisma::{ prisma::{device, file_path, location, object}, prisma_sync, }; -use sd_sync::{sync_entry, OperationFactory}; +use sd_sync::{option_sync_db_entry, sync_db_entry, sync_entry, OperationFactory}; use sd_utils::{ - db::{inode_from_db, inode_to_db, maybe_missing}, + chain_optional_iter, + db::{inode_from_db, inode_to_db, maybe_missing, size_in_bytes_to_db}, error::FileIOError, - msgpack, }; #[cfg(target_family = "unix")] @@ -354,32 +356,32 @@ async fn inner_create_file( let device_pub_id = sync.device_pub_id.to_db(); + let (sync_params, db_params) = [ + sync_db_entry!(date_created, object::date_created), + sync_db_entry!(int_kind, object::kind), + ( + sync_entry!( + prisma_sync::device::SyncId { + pub_id: device_pub_id.clone() + }, + object::device + ), + object::device::connect(device::pub_id::equals(device_pub_id)), + ), + ] + .into_iter() + .unzip::<_, _, Vec<_>, Vec<_>>(); + sync.write_op( db, sync.shared_create( prisma_sync::object::SyncId { pub_id: pub_id.to_db(), }, - [ - sync_entry!(date_created, object::date_created), - sync_entry!(int_kind, object::kind), - sync_entry!( - prisma_sync::device::SyncId { - pub_id: device_pub_id.clone() - }, - object::device - ), - ], + sync_params, ), db.object() - .create( - pub_id.into(), - vec![ - object::date_created::set(Some(date_created)), - object::kind::set(Some(int_kind)), - object::device::connect(device::pub_id::equals(device_pub_id)), - ], - ) + .create(pub_id.into(), db_params) .select(object_ids::select()), ) .await? @@ -391,17 +393,21 @@ async fn inner_create_file( prisma_sync::location::SyncId { pub_id: created_file.pub_id.clone(), }, - file_path::object::NAME, - msgpack!(prisma_sync::object::SyncId { - pub_id: object_pub_id.clone() - }), - ), - db.file_path().update( - file_path::pub_id::equals(created_file.pub_id.clone()), - vec![file_path::object::connect(object::pub_id::equals( - object_pub_id.clone(), - ))], + [sync_entry!( + prisma_sync::object::SyncId { + pub_id: object_pub_id.clone() + }, + file_path::object + )], ), + db.file_path() + .update( + file_path::pub_id::equals(created_file.pub_id.clone()), + vec![file_path::object::connect(object::pub_id::equals( + object_pub_id.clone(), + ))], + ) + .select(file_path::select!({ id })), ) .await?; @@ -590,34 +596,22 @@ async fn inner_update_file( let is_hidden = path_is_hidden(full_path, &fs_metadata); if file_path.cas_id.as_deref() != cas_id.as_ref().map(CasId::as_str) { - let (sync_params, db_params): (Vec<_>, Vec<_>) = { - use file_path::*; - + let (sync_params, db_params) = chain_optional_iter( [ - ( - (cas_id::NAME, msgpack!(file_path.cas_id)), - Some(cas_id::set(file_path.cas_id.clone())), + sync_db_entry!( + size_in_bytes_to_db(fs_metadata.len()), + file_path::size_in_bytes_bytes ), - ( - ( - size_in_bytes_bytes::NAME, - msgpack!(fs_metadata.len().to_be_bytes().to_vec()), - ), - Some(size_in_bytes_bytes::set(Some( - fs_metadata.len().to_be_bytes().to_vec(), - ))), + sync_db_entry!( + DateTime::::from(fs_metadata.modified_or_now()), + file_path::date_modified ), - { - let date = DateTime::::from(fs_metadata.modified_or_now()).into(); - - ( - (date_modified::NAME, msgpack!(date)), - Some(date_modified::set(Some(date))), - ) - }, - { - // TODO: Should this be a skip rather than a null-set? - let checksum = if file_path.integrity_checksum.is_some() { + ], + [ + option_sync_db_entry!(file_path.cas_id.clone(), file_path::cas_id), + option_sync_db_entry!( + if file_path.integrity_checksum.is_some() { + // TODO: Should this be a skip rather than a null-set? // If a checksum was already computed, we need to recompute it Some( file_checksum(full_path) @@ -626,68 +620,39 @@ async fn inner_update_file( ) } else { None - }; - - ( - (integrity_checksum::NAME, msgpack!(checksum)), - Some(integrity_checksum::set(checksum)), - ) - }, - { - if current_inode != inode { - ( - (inode::NAME, msgpack!(inode)), - Some(inode::set(Some(inode_to_db(inode)))), - ) - } else { - ((inode::NAME, msgpack!(nil)), None) - } - }, - { - if is_hidden != file_path.hidden.unwrap_or_default() { - ( - (hidden::NAME, msgpack!(inode)), - Some(hidden::set(Some(is_hidden))), - ) - } else { - ((hidden::NAME, msgpack!(nil)), None) - } - }, - ] - .into_iter() - .filter_map(|(sync_param, maybe_db_param)| { - maybe_db_param.map(|db_param| (sync_param, db_param)) - }) - .unzip() - }; - - let ops = sync_params - .into_iter() - .map(|(field, value)| { - sync.shared_update( - prisma_sync::file_path::SyncId { - pub_id: file_path.pub_id.clone(), }, - field, - value, - ) - }) - .collect::>(); - - if !ops.is_empty() && !db_params.is_empty() { - // file content changed - sync.write_ops( - db, - ( - ops, - db.file_path().update( - file_path::pub_id::equals(file_path.pub_id.clone()), - db_params, - ), + file_path::integrity_checksum ), - ) - .await?; - } + option_sync_db_entry!( + (current_inode != inode).then(|| inode_to_db(inode)), + file_path::inode + ), + option_sync_db_entry!( + (is_hidden != file_path.hidden.unwrap_or_default()).then_some(is_hidden), + file_path::hidden + ), + ], + ) + .into_iter() + .unzip::<_, _, Vec<_>, Vec<_>>(); + + // file content changed + sync.write_op( + db, + sync.shared_update( + prisma_sync::file_path::SyncId { + pub_id: file_path.pub_id.clone(), + }, + sync_params, + ), + db.file_path() + .update( + file_path::pub_id::equals(file_path.pub_id.clone()), + db_params, + ) + .select(file_path::select!({ id })), + ) + .await?; if let Some(ref object) = file_path.object { let int_kind = kind as i32; @@ -699,19 +664,18 @@ async fn inner_update_file( .await? == 1 { if object.kind.map(|k| k != int_kind).unwrap_or_default() { + let (sync_param, db_param) = sync_db_entry!(int_kind, object::kind); sync.write_op( db, sync.shared_update( prisma_sync::object::SyncId { pub_id: object.pub_id.clone(), }, - object::kind::NAME, - msgpack!(int_kind), - ), - db.object().update( - object::id::equals(object.id), - vec![object::kind::set(Some(int_kind))], + [sync_param], ), + db.object() + .update(object::id::equals(object.id), vec![db_param]) + .select(object::select!({ id })), ) .await?; } @@ -722,31 +686,31 @@ async fn inner_update_file( let device_pub_id = sync.device_pub_id.to_db(); + let (sync_params, db_params) = [ + sync_db_entry!(date_created, object::date_created), + sync_db_entry!(int_kind, object::kind), + ( + sync_entry!( + prisma_sync::device::SyncId { + pub_id: device_pub_id.clone() + }, + object::device + ), + object::device::connect(device::pub_id::equals(device_pub_id)), + ), + ] + .into_iter() + .unzip::<_, _, Vec<_>, Vec<_>>(); + sync.write_op( db, sync.shared_create( prisma_sync::object::SyncId { pub_id: pub_id.to_db(), }, - [ - sync_entry!(date_created, object::date_created), - sync_entry!(int_kind, object::kind), - sync_entry!( - prisma_sync::device::SyncId { - pub_id: device_pub_id.clone() - }, - object::device - ), - ], - ), - db.object().create( - pub_id.to_db(), - vec![ - object::date_created::set(Some(date_created)), - object::kind::set(Some(int_kind)), - object::device::connect(device::pub_id::equals(device_pub_id)), - ], + sync_params, ), + db.object().create(pub_id.to_db(), db_params), ) .await?; @@ -756,17 +720,21 @@ async fn inner_update_file( prisma_sync::location::SyncId { pub_id: file_path.pub_id.clone(), }, - file_path::object::NAME, - msgpack!(prisma_sync::object::SyncId { - pub_id: pub_id.to_db() - }), - ), - db.file_path().update( - file_path::pub_id::equals(file_path.pub_id.clone()), - vec![file_path::object::connect(object::pub_id::equals( - pub_id.into(), - ))], + [sync_entry!( + prisma_sync::object::SyncId { + pub_id: pub_id.to_db() + }, + file_path::object + )], ), + db.file_path() + .update( + file_path::pub_id::equals(file_path.pub_id.clone()), + vec![file_path::object::connect(object::pub_id::equals( + pub_id.into(), + ))], + ) + .select(file_path::select!({ id })), ) .await?; } @@ -874,21 +842,22 @@ async fn inner_update_file( invalidate_query!(library, "search.paths"); invalidate_query!(library, "search.objects"); } else if is_hidden != file_path.hidden.unwrap_or_default() { - sync.write_ops( + let (sync_param, db_param) = sync_db_entry!(is_hidden, file_path::hidden); + + sync.write_op( db, - ( - vec![sync.shared_update( - prisma_sync::file_path::SyncId { - pub_id: file_path.pub_id.clone(), - }, - file_path::hidden::NAME, - msgpack!(is_hidden), - )], - db.file_path().update( - file_path::pub_id::equals(file_path.pub_id.clone()), - vec![file_path::hidden::set(Some(is_hidden))], - ), + sync.shared_update( + prisma_sync::file_path::SyncId { + pub_id: file_path.pub_id.clone(), + }, + [sync_param], ), + db.file_path() + .update( + file_path::pub_id::equals(file_path.pub_id.clone()), + vec![db_param], + ) + .select(file_path::select!({ id })), ) .await?; @@ -972,7 +941,7 @@ pub(super) async fn rename( .await?; let total_paths_count = paths.len(); - let (sync_params, db_params): (Vec<_>, Vec<_>) = paths + let (sync_params, db_params) = paths .into_iter() .filter_map(|path| path.materialized_path.map(|mp| (path.id, path.pub_id, mp))) .map(|(id, pub_id, mp)| { @@ -981,19 +950,20 @@ pub(super) async fn rename( &format!("{}/{}/", new_parts.materialized_path, new_parts.name), ); + let (sync_param, db_param) = + sync_db_entry!(new_path, file_path::materialized_path); + ( sync.shared_update( sd_prisma::prisma_sync::file_path::SyncId { pub_id }, - file_path::materialized_path::NAME, - msgpack!(&new_path), - ), - db.file_path().update( - file_path::id::equals(id), - vec![file_path::materialized_path::set(Some(new_path))], + [sync_param], ), + db.file_path() + .update(file_path::id::equals(id), vec![db_param]) + .select(file_path::select!({ id })), ) }) - .unzip(); + .unzip::<_, _, Vec<_>, Vec<_>>(); if !sync_params.is_empty() && !db_params.is_empty() { sync.write_ops(db, (sync_params, db_params)).await?; @@ -1002,65 +972,38 @@ pub(super) async fn rename( trace!(%total_paths_count, "Updated file_paths;"); } - let is_hidden = path_is_hidden(new_path, &new_path_metadata); - - let date_modified = DateTime::::from(new_path_metadata.modified_or_now()).into(); - - let (sync_params, db_params): (Vec<_>, Vec<_>) = [ - ( - ( - file_path::materialized_path::NAME, - msgpack!(new_path_materialized_str), - ), - file_path::materialized_path::set(Some(new_path_materialized_str)), + let (sync_params, db_params) = [ + sync_db_entry!(new_path_materialized_str, file_path::materialized_path), + sync_db_entry!(new_parts.name.to_string(), file_path::name), + sync_db_entry!(new_parts.extension.to_string(), file_path::extension), + sync_db_entry!( + DateTime::::from(new_path_metadata.modified_or_now()), + file_path::date_modified ), - ( - (file_path::name::NAME, msgpack!(new_parts.name)), - file_path::name::set(Some(new_parts.name.to_string())), - ), - ( - (file_path::extension::NAME, msgpack!(new_parts.extension)), - file_path::extension::set(Some(new_parts.extension.to_string())), - ), - ( - (file_path::date_modified::NAME, msgpack!(&date_modified)), - file_path::date_modified::set(Some(date_modified)), - ), - ( - (file_path::hidden::NAME, msgpack!(is_hidden)), - file_path::hidden::set(Some(is_hidden)), + sync_db_entry!( + path_is_hidden(new_path, &new_path_metadata), + file_path::hidden ), ] .into_iter() - .unzip(); + .unzip::<_, _, Vec<_>, Vec<_>>(); - let ops = sync_params - .into_iter() - .map(|(k, v)| { - sync.shared_update( - prisma_sync::file_path::SyncId { - pub_id: file_path.pub_id.clone(), - }, - k, - v, - ) - }) - .collect::>(); + sync.write_op( + db, + sync.shared_update( + prisma_sync::file_path::SyncId { + pub_id: file_path.pub_id.clone(), + }, + sync_params, + ), + db.file_path() + .update(file_path::pub_id::equals(file_path.pub_id), db_params) + .select(file_path::select!({ id })), + ) + .await?; - if !ops.is_empty() && !db_params.is_empty() { - sync.write_ops( - db, - ( - ops, - db.file_path() - .update(file_path::pub_id::equals(file_path.pub_id), db_params), - ), - ) - .await?; - - invalidate_query!(library, "search.paths"); - invalidate_query!(library, "search.objects"); - } + invalidate_query!(library, "search.paths"); + invalidate_query!(library, "search.objects"); } Ok(()) @@ -1084,19 +1027,20 @@ pub(super) async fn remove( &location_path, full_path, )?) + .select(file_path_watcher_remove::select()) .exec() .await? else { return Ok(()); }; - remove_by_file_path(location_id, full_path, &file_path, library).await + remove_by_file_path(location_id, full_path, file_path, library).await } async fn remove_by_file_path( location_id: location::id::Type, path: impl AsRef + Send, - file_path: &file_path::Data, + file_path: file_path_watcher_remove::Data, library: &Library, ) -> Result<(), LocationManagerError> { // check file still exists on disk @@ -1120,28 +1064,42 @@ async fn remove_by_file_path( delete_directory( library, location_id, - Some(&IsolatedFilePathData::try_from(file_path)?), + Some(&IsolatedFilePathData::try_from(&file_path)?), ) .await?; } else { sync.write_op( db, sync.shared_delete(prisma_sync::file_path::SyncId { - pub_id: file_path.pub_id.clone(), + pub_id: file_path.pub_id, }), db.file_path().delete(file_path::id::equals(file_path.id)), ) .await?; - if let Some(object_id) = file_path.object_id { - db.object() - .delete_many(vec![ - object::id::equals(object_id), + if let Some(object) = file_path.object { + // If this object doesn't have any other file paths, delete it + if db + .object() + .count(vec![ + object::id::equals(object.id), // https://www.prisma.io/docs/reference/api-reference/prisma-client-reference#none object::file_paths::none(vec![]), ]) .exec() + .await? == 1 + { + sync.write_op( + db, + sync.shared_delete(prisma_sync::object::SyncId { + pub_id: object.pub_id, + }), + db.object() + .delete(object::id::equals(object.id)) + .select(object::select!({ id })), + ) .await?; + } } } } @@ -1210,6 +1168,7 @@ pub(super) async fn recalculate_directories_size( candidates: &mut HashMap, buffer: &mut Vec<(PathBuf, Instant)>, location_id: location::id::Type, + location_pub_id: location::pub_id::Type, library: &Library, ) -> Result<(), LocationManagerError> { let mut location_path_cache = None; @@ -1268,7 +1227,7 @@ pub(super) async fn recalculate_directories_size( } if should_update_location_size { - update_location_size(location_id, library).await?; + update_location_size(location_id, location_pub_id, library).await?; } if should_invalidate { diff --git a/core/src/location/manager/watcher/windows.rs b/core/src/location/manager/watcher/windows.rs index a9b24c54c..bd85693e8 100644 --- a/core/src/location/manager/watcher/windows.rs +++ b/core/src/location/manager/watcher/windows.rs @@ -39,6 +39,7 @@ use super::{ #[derive(Debug)] pub(super) struct EventHandler { location_id: location::id::Type, + location_pub_id: location::pub_id::Type, library: Arc, node: Arc, last_events_eviction_check: Instant, @@ -54,12 +55,18 @@ pub(super) struct EventHandler { } impl super::EventHandler for EventHandler { - fn new(location_id: location::id::Type, library: Arc, node: Arc) -> Self + fn new( + location_id: location::id::Type, + location_pub_id: location::pub_id::Type, + library: Arc, + node: Arc, + ) -> Self where Self: Sized, { Self { location_id, + location_pub_id, library, node, last_events_eviction_check: Instant::now(), @@ -277,6 +284,7 @@ impl super::EventHandler for EventHandler { &mut self.to_recalculate_size, &mut self.path_and_instant_buffer, self.location_id, + self.location_pub_id.clone(), &self.library, ) .await diff --git a/core/src/location/mod.rs b/core/src/location/mod.rs index a1fd20073..a4a998995 100644 --- a/core/src/location/mod.rs +++ b/core/src/location/mod.rs @@ -18,9 +18,9 @@ use sd_prisma::{ }; use sd_sync::*; use sd_utils::{ - db::{maybe_missing, size_in_bytes_to_db, MissingFieldError}, + db::{maybe_missing, size_in_bytes_from_db, size_in_bytes_to_db}, error::{FileIOError, NonUtf8PathError}, - msgpack, uuid_to_bytes, + uuid_to_bytes, }; use std::{ @@ -304,63 +304,36 @@ impl LocationUpdateArgs { let name = self.name.clone(); - let (sync_params, db_params): (Vec<_>, Vec<_>) = [ - self.name - .filter(|name| location.name.as_ref() != Some(name)) - .map(|v| { - ( - (location::name::NAME, msgpack!(v)), - location::name::set(Some(v)), - ) - }), - self.generate_preview_media.map(|v| { - ( - (location::generate_preview_media::NAME, msgpack!(v)), - location::generate_preview_media::set(Some(v)), - ) - }), - self.sync_preview_media.map(|v| { - ( - (location::sync_preview_media::NAME, msgpack!(v)), - location::sync_preview_media::set(Some(v)), - ) - }), - self.hidden.map(|v| { - ( - (location::hidden::NAME, msgpack!(v)), - location::hidden::set(Some(v)), - ) - }), - self.path.clone().map(|v| { - ( - (location::path::NAME, msgpack!(v)), - location::path::set(Some(v)), - ) - }), + let (sync_params, db_params) = [ + option_sync_db_entry!( + self.name + .filter(|name| location.name.as_ref() != Some(name)), + location::name + ), + option_sync_db_entry!( + self.generate_preview_media, + location::generate_preview_media + ), + option_sync_db_entry!(self.sync_preview_media, location::sync_preview_media), + option_sync_db_entry!(self.hidden, location::hidden), + option_sync_db_entry!(self.path.clone(), location::path), ] .into_iter() .flatten() - .unzip(); + .unzip::<_, _, Vec<_>, Vec<_>>(); if !sync_params.is_empty() { - sync.write_ops( + sync.write_op( db, - ( - sync_params - .into_iter() - .map(|p| { - sync.shared_update( - prisma_sync::location::SyncId { - pub_id: location.pub_id.clone(), - }, - p.0, - p.1, - ) - }) - .collect(), - db.location() - .update(location::id::equals(self.id), db_params), + sync.shared_update( + prisma_sync::location::SyncId { + pub_id: location.pub_id.clone(), + }, + sync_params, ), + db.location() + .update(location::id::equals(self.id), db_params) + .select(location::select!({ id })), ) .await?; @@ -651,33 +624,25 @@ pub async fn relink_location( .map(str::to_string) .ok_or_else(|| NonUtf8PathError(location_path.into()))?; - sync.write_op( - db, - sync.shared_update( - prisma_sync::location::SyncId { - pub_id: pub_id.clone(), - }, - location::path::NAME, - msgpack!(path), - ), - db.location().update( - location::pub_id::equals(pub_id.clone()), - vec![location::path::set(Some(path))], - ), - ) - .await?; + let (sync_param, db_param) = sync_db_entry!(path, location::path); - let location_id = db - .location() - .find_unique(location::pub_id::equals(pub_id)) - .select(location::select!({ id })) - .exec() + let location_id = sync + .write_op( + db, + sync.shared_update( + prisma_sync::location::SyncId { + pub_id: pub_id.clone(), + }, + [sync_param], + ), + db.location() + .update(location::pub_id::equals(pub_id.clone()), vec![db_param]) + .select(location::select!({ id })), + ) .await? - .ok_or_else(|| { - LocationError::MissingField(MissingFieldError::new("missing id of location")) - })?; + .id; - Ok(location_id.id) + Ok(location_id) } #[derive(Debug)] @@ -1002,45 +967,44 @@ async fn check_nested_location( #[instrument(skip_all, err)] pub async fn update_location_size( location_id: location::id::Type, + location_pub_id: location::pub_id::Type, library: &Library, -) -> Result<(), QueryError> { - let Library { db, .. } = library; +) -> Result<(), sd_core_sync::Error> { + let Library { db, sync, .. } = library; - let total_size = db - .file_path() - .find_many(vec![ - file_path::location_id::equals(Some(location_id)), - file_path::materialized_path::equals(Some("/".to_string())), - ]) - .select(file_path::select!({ size_in_bytes_bytes })) - .exec() - .await? - .into_iter() - .filter_map(|file_path| { - file_path.size_in_bytes_bytes.map(|size_in_bytes_bytes| { - u64::from_be_bytes([ - size_in_bytes_bytes[0], - size_in_bytes_bytes[1], - size_in_bytes_bytes[2], - size_in_bytes_bytes[3], - size_in_bytes_bytes[4], - size_in_bytes_bytes[5], - size_in_bytes_bytes[6], - size_in_bytes_bytes[7], - ]) + let total_size = size_in_bytes_to_db( + db.file_path() + .find_many(vec![ + file_path::location_id::equals(Some(location_id)), + file_path::materialized_path::equals(Some("/".to_string())), + ]) + .select(file_path::select!({ size_in_bytes_bytes })) + .exec() + .await? + .into_iter() + .filter_map(|file_path| { + file_path + .size_in_bytes_bytes + .map(|size_in_bytes_bytes| size_in_bytes_from_db(&size_in_bytes_bytes)) }) - }) - .sum::(); + .sum::(), + ); - db.location() - .update( - location::id::equals(location_id), - vec![location::size_in_bytes::set(Some( - total_size.to_be_bytes().to_vec(), - ))], - ) - .exec() - .await?; + let (sync_param, db_param) = sync_db_entry!(total_size, location::size_in_bytes); + + sync.write_op( + db, + sync.shared_update( + prisma_sync::location::SyncId { + pub_id: location_pub_id, + }, + [sync_param], + ), + db.location() + .update(location::id::equals(location_id), vec![db_param]) + .select(location::select!({ id })), + ) + .await?; invalidate_query!(library, "locations.list"); invalidate_query!(library, "locations.get"); @@ -1100,69 +1064,60 @@ pub async fn create_file_path( location_id, ))?; - let (sync_params, db_params): (Vec<_>, Vec<_>) = { - use file_path::{ - cas_id, date_created, date_indexed, date_modified, device, extension, hidden, inode, - is_dir, location, materialized_path, name, size_in_bytes_bytes, - }; + let device_pub_id = sync.device_pub_id.to_db(); - let device_pub_id = sync.device_pub_id.to_db(); - - [ - ( - sync_entry!( - prisma_sync::location::SyncId { - pub_id: location.pub_id - }, - location - ), - location::connect(prisma::location::id::equals(location.id)), + let (sync_params, db_params) = [ + ( + sync_entry!( + prisma_sync::location::SyncId { + pub_id: location.pub_id + }, + file_path::location ), - ( - sync_entry!(cas_id, cas_id), - cas_id::set(cas_id.map(Into::into)), + file_path::location::connect(prisma::location::id::equals(location.id)), + ), + ( + sync_entry!(cas_id, file_path::cas_id), + file_path::cas_id::set(cas_id.map(Into::into)), + ), + sync_db_entry!(materialized_path, file_path::materialized_path), + sync_db_entry!(name, file_path::name), + sync_db_entry!(extension, file_path::extension), + sync_db_entry!( + size_in_bytes_to_db(metadata.size_in_bytes), + file_path::size_in_bytes_bytes + ), + sync_db_entry!(inode_to_db(metadata.inode), file_path::inode), + sync_db_entry!(is_dir, file_path::is_dir), + sync_db_entry!(metadata.created_at, file_path::date_created), + sync_db_entry!(metadata.modified_at, file_path::date_modified), + sync_db_entry!(indexed_at, file_path::date_indexed), + sync_db_entry!(metadata.hidden, file_path::hidden), + ( + sync_entry!( + prisma_sync::device::SyncId { + pub_id: device_pub_id.clone() + }, + file_path::device ), - sync_db_entry!(materialized_path, materialized_path), - sync_db_entry!(name, name), - sync_db_entry!(extension, extension), - sync_db_entry!( - size_in_bytes_to_db(metadata.size_in_bytes), - size_in_bytes_bytes - ), - sync_db_entry!(inode_to_db(metadata.inode), inode), - sync_db_entry!(is_dir, is_dir), - sync_db_entry!(metadata.created_at, date_created), - sync_db_entry!(metadata.modified_at, date_modified), - sync_db_entry!(indexed_at, date_indexed), - sync_db_entry!(metadata.hidden, hidden), - ( - sync_entry!( - prisma_sync::device::SyncId { - pub_id: device_pub_id.clone() - }, - device - ), - device::connect(prisma::device::pub_id::equals(device_pub_id)), - ), - ] - .into_iter() - .unzip() - }; + file_path::device::connect(prisma::device::pub_id::equals(device_pub_id)), + ), + ] + .into_iter() + .unzip::<_, _, Vec<_>, Vec<_>>(); let pub_id = sd_utils::uuid_to_bytes(&Uuid::now_v7()); - let created_path = sync - .write_op( - db, - sync.shared_create( - prisma_sync::file_path::SyncId { - pub_id: pub_id.clone(), - }, - sync_params, - ), - db.file_path().create(pub_id, db_params), - ) - .await?; - - Ok(created_path) + sync.write_op( + db, + sync.shared_create( + prisma_sync::file_path::SyncId { + pub_id: pub_id.clone(), + }, + sync_params, + ), + db.file_path().create(pub_id, db_params), + ) + .await + .map_err(Into::into) } diff --git a/core/src/object/fs/old_copy.rs b/core/src/object/fs/old_copy.rs index 8b760b920..2d7b0fb70 100644 --- a/core/src/object/fs/old_copy.rs +++ b/core/src/object/fs/old_copy.rs @@ -323,8 +323,8 @@ impl StatefulJob for OldFileCopierJobInit { .await?; dirs.extend(more_dirs); - let (dir_source_file_data, dir_target_full_path): (Vec<_>, Vec<_>) = - dirs.into_iter().unzip(); + let (dir_source_file_data, dir_target_full_path) = + dirs.into_iter().unzip::<_, _, Vec<_>, Vec<_>>(); let step_files = dir_source_file_data .into_iter() diff --git a/core/src/object/tag/mod.rs b/core/src/object/tag/mod.rs index 98238462b..34b609a83 100644 --- a/core/src/object/tag/mod.rs +++ b/core/src/object/tag/mod.rs @@ -23,14 +23,14 @@ impl TagCreateArgs { ) -> Result { let pub_id = Uuid::now_v7().as_bytes().to_vec(); - let (sync_params, db_params): (Vec<_>, Vec<_>) = [ + let (sync_params, db_params) = [ sync_db_entry!(self.name, tag::name), sync_db_entry!(self.color, tag::color), sync_db_entry!(false, tag::is_hidden), sync_db_entry!(Utc::now(), tag::date_created), ] .into_iter() - .unzip(); + .unzip::<_, _, Vec<_>, Vec<_>>(); sync.write_op( db, diff --git a/core/src/object/validation/old_validator_job.rs b/core/src/object/validation/old_validator_job.rs index d90fc56cb..7ddd42938 100644 --- a/core/src/object/validation/old_validator_job.rs +++ b/core/src/object/validation/old_validator_job.rs @@ -15,8 +15,8 @@ use sd_prisma::{ prisma::{file_path, location}, prisma_sync, }; -use sd_sync::OperationFactory; -use sd_utils::{db::maybe_missing, error::FileIOError, msgpack}; +use sd_sync::{sync_db_entry, OperationFactory}; +use sd_utils::{db::maybe_missing, error::FileIOError}; use std::{ hash::{Hash, Hasher}, @@ -157,19 +157,22 @@ impl StatefulJob for OldObjectValidatorJobInit { .await .map_err(|e| ValidatorError::FileIO(FileIOError::from((full_path, e))))?; + let (sync_param, db_param) = sync_db_entry!(checksum, file_path::integrity_checksum); + sync.write_op( db, sync.shared_update( prisma_sync::file_path::SyncId { pub_id: file_path.pub_id.clone(), }, - file_path::integrity_checksum::NAME, - msgpack!(&checksum), - ), - db.file_path().update( - file_path::pub_id::equals(file_path.pub_id.clone()), - vec![file_path::integrity_checksum::set(Some(checksum))], + [sync_param], ), + db.file_path() + .update( + file_path::pub_id::equals(file_path.pub_id.clone()), + vec![db_param], + ) + .select(file_path::select!({ id })), ) .await?; } diff --git a/core/src/old_job/manager.rs b/core/src/old_job/manager.rs index f47164759..c9e5cc892 100644 --- a/core/src/old_job/manager.rs +++ b/core/src/old_job/manager.rs @@ -320,6 +320,7 @@ impl OldJobs { job::id::equals(job.id.as_bytes().to_vec()), vec![job::status::set(Some(JobStatus::Canceled as i32))], ) + .select(job::select!({ id })) .exec() .await?; } diff --git a/core/src/old_job/report.rs b/core/src/old_job/report.rs index ed40df23d..af7333267 100644 --- a/core/src/old_job/report.rs +++ b/core/src/old_job/report.rs @@ -395,6 +395,7 @@ impl OldJobReport { job::date_completed::set(self.completed_at.map(Into::into)), ], ) + .select(job::select!({ id })) .exec() .await?; Ok(()) diff --git a/core/src/volume/mod.rs b/core/src/volume/mod.rs index 50d9d1b15..9519d639a 100644 --- a/core/src/volume/mod.rs +++ b/core/src/volume/mod.rs @@ -7,7 +7,7 @@ use sd_prisma::{ prisma::{device, storage_statistics, PrismaClient}, prisma_sync, }; -use sd_sync::{sync_entry, OperationFactory}; +use sd_sync::{sync_db_not_null_entry, sync_entry, OperationFactory}; use sd_utils::uuid_to_bytes; use std::{ @@ -531,67 +531,66 @@ async fn update_storage_statistics( .map(|s| s.pub_id); if let Some(storage_statistics_pub_id) = storage_statistics_pub_id { - sync.write_ops( - db, - ( - [ - sync_entry!(total_capacity, storage_statistics::total_capacity), - sync_entry!(available_capacity, storage_statistics::available_capacity), - ] - .into_iter() - .map(|(field, value)| { - sync.shared_update( - prisma_sync::storage_statistics::SyncId { - pub_id: storage_statistics_pub_id.clone(), - }, - field, - value, - ) - }) - .collect(), - db.storage_statistics() - .update( - storage_statistics::pub_id::equals(storage_statistics_pub_id), - vec![ - storage_statistics::total_capacity::set(total_capacity as i64), - storage_statistics::available_capacity::set(available_capacity as i64), - ], - ) - // We don't need any data here, just the id avoids receiving the entire object - // as we can't pass an empty select macro call - .select(storage_statistics::select!({ id })), + let (sync_params, db_params) = [ + sync_db_not_null_entry!(total_capacity as i64, storage_statistics::total_capacity), + sync_db_not_null_entry!( + available_capacity as i64, + storage_statistics::available_capacity ), + ] + .into_iter() + .unzip::<_, _, Vec<_>, Vec<_>>(); + + sync.write_op( + db, + sync.shared_update( + prisma_sync::storage_statistics::SyncId { + pub_id: storage_statistics_pub_id.clone(), + }, + sync_params, + ), + db.storage_statistics() + .update( + storage_statistics::pub_id::equals(storage_statistics_pub_id), + db_params, + ) + // We don't need any data here, just the id avoids receiving the entire object + // as we can't pass an empty select macro call + .select(storage_statistics::select!({ id })), ) .await?; } else { let new_storage_statistics_id = uuid_to_bytes(&Uuid::now_v7()); + let (sync_params, db_params) = [ + sync_db_not_null_entry!(total_capacity as i64, storage_statistics::total_capacity), + sync_db_not_null_entry!( + available_capacity as i64, + storage_statistics::available_capacity + ), + ( + sync_entry!( + prisma_sync::device::SyncId { + pub_id: device_pub_id.clone() + }, + storage_statistics::device + ), + storage_statistics::device::connect(device::pub_id::equals(device_pub_id)), + ), + ] + .into_iter() + .unzip::<_, _, Vec<_>, Vec<_>>(); + sync.write_op( db, sync.shared_create( prisma_sync::storage_statistics::SyncId { pub_id: new_storage_statistics_id.clone(), }, - [ - sync_entry!(total_capacity, storage_statistics::total_capacity), - sync_entry!(available_capacity, storage_statistics::available_capacity), - sync_entry!( - prisma_sync::device::SyncId { - pub_id: device_pub_id.clone() - }, - storage_statistics::device - ), - ], + sync_params, ), db.storage_statistics() - .create( - new_storage_statistics_id, - vec![ - storage_statistics::total_capacity::set(total_capacity as i64), - storage_statistics::available_capacity::set(available_capacity as i64), - storage_statistics::device::connect(device::pub_id::equals(device_pub_id)), - ], - ) + .create(new_storage_statistics_id, db_params) // We don't need any data here, just the id avoids receiving the entire object // as we can't pass an empty select macro call .select(storage_statistics::select!({ id })), diff --git a/crates/crypto/src/cloud/decrypt.rs b/crates/crypto/src/cloud/decrypt.rs index c45a99110..94913f64b 100644 --- a/crates/crypto/src/cloud/decrypt.rs +++ b/crates/crypto/src/cloud/decrypt.rs @@ -31,7 +31,7 @@ impl OneShotDecryption for SecretKey { EncryptedBlockRef { nonce, cipher_text }: EncryptedBlockRef<'_>, ) -> Result, Error> { XChaCha20Poly1305::new(&self.0) - .decrypt(&nonce, cipher_text) + .decrypt(nonce, cipher_text) .map_err(|aead::Error| Error::Decrypt) } diff --git a/crates/crypto/src/cloud/secret_key.rs b/crates/crypto/src/cloud/secret_key.rs index c1df94f9f..2477684ad 100644 --- a/crates/crypto/src/cloud/secret_key.rs +++ b/crates/crypto/src/cloud/secret_key.rs @@ -191,7 +191,7 @@ mod tests { let EncryptedBlock { nonce, cipher_text } = key.encrypt(message, &mut rng).unwrap(); let mut bytes = Vec::with_capacity(nonce.len() + cipher_text.len()); - bytes.extend_from_slice(&nonce); + bytes.extend_from_slice(nonce.as_slice()); bytes.extend(cipher_text); assert_eq!( diff --git a/crates/crypto/src/primitives.rs b/crates/crypto/src/primitives.rs index a37981a01..1d8335fc1 100644 --- a/crates/crypto/src/primitives.rs +++ b/crates/crypto/src/primitives.rs @@ -16,7 +16,7 @@ pub struct EncryptedBlock { } pub struct EncryptedBlockRef<'e> { - pub nonce: OneShotNonce, + pub nonce: &'e OneShotNonce, pub cipher_text: &'e [u8], } @@ -25,7 +25,7 @@ impl<'e> From<&'e [u8]> for EncryptedBlockRef<'e> { let (nonce, cipher_text) = cipher_text.split_at(size_of::()); Self { - nonce: OneShotNonce::try_from(nonce).expect("we split the correct amount"), + nonce: nonce.try_into().expect("we split the correct amount"), cipher_text, } } diff --git a/crates/sync-generator/src/model.rs b/crates/sync-generator/src/model.rs index 767c1d820..e171634b8 100644 --- a/crates/sync-generator/src/model.rs +++ b/crates/sync-generator/src/model.rs @@ -46,7 +46,7 @@ pub fn module((model, sync_type): ModelWithSyncType<'_>) -> Module { RefinedFieldWalker::Scalar(scalar_field) => { (!scalar_field.is_in_required_relation()).then(|| { quote! { - #model_name_snake::#field_name_snake::set(::rmpv::ext::from_value(val).unwrap()), + #model_name_snake::#field_name_snake::set(::rmpv::ext::from_value(val)?), } }) } @@ -59,11 +59,19 @@ pub fn module((model, sync_type): ModelWithSyncType<'_>) -> Module { |i| { if i.count() == 1 { Some(quote! {{ - let val: std::collections::HashMap = ::rmpv::ext::from_value(val).unwrap(); - let val = val.into_iter().next().unwrap(); + + let (field, value) = ::rmpv + ::ext + ::from_value::>(val)? + .into_iter() + .next() + .ok_or(Error::MissingRelationData { + field: field.to_string(), + model: #relation_model_name_snake::NAME.to_string() + })?; #model_name_snake::#field_name_snake::connect( - #relation_model_name_snake::UniqueWhereParam::deserialize(&val.0, val.1).unwrap() + #relation_model_name_snake::UniqueWhereParam::deserialize(&field, value)? ) }}) } else { @@ -81,10 +89,13 @@ pub fn module((model, sync_type): ModelWithSyncType<'_>) -> Module { } else { quote! { impl #model_name_snake::SetParam { - pub fn deserialize(field: &str, val: ::rmpv::Value) -> Option { - Some(match field { + pub fn deserialize(field: &str, val: ::rmpv::Value) -> Result { + Ok(match field { #(#field_matches)* - _ => return None + _ => return Err(Error::FieldNotFound { + field: field.to_string(), + model: #model_name_snake::NAME.to_string(), + }), }) } } @@ -97,9 +108,12 @@ pub fn module((model, sync_type): ModelWithSyncType<'_>) -> Module { Module::new( model.name(), quote! { - use super::prisma::*; + use super::Error; + use prisma_client_rust::scalar_types::*; + use super::prisma::*; + #sync_id #set_param_impl @@ -172,7 +186,7 @@ fn process_unique_params(model: Walker<'_, ModelId>, model_name_snake: &Ident) - Some(quote!(#model_name_snake::#field_name_snake::NAME => #model_name_snake::#field_name_snake::equals( - ::rmpv::ext::from_value(val).unwrap() + ::rmpv::ext::from_value(val)? ), )) } @@ -185,10 +199,13 @@ fn process_unique_params(model: Walker<'_, ModelId>, model_name_snake: &Ident) - } else { quote! { impl #model_name_snake::UniqueWhereParam { - pub fn deserialize(field: &str, val: ::rmpv::Value) -> Option { - Some(match field { + pub fn deserialize(field: &str, val: ::rmpv::Value) -> Result { + Ok(match field { #(#field_matches)* - _ => return None + _ => return Err(Error::FieldNotFound { + field: field.to_string(), + model: #model_name_snake::NAME.to_string(), + }) }) } } diff --git a/crates/sync-generator/src/sync_data.rs b/crates/sync-generator/src/sync_data.rs index 9e9fdd937..e8ee713e6 100644 --- a/crates/sync-generator/src/sync_data.rs +++ b/crates/sync-generator/src/sync_data.rs @@ -7,7 +7,7 @@ use prisma_models::walkers::{FieldWalker, ScalarFieldWalker}; use crate::{ModelSyncType, ModelWithSyncType}; pub fn enumerate(models: &[ModelWithSyncType<'_>]) -> TokenStream { - let (variants, matches): (Vec<_>, Vec<_>) = models + let (variants, matches) = models .iter() .filter_map(|(model, sync_type)| { let model_name_snake = snake_ident(model.name()); @@ -26,12 +26,12 @@ pub fn enumerate(models: &[ModelWithSyncType<'_>]) -> TokenStream { quote!(#model_name_pascal(#model_name_snake::SyncId, sd_sync::CRDTOperationData)), quote! { #model_name_snake::MODEL_ID => - Self::#model_name_pascal(rmpv::ext::from_value(op.record_id).ok()?, op.data) + Self::#model_name_pascal(rmpv::ext::from_value(op.record_id)?, op.data) }, ) }) }) - .unzip(); + .unzip::<_, _, Vec<_>, Vec<_>>(); let exec_matches = models.iter().filter_map(|(model, sync_type)| { let model_name_pascal = pascal_ident(model.name()); @@ -54,20 +54,22 @@ pub fn enumerate(models: &[ModelWithSyncType<'_>]) -> TokenStream { }) }); + let error_enum = declare_error_enum(); + quote! { pub enum ModelSyncData { #(#variants),* } impl ModelSyncData { - pub fn from_op(op: sd_sync::CRDTOperation) -> Option { - Some(match op.model_id { + pub fn from_op(op: sd_sync::CRDTOperation) -> Result { + Ok(match op.model_id { #(#matches),*, - _ => return None + _ => return Err(Error::InvalidModelId(op.model_id)), }) } - pub async fn exec(self, db: &prisma::PrismaClient) -> prisma_client_rust::Result<()> { + pub async fn exec(self, db: &prisma::PrismaClient) -> Result<(), Error> { match self { #(#exec_matches),* } @@ -75,6 +77,69 @@ pub fn enumerate(models: &[ModelWithSyncType<'_>]) -> TokenStream { Ok(()) } } + + #error_enum + } +} + +fn declare_error_enum() -> TokenStream { + quote! { + #[derive(Debug)] + pub enum Error { + Rmpv(rmpv::ext::Error), + RmpSerialize(rmp_serde::encode::Error), + Prisma(prisma_client_rust::QueryError), + InvalidModelId(sd_sync::ModelId), + FieldNotFound { field: String, model: String }, + MissingRelationData { field: String, model: String }, + RelatedEntryNotFound { field: String, model: String }, + } + + impl From for Error { + fn from(e: rmpv::ext::Error) -> Self { + Self::Rmpv(e) + } + } + + impl From for Error { + fn from(e: rmp_serde::encode::Error) -> Self { + Self::RmpSerialize(e) + } + } + + impl From for Error { + fn from(e: prisma_client_rust::QueryError) -> Self { + Self::Prisma(e) + } + } + + impl std::fmt::Display for Error { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Rmpv(e) => write!(f, "Failed to serialize or deserialize rmpv data: {e}"), + Self::RmpSerialize(e) => write!(f, "Failed to serialize rmp data: {e}"), + Self::Prisma(e) => write!(f, "Prisma error: {e}"), + Self::InvalidModelId(id) => write!(f, "Invalid model id: {id}"), + Self::FieldNotFound { field, model } => { + write!(f, "Field '{field}' not found in model '{model}'") + } + Self::MissingRelationData { field, model } => { + write!( + f, + "Field '{field}' missing relation data in model '{model}'" + ) + } + Self::RelatedEntryNotFound { field, model } => { + write!( + f, + "Related entry for field '{field}' not found in table '{model}'" + ) + } + } + } + } + + impl std::error::Error for Error {} } } @@ -103,6 +168,7 @@ fn handle_crdt_ops_relation( .and_then(|(_m, sync)| sync.as_ref()) .map(|sync| snake_ident(sync.sync_id()[0].name())) .expect("missing sync id field name for relation"); + let item_model_name_snake = snake_ident(item.related_model().name()); let item_field_name_snake = snake_ident(item.name()); @@ -155,11 +221,15 @@ fn handle_crdt_ops_relation( vec![], ) .exec() - .await - .ok(); + .await?; }, - sd_sync::CRDTOperationData::Update { field, value } => { - let data = vec![prisma::#model_name_snake::SetParam::deserialize(&field, value).unwrap()]; + + sd_sync::CRDTOperationData::Update(data) => { + let data = data.into_iter() + .map(|(field, value)| { + prisma::#model_name_snake::SetParam::deserialize(&field, value) + }) + .collect::, _>>()?; db.#model_name_snake() .upsert( @@ -171,15 +241,14 @@ fn handle_crdt_ops_relation( data, ) .exec() - .await - .ok(); + .await?; }, + sd_sync::CRDTOperationData::Delete => { db.#model_name_snake() .delete(id) .exec() - .await - .ok(); + .await?; }, } } @@ -198,8 +267,10 @@ fn handle_crdt_ops_shared( .expect("missing fields") .next() .expect("empty fields"); + let id_name_snake = snake_ident(scalar_field.name()); let field_name_snake = snake_ident(rel.name()); + let opposite_model_name_snake = snake_ident( rel.opposite_relation_field() .expect("missing opposite relation field") @@ -211,12 +282,16 @@ fn handle_crdt_ops_shared( id.#field_name_snake.pub_id.clone() )); + let pub_id_field = format!("{field_name_snake}::pub_id"); + let rel_fetch = quote! { let rel = db.#opposite_model_name_snake() .find_unique(#relation_equals_condition) .exec() - .await? - .unwrap(); + .await?.ok_or_else(|| Error::RelatedEntryNotFound { + field: #pub_id_field.to_string(), + model: prisma::#opposite_model_name_snake::NAME.to_string(), + })?; }; ( @@ -226,6 +301,7 @@ fn handle_crdt_ops_shared( relation_equals_condition, ) } + RefinedFieldWalker::Scalar(s) => { let field_name_snake = snake_ident(s.name()); let thing = quote!(id.#field_name_snake.clone()); @@ -238,24 +314,12 @@ fn handle_crdt_ops_shared( #get_id match data { - sd_sync::CRDTOperationData::Create(data) => { - let data: Vec<_> = data.into_iter().map(|(field, value)| { - prisma::#model_name_snake::SetParam::deserialize(&field, value).unwrap() - }).collect(); - - db.#model_name_snake() - .upsert( - prisma::#model_name_snake::#id_name_snake::equals(#equals_value), - prisma::#model_name_snake::create(#create_id, data.clone()), - data - ) - .exec() - .await?; - }, - sd_sync::CRDTOperationData::Update { field, value } => { - let data = vec![ - prisma::#model_name_snake::SetParam::deserialize(&field, value).unwrap() - ]; + sd_sync::CRDTOperationData::Create(data) | sd_sync::CRDTOperationData::Update(data) => { + let data = data.into_iter() + .map(|(field, value)| { + prisma::#model_name_snake::SetParam::deserialize(&field, value) + }) + .collect::, _>>()?; db.#model_name_snake() .upsert( @@ -266,6 +330,7 @@ fn handle_crdt_ops_shared( .exec() .await?; }, + sd_sync::CRDTOperationData::Delete => { db.#model_name_snake() .delete(prisma::#model_name_snake::#id_name_snake::equals(#equals_value)) @@ -275,8 +340,8 @@ fn handle_crdt_ops_shared( db.crdt_operation() .delete_many(vec![ prisma::crdt_operation::model::equals(#model_id as i32), - prisma::crdt_operation::record_id::equals(rmp_serde::to_vec(&id).unwrap()), - prisma::crdt_operation::kind::equals(sd_sync::OperationKind::Create.to_string()) + prisma::crdt_operation::record_id::equals(rmp_serde::to_vec(&id)?), + prisma::crdt_operation::kind::equals(sd_sync::OperationKind::Create.to_string()), ]) .exec() .await?; diff --git a/crates/sync/src/crdt.rs b/crates/sync/src/crdt.rs index 13eda3ffa..3cbdf23d2 100644 --- a/crates/sync/src/crdt.rs +++ b/crates/sync/src/crdt.rs @@ -7,7 +7,7 @@ use uhlc::NTP64; pub enum OperationKind<'a> { Create, - Update(&'a str), + Update(Vec<&'a str>), Delete, } @@ -15,7 +15,7 @@ impl fmt::Display for OperationKind<'_> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { OperationKind::Create => write!(f, "c"), - OperationKind::Update(field) => write!(f, "u:{field}"), + OperationKind::Update(fields) => write!(f, "u:{}:", fields.join(":")), OperationKind::Delete => write!(f, "d"), } } @@ -26,7 +26,7 @@ pub enum CRDTOperationData { #[serde(rename = "c")] Create(BTreeMap), #[serde(rename = "u")] - Update { field: String, value: rmpv::Value }, + Update(BTreeMap), #[serde(rename = "d")] Delete, } @@ -41,7 +41,9 @@ impl CRDTOperationData { pub fn as_kind(&self) -> OperationKind<'_> { match self { Self::Create(_) => OperationKind::Create, - Self::Update { field, .. } => OperationKind::Update(field), + Self::Update(fields_and_values) => { + OperationKind::Update(fields_and_values.keys().map(String::as_str).collect()) + } Self::Delete => OperationKind::Delete, } } diff --git a/crates/sync/src/factory.rs b/crates/sync/src/factory.rs index 9fed9a52f..7c73f8b5f 100644 --- a/crates/sync/src/factory.rs +++ b/crates/sync/src/factory.rs @@ -46,15 +46,16 @@ pub trait OperationFactory { fn shared_update( &self, id: impl SyncId, - field: impl Into, - value: rmpv::Value, + values: impl IntoIterator + 'static, ) -> CRDTOperation { self.new_op( &id, - CRDTOperationData::Update { - field: field.into(), - value, - }, + CRDTOperationData::Update( + values + .into_iter() + .map(|(name, value)| (name.to_string(), value)) + .collect(), + ), ) } @@ -77,20 +78,23 @@ pub trait OperationFactory { ), ) } + fn relation_update( &self, id: impl RelationSyncId, - field: impl Into, - value: rmpv::Value, + values: impl IntoIterator + 'static, ) -> CRDTOperation { self.new_op( &id, - CRDTOperationData::Update { - field: field.into(), - value, - }, + CRDTOperationData::Update( + values + .into_iter() + .map(|(name, value)| (name.to_string(), value)) + .collect(), + ), ) } + fn relation_delete( &self, id: impl RelationSyncId, @@ -101,9 +105,14 @@ pub trait OperationFactory { #[macro_export] macro_rules! sync_entry { + (nil, $($prisma_column_module:tt)+) => { + ($($prisma_column_module)+::NAME, ::sd_utils::msgpack!(nil)) + }; + ($value:expr, $($prisma_column_module:tt)+) => { ($($prisma_column_module)+::NAME, ::sd_utils::msgpack!($value)) - } + }; + } #[macro_export] @@ -124,6 +133,28 @@ macro_rules! sync_db_entry { }} } +#[macro_export] +macro_rules! sync_db_nullable_entry { + ($value:expr, $($prisma_column_module:tt)+) => {{ + let value = $value.into(); + ( + $crate::sync_entry!(&value, $($prisma_column_module)+), + $($prisma_column_module)+::set(value) + ) + }} +} + +#[macro_export] +macro_rules! sync_db_not_null_entry { + ($value:expr, $($prisma_column_module:tt)+) => {{ + let value = $value.into(); + ( + $crate::sync_entry!(&value, $($prisma_column_module)+), + $($prisma_column_module)+::set(value) + ) + }} +} + #[macro_export] macro_rules! option_sync_db_entry { ($value:expr, $($prisma_column_module:tt)+) => { diff --git a/crates/utils/Cargo.toml b/crates/utils/Cargo.toml index 45891b344..2960a21c4 100644 --- a/crates/utils/Cargo.toml +++ b/crates/utils/Cargo.toml @@ -14,6 +14,8 @@ sd-prisma = { path = "../prisma" } # Workspace dependencies chrono = { workspace = true } prisma-client-rust = { workspace = true } +rmp-serde = { workspace = true } +rmpv = { workspace = true } rspc = { workspace = true, features = ["unstable"] } thiserror = { workspace = true } tracing = { workspace = true } From 3e763295f2adad4b5fa851a9d1bfa5c1c98335bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADtor=20Vasconcellos?= Date: Sat, 19 Oct 2024 08:06:11 -0300 Subject: [PATCH 04/21] Remove obsolete settings cloud page - Fixed some typings - Update vite --- apps/desktop/package.json | 4 +- apps/mobile/package.json | 3 +- .../components/modal/ImportLibraryModal.tsx | 4 +- .../src/navigation/tabs/SettingsStack.tsx | 12 - .../library/CloudSettings/CloudSettings.tsx | 130 --------- .../library/CloudSettings/Instance.tsx | 64 ----- .../library/CloudSettings/Library.tsx | 66 ----- .../settings/library/CloudSettings/Login.tsx | 45 --- .../library/CloudSettings/ThisInstance.tsx | 76 ----- .../screens/settings/library/SyncSettings.tsx | 158 ----------- apps/storybook/package.json | 2 +- apps/web/package.json | 4 +- .../SidebarLayout/LibrariesDropdown.tsx | 22 +- interface/app/$libraryId/debug/actors.tsx | 69 ----- interface/app/$libraryId/debug/cloud.tsx | 262 ------------------ interface/app/$libraryId/debug/index.ts | 6 - interface/app/$libraryId/index.tsx | 5 +- .../$libraryId/settings/client/general.tsx | 4 +- .../app/$libraryId/settings/library/index.tsx | 1 - .../app/$libraryId/settings/library/sync.tsx | 228 --------------- .../settings/node/libraries/JoinDialog.tsx | 99 ------- .../settings/node/libraries/ListItem.tsx | 43 +-- .../settings/node/libraries/index.tsx | 18 +- interface/app/onboarding/index.tsx | 2 - interface/app/onboarding/join-library.tsx | 84 ------ interface/app/onboarding/new-library.tsx | 15 - interface/components/Authentication.tsx | 4 +- interface/components/Login.tsx | 2 +- interface/package.json | 3 +- package.json | 4 +- packages/config/package.json | 5 +- .../@react-navigation__drawer@6.6.15.patch | 54 +--- pnpm-lock.yaml | Bin 1193120 -> 1188670 bytes 33 files changed, 33 insertions(+), 1465 deletions(-) delete mode 100644 apps/mobile/src/screens/settings/library/CloudSettings/CloudSettings.tsx delete mode 100644 apps/mobile/src/screens/settings/library/CloudSettings/Instance.tsx delete mode 100644 apps/mobile/src/screens/settings/library/CloudSettings/Library.tsx delete mode 100644 apps/mobile/src/screens/settings/library/CloudSettings/Login.tsx delete mode 100644 apps/mobile/src/screens/settings/library/CloudSettings/ThisInstance.tsx delete mode 100644 apps/mobile/src/screens/settings/library/SyncSettings.tsx delete mode 100644 interface/app/$libraryId/debug/actors.tsx delete mode 100644 interface/app/$libraryId/debug/cloud.tsx delete mode 100644 interface/app/$libraryId/debug/index.ts delete mode 100644 interface/app/$libraryId/settings/library/sync.tsx delete mode 100644 interface/app/$libraryId/settings/node/libraries/JoinDialog.tsx delete mode 100644 interface/app/onboarding/join-library.tsx diff --git a/apps/desktop/package.json b/apps/desktop/package.json index 08058a5de..5fd357b0e 100644 --- a/apps/desktop/package.json +++ b/apps/desktop/package.json @@ -41,7 +41,7 @@ "@types/react-dom": "^18.2.22", "sass": "^1.72.0", "typescript": "^5.6.2", - "vite": "^5.2.0", - "vite-tsconfig-paths": "^4.3.2" + "vite": "^5.4.9", + "vite-tsconfig-paths": "^5.0.1" } } diff --git a/apps/mobile/package.json b/apps/mobile/package.json index 31c28d3f3..741a20ee0 100644 --- a/apps/mobile/package.json +++ b/apps/mobile/package.json @@ -22,11 +22,10 @@ "@gorhom/bottom-sheet": "^4.6.1", "@hookform/resolvers": "^3.1.0", "@oscartbeaumont-sd/rspc-client": "github:spacedriveapp/rspc#path:packages/client&bc882f4724", - "@oscartbeaumont-sd/rspc-react": "github:spacedriveapp/rspc#path:packages/react&bc882f4724", "@react-native-async-storage/async-storage": "~1.23.1", "@react-native-masked-view/masked-view": "^0.3.1", "@react-navigation/bottom-tabs": "^6.5.19", - "@react-navigation/drawer": "^6.6.14", + "@react-navigation/drawer": "^6.6.15", "@react-navigation/native": "^6.1.16", "@react-navigation/native-stack": "^6.9.25", "@sd/assets": "workspace:*", diff --git a/apps/mobile/src/components/modal/ImportLibraryModal.tsx b/apps/mobile/src/components/modal/ImportLibraryModal.tsx index dddedcc53..fac41a218 100644 --- a/apps/mobile/src/components/modal/ImportLibraryModal.tsx +++ b/apps/mobile/src/components/modal/ImportLibraryModal.tsx @@ -57,11 +57,11 @@ const ImportModalLibrary = forwardRef((_, ref) => { description="No cloud libraries available to join" /> } - keyExtractor={(item) => item.uuid} + keyExtractor={(item) => item.pub_id} showsVerticalScrollIndicator={false} renderItem={({ item }) => ( diff --git a/apps/mobile/src/navigation/tabs/SettingsStack.tsx b/apps/mobile/src/navigation/tabs/SettingsStack.tsx index 07cdc3688..c07f1111d 100644 --- a/apps/mobile/src/navigation/tabs/SettingsStack.tsx +++ b/apps/mobile/src/navigation/tabs/SettingsStack.tsx @@ -14,12 +14,10 @@ import PrivacySettingsScreen from '~/screens/settings/client/PrivacySettings'; import AboutScreen from '~/screens/settings/info/About'; import DebugScreen from '~/screens/settings/info/Debug'; import SupportScreen from '~/screens/settings/info/Support'; -import CloudSettings from '~/screens/settings/library/CloudSettings/CloudSettings'; import EditLocationSettingsScreen from '~/screens/settings/library/EditLocationSettings'; import LibraryGeneralSettingsScreen from '~/screens/settings/library/LibraryGeneralSettings'; import LocationSettingsScreen from '~/screens/settings/library/LocationSettings'; import NodesSettingsScreen from '~/screens/settings/library/NodesSettings'; -import SyncSettingsScreen from '~/screens/settings/library/SyncSettings'; import TagsSettingsScreen from '~/screens/settings/library/TagsSettings'; import SettingsScreen from '~/screens/settings/Settings'; @@ -106,16 +104,6 @@ export default function SettingsStack() { component={TagsSettingsScreen} options={{ header: () =>
}} /> -
}} - /> -
}} - /> {/* { - return ( - - - - ); -}; - -const AuthSensitiveChild = () => { - const authState = useAuthStateSnapshot(); - if (authState.status === 'loggedIn') return ; - if (authState.status === 'notLoggedIn' || authState.status === 'loggingIn') return ; - - return null; -}; - -const Authenticated = () => { - const { library } = useLibraryContext(); - const cloudLibrary = useLibraryQuery(['cloud.library.get'], { retry: false }); - const createLibrary = useLibraryMutation(['cloud.library.create']); - - const cloudInstances = useMemo( - () => - cloudLibrary.data?.instances.filter( - (instance) => instance.uuid !== library.instance_id - ), - [cloudLibrary.data, library.instance_id] - ); - - if (cloudLibrary.isLoading) { - return ( - - - - ); - } - - return ( - - {cloudLibrary.data ? ( - - - - - - - - {cloudInstances?.length} - - - Instances - - - - - } - contentContainerStyle={twStyle( - cloudInstances?.length === 0 && 'flex-row' - )} - showsHorizontalScrollIndicator={false} - ItemSeparatorComponent={() => } - renderItem={({ item }) => } - keyExtractor={(item) => item.id} - numColumns={1} - /> - - - - ) : ( - - - - - Uploading your library to the cloud will allow you to access your - library from other devices using your account & importing. - - - - - )} - - ); -}; - -export default CloudSettings; diff --git a/apps/mobile/src/screens/settings/library/CloudSettings/Instance.tsx b/apps/mobile/src/screens/settings/library/CloudSettings/Instance.tsx deleted file mode 100644 index dbac4a60a..000000000 --- a/apps/mobile/src/screens/settings/library/CloudSettings/Instance.tsx +++ /dev/null @@ -1,64 +0,0 @@ -import { Text, View } from 'react-native'; -import { CloudInstance, HardwareModel } from '@sd/client'; -import { Icon } from '~/components/icons/Icon'; -import { hardwareModelToIcon } from '~/components/overview/Devices'; -import { tw } from '~/lib/tailwind'; - -import { InfoBox } from './CloudSettings'; - -interface Props { - data: CloudInstance; -} - -const Instance = ({ data }: Props) => { - return ( - - - - - - - {data.metadata.name} - - - - Id: - - {data.id} - - - - - - - - UUID: - - {data.uuid} - - - - - - - - Public key: - - {data.identity} - - - - - - ); -}; - -export default Instance; diff --git a/apps/mobile/src/screens/settings/library/CloudSettings/Library.tsx b/apps/mobile/src/screens/settings/library/CloudSettings/Library.tsx deleted file mode 100644 index 7e385bd31..000000000 --- a/apps/mobile/src/screens/settings/library/CloudSettings/Library.tsx +++ /dev/null @@ -1,66 +0,0 @@ -import { CheckCircle, XCircle } from 'phosphor-react-native'; -import { useMemo } from 'react'; -import { Text, View } from 'react-native'; -import { CloudLibrary, useLibraryContext, useLibraryMutation } from '@sd/client'; -import Card from '~/components/layout/Card'; -import { Button } from '~/components/primitive/Button'; -import { Divider } from '~/components/primitive/Divider'; -import { SettingsTitle } from '~/components/settings/SettingsContainer'; -import { tw } from '~/lib/tailwind'; -import { logout, useAuthStateSnapshot } from '~/stores/auth'; - -import { InfoBox } from './CloudSettings'; - -interface LibraryProps { - cloudLibrary?: CloudLibrary; -} - -const Library = ({ cloudLibrary }: LibraryProps) => { - const authState = useAuthStateSnapshot(); - const { library } = useLibraryContext(); - const syncLibrary = useLibraryMutation(['cloud.library.sync']); - const thisInstance = useMemo( - () => cloudLibrary?.instances.find((instance) => instance.uuid === library.instance_id), - [cloudLibrary, library.instance_id] - ); - - return ( - - - Library - {authState.status === 'loggedIn' && ( - - )} - - - Name - - {cloudLibrary?.name} - - - - ); -}; - -export default Library; diff --git a/apps/mobile/src/screens/settings/library/CloudSettings/Login.tsx b/apps/mobile/src/screens/settings/library/CloudSettings/Login.tsx deleted file mode 100644 index 88738c329..000000000 --- a/apps/mobile/src/screens/settings/library/CloudSettings/Login.tsx +++ /dev/null @@ -1,45 +0,0 @@ -import { Text, View } from 'react-native'; -import { Icon } from '~/components/icons/Icon'; -import Card from '~/components/layout/Card'; -import { Button } from '~/components/primitive/Button'; -import { tw } from '~/lib/tailwind'; -import { cancel, login, useAuthStateSnapshot } from '~/stores/auth'; - -const Login = () => { - const authState = useAuthStateSnapshot(); - const buttonText = { - notLoggedIn: 'Login', - loggingIn: 'Cancel' - }; - return ( - - - - - - Cloud Sync will upload your library to the cloud so you can access your - library from other devices by importing it from the cloud. - - - {(authState.status === 'notLoggedIn' || authState.status === 'loggingIn') && ( - - )} - - - ); -}; - -export default Login; diff --git a/apps/mobile/src/screens/settings/library/CloudSettings/ThisInstance.tsx b/apps/mobile/src/screens/settings/library/CloudSettings/ThisInstance.tsx deleted file mode 100644 index 041d6591c..000000000 --- a/apps/mobile/src/screens/settings/library/CloudSettings/ThisInstance.tsx +++ /dev/null @@ -1,76 +0,0 @@ -import { useMemo } from 'react'; -import { Text, View } from 'react-native'; -import { CloudLibrary, HardwareModel, useLibraryContext } from '@sd/client'; -import { Icon } from '~/components/icons/Icon'; -import Card from '~/components/layout/Card'; -import { hardwareModelToIcon } from '~/components/overview/Devices'; -import { Divider } from '~/components/primitive/Divider'; -import { tw } from '~/lib/tailwind'; - -import { InfoBox } from './CloudSettings'; - -interface ThisInstanceProps { - cloudLibrary?: CloudLibrary; -} - -const ThisInstance = ({ cloudLibrary }: ThisInstanceProps) => { - const { library } = useLibraryContext(); - const thisInstance = useMemo( - () => cloudLibrary?.instances.find((instance) => instance.uuid === library.instance_id), - [cloudLibrary, library.instance_id] - ); - - if (!thisInstance) return null; - - return ( - - - This Instance - - - - - - {thisInstance.metadata.name} - - - - - - Id: - {thisInstance.id} - - - - - - - UUID: - - {thisInstance.uuid} - - - - - - - - Publc Key: - - {thisInstance.identity} - - - - - - ); -}; - -export default ThisInstance; diff --git a/apps/mobile/src/screens/settings/library/SyncSettings.tsx b/apps/mobile/src/screens/settings/library/SyncSettings.tsx deleted file mode 100644 index d6cb460de..000000000 --- a/apps/mobile/src/screens/settings/library/SyncSettings.tsx +++ /dev/null @@ -1,158 +0,0 @@ -import { inferSubscriptionResult } from '@oscartbeaumont-sd/rspc-client'; -import { useIsFocused } from '@react-navigation/native'; -import { MotiView } from 'moti'; -import { Circle } from 'phosphor-react-native'; -import React, { useEffect, useRef, useState } from 'react'; -import { Text, View } from 'react-native'; -import { - Procedures, - useLibraryMutation, - useLibraryQuery, - useLibrarySubscription -} from '@sd/client'; -import { Icon } from '~/components/icons/Icon'; -import Card from '~/components/layout/Card'; -import { ModalRef } from '~/components/layout/Modal'; -import ScreenContainer from '~/components/layout/ScreenContainer'; -import CloudModal from '~/components/modal/cloud/CloudModal'; -import { Button } from '~/components/primitive/Button'; -import { tw } from '~/lib/tailwind'; -import { SettingsStackScreenProps } from '~/navigation/tabs/SettingsStack'; - -const SyncSettingsScreen = ({ navigation }: SettingsStackScreenProps<'SyncSettings'>) => { - const syncEnabled = useLibraryQuery(['sync.enabled']); - const [data, setData] = useState>({}); - const modalRef = useRef(null); - - const [startBackfill, setStart] = useState(false); - const pageFocused = useIsFocused(); - const [showCloudModal, setShowCloudModal] = useState(false); - - useLibrarySubscription(['library.actors'], { onData: setData }); - - useEffect(() => { - if (startBackfill === true) { - navigation.navigate('BackfillWaitingStack', { - screen: 'BackfillWaiting' - }); - setTimeout(() => setShowCloudModal(true), 1000); - } - }, [startBackfill, navigation]); - - useEffect(() => { - if (pageFocused && showCloudModal) modalRef.current?.present(); - return () => { - if (showCloudModal) setShowCloudModal(false); - }; - }, [pageFocused, showCloudModal]); - - return ( - - {syncEnabled.data === false ? ( - - - - - - With Sync, you can share your library with other devices using P2P - technology. - - - Additionally, allowing you to enable Cloud services to upload your - library to the cloud, making it accessible on any of your devices. - - - - - - ) : ( - - {Object.keys(data).map((key) => { - return ( - - - - {key} - - {data[key] ? : } - - ); - })} - - )} - - - ); -}; - -export default SyncSettingsScreen; - -function OnlineIndicator({ online }: { online: boolean }) { - const size = 6; - return ( - - {online ? ( - - - - - ) : ( - - )} - - ); -} - -function StartButton({ name }: { name: string }) { - const startActor = useLibraryMutation(['library.startActor']); - return ( - - ); -} - -function StopButton({ name }: { name: string }) { - const stopActor = useLibraryMutation(['library.stopActor']); - return ( - - ); -} diff --git a/apps/storybook/package.json b/apps/storybook/package.json index 0f5786227..5dfe9f15f 100644 --- a/apps/storybook/package.json +++ b/apps/storybook/package.json @@ -30,6 +30,6 @@ "storybook": "^8.0.1", "tailwindcss": "^3.4.10", "typescript": "^5.6.2", - "vite": "^5.2.0" + "vite": "^5.4.9" } } diff --git a/apps/web/package.json b/apps/web/package.json index b487de1d8..8fa032697 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -41,7 +41,7 @@ "rollup-plugin-visualizer": "^5.12.0", "start-server-and-test": "^2.0.3", "typescript": "^5.6.2", - "vite": "^5.2.0", - "vite-tsconfig-paths": "^4.3.2" + "vite": "^5.4.9", + "vite-tsconfig-paths": "^5.0.1" } } diff --git a/interface/app/$libraryId/Layout/Sidebar/SidebarLayout/LibrariesDropdown.tsx b/interface/app/$libraryId/Layout/Sidebar/SidebarLayout/LibrariesDropdown.tsx index 0511cf33b..44515a41c 100644 --- a/interface/app/$libraryId/Layout/Sidebar/SidebarLayout/LibrariesDropdown.tsx +++ b/interface/app/$libraryId/Layout/Sidebar/SidebarLayout/LibrariesDropdown.tsx @@ -1,9 +1,7 @@ -import { CloudArrowDown, Gear, Lock, Plus } from '@phosphor-icons/react'; +import { Gear, Plus } from '@phosphor-icons/react'; import clsx from 'clsx'; import { useClientContext } from '@sd/client'; import { dialogManager, Dropdown, DropdownMenu } from '@sd/ui'; -import JoinDialog from '~/app/$libraryId/settings/node/libraries/JoinDialog'; -import RequestAddDialog from '~/components/RequestAddDialog'; import { useLocale } from '~/hooks'; import CreateDialog from '../../../settings/node/libraries/CreateDialog'; @@ -64,17 +62,6 @@ export default () => { onClick={() => dialogManager.create((dp) => )} className="font-medium" /> - - dialogManager.create((dp) => ( - - )) - } - className="font-medium" - /> { to="settings/library/general" className="font-medium" /> - {/* alert('TODO: Not implemented yet!')} - className="font-medium" - /> */} ); }; diff --git a/interface/app/$libraryId/debug/actors.tsx b/interface/app/$libraryId/debug/actors.tsx deleted file mode 100644 index a744583a6..000000000 --- a/interface/app/$libraryId/debug/actors.tsx +++ /dev/null @@ -1,69 +0,0 @@ -import { inferSubscriptionResult } from '@oscartbeaumont-sd/rspc-client'; -import { useMemo, useState } from 'react'; -import { Procedures, useLibraryMutation, useLibrarySubscription } from '@sd/client'; -import { Button } from '@sd/ui'; -import { useRouteTitle } from '~/hooks/useRouteTitle'; - -// @million-ignore -export const Component = () => { - useRouteTitle('Actors'); - - const [data, setData] = useState>({}); - - useLibrarySubscription(['library.actors'], { onData: setData }); - - const sortedData = useMemo(() => { - const sorted = Object.entries(data).sort(([a], [b]) => a.localeCompare(b)); - return sorted; - }, [data]); - - return ( -
- - - - - - {sortedData.map(([name, running]) => ( - - - - - - ))} -
NameRunning
{name} - {running ? 'Running' : 'Not Running'} - - {running ? : } -
-
- ); -}; - -function StartButton({ name }: { name: string }) { - const startActor = useLibraryMutation(['library.startActor']); - - return ( - - ); -} - -function StopButton({ name }: { name: string }) { - const stopActor = useLibraryMutation(['library.stopActor']); - - return ( - - ); -} diff --git a/interface/app/$libraryId/debug/cloud.tsx b/interface/app/$libraryId/debug/cloud.tsx deleted file mode 100644 index dfc60cbea..000000000 --- a/interface/app/$libraryId/debug/cloud.tsx +++ /dev/null @@ -1,262 +0,0 @@ -import { CheckCircle, XCircle } from '@phosphor-icons/react'; -import { Suspense, useMemo } from 'react'; -import { - auth, - HardwareModel, - useBridgeQuery, - useLibraryContext, - useLibraryMutation, - useLibraryQuery -} from '@sd/client'; -import { Button, Card, Loader, tw } from '@sd/ui'; -import { Icon } from '~/components'; -import { AuthRequiredOverlay } from '~/components/AuthRequiredOverlay'; -import { LoginButton } from '~/components/LoginButton'; -import { useLocale, useRouteTitle } from '~/hooks'; -import { hardwareModelToIcon } from '~/util/hardware'; - -const DataBox = tw.div`max-w-[300px] rounded-md border border-app-line/50 bg-app-lightBox/20 p-2`; -const Count = tw.div`min-w-[20px] flex h-[20px] px-1 items-center justify-center rounded-full border border-app-button/40 text-[9px]`; - -export const Component = () => { - useRouteTitle('Cloud'); - - // const authState = auth.useStateSnapshot(); - - // const authSensitiveChild = () => { - // if (authState.status === 'loggedIn') return ; - // if (authState.status === 'notLoggedIn' || authState.status === 'loggingIn') - // return ( - //
- // - //
- // - //

- // To access cloud related features, please login - //

- //
- // - // - //
- // ); - - // return null; - // }; - - // return
{Authenticated()}
; - return
; -}; - -// million-ignore -// function Authenticated() { -// const { library } = useLibraryContext(); -// const cloudLibrary: any = useLibraryQuery(['cloud.library.get'], { -// suspense: true, -// retry: false -// }); -// const getCloudDevice = useBridgeQuery(['cloud.devices.get'], { -// suspense: true, -// retry: false -// }); -// const cloudDevicesList = useBridgeQuery(['cloud.devices.list'], { -// suspense: true, -// retry: false -// }); -// console.log('[DEBUG] fetch cloud device:', getCloudDevice.data); -// console.log('[DEBUG] cloudDevicesList', cloudDevicesList.data); -// const createLibrary = useLibraryMutation(['cloud.library.create']); -// const { t } = useLocale(); - -// const thisInstance = useMemo(() => { -// if (!cloudLibrary.data) return undefined; -// return cloudLibrary.data.instances.find( -// (instance: any) => instance.uuid === library.instance_id -// ); -// }, [cloudLibrary.data, library.instance_id]); - -// return ( -// -// -// -// } -// > -// {cloudLibrary.data ? ( -//
-// -// {thisInstance && } -// -//
-// ) : ( -//
-// -// -//
-// -//

-// {t('cloud_connect_description')} -//

-//
-// -// -//
-// )} -// -// ); -// } - -// // million-ignore -// const Instances = ({ instances }: { instances: any[] }) => { -// const { library } = useLibraryContext(); -// const filteredInstances = instances.filter((instance) => instance.uuid !== library.instance_id); -// return ( -//
-//
-//

Instances

-// {filteredInstances.length} -//
-//
-// {filteredInstances.map((instance) => ( -// -//
-// -//

-// {instance.metadata.name} -//

-//
-//
-// -//

-// Id:{' '} -// {instance.id} -//

-// -// -//

-// UUID:{' '} -// -// {instance.uuid} -// -//

-// -// -//

-// Public Key:{' '} -// -// {instance.identity} -// -//

-// -//
-// -// ))} -//
-//
-// ); -// }; - -// interface LibraryProps { -// cloudLibrary: any; -// thisInstance: any | undefined; -// } - -// // million-ignore -// const Library = ({ thisInstance, cloudLibrary }: LibraryProps) => { -// const syncLibrary = useLibraryMutation(['cloud.library.sync']); -// return ( -//
-//

Library

-// -//

-// Name: {cloudLibrary.name} -//

-// -// -//
-// ); -// }; - -// interface ThisInstanceProps { -// instance: any; -// } - -// // million-ignore -// const ThisInstance = ({ instance }: ThisInstanceProps) => { -// return ( -//
-//

This Instance

-// -//
-// -//

-// {instance.metadata.name} -//

-//
-//
-// -//

-// Id: {instance.id} -//

-// -// -//

-// UUID: {instance.uuid} -//

-// -// -//

-// Public Key:{' '} -// {instance.identity} -//

-// -//
-// -//
-// ); -// }; diff --git a/interface/app/$libraryId/debug/index.ts b/interface/app/$libraryId/debug/index.ts deleted file mode 100644 index 4cf60b56c..000000000 --- a/interface/app/$libraryId/debug/index.ts +++ /dev/null @@ -1,6 +0,0 @@ -import { RouteObject } from 'react-router'; - -export const debugRoutes = [ - { path: 'cloud', lazy: () => import('./cloud') }, - { path: 'actors', lazy: () => import('./actors') } -] satisfies RouteObject[]; diff --git a/interface/app/$libraryId/index.tsx b/interface/app/$libraryId/index.tsx index 6be3b0dd3..10a95e97a 100644 --- a/interface/app/$libraryId/index.tsx +++ b/interface/app/$libraryId/index.tsx @@ -3,15 +3,13 @@ import { type RouteObject } from 'react-router-dom'; import { guessOperatingSystem } from '~/hooks'; import { Platform } from '~/util/Platform'; -import { debugRoutes } from './debug'; import settingsRoutes from './settings'; // Routes that should be contained within the standard Page layout const pageRoutes: RouteObject = { lazy: () => import('./PageLayout'), children: [ - { path: 'overview', lazy: () => import('./overview') }, - { path: 'debug', children: debugRoutes } + { path: 'overview', lazy: () => import('./overview') } ] }; @@ -38,7 +36,6 @@ function loadTopBarRoutes() { ...explorerRoutes, pageRoutes, { path: 'settings', lazy: () => import('./settings/Layout'), children: settingsRoutes }, - { path: 'debug', children: debugRoutes } ]; } else return [...explorerRoutes, pageRoutes]; } diff --git a/interface/app/$libraryId/settings/client/general.tsx b/interface/app/$libraryId/settings/client/general.tsx index 4b42f6753..bba74c7f6 100644 --- a/interface/app/$libraryId/settings/client/general.tsx +++ b/interface/app/$libraryId/settings/client/general.tsx @@ -50,7 +50,7 @@ export const Component = () => { schema: z .object({ name: z.string().min(1).max(250).optional(), - image_labeler_version: z.string().optional(), + // image_labeler_version: z.string().optional(), background_processing_percentage: z.coerce .number({ invalid_type_error: 'Must use numbers from 0 to 100' @@ -63,7 +63,7 @@ export const Component = () => { reValidateMode: 'onChange', defaultValues: { name: node.data?.name, - image_labeler_version: node.data?.image_labeler_version ?? undefined + // image_labeler_version: node.data?.image_labeler_version ?? undefined // background_processing_percentage: // node.data?.preferences.thumbnailer.background_processing_percentage || 50 } diff --git a/interface/app/$libraryId/settings/library/index.tsx b/interface/app/$libraryId/settings/library/index.tsx index fd05bbd68..9be623d80 100644 --- a/interface/app/$libraryId/settings/library/index.tsx +++ b/interface/app/$libraryId/settings/library/index.tsx @@ -8,7 +8,6 @@ export default [ // { path: 'keys', lazy: () => import('./keys') }, { path: 'security', lazy: () => import('./security') }, { path: 'sharing', lazy: () => import('./sharing') }, - { path: 'sync', lazy: () => import('./sync') }, { path: 'general', lazy: () => import('./general') }, { path: 'tags', lazy: () => import('./tags') }, // { path: 'saved-searches', lazy: () => import('./saved-searches') }, diff --git a/interface/app/$libraryId/settings/library/sync.tsx b/interface/app/$libraryId/settings/library/sync.tsx deleted file mode 100644 index fe6124525..000000000 --- a/interface/app/$libraryId/settings/library/sync.tsx +++ /dev/null @@ -1,228 +0,0 @@ -import { inferSubscriptionResult } from '@oscartbeaumont-sd/rspc-client'; -import clsx from 'clsx'; -import { useEffect, useState } from 'react'; -import { - Procedures, - useFeatureFlag, - useLibraryMutation, - useLibraryQuery, - useLibrarySubscription, - useZodForm -} from '@sd/client'; -import { Button, Dialog, dialogManager, useDialog, UseDialogProps, z } from '@sd/ui'; -import { useLocale } from '~/hooks'; - -import { Heading } from '../Layout'; -import Setting from '../Setting'; - -const ACTORS = { - Ingest: 'Sync Ingest', - CloudSend: 'Cloud Sync Sender', - CloudReceive: 'Cloud Sync Receiver', - CloudIngest: 'Cloud Sync Ingest' -}; - -export const Component = () => { - const { t } = useLocale(); - - const syncEnabled = useLibraryQuery(['sync.enabled']); - - const backfillSync = useLibraryMutation(['sync.backfill'], { - onSuccess: async () => { - await syncEnabled.refetch(); - } - }); - - const [data, setData] = useState>({}); - - useLibrarySubscription(['library.actors'], { onData: setData }); - - const cloudSync = useFeatureFlag('cloudSync'); - - return ( - <> - - {syncEnabled.data === false ? ( - -
- -
- - ) : ( - <> - - {t('ingester')} - - - } - description={t('injester_description')} - > -
- {data[ACTORS.Ingest] ? ( - - ) : ( - - )} -
- - - {cloudSync && } - - )} - - ); -}; - -function SyncBackfillDialog(props: UseDialogProps & { onEnabled: () => void }) { - const form = useZodForm({ schema: z.object({}) }); - const dialog = useDialog(props); - const { t } = useLocale(); - - const enableSync = useLibraryMutation(['sync.backfill'], {}); - - // dialog is in charge of enabling sync - useEffect(() => { - form.handleSubmit( - async () => { - await enableSync.mutateAsync(null).then(() => (dialog.state.open = false)); - await props.onEnabled(); - }, - () => {} - )(); - // eslint-disable-next-line react-hooks/exhaustive-deps - }, []); - - return ( - - ); -} - -function CloudSync({ data }: { data: inferSubscriptionResult }) { - const { t } = useLocale(); - return ( - <> -
-

{t('cloud_sync')}

-

{t('cloud_sync_description')}

-
- - {t('sender')} - - } - description={t('sender_description')} - > -
- {data[ACTORS.CloudSend] ? ( - - ) : ( - - )} -
- - - {t('receiver')} - - - } - description={t('receiver_description')} - > -
- {data[ACTORS.CloudReceive] ? ( - - ) : ( - - )} -
- - - {t('ingester')} - - - } - description={t('ingester_description')} - > -
- {data[ACTORS.CloudIngest] ? ( - - ) : ( - - )} -
- - - ); -} - -function StartButton({ name }: { name: string }) { - const startActor = useLibraryMutation(['library.startActor']); - const { t } = useLocale(); - - return ( - - ); -} - -function StopButton({ name }: { name: string }) { - const stopActor = useLibraryMutation(['library.stopActor']); - const { t } = useLocale(); - - return ( - - ); -} - -function OnlineIndicator({ online }: { online: boolean }) { - return ( -
- ); -} diff --git a/interface/app/$libraryId/settings/node/libraries/JoinDialog.tsx b/interface/app/$libraryId/settings/node/libraries/JoinDialog.tsx deleted file mode 100644 index e57f5340c..000000000 --- a/interface/app/$libraryId/settings/node/libraries/JoinDialog.tsx +++ /dev/null @@ -1,99 +0,0 @@ -import { useQueryClient } from '@tanstack/react-query'; -import { useNavigate } from 'react-router'; -import { LibraryConfigWrapped, useBridgeMutation, useBridgeQuery, useZodForm } from '@sd/client'; -import { Dialog, Loader, Select, SelectOption, toast, useDialog, UseDialogProps, z } from '@sd/ui'; -import { useLocale } from '~/hooks'; -import { usePlatform } from '~/util/Platform'; - -const schema = z.object({ - libraryId: z.string().refine((value) => value !== 'select_library', { - message: 'Please select a library' - }) -}); - -export default (props: UseDialogProps & { librariesCtx: LibraryConfigWrapped[] | undefined }) => { - const cloudLibraries = useBridgeQuery(['cloud.library.list']); - const joinLibrary = useBridgeMutation(['cloud.library.join']); - - const { t } = useLocale(); - const navigate = useNavigate(); - const platform = usePlatform(); - const queryClient = useQueryClient(); - - const form = useZodForm({ schema, defaultValues: { libraryId: 'select_library' } }); - - // const queryClient = useQueryClient(); - // const submitPlausibleEvent = usePlausibleEvent(); - // const platform = usePlatform(); - - const onSubmit = form.handleSubmit(async (data) => { - try { - const library = await joinLibrary.mutateAsync(data.libraryId); - - queryClient.setQueryData(['library.list'], (libraries: any) => { - // The invalidation system beat us to it - if ((libraries || []).find((l: any) => l.uuid === library.uuid)) return libraries; - - return [...(libraries || []), library]; - }); - - if (platform.refreshMenuBar) platform.refreshMenuBar(); - - navigate(`/${library.uuid}`, { replace: true }); - } catch (e: any) { - console.error(e); - toast.error(e); - } - }); - - return ( - -
- {cloudLibraries.isLoading && ( -
- - {t('loading')}... -
- )} - {cloudLibraries.data && ( - - )} -
- - ); -}; diff --git a/interface/app/$libraryId/settings/node/libraries/ListItem.tsx b/interface/app/$libraryId/settings/node/libraries/ListItem.tsx index 78bc8beec..0d1bc8550 100644 --- a/interface/app/$libraryId/settings/node/libraries/ListItem.tsx +++ b/interface/app/$libraryId/settings/node/libraries/ListItem.tsx @@ -1,13 +1,12 @@ import { CaretRight, Pencil, Trash } from '@phosphor-icons/react'; import { AnimatePresence, motion } from 'framer-motion'; -import { Key, useState } from 'react'; -import { LibraryConfigWrapped, useBridgeQuery } from '@sd/client'; +import { useState } from 'react'; +import { LibraryConfigWrapped } from '@sd/client'; import { Button, ButtonLink, Card, dialogManager, Tooltip } from '@sd/ui'; import { Icon } from '~/components'; import { useAccessToken, useLocale } from '~/hooks'; import DeleteDialog from './DeleteDialog'; -import DeviceItem from './DeviceItem'; interface Props { library: LibraryConfigWrapped; @@ -19,7 +18,6 @@ export default (props: Props) => { const [isExpanded, setIsExpanded] = useState(false); const accessToken = useAccessToken(); - const cloudDevicesList = useBridgeQuery(['cloud.devices.list', { access_token: accessToken }]); const toggleExpansion = () => { setIsExpanded((prev) => !prev); }; @@ -86,43 +84,6 @@ export default (props: Props) => { className="relative mt-2 flex origin-top flex-col gap-1 pl-8" >
- - {cloudDevicesList.data?.map( - ( - device: { - pub_id: Key | null | undefined; - name: string; - os: string; - storage_size: bigint; - used_storage: bigint; - created_at: string; - device_model: string; - }, - index: number - ) => ( -
- -
-
- -
- -
- ) - )} )} diff --git a/interface/app/$libraryId/settings/node/libraries/index.tsx b/interface/app/$libraryId/settings/node/libraries/index.tsx index dc8ff541b..db7d6f5d8 100644 --- a/interface/app/$libraryId/settings/node/libraries/index.tsx +++ b/interface/app/$libraryId/settings/node/libraries/index.tsx @@ -1,18 +1,15 @@ -import { useBridgeQuery, useClientContext, useFeatureFlag, useLibraryContext } from '@sd/client'; +import { useBridgeQuery, useClientContext, useLibraryContext } from '@sd/client'; import { Button, dialogManager } from '@sd/ui'; import { useLocale } from '~/hooks'; import { Heading } from '../../Layout'; import CreateDialog from './CreateDialog'; -import JoinDialog from './JoinDialog'; import ListItem from './ListItem'; export const Component = () => { const librariesQuery = useBridgeQuery(['library.list']); const libraries = librariesQuery.data; - const cloudEnabled = useFeatureFlag('cloudSync'); - const { library } = useLibraryContext(); const { libraries: librariesCtx } = useClientContext(); const librariesCtxData = librariesCtx.data; @@ -35,19 +32,6 @@ export const Component = () => { > {t('add_library')} - {cloudEnabled && ( - - )}
} /> diff --git a/interface/app/onboarding/index.tsx b/interface/app/onboarding/index.tsx index f57a4e379..22ba946c3 100644 --- a/interface/app/onboarding/index.tsx +++ b/interface/app/onboarding/index.tsx @@ -4,7 +4,6 @@ import { onboardingStore } from '@sd/client'; import { useOnboardingContext } from './context'; import CreatingLibrary from './creating-library'; import { FullDisk } from './full-disk'; -import { JoinLibrary } from './join-library'; import Locations from './locations'; import NewLibrary from './new-library'; import PreRelease from './prerelease'; @@ -38,7 +37,6 @@ export default [ // path: 'login' // }, { Component: NewLibrary, path: 'new-library' }, - { Component: JoinLibrary, path: 'join-library' }, { Component: FullDisk, path: 'full-disk' }, { Component: Locations, path: 'locations' }, { Component: Privacy, path: 'privacy' }, diff --git a/interface/app/onboarding/join-library.tsx b/interface/app/onboarding/join-library.tsx deleted file mode 100644 index d4bbed977..000000000 --- a/interface/app/onboarding/join-library.tsx +++ /dev/null @@ -1,84 +0,0 @@ -import { useQueryClient } from '@tanstack/react-query'; -import { useNavigate } from 'react-router'; -import { - resetOnboardingStore, - useBridgeMutation, - useBridgeQuery, - useLibraryMutation -} from '@sd/client'; -import { Button } from '@sd/ui'; -import { Icon } from '~/components'; -import { AuthRequiredOverlay } from '~/components/AuthRequiredOverlay'; -import { useLocale, useRouteTitle } from '~/hooks'; -import { usePlatform } from '~/util/Platform'; - -import { OnboardingContainer, OnboardingDescription, OnboardingTitle } from './components'; - -export function JoinLibrary() { - const { t } = useLocale(); - - useRouteTitle('Join Library'); - - return ( - - - {t('join_library')} - {t('join_library_description')} - -
- Cloud Libraries -
    - - -
-
- - ); -} - -function CloudLibraries() { - const { t } = useLocale(); - - const cloudLibraries = useBridgeQuery(['cloud.library.list']); - const joinLibrary = useBridgeMutation(['cloud.library.join']); - - const navigate = useNavigate(); - const queryClient = useQueryClient(); - const platform = usePlatform(); - - if (cloudLibraries.isLoading) return {t('loading')}...; - - return ( - <> - {cloudLibraries.data?.map((cloudLibrary) => ( -
  • - {cloudLibrary.name} - -
    • - ))} - - ); -} diff --git a/interface/app/onboarding/new-library.tsx b/interface/app/onboarding/new-library.tsx index 6242a2d1e..36d6ea453 100644 --- a/interface/app/onboarding/new-library.tsx +++ b/interface/app/onboarding/new-library.tsx @@ -1,6 +1,5 @@ import { useState } from 'react'; import { useNavigate } from 'react-router'; -import { useFeatureFlag } from '@sd/client'; import { Button, Form, InputField } from '@sd/ui'; import { Icon } from '~/components'; import { useLocale, useOperatingSystem } from '~/hooks'; @@ -21,8 +20,6 @@ export default function OnboardingNewLibrary() { // TODO }; - const cloudFeatureFlag = useFeatureFlag('cloudSync'); - return (
    */} - {cloudFeatureFlag && ( - <> - {t('or')} - - - )} )} diff --git a/interface/components/Authentication.tsx b/interface/components/Authentication.tsx index 4d530bb32..6c7667b80 100644 --- a/interface/components/Authentication.tsx +++ b/interface/components/Authentication.tsx @@ -1,4 +1,4 @@ -import { AlphaRSPCError } from '@oscartbeaumont-sd/rspc-client/v2'; +import { AlphaRSPCError } from '@oscartbeaumont-sd/rspc-client/src/v2'; import { GoogleLogo, Icon } from '@phosphor-icons/react'; import { Apple, Github } from '@sd/assets/svgs/brands'; import { UseMutationResult } from '@tanstack/react-query'; @@ -7,7 +7,7 @@ import clsx from 'clsx'; import { motion } from 'framer-motion'; import { Dispatch, SetStateAction, useState } from 'react'; import { getAuthorisationURLWithQueryParamsAndSetState } from 'supertokens-web-js/recipe/thirdparty'; -import { Card, Divider, toast } from '@sd/ui'; +import { Card, toast } from '@sd/ui'; import { Icon as Logo } from '~/components'; import { useIsDark } from '~/hooks'; diff --git a/interface/components/Login.tsx b/interface/components/Login.tsx index 463029c57..cd3c0b3f9 100644 --- a/interface/components/Login.tsx +++ b/interface/components/Login.tsx @@ -1,4 +1,4 @@ -import { AlphaRSPCError } from '@oscartbeaumont-sd/rspc-client/v2'; +import { AlphaRSPCError } from '@oscartbeaumont-sd/rspc-client/src/v2'; import { ArrowLeft } from '@phosphor-icons/react'; import { UseMutationResult } from '@tanstack/react-query'; import clsx from 'clsx'; diff --git a/interface/package.json b/interface/package.json index dda41dbc4..40bb3d7dc 100644 --- a/interface/package.json +++ b/interface/package.json @@ -13,6 +13,7 @@ "@dnd-kit/utilities": "^3.2.2", "@headlessui/react": "^1.7.17", "@icons-pack/react-simple-icons": "^9.1.0", + "@oscartbeaumont-sd/rspc-client": "github:spacedriveapp/rspc#path:packages/client", "@phosphor-icons/react": "^2.0.13", "@radix-ui/react-dialog": "^1.0.5", "@radix-ui/react-dropdown-menu": "^2.0.6", @@ -82,7 +83,7 @@ "tailwindcss": "^3.4.10", "type-fest": "^4.13.0", "typescript": "^5.6.2", - "vite": "^5.2.0", + "vite": "^5.4.9", "vite-plugin-svgr": "^3.3.0" } } diff --git a/package.json b/package.json index 80924fac7..ea61bc8d9 100644 --- a/package.json +++ b/package.json @@ -62,7 +62,7 @@ "turbo": "^1.12.5", "turbo-ignore": "^1.12.5", "typescript": "^5.6.2", - "vite": "^5.2.0" + "vite": "^5.4.9" }, "engines": { "pnpm": ">=9.0.0", @@ -73,5 +73,5 @@ "eslintConfig": { "root": true }, - "packageManager": "pnpm@9.9.0" + "packageManager": "pnpm@9.12.2" } diff --git a/packages/config/package.json b/packages/config/package.json index 90ee1a5fc..fab5fce96 100644 --- a/packages/config/package.json +++ b/packages/config/package.json @@ -11,6 +11,7 @@ "lint": "eslint . --cache" }, "devDependencies": { + "@babel/preset-typescript": "^7.24.0", "@typescript-eslint/eslint-plugin": "^8.8.0", "@typescript-eslint/parser": "^8.8.0", "@vitejs/plugin-react-swc": "^3.6.0", @@ -26,8 +27,8 @@ "eslint-utils": "^3.0.0", "regexpp": "^3.2.0", "vite-plugin-html": "^3.2.2", - "vite-plugin-i18next-loader": "^2.0.12", - "vite-plugin-inspect": "^0.8.3", + "vite-plugin-i18next-loader": "^2.0.14", + "vite-plugin-inspect": "^0.8.7", "vite-plugin-solid": "^2.10.2", "vite-plugin-svgr": "^3.3.0" }, diff --git a/patches/@react-navigation__drawer@6.6.15.patch b/patches/@react-navigation__drawer@6.6.15.patch index fa4c115a9..067d678a9 100644 --- a/patches/@react-navigation__drawer@6.6.15.patch +++ b/patches/@react-navigation__drawer@6.6.15.patch @@ -1,59 +1,21 @@ diff --git a/src/views/modern/Drawer.tsx b/src/views/modern/Drawer.tsx -index 9909e9698e51379de6469eb2053a1432636d0c7d..220fa07f6784c5da13e6949e9c4893e015a5d1f8 100644 +index 9909e96..a7dd9b7 100644 --- a/src/views/modern/Drawer.tsx +++ b/src/views/modern/Drawer.tsx -@@ -1,26 +1,27 @@ +@@ -1,5 +1,6 @@ import * as React from 'react'; import { -- I18nManager, -- InteractionManager, -- Keyboard, -- Platform, -- StatusBar, -- StyleSheet, -- View, -+ Dimensions, -+ I18nManager, -+ InteractionManager, -+ Keyboard, -+ Platform, -+ StatusBar, -+ StyleSheet, -+ View, - } from 'react-native'; - import { -- PanGestureHandler, -- PanGestureHandlerGestureEvent, -- State as GestureState, -+ PanGestureHandler, -+ PanGestureHandlerGestureEvent, -+ State as GestureState, - } from 'react-native-gesture-handler'; - import Animated, { -- interpolate, -- runOnJS, -- useAnimatedGestureHandler, -- useAnimatedStyle, -- useDerivedValue, -- useSharedValue, -- withSpring, -+ interpolate, -+ runOnJS, -+ useAnimatedGestureHandler, -+ useAnimatedStyle, -+ useDerivedValue, -+ useSharedValue, -+ withSpring, - } from 'react-native-reanimated'; - - import type { DrawerProps } from '../../types'; ++ Dimensions, + I18nManager, + InteractionManager, + Keyboard, @@ -72,7 +73,8 @@ export default function Drawer({ const percentage = Number(width.replace(/%$/, '')); if (Number.isFinite(percentage)) { - return dimensions.width * (percentage / 100); -+ const dimensionsWidth = Dimensions.get("screen").width -+ return dimensionsWidth * (percentage / 100); ++ const dimensionsWidth = Dimensions.get("screen").width ++ return dimensionsWidth * (percentage / 100); } } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 634052ff92d012d1ee218a57aa513966d6bae2b7..9b68549c9bfefcdd68ee98abc3d09c383ee70309 100644 GIT binary patch delta 7047 zcmbt3YgAk3k>?_W;JaMuO6K8!F|v&vKsJ)l3mn@Z9s&VE;voU5V-tbnDis={EIl94B#`Y@0Nv4&6S|$Hw08O1$io zX7`-xhtBZ6su~I~`&7h!c*Qvg=FLHjPHE)+no0vcW;8%Bhw) z6!uEH(^)kjlgTyJYNtx$bSTt`jzBJP>+G4awsot-3t^o|RFdfVMwnGMwVJ*5jVqZ6 z*~=LqNTYdRNTah8Md`VTJ2{0g^(`a@Z!cdoyYSeVj0BaxmQVu0741k|9pWW+Kb4)x zKe-_h{`MNKtfHzyGiy58EQ6DEX{EgqmAZ1I+N-qbt7Lw;#1b!RZ<@j*jl! z311N+)oj~;klf7qR)u12pd6klJYp*eAXomg&N0w-5q2m&N(vxSOl)~z z?7IXr_T4nj38*VM|c2)b=*D zbV=j(m~^nN+tC&_d$meqLwLjk9ls$;m;=MtR_g+WL5&`SWyCGaul}4KyPv3rv#+I>ndL)crn;7pUK+A{I$W(o zdULg0uUGeYZE=aC&EBpssKbMmu~Ac%t1}i>G)FsnJ(jRT+p36o>SZCftuZz>ZW#6S z*1PPD-R>5jQsdKtaFDy1`TIWZhigz^#2pR>{LVlWet8d1(miI522EbW(9poR%wUR} z-1?rdyfvh>IeS_>4OVrRVZ_|&a{E0F>rkXK))4aPRMJSb%RbcPiuW2^5s7*v6o>`g z)=IC=9aHEGfi5_DYgz%!lyXSsy`$Wp6Chg0-C`bV_QvGBN?E5-GH9%dw^a4GDhDgW z)*g#zIPMydj}HyY^*cFY2 zBHN`>XT;|YL@R>fA?YIRwn!`#3WlTN!C+V%jJlloKNQ6YG*ST_)7<-QjY8TEZRtC^N_}%Rt~tk0;XuEN7mEc5tT>=I=!Ag3tjp{&A!@IRR*xA|CuJU!ZyL>@KoyR;}O-V_nrP<|=1jIM6Lu z_es@3YnxXy>~y;ARkp6?sHbDJqrSndv3F`({ez8$xVFAU=ds1wo0>r`CpN(JTvi6W z^L*B3NbA7CDedhI#uNsNU9QlojMdW0c87Y*KJE#dN23~RTwm>~3Uvp}25(r^6>+uM z2V|xev(YkKX_53=ybh;X8Sq4gwNZa-m8QGd8Z%bfgH3Su-_wg=0n#=yFP_eNFO5*a z-GAhknd8HwBa*gA)U0VAH96adZ651*TVtIiWY(G04Y8ru9U zBSf`AunPovFyGJTLAw~;#T0Eq7kDr!N4LOTdZd8czlXNM%r^AM)K=60^EM=c+Il2~ zht`uOm@Y+`FyTdoiOD~vL%bZV0ht`dZ`^t1%XbyY@vfP@s0C&i%omF)fj8x7bVc60 z7OnjPc?{lpi2&2z5t&lNR86kKiey5864A>n#c-Dj@!^~Ton?=OmX`%ESBcV@IVC!s zw#o`cKOlKv-7n0C&v&2*oKd4SQ_rDRsNF@@L%#;){Ucf($g0t2{|Fm{$dmkBpj(g* zYAs|toZo@6{}FMLRUK^Ffu8>&5(aBlPJ9uD)j7F&GwXvj%#U}Wzoo-I-I9s>r-eCi zRfo<4e&R7f9r`S7+1bBaerjdrTTnJ{*|s<~K0p-1??gWo!OZK}2Gcd96#CZ_JVxw9 zucZO~a~v>1H>!krEs3K|M$oX=h0Vbt7>t;}p85&8jb$l<1rc=%toNc6=1 zi*g94{a2)ielfKUuHTDYFq2D);Y21ShtM6U5X1p=|1xYV)czHfgE)kCupSS<#5>5! z;N^FS6b*e0Z`rk*EJ|&z9mXos1Q~!U!$=FJd6W-x8Mw)rC?)d18bQ9)wjb2v^CiW} zqUgS5Sb0hu)^#Ja#Bf9qn zR$>hV;;B##Kq!k^3sfAvnWCMzjzbkMOO_=jchTS*$0os6#ld(T^)Iud;gLI5A@r%E znf9R`)-Eb$*FMySO_KE8o#-Ic?#24eeUG0HXaZj^Ow$jh1K5eCyHUo}1iEvXf`ULL z*3^A>O4Ajx{V9^^I}s233PK394_P5}4?49hhX*I_!ACG%Li-^}cjP{7suf{)dH*Ud zd#z+HT-%T1YlUP97G6a+eLosqB?B)DVc`LEn0faBWKK_EChABLocTKT?~3%|orDMk zhgc^s8iEH)FnSf;^&|L5R}QCc&EkPE47V=JX2Q$yl)(FGWU4lhYO83{3 z8$ogmg;R2umU*nwalo-AguLT7Fj#^uGLT)viFW=kNC;=@hz!u}CUZc%2Z#Gjh`|^} zgq}rrvxp)%H+6$4lQH%3)RJ#8i4ZPq!aej?VM(|~>BKT1A=oi8wDLcRSD7v`l+D7sg zSa=lWIy+wmQpynyaqg`GdeUk}q?UBZ6K@H3dvc~p`N zH!)PWv4rz)p{EzuzAaQH^Vr+y6HZEl|GA8C4|Qqjfb$suDNJ#ovUnp2Y z5{yTQY$z=yA6&I;_T!AQ#48I9uOa7Id+!001()=qLUIkfFTlmvlb2F+u|r7KFn&Hc z1pUR>oC{ji!I<*MukzUHj9}~~J_VEAt6)w$i{n< zup9W{0{a~H5ZgjAc&w1kg#m#eKh+IZy@;Aq4ZzFC$!rLn#!kIZghN2Q0oR^+A6*Rd zURn&MQhX)6fQ77dk}G?Q7+gAPdhj;fHD<#5*+e#z_YnCD^(yK!)%Ydt(?_v=GA@w0 zC|4sce)FtfiWBk3z@LDOHzpD=>YNck#Z3)~Ne0g*uc_GS(@uEd5VeE-Q{sT8^nk}rB-9+GN@2#rNr$#x z&f3&cJc*V5%@In+4xjD^JaJ0W4YO0HUF^WA7Ct^k)v#lxBw`~JJ2)A*EfHoiR}xf- z=%1Q8_3N`q_~wTcg86$fS9XHL;0eY|6lf>E9O#Rb4CW7Wa^TdTs0|=kpteHjRs1yf9F@le z-=LZ|t`e2;5!IDk;%R3V#3_*i_PvBtsPREgCS3Z6+OmXP!;0L(_&=sZiy}YG5|uFT z{FXZBUlnvmto#~~_V9ZWlV`RtclPn`S-H~Xa3&4ERz9A8s4u+gBK#cJecBCSfp1VbSt6;=)sQ zoNKP%%jZJkUcT}s1cMQON+&ch{7rtbaAg>^T`aGVSIC%>gM7Lyl>>3wrxGuzvYGzp z_|KJREzRT9ASZ-H(ElDkpSgIMe^9|nh%pCf!7=h?VV&y)O&NH!_IONK%#Vkh+t?Y# zb}=4esF{D;C^*aIuV_$<6@Y!Ef|Kd0?czH?dYroncHc(lFnh`b^{Z|?gY(!k!c}%V z&??v|T9(X%%nP`v&iS#`*LMp93}=tvm&z4#xNl6bmMKUGvUJOP7H#&Gw*@^n!+!g= z;PsnfzrG~sy$QyQyd(JgM%Hdy;Nc8<1K`>xgf}y2Tv<-?vzfnM6F4|uicTlk_u4$> zI)|nSwvbf8gt*no1NZSnx5<*{x#7IqqRz12 z9kBVfi#6;WxXE^}^F(n%lQ5lZ1sP9N08eC!a`0FOLX@Z)Ih+HrAvT~DU=s+|rb>E2 zmZ%Vqn$Et4``qpIg9fQF%kK-+ZglWfOI-f`@>(z2PqO8m2&eQfWTGGjttvzY=+%e{Q zHnm~MN2DWic%0KGZ|$oHTeIEWt^a5ZFk}cUX+3-IAe}o52R5|6^vn=EX5`>=cR%<5qeXjk zpe0z%Xg&UUXRGxELF?1y0R}Om8j-@e_xNUXrWJEL9(rBW>U#T1U)rDI$=m`bo@VrNv&ncVREY{Ic|o-t6~M3iu(j|Oj%@^U ztZR5q^=to+;gLIV?kDbhoJQXFc-Qs^p=9jZ;gbgg4tq7EDe8@)NohP#cBLCmjc069 zI*}AxVGA82d#j=85o+H6i03`NXRuf`e4gsSdZ zC1gq^GCFnF6qv8iUE5U%GE)_PbMj;e#MMr6`e@I@3TTUf(jztDrjzq;u> z7tey$#ILqWY;CVQR{TN=$xL?j!QKam3Tb9dpCTR-Zcos`^z*kf$OHf+=-;B z9!y3;UX#}|;gWk~HJ4i=@ul5XyDy&~Ck!&1(B!T={gZB2Qmbgj^ksw9;t9A6#@bX< zo;4V%`i3ziYHAAVRMlNbIGrXKgWJtsX5}(wga1L+>lv65a{_SUIRlR8{s+S}BtIJdRSst`GVHT5&F08uOV7L@;cT z=#?3FwQBc;^im-_>qGj|Ui!~EO0-h3vowj?StjmlmRzZcdeUAm8x7&mL_M1}C#4Q; zDenx(L%yQSNND6WQO)PmR1D<_aV|ENi&yflLe!QyTh7M7Ai_}wlVEj>Xl+mtT~ZL2*ZwqC7{ly^J--(8SubviQc`(F12_2TqvZS~{c{T*YkGAeRoxnkbG|Z`kV#`EXEdNys}kB! zOr_QZoQ0aOQ6+RLr@?J^6H`fzGCW=_nhf@^c*-QqN=k`D*&MEm6Lpk+#UIkQ!W%yN z3K$8}f80qrnW7q1HM5nhNbc}@6$NcAl}Uz)X_cfnnQqkNvT=_ohzxaltHGSlD6G_K zZU3F7u^KVyR7`suxRUchnvBnj;$6A7Bl^@$2XU_+xM(6MPQ2$Xc{S zEbdIPF%_+vqyb&jFIARHT2)pX%eb4KlD8b6@`}SEuR5j3NaB+5e-)r`Jowg~vRA*02TU>RcyAUFbxkN%6 zx7l^ISl%&NlNhxAY*b|vX>5jAV!~k^ui8CAXVn}LD}-b6W+O-#BL!Qg=%{(DZgDY2 zd9PEvd+BU=E{7*64`h-=My(vvJF5x5Ph3_CLvF7q zRu!kc)kfScvzDDi%vEj1fuHUjhSLxAFvx#Pbk}Yt&%f1u{T(zFoO`HyyQdKvizVxPwYd_`Dhd|DlXFe0Dmi~qly$oar`X_f*G+a!u`bJK#$*|p+8foyRPynxIHz_w z3|V=xTGBdF^`zKc(KtO4zrK+O!rR8K4M1#WN;%RiF-#!HhDzyZWK;|{A7}3H2E-zt zq#26Y!}iFeQ|l-E1&!H1tqY|c_CikKB?6X)$5fVv>ve^%VV@5A#A6=2Juqph8KrT* zR241?la_iWWi)zn)=EYfixXlv_bPJ}yt{(F;uBl9Fv+c3dLKUo#TaY5M;eI6&CzTs z@5}|v+45M%Zj4ziK8HgiZ&=b!mDOmmxa*?WSjiCbrS+kvMK0DQGme7IW+{rJjgner zkc^2v(U{}5;GHoz7) zbF0@Du8AjQ?y4d+Wv-=VwT8|ZvDOoHMLy#<_|nsPi!vaUxC%;PEoDPDjR{pXtSFC< zxe3DOo=Qa|qU_kT*;{j}OH&i|;^del6c?eoU|k6>KgzlSzI|)=R*!qkH*KGYI7I1s zRFO!fD@u!dQl=w>E`6~Wm8&$>Y&fTlxomZrxgyO@xzz>zSUQ{#=BkCN#4*;WNh;2y zLalcio#LQIT#Jv1V56V00cMZWHBwv79t1(2jOr>9s~u@r>D}&^Z0hDYa=l=~lYJlDbw=wWh*Z<#=9X zGurG5slJ{x`D4|z+7T}%9ip6kLTA!WC#~sNU7eG8rwYnUrDCg33spg%8D5<0S_f;t z#ySS4zs7nS-Z;rBkT=b-lpdOrB$?PKpJcDayuEO!7hen9t1v0-QDAHsp2U0L*u&@| zxpxb86i%GP2Ep+Tb`9`0VN76u725_kU4e06W)Q>SnH^|V*pIDj&A-ir@i4}N869Q< zVkfrh?+QK()9+yY|4?Q(97SrL+=~5*4l|#@hG35tW42Dca|Qg*4oVjb-%qK*VB0Qa z$UpDEp1pjFOy7lFffzQn1s&Onl}N`<>^&MhD8bS&x*6Fpe=lbZ9NL8pCJ5XNgalgy z2Ueo+kPj@W{Uw-$&H_U+F0cno8a#wlzCLKm@ha6E$TfKdl_Jte0f z_9swS&dYEnJl4zJu|#sm-@sl0`*RozPUMh)2L{;d;kz+xH57h?-3X^%#N-`=DFlfS zchQz4{C;VLZV)--bU_)}aUzAzc$ppOvVIrFg&S6|dtg|Jujv?r=x$pQ@F9|cC5wcx zJ4^YDz_vsFbtL6j7Nv$`KPnBBF4~TOStZT|o&(o{;LDsrIP*W~Ae4LHnk@FqWh6v$ z8ObFbb4w%|%CswhXW{k1`5bb>+2*1@k&bJYtdfDC7hNJoZ=rR=`2v=w1gxcM^Zipsj)5S#b1=coOq>f_(|m z8$$?z8zDZ-R!1&s`@Nln9gaG7`{i`K@J$Jg2Z1Rp2-|YyKcow%Q)s2V5^oQyqv?ln})&sZO9#$bS%Pn z9ga(gH*QC{c>ZHp6VPfJMyIizRN1VD`|rROEGPERc7W{@m$_FkO4|nbor^jax68#N z7>g>MTRLVeZnu8eX5N3JmTCCpWsY8iE%)&Zi_M2adE||YpIr3DWw4KU17X*I@6a;h zHSq4zmQbOTeehFDr(D=xs+t9#jyh4G{&aYeZgKk&y7_ysm0-RH6-?W^i`#)q9w!lO z6RqRpStsgpyn9j5QFAcEya4Ng!js7F&+NoUmj!N-?mJ7_CWKQa+5kCtA9g<~^Od*& z&f94i`RZq}Z}L0)!*(8<{P#KRpLi5gKg@iALr|@uAI^RseS434c-=r8KuP!fchIR8 zNrX#XaDi?nPGgZ@`YtADC#wK%O43%4e|`@8;=;b*;PnFW0@h8QegXS0dPn!}hdG1f z`IoSL-IU<9)YoM-eEeq^0Y`t@`DmG7_7&9OW`2P(ZHa_LBwuB9M4akFO?>bh*l%EQ zwVL`Co`&;BP!V4o#W7U-G$xt-4R#zQI;F+73ER~!r7B39y1>xG;x9Y zN8nEC&N2cg2;5E1ZmyIjQB02U;t;ezj&8ONJL`cc4RrO#E0N{9>4ufL+NP1Iq>}FXl(dR z0yWsByV#M3uubq7<$}ea^}`}-+jZ@4=OX{VApSqyhPN+SO~hJmUrmnqsL$O!7@s_R z1wPW%b~6y$kuz@HhK!)x)!ve5d*ecawb93UYzDb=7ycD5IBrDe7u<@BL}j*6SLpc0 zP8+@U7;ill$1ULaYu_sPp@Y}9Z3~=t^ENN#Hcq0sCYnLduEK_4{UO{9!n^U8Vg8f& zb!7e!zFG;hSMxTI-}nwbfTKd=2*@{14DR) z0QL(0)FlLo5brR6=;rMt-LK(Obh!H`_z1lD6PyLJO>Q5_`5mt8rX&x;fw%ALH?xQZ?|T=za}6$oCz8RSjB$E{q-+9C4wi}-7t#ZkBgMzj1rIB_Sx z2Szx3GZdUeGxz4oVH$}LPaTE7^!D$D<2v3| zM|Q9|WZ_KT7lz;!HLf}GMGg!0{iRO_)-Q7YncR=FpJ0%`r?ZJoAWHEzf)U5okio6& z>gK;QvTvPHZ6JG0>~r!iu@pkLvW4&!2Ym=cGwe;&3WwntHe)%i4bdz?&~S8+qTWu; zD|?oM+Y?NFN0P`#N7+BaW-k#eLS~mk`XG2S+prw6UX3jR){{f4IngDw%@a~|mg#D+{#r*tB zY%2&p&t647eUfe5w3s6{nEW~0xwsQkk00-q9I@WC3{nj%G$l8V}kS7cKr;ISv%)`T01!bEjLq79!%2VRME3|$Z%9Q51C#o-+Ba8|n&Iepuy4mU);@Hx(@9VGJwj#Ua6=YM$ z4e{HRPQHH)_Zrk6pW|$YJN4X^{@0eSE??y#YArU!q>Wx!v5z_l-JpY5#P zVWXde-~JI-*RgQ&dG3Wx)T!DHZUCAWxErYkhXZ#S{RaQa1@4y4_Tg>lA?3{rTs=AR zHkVERf3W~!d4y`ytH@XW$mP+fhN^-`e~Ye*`!#qEseg~lp|xELXEW$li;S4r!X-wC zZHsd($p`_@mXH(se#m3f6(bT*Y~UGJNztf_`s{l^9n^zeCLSJU~ZQL+yM?^-^J7@L0^%&6(i*72)5Aad;r{#q_Lg#1UybDQ~J z*O8AU`1^ZlVtD-fd;t(eem~ssarD#ijxY2L!G^!^nII^kf2a@gm{(pUkd2@osVVW- zb~@u@MLv`27i|dYgMF9NAP5s{C}hhRH}m4TYBV&JOvIK!H$l!z?^({giKCG1%2vyn zWmLmdr^A<14a1#fe!F8K3x=uDeeZJgT0mXxLv}CrIo3xAd$`EdUDImd;CHEhjI$#O8h;36V1TT z>uHmwgjX)|)o{|p*#Xa7;143tzxV_85L~#x-ve(Q;$Woaef~juyVJe!Xa0NCf_mqL zPrc6{2O_{(MNVGi|BCiE-mg2B2`Zf)G<@H4{WrA5pk^2V$CuC*#4*L|zK_@6-!s~N z6mQFYqpSZ~xaOC<-o8>klc^RKVtg*xCxlrQrx%WEIDPQYk5M{*lHM;~V5Q#E3HS%R R=yxZ*|620NuKqg({|7A$(Qg0% From 97950140997feb220bd37273db09c207587ac932 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADtor=20Vasconcellos?= Date: Sat, 19 Oct 2024 08:20:18 -0300 Subject: [PATCH 05/21] autoformat --- interface/app/$libraryId/index.tsx | 6 ++---- interface/app/$libraryId/settings/client/general.tsx | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/interface/app/$libraryId/index.tsx b/interface/app/$libraryId/index.tsx index 10a95e97a..6ff9f4b3e 100644 --- a/interface/app/$libraryId/index.tsx +++ b/interface/app/$libraryId/index.tsx @@ -8,9 +8,7 @@ import settingsRoutes from './settings'; // Routes that should be contained within the standard Page layout const pageRoutes: RouteObject = { lazy: () => import('./PageLayout'), - children: [ - { path: 'overview', lazy: () => import('./overview') } - ] + children: [{ path: 'overview', lazy: () => import('./overview') }] }; // Routes that render the explorer and don't need padding and stuff @@ -35,7 +33,7 @@ function loadTopBarRoutes() { return [ ...explorerRoutes, pageRoutes, - { path: 'settings', lazy: () => import('./settings/Layout'), children: settingsRoutes }, + { path: 'settings', lazy: () => import('./settings/Layout'), children: settingsRoutes } ]; } else return [...explorerRoutes, pageRoutes]; } diff --git a/interface/app/$libraryId/settings/client/general.tsx b/interface/app/$libraryId/settings/client/general.tsx index bba74c7f6..2196b0e27 100644 --- a/interface/app/$libraryId/settings/client/general.tsx +++ b/interface/app/$libraryId/settings/client/general.tsx @@ -62,7 +62,7 @@ export const Component = () => { .strict(), reValidateMode: 'onChange', defaultValues: { - name: node.data?.name, + name: node.data?.name // image_labeler_version: node.data?.image_labeler_version ?? undefined // background_processing_percentage: // node.data?.preferences.thumbnailer.background_processing_percentage || 50 From b29d165fed3a3151bea7615b50db2eb945ceb875 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADtor=20Vasconcellos?= Date: Sat, 19 Oct 2024 08:35:53 -0300 Subject: [PATCH 06/21] autoformat --- interface/app/$libraryId/Explorer/FilePath/Original.tsx | 2 +- .../$libraryId/settings/library/locations/AddLocationButton.tsx | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/interface/app/$libraryId/Explorer/FilePath/Original.tsx b/interface/app/$libraryId/Explorer/FilePath/Original.tsx index 451b13d35..b1b14ef75 100644 --- a/interface/app/$libraryId/Explorer/FilePath/Original.tsx +++ b/interface/app/$libraryId/Explorer/FilePath/Original.tsx @@ -83,7 +83,7 @@ const TEXT_RENDERER: OriginalRenderer = (props) => ( onLoad={props.onLoad} onError={props.onError} className={clsx( - 'textviewer-scroll size-full overflow-y-auto whitespace-pre-wrap break-words px-4 font-mono', + 'textviewer-scroll font-mono size-full overflow-y-auto whitespace-pre-wrap break-words px-4', !props.mediaControls ? 'overflow-hidden' : 'overflow-auto', props.className, props.frame && [props.frameClassName, '!bg-none p-2'] diff --git a/interface/app/$libraryId/settings/library/locations/AddLocationButton.tsx b/interface/app/$libraryId/settings/library/locations/AddLocationButton.tsx index 23b70e901..e9cc0fd75 100644 --- a/interface/app/$libraryId/settings/library/locations/AddLocationButton.tsx +++ b/interface/app/$libraryId/settings/library/locations/AddLocationButton.tsx @@ -71,7 +71,7 @@ export const AddLocationButton = ({ {...props} > {path ? ( -
    +
    Date: Sat, 19 Oct 2024 12:45:57 -0300 Subject: [PATCH 07/21] Fix onbording test --- interface/app/onboarding/prerelease.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/interface/app/onboarding/prerelease.tsx b/interface/app/onboarding/prerelease.tsx index 156d6e5a5..5432ebec5 100644 --- a/interface/app/onboarding/prerelease.tsx +++ b/interface/app/onboarding/prerelease.tsx @@ -17,7 +17,7 @@ export default function OnboardingPreRelease() {
    From 2c91484dd460dc17bba8c791e03dfb2666ac934f Mon Sep 17 00:00:00 2001 From: Ericson Soares Date: Sun, 20 Oct 2024 22:26:06 -0300 Subject: [PATCH 08/21] Some tweaks on sync messages send and compression --- core/crates/cloud-services/src/sync/send.rs | 39 ++++++++++++++++---- crates/sync/src/compressed.rs | 41 +++++++++++++++------ 2 files changed, 61 insertions(+), 19 deletions(-) diff --git a/core/crates/cloud-services/src/sync/send.rs b/core/crates/cloud-services/src/sync/send.rs index 4fd3842da..c0ab06e88 100644 --- a/core/crates/cloud-services/src/sync/send.rs +++ b/core/crates/cloud-services/src/sync/send.rs @@ -34,14 +34,14 @@ use tokio::{ sync::{broadcast, Notify}, time::sleep, }; -use tracing::error; +use tracing::{debug, error}; use uuid::Uuid; use super::{SyncActors, ONE_MINUTE}; const TEN_SECONDS: Duration = Duration::from_secs(10); -const MESSAGES_COLLECTION_SIZE: u32 = 100_000; +const MESSAGES_COLLECTION_SIZE: u32 = 10_000; enum RaceNotifiedOrStopped { Notified, @@ -147,6 +147,8 @@ impl Sender { } async fn run_loop_iteration(&mut self) -> Result { + debug!("Starting cloud sender actor loop iteration"); + let current_device_pub_id = devices::PubId(Uuid::from(&self.sync.device_pub_id)); let (key_hash, secret_key) = self @@ -166,6 +168,11 @@ impl Sender { let mut status = LoopStatus::Idle; let mut new_latest_timestamp = current_latest_timestamp; + + debug!( + chunk_size = MESSAGES_COLLECTION_SIZE, + "Trying to fetch chunk of sync messages from the database" + ); while let Some(ops_res) = crdt_ops_stream.next().await { let ops = ops_res?; @@ -173,9 +180,13 @@ impl Sender { break; }; + debug!("Got first and last sync messages"); + #[allow(clippy::cast_possible_truncation)] let operations_count = ops.len() as u32; + debug!(operations_count, "Got chunk of sync messages"); + new_latest_timestamp = last.timestamp; let start_time = timestamp_to_datetime(first.timestamp); @@ -188,6 +199,16 @@ impl Sender { let messages_bytes = rmp_serde::to_vec_named(&compressed_ops) .map_err(Error::SerializationFailureToPushSyncMessages)?; + let encrypted_messages = + encrypt_messages(&secret_key, &mut self.rng, messages_bytes).await?; + + let encrypted_messages_size = encrypted_messages.len(); + + debug!( + operations_count, + encrypted_messages_size, "Sending sync messages to cloud", + ); + self.cloud_client .sync() .messages() @@ -202,20 +223,22 @@ impl Sender { key_hash: key_hash.clone(), operations_count, time_range: (start_time, end_time), - encrypted_messages: encrypt_messages( - &secret_key, - &mut self.rng, - messages_bytes, - ) - .await?, + encrypted_messages, }) .await??; + debug!( + operations_count, + encrypted_messages_size, "Sent sync messages to cloud", + ); + status = LoopStatus::SentMessages; } self.maybe_latest_timestamp = Some(new_latest_timestamp); + debug!("Finished cloud sender actor loop iteration"); + Ok(status) } diff --git a/crates/sync/src/compressed.rs b/crates/sync/src/compressed.rs index aa084c4b7..47c38e7fe 100644 --- a/crates/sync/src/compressed.rs +++ b/crates/sync/src/compressed.rs @@ -1,6 +1,6 @@ use crate::{CRDTOperation, CRDTOperationData, DevicePubId, ModelId, RecordId}; -use std::collections::BTreeMap; +use std::collections::{hash_map::Entry, BTreeMap, HashMap}; use serde::{Deserialize, Serialize}; use uhlc::NTP64; @@ -17,11 +17,16 @@ pub struct CompressedCRDTOperationsPerModelPerDevice( ); impl CompressedCRDTOperationsPerModelPerDevice { + /// Creates a new [`CompressedCRDTOperationsPerModelPerDevice`] from a vector of [`CRDTOperation`]s. + /// + /// # Panics + /// + /// Will panic if for some reason `rmp_serde::to_vec` fails to serialize a `rmpv::Value` to bytes. #[must_use] pub fn new(ops: Vec) -> Self { let mut compressed_map = BTreeMap::< DevicePubId, - BTreeMap)>>, + BTreeMap, (RecordId, Vec)>>, >::new(); for CRDTOperation { @@ -38,14 +43,21 @@ impl CompressedCRDTOperationsPerModelPerDevice { .entry(model_id) .or_default(); - // Can't use RecordId as a key because rmpv::Value doesn't implement Hash + Eq - if let Some((_, ops)) = records - .iter_mut() - .find(|(current_record_id, _)| *current_record_id == record_id) - { - ops.push(CompressedCRDTOperation { timestamp, data }); - } else { - records.push((record_id, vec![CompressedCRDTOperation { timestamp, data }])); + // Can't use RecordId as a key because rmpv::Value doesn't implement Hash + Eq. + // So we use it's serialized bytes as a key. + let record_id_bytes = + rmp_serde::to_vec(&record_id).expect("already serialized to Value"); + + match records.entry(record_id_bytes) { + Entry::Occupied(mut entry) => { + entry + .get_mut() + .1 + .push(CompressedCRDTOperation { timestamp, data }); + } + Entry::Vacant(entry) => { + entry.insert((record_id, vec![CompressedCRDTOperation { timestamp, data }])); + } } } @@ -55,7 +67,14 @@ impl CompressedCRDTOperationsPerModelPerDevice { .map(|(device_pub_id, model_map)| { ( device_pub_id, - CompressedCRDTOperationsPerModel(model_map.into_iter().collect()), + CompressedCRDTOperationsPerModel( + model_map + .into_iter() + .map(|(model_id, ops_per_record_map)| { + (model_id, ops_per_record_map.into_values().collect()) + }) + .collect(), + ), ) }) .collect(), From 17d82ee20d3e93d514c6c86ec48880c67a8f4ca7 Mon Sep 17 00:00:00 2001 From: Ericson Soares Date: Mon, 21 Oct 2024 00:32:00 -0300 Subject: [PATCH 09/21] Ensure single cloud bootstrap ever --- core/crates/cloud-services/src/client.rs | 10 +++----- core/src/api/cloud/mod.rs | 29 +++++++++++++++++++----- 2 files changed, 26 insertions(+), 13 deletions(-) diff --git a/core/crates/cloud-services/src/client.rs b/core/crates/cloud-services/src/client.rs index 71d4da975..d9ec361e1 100644 --- a/core/crates/cloud-services/src/client.rs +++ b/core/crates/cloud-services/src/client.rs @@ -2,11 +2,7 @@ use crate::p2p::{NotifyUser, UserResponse}; use sd_cloud_schema::{Client, Service, ServicesALPN}; -use std::{ - net::SocketAddr, - sync::{atomic::AtomicBool, Arc}, - time::Duration, -}; +use std::{net::SocketAddr, sync::Arc, time::Duration}; use futures::Stream; use iroh_net::relay::RelayUrl; @@ -15,7 +11,7 @@ use quinn::{crypto::rustls::QuicClientConfig, ClientConfig, Endpoint}; use reqwest::{IntoUrl, Url}; use reqwest_middleware::{reqwest, ClientBuilder, ClientWithMiddleware}; use reqwest_retry::{policies::ExponentialBackoff, RetryTransientMiddleware}; -use tokio::sync::RwLock; +use tokio::sync::{Mutex, RwLock}; use tracing::warn; use super::{ @@ -53,7 +49,7 @@ pub struct CloudServices { notify_user_rx: flume::Receiver, user_response_tx: flume::Sender, pub(crate) user_response_rx: flume::Receiver, - pub has_bootstrapped: Arc, + pub has_bootstrapped: Arc>, } impl CloudServices { diff --git a/core/src/api/cloud/mod.rs b/core/src/api/cloud/mod.rs index a47ced682..dd2bde419 100644 --- a/core/src/api/cloud/mod.rs +++ b/core/src/api/cloud/mod.rs @@ -15,7 +15,7 @@ use sd_cloud_schema::{ use sd_crypto::{CryptoRng, SeedableRng}; use sd_utils::error::report_error; -use std::{pin::pin, sync::atomic::Ordering}; +use std::pin::pin; use async_stream::stream; use futures::{FutureExt, StreamExt}; @@ -50,7 +50,19 @@ pub(crate) fn mount() -> AlphaRouter { |node, (access_token, refresh_token): (auth::AccessToken, auth::RefreshToken)| async move { use sd_cloud_schema::devices; - if node.cloud_services.has_bootstrapped.load(Ordering::Acquire) { + // Only allow a single bootstrap request in flight at a time + let mut has_bootstrapped_lock = node + .cloud_services + .has_bootstrapped + .try_lock() + .map_err(|_| { + rspc::Error::new( + rspc::ErrorCode::Conflict, + String::from("Bootstrap in progress"), + ) + })?; + + if *has_bootstrapped_lock { return Err(rspc::Error::new( rspc::ErrorCode::Conflict, String::from("Already bootstrapped"), @@ -210,9 +222,7 @@ pub(crate) fn mount() -> AlphaRouter { .try_join() .await?; - node.cloud_services - .has_bootstrapped - .store(true, Ordering::Release); + *has_bootstrapped_lock = true; Ok(()) }, @@ -242,7 +252,14 @@ pub(crate) fn mount() -> AlphaRouter { .procedure( "hasBootstrapped", R.query(|node, _: ()| async move { - Ok(node.cloud_services.has_bootstrapped.load(Ordering::Relaxed)) + // If we can't lock immediately, it means that there is a bootstrap in progress + // so we didn't bootstrapped yet + Ok(node + .cloud_services + .has_bootstrapped + .try_lock() + .map(|lock| *lock) + .unwrap_or(false)) }), ) } From ea92383b784f4266189e03008361edeeecd631d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADtor=20Vasconcellos?= Date: Mon, 21 Oct 2024 12:47:40 -0300 Subject: [PATCH 10/21] Improve file thumbnails and Quick Preview (+ some code clean-up and rust deps update) (#2758) * Update rspc, prisma-client-rust, axum and tanstack-query - Deleted some unused examples and fully commented out frontend code - Implement many changes required due to the updates - Update most rust dependencies * Re-enable p2p * Fix server * Auto format * Fix injected script format - Update some github actions - Update pnpm lock file * Fix devtools showing up when app opens - Fix million complaining about Sparkles component * Fix sd-server * Fix and improve thumbnails rendering - Fix core always saying a new thumbnail was generated even for files that it skiped thumbnail generation - Rewrite FileThumb and improve related components * Ignore tmp files when running prettier * Improve FileThumb component performance - Rework useExplorerDraggable and useExplorerItemData hooks due to reduce unecessary re-renders * More fixes for thumb component - A couple of minor performance improvements to frontend code * auto format * Fix Thumbnail and QuickPreview * Fix logic for when to show 'fail to load original' error message in QuickPreview - Updated prisma-client-rust, libp2p, tauri, tauri-specta, rspc and hyper * Fix type checking - Format scripts * Add script prettier config * Fix serde missing feature - Use rust-libp2p spacedrive fork again - Update rspc * Autoformat + fix pnpm lock * Fix thumbnail first load again * Autoformat * autoformat * Fix rust-libp2p fork url again? * Remove usePathsInfiniteQuery hook * Update tauri 2.0.6 --- .github/actions/setup-pnpm/action.yml | 4 +- .github/actions/setup-system/action.yml | 2 +- .prettierignore | 2 + Cargo.lock | Bin 286608 -> 275006 bytes Cargo.toml | 62 +- apps/desktop/package.json | 8 +- apps/desktop/src-tauri/Cargo.toml | 13 +- apps/desktop/src-tauri/src/tauri_plugins.rs | 94 +-- apps/desktop/src/patches.ts | 2 +- apps/landing/package.json | 2 +- apps/mobile/modules/sd-core/src/index.ts | 6 +- apps/mobile/package.json | 10 +- .../src/components/browse/BrowseLocations.tsx | 3 +- .../src/components/drawer/DrawerLocations.tsx | 3 +- .../src/components/explorer/Explorer.tsx | 4 +- .../explorer/sections/FavoriteButton.tsx | 4 +- apps/mobile/src/components/job/JobGroup.tsx | 2 +- .../src/components/modal/AddTagModal.tsx | 4 +- .../components/modal/CreateLibraryModal.tsx | 2 +- .../components/modal/ImportLibraryModal.tsx | 4 +- .../src/components/modal/ImportModal.tsx | 2 +- .../confirmModals/DeleteLibraryModal.tsx | 4 +- .../confirmModals/DeleteLocationModal.tsx | 4 +- .../modal/confirmModals/DeleteTagModal.tsx | 4 +- .../modal/inspector/ActionsModal.tsx | 4 +- .../modal/inspector/RenameModal.tsx | 2 +- .../components/modal/tag/CreateTagModal.tsx | 2 +- .../components/modal/tag/UpdateTagModal.tsx | 6 +- .../src/components/overview/Devices.tsx | 4 +- .../src/components/overview/OverviewStats.tsx | 4 +- .../search/filters/SavedSearches.tsx | 2 +- apps/mobile/src/hooks/useFiltersSearch.ts | 3 +- apps/mobile/src/hooks/useSavedSearch.ts | 5 +- apps/mobile/src/screens/BackfillWaiting.tsx | 6 +- apps/mobile/src/screens/browse/Location.tsx | 7 +- apps/mobile/src/screens/search/Search.tsx | 7 +- .../library/CloudSettings/CloudSettings.tsx | 4 +- .../library/CloudSettings/Library.tsx | 2 +- .../settings/library/EditLocationSettings.tsx | 28 +- .../screens/settings/library/SyncSettings.tsx | 10 +- apps/mobile/src/stores/auth.ts | 2 +- apps/server/Cargo.toml | 13 +- apps/server/src/main.rs | 34 +- apps/web/package.json | 4 +- apps/web/src/patches.ts | 2 +- core/Cargo.toml | 9 +- core/crates/heavy-lifting/Cargo.toml | 2 +- .../media_processor/helpers/thumbnailer.rs | 23 +- .../src/media_processor/tasks/thumbnailer.rs | 19 +- core/crates/indexer-rules/Cargo.toml | 2 +- core/src/custom_uri/async_read_body.rs | 61 -- core/src/custom_uri/mod.rs | 28 +- core/src/custom_uri/serve_file.rs | 25 +- core/src/custom_uri/utils.rs | 23 +- core/src/p2p/operations/rspc.rs | 20 +- crates/ai/Cargo.toml | 2 +- crates/ffmpeg/src/thumbnailer.rs | 20 +- crates/p2p/Cargo.toml | 8 +- crates/prisma-cli/Cargo.toml | 4 +- crates/sync/example/Cargo.toml | 27 - crates/sync/example/README.md | 18 - .../sync/example/prisma/migrations/.gitkeep | 0 crates/sync/example/prisma/schema.prisma | 34 - crates/sync/example/src/api/mod.rs | 175 ----- crates/sync/example/src/main.rs | 45 -- crates/sync/example/src/utils.rs | 28 - crates/sync/example/web/.gitignore | 2 - crates/sync/example/web/README.md | 34 - crates/sync/example/web/index.html | 15 - crates/sync/example/web/package.json | 24 - crates/sync/example/web/postcss.config.js | 6 - crates/sync/example/web/src/App.tsx | 172 ----- crates/sync/example/web/src/index.css | 3 - crates/sync/example/web/src/index.tsx | 16 - crates/sync/example/web/src/test.ts | 47 -- crates/sync/example/web/src/utils/bindings.ts | 80 --- crates/sync/example/web/src/utils/rspc.ts | 29 - crates/sync/example/web/tailwind.config.js | 8 - crates/sync/example/web/tsconfig.json | 13 - crates/sync/example/web/vite.config.ts | 12 - .../Explorer/ContextMenu/OpenWith.tsx | 10 +- .../Explorer/ContextMenu/SharedItems.tsx | 2 +- .../$libraryId/Explorer/ExplorerTagBar.tsx | 2 +- .../Explorer/FilePath/DecryptDialog.tsx | 179 ----- .../Explorer/FilePath/DeleteDialog.tsx | 2 +- .../Explorer/FilePath/EncryptDialog.tsx | 177 ----- .../Explorer/FilePath/EraseDialog.tsx | 68 -- .../Explorer/FilePath/ErrorBarrier.tsx | 40 ++ .../$libraryId/Explorer/FilePath/Image.tsx | 11 +- .../Explorer/FilePath/LayeredFileIcon.tsx | 23 +- .../$libraryId/Explorer/FilePath/Original.tsx | 133 ++-- .../$libraryId/Explorer/FilePath/Thumb.tsx | 614 +++++++++++------- .../Explorer/Inspector/FavoriteButton.tsx | 2 +- .../$libraryId/Explorer/Inspector/index.tsx | 2 +- .../Explorer/QuickPreview/index.tsx | 189 +++--- .../Explorer/View/Grid/DragSelect/index.tsx | 64 +- .../Explorer/View/GridView/Item/index.tsx | 54 +- .../Explorer/View/GridView/index.tsx | 12 +- .../Explorer/View/ListView/index.tsx | 62 +- .../Explorer/View/MediaView/Item.tsx | 2 +- .../Explorer/View/MediaView/index.tsx | 28 +- .../Explorer/View/RenamableItemText.tsx | 6 +- .../app/$libraryId/Explorer/View/ViewItem.tsx | 11 +- interface/app/$libraryId/Explorer/index.tsx | 2 +- interface/app/$libraryId/Explorer/store.ts | 15 +- .../app/$libraryId/Explorer/useExplorer.ts | 57 +- .../Explorer/useExplorerDraggable.tsx | 56 +- .../Explorer/useExplorerItemData.tsx | 99 ++- .../Explorer/useExplorerPreferences.ts | 2 +- .../app/$libraryId/Layout/CMDK/index.tsx | 5 +- .../Layout/CMDK/pages/CMDKLocations.tsx | 5 +- .../$libraryId/Layout/CMDK/pages/CMDKTags.tsx | 3 +- .../Layout/Sidebar/DebugPopover.tsx | 15 +- .../Layout/Sidebar/JobManager/JobGroup.tsx | 2 +- .../Layout/Sidebar/JobManager/index.tsx | 2 +- .../Layout/Sidebar/SidebarLayout/Footer.tsx | 2 +- .../Sidebar/sections/Locations/index.tsx | 5 +- .../Layout/Sidebar/sections/Tags/index.tsx | 3 +- interface/app/$libraryId/Spacedrop/index.tsx | 2 +- interface/app/$libraryId/debug/actors.tsx | 12 +- interface/app/$libraryId/debug/cloud.tsx | 6 +- interface/app/$libraryId/ephemeral.tsx | 31 +- interface/app/$libraryId/location/$id.tsx | 3 +- interface/app/$libraryId/overview/index.tsx | 5 +- interface/app/$libraryId/search/Filters.tsx | 6 +- .../$libraryId/settings/client/account.tsx | 14 +- .../$libraryId/settings/client/backups.tsx | 8 +- .../settings/client/network/index.tsx | 2 +- .../$libraryId/settings/library/general.tsx | 2 +- .../app/$libraryId/settings/library/index.tsx | 1 - .../library/keys/BackupRestoreDialog.tsx | 121 ---- .../settings/library/keys/KeyViewerDialog.tsx | 160 ----- .../library/keys/MasterPasswordDialog.tsx | 187 ------ .../settings/library/keys/index.tsx | 293 --------- .../settings/library/locations/$id.tsx | 2 +- .../settings/library/saved-searches/index.tsx | 2 +- .../app/$libraryId/settings/library/sync.tsx | 12 +- .../settings/library/tags/CreateDialog.tsx | 2 +- .../settings/node/libraries/DeleteDialog.tsx | 2 +- .../settings/resources/changelog.tsx | 7 +- .../settings/resources/dependencies.tsx | 51 -- interface/app/index.tsx | 2 +- interface/app/onboarding/join-library.tsx | 4 +- interface/components/Devtools.tsx | 16 +- interface/components/Sparkles.tsx | 5 +- interface/hooks/useHomeDir.ts | 13 +- interface/hooks/useOperatingSystem.ts | 16 +- interface/package.json | 10 +- interface/util/useTraceUpdate.tsx | 28 + package.json | 2 +- packages/client/package.json | 10 +- packages/client/src/core.ts | 2 +- packages/client/src/explorer/index.ts | 1 - .../src/explorer/useExplorerInfiniteQuery.ts | 3 +- .../client/src/explorer/useExplorerQuery.ts | 9 +- .../src/explorer/useObjectsInfiniteQuery.ts | 19 +- .../explorer/useObjectsOffsetInfiniteQuery.ts | 7 +- .../src/explorer/usePathsExplorerQuery.ts | 2 - .../src/explorer/usePathsInfiniteQuery.ts | 134 ---- .../explorer/usePathsOffsetInfiniteQuery.ts | 9 +- .../client/src/hooks/useClientContext.tsx | 20 +- packages/client/src/index.ts | 2 +- packages/client/src/lib/humanizeSize.ts | 7 + packages/client/src/rspc-cursed.ts | 18 +- packages/client/src/rspc.tsx | 11 +- packages/client/src/solid/index.ts | 1 - packages/client/src/solid/solid.solid.tsx | 1 - .../client/src/solid/useUniversalQuery.ts | 14 - packages/client/src/stores/auth.ts | 2 +- packages/client/src/stores/debugState.ts | 4 +- packages/config/vite/narrowSolidPlugin.ts | 8 +- packages/ui/package.json | 2 +- pnpm-lock.yaml | Bin 1062415 -> 1054980 bytes scripts/list-dup-deps.sh | 9 - scripts/preprep.mjs | 5 +- scripts/tauri.mjs | 3 +- scripts/utils/fetch.mjs | 2 +- 177 files changed, 1379 insertions(+), 3349 deletions(-) delete mode 100644 core/src/custom_uri/async_read_body.rs delete mode 100644 crates/sync/example/Cargo.toml delete mode 100644 crates/sync/example/README.md delete mode 100644 crates/sync/example/prisma/migrations/.gitkeep delete mode 100644 crates/sync/example/prisma/schema.prisma delete mode 100644 crates/sync/example/src/api/mod.rs delete mode 100644 crates/sync/example/src/main.rs delete mode 100644 crates/sync/example/src/utils.rs delete mode 100644 crates/sync/example/web/.gitignore delete mode 100644 crates/sync/example/web/README.md delete mode 100644 crates/sync/example/web/index.html delete mode 100644 crates/sync/example/web/package.json delete mode 100644 crates/sync/example/web/postcss.config.js delete mode 100644 crates/sync/example/web/src/App.tsx delete mode 100644 crates/sync/example/web/src/index.css delete mode 100644 crates/sync/example/web/src/index.tsx delete mode 100644 crates/sync/example/web/src/test.ts delete mode 100644 crates/sync/example/web/src/utils/bindings.ts delete mode 100644 crates/sync/example/web/src/utils/rspc.ts delete mode 100644 crates/sync/example/web/tailwind.config.js delete mode 100644 crates/sync/example/web/tsconfig.json delete mode 100644 crates/sync/example/web/vite.config.ts delete mode 100644 interface/app/$libraryId/Explorer/FilePath/DecryptDialog.tsx delete mode 100644 interface/app/$libraryId/Explorer/FilePath/EncryptDialog.tsx delete mode 100644 interface/app/$libraryId/Explorer/FilePath/EraseDialog.tsx create mode 100644 interface/app/$libraryId/Explorer/FilePath/ErrorBarrier.tsx delete mode 100644 interface/app/$libraryId/settings/library/keys/BackupRestoreDialog.tsx delete mode 100644 interface/app/$libraryId/settings/library/keys/KeyViewerDialog.tsx delete mode 100644 interface/app/$libraryId/settings/library/keys/MasterPasswordDialog.tsx delete mode 100644 interface/app/$libraryId/settings/library/keys/index.tsx delete mode 100644 interface/app/$libraryId/settings/resources/dependencies.tsx create mode 100644 interface/util/useTraceUpdate.tsx delete mode 100644 packages/client/src/explorer/usePathsInfiniteQuery.ts delete mode 100644 packages/client/src/solid/useUniversalQuery.ts delete mode 100755 scripts/list-dup-deps.sh diff --git a/.github/actions/setup-pnpm/action.yml b/.github/actions/setup-pnpm/action.yml index 89cda1ffc..baf889960 100644 --- a/.github/actions/setup-pnpm/action.yml +++ b/.github/actions/setup-pnpm/action.yml @@ -9,9 +9,7 @@ runs: using: 'composite' steps: - name: Install pnpm - uses: pnpm/action-setup@v3 - with: - version: 9.0.6 + uses: pnpm/action-setup@v4 - name: Install Node.js uses: actions/setup-node@v4 diff --git a/.github/actions/setup-system/action.yml b/.github/actions/setup-system/action.yml index d6287844f..6b934c9e0 100644 --- a/.github/actions/setup-system/action.yml +++ b/.github/actions/setup-system/action.yml @@ -29,7 +29,7 @@ runs: - name: Install LLVM and Clang if: ${{ runner.os == 'Windows' }} - uses: KyleMayes/install-llvm-action@v1 + uses: KyleMayes/install-llvm-action@v2 with: cached: ${{ steps.cache-llvm-restore.outputs.cache-hit }} version: '15' diff --git a/.prettierignore b/.prettierignore index 4d06ac882..96e019a26 100644 --- a/.prettierignore +++ b/.prettierignore @@ -35,3 +35,5 @@ package*.json # Dont format locales json interface/locales + +scripts/utils/.tmp/* diff --git a/Cargo.lock b/Cargo.lock index 234727c7242447f9d4618a26bd20b09f9f1c1e81..a453654f593a9481d5447218662abd971447730b 100644 GIT binary patch delta 26316 zcmcJX33wOfx%RJ_`6nSLJ0xKbi#yBgi%0@ity+;JSl3pa6{8R!1lL-%wbrd}qr7!N zz7doG4ZQy-MX|*=M`>?OvM4n@FLm6|@?2fleb3VaJqdmO>gthgn_d_uj-{uT zZu*IBr+SoSsh#;jXvF!dgB$bDl|R-b^U`ipJpF4mYBfyHHTbE>bUZ&vJj>T(Hwwd0 z&+ItWt<<-(z)1sFH!UMEk~H?C%uc+-O-#y{#uYbNq3GzWx z+FQIRvz)+=?KtrL(6J214jkJx1K-rWz(|tR4t&Qmd?N@`I}RN^4t;}LarHP3M0ldw z)O*}ORav^P(l*0$Lo@Xa)3zKVOF|>fd~PwcqA2mwD2gN7cljf+P16bEQ0Mj&-{zr` z$j{O^lDC=H3~myqyr{Lcda-UMhG`|4Zg^=Hv#dOLz(ozuaw5x(oron(V=H!o*w@2= zPl;phH4Dtt5sR+j-kuozQzhSUZCi_-*jB7tNswA@lGr>3cf*ZFtg0Tkwwr{OYsRMM zo2JEcS}cUci;K*d;uu&rFOK+e~YasO`k0Qs0fO&~qK%PBVuU2{Y4o z6W#LTBs978$YoWM%#35(upF1g^;r)-Q}xY!ZQaOx#^{>flZW@M$q)Q{N2_a@j*}Tl zY+AnI8U{zewp_>ZB9@E~vkc$L*cZALI-#B>VP*tInxvWTI;I=tn|5yy`bKrI82g@D zn?F3RF+X(dsbb@Ud}QnB+Wh^ouWbLZ|J{7Dw)OA3)#m3<{GFJ&R$&8~MjWPIk~*S3nuzpF`+{HakJaikH9gMtiHH+o=`Uo~3(X>|3TAI=1EU-Pkd_*p3|6Gu${x0zV65c8wpi0MO8j^W!O$JZi6`G#dgMpnJGwq3P)R%$?glrb_N zHg%wQbDq{$Jn}WA{{MNOwH(T{JIQD0dU<~pd4KsUe`^re4Ocs+J2GBaj-zLxpXiM(A2ZJwJ?zhw`@#D`r<@j?Wp1UC(w?Kl9~Q%x5lb$Zs_Vh?@^o zhvd(iyVMd&{3&|=dvlz4e?-O1*IAX`Vb1Jv*#Br5Cl;Ni z?k#>;R~eQc(mF_<^G*AW>OH-+w?=fdX(RH(4r~;E8m6>QCt>lhHqqEmeX?!lXP&L6 zHisjzWhhHXK|#vvv~|a_vMh@%ljvZ0N$i&KOl+*J)b#c{XfLIA?7{a{#FzX^TWc1D zMi8^!gl`$1$rO?2q=w-Vd?ON9=q4Ox{z7Ik0-1UJ*bkF1$ujbqXrIcNT{26a+56); ztt3t2PG-1SU=z$au^a^MM)$x1QECK1L`>ownNDDNcKI`jBeOsnnn6^JNr%mz(0jz; zM{1(&PNn}L3%ffP&krZ3U2{9T(#2EecJ{P(r>9O2LT=3sZQGr&?Ms`q9ooEN-glfh zi5|;c9y);;`9^v1-lmR`syJa1S**0|k+{0WrL!=S0kY53pR2?PwsLv1&jG9>Dx1G_LAJcy&F#$P$vclT#Z<6 z6ZEp!bO9%B5P&{F3xVx$=z}cgm3x*;T8)4<{N3QbiQIgjuy5p7-q$Dw9-{Qk@0&YF z@p{+Jy;Kq79iAKjWgBe9fC!d&?48&RSxW$5?D}MqEb!Quk!uiCvLv&4fqD#bv-n6W zh|9MvKL1P|n%AB@w5bXm;?9L9rQMu#XNo>~VRt&k>orflL(Tv6Mq8^B8Mf(=Aye{s zraK{t2k7FsK&d#3+|b}H0b_xCAWw&T11e^qU(Synxn)2(Xa1!8&dSK%pUr<-5uY#B zI`TsoeM_uL$?Rh;Zpdd`JTbqoXMFEvi_hyX1}-f@pl3%uNiea1QGpi|vG@ju%=J9p zcW9bkdDOii3FQf4nOQgRMQSB+SWTHN=Z_S6q;%vHzTKSYa3Rts>@JNUq-dwtl@Q-txlE2pElG#bfFq(dPoKfA5oba38@GvGQ1T z&h$9ZLnn3tem;lEd8_rKeI1XIa*Jo`bo9G;YdSlix%6OTavB=36J=WNJ$=%aS zV7Kb7(YEtQ`KIL~^8xp_?u^JM)(=%`^0oILo{xFIpV)A^>@O9Hs&usT0f-3QJq~$ zdiwluVY4~Kouc)|7OIxkWX}cqIG6uu1VT63@TMXnH_OM2Fcq`1Dm_Gd64|dx0`xb{^2l9 zEF4QTNqh=^%1|RTZ3@1Cp9=yrNKFnmB>*+G#J)u5&;cb=3+Yr{cIKt2#}W(WOrpt* z<0`x7KYD45xav}+qs5GXuogfjv?DX~65Y3CDj{mY1OW2Ef}UXol4}6k^4WG6xS8+i z9Iz}EE5D^R^?vi^y;bo_0IIPZ(xhpo0lD4v;|SnMgbO_129=X>c|=_%Rte@UE2cPg z0>dN4>XdJepFel=IC1ZRYJDF5X-JA(g>%|3fzD~I?jMBfus9e-{d#hA-WzK zA%#1;f=bfWJu?6sTP}EAR>UJ-8?N!T#oDvA(Y>=?Tce2iUnQd?RGfB_n4#}e^@YGe z0y}vHsG$Roc%c#LGXUNApaz?oo1(>Hiw8QPILH_Mtx+tjD1F3dzS<`r_-UVf^?UXC zuU>zrc3yH)xcDTYuT{cUNDA(eR}>S`Lywf7a-AUJb$j5H44#Lj&PduZtW6>*8;VW` zKu!{MLoqH^n{Hn7M$P7ZoImr|e)+OD>v}i7^}CvUX1})SE~ibWh!0t4FJ^6=%mkI{ z)UUt{ORgTPABCPo_Z~Eh?Xce{xOt1ZII*SDB-Zbz4HW}6D#Jxfvob8d@4W?LV4}89 zH+WUP+=#yIWU{af3=RZjmV|t92GsSzxPd7F244qU)J}o5PL?^gxKdTdh@Ys+F1@SY zKTeVN+G2u*!I7EAuSsNM1QwMm0J=oIWNQu^xY9A~)CkGIz!n`MiNY9E104XwZ@yX? zw0W`XdSCkRTS~6HO$D5pfe!g2V-l~&;Fl5)%C^kp$KTVcw@X|~Sxcxjj}rmnkN8@yNXkKl zgp{Q<#Wkx!OaQBr$viVlGcqyBYAbWJd6$(}&$G`PN@&&sdNT=!94sOkfHh)$Szb5e zwgR3abGUzMOwOPSvNl4GTEU|Z;A7ds%CiJ9)}l{7V2`@`?j?&MO~^W8<%{YNaq)}l z@bcrXi6B{K_Tyc84i}~#OZ6nCEYQKsaTc&=c@RSqNg=g)z{Y^Rv59URqW~6>Ql1d) zZF!fYMSF+Z-b(bPu4Cb3kOOA{prH92UMeRn)&pu#$CM;j3eguifYyX}3ohr~!-Q_R zTA$)=MR`#v-`{lD45%xJH#VBiR;H$a>x7UbPH+f4htMMl7t%#c6$Dgf|3cK6#OzRn zQ(2B}M{eVF+JNHjiZY|V__)8aSFK{@n|eBCa3VN+lv(U&(0UM?6pK6sha2i5mUWU2 zMMy{nU-O!YAe?WwH2Io9y}}ryG!@SdQ?Av-O{=x`>Ci`h1j33z{~dlZ@6^4Eu_Kf@in#Z3f@f;m9!umpflsU?ck3<` zGywBJ2pdzf(Fx~Ws&Q_S*qg)&OHLcI44c^E+IHn55e#;)a!#{qlI zmSB-BpTZS-GT_LN>r;mjM9DvFa>&9k_QWfLwMNl6O6e!QG+H@c)ci(ipFzEm>QI?f zYt$iR3EmL_)@LytpR_~4ZvfRo5QU`M;C!f!2o_+Dh~OI)tlgUuaf(&DC?gAhywa@} z_fJ&pTG5&*qnGZfRK(axO5b9@KFaD^`Ex4|P)3S34P~Gh(4clKZLL(ML9LLN1A-7+ zAY_%Gc8T6J_|Mm z@DX!Hfqyp1g(b4kOkhMQ{K<~2Q%V#7jX0Ejk`GDz>RZ3@Dc<_!Pb;H~2Ylr)rL62) zp!W>jO=Xyllx8aGA^`7&`1)-;~u30S54F z#c96i_YGqB;mU5}vuVm;(L7k~xc$)SN?p-@fKpLQ8ml<3GeHzXYi;_o*7R>4*X>)L z&ho{T1o|#VvdyHjA`bq#(p-WHSYkp10gGCfWCNlmgi-vUi?Hda0+BCaNRSr2kXW6_ zeF&&Qq6o-Ls1$&2(eHTWEEzS&h(o@v^ecLgSH4;+h9(4|4YMk9T09?NECAN{pc9BC zQW3a^Yw)n33zruS3oSX&fCYy^qq5K)YHp&2$ST4N zPcg+?sdAwh{AE4~w2KUCLr0-6Ih0*6abQg%aNv?J5rcpU0BDoXCv`xLg%A}~V&t7H zDCxmclo0#O-4Ynety2HEI-q=I#iF^&!L|8!YCUtd}KsaBfY^P z$nZoVP4PuBk!Bt^DL}ael8|-fZeI%T-KC7uVBf^d5z55k<}PJXC66YxwV`-MwDdh|Q(pda>v0_w2@M~?(3@S0M z#3LfOIXY^i+&Y;hqb!7UW_wgp2I$sfwb7bkfl&waz%&Wp94!jKevL(by7CuAZ1_MK zAkKY8{Zeu7S-gS#yrbKvQ(GahARVIEawu)bq)Mi^1O+5S6hK)pm@m<=IU!z9+6UYzwonF; zA%A(RvWpmfw>&H3PfMf8xzRXA47`Q3IpAjEJOulWeB3S)bNd_WZaZM{?q9{WmJLXm9=Z>lo zKfRL!|Lfb7>D4v=^j@W}Hm`e$SaBCn{j%GYasP4~&)lvI*>NiZjyqAT_>fpi>K`Kp zu|E-Scg%8cdB9QP&c}$@d#h?+vFT3b>)RexTz;3*DL;P9zuQiAV*k69-Np3V6#a{w znMR@9qlDtI8`RA^_riOWX=3j?zv$^Hqtl7^D%sZOnIje+pp28pW7lDwo6Db57jz~i z(%Sx@)QMF;P=+e*j#}tXZyPM;-lzOQ)^$5-drHEljPSMMwDK|6RouKjqYa z^a!Wkdr+BHMa5LGU0s5rY?xU7AgeHKxw@-5Z?SmvLGEDVt~)He{loLt|8oE)7#t+- zpH&$mYI`}L|0dwp$sj#wVBb^I?!}!8x|*jqj|cPasPik)?kKV1Ud~{d5FWjcuWEmy z4B37k=szL*$3Ier)|&{$;k<<>g~GT^+3$;-*oN))-iRM77m4OIU-YnUi~QxIT>r2# zRQ&K^CH|*JnV>j3=uB=Aro_0r2?^dKJ2Gf*Ihi$EH}6IH`ZssfA>6w7o1xwAtCg8s zPrx@s2R0HN@a{ulTGg6TUbEQnxbk9+c=9Q> z<;QEW$AF$;j-^#YN?+(U+#E6-J}_<%J~%_nfp?YMFV&#DY4kwKS&DVsjaIdsDfzKi z&Q(Si_dKoeKQABjOh*d}o^+VOqH-tLzrg&y1OJMCLwO%zH^hn*>_Erci?1 zfGOiS^7HyM6^`40>w9ZNq46#l_rX*r)hO!*mWSPd!AKR7yGYMKJJr0zp1SS z6&R!rqfBjwJ0!zzDeb%nz4F0;|B??%VoGjA14usH95A$aXwiIKK5t3ma6W5(T4saJ zY%=zs?!~gax29Odd%ss26tT~%yo@Zdq->gC5b>q4DT2M1JPkS%zB!y`o{daFaSV|z zEeRNZAU0qc;TIu0t0l)4<3{oH3yPzHvlcxg{ZAMHsVE>oxFWTHbj2nP9#!8*PT@sC zCrcFw;|3HaK0WvgN-Anz=~i*&nu+BvvvpRkSEd##URPFC#PZ!C#bCZ7yjS?HF`pvg zCGgbjP5d+{H72r~WIl9m|gf)4vlnyu_C68y_}cG2ij6qK@Wj)vtUuAxy3A%86QQ#aH{J| zN@*02I-u>z&1hq?Qp#7qKR`Tvt=6}0e%jOB884Pa#u%~w2DQF8cO!JR=vXLa0_l>; zq-oywr9BXv0PLEJVGzqzz-MAt_A}%%YGvtm_jQPT>`u~C5L91*boGf(-&K~2bss9V z#ZB)im#SYRPi+q)4&SKmQh;Qqs)}2z`yb`A{>A-$)a!Oh5o%@Lf+dOQKTb_sc_j%1ri-M+@ndUXkw8&_`bP7|gu-D* zKA@^bH^&f)FpE!JipP}Z@U+!9g=cc8XxUvID84pP9oSD6aFa`-9+V)$iQGij1a)j} zS5FxCEEbpTu7<6M{TT{;iZKp}4O)gXK~!W%NW(*dq=<5jV;7cIzHFp4E5X2t4iw@e zM7AeB=I3C>Y@4X=B`-g|SUg^})IM82TrAj4ojYB|u2QLlbD-Epc*9DDc42wA z2~TOm#Pz$YM_@ML|4?q_0!oLS>+o11jsrx*5GDcXnzn=tKUP8%g0TvumkzVE4l!v3 zWklWbe51tF`Px|dkfV!>c2^Hl>&?lF&zK)A03()*sfU#%W+iB9_{m0p1P!!ZJ8?v~(cVn$b2Z?RMLuK9;*~@+qA?;^T3u{ja;EdDpEA zNvIc>P1BkWUJ#ueZ++M@Bv-$)q^mPq&^>>04@$t~&V--u%sPRk+y4m8=gD0Q_(6Oi#B*3_8c>izSw^I*I7Qyn(Gsxa zl+7+ygbMl0soM=9ZCVxFY~39C3s#}cnxtGXiI zM;$4eH;`W78i}qjjgp8%hJ_nOs*a%U>F|*b1N_ucsOxmVfTwWzp;%xTf#pt7W|C^C zxa97>V(KJys$%Ab+9B55s#T_e6v%#JSwLdIxlQg5T?`_!OGl>@ zBjgi(>B^DZaJFS>2N%n9^=RoQ6Duusyclk(FVvX(fIVc37j7(xY;Xzjpr8*I{yavS zL1yMK!mOq#icpBEYQjUZXOPqgv98PCEZ6qR2AAcQx^HQA;_cFeg&Bl<1b0Ss9iVIB zXD0L$3+2jUSwM55kp;&ui5Tm)TvY4`U|G^O!rTc6Tc5nH)r(7Qb^qcbSG`j$-VIb$ zEk2p1sx|r5@3hUpOp2A$LT^{^*+o8y1+u5Gr?CyN{Lo3@2dUU}3|#)SQ0*wMCS^4s5w}I%hiWb54EnF=wn9+h53&?` z7Z{0DVH5vAu=pO4sfh2mXT>W=XpQ-diQ~4e|CuMMee=&gsV&Zn)Rv04a-r5S9W{eK z24tE9p8^I&mV?5R1jIi`@>7gMLiZCW&Hrq10w+Xr6EYy9S0A;-|zi2&9NN zCO%utr?kw-+b+C?>f2Snx-fdwMno}=8g>ycnXv6kY{!^sSOO_4VjGdo9~qud9g|le zjGwZ_ai&y4AQ+j9#Lip_IX^{4jjJ|DPWw*8fdy z>f70!#+YI{L7=aE%3k^WbRhTFPgAX-viqxA%F>*P+nQD?Q><%N_7o>Bh994OD$R~l zPFMHab`SrH2hKN5Y7jHdf(zc`Om*baGgK>Ij|0&c3(uu{@8!k71ny|CIQ!`plA6uR5!&c(BkkgzFpDHy=Q6aln1 zNrECW>8T>^$!1FmIjRtTnnVSHg?TZn{fk$9r;oV%Yt%32pHHw~Jeuf&?M^z1=nakB zGW}!YARYiw68e3?0}g?L&L-*0AXw0V;^LUV&OjStwY^K6b^&&^qrRiQPy`pKP6?RG zWOB)ORd1&(l5wGW#?H8~Y>E(jUIchrbdg%9JK-a$%Il~oTnu1@$ui9;M!C77t?yiP zU8(La-nkS6-|#(km1w$L-G8Ut$+}auUBuZ3YXilrm#fV~N_17;%hqk5E32DC-$VF` z)6UjLh!d_*&)X??0yQ*H9DgN2kZ124`ETx|yf4(zd}ZAbG4U#OTHd^Rd_Lpr{lyGb znw+juN1+yK{e+eicP_a`Z75#4npgNmIG%=o)M$#j@2k71`E#2f1##Ou*#9v3P}{>V z(ojfR!eYX)%kZC4ZUL7#RXA%Th=HgApi;b2?7M#H1^g!)+H2RTd-gA%vQ(fZFG&_k zq}wcor$d)6Q%5RBzBX#de_VF*LYsO(dB85fgpvl#$e;jPDLoLWOXjnwZ~zE2J>xT% zy_6Qb(b6&hU7)d^z2jp}Ui)CwxtpIk`Mh$okZ5NLCF3(yi&B-D?@F!(^g z4W*4*W^LIxYDSDmkl(@z36`@lZnQ=qQ1WHd#r}s1lcJ_98_ue@LwrSNsqmVAL>q|J1k`P-Jyv(5;CN6VQsy z5O*;E7)m{bCZ!;xua-KCW>I)Oa3TIgRxQlKcN@hsAEQ1VU#K4!$NxxeR0aQS5PzPi z3@BDStlqUVf=@NG-Fv7uO1$={s{h*)&-u!0SC0@~OMnn}%m1u+lak}Rs{u!}T1D-x z(piO(3tKxy00T)RO~Lw7{$uf`2!^0_DKJ=Wns%_;p<+mPAiRofxvkc{rOUSMEA$Q> zX$%%GU#AWF=Q#so9;bqm!v@4rkE`9|%9F9ReV}=K^~=c%)A?EFJPE3*9R-7*P(SUL zEI4iP#3i1#l? zhd}dConbhZIFhP_ZHa*%_pPCe@FBIac=<{7{M!GmgLa#UcYmF>d-38rb&RT7W*$zW z?2?9Xs7fddS-wF?V8w!ch0&KCcrny0y@niY$v!dSL^|VTQ#;mdj**m1%C7n1htI2T zsba-!)DcN-$sSpXe7JF}{gC}GO=es)A(_*#&hO*qhi(gzXs{Jxjtc`^@@^VLLEauW z76<=U-B;Bt*)}Iz$AA<}u)&o^BV6ipE7T_pbby;ea>)3T<}HAHfJ2uaYYGeGE8?Ye z9J0il9hH;r(Hg|87t{dOhK9qXzg8>akr#*%AI*TYg^*&KrN~p%APgyxzH8<)xNxWN zHMrfNLuB#o#Y7?AA?F&WAuUMA4>(e*4%EMXuI*i3_x2aTJM}N)dZcnhseqZKenm5a zQ7X-{F+?*Y4lQ^lgbwGQeqY!R3Mr&JIT8Yzjvh8!&ZlXVmn;^%q>fPX$0FK>sZ9|i zXnR5TLA=9W3ME0;E&+=+9_(t2kYSzBOz3ApghWFEwplO*c2pfJBlMR+<}K^hCcrSD^Bkrd`jVPH!KzE|3f-NU z=eR*Ss|kX#znSd^F9z39s$=?>F;$Qr#)#>!tNZR0d?9&=GOG1^gNL~B2DOh^I+;}H zzoEA5ln;ci8Ytd>LtVU6uDuPep@M&jcThR-u(#Bm26T!6ORqh1{dFWSDjA%d=~ZgtcU5&_}fOZ^84?pBZrs4 zyPV*Hr*Lky1m4y*Af~;q?jz>EuL^m^Jw@|pYX4&Y57cuiVp)Gg&Nn_)gJQ(Ts!^ku z#bKYThO*scmupU^nAz5c@X&qQUAt`e&!w%u=-gHNON}t5DRZXNgk=(yAl}hZbki$i zDxgAgAT$+8k^?0O1sZ)^NAr~;FJ%^SmbfWqHdGq(4Hvk@lp)$#DmK0LTeS8TA5ke_ z=!ldh2&@EJn%-ee8P`M$DHdm+j#b1TdMPpJ(+5l`g#jU;DvQ68*J$;ncfK#IVbshyDH4G5rJD~9qznE%*+#5)m zFTr42B3a<1DZvZ_5vWJe=4d+02+c{PS4|C{uh4c;-6?t7)W%};Xsw^@WZ&-N4jZdA zE$vbJ<-d%+1kZ}E0MQ0=3r&SWlUxGf={Z!gCIu{VH?=FqeGCwwDhf{+BK!|9y!3>n zGJafj+*on=Xsx!G(yUEX?JCY4ukFt# zZfanb7UOQ94ryX#R2upp@;FXv$ulzP$faRO3aWA>2`@e+aWOyyZ$6JM@loiB6Hnq3 zJ11yUwyxGg6Npe9>$EwoCKK0aaA6>iY}11eMxB)%`SjV5c`<*a_!%Tk3-jv8ox&g+WUQqxsLV|Enii; zTXA@x^;1f$+bU;-F?I~uK+Z&=V~S4!D+Tx#SYFTtOzinXK3%rRg>Q-_|7tf=p8!~M-H(O1Ha4} zTJvS?fE|7HlGxu;jE|7a(-c|elCXxz;=3h3ds)3WaW!+M+O8nt(p)uE;Ew1kF1s4# zuO?R=vGEY?kezZ{Wvcn+m1;xr^Fy_zReE&C(ruf)=w20u>f zoEEfXgTYHEfCN}3{t@%(nh)UeW!WYbJA7A+*#fB2$OENM)S$!$x6x)3=ZpS=9A&Ui zFPN>h(+qkf{Ce{o0O2QdwEh*j5y_Mzi`|db9@Rj$=Y1XGjwwPJ4e0b{6T0|t6bRYi zi^L~YCc0J73hA~%GStyiJt|Q;ZyBEk?$ePY%Q1T+#2v?Jdl%1qO-GzjKZ4b!oBk`44DLEk-f+y{3(Z+Ax)t0Nv#YM0rUVq4wq<%8I1N$ z**T%hMGNp<;q$7FysJJ+dV)5kR%oh}lXZ$&*=m8J%p40GLpVSn>;b^C>kV@Q?}r9r zG8VN86$RP>^B5`UsoOoL1blL;S53SiUcmIZ%|m}g`-9K}cVyc0;+)TDr(HHz`*FVf zVw_z}17K_n76|Fo@Q61sk^BwKLr2EK7=Yqz)9Hwf3@8i^jp)21=p`^9Nmj~GTb^ay zNkokOI<;r>^NyAr2Q4okX$ys%-ftSJk(yyZ<)8_KH1Iwxh%{HC;vni_qLR}P%IBr| zO|N#i75AO24N>xMjc1|+SRsh0gcB$ShQZJsh?fP#kLZhhXF?a~)N`b1M)_D28gld; z!a0Q=&2C1OGJ%jQy60&l6q#*GUtN8%W?0bhOgYN)i3SVW^cX)_7WYO992M= zgkkao+zXQ_Y5n4{$>SWvbG6;XkI&P_%Tu|>V(pmf zWvNxa!{rQt#20 zZMi1R-OzrN(+tx?N3-FNm~aE*<4VASYXqFg7*?1%Pz+tH7_F#j`6ao!<-xdTZw%FE zF40z%m!#W|wk38yC>4+->3y~=-crgaY?N}E1vRjILvmzPIgtk&6|jxkkV7I?U5*dy zg;TYC)#@GUAW5e2@l2det_nvO(HjV&fJ)>9A??vFYXh5D2gp31rN^{HcPl3hCnIt1 z_J_~VRI&Cf?Z5JUUc?Lnf?=u)WQ57x@uHYTDGgL$6Dd(r3v+~!$QhO+ha)&NgfU@| zhCG4?ZL=i}y6@TA6B4u?LE{@C4=))08#x+>?q7~EFj_Jl%xvX!@@fe%a(D-t2SulB zoF@cv4y3uFTJb-kYJG}(zo~UA20tb5g+n%B+n1<}SYA1(p$^o0)};^zs+w6DT1@@6_H<2YAini?oBwi;)F*m27yn zD6tnK0(;M=9hT(5zM~+KmI&Nt>=EiD$-HoJ3~C|Q$DArB^lgs1#j1-n21+p}wK(T$ z?Fa=etNBK4bg}6g?I+6CcIbk3=))8_jjdIZ<_v9wt3~gIGT_yce2F_TpU(m^aRd7> zlk=$6pzkoe$;l9KTlhYbxau&Hsj-jjx~><`UaO7H#~(X_Szgo#gRj%3EBTwX9mQEU zXkFSiV88wbK*Efb+ITT`ISYN%3a$5F|Ix0F>Lo$r2qP?UG^DMWF$u(2Dj=p;P)Ojg zVnzkAnlU{1Q(?H}_+O@(K=${6 zktdklA=>xGwYK4cnm%*8!-XeF$1j=x^}KN%V@2e!PT9x{qL4?Q7$#=}a1)SBOl%H>S|A#IH!{{AEFgYBpEE?U+oZu^loOf)^LIb!5H z4B*pKOsrUBz?6g!;GRPT!?Yn@8U8E-eMOFLfV*N08O582gg{2%SD0-J?VXBow@{va z@UZr~eEvCYGfa*db`K~8W)sM1w}e61Q%PFNajgkslQASS;sWEVP17~Kq*%SNuyIy_ zT!y&v8Lg>U@`&~zt#9pW#|&xo#m&TWGBpWb8mzFi{vrNC^WqW$7{WC`{W7y#UQY6V zh%a=_Gkz1%Ltb#K`1}br^RhPpZs1x8pCoB3kxBruO^GZAnlfdURbZeIzDGD>Y)(kW zv@FmYz~mijXb^T#jxZGif3Ecr$NfniP(1%1+Sk-_!g32O5_p!GMMGgo6E~)7tWqH= z2zeDQMRq{|*-JydoW_Bh7?lL|jMAG4SovDub9vpDe$EXa^DA%;1CL>6<*W|8CN#G$Rw zK;z?!rD~9@H|Jvukqo7_o2o4SZThN8aFdYm=TLP#UZdTO1yf>YkmR8l?7Vc z3_>m4%sdX`Z6s}i1&bPl+r|$MD~kh%{tSQ_R51FV3+mvg5XBfW4y3?pSiYs>p3+9; zt1e+^uB6Fng63tgt;v4$ck_bfs1Z3}l4%gkN8%Ytlafp)Ye`rpN3zKN{mn0_`DyJH zd7x%sdBeYh=a6yJs8Dij0PXz98DvF*6udEO2>p$431Xn!2;r0QJF(~a|FFC{t6GC$ zNMybMAsA~BGnr%?2bBxlJ5P@JjA#d(Vf*7%1TOK;&?DgKX(jnkDCj<)p589V^LW$g%%U1sP{-6yh`n{lis)^H{ zlGBf+{Ynm7LJpzSBm@FzfP*+OuspG8VC*n}ARpOL7Z`f(n%<_QaP&!mn`rO%t>KO&k!IFUm`XxCU1hI zpfN#Wkzg)cqd0d@vdzY~v=ggZ4m0=|!%TkxwH+=mtc=7L++-N<&}Y$sLHqQ1;QQls z(-nookzmH?UnCCvCslUogSWKK3 z08vU3n0S(9)ys4hu1}8+6ZLGfsdT6dK$xK0shO>4MXliRbPwwdk_O_{Jt}VRcoZd+|}@K$dklf7*l? zlC2zUM&u(V0Nj~yA{&qpoJnqk9=bTB^_SmamaL?l&fNgL_5F8AG^LP7go3aJm~o%n z3n?0ajo*Nn3LwF$PFsU4FPH#GUx1tugY%o!#?vjwC|5n1+n~`#2N~h;4ccJwOig9H z=-i+UFIK;&T~Z@f-Gc!XTU|L{2!k;Yi)&M7fY7P>Xg`qKS5E38kno|*?xXO>H;Do! z+t)zUr4Mc0ZeSHYVL(3V+(w~)q*aOuA8X@k%8HqAhLceiLdausJ=la0ETjsa3B{A_ zgDS~PAIeKPr%~2Rt~5aLi=3Bo-xUaa=adAq@&=sdHMED2)h`$It*AAktF6%i$$$KI>lnU$_s~j*xOH^p zw14@xlF#3Ik@{`Y_mwWqJy1$g8wNu&f=(tp37*mE=oqx9a5Eiz3XEXb2OJf3 zBWXDWhS>~om4ycyT_17D@JhY-`?v~2??+S)5c@V)=m38m*TubKE3Y@R#AOjDr&{8V zp;J-5UpdPiqd)-rm0)hC{{_B1h9y7)k)i=ZCIfH+C`2|}c+56OT`x^=;YyZkE z6|rOirhO=PtnVgXdyWvG!lH*t&S%AA67y}@ut*|gz8C&!`Y>p5!)JqkhOok35bgEg z*Gum1SN!bDl}pr;NWvBe3(1k>$DuxmPIOJ;>6Sydpc*}d9(FHyn@Gg`GyoqvU3Qn! zR00)Hj^!%;=TNyP@|lQxq=l1b>0Om;7U%nS%9O_G3sa3t(XMv}~A5=tU1fCa7? zU<04h5)`CKiKF}##00Ux0jYWrkdmXvW1~cm4G(gEYyW3bIC$>!T*$p1O(w}M-?zS1 z-uGQ=fA^Qomv0(-?+Kc^|M&W$PCs`&(@9-F_EIwqLf6eBH%z10&HOmcqQW!NG)>$r zw^BFElPI?%E03+f4b0RJ)W%M&rTmBXQva8XF%9bAmBymZFtXer$ob3(i+Y7pxrLF>I*v4G;(~`ii*fDV%xG(D~T-M55pjF!raSUGqIw`o*Y z6_FEXc^a636_;1M-BKxLj*Y>bJ*eT4|rUpexEO&&nOI@GX-kFwMlb zf+(_*AhL=yOp45Qf;eY+Gmk$GO(*a|)-_68EBAxyYUPW~lhi%`qAlo*{LC?9)AwAf zNa7+%gCex^%raBcw|qA(lFUwB)3Thf@Z&7YLRa1-O5Mcr153@kPM_3&?buD4`uuDz zmlQc0lx1d|dNKbjOD)UK+3m=(c<+d1N`2e&P16lrcHb(@BIW+3XU0hym0QkZ)5rhm z%6k`e28CZ_UY5m$>3CLFuxMe%jwE)Gu(M%qhCGgGhh9*aN#XNIf$v0?>8EBsmv)wrVVdn|Wk=V2c6oB^*LByP&@A_s#>LeFzp11rs0(llmU>?p?& zvLMTIU(KAUyXw5t^^xTzPmC-xe@fZeI#qS`W14or|8va5Hk~Nu{*hOhk>hzjS4w=Y zo#siNc~0b{v0qr89a!=&(mZuy*Urne4`Ve?>{CuVqqVre%OkX^xmW4K$_U+}yF*quh77zn{lN9@>eOMW!Ekg%<{{&BI4lj1?!i7sv5A7P!Sk z&3w^lRTtc*jVZq~d-RwjUdc6NWvupK)@(KQK^Ag|HG4KcX}9cIvxm3MTC!?Y@0_G3 zTg_i{+x_~_R`=$8{N8fg9aHtbrB5tRopYZ$do|Xu;VHeLG~4H?&mF0?mUGR<^2+w{ zs{4m>JiMrgy&#Bf7gypInNtKs;Afonzz!lCFY8(yz9RD-m;LZ;JOV};2X2PH70)Ne z)}%JC)`yp$m^)+E|KmBAbuTM7&21^yS#9;ldD@;@F1%<#hnJ?dWBV9=5wa0@6Yh`) z;>InO4d@rq^Y{$4Xt%l+pJ~X z%e&Vs?m4?V%ggUOGu4?l8?F7V?m4>J(5@fb>A00k3KA0&3MFD;KzL-lfoT<17UX^w z*;tvw&N+5!+ld(lnPa&aj#H>BZp0$a^CzkeKIX;QOuW#y1BWLwZ2}AKF$prCBkj9x z77{)%6aIBtRC;rPbqayLm+zs)J z@`NTH(GOf)dFaP^=$jr3M7Xh%%**paJnPj<^bhr~i6$DlAE^s2)fRT*c(XM0tk4PZ ztTry6*Y+|i@@&jB@M0$>zIudR*UDl);EdovtuVCt+fgh&wpCsJhBmb9n!kUOn4H<} zsk^T*T8wzjYIWXbV|O)TuZH2}RrB|*AOHU6=6|zs;BN-yL3>Rpzxtt8HSR59M;_%V zK{t=$kY&pqLSd98fyL68d6v3G?xwMwra_h_LFf}Ye3OG@$7~KyUpahr@)Bc8|2c3@Ff{2jh zp&2Eij}y;hZe%;DnWS-u&o^4jUmY=CU3#u|TxZ}GK@?b7lHo@EOcptC%`i?v2j8BB zF^+=F!NM{_ugJ)tthBHTFA4)IN_?fA$>E+mTWc&wjTluvdDPhc9~^nSapJ*!`SPr1 zdC!^M>5{p~CmBEC79?JIR%A(9m^R^%j56kc5&~kco&ofpIJ5tr<04&hr%uQ8FeAdMZDwIn zIFTP_UV?8Ym=aVnIn7ua?wHHt_CwdPox(H;tOW}ptL&BQPHgPoc*3wo%__HkY|(tv zcXQv&k^=t_kjSKVLDb1i5<_e&iDQp=oCQUZXJMMg366k0mYgl{(keLipZbY0O-c_m zLaiT1BoJOqwh)F~InJ}pNh0o#NjbTNePZ}#NP6M3D!4o+A=Ak7g!re{{+u{>@5l8K zWn12?`TbV@d9A$iz=a()4m` zW1^J!qz>Af>n{G8KE6EjvFcG3vP!^{M^- z>5pq_%REjrX)kZ!l2HaRhdLnDvaHbZQqL#r=PZYDjMZ_}$i_3GgXa-7Ld)lVPT*8g z>VnB+0`FWjy!_6xDMQ3-wcBdz(b@!c?mT0}gtVtGpL1GY+`FVZUCE!?4u5?8I3A~M z`Rwu!OMfZ<#BMjMe^UPV!pZgIrF`?ky~>RkX_F1`uq5F{Mm`ks5-g?M%r9IHHMLO?kf%o+>&(Sh zg-%ZJbP^}F69U-O{_woV4b3UFdlz=3lFizQ6fcBE_L1r&^%I=FnYk{pg(8puMEX%+ z`X&b{P9&M)fQMvTGv6XPs&CyMvJC2q$F!Mc^F{mh|JT~EQ8I^4DkJ`~Fp=ITG5XW}q5KtRE+@gA$jOqPveD>O5Rm9q@{3^GR zBzd>~FI+iGFW>n3DD`Sf!y)r?-jK~AeJcvd6Ul=@d@)Z;Oy;)aeKFB8E(i)11*Euw z6idVnI6=Ag{>l9(T&?MYv9SL{b+A^Ck>y=qYU+RRi+|nz+o9$2*S=c+!~UOMH&hb~ zjM^==H7FB;VwH%@&&b_~5|r#FId*LFPfTJ< zfNzX&yX4!JNBD6_Br+mIh&v#pI8@7IJIXpnpR_FvIq{B12aLuoXWscSHD}L;<2nn@ zw3oS*%Si?r65?8j7OoTGzr8T!A|}<58{yC8|6KfI4xT|VNijE&Je4!PHdHLjX|HEy zr`-8y1HX0ELnl#|9rU&D?eM#4ceSb`eu$SNN`h`gcob4^f)mBB?fXs~b7HG9(B^29 z>f$U+pVy3IGY;?oHU(Fl#cJl^dW+h;9}fACca1N*zdXHv`d$6{fUUoF&(j0HEMNF~ zwCfDurZM`Y(WepiyO+gFs}-=+Wq;QvsqZh=r}TfP|MFqt|7u>23>2KQf zkUli)j+f6_wYpcmnHx!G>bQkuIwTSJ%fh1$!%*-Ygi&%aRuczIX^HnSJ;!kXKWs@X zC2zxm)Y=gwR6l%ZdbwZ~ez7P>jmSxTLXelpR)yd_DUV15;B2}vIg67KEn}<&^Cu>G zxu25EQrk=Qpf-Jt+V0#IZJgRX$(S_YsvduMc>k;4`?tYk`poxV9@x6af3R*yfBd+s zS3t@6PYx?LJ<+PV)&Xl^59G>ZIt7WdPn~CDvzeHL+_Q*Wi+qNPDYQ9K_c6dWXnXp=?MGzkf;hbJO?N&(Db zwgx!HrZ{)YwJ)})*XK0sRjOyFmljW#B{*}zGk}Lc7-QVE=_fWgkxx2GV#$M22}>5<}Ub9QQ;? z;Cm_AD?m*kahRGUh+V1<7L9x>-A-S0yQcU75;uyRm@mSb2B#4@1j1s4VNIi5kK*HaCHn}X1o6>JTzgbhw4{6l! zo|KLNezr%cNSKwkbkp1+1lVSh zIiQ&U1c~p4fm*)Rn4;c%P;V+b-`cC}{LRJXjax>kw{OtQ1KLmPUe(!`pE+M}NgqdC z1iNqlB^fRLI_8MrK^?Lkycnes#RDZu9R%U3{XM@eGoKxz{CQpJKT1E@qCW3qdvpwO)+g3%>rF8dqXq~&$*u;I5u%g0`yBDi-4QOPgT zz@|L*-2#*{q4)+4pu~#eh`3^Az6C%5ut%wj6DMWMd2^}}wy#`qOzYr7jw@IFW>WdU z8(kHi9=ig3=WIE}mK9UdQOabzJLLeVjctQA`xI9eAQK1KE^P7w))Z8POQ#jUT*`yK zzIAN>18<&VlxrWOz~J3jy95j8bdt_d=L5W@p6jG!`!*T;pB{`>y&5;jG_?PR`qs~ph5H3RC+i~@qqAYM~7 z1Ti=k@k_3l2+K{B#eSZC6UY1Itf9cT=9Qe6>)Nmv9#y3a`W7YL(b|3);LRz z9oI0cc8{U84^j91t9ERMYf`!M=t1IRwBW2x7BYGYA=k|7HKN&GH{Dj_-uX=VO4Cg%Wi?dVe@VTJA^9<$&n$!@Pi-- zQvBQCg`8eD6G+3FOE~!XbbXk5p;=p@HlC<0m>=SKKruMkWFV4i!K8#4#FQBj>G8IH z4(LpE&qfmG$un`8RER+Jp{-U-CC#a>_|{l0DerueVluLUeguAsEU+d*iU;526b_ge z3&Hcm4gh3}d+Dd&$8z-UNsTGqocZhoB zKy6yBHbZ-SNc|kD^?PY!4JvJ6%sN9tDI7^0f`VGaL|}o`W1EP3urc6u1kIGSaXmbR zlv`vCR9Oyw3WPluL)=HL*;|`cu6t{!YTlz^_*8ImQn+onIrfvEy)5o+v)kQTZEtOw zu0D0RP%a>kQjJE0etBV+jZaxC@v3C|DIwMZD95ls$s{!bqLv(twcv02Qo^*V4UcOL zl1K;b)J1c&Y3hRgv=;S~eYLC9H#)WPYW?F{4C3k96+Npi zFttY5SoTSrraIMPx$a(rYZe~D%ArUx0wB3O!bl;4NfQGiGDt2bD+x#-y^H`4^NXwY zQgAJC0ZyHrxc*p>>UYlDtA5$(KQI>EI?U3Vs%pw^chr_^w28Ipq4tJ0Se486O|9)S zPy2nNx_d9}xDL)_EDnRJEs9umlQ2cLWQ7^fEZ&eR2Nw)jOJd_B;2ea89RCpuWssLE ze$}cr_Zh=S2yZiBk7aC4DwpluTKh+*_Df28;8@;%;D>7L!P<*zYitlRQjlZLuLXEP zl~?K%G|DH* zfj5E_g;Wc1oiPtWO5$NzQv8GA1%U)=VgWDX0novpW~{EM4tj|@9`a$$#?XYwk^t3# z^8(Bv5fOleWJSCh2~b3+f(0fm!4u)#$O8(lpKy?c#Sd5y!0249H~@N$!Vu(blknIolkJ&TF z*UexX#~1gl%quESUH=hnNx5OHQ~FRf z?g;H<0R|zX$d58MfgBFLC>C%PVFbpQOCeKa1yCo3WV+Cs0pWv67$E|72wod)d)?uq zb}}J#=#hB7j5cNl?luMwRd8g&gApHt0m0WW|RLmC%&6VP}U=*WdVBq5Wt z1UDyu6UGjtLfJri2>DKobds2o3!bsAL*M2cgOfp=QKJ?SNOb~XL&AYM6K$Yjk`yQ_Oc7+@lkj+IflaR^f*(I8!@#*MVas zQ?1H8OPesoX?H4zv@-9@>csl0o@IiUd1^~1Y0jFnw122{wbnvTcT?}NogjY*j?ljp zv4iUX1XvAHZGU1XUN!OaUh;lvqxwiHrbff^tuCC9Vt* z2Udy$r*K9BX2e4H*r)!ng`F^gq zliFmMnqP9w3Abo7%FF*cO#SjMo_JHKEm6N(tIt#yT&3-egkp@E(P~Uqn@_=@y4GnU z-p}o~z0T=yaUal5;4wz1k6))vR~^q`?(6aNwc#^Mt1uy!AYcz9Po_fJu`b6d5 zN^1P-SI9vAbd8p)rC0I~KdrQhYVU0R4lZ(|T7I5BPR;+KHotnT!S!uY zr(CTKAHKBj419n^C8exeiQu1kN*|-l$9bFSw*whhPd4{U+IaqGY^$;N2eSi%&sNEe zhN*S8X(QF;8?>S74`0#_99*euZfEJ2qyFtWt$U!H2zj18>+7-ZdQ9}YCu)a?*a?~(fEgjFP_kwe6#y?W zN&!;j44E659RWcq9e{aB=HYUI;fVcqRqD1X=LRhvyoXx*ls=(a{_?QAGX5s@;tkq( z!()Fw5SM}19Hl;cqgK3!F{s~{+5zel>$TC_p6C3DSmqmF)J7|{o;lXe{tVsEuuBUEo8)i1N zsL$T2&8*wj4@+(AxZ!8!eJi_rmbcAs6B#1U{lVD&pun;6IQF{^K>Ho|m&V(*w)%rC zxLun#OvF|3(%vQU2Xuc)htB_swnyE=aL2`l`$4_jdqkk^uDq35tEDGH&s?-2!oGmIDAGr@I924w> zq}#D8QNWd(n$)-YwIkF!k3c6%DF&Q`T+BhSgQA|Q2>u%I2MG@P3?bnWPoh8sQKWK$ z;Q$%to+1zpm3Fq#$^~n8e?xn*QPdgg%lBz5tB~Km*;))s-C{Ow@!O{^K1_OQQdnn8E5ZW zV-NMY`ytM&HWq+GP&QCos!H;GP-W0UR9KV%6e-}M6f_h_2ssMK9x7A{(A=ec1IPgm zN~*x-CGpB7Nng)dm4g=OlJ%deF1eqmdPzue1ZW5T0e&wEzd+>b&~_{{co{HmR9zn| zO%!KPEkMOFWitY1RK?WfX1#L&)Of$z(g!q6E1y>MIG`(WV9?Ndp(xAPShOTyw*gy7 z&7b5ZjW|GjJS;+17%RkF?5`UlHK0GIs^$jeY|4L7+pCMkNc~t{t0>7j_c95VL{H0?(p0Lvt=#NJI)j9maU-N7}gB zZ+=8sTOMI9=tRSf00a95nSirJDh$A!0+)b(!wSK#z>pxmiI9c505Ksfq6PI>H7E%Z zeeYy-K_9U4_a448lVtzQZ= z0(%7x?*aWgq70>ahM=VuXH&pgAXSisQTM`MLW2i97Ze_-QeAf{_mX6?B1dk1T$?*( z;Jxy*dwcp;<$b76dwAC+`57zK1s~Cq&IsxwLG4mRN$d*Q!8UN1qNE~7X`q54l(R-@ z57`btOPiO7e95SxBxv+Olk_fV5;f4|V|th8t9D#j$&RA-dO|d~*k`Qj7DNPnT>a<0 zao_3P%THUpv}cw4m};@Q!iqEEIQ29OuYhn>;+CQTM<_O<%R$-EW4 zbM=?S1upAO=Sbtutl8z-OZQc`v}t3i5?SO_=MOxDECjn*EyVIOmMuj&H||@3Kbl4flmQtwGnbcw!lmyNr>@@NdjPxJ#oxMz-Xi8 z0|$-KhkylFTJurw{rV>l?)eUHKWH^Oz5W0DUH`|+sjbgwyVZX3xOQ)Yy74d^e?;FC z5UEIsAV@)jL7k+v34ImHLtAuYpo|4XoasP?BgcRS2grt%Kzb5ZatpQpGuoqN&$Ii~ z-hNj5gkE0LxL`gCV%$H943Tkzq9Ox8@yj{$=`qYeNssQv#D64it+CWAr|2k^UwimQZ3~BBhyZ@}(6EA4X^!J1N zRJql*qN#S{OWHhLb8FAPqWxp^&?H%!FFrF*mDO*=NL0|W!;-*EOW%+55v1tGNPlom zh;d*;ArcCqYD7|n5ELO95I1IsyHuA4$f|z++AQ_h8`}8VuV2?LnK;0dH#h4I>WT*a z;gM~#dXi73i}SP7yjR7o`YhGcsNY&%WK2~2rN-J3L-b?y+Ka>WK(9HY^%%le*0t;wO0z(f%5f9J_3*!zK zn#NW1xpGsZ>O2M%P9D!#+?^@yWi9SVQD37%GM4*Fng0n$om!V5TL1B@> z;l$|6;Qw<95lEs#V>7SV&uG1-iIz2e?G*Lf$w(EXRj9)zccMiPc|B1CQ4vWa>k6(< z3We!_qQ@|6KJn8*o#ElWh^A;7k;B6astqe>y;(f0P5o$!KDB)6a@Y>E;<8O7f55{& z0t5PZXjv|3-V(M6k|P(X3J)b#4qO9y3AZcek7KMF_sY394=eK_t?CXW6%uN@Ai|JP zA>M&d1>+~bR`ZAGbYPpa*A`h=R^rk|h< zSd%#oYg#>1f5ou&chrhYX@VmaL>J@0U`i)efl)&DuyX|R>g`2ShAf$sDW%U5ND9RZ z4VO4YrBrFCwU)7zePrRT_QHQoQw5e0pYs1v3kLi=u`ls~K>bORIQq9^! zze68FtV4lPKJZLTr!QX2=7g%%KOmIfpaUOO6M_bfMTGl_0YLG%S3voHvSP}y}Hj8?fOr4>~WBv&QMcd zgA^4(9|=7zBlPJe(7SXyBGczFa0qlJ<8?&?1}O@6NmUcm%!m5Hr29oQ_2R&#WZMSp z9cAiMCrB)6x9Z)qw8$-8iWe{AFYhV${k1%Zl(?sv`e$}2%hsY!vUEpX_jlC3SJ47> zm!-c|kLobnXoh8!0-FR6jLI9PogN$MujlBq#@GQS&0qo4L5T4=V8ni;k0{ZyO6^(u zsH5*u7q*a_*7{t1jb8TOEQ~s59fu5gjwB4h7fM8K5SV1kDF;XLge-_CyyBeP2Qrd% zr~buZQ8JVlyePhOn7TI5kCE$j3M~xx2D=3jhu1*r=@MEHgrF!A-2;h2jG#7{MhtZAB!9l2n=9z|7+Q$mw z27VFA4|15mqODN4Sgtl&T^Z^75BR)Iq28!oj`Wu57I-0QB6^C^IpV)i4y1M!-K1xW zkU&2fF*_iSia9UH%LiMPOU?(1-M?>dAQ|__Ny^ zXay$rqlUI{eelRadm`kbhz1G)4v;2K6p2V~Xn#W;75lPe!fIOJ0C?K!CZ9h*Zys8u z$hZx;!L-_s4$yBOQGcu2=Rd69U??XcJ`%+s9c3a!>M>b^qL~pC6nW&gT(6RGBR6M@ z5wUTe&|Jb_lU?y2{ze&paguuSNPSUtu}%utkc?6qZDHam6sSE!TPkhB$ZA|@OJqkh z9wV~EcL_OxZKfe5oJ+v1K$30k|7z5c`iR|_d_LDwSkHTCqV#w7LDLj7mJ|IaScPpYDfXn$#U;gx|a zPzvWfw**&dIkbqq4D6t{pZd_Fa3{FYe@Cs0R0|{JDXKU*092s-I94}B+FtXI)3-LN z(Tns&UF4;hATkeJB*;wwY=E|eLq{e8CIDkk*DKjCa&KOS41w-A(mUv0KFDSOG~Q3RD&*t~dW)ZW+U* z?5vknN%zj}ELEwr(+^!(`_U`)Hg&>c-Kv~U?aIaaD(!=Nq}f?_U!H=Bb>}PFTv4@E zPf;w5SLNHM7u0>Pt-78ASLjXQ8E=jCsA?M@)0eL}D_=QSI0wplP}`$?e$IIHm-$do zu7Y|wCDo^?DA5o2;GSC@ne!;~GJU+dXR#FLu|7(DYZAeU;UB|gMu*(>#9eU1Xs0`j zS6|Ka@Pob07*q)-=lTumt@*J1gDcpi<`nt?yDqrBSAW$jg+6P*TY5@6O}}Q>-M&(= zNZaD%e|kP@ZALieTLXb z_X7$H~I{9b*=U><+=3O(G#D>%JRU%l&Y z@m@B9e>!;3MrsRI>NS1W-CqVyZNn&r&m5e+f~=v=hwkJ&rTcZmnfd_}>R^%t{Xsx$ zSx>p*Clgir0$6asXpqE<}Q#!v2x&zb|8)}2*9 zKPz6@UC&2gXqzBtuDDg-UyWK#5p(5P`hmN6Ejxd!jZlZ3t(y{DMvMWdUDeZ5amLk@ zkWGuJ=FK*vxjbP)v)a4{m;1A`^;Nt0p$7Te@siaK?BZVPZ>#kWRZltXT~AppW1a96 zF~6+wD9D}dm8Z(r=v@<5oRRmf9vG}KX8`6?8`kLEJOAZto`%Yv^eH_UvBO`lka=}^ zs_(UHU;VHh;KyyY`p=Q{x?$~rm5L!zBuUfv^lfekA>s`6sk;q$!CA%x)%^p6N$0Lb zCDD8iWp4f{PVD=J+t{+k8Y+N;bP+ztT!j<{gMyAM55Fo{1z-ftFG0e%IXEI3Cg6%d z7a(M+)*?!1hHlVn&R-7(wbtX*@)2lsde-V~Q|pJMUjJ3PhdF0tBF{$*U9OuyW%Rq$ z4nUTBi>wG4_yy!ERbvl0d`3+J;)yf|UM!&LAVC~RIRe83h77$(?+8C6Yyc@SG=!!& zS_6m!^j~keo<*y)ct3kopSV~3A>cJu^~keX1CyP0d^VBu+`z+c7iuHII6^gD2&DDO z!8B&0y{7MwW==G!gcV8x5QbbHg=!3!Lyk;?8Q>9RBXV03G z1bFeQYe3E6Fyu)@0Py|M5mHt{#2_q2 z6Hm34Atq-05j{1yKv*K)+LlB@=)M%Bn86@P#$-6)AG*!5 zn9e}DM(G>_l7rg8pw!w7c=(=sMyq+B)}Nzuyv~5A%MKIF6@{$;1qI!JP|0A=G=)+B zG5rQ5w2TkpYBYJVBXkM?j8b%g3?XW!TnyFLE~B-Tu|Wgf=yh6Ud3S|wms?(&rkXDY zGCcHiRJ`kTeNk=IXY|hwA776;JK}H{^;Z43PC!j&crly{Oi0Lc7A+<55$cN03~EZi zN|~p?|H5v`03PPJ$z(Yhh~zlsL#MW=fog21y6;a$Ywa(W>z6fXwtBh|Z$LLBU;-Sj z7y}D$(LVy(!}c>}5X_x(NhOBqWmKlj4q{>uZMDF*)M1_#iJ8wR-?^xMX|?2PeR_Qw zl})jUl9(<{2Kz9lg@Y6^cZ=s^3JPEgWIExSraJ>oK?UJC95c(un-7j)}{2k5va= zqdVoDKcp9&TG62+g&`%}y;lGBE=-f&D@{A^2K}h_4)QV~L``oq#;Hr! zU@tf1dXt)ZD!=MFgYv_^Q9oC0J`ceNJw5QtWIZO6Dj6|D_acY_)5qxXq0tS*JYjGP zD3=H1PazCC&P+xcz3G92k*kb^0(hJ{buQQa{z9^fy?!%!TX#=ah3{>zIsey*c~#`A z8dF5VfzxBak^0Mey>CK249w}xmlfSh^Qv#eRX1(WyN2Q!D`Hozi}2&WKOW-g#;5cl z>hBu}h~In)DCoeO^aX0&5>%gGyb&z;hWqg^(n<;2PlqS{EHVv1nxF*Iw`5KeV;R6h z=vhFb0!2;>1fV9O4JPhDjnRWy?=a=HhpSW9>qiZCj?I`eZLn+R(e?TaNpyVP;MeQ* z&p6`3X~L{taeGjcHlH1SwCx26_mmnh--kc^>IVI9EYE zJCSMBAxr5?qpHI@5ZLGzN83Z4=T!QF*4m5x`aB(K+)@h{FvWs?A!rrYzmT{oT`D-P zg0TXKj^JK|Yms?!tRUq6v&QnVZce!wZITjP$ii;A^psZToOtRG#HQ)36abbO-#2w z3Qz8_db2TFZWJ;4ZB&huf5Rtn^A_qZxB^BsN>c;qn)DNLVMe^cyo5RPR*A~AsL|go zjd}2EV81Cn6|_aU^}z|4aFh+eq;( z^AYIvj47ySegmYYk(A#M4`{(=t`?7kH-Sc@yKeB93{}6lkKVtLe{Pkf8KQo3AH2r= zZvha$+)Yj)D`NvVLS-!`X8_N@!!d#h5LTEb zQAlu9SUZ}e>CytumL57%O~$X1iO@2~bOo`m>FUPK1c)k8`rhVU2}o7NKA^o*uAe_t z{dpdl;CCL{;=2!) z*u!dfe_uava0bATNAyo?T3Q}GDd}XG2vHnW0VJKr=rv4|lPoC_OV!1<7aMU47}+Oz6vH;zP9yYx}LOOx6_gjIoB_$)Qqul`m4$F?8Ue)*Vwj5fB7 z7`_wytX_FW-&dXXV|};1`PbVf=St@K1Et0wb8cN72q8hm<{|mLsPz_0b(U#{eS6i43KvGa)`S3PkIZP|Hcbwrlg#v-bS}bZ6+ht z=?lhMV5DgaB+zn0!ZfiWp)!NfC$&V7$wUk$YOy-isK2Fu)<>XPuMbJCt^2b+THo<* zoBj$!b>&$^eDX3>#T2JhRaD4S*4QsO1kqGx#kvG2d>q9!2?a3;HdlOV-CyT^-tHYPj@M_JZvzd*(9KYHZa|eI+DVZK`5e|LEuR; zM*6|*AcU8kJ1RqV4;@5Z)}~(mZ@qo{vNWnWZ|f6{-oCsNS->&Mh;)~nD1mK;vxB7; ztrOEa=+4GSi1acjg=_^~tB5?9PsL>6oMDV^B_kv>H`HE#Ti?5WaL^N~t%fmeto<%o z2QKVHX6GPE`IEKwra*6(*80i%&by3`9%Ft6%BYH)NejvUKRwW~#BV9%T3dK%xz*qx>E^IW~W&}I* zo046?6+$zrjgNq(y)e>vz!Aq$OTyHU2hWJN&Y#_V-D~ z7aMn30IGPBw%v^eHDrcy$S%809f=>rx0{>)2Q>h>hB0wq!#Jq_6FD9Pz&ns9 zQp>d&$4^|<7pDMib9RQ9*WQiv=)Y$fb5+w!qhlApmE;b!2WJ{z({}NV2GfLXNc@|- z8~X@~FiJJeG7eEkevQ!--@F+7fShHxx_l4gL+Y5n;EU1cqa3H90r4t@F%nATK+FJR z3I-fMVlorF6~Q@p4+I)yI^rf`075(9$du7?=ruNfgJ5#rp2lJ759gtHqda5=SwO4+ zXF!dPnw1eLq}L3ChT;RM!qqeG4>=)_Das6{!HZmmx%8-VsyXOVe+^OZ3^OLxF5Sym zSa)T@)V+Bq1?gY+Ge!@Lv8&CRYkZfisDcWtAkZqMeH~b1V9@)B z5j|;7EUrRe1(};HLy#-qA)$1Ahw)}*_VV%@f6lYq(P@0UTy`~46t$8I5={36yeIq? zdKBbnRM4Ibh33;DjJk*f!efk`j6IVXRTQv?4Jgu8|8c;D$nw$cogDkxo=0LBaSs(i%`W22Ud7k**4nqqWw)bFeW^Qyot- zv9Ta<$e1-zAXCW@DrG7`HLS*B&&V9f+JndHDy^4Q=(G6AX!?yGkG zA;#po;L651qFva(_VA&`A@%!oNUITzCg3`fzX8IsLqM6L#9_b1Qz29Y2`ky*2B zH5II=V7VW*E4@fakc7adh6aRXbclRe0D~)g8M-!DH4jtA9ghCt%)^b3S52bZ8dFy< zXMz?H+f~P$fb6*SW?I?an9?vr4L!nGSbq${A0-}R&8XLrI1+^relaS8$sYg^Xm98z z;st>>SXYLOAkdMKfXJUXV0k(7-&$q#sHJlc>qz6{^<6>&!)R2{2HK@*wjn0~5|kVS z4H2>`P898DFd(#Wab4hfx+vI2*?DH!7xl5TbB{7^t6vU;(8N5-!A-_=hSFs{ok$e$KXBceHVB{kjY$;YihOKoH9UnKOu(>?$E=ghdfiE}AQ!t~ENXP*pKv701dHTUx+m-W;cZ1{-u$ zo?7@llID(&84uU5CO{3@7jOVsGh@eaa2$B(I&c%z5HgMh|HjTBnnFoI9~nc15UPr@ zn*OT`9&WAOcD%7h1LL4s3o(Qn?7sb%KxNQpjQ9ee5&8&ZOC(>+g~P2?N

    6QMPPd71cX zz%a-IJRy}Z%k=FJ&*?5&Xr+g5F zF=GsF$1ihfPB1-63{*r^OSuEhA+QnQ10Rl4r#Qs8Qk))4QZ>glP;n3V#sfbko_;+x zI?Ii>FYFMlKNB3~OA1UK#4e#ZpsbL@G3gG)8g&8bx)3zf-_SA~FJf%OR9MPS9}^mw zpIQ4!Vq86J(BI3@q}s(xjfI2h-y6M5IO;wUw>0A4wb8Za<;IOwxn50{sxetA(KJ(O zfwrgv$-k&OX?{zYUWO){VgcDA$-EdG)+^=+E>7bZ0~1MX@r_K-W~>vSt3TNE+NAD1 zoV|Wzg)vP;#kSgf5`zZcSz&xg)4X!y%@9U>zXJS`I$0-4V%mr2lE#hJd#L9GMDNLJ2JBN#uvxI zZ46VNzDsW|3u~f!s@E8*UOtMkAy-@TYfr8=9%vk}%?r*ohSu&r-}tj$j(mZ)M<{G@ zL`3Jwp^;(qJO+6U)G0*>Kpaz~nEr?L04$?R;qG*w;jyuRte|?J2xIT1@5z z$~5DELX;|~p0fehjow*LA&}M_DL$Ckg|=dfNDpSz)?H+LR5SQ#9Z0Me;GK(&Uk}=( z@mZs#yt#KlCtyDf$=Hzap|l);zf+eo3X!=QReupKO@>5ccRZ+6oifM_zR#n!w`nSt zZ?05W+oPW~Hk7w4WF{N9IW?#ZHDVT=kbP`0-jSe9Cm*~g>j3=D>@tZ3G<-sQ;Zez4 zpy{o!%1uXKVvO1`o0_J=ON>X$|Gsenp9jH@Gv6B+oB_YW^YZ00sCM)YsK!Wx`M3tX zHfZzNS%NIpHy{ZjA_zwy)+>yL$+g=qH8#tk+j@?^Xg-gKBO=w4&$R$q@Kp#0;*&4| zU>lO?!k9DNl<#WM)`}30qzrt=rWu-qxjsw{24}RIc9}7A{JTiDYGfT_#iprqE+Z%| zJq8oqb1@tA(a#%WH1*gOfc^Z*QEJmS0LifjLIb`BJb{UA;6V&A=7=KB1{3ESFvMPj z%r@BxW5bXwpnAupLamUbMQY|ODz2U{7|)ka9I~K`{6RiahxI_p^Y7pvBvofikno{; zz|r!Z7J+LRuSF1(uW@nLkX%7PX)d>cJpRpf#y6Ts)VJ=XFY4fP88k96yA{lzodN^n z%6uRXrk$vTDiC=AXeKzB40V;XitCVRgAf6NNtW4C&VF=n_3D>!!^eM-TjM37)S%nO`CzzbewTNw~)~SGi8^{`QopkC@F)*19y*J-`fCT`W!$IR{ zNyT_Sh%_N#{(8M}NO|v7R05PBQC2ky!lvNUu`ha!Xq84LKvD#V$8=8|Gua%S;4BCs z67GUi$jDRn=Q@&%>N$UR1Gd$FA9bPhK5^3M5&%VJ8R>6zn1)JPCtnBxPLpZa2&efn z2cvXk0J3~v1Y#=12{Hvv2IkkVx$#DF-Q`M@-V|KyJTt?Ye~aKh#0KDFVYcXJ=389= zbR?G~Ye+kxTtFTLxMuh==oTy|lML7%b>4dHlJiTQDLf>o6i1$P2NH+=c;@CoD}h^y z;z05!mdC<5;z8>>s`S<0r`BR3g0G;J5!2=7}v|8`^1uKl3M_LAbTn&Uu7&%`?ZC48*dxxsBZzSGax_^W6@_wUoYO3k@qla#Ag|R zPvu9)G+~+igq}ElA~Rm;@fU8BIk0tx6oE! z5G;-0Bxv-m69i=0zDoQYobJsA^lI2)$&coqKda;q#>u4;nBP} zC!4D=CtILmX)YjPfi*{(K{%j%26y3mit_!coSqr_SAw7(1@q0uAyS~-x!E{T9rU_H z7P{*hget8p^hVI##*AK)2yBsJ2y&H72d);#gM%rprqcB(!ZgG#VXoHhO%W}hJy8Gl z0|LlX58*dRggYVKL1ggO&>6@GkW$l6BP}!>L*`WDOPLeRh~=1QdKbSz}sf z7ZzVRTs{0fD=|-cssMVxNstH-+i5c6 zOGAztdo_x;d+(Vk@BaITjb3@kE(V-1;?*I2TTzYQNuni27vc4{%9!$$@-5fH%h=fsmOd$op{Ufxdyk$wwrF(eMUU zMwpOqa?&71a3hE?5R@UV#Gk!utmQ8n%cuVjt&#GL2R=2!d`nu~sbghaIP?Y*I-oNN zaM0|;OY}F)YX~p{R3{W)C{;)ms~+pxDUTb+H>!Q+$X70?ASguzD#GdGsChSbn`Z!z z!?ksc^w`cv7(t%~8JzJyQGlGsgLY?8|G<(!Q;)K>N4;8Dmkc_^Hu1 zSgkth_2-R!)sIGKqiUmnX56ijYd`qBVGWibHU9XUJN@PTYCrzD!62!7k4M|}&Z|a~ z`r{VkHMPDVW?<^LV6uC@N6W&p0u=asH;1nq08}vYoH-w)5Y%jZDweXJQT_5+1(`iA z1-+8bb4(b7!{8Vz6^%4ytBt=f_LBs3h#J?0tN#2ooZ$t({1c)l@B~Uk z`kSy0`4A|3%M=Qz62>gxT+kaJHH3r%-QW`j%%NeF1`0$hfY@M~bs%#KvHFc)gAH@t zX4Um;mg|gP8Z+u2ubQgvea-moKqbI0hpA1k8(ZG>`$_7hHz--?nJuTvBMz;-w$(U# zCK{ior^B|0ghxIdBA)`Luq9^k9YcnBfS^I=RiEdBl0=|FODKuGj2V#6zDfHi<5ubo zc-vs4_N6`P>D3~ogR2IJIPimaeRfU2#COIw+&FUEqDohJr6}v#z(BI*i49W*+;~{q zZ(_sE4eH6~Pz2&T$+MXcAhj)q?#F!rA*uvk3k9OKkQC_;IRx242n z5Xb+t7dIEWBBir( .fallback(|| async { "404 Not Found: We're past the event horizon..." }); // Only allow current device to access it - let listenera = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).await?; - let listen_addra = listenera.local_addr()?; - let listenerb = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).await?; - let listen_addrb = listenerb.local_addr()?; - let listenerc = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).await?; - let listen_addrc = listenerc.local_addr()?; - let listenerd = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).await?; - let listen_addrd = listenerd.local_addr()?; - - // let listen_addr = listener.local_addr()?; // We get it from a listener so `0` is turned into a random port + let listener = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).await?; + let listen_addr = listener.local_addr()?; // We get it from a listener so `0` is turned into a random port let (tx, mut rx) = tokio::sync::mpsc::channel(1); - info!("Internal server listening on: http://{listen_addra:?} http://{listen_addrb:?} http://{listen_addrc:?} http://{listen_addrd:?}"); - let server = axum::Server::builder(CombinedIncoming { - a: AddrIncoming::from_listener(listenera)?, - b: AddrIncoming::from_listener(listenerb)?, - c: AddrIncoming::from_listener(listenerc)?, - d: AddrIncoming::from_listener(listenerd)?, - }); + info!("Internal server listening on: http://{listen_addr:?}"); tokio::spawn(async move { - server - .serve(app.into_make_service()) - .with_graceful_shutdown(async { + axum::serve(listener, app) + .with_graceful_shutdown(async move { rx.recv().await; }) .await @@ -96,12 +79,7 @@ pub async fn sd_server_plugin( }); let script = format!( - r#"window.__SD_CUSTOM_SERVER_AUTH_TOKEN__ = "{auth_token}"; window.__SD_CUSTOM_URI_SERVER__ = [{}];"#, - [listen_addra, listen_addrb, listen_addrc, listen_addrd] - .iter() - .map(|addr| format!("'http://{addr}'")) - .collect::>() - .join(","), + r#"window.__SD_CUSTOM_SERVER_AUTH_TOKEN__ = "{auth_token}"; window.__SD_CUSTOM_URI_SERVER__ = ['http://{listen_addr}'];"#, ); Ok(tauri::plugin::Builder::new("sd-server") @@ -127,15 +105,12 @@ struct QueryParams { token: Option, } -async fn auth_middleware( +async fn auth_middleware( Query(query): Query, State(auth_token): State, - request: Request, - next: Next, -) -> Result -where - B: Send, -{ + request: Request, + next: Next, +) -> Result { let req = if query.token.as_ref() != Some(&auth_token) { let (mut parts, body) = request.into_parts(); @@ -158,38 +133,3 @@ where Ok(next.run(req).await) } - -struct CombinedIncoming { - a: AddrIncoming, - b: AddrIncoming, - c: AddrIncoming, - d: AddrIncoming, -} - -impl Accept for CombinedIncoming { - type Conn = ::Conn; - type Error = ::Error; - - fn poll_accept( - mut self: Pin<&mut Self>, - cx: &mut Context<'_>, - ) -> Poll>> { - if let Poll::Ready(Some(value)) = Pin::new(&mut self.a).poll_accept(cx) { - return Poll::Ready(Some(value)); - } - - if let Poll::Ready(Some(value)) = Pin::new(&mut self.b).poll_accept(cx) { - return Poll::Ready(Some(value)); - } - - if let Poll::Ready(Some(value)) = Pin::new(&mut self.c).poll_accept(cx) { - return Poll::Ready(Some(value)); - } - - if let Poll::Ready(Some(value)) = Pin::new(&mut self.d).poll_accept(cx) { - return Poll::Ready(Some(value)); - } - - Poll::Pending - } -} diff --git a/apps/desktop/src/patches.ts b/apps/desktop/src/patches.ts index 8fd5d4500..f1c33a0a7 100644 --- a/apps/desktop/src/patches.ts +++ b/apps/desktop/src/patches.ts @@ -1,4 +1,4 @@ -import { tauriLink } from '@oscartbeaumont-sd/rspc-tauri/src/v2'; +import { tauriLink } from '@spacedrive/rspc-tauri/src/v2'; globalThis.isDev = import.meta.env.DEV; globalThis.rspcLinks = [ diff --git a/apps/landing/package.json b/apps/landing/package.json index 23fedacdf..6cc22e363 100644 --- a/apps/landing/package.json +++ b/apps/landing/package.json @@ -49,7 +49,7 @@ "three": "^0.161.0", "tsparticles": "^3.3.0", "unist-util-visit": "^5.0.0", - "zod": "~3.22.4" + "zod": "^3.23" }, "devDependencies": { "@next/bundle-analyzer": "^13.5.6", diff --git a/apps/mobile/modules/sd-core/src/index.ts b/apps/mobile/modules/sd-core/src/index.ts index fb034782a..dfb338f36 100644 --- a/apps/mobile/modules/sd-core/src/index.ts +++ b/apps/mobile/modules/sd-core/src/index.ts @@ -1,4 +1,4 @@ -import { AlphaRSPCError, Link, RspcRequest } from '@oscartbeaumont-sd/rspc-client/src/v2'; +import { Link, RSPCError, RspcRequest } from '@spacedrive/rspc-client'; import { EventEmitter, requireNativeModule } from 'expo-modules-core'; // It loads the native module object from the JSI or falls back to @@ -15,7 +15,7 @@ export function reactNativeLink(): Link { string, { resolve: (result: any) => void; - reject: (error: Error | AlphaRSPCError) => void; + reject: (error: Error | RSPCError) => void; } >(); @@ -29,7 +29,7 @@ export function reactNativeLink(): Link { activeMap.delete(id); } else if (result.type === 'error') { const { message, code } = result.data; - activeMap.get(id)?.reject(new AlphaRSPCError(code, message)); + activeMap.get(id)?.reject(new RSPCError(code, message)); activeMap.delete(id); } else { console.error(`rspc: received event of unknown type '${result.type}'`); diff --git a/apps/mobile/package.json b/apps/mobile/package.json index 5dc682437..525c790ba 100644 --- a/apps/mobile/package.json +++ b/apps/mobile/package.json @@ -21,8 +21,8 @@ "@dr.pogodin/react-native-fs": "^2.24.1", "@gorhom/bottom-sheet": "^4.6.1", "@hookform/resolvers": "^3.1.0", - "@oscartbeaumont-sd/rspc-client": "github:spacedriveapp/rspc#path:packages/client&bc882f4724", - "@oscartbeaumont-sd/rspc-react": "github:spacedriveapp/rspc#path:packages/react&bc882f4724", + "@spacedrive/rspc-client": "github:spacedriveapp/rspc#path:packages/client&6a77167495", + "@spacedrive/rspc-react": "github:spacedriveapp/rspc#path:packages/react&6a77167495", "@react-native-async-storage/async-storage": "~1.23.1", "@react-native-masked-view/masked-view": "^0.3.1", "@react-navigation/bottom-tabs": "^6.5.19", @@ -32,7 +32,7 @@ "@sd/assets": "workspace:*", "@sd/client": "workspace:*", "@shopify/flash-list": "1.6.4", - "@tanstack/react-query": "^4.36.1", + "@tanstack/react-query": "^5.59", "babel-preset-solid": "^1.9.0", "class-variance-authority": "^0.7.0", "dayjs": "^1.11.10", @@ -74,8 +74,8 @@ "twrnc": "^4.1.0", "use-count-up": "^3.0.1", "use-debounce": "^9.0.4", - "valtio": "^1.11.2", - "zod": "~3.22.4" + "valtio": "^2.0", + "zod": "^3.23" }, "devDependencies": { "@babel/core": "^7.24.0", diff --git a/apps/mobile/src/components/browse/BrowseLocations.tsx b/apps/mobile/src/components/browse/BrowseLocations.tsx index 75787d7a4..4159fbe7f 100644 --- a/apps/mobile/src/components/browse/BrowseLocations.tsx +++ b/apps/mobile/src/components/browse/BrowseLocations.tsx @@ -1,4 +1,5 @@ import { useNavigation } from '@react-navigation/native'; +import { keepPreviousData } from '@tanstack/react-query'; import { Plus } from 'phosphor-react-native'; import { useRef, useState } from 'react'; import { FlatList, Text, View } from 'react-native'; @@ -22,7 +23,7 @@ const BrowseLocations = () => { const modalRef = useRef(null); const [showAll, setShowAll] = useState(false); - const result = useLibraryQuery(['locations.list'], { keepPreviousData: true }); + const result = useLibraryQuery(['locations.list'], { placeholderData: keepPreviousData }); const locations = result.data; return ( diff --git a/apps/mobile/src/components/drawer/DrawerLocations.tsx b/apps/mobile/src/components/drawer/DrawerLocations.tsx index 2e1d8995c..0a5f87559 100644 --- a/apps/mobile/src/components/drawer/DrawerLocations.tsx +++ b/apps/mobile/src/components/drawer/DrawerLocations.tsx @@ -1,5 +1,6 @@ import { DrawerNavigationHelpers } from '@react-navigation/drawer/lib/typescript/src/types'; import { useNavigation } from '@react-navigation/native'; +import { keepPreviousData } from '@tanstack/react-query'; import { useRef } from 'react'; import { Pressable, Text, View } from 'react-native'; import { @@ -73,7 +74,7 @@ const DrawerLocations = () => { const modalRef = useRef(null); - const result = useLibraryQuery(['locations.list'], { keepPreviousData: true }); + const result = useLibraryQuery(['locations.list'], { placeholderData: keepPreviousData }); const locations = result.data || []; return ( diff --git a/apps/mobile/src/components/explorer/Explorer.tsx b/apps/mobile/src/components/explorer/Explorer.tsx index 10b448d7c..d3281aa7d 100644 --- a/apps/mobile/src/components/explorer/Explorer.tsx +++ b/apps/mobile/src/components/explorer/Explorer.tsx @@ -1,6 +1,6 @@ import { useNavigation } from '@react-navigation/native'; import { FlashList } from '@shopify/flash-list'; -import { UseInfiniteQueryResult } from '@tanstack/react-query'; +import { InfiniteData, UseInfiniteQueryResult } from '@tanstack/react-query'; import * as Haptics from 'expo-haptics'; import { useRef } from 'react'; import { ActivityIndicator } from 'react-native'; @@ -32,7 +32,7 @@ type ExplorerProps = { items: ExplorerItem[] | null; /** Function to fetch next page of items. */ loadMore: () => void; - query: UseInfiniteQueryResult>; + query: UseInfiniteQueryResult>>; count?: number; empty?: never; isEmpty?: never; diff --git a/apps/mobile/src/components/explorer/sections/FavoriteButton.tsx b/apps/mobile/src/components/explorer/sections/FavoriteButton.tsx index 5a09e515c..2dd851b7a 100644 --- a/apps/mobile/src/components/explorer/sections/FavoriteButton.tsx +++ b/apps/mobile/src/components/explorer/sections/FavoriteButton.tsx @@ -12,7 +12,7 @@ type Props = { const FavoriteButton = (props: Props) => { const [favorite, setFavorite] = useState(props.data.favorite); - const { mutate: toggleFavorite, isLoading } = useLibraryMutation('files.setFavorite', { + const { mutate: toggleFavorite, isPending } = useLibraryMutation('files.setFavorite', { onSuccess: () => { // TODO: Invalidate search queries setFavorite(!favorite); @@ -22,7 +22,7 @@ const FavoriteButton = (props: Props) => { return ( toggleFavorite({ id: props.data.id, favorite: !favorite })} style={props.style} > diff --git a/apps/mobile/src/components/job/JobGroup.tsx b/apps/mobile/src/components/job/JobGroup.tsx index 26dc832ac..9000a7b0a 100644 --- a/apps/mobile/src/components/job/JobGroup.tsx +++ b/apps/mobile/src/components/job/JobGroup.tsx @@ -191,7 +191,7 @@ function Options({ activeJob, group, setShowChildJobs, showChildJobs }: OptionsP const clearJob = useLibraryMutation(['jobs.clear'], { onSuccess: () => { - rspc.queryClient.invalidateQueries(['jobs.reports']); + rspc.queryClient.invalidateQueries({ queryKey: ['jobs.reports'] }); } }); diff --git a/apps/mobile/src/components/modal/AddTagModal.tsx b/apps/mobile/src/components/modal/AddTagModal.tsx index 929820c4c..ae69a74ff 100644 --- a/apps/mobile/src/components/modal/AddTagModal.tsx +++ b/apps/mobile/src/components/modal/AddTagModal.tsx @@ -35,8 +35,8 @@ const AddTagModal = forwardRef((_, ref) => { const mutation = useLibraryMutation(['tags.assign'], { onSuccess: () => { // this makes sure that the tags are updated in the UI - rspc.queryClient.invalidateQueries(['tags.getForObject']); - rspc.queryClient.invalidateQueries(['search.paths']); + rspc.queryClient.invalidateQueries({ queryKey: ['tags.getForObject'] }); + rspc.queryClient.invalidateQueries({ queryKey: ['search.paths'] }); modalRef.current?.dismiss(); } }); diff --git a/apps/mobile/src/components/modal/CreateLibraryModal.tsx b/apps/mobile/src/components/modal/CreateLibraryModal.tsx index 346988473..83d005bfd 100644 --- a/apps/mobile/src/components/modal/CreateLibraryModal.tsx +++ b/apps/mobile/src/components/modal/CreateLibraryModal.tsx @@ -17,7 +17,7 @@ const CreateLibraryModal = forwardRef((_, ref) => { const submitPlausibleEvent = usePlausibleEvent(); - const { mutate: createLibrary, isLoading: createLibLoading } = useBridgeMutation( + const { mutate: createLibrary, isPending: createLibLoading } = useBridgeMutation( 'library.create', { onSuccess: (lib) => { diff --git a/apps/mobile/src/components/modal/ImportLibraryModal.tsx b/apps/mobile/src/components/modal/ImportLibraryModal.tsx index 82de9fdd4..f15edb421 100644 --- a/apps/mobile/src/components/modal/ImportLibraryModal.tsx +++ b/apps/mobile/src/components/modal/ImportLibraryModal.tsx @@ -100,7 +100,7 @@ const CloudLibraryCard = ({ data, modalRef, navigation }: Props) => { diff --git a/apps/mobile/src/components/overview/Devices.tsx b/apps/mobile/src/components/overview/Devices.tsx index 8643d08af..1f65407b9 100644 --- a/apps/mobile/src/components/overview/Devices.tsx +++ b/apps/mobile/src/components/overview/Devices.tsx @@ -1,5 +1,5 @@ import * as RNFS from '@dr.pogodin/react-native-fs'; -import { AlphaRSPCError } from '@oscartbeaumont-sd/rspc-client/src/v2'; +import { RSPCError } from '@spacedrive/rspc-client'; import { UseQueryResult } from '@tanstack/react-query'; import React, { useEffect, useState } from 'react'; import { Platform, Text, View } from 'react-native'; @@ -16,7 +16,7 @@ import StatCard from './StatCard'; interface Props { node: NodeState | undefined; - stats: UseQueryResult; + stats: UseQueryResult; } export function hardwareModelToIcon(hardwareModel: HardwareModel) { diff --git a/apps/mobile/src/components/overview/OverviewStats.tsx b/apps/mobile/src/components/overview/OverviewStats.tsx index 2a7ecd0de..3ba73a8ce 100644 --- a/apps/mobile/src/components/overview/OverviewStats.tsx +++ b/apps/mobile/src/components/overview/OverviewStats.tsx @@ -1,5 +1,5 @@ import * as RNFS from '@dr.pogodin/react-native-fs'; -import { AlphaRSPCError } from '@oscartbeaumont-sd/rspc-client/src/v2'; +import { RSPCError } from '@spacedrive/rspc-client'; import { UseQueryResult } from '@tanstack/react-query'; import { useEffect, useState } from 'react'; import { Platform, Text, View } from 'react-native'; @@ -47,7 +47,7 @@ const StatItem = ({ title, bytes, isLoading, style }: StatItemProps) => { }; interface Props { - stats: UseQueryResult; + stats: UseQueryResult; } const OverviewStats = ({ stats }: Props) => { diff --git a/apps/mobile/src/components/search/filters/SavedSearches.tsx b/apps/mobile/src/components/search/filters/SavedSearches.tsx index e4dacb47e..5a63ac22f 100644 --- a/apps/mobile/src/components/search/filters/SavedSearches.tsx +++ b/apps/mobile/src/components/search/filters/SavedSearches.tsx @@ -71,7 +71,7 @@ const SavedSearch = ({ search }: Props) => { const dataForSearch = useSavedSearch(search); const rspc = useRspcLibraryContext(); const deleteSearch = useLibraryMutation('search.saved.delete', { - onSuccess: () => rspc.queryClient.invalidateQueries(['search.saved.list']) + onSuccess: () => rspc.queryClient.invalidateQueries({ queryKey: ['search.saved.list'] }) }); return ( { diff --git a/apps/mobile/src/hooks/useSavedSearch.ts b/apps/mobile/src/hooks/useSavedSearch.ts index 0bdaa7e59..cf61b0ecc 100644 --- a/apps/mobile/src/hooks/useSavedSearch.ts +++ b/apps/mobile/src/hooks/useSavedSearch.ts @@ -1,4 +1,5 @@ import { IconTypes } from '@sd/assets/util'; +import { keepPreviousData } from '@tanstack/react-query'; import { useCallback, useMemo } from 'react'; import { SavedSearch, SearchFilterArgs, Tag, useLibraryQuery } from '@sd/client'; import { kinds } from '~/components/search/filters/Kind'; @@ -44,11 +45,11 @@ export function useSavedSearch(search: SavedSearch) { }; const locations = useLibraryQuery(['locations.list'], { - keepPreviousData: true, + placeholderData: keepPreviousData, enabled: filterKeys.includes('locations') }); const tags = useLibraryQuery(['tags.list'], { - keepPreviousData: true, + placeholderData: keepPreviousData, enabled: filterKeys.includes('tags') }); diff --git a/apps/mobile/src/screens/BackfillWaiting.tsx b/apps/mobile/src/screens/BackfillWaiting.tsx index ab13c54e7..10f7e76ba 100644 --- a/apps/mobile/src/screens/BackfillWaiting.tsx +++ b/apps/mobile/src/screens/BackfillWaiting.tsx @@ -52,10 +52,8 @@ const BackfillWaiting = () => { const syncEnabled = useLibraryQuery(['sync.enabled']); useEffect(() => { - (async () => { - await enableSync.mutateAsync(null); - })(); - }, []); + enableSync.mutate(null); + }, [enableSync]); return ( diff --git a/apps/mobile/src/screens/browse/Location.tsx b/apps/mobile/src/screens/browse/Location.tsx index 81f9a152c..ca1bff307 100644 --- a/apps/mobile/src/screens/browse/Location.tsx +++ b/apps/mobile/src/screens/browse/Location.tsx @@ -62,10 +62,13 @@ export default function LocationScreen({ navigation, route }: BrowseStackScreenP filters: [...defaultFilters, ...layoutFilter].filter(Boolean), take: 30 }, - order, - onSuccess: () => getExplorerStore().resetNewThumbnails() + order }); + useEffect(() => { + getExplorerStore().resetNewThumbnails(); + }, [path]); + useEffect(() => { // Set screen title to location. if (path && path !== '') { diff --git a/apps/mobile/src/screens/search/Search.tsx b/apps/mobile/src/screens/search/Search.tsx index f0f0a3ad0..bf6dd3a0f 100644 --- a/apps/mobile/src/screens/search/Search.tsx +++ b/apps/mobile/src/screens/search/Search.tsx @@ -1,6 +1,6 @@ import { useIsFocused } from '@react-navigation/native'; import { ArrowLeft, DotsThree, FunnelSimple } from 'phosphor-react-native'; -import { Suspense, useDeferredValue, useMemo, useState } from 'react'; +import { Suspense, useDeferredValue, useEffect, useMemo, useState } from 'react'; import { ActivityIndicator, Platform, Pressable, TextInput, View } from 'react-native'; import { useSafeAreaInsets } from 'react-native-safe-area-context'; import { ObjectKindEnum, useLibraryQuery, usePathsExplorerQuery } from '@sd/client'; @@ -41,10 +41,11 @@ const SearchScreen = ({ navigation }: SearchStackScreenProps<'Search'>) => { filters: [...layoutSearchFilter, ...searchStore.mergedFilters] }, enabled: isFocused && searchStore.mergedFilters.length >= 1, // only fetch when screen is focused & filters are applied - suspense: true, - onSuccess: () => getExplorerStore().resetNewThumbnails() + suspense: true }); + useEffect(() => getExplorerStore().resetNewThumbnails(), [objects]); + useFiltersSearch(deferredSearch); const appliedFiltersLength = Object.keys(searchStore.appliedFilters).length; diff --git a/apps/mobile/src/screens/settings/library/CloudSettings/CloudSettings.tsx b/apps/mobile/src/screens/settings/library/CloudSettings/CloudSettings.tsx index f121eb8c4..892556d0e 100644 --- a/apps/mobile/src/screens/settings/library/CloudSettings/CloudSettings.tsx +++ b/apps/mobile/src/screens/settings/library/CloudSettings/CloudSettings.tsx @@ -111,10 +111,10 @@ const Authenticated = () => { ); @@ -147,11 +147,11 @@ function StopButton({ name }: { name: string }) { ); diff --git a/apps/mobile/src/stores/auth.ts b/apps/mobile/src/stores/auth.ts index 3ef1079d8..336b3ff22 100644 --- a/apps/mobile/src/stores/auth.ts +++ b/apps/mobile/src/stores/auth.ts @@ -1,4 +1,4 @@ -import { RSPCError } from '@oscartbeaumont-sd/rspc-client'; +import { RSPCError } from '@spacedrive/rspc-client'; import { Linking } from 'react-native'; import { createMutable } from 'solid-js/store'; import { nonLibraryClient, useSolidStore } from '@sd/client'; diff --git a/apps/server/Cargo.toml b/apps/server/Cargo.toml index f649ab23b..f4050942f 100644 --- a/apps/server/Cargo.toml +++ b/apps/server/Cargo.toml @@ -16,12 +16,13 @@ default = [] sd-core = { path = "../../core", features = ["ffmpeg", "heif"] } # Workspace dependencies -axum = { workspace = true, features = ["headers"] } -http = { workspace = true } -rspc = { workspace = true, features = ["axum"] } -tempfile = { workspace = true } -tokio = { workspace = true, features = ["rt-multi-thread", "signal", "sync"] } -tracing = { workspace = true } +axum = { workspace = true } +axum-extra = { workspace = true, features = ["typed-header"] } +http = { workspace = true } +rspc = { workspace = true, features = ["axum"] } +tempfile = { workspace = true } +tokio = { workspace = true, features = ["rt-multi-thread", "signal", "sync"] } +tracing = { workspace = true } # Specific Desktop dependencies include_dir = "0.7.3" diff --git a/apps/server/src/main.rs b/apps/server/src/main.rs index 252d45676..5a6304bf3 100644 --- a/apps/server/src/main.rs +++ b/apps/server/src/main.rs @@ -1,12 +1,15 @@ use std::{collections::HashMap, env, net::SocketAddr, path::Path}; use axum::{ + body::Body, extract::{FromRequestParts, State}, - headers::{authorization::Basic, Authorization}, http::Request, middleware::Next, response::{IntoResponse, Response}, routing::get, +}; +use axum_extra::{ + headers::{authorization::Basic, Authorization}, TypedHeader, }; use sd_core::{custom_uri, Node}; @@ -24,11 +27,7 @@ pub struct AppState { auth: HashMap, } -async fn basic_auth( - State(state): State, - request: Request, - next: Next, -) -> Response { +async fn basic_auth(State(state): State, request: Request, next: Next) -> Response { let request = if !state.auth.is_empty() { let (mut parts, body) = request.into_parts(); @@ -175,10 +174,7 @@ async fn main() { .route( "/", get(|| async move { - use axum::{ - body::{self, Full}, - response::Response, - }; + use axum::{body::Body, response::Response}; use http::{header, HeaderValue, StatusCode}; match ASSETS_DIR.get_file("index.html") { @@ -188,11 +184,11 @@ async fn main() { header::CONTENT_TYPE, HeaderValue::from_str("text/html").unwrap(), ) - .body(body::boxed(Full::from(file.contents()))) + .body(Body::from(file.contents())) .unwrap(), None => Response::builder() .status(StatusCode::NOT_FOUND) - .body(body::boxed(axum::body::Empty::new())) + .body(Body::empty()) .unwrap(), } }), @@ -201,10 +197,7 @@ async fn main() { "/*id", get( |axum::extract::Path(path): axum::extract::Path| async move { - use axum::{ - body::{self, Empty, Full}, - response::Response, - }; + use axum::{body::Body, response::Response}; use http::{header, HeaderValue, StatusCode}; let path = path.trim_start_matches('/'); @@ -218,7 +211,7 @@ async fn main() { ) .unwrap(), ) - .body(body::boxed(Full::from(file.contents()))) + .body(Body::from(file.contents())) .unwrap(), None => match ASSETS_DIR.get_file("index.html") { Some(file) => Response::builder() @@ -227,11 +220,11 @@ async fn main() { header::CONTENT_TYPE, HeaderValue::from_str("text/html").unwrap(), ) - .body(body::boxed(Full::from(file.contents()))) + .body(Body::from(file.contents())) .unwrap(), None => Response::builder() .status(StatusCode::NOT_FOUND) - .body(body::boxed(Empty::new())) + .body(Body::empty()) .unwrap(), }, } @@ -254,8 +247,7 @@ async fn main() { let mut addr = "[::]:8080".parse::().unwrap(); // This listens on IPv6 and IPv4 addr.set_port(port); info!("Listening on http://localhost:{}", port); - axum::Server::bind(&addr) - .serve(app.into_make_service()) + axum::serve(tokio::net::TcpListener::bind(addr).await.unwrap(), app) .with_graceful_shutdown(signal) .await .expect("Error with HTTP server!"); diff --git a/apps/web/package.json b/apps/web/package.json index b487de1d8..ad2a51ebf 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -17,10 +17,10 @@ "lint": "eslint src --cache" }, "dependencies": { - "@oscartbeaumont-sd/rspc-client": "github:spacedriveapp/rspc#path:packages/client&bc882f4724", + "@spacedrive/rspc-client": "github:spacedriveapp/rspc#path:packages/client&6a77167495", "@sd/client": "workspace:*", "@sd/interface": "workspace:*", - "@tanstack/react-query": "^4.36.1", + "@tanstack/react-query": "^5.59", "html-to-image": "^1.11.11", "html2canvas": "^1.4.1", "react": "^18.2.0", diff --git a/apps/web/src/patches.ts b/apps/web/src/patches.ts index 2faac6997..a545a1217 100644 --- a/apps/web/src/patches.ts +++ b/apps/web/src/patches.ts @@ -1,4 +1,4 @@ -import { wsBatchLink } from '@oscartbeaumont-sd/rspc-client/src/v2'; +import { wsBatchLink } from '@spacedrive/rspc-client'; globalThis.isDev = import.meta.env.DEV; globalThis.rspcLinks = [ diff --git a/core/Cargo.toml b/core/Cargo.toml index 49aac5516..3d3762464 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -64,7 +64,7 @@ reqwest = { workspace = true, features = ["json", "native-tls-vendor rmp-serde = { workspace = true } rmpv = { workspace = true } rspc = { workspace = true, features = ["alpha", "axum", "chrono", "unstable", "uuid"] } -serde = { workspace = true, features = ["derive"] } +serde = { workspace = true, features = ["derive", "rc"] } serde_json = { workspace = true } specta = { workspace = true } strum = { workspace = true, features = ["derive"] } @@ -85,15 +85,16 @@ ctor = "0.2.8" directories = "5.0" flate2 = "1.0" hostname = "0.4.0" -http-body = "0.4.6" # Update blocked by http +http-body = "1.0" http-range = "0.1.5" -int-enum = "0.5" # Update blocked due to API breaking changes +hyper-util = { version = "0.1.9", features = ["tokio"] } +int-enum = "0.5" # Update blocked due to API breaking changes mini-moka = "0.10.3" serde-hashkey = "0.4.5" serde_repr = "0.1.19" serde_with = "3.8" slotmap = "1.0" -sysinfo = "0.29.11" # Update blocked due to API breaking changes +sysinfo = "0.29.11" # Update blocked due to API breaking changes tar = "0.4.41" tower-service = "0.3.2" tracing-appender = "0.2.3" diff --git a/core/crates/heavy-lifting/Cargo.toml b/core/crates/heavy-lifting/Cargo.toml index 75e99359c..20743e9fa 100644 --- a/core/crates/heavy-lifting/Cargo.toml +++ b/core/crates/heavy-lifting/Cargo.toml @@ -44,7 +44,7 @@ prisma-client-rust = { workspace = true } rmp-serde = { workspace = true } rmpv = { workspace = true } rspc = { workspace = true } -serde = { workspace = true, features = ["derive"] } +serde = { workspace = true, features = ["derive", "rc"] } serde_json = { workspace = true } specta = { workspace = true } strum = { workspace = true, features = ["derive", "phf"] } diff --git a/core/crates/heavy-lifting/src/media_processor/helpers/thumbnailer.rs b/core/crates/heavy-lifting/src/media_processor/helpers/thumbnailer.rs index d1637ebb3..6d919fbae 100644 --- a/core/crates/heavy-lifting/src/media_processor/helpers/thumbnailer.rs +++ b/core/crates/heavy-lifting/src/media_processor/helpers/thumbnailer.rs @@ -25,7 +25,8 @@ use image::{imageops, DynamicImage, GenericImageView}; use serde::{Deserialize, Serialize}; use specta::Type; use tokio::{ - fs, io, + fs::{self, File}, + io::{self, AsyncWriteExt}, sync::{oneshot, Mutex}, task::spawn_blocking, time::{sleep, Instant}, @@ -450,15 +451,29 @@ async fn generate_image_thumbnail( trace!("Created shard directory and writing it to disk"); - let res = fs::write(output_path, &webp).await.map_err(|e| { + let mut file = File::create(output_path).await.map_err(|e| { + thumbnailer::NonCriticalThumbnailerError::SaveThumbnail( + file_path.clone(), + FileIOError::from((output_path, e)).to_string(), + ) + })?; + + file.write_all(&webp).await.map_err(|e| { + thumbnailer::NonCriticalThumbnailerError::SaveThumbnail( + file_path.clone(), + FileIOError::from((output_path, e)).to_string(), + ) + })?; + + file.sync_all().await.map_err(|e| { thumbnailer::NonCriticalThumbnailerError::SaveThumbnail( file_path, FileIOError::from((output_path, e)).to_string(), ) - }); + })?; trace!("Wrote thumbnail to disk"); - res + return Ok(()); } #[instrument( diff --git a/core/crates/heavy-lifting/src/media_processor/tasks/thumbnailer.rs b/core/crates/heavy-lifting/src/media_processor/tasks/thumbnailer.rs index 0180014a9..1497e3cc4 100644 --- a/core/crates/heavy-lifting/src/media_processor/tasks/thumbnailer.rs +++ b/core/crates/heavy-lifting/src/media_processor/tasks/thumbnailer.rs @@ -379,21 +379,20 @@ fn process_thumbnail_generation_output( match status { GenerationStatus::Generated => { *generated += 1; + // This if is REALLY needed, due to the sheer performance of the thumbnailer, + // I restricted to only send events notifying for thumbnails in the current + // opened directory, sending events for the entire location turns into a + // humongous bottleneck in the frontend lol, since it doesn't even knows + // what to do with thumbnails for inner directories lol + // - fogodev + if with_priority { + reporter.new_thumbnail(thumb_key); + } } GenerationStatus::Skipped => { *skipped += 1; } } - - // This if is REALLY needed, due to the sheer performance of the thumbnailer, - // I restricted to only send events notifying for thumbnails in the current - // opened directory, sending events for the entire location turns into a - // humongous bottleneck in the frontend lol, since it doesn't even knows - // what to do with thumbnails for inner directories lol - // - fogodev - if with_priority { - reporter.new_thumbnail(thumb_key); - } } Err(e) => { errors.push(media_processor::NonCriticalMediaProcessorError::from(e).into()); diff --git a/core/crates/indexer-rules/Cargo.toml b/core/crates/indexer-rules/Cargo.toml index 218f04d75..472bd2442 100644 --- a/core/crates/indexer-rules/Cargo.toml +++ b/core/crates/indexer-rules/Cargo.toml @@ -19,7 +19,7 @@ globset = { workspace = true, features = ["serde1"] } prisma-client-rust = { workspace = true } rmp-serde = { workspace = true } rspc = { workspace = true } -serde = { workspace = true, features = ["derive"] } +serde = { workspace = true, features = ["derive", "rc"] } specta = { workspace = true } thiserror = { workspace = true } tokio = { workspace = true, features = ["fs"] } diff --git a/core/src/custom_uri/async_read_body.rs b/core/src/custom_uri/async_read_body.rs deleted file mode 100644 index 1a1cc523a..000000000 --- a/core/src/custom_uri/async_read_body.rs +++ /dev/null @@ -1,61 +0,0 @@ -use std::{ - io, - pin::Pin, - task::{Context, Poll}, -}; - -use axum::http::HeaderMap; -use bytes::Bytes; -use futures::Stream; -use http_body::Body; -use pin_project_lite::pin_project; -use tokio::io::{AsyncRead, AsyncReadExt, Take}; -use tokio_util::io::ReaderStream; - -// This code was taken from: https://github.com/tower-rs/tower-http/blob/e8eb54966604ea7fa574a2a25e55232f5cfe675b/tower-http/src/services/fs/mod.rs#L30 -pin_project! { - // NOTE: This could potentially be upstreamed to `http-body`. - /// Adapter that turns an [`impl AsyncRead`][tokio::io::AsyncRead] to an [`impl Body`][http_body::Body]. - #[derive(Debug)] - pub struct AsyncReadBody { - #[pin] - reader: ReaderStream, - } -} - -impl AsyncReadBody -where - T: AsyncRead, -{ - pub(crate) fn with_capacity_limited( - read: T, - capacity: usize, - max_read_bytes: u64, - ) -> AsyncReadBody> { - AsyncReadBody { - reader: ReaderStream::with_capacity(read.take(max_read_bytes), capacity), - } - } -} - -impl Body for AsyncReadBody -where - T: AsyncRead, -{ - type Data = Bytes; - type Error = io::Error; - - fn poll_data( - self: Pin<&mut Self>, - cx: &mut Context<'_>, - ) -> Poll>> { - self.project().reader.poll_next(cx) - } - - fn poll_trailers( - self: Pin<&mut Self>, - _cx: &mut Context<'_>, - ) -> Poll, Self::Error>> { - Poll::Ready(Ok(None)) - } -} diff --git a/core/src/custom_uri/mod.rs b/core/src/custom_uri/mod.rs index 33af7c727..89bd85122 100644 --- a/core/src/custom_uri/mod.rs +++ b/core/src/custom_uri/mod.rs @@ -29,7 +29,7 @@ use std::{ use async_stream::stream; use axum::{ - body::{self, Body, BoxBody, Full, StreamBody}, + body::Body, extract::{self, State}, http::{HeaderMap, HeaderValue, Request, Response, StatusCode}, middleware, @@ -38,8 +38,8 @@ use axum::{ Router, }; use bytes::Bytes; -use http_body::combinators::UnsyncBoxBody; use hyper::{header, upgrade::OnUpgrade}; +use hyper_util::rt::TokioIo; use mini_moka::sync::Cache; use tokio::{ fs::{self, File}, @@ -50,7 +50,6 @@ use uuid::Uuid; use self::{serve_file::serve_file, utils::*}; -mod async_read_body; mod mpsc_to_async_write; mod serve_file; mod utils; @@ -97,7 +96,7 @@ async fn request_to_remote_node( p2p: Arc, identity: RemoteIdentity, mut request: Request, -) -> Response> { +) -> Response { let request_upgrade_header = request.headers().get(header::UPGRADE).cloned(); let maybe_client_upgrade = request.extensions_mut().remove::(); @@ -121,17 +120,20 @@ async fn request_to_remote_node( }; tokio::spawn(async move { - let Ok(mut request_upgraded) = request_upgraded.await.map_err(|e| { + let Ok(request_upgraded) = request_upgraded.await.map_err(|e| { warn!(?e, "Error upgrading websocket request;"); }) else { return; }; - let Ok(mut response_upgraded) = response_upgraded.await.map_err(|e| { + let Ok(response_upgraded) = response_upgraded.await.map_err(|e| { warn!(?e, "Error upgrading websocket response;"); }) else { return; }; + let mut request_upgraded = TokioIo::new(request_upgraded); + let mut response_upgraded = TokioIo::new(response_upgraded); + copy_bidirectional(&mut request_upgraded, &mut response_upgraded) .await .map_err(|e| { @@ -147,7 +149,7 @@ async fn request_to_remote_node( async fn get_or_init_lru_entry( state: &LocalState, extract::Path((lib_id, loc_id, path_id)): ExtractedPath, -) -> Result<(CacheValue, Arc), Response> { +) -> Result<(CacheValue, Arc), Response> { let library_id = Uuid::from_str(&lib_id).map_err(bad_request)?; let location_id = loc_id.parse::().map_err(bad_request)?; let file_path_id = path_id @@ -245,7 +247,7 @@ pub fn base_router() -> Router { } else { StatusCode::INTERNAL_SERVER_ERROR }) - .body(body::boxed(Full::from(""))) + .body(Body::from("")) })?; let metadata = file.metadata().await; serve_file( @@ -290,7 +292,7 @@ pub fn base_router() -> Router { } else { StatusCode::INTERNAL_SERVER_ERROR }) - .body(body::boxed(Full::from(""))) + .body(Body::from("")) })?; let resp = InfallibleResponse::builder().header( @@ -335,11 +337,11 @@ pub fn base_router() -> Router { // TODO: Content Type Ok(InfallibleResponse::builder().status(StatusCode::OK).body( - body::boxed(StreamBody::new(stream! { + Body::from_stream(stream! { while let Some(item) = rx.recv().await { yield item; } - })), + }), )) } } @@ -364,7 +366,7 @@ pub fn base_router() -> Router { } else { StatusCode::INTERNAL_SERVER_ERROR }) - .body(body::boxed(Full::from(""))) + .body(Body::from("")) })?; let resp = InfallibleResponse::builder().header( @@ -453,7 +455,7 @@ async fn infer_the_mime_type( ext: &str, file: &mut File, metadata: &Metadata, -) -> Result> { +) -> Result> { let ext = ext.to_lowercase(); let mime_type = match ext.as_str() { // AAC audio diff --git a/core/src/custom_uri/serve_file.rs b/core/src/custom_uri/serve_file.rs index fd80a69e5..460684cf5 100644 --- a/core/src/custom_uri/serve_file.rs +++ b/core/src/custom_uri/serve_file.rs @@ -3,18 +3,18 @@ use crate::util::InfallibleResponse; use std::{fs::Metadata, time::UNIX_EPOCH}; use axum::{ - body::{self, BoxBody, Full, StreamBody}, + body::Body, http::{header, request, HeaderValue, Method, Response, StatusCode}, }; use http_range::HttpRange; use tokio::{ fs::File, - io::{self, AsyncSeekExt, SeekFrom}, + io::{self, AsyncReadExt, AsyncSeekExt, SeekFrom}, }; use tokio_util::io::ReaderStream; use tracing::error; -use super::{async_read_body::AsyncReadBody, utils::*}; +use super::utils::*; // default capacity 64KiB const DEFAULT_CAPACITY: usize = 65536; @@ -31,7 +31,7 @@ pub(crate) async fn serve_file( metadata: io::Result, req: request::Parts, mut resp: InfallibleResponse, -) -> Result, Response> { +) -> Result, Response> { if let Ok(metadata) = metadata { // We only accept range queries if `files.metadata() == Ok(_)` // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Accept-Ranges @@ -48,7 +48,7 @@ pub(crate) async fn serve_file( return Ok(resp .status(StatusCode::OK) .header("Content-Length", HeaderValue::from_static("0")) - .body(body::boxed(Full::from("")))); + .body(Body::from(""))); } // ETag @@ -73,9 +73,7 @@ pub(crate) async fn serve_file( // Used for normal requests if let Some(etag) = req.headers.get("If-None-Match") { if etag.as_bytes() == etag_header.as_bytes() { - return Ok(resp - .status(StatusCode::NOT_MODIFIED) - .body(body::boxed(Full::from("")))); + return Ok(resp.status(StatusCode::NOT_MODIFIED).body(Body::from(""))); } } @@ -104,7 +102,7 @@ pub(crate) async fn serve_file( .map_err(internal_server_error)?, ) .status(StatusCode::RANGE_NOT_SATISFIABLE) - .body(body::boxed(Full::from("")))); + .body(Body::from(""))); } let range = ranges.first().expect("checked above"); @@ -116,7 +114,7 @@ pub(crate) async fn serve_file( .map_err(internal_server_error)?, ) .status(StatusCode::RANGE_NOT_SATISFIABLE) - .body(body::boxed(Full::from("")))); + .body(Body::from(""))); } file.seek(SeekFrom::Start(range.start)) @@ -140,14 +138,13 @@ pub(crate) async fn serve_file( HeaderValue::from_str(&range.length.to_string()) .map_err(internal_server_error)?, ) - .body(body::boxed(AsyncReadBody::with_capacity_limited( - file, + .body(Body::from_stream(ReaderStream::with_capacity( + file.take(range.length), DEFAULT_CAPACITY, - range.length, )))); } } } - Ok(resp.body(body::boxed(StreamBody::new(ReaderStream::new(file))))) + Ok(resp.body(Body::from_stream(ReaderStream::new(file)))) } diff --git a/core/src/custom_uri/utils.rs b/core/src/custom_uri/utils.rs index 645da5106..cb54b815f 100644 --- a/core/src/custom_uri/utils.rs +++ b/core/src/custom_uri/utils.rs @@ -3,50 +3,49 @@ use crate::util::InfallibleResponse; use std::{fmt::Debug, panic::Location}; use axum::{ - body::{self, BoxBody}, + body::Body, http::{self, HeaderValue, Method, Request, Response, StatusCode}, middleware::Next, }; -use http_body::Full; use tracing::debug; #[track_caller] -pub(crate) fn bad_request(e: impl Debug) -> http::Response { +pub(crate) fn bad_request(e: impl Debug) -> http::Response { debug!(caller = %Location::caller(), ?e, "400: Bad Request;"); InfallibleResponse::builder() .status(StatusCode::BAD_REQUEST) - .body(body::boxed(Full::from(""))) + .body(Body::from("")) } #[track_caller] -pub(crate) fn not_found(e: impl Debug) -> http::Response { +pub(crate) fn not_found(e: impl Debug) -> http::Response { debug!(caller = %Location::caller(), ?e, "404: Not Found;"); InfallibleResponse::builder() .status(StatusCode::NOT_FOUND) - .body(body::boxed(Full::from(""))) + .body(Body::from("")) } #[track_caller] -pub(crate) fn internal_server_error(e: impl Debug) -> http::Response { +pub(crate) fn internal_server_error(e: impl Debug) -> http::Response { debug!(caller = %Location::caller(), ?e, "500: Internal Server Error;"); InfallibleResponse::builder() .status(StatusCode::INTERNAL_SERVER_ERROR) - .body(body::boxed(Full::from(""))) + .body(Body::from("")) } #[track_caller] -pub(crate) fn not_implemented(e: impl Debug) -> http::Response { +pub(crate) fn not_implemented(e: impl Debug) -> http::Response { debug!(caller = %Location::caller(), ?e, "501: Not Implemented;"); InfallibleResponse::builder() .status(StatusCode::NOT_IMPLEMENTED) - .body(body::boxed(Full::from(""))) + .body(Body::from("")) } -pub(crate) async fn cors_middleware(req: Request, next: Next) -> Response { +pub(crate) async fn cors_middleware(req: Request, next: Next) -> Response { if req.method() == Method::OPTIONS { return Response::builder() .header("Access-Control-Allow-Methods", "GET, HEAD, POST, OPTIONS") @@ -54,7 +53,7 @@ pub(crate) async fn cors_middleware(req: Request, next: Next) -> Respon .header("Access-Control-Allow-Headers", "*") .header("Access-Control-Max-Age", "86400") .status(StatusCode::OK) - .body(body::boxed(Full::from(""))) + .body(Body::from("")) .expect("Invalid static response!"); } diff --git a/core/src/p2p/operations/rspc.rs b/core/src/p2p/operations/rspc.rs index ed86c0912..e5e0b3cc8 100644 --- a/core/src/p2p/operations/rspc.rs +++ b/core/src/p2p/operations/rspc.rs @@ -1,9 +1,11 @@ use std::{error::Error, sync::Arc}; -use axum::{body::Body, http, Router}; -use hyper::{server::conn::Http, Response}; +use axum::{extract::Request, http, Router}; +use hyper::{body::Incoming, client::conn::http1::handshake, server::conn::http1, Response}; +use hyper_util::rt::TokioIo; use sd_p2p::{RemoteIdentity, UnicastStream, P2P}; use tokio::io::AsyncWriteExt; +use tower_service::Service; use tracing::debug; use crate::{p2p::Header, Node}; @@ -13,7 +15,7 @@ pub async fn remote_rspc( p2p: Arc, identity: RemoteIdentity, request: http::Request, -) -> Result, Box> { +) -> Result, Box> { let peer = p2p .peers() .get(&identity) @@ -23,7 +25,7 @@ pub async fn remote_rspc( stream.write_all(&Header::RspcRemote.to_bytes()).await?; - let (mut sender, conn) = hyper::client::conn::handshake(stream).await?; + let (mut sender, conn) = handshake(TokioIo::new(stream)).await?; tokio::task::spawn(async move { if let Err(e) = conn.await { println!("Connection error: {:?}", e); @@ -49,10 +51,12 @@ pub(crate) async fn receiver( todo!("No way buddy!"); } - Http::new() - .http1_only(true) - .http1_keep_alive(true) - .serve_connection(stream, service) + let hyper_service = + hyper::service::service_fn(move |request: Request| service.clone().call(request)); + + http1::Builder::new() + .keep_alive(true) + .serve_connection(TokioIo::new(stream), hyper_service) .with_upgrades() .await .map_err(Into::into) diff --git a/crates/ai/Cargo.toml b/crates/ai/Cargo.toml index e24bd3c0c..b521ee93d 100644 --- a/crates/ai/Cargo.toml +++ b/crates/ai/Cargo.toml @@ -38,7 +38,7 @@ uuid = { workspace = true, features = ["serde", "v4"] } # Specific AI dependencies # Note: half and ndarray version must be the same as used in ort -half = { version = "2.1", features = ['num-traits'] } +half = { version = "2.4", features = ['num-traits'] } ndarray = "0.15" url = '2.5' diff --git a/crates/ffmpeg/src/thumbnailer.rs b/crates/ffmpeg/src/thumbnailer.rs index afd008813..6333e34c4 100644 --- a/crates/ffmpeg/src/thumbnailer.rs +++ b/crates/ffmpeg/src/thumbnailer.rs @@ -4,7 +4,7 @@ use std::{io, ops::Deref, path::Path}; use image::{imageops, DynamicImage, RgbImage}; use sd_utils::error::FileIOError; -use tokio::{fs, task::spawn_blocking}; +use tokio::{fs, io::AsyncWriteExt, task::spawn_blocking}; use tracing::error; use webp::Encoder; @@ -37,12 +37,18 @@ impl Thumbnailer { .await .map_err(|e| FileIOError::from((path, e)))?; - fs::write( - output_thumbnail_path, - &*self.process_to_webp_bytes(video_file_path).await?, - ) - .await - .map_err(|e| FileIOError::from((output_thumbnail_path, e)).into()) + let webp = self.process_to_webp_bytes(video_file_path).await?; + let mut file = fs::File::create(output_thumbnail_path) + .await + .map_err(|e: io::Error| FileIOError::from((output_thumbnail_path, e)))?; + + file.write_all(&webp) + .await + .map_err(|e| FileIOError::from((output_thumbnail_path, e)))?; + + file.sync_all() + .await + .map_err(|e| FileIOError::from((output_thumbnail_path, e)).into()) } /// Processes an video input file and returns a webp encoded thumbnail as bytes diff --git a/crates/p2p/Cargo.toml b/crates/p2p/Cargo.toml index 14d21c20b..4727ec7f9 100644 --- a/crates/p2p/Cargo.toml +++ b/crates/p2p/Cargo.toml @@ -31,11 +31,11 @@ uuid = { workspace = true, features = ["serde"] } # Specific P2P dependencies dns-lookup = "2.0" -flume = "=0.11.0" # Must match version used by `mdns-sd` +flume = "=0.11.1" # Must match version used by `mdns-sd` hash_map_diff = "0.2.0" if-watch = { version = "=3.2.0", features = ["tokio"] } # Override features used by libp2p-quic -libp2p-stream = "=0.1.0-alpha" # Update blocked due to custom patch -mdns-sd = "0.11.1" +libp2p-stream = "=0.2.0-alpha" # Update blocked due to custom patch +mdns-sd = "0.11.5" rand_core = "0.6.4" stable-vec = "0.4.1" sync_wrapper = "1.0" @@ -43,7 +43,7 @@ zeroize = { version = "1.8", features = ["derive"] } [dependencies.libp2p] features = ["autonat", "dcutr", "macros", "noise", "quic", "relay", "serde", "tokio", "yamux"] -version = "=0.53.2" # Update blocked due to custom patch +version = "=0.54.1" # Update blocked due to custom patch [dev-dependencies] tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } diff --git a/crates/prisma-cli/Cargo.toml b/crates/prisma-cli/Cargo.toml index 37c764459..c5c0fb669 100644 --- a/crates/prisma-cli/Cargo.toml +++ b/crates/prisma-cli/Cargo.toml @@ -14,5 +14,5 @@ sd-sync-generator = { path = "../sync-generator" } [dependencies.prisma-client-rust-generator] default-features = false features = ["migrations", "specta", "sqlite", "sqlite-create-many"] -git = "https://github.com/brendonovich/prisma-client-rust" -rev = "4f9ef9d38c" +git = "https://github.com/spacedriveapp/prisma-client-rust" +rev = "b22ad7dc7d" diff --git a/crates/sync/example/Cargo.toml b/crates/sync/example/Cargo.toml deleted file mode 100644 index 6893daee8..000000000 --- a/crates/sync/example/Cargo.toml +++ /dev/null @@ -1,27 +0,0 @@ -[package] -name = "sd-sync-example" -version = "0.1.0" - -edition.workspace = true -license.workspace = true -publish = false -repository.workspace = true -rust-version.workspace = true - -[dependencies] -# Spacedrive Sub-crates -sd-sync = { path = ".." } - -# Workspace dependencies -axum = { workspace = true } -http = { workspace = true } -prisma-client-rust = { workspace = true } -rspc = { workspace = true, features = ["axum"] } -serde = { workspace = true, features = ["derive"] } -serde_json = { workspace = true } -tokio = { workspace = true, features = ["full"] } -uuid = { workspace = true, features = ["v4"] } - -# Specific Core dependencies -dotenv = "0.15.0" -tower-http = { version = "0.4.4", features = ["cors"] } # Update blocked by http diff --git a/crates/sync/example/README.md b/crates/sync/example/README.md deleted file mode 100644 index c3adc9cab..000000000 --- a/crates/sync/example/README.md +++ /dev/null @@ -1,18 +0,0 @@ -# Create rspc app - -This app was scaffolded using the [create-rspc-app](https://rspc.dev) CLI. - -## Usage - -```bash -# Terminal One -cd web -pnpm i -pnpm dev - -# Terminal Two -cd api/ -cargo prisma generate -cargo prisma db push -cargo run -``` diff --git a/crates/sync/example/prisma/migrations/.gitkeep b/crates/sync/example/prisma/migrations/.gitkeep deleted file mode 100644 index e69de29bb..000000000 diff --git a/crates/sync/example/prisma/schema.prisma b/crates/sync/example/prisma/schema.prisma deleted file mode 100644 index efc4ee911..000000000 --- a/crates/sync/example/prisma/schema.prisma +++ /dev/null @@ -1,34 +0,0 @@ -// This is your Prisma schema file, -// learn more about it in the docs: https://pris.ly/d/prisma-schema - -datasource db { - provider = "sqlite" - url = "file:dev.db" -} - -generator client { - provider = "cargo prisma" - output = "../src/prisma.rs" -} - -generator sync { - provider = "cargo run -p prisma-cli --bin sync --" - output = "../src/prisma_sync.rs" -} - -/// @owned -model FilePath { - id Bytes @id - path String - - object Object? @relation(fields: [object_id], references: [id]) - object_id Bytes? -} - -/// @shared -model Object { - id Bytes @id - name String - - paths FilePath[] @relation() -} diff --git a/crates/sync/example/src/api/mod.rs b/crates/sync/example/src/api/mod.rs deleted file mode 100644 index 14cc6d257..000000000 --- a/crates/sync/example/src/api/mod.rs +++ /dev/null @@ -1,175 +0,0 @@ -use std::collections::HashMap; -use std::sync::Arc; - -use rspc::*; -use sd_sync::*; -use serde_json::*; -use std::path::PathBuf; -use tokio::sync::Mutex; -use uuid::Uuid; - -use crate::prisma::{file_path, PrismaClient}; - -pub struct Ctx { - pub dbs: HashMap, - pub prisma: PrismaClient, -} - -type Router = rspc::Router>>; - -fn to_map(v: &impl serde::Serialize) -> serde_json::Map { - match to_value(v).unwrap() { - Value::Object(m) => m, - _ => unreachable!(), - } -} - -pub(crate) fn new() -> RouterBuilder>> { - Router::new() - .config(Config::new().export_ts_bindings( - PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("web/src/utils/bindings.ts"), - )) - .mutation("testCreate", |r| { - r(|ctx, _: String| async move { - let prisma = &ctx.lock().await.prisma; - - let res = prisma - .file_path() - .create(vec![], String::new(), vec![]) - .exec_raw() - .await - .unwrap(); - - file_path::Create::operation_from_data(&res); - - Ok(()) - }) - }) - .mutation("createDatabase", |r| { - r(|ctx, _: String| async move { - let dbs = &mut ctx.lock().await.dbs; - let uuid = Uuid::new_v4(); - - dbs.insert(uuid, Db::new(uuid)); - - let ids = dbs.keys().copied().collect::>(); - - for db in dbs.values_mut() { - for id in &ids { - db.register_node(*id); - } - } - - Ok(uuid) - }) - }) - .mutation("removeDatabases", |r| { - r(|ctx, _: String| async move { - let dbs = &mut ctx.lock().await.dbs; - - dbs.drain(); - - Ok(()) - }) - }) - .query("dbs", |r| { - r(|ctx, _: String| async move { - let dbs = &mut ctx.lock().await.dbs; - - Ok(dbs.iter().map(|(id, _)| *id).collect::>()) - }) - }) - .query("db.tags", |r| { - r(|ctx, id: String| async move { - let dbs = &mut ctx.lock().await.dbs; - - let id = id.parse().unwrap(); - - Ok(dbs.get(&id).unwrap().tags.clone()) - }) - }) - .query("file_path.list", |r| { - r(|ctx, id: String| async move { - let dbs = &mut ctx.lock().await.dbs; - - let db = dbs.get(&id.parse().unwrap()).unwrap(); - - let file_paths = db.file_paths.values().map(Clone::clone).collect::>(); - - Ok(file_paths) - }) - }) - .mutation("file_path.create", |r| { - r(|ctx, db: String| async move { - let dbs = &mut ctx.lock().await.dbs; - - let db = dbs.get_mut(&db.parse().unwrap()).unwrap(); - - let id = Uuid::new_v4(); - - let file_path = FilePath { - id, - path: String::new(), - file: None, - }; - - let op = db.create_crdt_operation(CRDTOperationType::Owned(OwnedOperation { - model: "FilePath".to_string(), - items: vec![OwnedOperationItem { - id: serde_json::to_value(id).unwrap(), - data: OwnedOperationData::Create( - serde_json::from_value(serde_json::to_value(&file_path).unwrap()) - .unwrap(), - ), - }], - })); - - db.receive_crdt_operations(vec![op]); - - file_path - }) - }) - .query("message.list", |r| { - r(|ctx, id: String| async move { - let dbs = &mut ctx.lock().await.dbs; - - let db = dbs.get(&id.parse().unwrap()).unwrap(); - - Ok(db._operations.clone()) - }) - }) - .mutation("pullOperations", |r| { - r(|ctx, db_id: String| async move { - let dbs = &mut ctx.lock().await.dbs; - - let db_id = db_id.parse().unwrap(); - - let ops = dbs.values().flat_map(|db| db._operations.clone()).collect(); - - let db = dbs.get_mut(&db_id).unwrap(); - - db.receive_crdt_operations(ops); - - Ok(()) - }) - }) - .query("operations", |r| { - r(|ctx, _: String| async move { - let dbs = &mut ctx.lock().await.dbs; - - let mut hashmap = HashMap::new(); - - for db in dbs.values_mut() { - for op in &db._operations { - hashmap.insert(op.id, op.clone()); - } - } - - let mut array = hashmap.into_values().collect::>(); - - array.sort_by(|a, b| a.id.partial_cmp(&b.id).unwrap()); - - Ok(array) - }) - }) -} diff --git a/crates/sync/example/src/main.rs b/crates/sync/example/src/main.rs deleted file mode 100644 index a04eae216..000000000 --- a/crates/sync/example/src/main.rs +++ /dev/null @@ -1,45 +0,0 @@ -use api::Ctx; -use axum::{ - http::{HeaderValue, Method}, - routing::get, -}; -use std::{net::SocketAddr, sync::Arc}; -use tokio::sync::Mutex; -use tower_http::cors::CorsLayer; - -mod api; -mod prisma; -mod prisma_sync; -mod utils; - -async fn router() -> axum::Router { - let router = api::new().build().arced(); - - let ctx = Arc::new(Mutex::new(Ctx { - dbs: Default::default(), - prisma: prisma::new_client().await.unwrap(), - })); - - axum::Router::new() - .route("/", get(|| async { "Hello 'rspc'!" })) - .route("/rspc/:id", router.endpoint(move || ctx.clone()).axum()) - .layer( - CorsLayer::new() - .allow_origin("http://localhost:3000".parse::().unwrap()) - .allow_headers(vec![http::header::CONTENT_TYPE]) - .allow_methods([Method::GET, Method::POST]), - ) -} - -#[tokio::main] -async fn main() { - dotenv::dotenv().ok(); - - let addr = "[::]:9000".parse::().unwrap(); // This listens on IPv6 and IPv4 - println!("{} listening on http://{}", env!("CARGO_CRATE_NAME"), addr); - axum::Server::bind(&addr) - .serve(router().await.into_make_service()) - .with_graceful_shutdown(utils::axum_shutdown_signal()) - .await - .expect("Error with HTTP server!"); -} diff --git a/crates/sync/example/src/utils.rs b/crates/sync/example/src/utils.rs deleted file mode 100644 index f6437d365..000000000 --- a/crates/sync/example/src/utils.rs +++ /dev/null @@ -1,28 +0,0 @@ -use tokio::signal; - -/// shutdown_signal will inform axum to gracefully shutdown when the process is asked to shutdown. -pub async fn axum_shutdown_signal() { - let ctrl_c = async { - signal::ctrl_c() - .await - .expect("failed to install Ctrl+C handler"); - }; - - #[cfg(unix)] - let terminate = async { - signal::unix::signal(signal::unix::SignalKind::terminate()) - .expect("failed to install signal handler") - .recv() - .await; - }; - - #[cfg(not(unix))] - let terminate = std::future::pending::<()>(); - - tokio::select! { - _ = ctrl_c => {}, - _ = terminate => {}, - } - - println!("signal received, starting graceful shutdown"); -} diff --git a/crates/sync/example/web/.gitignore b/crates/sync/example/web/.gitignore deleted file mode 100644 index 76add878f..000000000 --- a/crates/sync/example/web/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -node_modules -dist \ No newline at end of file diff --git a/crates/sync/example/web/README.md b/crates/sync/example/web/README.md deleted file mode 100644 index 434f7bb9d..000000000 --- a/crates/sync/example/web/README.md +++ /dev/null @@ -1,34 +0,0 @@ -## Usage - -Those templates dependencies are maintained via [pnpm](https://pnpm.io) via `pnpm up -Lri`. - -This is the reason you see a `pnpm-lock.yaml`. That being said, any package manager will work. This file can be safely be removed once you clone a template. - -```bash -$ npm install # or pnpm install or yarn install -``` - -### Learn more on the [Solid Website](https://solidjs.com) and come chat with us on our [Discord](https://discord.com/invite/solidjs) - -## Available Scripts - -In the project directory, you can run: - -### `npm dev` or `npm start` - -Runs the app in the development mode.
    -Open [http://localhost:3000](http://localhost:3000) to view it in the browser. - -The page will reload if you make edits.
    - -### `npm run build` - -Builds the app for production to the `dist` folder.
    -It correctly bundles Solid in production mode and optimizes the build for the best performance. - -The build is minified and the filenames include the hashes.
    -Your app is ready to be deployed! - -## Deployment - -You can deploy the `dist` folder to any static host provider (netlify, surge, now, etc.) diff --git a/crates/sync/example/web/index.html b/crates/sync/example/web/index.html deleted file mode 100644 index f22a9d4f1..000000000 --- a/crates/sync/example/web/index.html +++ /dev/null @@ -1,15 +0,0 @@ - - - - - - - Solid App - - - -

    - - - - diff --git a/crates/sync/example/web/package.json b/crates/sync/example/web/package.json deleted file mode 100644 index 0c95d0ba5..000000000 --- a/crates/sync/example/web/package.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "name": "example-2", - "version": "0.0.0", - "description": "", - "scripts": { - "dev": "vite", - "build": "vite build", - "serve": "vite preview", - "typecheck": "tsc --noEmit" - }, - "license": "MIT", - "dependencies": { - "clsx": "^2.0.0", - "react": "^18.2.0", - "react-dom": "^18.2.0", - "solid-js": "^1.8.3" - }, - "devDependencies": { - "@tanstack/react-query": "^4.36.1", - "typescript": "^5.6.2", - "vite": "^5.2.0", - "tailwindcss": "^3.3.3" - } -} diff --git a/crates/sync/example/web/postcss.config.js b/crates/sync/example/web/postcss.config.js deleted file mode 100644 index 054c147cb..000000000 --- a/crates/sync/example/web/postcss.config.js +++ /dev/null @@ -1,6 +0,0 @@ -module.exports = { - plugins: { - tailwindcss: {}, - autoprefixer: {} - } -}; diff --git a/crates/sync/example/web/src/App.tsx b/crates/sync/example/web/src/App.tsx deleted file mode 100644 index 6c90cbd4b..000000000 --- a/crates/sync/example/web/src/App.tsx +++ /dev/null @@ -1,172 +0,0 @@ -// import clsx from 'clsx'; -// import { Suspense, useState } from 'react'; -// import { tests } from './test'; -// import { CRDTOperationType, rspc } from './utils/rspc'; - -// export function App() { -// const dbs = rspc.useQuery(['dbs', 'cringe']); - -// const operations = rspc.useQuery(['operations', 'cringe']); - -// const createDb = rspc.useMutation('createDatabase'); -// const removeDbs = rspc.useMutation('removeDatabases'); -// const testCreate = rspc.useMutation('testCreate'); - -// return ( -//
    -//
    -//
    -// -// -// -//
    -//
      -// {Object.entries(tests).map(([key, test]) => ( -//
    • -// -//
      • -// ))} -//
    -//
    -//
    -//
      -// {dbs.data?.map((id) => ( -// -// -// -// ))} -//
    -//
    -//
    -//

    All Operations

    -//
      -// {operations.data?.map((op) => ( -//
    • -//

      ID: {op.id}

      -//

      Timestamp: {op.timestamp.toString()}

      -//

      Node: {op.node}

      -//
      • -// ))} -//
    -//
    -//
    -// ); -// } - -// interface DatabaseViewProps { -// id: string; -// } -// const TABS = ['File Paths', 'Objects', 'Tags', 'Operations']; - -// function DatabaseView(props: DatabaseViewProps) { -// const [currentTab, setCurrentTab] = useState<(typeof TABS)[number]>('Operations'); - -// const pullOperations = rspc.useMutation('pullOperations'); - -// return ( -//
    -//
    -//

    {props.id}

    -// -//
    -//
    -// -// -// {currentTab === 'File Paths' && } -// {currentTab === 'Operations' && } -// -//
    -//
    -// ); -// } - -// function FilePathList(props: { db: string }) { -// const createFilePath = rspc.useMutation('file_path.create'); -// const filePaths = rspc.useQuery(['file_path.list', props.db]); - -// return ( -//
    -// {filePaths.data && ( -//
      -// {filePaths.data -// .sort((a, b) => a.id.localeCompare(b.id)) -// .map((path) => ( -//
    • {JSON.stringify(path)}
      • -// ))} -//
    -// )} -// -//
    -// ); -// } - -// function messageType(msg: CRDTOperationType) { -// if ('items' in msg) { -// return 'Owned'; -// } else if ('record_id' in msg) { -// return 'Shared'; -// } -// } - -// function OperationList(props: { db: string }) { -// const messages = rspc.useQuery(['message.list', props.db]); - -// return ( -//
    -// {messages.data && ( -// -// {messages.data -// .sort((a, b) => Number(a.timestamp - b.timestamp)) -// .map((message) => ( -// -// -// -// -// -// ))} -//
    {message.id} -// {new Date( -// Number(message.timestamp) / 10000000 -// ).toLocaleTimeString()} -// -// {messageType(message.typ)} -//
    -// )} -//
    -// ); -// } - -// const ButtonStyles = 'bg-blue-500 text-white px-2 py-1 rounded-md'; - -export {}; diff --git a/crates/sync/example/web/src/index.css b/crates/sync/example/web/src/index.css deleted file mode 100644 index b5c61c956..000000000 --- a/crates/sync/example/web/src/index.css +++ /dev/null @@ -1,3 +0,0 @@ -@tailwind base; -@tailwind components; -@tailwind utilities; diff --git a/crates/sync/example/web/src/index.tsx b/crates/sync/example/web/src/index.tsx deleted file mode 100644 index 6c9a09bac..000000000 --- a/crates/sync/example/web/src/index.tsx +++ /dev/null @@ -1,16 +0,0 @@ -// /* @refresh reload */ -// import { Suspense } from 'react'; -// import { createRoot } from 'react-dom/client'; -// import { App } from './App'; -// import './index.css'; -// import { queryClient, rspc, rspcClient } from './utils/rspc'; - -// createRoot(document.getElementById('root') as HTMLElement).render( -// -// -// -// -// -// ); - -export {}; diff --git a/crates/sync/example/web/src/test.ts b/crates/sync/example/web/src/test.ts deleted file mode 100644 index e516517cc..000000000 --- a/crates/sync/example/web/src/test.ts +++ /dev/null @@ -1,47 +0,0 @@ -// import { queryClient, rspcClient } from './utils/rspc'; - -// function test(fn: () => Promise) { -// return async () => { -// await fn(); -// queryClient.invalidateQueries(); -// }; -// } - -// const wait = (ms: number) => new Promise((res) => setTimeout(res, ms)); - -// export const tests = { -// three: { -// name: 'Three', -// run: test(async () => { -// const [db1, db2, db3] = await Promise.all([ -// rspcClient.mutation(['createDatabase', ' ']), -// rspcClient.mutation(['createDatabase', ' ']), -// rspcClient.mutation(['createDatabase', ' ']) -// ]); - -// const dbs = await rspcClient.query(['dbs', 'cringe']); - -// for (const db of dbs) { -// await rspcClient.mutation(['file_path.create', db]); -// } - -// for (const db of dbs) { -// await rspcClient.mutation(['pullOperations', db]); -// } - -// await rspcClient.mutation(['file_path.create', dbs[0]]); -// await rspcClient.mutation(['file_path.create', dbs[0]]); - -// for (const db of dbs) { -// await rspcClient.mutation(['pullOperations', db]); -// } - -// await rspcClient.mutation(['pullOperations', dbs[1]]); -// await rspcClient.mutation(['pullOperations', dbs[1]]); -// await rspcClient.mutation(['pullOperations', dbs[1]]); -// await rspcClient.mutation(['pullOperations', dbs[1]]); -// }) -// } -// }; - -export {}; diff --git a/crates/sync/example/web/src/utils/bindings.ts b/crates/sync/example/web/src/utils/bindings.ts deleted file mode 100644 index 3d31e95bb..000000000 --- a/crates/sync/example/web/src/utils/bindings.ts +++ /dev/null @@ -1,80 +0,0 @@ -// This file was generated by [rspc](https://github.com/oscartbeaumont/rspc). Do not edit this file manually. - -export type Procedures = { - queries: - | { key: 'db.tags'; input: string; result: Record } - | { key: 'dbs'; input: string; result: Array } - | { key: 'file_path.list'; input: string; result: Array } - | { key: 'message.list'; input: string; result: Array } - | { key: 'operations'; input: string; result: Array }; - mutations: - | { key: 'createDatabase'; input: string; result: string } - | { key: 'file_path.create'; input: string; result: FilePath } - | { key: 'pullOperations'; input: string; result: null } - | { key: 'removeDatabases'; input: string; result: null } - | { key: 'testCreate'; input: string; result: null }; - subscriptions: never; -}; - -export interface CRDTOperation { - node: string; - timestamp: bigint; - id: string; - typ: CRDTOperationType; -} - -export type CRDTOperationType = SharedOperation | RelationOperation | OwnedOperation; - -export interface Color { - red: number; - green: number; - blue: number; -} - -export interface FilePath { - id: string; - path: string; - file: string | null; -} - -export interface OwnedOperation { - model: string; - items: Array; -} - -export type OwnedOperationData = - | { Create: Record } - | { Update: Record } - | 'Delete'; - -export interface OwnedOperationItem { - id: any; - data: OwnedOperationData; -} - -export interface RelationOperation { - relation_item: string; - relation_group: string; - relation: string; - data: RelationOperationData; -} - -export type RelationOperationData = 'Create' | { Update: { field: string; value: any } } | 'Delete'; - -export interface SharedOperation { - record_id: string; - model: string; - data: SharedOperationData; -} - -export type SharedOperationCreateData = { Unique: Record } | 'Atomic'; - -export type SharedOperationData = - | { Create: SharedOperationCreateData } - | { Update: { field: string; value: any } } - | 'Delete'; - -export interface Tag { - color: Color; - name: string; -} diff --git a/crates/sync/example/web/src/utils/rspc.ts b/crates/sync/example/web/src/utils/rspc.ts deleted file mode 100644 index 9991bd5f0..000000000 --- a/crates/sync/example/web/src/utils/rspc.ts +++ /dev/null @@ -1,29 +0,0 @@ -// import { createClient, httpLink } from '@oscartbeaumont-sd/rspc-client'; -// import { createReactHooks } from '@oscartbeaumont-sd/rspc-react'; -// import { QueryClient } from '@tanstack/react-query'; -// import type { Procedures } from './bindings'; - -// export * from './bindings'; - -// // These are generated by rspc in Rust for you. - -// const rspc = createReactHooks(); - -// const rspcClient = rspc.createClient({ -// links: [httpLink({ url: 'http://localhost:9000/rspc' })] -// }); - -// const queryClient = new QueryClient({ -// defaultOptions: { -// queries: { -// suspense: true -// }, -// mutations: { -// onSuccess: () => queryClient.invalidateQueries() -// } -// } -// }); - -// export { rspc, rspcClient, queryClient }; - -export {}; diff --git a/crates/sync/example/web/tailwind.config.js b/crates/sync/example/web/tailwind.config.js deleted file mode 100644 index 7cf6cc57b..000000000 --- a/crates/sync/example/web/tailwind.config.js +++ /dev/null @@ -1,8 +0,0 @@ -/** @type {import('tailwindcss').Config} */ -module.exports = { - content: ['./index.html', './src/**/*.{js,ts,jsx,tsx}'], - theme: { - extend: {} - }, - plugins: [] -}; diff --git a/crates/sync/example/web/tsconfig.json b/crates/sync/example/web/tsconfig.json deleted file mode 100644 index 1d5d18140..000000000 --- a/crates/sync/example/web/tsconfig.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "compilerOptions": { - "target": "ESNext", - "module": "ESNext", - "moduleResolution": "node", - "allowSyntheticDefaultImports": true, - "esModuleInterop": true, - "jsx": "react-jsx", - "types": ["vite/client"], - "noEmit": true, - "isolatedModules": true - } -} diff --git a/crates/sync/example/web/vite.config.ts b/crates/sync/example/web/vite.config.ts deleted file mode 100644 index 29f022f92..000000000 --- a/crates/sync/example/web/vite.config.ts +++ /dev/null @@ -1,12 +0,0 @@ -import react from '@vitejs/plugin-react'; -import { defineConfig } from 'vite'; - -export default defineConfig({ - plugins: [react()], - server: { - port: 3000 - }, - build: { - target: 'esnext' - } -}); diff --git a/interface/app/$libraryId/Explorer/ContextMenu/OpenWith.tsx b/interface/app/$libraryId/Explorer/ContextMenu/OpenWith.tsx index 381548a0a..d11ba36bc 100644 --- a/interface/app/$libraryId/Explorer/ContextMenu/OpenWith.tsx +++ b/interface/app/$libraryId/Explorer/ContextMenu/OpenWith.tsx @@ -81,9 +81,9 @@ const Items = ({ const paths = selectedEphemeralPaths.map((obj) => obj.path); const { t } = useLocale(); - const { data: apps } = useQuery( - ['openWith', ids, paths], - async () => { + const { data: apps } = useQuery({ + queryKey: ['openWith', ids, paths], + queryFn: async () => { const handleError = (res: Result) => { if (res?.status === 'error') { toast.error('Failed to get applications capable to open file'); @@ -104,8 +104,8 @@ const Items = ({ .then((res) => res.flat()) .then((res) => res.sort((a, b) => a.name.localeCompare(b.name))); }, - { initialData: [] } - ); + initialData: [] + }); return ( <> diff --git a/interface/app/$libraryId/Explorer/ContextMenu/SharedItems.tsx b/interface/app/$libraryId/Explorer/ContextMenu/SharedItems.tsx index 1c79d03ac..41a882e0e 100644 --- a/interface/app/$libraryId/Explorer/ContextMenu/SharedItems.tsx +++ b/interface/app/$libraryId/Explorer/ContextMenu/SharedItems.tsx @@ -224,7 +224,7 @@ const SpacedropNodes = () => { { spacedrop.mutateAsync({ identity: id, diff --git a/interface/app/$libraryId/Explorer/ExplorerTagBar.tsx b/interface/app/$libraryId/Explorer/ExplorerTagBar.tsx index aca3ec3f1..2dd8ef46e 100644 --- a/interface/app/$libraryId/Explorer/ExplorerTagBar.tsx +++ b/interface/app/$libraryId/Explorer/ExplorerTagBar.tsx @@ -117,7 +117,7 @@ export const ExplorerTagBar = () => { const { data: allTags = [] } = useLibraryQuery(['tags.list']); const mutation = useLibraryMutation(['tags.assign'], { - onSuccess: () => rspc.queryClient.invalidateQueries(['search.paths']) + onSuccess: () => rspc.queryClient.invalidateQueries({ queryKey: ['search.paths'] }) }); const { t } = useLocale(); diff --git a/interface/app/$libraryId/Explorer/FilePath/DecryptDialog.tsx b/interface/app/$libraryId/Explorer/FilePath/DecryptDialog.tsx deleted file mode 100644 index 0f2cbbc2e..000000000 --- a/interface/app/$libraryId/Explorer/FilePath/DecryptDialog.tsx +++ /dev/null @@ -1,179 +0,0 @@ -// import { RadioGroup } from '@headlessui/react'; -// import { Info } from '@phosphor-icons/react'; -// import { useLibraryMutation, useLibraryQuery } from '@sd/client'; -// import { Button, Dialog, Tooltip, UseDialogProps, useDialog } from '@sd/ui'; -// import { PasswordInput, Switch, useZodForm, z } from '@sd/ui/src/forms'; -// import { showAlertDialog } from '~/components'; -// import { usePlatform } from '~/util/Platform'; - -// const schema = z.object({ -// type: z.union([z.literal('password'), z.literal('key')]), -// outputPath: z.string(), -// mountAssociatedKey: z.boolean(), -// password: z.string(), -// saveToKeyManager: z.boolean() -// }); - -// interface Props extends UseDialogProps { -// location_id: number; -// path_id: number; -// } - -// export default (props: Props) => { -// const platform = usePlatform(); - -// const mountedUuids = useLibraryQuery(['keys.listMounted'], { -// onSuccess: (data) => { -// hasMountedKeys = data.length > 0 ? true : false; -// if (!hasMountedKeys) { -// form.setValue('type', 'password'); -// } else { -// form.setValue('type', 'key'); -// } -// } -// }); - -// let hasMountedKeys = -// mountedUuids.data !== undefined && mountedUuids.data.length > 0 ? true : false; - -// const decryptFile = useLibraryMutation('files.decryptFiles', { -// onSuccess: () => { -// showAlertDialog({ -// title: 'Success', -// value: 'The decryption job has started successfully. You may track the progress in the job overview panel.' -// }); -// }, -// onError: () => { -// showAlertDialog({ -// title: 'Error', -// value: 'The decryption job failed to start.' -// }); -// } -// }); - -// const form = useZodForm({ -// defaultValues: { -// type: hasMountedKeys ? 'key' : 'password', -// saveToKeyManager: true, -// outputPath: '', -// password: '', -// mountAssociatedKey: true -// }, -// schema -// }); - -// return ( -// -// decryptFile.mutateAsync({ -// location_id: props.location_id, -// file_path_ids: [props.path_id], -// output_path: data.outputPath !== '' ? data.outputPath : null, -// mount_associated_key: data.mountAssociatedKey, -// password: data.type === 'password' ? data.password : null, -// save_to_library: data.type === 'password' ? data.saveToKeyManager : null -// }) -// )} -// title="Decrypt a file" -// description="Leave the output file blank for the default." -// loading={decryptFile.isLoading} -// ctaLabel="Decrypt" -// > -//
    -//

    Key Type

    -// form.setValue('type', e)} -// className="mt-2 flex flex-row gap-2" -// > -// -// {({ checked }) => ( -// -// )} -// -// -// {({ checked }) => ( -// -// )} -// -// - -// {form.watch('type') === 'key' && ( -//
    -// form.setValue('mountAssociatedKey', e)} -// /> -// -// Automatically mount key -// -// -// -// -//
    -// )} - -// {form.watch('type') === 'password' && ( -// <> -// - -//
    -// -// -// Save to Key Manager -// -// -// -// -//
    -// -// )} - -//

    Output file

    -// -//
    -//     -// ); -// }; diff --git a/interface/app/$libraryId/Explorer/FilePath/DeleteDialog.tsx b/interface/app/$libraryId/Explorer/FilePath/DeleteDialog.tsx index bd9d2fdb0..8ce5a5184 100644 --- a/interface/app/$libraryId/Explorer/FilePath/DeleteDialog.tsx +++ b/interface/app/$libraryId/Explorer/FilePath/DeleteDialog.tsx @@ -110,7 +110,7 @@ export default (props: Props) => { dialog={useDialog(props)} title={t('delete_dialog_title', { prefix, type: translatedType })} description={description} - loading={deleteFile.isLoading} + loading={deleteFile.isPending} ctaLabel={t('delete_forever')} ctaSecondLabel={t('move_to_trash')} closeLabel={t('close')} diff --git a/interface/app/$libraryId/Explorer/FilePath/EncryptDialog.tsx b/interface/app/$libraryId/Explorer/FilePath/EncryptDialog.tsx deleted file mode 100644 index 9daf21a7a..000000000 --- a/interface/app/$libraryId/Explorer/FilePath/EncryptDialog.tsx +++ /dev/null @@ -1,177 +0,0 @@ -// import { -// Algorithm, -// hashingAlgoSlugSchema, -// slugFromHashingAlgo, -// useLibraryMutation, -// useLibraryQuery -// } from '@sd/client'; -// import { Button, Dialog, Select, SelectOption, UseDialogProps, useDialog } from '@sd/ui'; -// import { CheckBox, useZodForm, z } from '@sd/ui/src/forms'; -// import { showAlertDialog } from '~/components'; -// import { usePlatform } from '~/util/Platform'; -// import { KeyListSelectOptions } from '../../KeyManager/List'; - -// interface Props extends UseDialogProps { -// location_id: number; -// path_id: number; -// } - -// const schema = z.object({ -// key: z.string(), -// encryptionAlgo: z.string(), -// hashingAlgo: hashingAlgoSlugSchema, -// metadata: z.boolean(), -// previewMedia: z.boolean(), -// outputPath: z.string() -// }); - -// export default (props: Props) => { -// const platform = usePlatform(); - -// const UpdateKey = (uuid: string) => { -// form.setValue('key', uuid); -// const hashAlg = keys.data?.find((key) => { -// return key.uuid === uuid; -// })?.hashing_algorithm; -// hashAlg && form.setValue('hashingAlgo', slugFromHashingAlgo(hashAlg)); -// }; - -// const keys = useLibraryQuery(['keys.list']); -// const mountedUuids = useLibraryQuery(['keys.listMounted'], { -// onSuccess: (data) => { -// UpdateKey(data[0] ?? ''); -// } -// }); - -// const encryptFile = useLibraryMutation('files.encryptFiles', { -// onSuccess: () => { -// showAlertDialog({ -// title: 'Success', -// value: 'The encryption job has started successfully. You may track the progress in the job overview panel.' -// }); -// }, -// onError: () => { -// showAlertDialog({ -// title: 'Error', -// value: 'The encryption job failed to start.' -// }); -// } -// }); - -// const form = useZodForm({ -// defaultValues: { encryptionAlgo: 'XChaCha20Poly1305', outputPath: '' }, -// schema -// }); - -// return ( -// -// encryptFile.mutateAsync({ -// algorithm: data.encryptionAlgo as Algorithm, -// key_uuid: data.key, -// location_id: props.location_id, -// file_path_ids: [props.path_id], -// metadata: data.metadata, -// preview_media: data.previewMedia -// }) -// )} -// dialog={useDialog(props)} -// title="Encrypt a file" -// description="Configure your encryption settings. Leave the output file blank for the default." -// loading={encryptFile.isLoading} -// ctaLabel="Encrypt" -// > -//
    -//
    -// Key -// -//
    -//
    -// Output file - -// -//
    -//
    - -//
    -//
    -// Encryption -// -//
    -//
    -// Hashing -// -//
    -//
    - -//
    -//
    -// Metadata -// -//
    -//
    -// Preview Media -// -//
    -//
    -//     -// ); -// }; diff --git a/interface/app/$libraryId/Explorer/FilePath/EraseDialog.tsx b/interface/app/$libraryId/Explorer/FilePath/EraseDialog.tsx deleted file mode 100644 index 4bd84d28e..000000000 --- a/interface/app/$libraryId/Explorer/FilePath/EraseDialog.tsx +++ /dev/null @@ -1,68 +0,0 @@ -// import { useState } from 'react'; -// import { FilePath, useLibraryMutation, useZodForm } from '@sd/client'; -// import { Dialog, Slider, useDialog, UseDialogProps, z } from '@sd/ui'; -// import { useLocale } from '~/hooks'; - -// interface Props extends UseDialogProps { -// locationId: number; -// filePaths: FilePath[]; -// } - -// const schema = z.object({ -// passes: z.number() -// }); - -// export default (props: Props) => { -// const { t } = useLocale(); -// const eraseFile = useLibraryMutation('files.eraseFiles'); - -// const form = useZodForm({ -// schema, -// defaultValues: { -// passes: 4 -// } -// }); - -// const [passes, setPasses] = useState([4]); - -// return ( -// -// eraseFile.mutateAsync({ -// location_id: props.locationId, -// file_path_ids: props.filePaths.map((p) => p.id), -// passes: data.passes.toString() -// }) -// )} -// dialog={useDialog(props)} -// title={t('erase_a_file')} -// description={t('erase_a_file_description')} -// loading={eraseFile.isLoading} -// ctaLabel={t('erase')} -// > -//
    -// {t('number_of_passes')} - -//
    -//
    -// { -// setPasses(val); -// form.setValue('passes', val[0] ?? 1); -// }} -// /> -//
    -// {passes} -//
    -//
    - -// {/*

    TODO: checkbox for "erase all matching files" (only if a file is selected)

    */} -//     -// ); -// }; diff --git a/interface/app/$libraryId/Explorer/FilePath/ErrorBarrier.tsx b/interface/app/$libraryId/Explorer/FilePath/ErrorBarrier.tsx new file mode 100644 index 000000000..9290ca589 --- /dev/null +++ b/interface/app/$libraryId/Explorer/FilePath/ErrorBarrier.tsx @@ -0,0 +1,40 @@ +import React, { Component, ReactNode } from 'react'; + +interface ErrorBarrierProps { + onError: (error: Error, info: React.ErrorInfo) => void; + children: ReactNode; +} + +interface ErrorBarrierState { + hasError: boolean; +} + +export class ErrorBarrier extends Component { + constructor(props: ErrorBarrierProps) { + super(props); + this.state = { hasError: false }; + } + + static getDerivedStateFromError(error: Error) { + // Update state so the next render will show the fallback UI. + return { hasError: true }; + } + + componentDidCatch(error: Error, info: React.ErrorInfo) { + // Call the onError function passed as a prop + this.props.onError(error, info); + // Reset the error state after calling onError + Promise.resolve().then(() => this.setState({ hasError: false })); + } + + render() { + if (this.state.hasError) { + // Render nothing since the parent component will handle the error + return null; + } + + return this.props.children; + } +} + +export default ErrorBarrier; diff --git a/interface/app/$libraryId/Explorer/FilePath/Image.tsx b/interface/app/$libraryId/Explorer/FilePath/Image.tsx index 5be914315..e7036595b 100644 --- a/interface/app/$libraryId/Explorer/FilePath/Image.tsx +++ b/interface/app/$libraryId/Explorer/FilePath/Image.tsx @@ -1,14 +1,7 @@ import { ComponentProps, forwardRef } from 'react'; -import { useSize } from './utils'; - -export interface ImageProps extends ComponentProps<'img'> { - extension?: string; - size: ReturnType; -} - -export const Image = forwardRef( - ({ crossOrigin, size, ...props }, ref) => ( +export const Image = forwardRef>( + ({ crossOrigin, ...props }, ref) => ( { +interface LayeredFileIconProps extends Omit, 'src'> { kind: ObjectKindKey; + isDir: boolean; extension: string | null; + customIcon: IconTypes | null; } const SUPPORTED_ICONS = ['Document', 'Code', 'Text', 'Config']; @@ -17,8 +20,18 @@ const positionConfig: Record = { }; const LayeredFileIcon = forwardRef( - ({ kind, extension, ...props }, ref) => { - const iconImg = ; + ({ kind, isDir, extension, customIcon, ...props }, ref) => { + const isDark = useIsDark(); + + const src = useMemo( + () => + customIcon + ? getIconByName(customIcon, isDark) + : getIcon(kind, isDark, extension, isDir), + [customIcon, isDark, kind, extension, isDir] + ); + + const iconImg = {`${kind}; if (SUPPORTED_ICONS.includes(kind) === false) { return iconImg; diff --git a/interface/app/$libraryId/Explorer/FilePath/Original.tsx b/interface/app/$libraryId/Explorer/FilePath/Original.tsx index 451b13d35..6b319793e 100644 --- a/interface/app/$libraryId/Explorer/FilePath/Original.tsx +++ b/interface/app/$libraryId/Explorer/FilePath/Original.tsx @@ -8,27 +8,28 @@ import { useState, type VideoHTMLAttributes } from 'react'; -import { getItemFilePath, useLibraryContext } from '@sd/client'; +import { ObjectKindKey, useLibraryContext } from '@sd/client'; import i18n from '~/app/I18n'; import { PDFViewer, TextViewer } from '~/components'; -import { useLocale } from '~/hooks'; +import { useIsDark, useLocale } from '~/hooks'; import { pdfViewerEnabled } from '~/util/pdfViewer'; import { usePlatform } from '~/util/Platform'; import { useExplorerContext } from '../Context'; import { explorerStore } from '../store'; -import { ExplorerItemData } from '../useExplorerItemData'; import { Image } from './Image'; import { useBlackBars, useSize } from './utils'; interface OriginalRendererProps { src: string; - className: string; - frameClassName: string; - itemData: ExplorerItemData; - isDark: boolean; + fileId: number | null; + locationId: number | null; + path: string | null; + className?: string; + frameClassName?: string; + kind: ObjectKindKey; + extension: string | null; childClassName?: string; - size?: number; magnification?: number; mediaControls?: boolean; frame?: boolean; @@ -37,44 +38,53 @@ interface OriginalRendererProps { blackBars?: boolean; blackBarsSize?: number; onLoad?(): void; - onError?(e: ErrorEvent | SyntheticEvent): void; } export function Original({ - itemData, - filePath, + path, + fileId, + locationId, ...props -}: Omit & { - filePath: ReturnType; -}) { - const [error, setError] = useState(false); - if (error) throw new Error('onError'); +}: Omit) { + const [error, setError] = useState(null); + if (error != null) throw error; const Renderer = useMemo(() => { - const kind = originalRendererKind(itemData); + const kind = originalRendererKind(props.kind, props.extension); return ORIGINAL_RENDERERS[kind]; - }, [itemData]); + }, [props.kind, props.extension]); if (!Renderer) throw new Error('no renderer!'); const platform = usePlatform(); const { library } = useLibraryContext(); const { parent } = useExplorerContext(); + locationId = locationId ?? (parent?.type === 'Location' ? parent.location.id : null); const src = useMemo(() => { - const locationId = - itemData.locationId ?? (parent?.type === 'Location' ? parent.location.id : null); - - if (filePath && (itemData.extension !== 'pdf' || pdfViewerEnabled())) { - if ('id' in filePath && locationId) - return platform.getFileUrl(library.uuid, locationId, filePath.id); - else if ('path' in filePath) return platform.getFileUrlByPath(filePath.path); + if (props.extension !== 'pdf' || pdfViewerEnabled()) { + if (fileId != null && locationId) + return platform.getFileUrl(library.uuid, locationId, fileId); + else if (path) return platform.getFileUrlByPath(path); } - }, [itemData, filePath, library.uuid, parent, platform]); + }, [props.extension, fileId, locationId, platform, library.uuid, path]); if (src === undefined) throw new Error('no src!'); - return setError(true)} {...props} />; + return ( + + setError( + ('error' in event && event.error instanceof Error && event.error) || + new Error( + ('message' in event && event.message) || 'Filetype is not supported yet' + ) + ) + } + {...props} + /> + ); } const TEXT_RENDERER: OriginalRenderer = (props) => ( @@ -89,18 +99,20 @@ const TEXT_RENDERER: OriginalRenderer = (props) => ( props.frame && [props.frameClassName, '!bg-none p-2'] )} codeExtension={ - ((props.itemData.kind === 'Code' || props.itemData.kind === 'Config') && - props.itemData.extension) || - '' + ((props.kind === 'Code' || props.kind === 'Config') && props.extension) || '' } isSidebarPreview={props.isSidebarPreview} /> ); -type OriginalRenderer = (props: OriginalRendererProps) => JSX.Element; +type OriginalRenderer = ( + props: Omit & { + onError?(e: ErrorEvent | SyntheticEvent): void; + } +) => JSX.Element; -function originalRendererKind(itemData: ExplorerItemData) { - return itemData.extension === 'pdf' ? 'PDF' : itemData.kind; +function originalRendererKind(kind: ObjectKindKey, extension: string | null) { + return extension === 'pdf' ? 'PDF' : kind; } type OriginalRendererKind = ReturnType; @@ -135,44 +147,45 @@ const ORIGINAL_RENDERERS: { )} /> ), - Audio: (props) => ( - <> - - {props.mediaControls && ( - - )} - - ), + Audio: (props) => { + const isDark = useIsDark(); + return ( + <> + + {props.mediaControls && ( + + )} + + ); + }, Image: (props) => { const ref = useRef(null); - const size = useSize(ref); return (
    { + cover?: boolean; + blackBars?: boolean; + blackBarsSize?: number; + videoExtension?: string; +} + +const Thumbnail = memo( + forwardRef( + ( + { + blackBars, + blackBarsSize, + videoExtension: extension, + cover, + className, + style, + ...props + }, + _ref + ) => { + const ref = useRef(null); + useImperativeHandle( + _ref, + () => ref.current + ); + + const size = useSize(ref); + + const { style: blackBarsStyle } = useBlackBars(ref, size, { + size: blackBarsSize, + disabled: !blackBars + }); + + return ( + <> + + + {(cover || size.width > 80) && extension && ( +
    + {extension} +
    + )} + + ); + } + ) +); + +interface ThumbProps extends ThumbnailProps { + src?: string; + kind: ObjectKindKey; + path: string | null; + isDir: boolean; + frame: boolean; + fileId: number | null; + onLoad: () => void; + onError: (error: Error | ErrorEvent | SyntheticEvent) => void; + thumbType: ThumbType; + extension: string | null; + customIcon: IconTypes | null; + locationId: number | null; + pauseVideo: boolean; + magnification: number; + mediaControls: boolean; + frameClassName: string; + isSidebarPreview: boolean; +} + +const Thumb = memo( + forwardRef( + ( + { + src, + kind, + path, + frame, + isDir, + cover, + fileId, + thumbType, + extension, + blackBars, + className, + pauseVideo, + locationId, + customIcon, + magnification, + mediaControls, + blackBarsSize, + videoExtension, + frameClassName, + isSidebarPreview, + onLoad, + ...props + }, + _ref + ) => { + const ref = useRef(null); + useImperativeHandle( + _ref, + () => ref.current + ); + const [isLoading, setIsLoading] = useState(true); + + const handleLoad = useCallback(() => { + const img = ref.current; + setIsLoading(!(img == null || (img.naturalHeight > 0 && img.naturalWidth > 0))); + onLoad?.(); + }, [onLoad]); + + let thumb: JSX.Element | null = null; + + switch (thumbType) { + case 'original': + thumb = ( + + ); + break; + case 'thumbnail': + thumb = ( + + ); + break; + } + + return ( + <> + {}} + decoding="sync" + draggable={false} + extension={extension} + className={clsx(ThumbClasses, className, !isLoading && 'hidden')} + customIcon={customIcon} + /> + {thumb ?? null} + + ); + } + ) +); + +export interface FileThumbProps { data: ExplorerItem; loadOriginal?: boolean; size?: number; cover?: boolean; frame?: boolean; - onLoad?: (state: ThumbType) => void; - onError?: (state: ThumbType, error: Error) => void; + onLoad?: (type: ThumbType) => void; + onError?: (state: LoadState, error: Error) => void; blackBars?: boolean; blackBarsSize?: number; extension?: boolean; @@ -43,253 +259,181 @@ export interface ThumbProps { magnification?: number; } -type ThumbType = { variant: 'original' } | { variant: 'thumbnail' } | { variant: 'icon' }; -type LoadState = { - [K in 'original' | 'thumbnail' | 'icon']: 'notLoaded' | 'loaded' | 'error'; -}; +/** + * This component is used to render a thumbnail of a file or folder. + * It will automatically choose the best thumbnail to display based on the item data. + * + * .. WARNING:: + * This Component is heavely used inside the explorer, and as such it is a performance critical component. + * Be careful with the performance of the code, make sure to always memoize any objects or functions to avoid unnecessary re-renders. + * + */ +export const FileThumb = memo( + forwardRef((props, ref) => { + const frame = useFrame(); + const platform = usePlatform(); + const itemData = useExplorerItemData(props.data); + const filePath = getItemFilePath(props.data); + const { library } = useLibraryContext(); + const [loadState, setLoadState] = useState({ + icon: 'normal', + original: 'normal', + thumbnail: 'normal' + }); -export const FileThumb = forwardRef((props, ref) => { - const isDark = useIsDark(); - const platform = usePlatform(); - const frame = useFrame(); + // WARNING: This is required so QuickPreview can work properly + useEffect(() => { + setLoadState({ + icon: 'normal', + original: 'normal', + thumbnail: 'normal' + }); + }, [props.data]); - const itemData = useExplorerItemData(props.data); - const filePath = getItemFilePath(props.data); + const thumbType = useMemo((): ThumbType => { + if (loadState.original !== 'error' && props.loadOriginal) return 'original'; + if (loadState.thumbnail !== 'error' && itemData.hasLocalThumbnail) return 'thumbnail'; + return 'icon'; + }, [itemData.hasLocalThumbnail, loadState, props.loadOriginal]); - const { library } = useLibraryContext(); - - const [loadState, setLoadState] = useState({ - original: 'notLoaded', - thumbnail: 'notLoaded', - icon: 'notLoaded' - }); - - const childClassName = 'max-h-full max-w-full object-contain'; - const frameClassName = clsx(frame.className, props.frameClassName); - - const thumbType = useMemo(() => { - const thumbType = 'thumbnail'; - - if (thumbType === 'thumbnail') - if ( - loadState.thumbnail !== 'error' && - itemData.hasLocalThumbnail && - itemData.thumbnailKey - ) - return { variant: 'thumbnail' }; - - return { variant: 'icon' }; - }, [itemData, loadState]); - - const src = useMemo(() => { - switch (thumbType.variant) { - case 'original': - if (filePath && (itemData.extension !== 'pdf' || pdfViewerEnabled())) { - if ('id' in filePath && itemData.locationId) - return platform.getFileUrl(library.uuid, itemData.locationId, filePath.id); - else if ('path' in filePath) return platform.getFileUrlByPath(filePath.path); + useEffect(() => { + let timeoutId = null; + // Reload thumbnail when it gets a notification from core that it has been generated + if (thumbType === 'icon' && loadState.thumbnail === 'error') { + for (const [, thumbId] of itemData.thumbnails) { + if (thumbId == null || !explorerStore.newThumbnails.has(thumbId)) continue; + // HACK: Delay removing the new thumbnail event from store + // to avoid some weird race condition with core that prevents + // us from accessing the new thumbnail immediately after it is created + timeoutId = setTimeout(() => explorerStore.removeThumbnail(thumbId), 0); + explorerStore.removeThumbnail(thumbId); + setLoadState((state) => ({ ...state, thumbnail: 'normal' })); + break; } - break; + } - case 'thumbnail': - if (itemData.thumbnailKey) - return platform.getThumbnailUrlByThumbKey(itemData.thumbnailKey); + return () => void (timeoutId && clearTimeout(timeoutId)); + }, [itemData.thumbnails, loadState.thumbnail, thumbType]); - break; - case 'icon': - if (itemData.customIcon) return getIconByName(itemData.customIcon as any, isDark); + const src = useMemo(() => { + switch (thumbType) { + case 'original': + if (filePath && (itemData.extension !== 'pdf' || pdfViewerEnabled())) { + if ('id' in filePath && itemData.locationId) + return platform.getFileUrl( + library.uuid, + itemData.locationId, + filePath.id + ); + else if ('path' in filePath) + return platform.getFileUrlByPath(filePath.path); + else setLoadState((state) => ({ ...state, [thumbType]: 'error' })); + } + break; - return getIcon( - // itemData.isDir || parent?.type === 'Node' ? 'Folder' : - itemData.kind, - isDark, - itemData.extension, - itemData.isDir - ); - } - }, [filePath, isDark, library.uuid, itemData, platform, thumbType]); + case 'thumbnail': { + const thumbnail = Array.from(itemData.thumbnails.keys()).find((key) => key); + if (thumbnail) return thumbnail; + else setLoadState((state) => ({ ...state, [thumbType]: 'error' })); - const onLoad = (s: 'original' | 'thumbnail' | 'icon') => { - setLoadState((state) => ({ ...state, [s]: 'loaded' })); - props.onLoad?.call(null, thumbType); - }; + break; + } + } + }, [ + filePath, + itemData.extension, + itemData.locationId, + itemData.thumbnails, + library.uuid, + platform, + thumbType + ]); - const onError = ( - s: 'original' | 'thumbnail' | 'icon', - event: ErrorEvent | SyntheticEvent - ) => { - setLoadState((state) => ({ ...state, [s]: 'error' })); + const onError = useCallback( + (event: Error | ErrorEvent | SyntheticEvent) => { + const rawError = + event instanceof Error + ? event + : ('error' in event && event.error) || + ('message' in event && event.message) || + 'Filetype is not supported yet'; - const rawError = - ('error' in event && event.error) || - ('message' in event && event.message) || - 'Filetype is not supported yet'; - - props.onError?.call( - null, - thumbType, - rawError instanceof Error ? rawError : new Error(rawError) + setLoadState((state) => { + state = { ...state, [thumbType]: 'error' }; + props.onError?.call( + null, + state, + rawError instanceof Error ? rawError : new Error(rawError) + ); + return state; + }); + }, + [props.onError, thumbType] ); - }; - const _childClassName = - typeof props.childClassName === 'function' - ? props.childClassName(thumbType) - : props.childClassName; + const onLoad = useCallback(() => { + props.onLoad?.call(null, thumbType); + }, [props.onLoad, thumbType]); - const className = clsx(childClassName, _childClassName); - - const thumbnail = (() => { - if (!src) return <>; - - switch (thumbType.variant) { - case 'thumbnail': - return ( - onLoad('thumbnail')} - onError={(e) => onError('thumbnail', e)} - decoding={props.size ? 'async' : 'sync'} - className={clsx( - props.cover - ? [ - 'min-h-full min-w-full object-cover object-center', - _childClassName - ] - : className, - props.frame && !(itemData.kind === 'Video' && props.blackBars) - ? frameClassName - : null - )} - crossOrigin="anonymous" // Here it is ok, because it is not a react attr - blackBars={props.blackBars && itemData.kind === 'Video' && !props.cover} - blackBarsSize={props.blackBarsSize} - extension={ - props.extension && itemData.extension && itemData.kind === 'Video' - ? itemData.extension - : undefined - } - /> - ); - - case 'icon': - return ( - + { + console.error('ErrorBoundary', error, info); + onError(error); + }, + [onError] + )} + > + onLoad('icon')} - onError={(e) => onError('icon', e)} - decoding={props.size ? 'async' : 'sync'} - className={className} - draggable={false} - /> - ); - default: - return <>; - } - })(); - - return ( -
    - {props.loadOriginal ? ( - - onLoad('original')} - onError={(e) => onError('original', e)} - filePath={filePath} - className={className} - frameClassName={frameClassName} - itemData={itemData} - isDark={isDark} - childClassName={childClassName} - size={props.size} - magnification={props.magnification} - mediaControls={props.mediaControls} - frame={props.frame} - isSidebarPreview={props.isSidebarPreview} - pauseVideo={props.pauseVideo} blackBars={props.blackBars} + className={ + typeof props.childClassName === 'function' + ? props.childClassName(thumbType) + : props.childClassName + } + customIcon={itemData.customIcon as IconTypes | null} + locationId={itemData.locationId} + pauseVideo={props.pauseVideo ?? false} blackBarsSize={props.blackBarsSize} + mediaControls={props.mediaControls ?? false} + magnification={props.magnification ?? 1} + frameClassName={clsx(frame.className, props.frameClassName)} + videoExtension={ + props.extension && itemData.extension && itemData.kind === 'Video' + ? itemData.extension + : undefined + } + isSidebarPreview={props.isSidebarPreview ?? false} /> - - ) : ( - thumbnail - )} -
    - ); -}); - -interface ThumbnailProps extends Omit { - cover?: boolean; - blackBars?: boolean; - blackBarsSize?: number; - extension?: string; -} - -const Thumbnail = forwardRef( - ({ blackBars, blackBarsSize, extension, cover, className, style, ...props }, _ref) => { - const ref = useRef(null); - useImperativeHandle( - _ref, - () => ref.current +     +
    ); - - const size = useSize(ref); - - const { style: blackBarsStyle } = useBlackBars(ref, size, { - size: blackBarsSize, - disabled: !blackBars - }); - - return ( - <> - - - {(cover || size.width > 80) && extension && ( -
    - {extension} -
    - )} - - ); - } + }) ); diff --git a/interface/app/$libraryId/Explorer/Inspector/FavoriteButton.tsx b/interface/app/$libraryId/Explorer/Inspector/FavoriteButton.tsx index 97abc0d27..71eea2117 100644 --- a/interface/app/$libraryId/Explorer/Inspector/FavoriteButton.tsx +++ b/interface/app/$libraryId/Explorer/Inspector/FavoriteButton.tsx @@ -14,7 +14,7 @@ export default function FavoriteButton(props: Props) { setFavorite(!!props.data?.favorite); }, [props.data]); - const { mutate: fileToggleFavorite, isLoading: isFavoriteLoading } = useLibraryMutation( + const { mutate: fileToggleFavorite, isPending: isFavoriteLoading } = useLibraryMutation( 'files.setFavorite' // { // onError: () => setFavorite(!!props.data?.favorite) diff --git a/interface/app/$libraryId/Explorer/Inspector/index.tsx b/interface/app/$libraryId/Explorer/Inspector/index.tsx index 08b3255d0..7d0226396 100644 --- a/interface/app/$libraryId/Explorer/Inspector/index.tsx +++ b/interface/app/$libraryId/Explorer/Inspector/index.tsx @@ -154,7 +154,7 @@ const Thumbnails = ({ items }: { items: ExplorerItem[] }) => { i === 2 && 'z-10 !h-[84%] !w-[84%] rotate-[7deg]' )} childClassName={(type) => - type.variant !== 'icon' && thumbs.length > 1 + type !== 'icon' && thumbs.length > 1 ? 'shadow-md shadow-app-shade' : undefined } diff --git a/interface/app/$libraryId/Explorer/QuickPreview/index.tsx b/interface/app/$libraryId/Explorer/QuickPreview/index.tsx index ad1b0fdc5..918af62c9 100644 --- a/interface/app/$libraryId/Explorer/QuickPreview/index.tsx +++ b/interface/app/$libraryId/Explorer/QuickPreview/index.tsx @@ -50,7 +50,7 @@ import ExplorerContextMenu, { SharedItems } from '../ContextMenu'; import { Conditional } from '../ContextMenu/ConditionalItem'; -import { FileThumb } from '../FilePath/Thumb'; +import { FileThumb, ThumbType } from '../FilePath/Thumb'; import { SingleItemMetadata } from '../Inspector'; import { explorerStore } from '../store'; import { useExplorerViewContext } from '../View/Context'; @@ -84,19 +84,31 @@ export const QuickPreview = () => { const { open, itemIndex } = useQuickPreviewStore(); const thumb = createRef(); - const [thumbErrorToast, setThumbErrorToast] = useState(); const [showMetadata, setShowMetadata] = useState(false); const [magnification, setMagnification] = useState(1); const [isContextMenuOpen, setIsContextMenuOpen] = useState(false); const [isRenaming, setIsRenaming] = useState(false); const [newName, setNewName] = useState(null); - const [thumbnailLoading, setThumbnailLoading] = useState<'notLoaded' | 'loaded' | 'error'>( - 'notLoaded' - ); + const [thumbnailLoading, setThumbnailLoading] = useState({ + icon: 'notLoaded', + thumbnail: 'notLoaded', + original: 'notLoaded' + } as { + [K in ThumbType]: 'notLoaded' | 'loaded' | 'error'; + }); // the purpose of these refs is to prevent "jittering" when zooming with trackpads, as the deltaY value can be very high const deltaYRef = useRef(0); const lastZoomTimeRef = useRef(0); + const hasError = useMemo( + () => Object.values(thumbnailLoading).some((status) => status === 'error'), + [thumbnailLoading] + ); + const isLoaded = useMemo( + () => Object.values(thumbnailLoading).some((status) => status === 'loaded'), + [thumbnailLoading] + ); + const { t } = useLocale(); const items = useMemo(() => { @@ -122,50 +134,26 @@ export const QuickPreview = () => { const renameFile = useLibraryMutation(['files.renameFile'], { onError: () => setNewName(null), - onSuccess: () => rspc.queryClient.invalidateQueries(['search.paths']) + onSuccess: () => rspc.queryClient.invalidateQueries({ queryKey: ['search.paths'] }) }); const renameEphemeralFile = useLibraryMutation(['ephemeralFiles.renameFile'], { onError: () => setNewName(null), - onSuccess: () => rspc.queryClient.invalidateQueries(['search.paths']) + onSuccess: () => rspc.queryClient.invalidateQueries({ queryKey: ['search.paths'] }) }); const changeCurrentItem = (index: number) => { if (items[index]) getQuickPreviewStore().itemIndex = index; }; - // Error toast - useEffect(() => { - if (!thumbErrorToast) return; - - let id: string | number | undefined; - toast.error( - (_id) => { - id = _id; - return thumbErrorToast; - }, - { - ref: thumb, - duration: Infinity, - onClose() { - id = undefined; - setThumbErrorToast(undefined); - } - } - ); - - return () => void toast.dismiss(id); - }, [thumb, thumbErrorToast]); - // Reset state useEffect(() => { setNewName(null); - setThumbErrorToast(undefined); setMagnification(1); + setThumbnailLoading({ icon: 'notLoaded', thumbnail: 'notLoaded', original: 'notLoaded' }); if (open || item) return; - setThumbnailLoading('notLoaded'); getQuickPreviewStore().open = false; getQuickPreviewStore().itemIndex = 0; setShowMetadata(false); @@ -344,18 +332,12 @@ export const QuickPreview = () => { )} >
    - {thumbnailLoading !== 'error' && - thumbnailLoading !== 'notLoaded' && - background && ( -
    - -
    -
    - )} + {!hasError && isLoaded && background && ( +
    + +
    +
    + )}
    { - {thumbnailLoading === 'error' && ( - -
    - -

    - {t('quickpreview_thumbnail_error_message')} -

    -
    -     - )} + {thumbnailLoading.original === 'error' && + thumbnailLoading.thumbnail === 'loaded' && ( + +
    + +

    + {t( + 'quickpreview_thumbnail_error_message' + )} +

    +
    +     + )} {items.length > 1 && (
    @@ -616,56 +603,42 @@ export const QuickPreview = () => {
    - {thumbnailLoading === 'error' ? ( - <> - - - ) : ( - { - setThumbnailLoading('loaded'); - if (type.variant === 'original') - setThumbErrorToast(undefined); - }} - onError={(type, error) => { - setThumbnailLoading('error'); - if (type.variant === 'original') - setThumbErrorToast({ - title: t('error_loading_original_file'), - body: error.message - }); - }} - loadOriginal - frameClassName="!border-0" - mediaControls - className={clsx( - thumbnailLoading === 'notLoaded' && 'hidden', - 'm-3 !w-auto flex-1 !overflow-hidden rounded', - !background && !icon && 'bg-app-box shadow' - )} - childClassName={clsx( - 'rounded', - kind === 'Text' && 'p-3', - !icon && 'h-full', - textKinds.includes(kind) && 'select-text' - )} - magnification={magnification} - /> - )} + { + setThumbnailLoading((obj) => ({ + ...obj, + [type]: 'loaded' + })); + }} + onError={(state, error) => { + console.error(error); + setThumbnailLoading((obj) => { + const newState = { ...obj }; + for (const [type, loadState] of Object.entries( + state + ) as [ThumbType, string][]) + if (loadState === 'error') newState[type] = 'error'; + + return newState; + }); + }} + loadOriginal + frameClassName="!border-0" + mediaControls + className={clsx( + !isLoaded && 'hidden', + 'm-3 !w-auto flex-1 !overflow-hidden rounded', + !background && !icon && 'bg-app-box shadow' + )} + childClassName={clsx( + 'rounded', + kind === 'Text' && 'p-3', + !icon && 'h-full', + textKinds.includes(kind) && 'select-text' + )} + magnification={magnification} + /> {explorerLayoutStore.showImageSlider && activeItem && ( diff --git a/interface/app/$libraryId/Explorer/View/Grid/DragSelect/index.tsx b/interface/app/$libraryId/Explorer/View/Grid/DragSelect/index.tsx index e58a56787..e93f85355 100644 --- a/interface/app/$libraryId/Explorer/View/Grid/DragSelect/index.tsx +++ b/interface/app/$libraryId/Explorer/View/Grid/DragSelect/index.tsx @@ -13,8 +13,16 @@ import { getElementIndex, SELECTABLE_DATA_ATTRIBUTE } from './util'; const CHROME_REGEX = /Chrome/; +type GridOpts = ReturnType>; + interface Props extends PropsWithChildren { - grid: ReturnType>; + columnCount: GridOpts['columnCount']; + gapY: GridOpts['gap']['y']; + getItem: GridOpts['getItem']; + totalColumnCount: GridOpts['totalColumnCount']; + totalCount: GridOpts['totalCount']; + totalRowCount: GridOpts['totalRowCount']; + virtualItemHeight: GridOpts['virtualItemHeight']; } export interface Drag { @@ -24,7 +32,7 @@ export interface Drag { endRow: number; } -export const DragSelect = ({ grid, children }: Props) => { +export const DragSelect = ({ children, ...props }: Props) => { const isChrome = CHROME_REGEX.test(navigator.userAgent); const { explorerOperatingSystem, matchingOperatingSystem } = useExplorerOperatingSystem(); @@ -62,7 +70,7 @@ export const DragSelect = ({ grid, children }: Props) => { function getGridItem(element: Element) { const index = getElementIndex(element); - return (index !== null && grid.getItem(index)) || undefined; + return (index !== null && props.getItem(index)) || undefined; } function handleScroll(e: SelectoEvents['scroll']) { @@ -176,9 +184,9 @@ export const DragSelect = ({ grid, children }: Props) => { // that are still in the DOM const elements: Element[] = []; - e.added.forEach((element) => { + for (const element of e.added) { const item = getGridItem(element); - if (!item?.data) return; + if (!item?.data) continue; // Add item to selected targets // Don't update selecto as it's already aware of it @@ -188,22 +196,22 @@ export const DragSelect = ({ grid, children }: Props) => { explorer.addSelectedItem(item.data); if (document.contains(element)) elements.push(element); - }); + } - e.removed.forEach((element) => { + for (const element of e.removed) { const item = getGridItem(element); - if (!item?.data) return; + if (!item?.data) continue; // Remove item from selected targets // Don't update selecto as it's already aware of it selectedTargets.removeSelectedTarget(String(item.id), { updateSelecto: false }); // Don't deselect item if element is unmounted by scroll - if (!document.contains(element)) return; + if (!document.contains(element)) continue; explorer.removeSelectedItem(item.data); elements.push(element); - }); + } const dragDirection = { x: inputEvent.x === e.rect.left ? 'left' : 'right', @@ -280,7 +288,7 @@ export const DragSelect = ({ grid, children }: Props) => { const addedRows = new Set(); const removedRows = new Set(); - columns.forEach((column) => { + for (const column of columns) { const { firstItem, lastItem } = columnItems[column]!; const { row: firstRow } = firstItem.item; @@ -353,7 +361,7 @@ export const DragSelect = ({ grid, children }: Props) => { // Remove row if dragged out of the last grid item // from a row that's above it - if (item.item.index === grid.totalCount - 1) { + if (item.item.index === props.totalCount - 1) { removedRows.add(item.item.row); } } @@ -372,21 +380,21 @@ export const DragSelect = ({ grid, children }: Props) => { // caches multiple rows at once, and the first one being removed if ( !isFirstRowInDrag && - firstRow === grid.totalRowCount - 2 && - firstItem.item.index + grid.totalColumnCount > grid.totalCount - 1 + firstRow === props.totalRowCount - 2 && + firstItem.item.index + props.totalColumnCount > props.totalCount - 1 ) { removedColumns.add(column); } // Return if first row equals the first/last row of the grid (depending on drag direction) // as there's no items to be selected beyond that point - if (!drag.current && (firstRow === 0 || firstRow === grid.totalRowCount - 1)) { - return; + if (!drag.current && (firstRow === 0 || firstRow === props.totalRowCount - 1)) { + continue; } // Return if column is already in drag range if (isColumnInDrag && isColumnInDragRange) { - return; + continue; } const viewTop = explorerView.ref.current?.getBoundingClientRect().top ?? 0; @@ -397,9 +405,9 @@ export const DragSelect = ({ grid, children }: Props) => { const hasEmptySpace = dragDirection.y === 'down' ? dragStart.y < itemTop : dragStart.y > itemBottom; - if (!hasEmptySpace) return; + if (!hasEmptySpace) continue; - // Get the heigh of the empty drag space between the start of the drag + // Get the height of the empty drag space between the start of the drag // and the first visible item const emptySpaceHeight = Math.abs( dragStart.y - (dragDirection.y === 'down' ? itemTop : itemBottom) @@ -407,8 +415,8 @@ export const DragSelect = ({ grid, children }: Props) => { // Check how many items we can fit into the empty space let itemsInEmptySpace = - (emptySpaceHeight - (grid.gap.y ?? 0)) / - (grid.virtualItemHeight + (grid.gap.y ?? 0)); + (emptySpaceHeight - (props.gapY ?? 0)) / + (props.virtualItemHeight + (props.gapY ?? 0)); if (itemsInEmptySpace > 1) { itemsInEmptySpace = Math.ceil(itemsInEmptySpace); @@ -416,15 +424,15 @@ export const DragSelect = ({ grid, children }: Props) => { itemsInEmptySpace = Math.round(itemsInEmptySpace); } - [...Array(itemsInEmptySpace)].forEach((_, i) => { + for (let i = 0; i < itemsInEmptySpace; i++) { i = dragDirection.y === 'down' ? itemsInEmptySpace - i : i + 1; const explorerItemIndex = firstItem.item.index + - (dragDirection.y === 'down' ? -i : i) * grid.columnCount; + (dragDirection.y === 'down' ? -i : i) * props.columnCount; - const item = grid.getItem(explorerItemIndex); - if (!item?.data) return; + const item = props.getItem(explorerItemIndex); + if (!item?.data) continue; // Set start row if not already set if (!drag.current && i === itemsInEmptySpace - 1) { @@ -438,13 +446,13 @@ export const DragSelect = ({ grid, children }: Props) => { explorer.addSelectedItem(item.data); } - return; + continue; } if (!isItemInDrag) explorer.removeSelectedItem(item.data); else explorer.addSelectedItem(item.data); - }); - }); + } + } const addedColumnsArray = [...addedColumns]; const removedColumnsArray = [...removedColumns]; diff --git a/interface/app/$libraryId/Explorer/View/GridView/Item/index.tsx b/interface/app/$libraryId/Explorer/View/GridView/Item/index.tsx index aeb40ee0f..a9d01994f 100644 --- a/interface/app/$libraryId/Explorer/View/GridView/Item/index.tsx +++ b/interface/app/$libraryId/Explorer/View/GridView/Item/index.tsx @@ -52,9 +52,10 @@ export const GridViewItem = memo((props: GridViewItemProps) => { ); }); -const InnerDroppable = () => { +const InnerDroppable = memo(() => { const item = useGridViewItemContext(); const { isDroppable } = useExplorerDroppableContext(); + return ( <>
    { (item.selected || isDroppable) && 'bg-app-selectedItem' )} > - +
    ); -}; +}); -const ItemFileThumb = () => { +const ItemFileThumb = memo((props: GridViewItemProps) => { const frame = useFrame(); - - const item = useGridViewItemContext(); - const isLabel = item.data.type === 'Label'; - const { attributes, listeners, style, setDraggableRef } = useExplorerDraggable({ - data: item.data + data: props.data }); + const isLabel = props.data.type === 'Label'; + return ( ({ + style, + ...attributes, + ...listeners + }), + [style, attributes, listeners] + )} /> ); -}; +}); -const ItemMetadata = () => { +const ItemMetadata = memo(() => { const item = useGridViewItemContext(); const { isDroppable } = useExplorerDroppableContext(); const explorerLayout = useExplorerLayoutStore(); @@ -123,9 +125,9 @@ const ItemMetadata = () => { {item.data.type === 'Label' && } ); -}; +}); -const ItemTags = () => { +const ItemTags = memo(() => { const item = useGridViewItemContext(); const object = getItemObject(item.data); const filePath = getItemFilePath(item.data); @@ -150,9 +152,9 @@ const ItemTags = () => { ))}
    ); -}; +}); -const ItemSize = () => { +const ItemSize = memo(() => { const item = useGridViewItemContext(); const { showBytesInGridView } = useExplorerContext().useSettingsSnapshot(); const isRenaming = useSelector(explorerStore, (s) => s.isRenaming); @@ -186,9 +188,9 @@ const ItemSize = () => { {`${bytes}`}
    ); -}; +}); -function LabelItemCount({ data }: { data: Extract }) { +const LabelItemCount = memo(({ data }: { data: Extract }) => { const { t } = useLocale(); const count = useLibraryQuery([ @@ -202,11 +204,11 @@ function LabelItemCount({ data }: { data: Extract {t('item_with_count', { count: count.data })}
    ); -} +}); diff --git a/interface/app/$libraryId/Explorer/View/GridView/index.tsx b/interface/app/$libraryId/Explorer/View/GridView/index.tsx index b5055ee49..507e5ee7d 100644 --- a/interface/app/$libraryId/Explorer/View/GridView/index.tsx +++ b/interface/app/$libraryId/Explorer/View/GridView/index.tsx @@ -1,5 +1,5 @@ import { Grid, useGrid } from '@virtual-grid/react'; -import { useCallback, useEffect, useMemo, useState } from 'react'; +import { useCallback } from 'react'; import { useExplorerLayoutStore } from '@sd/client'; import { useExplorerContext } from '../../Context'; @@ -50,7 +50,15 @@ export const GridView = () => { useKeySelection(grid, { scrollToEnd: true }); return ( - + {(index) => { const item = explorer.items?.[index]; diff --git a/interface/app/$libraryId/Explorer/View/ListView/index.tsx b/interface/app/$libraryId/Explorer/View/ListView/index.tsx index fc6ac953d..e792a19ca 100644 --- a/interface/app/$libraryId/Explorer/View/ListView/index.tsx +++ b/interface/app/$libraryId/Explorer/View/ListView/index.tsx @@ -130,47 +130,49 @@ export const ListView = memo(() => { const [backRange, frontRange] = getRangesByRow(range.start); if (backRange && frontRange) { - [...Array(backRange.sorted.end.index - backRange.sorted.start.index + 1)].forEach( - (_, i) => { - const index = backRange.sorted.start.index + i; + for (let i = backRange.sorted.start.index; i <= backRange.sorted.end.index; i++) { + const index = backRange.sorted.start.index + i; - if (index === range.start.index) return; + if (index === range.start.index) continue; - const row = rows[index]; + const row = rows[index]; - if (row) explorer.removeSelectedItem(row.original); - } - ); + if (row) explorer.removeSelectedItem(row.original); + } _ranges = _ranges.filter((_, i) => i !== backRange.index); } - [...Array(Math.abs(range.end.index - rowIndex) + (changeDirection ? 1 : 0))].forEach( - (_, i) => { - if (!range.direction || direction === range.direction) i += 1; + for ( + let i = 0; + i < Math.abs(range.end.index - rowIndex) + (changeDirection ? 1 : 0); + i++ + ) { + if (!range.direction || direction === range.direction) i += 1; - const index = range.end.index + (direction === 'down' ? i : -i); + const index = range.end.index + (direction === 'down' ? i : -i); - const row = rows[index]; + const row = rows[index]; - if (!row) return; + if (!row) continue; - const item = row.original; + const item = row.original; - if (uniqueId(item) === uniqueId(range.start.original)) return; + if (uniqueId(item) === uniqueId(range.start.original)) continue; - if ( - !range.direction || - direction === range.direction || - (changeDirection && - (range.direction === 'down' - ? index < range.start.index - : index > range.start.index)) - ) { - explorer.addSelectedItem(item); - } else explorer.removeSelectedItem(item); + if ( + !range.direction || + direction === range.direction || + (changeDirection && + (range.direction === 'down' + ? index < range.start.index + : index > range.start.index)) + ) { + explorer.addSelectedItem(item); + } else { + explorer.removeSelectedItem(item); } - ); + } let newRangeEnd = item; let removeRangeIndex: number | null = null; @@ -186,15 +188,13 @@ export const ListView = memo(() => { rowIndex ); - [...Array(removableRowsCount)].forEach((_, i) => { - i += 1; - + for (let i = 1; i <= removableRowsCount; i++) { const index = rowIndex + (direction === 'down' ? i : -i); const row = rows[index]; if (row) explorer.removeSelectedItem(row.original); - }); + } removeRangeIndex = i; break; diff --git a/interface/app/$libraryId/Explorer/View/MediaView/Item.tsx b/interface/app/$libraryId/Explorer/View/MediaView/Item.tsx index c5b1652cf..dd71f2b31 100644 --- a/interface/app/$libraryId/Explorer/View/MediaView/Item.tsx +++ b/interface/app/$libraryId/Explorer/View/MediaView/Item.tsx @@ -57,7 +57,7 @@ const ItemFileThumb = (props: Pick) => { filePath?.hidden && 'opacity-50' )} ref={setDraggableRef} - childClassName={({ variant }) => clsx(variant === 'icon' && 'size-2/4')} + childClassName={(type) => clsx(type === 'icon' && 'size-2/4')} childProps={{ style, ...attributes, diff --git a/interface/app/$libraryId/Explorer/View/MediaView/index.tsx b/interface/app/$libraryId/Explorer/View/MediaView/index.tsx index 6554d6a7d..cf8f6f866 100644 --- a/interface/app/$libraryId/Explorer/View/MediaView/index.tsx +++ b/interface/app/$libraryId/Explorer/View/MediaView/index.tsx @@ -82,11 +82,12 @@ export const MediaView = () => { let firstRowIndex: number | undefined = undefined; let lastRowIndex: number | undefined = undefined; + const scrollOffset = rowVirtualizer.scrollOffset ?? 0; // Find first row in viewport for (let i = 0; i < virtualRows.length; i++) { const row = virtualRows[i]!; - if (row.end >= rowVirtualizer.scrollOffset) { + if (row.end >= scrollOffset) { firstRowIndex = row.index; break; } @@ -95,7 +96,7 @@ export const MediaView = () => { // Find last row in viewport for (let i = virtualRows.length - 1; i >= 0; i--) { const row = virtualRows[i]!; - if (row.start <= rowVirtualizer.scrollOffset + rowVirtualizer.scrollRect.height) { + if (row.start <= scrollOffset + (rowVirtualizer.scrollRect?.height ?? 0)) { lastRowIndex = row.index; break; } @@ -163,15 +164,16 @@ export const MediaView = () => { ); } }, [ + isSortingByDate, + orderBy, + orderDirection, explorer.items, + rowVirtualizer.scrollOffset, + rowVirtualizer.scrollRect?.height, grid.columnCount, grid.options.count, - isSortingByDate, - rowVirtualizer.scrollOffset, - rowVirtualizer.scrollRect.height, - virtualRows, - orderBy, - orderDirection + dateFormat, + virtualRows ]); useKeySelection(grid); @@ -187,7 +189,15 @@ export const MediaView = () => { > {isSortingByDate && } - + {virtualRows.map((virtualRow) => ( {columnVirtualizer.getVirtualItems().map((virtualColumn) => { diff --git a/interface/app/$libraryId/Explorer/View/RenamableItemText.tsx b/interface/app/$libraryId/Explorer/View/RenamableItemText.tsx index b3852082e..2b53e43c8 100644 --- a/interface/app/$libraryId/Explorer/View/RenamableItemText.tsx +++ b/interface/app/$libraryId/Explorer/View/RenamableItemText.tsx @@ -58,17 +58,17 @@ export const RenamableItemText = ({ const renameFile = useLibraryMutation(['files.renameFile'], { onError: () => reset(), - onSuccess: () => rspc.queryClient.invalidateQueries(['search.paths']) + onSuccess: () => rspc.queryClient.invalidateQueries({ queryKey: ['search.paths'] }) }); const renameEphemeralFile = useLibraryMutation(['ephemeralFiles.renameFile'], { onError: () => reset(), - onSuccess: () => rspc.queryClient.invalidateQueries(['search.paths']) + onSuccess: () => rspc.queryClient.invalidateQueries({ queryKey: ['search.paths'] }) }); const renameLocation = useLibraryMutation(['locations.update'], { onError: () => reset(), - onSuccess: () => rspc.queryClient.invalidateQueries(['search.paths']) + onSuccess: () => rspc.queryClient.invalidateQueries({ queryKey: ['search.paths'] }) }); const reset = useCallback(() => { diff --git a/interface/app/$libraryId/Explorer/View/ViewItem.tsx b/interface/app/$libraryId/Explorer/View/ViewItem.tsx index 2537b854f..087c8b808 100644 --- a/interface/app/$libraryId/Explorer/View/ViewItem.tsx +++ b/interface/app/$libraryId/Explorer/View/ViewItem.tsx @@ -192,14 +192,15 @@ export const useViewItemDoubleClick = () => { } }, [ - searchParams, explorer.selectedItems, explorer.settingsStore.openOnDoubleClick, - library.uuid, - navigate, openFilePaths, - openEphemeralFiles, - updateAccessTime + updateAccessTime, + library.uuid, + t, + searchParams, + navigate, + openEphemeralFiles ] ); diff --git a/interface/app/$libraryId/Explorer/index.tsx b/interface/app/$libraryId/Explorer/index.tsx index 3dcb19261..2aa5971c0 100644 --- a/interface/app/$libraryId/Explorer/index.tsx +++ b/interface/app/$libraryId/Explorer/index.tsx @@ -61,7 +61,7 @@ export default function Explorer(props: PropsWithChildren) { // I had planned to somehow fetch the Object, but its a lot more work than its worth given // id have to fetch the file_path explicitly and patch the query // for now, it seems to work a treat just invalidating the whole query - rspc.queryClient.invalidateQueries(['search.paths']); + rspc.queryClient.invalidateQueries({ queryKey: ['search.paths'] }); } } }); diff --git a/interface/app/$libraryId/Explorer/store.ts b/interface/app/$libraryId/Explorer/store.ts index 4814f972a..bb463ab11 100644 --- a/interface/app/$libraryId/Explorer/store.ts +++ b/interface/app/$libraryId/Explorer/store.ts @@ -123,8 +123,19 @@ export function flattenThumbnailKey(thumbKey: ThumbKey) { export const explorerStore = proxy({ ...state, reset: (_state?: typeof state) => resetStore(explorerStore, _state || state), - addNewThumbnail: (thumbKey: ThumbKey) => { - explorerStore.newThumbnails.add(flattenThumbnailKey(thumbKey)); + addNewThumbnail: (thumbKey: ThumbKey | string) => { + thumbKey = typeof thumbKey === 'string' ? thumbKey : flattenThumbnailKey(thumbKey); + // HACK: Ensure store propagates changes + const newThumbnails = new Set(explorerStore.newThumbnails); + newThumbnails.add(thumbKey); + explorerStore.newThumbnails = newThumbnails; + }, + removeThumbnail: (thumbKey: ThumbKey | string) => { + thumbKey = typeof thumbKey === 'string' ? thumbKey : flattenThumbnailKey(thumbKey); + // HACK: Ensure store propagates changes + const newThumbnails = new Set(explorerStore.newThumbnails); + newThumbnails.delete(thumbKey); + explorerStore.newThumbnails = newThumbnails; }, resetCache: () => { explorerStore.newThumbnails.clear(); diff --git a/interface/app/$libraryId/Explorer/useExplorer.ts b/interface/app/$libraryId/Explorer/useExplorer.ts index 97af144b8..c656362c7 100644 --- a/interface/app/$libraryId/Explorer/useExplorer.ts +++ b/interface/app/$libraryId/Explorer/useExplorer.ts @@ -105,7 +105,7 @@ export function useExplorerSettings({ >; data?: T | null; }) { - const [store] = useState(() => proxy(settings)); + const store = useMemo(() => proxy(settings), [settings]); const updateSettings = useDebouncedCallback((settings: ExplorerSettings, data: T) => { onSettingsChanged?.(settings, data); @@ -149,15 +149,8 @@ function useSelectedItems(items: ExplorerItem[] | null) { const itemHashesWeakMap = useRef(new WeakMap()); // Store hashes of items instead as objects are unique by reference but we - // still need to differentate between item variants - const [selectedItemHashes, setSelectedItemHashes] = useState(() => ({ - value: new Set() - })); - - const updateHashes = useCallback( - () => setSelectedItemHashes((h) => ({ ...h })), - [setSelectedItemHashes] - ); + // still need to differentiate between item variants + const [selectedItemHashes, setSelectedItemHashes] = useState(() => new Set()); const itemsMap = useMemo( () => @@ -172,7 +165,7 @@ function useSelectedItems(items: ExplorerItem[] | null) { const selectedItems = useMemo( () => - [...selectedItemHashes.value].reduce((items, hash) => { + [...selectedItemHashes].reduce((items, hash) => { const item = itemsMap.get(hash); if (item) items.add(item.data); return items; @@ -194,37 +187,37 @@ function useSelectedItems(items: ExplorerItem[] | null) { (item: ExplorerItem | ExplorerItem[]) => { const items = Array.isArray(item) ? item : [item]; - for (let i = 0; i < items.length; i++) { - selectedItemHashes.value.add(getItemUniqueId(items[i]!)); - } - - updateHashes(); + setSelectedItemHashes((oldHashes) => { + const newHashes = new Set(oldHashes); + for (const it of items) newHashes.add(getItemUniqueId(it)); + return newHashes; + }); }, - [getItemUniqueId, selectedItemHashes.value, updateHashes] + [getItemUniqueId] ), removeSelectedItem: useCallback( (item: ExplorerItem | ExplorerItem[]) => { const items = Array.isArray(item) ? item : [item]; - - for (let i = 0; i < items.length; i++) { - selectedItemHashes.value.delete(getItemUniqueId(items[i]!)); - } - - updateHashes(); + setSelectedItemHashes((oldHashes) => { + const newHashes = new Set(oldHashes); + for (const it of items) newHashes.delete(getItemUniqueId(it)); + return newHashes; + }); }, - [getItemUniqueId, selectedItemHashes.value, updateHashes] + [getItemUniqueId] ), resetSelectedItems: useCallback( (items?: ExplorerItem[]) => { - selectedItemHashes.value.clear(); - items?.forEach((item) => selectedItemHashes.value.add(getItemUniqueId(item))); - updateHashes(); + if (items) { + const newHashes = new Set(); + for (const it of items) newHashes.add(getItemUniqueId(it)); + setSelectedItemHashes(newHashes); + } else { + setSelectedItemHashes(new Set()); + } }, - [getItemUniqueId, selectedItemHashes.value, updateHashes] + [getItemUniqueId] ), - isItemSelected: useCallback( - (item: ExplorerItem) => selectedItems.has(item), - [selectedItems] - ) + isItemSelected: (item: ExplorerItem) => selectedItems.has(item) }; } diff --git a/interface/app/$libraryId/Explorer/useExplorerDraggable.tsx b/interface/app/$libraryId/Explorer/useExplorerDraggable.tsx index 39c23511f..b32105624 100644 --- a/interface/app/$libraryId/Explorer/useExplorerDraggable.tsx +++ b/interface/app/$libraryId/Explorer/useExplorerDraggable.tsx @@ -1,5 +1,5 @@ import { useDraggable, UseDraggableArguments } from '@dnd-kit/core'; -import { CSSProperties, HTMLAttributes } from 'react'; +import { CSSProperties, HTMLAttributes, useCallback, useMemo } from 'react'; import { ExplorerItem } from '@sd/client'; import { explorerStore } from './store'; @@ -11,8 +11,26 @@ export interface UseExplorerDraggableProps extends Omit { - const disabled = props.disabled || !draggableTypes.includes(props.data.type); + const disabled = useMemo( + () => props.disabled || !draggableTypes.includes(props.data.type), + [props.disabled, props.data.type] + ); const { setNodeRef, ...draggable } = useDraggable({ ...props, @@ -20,30 +38,30 @@ export const useExplorerDraggable = (props: UseExplorerDraggableProps) => { disabled: disabled }); - const onMouseDown = () => { + const onMouseDown = useCallback(() => { if (!disabled) explorerStore.drag = { type: 'touched' }; - }; + }, [disabled]); - const onMouseLeave = () => { + const onMouseLeave = useCallback(() => { if (explorerStore.drag?.type !== 'dragging') explorerStore.drag = null; - }; + }, []); - const onMouseUp = () => (explorerStore.drag = null); - - const style = { - cursor: 'default', - outline: 'none' - } satisfies CSSProperties; + const onMouseUp = useCallback(() => { + explorerStore.drag = null; + }, []); return { ...draggable, setDraggableRef: setNodeRef, - listeners: { - ...draggable.listeners, - onMouseDown, - onMouseLeave, - onMouseUp - } satisfies HTMLAttributes, - style + listeners: useMemo( + () => ({ + ...draggable.listeners, + onMouseDown, + onMouseLeave, + onMouseUp + }), + [draggable.listeners, onMouseDown, onMouseLeave, onMouseUp] + ) satisfies HTMLAttributes, + style: DRAGGABLE_STYLE }; }; diff --git a/interface/app/$libraryId/Explorer/useExplorerItemData.tsx b/interface/app/$libraryId/Explorer/useExplorerItemData.tsx index 06d2f2af9..3aff87488 100644 --- a/interface/app/$libraryId/Explorer/useExplorerItemData.tsx +++ b/interface/app/$libraryId/Explorer/useExplorerItemData.tsx @@ -1,34 +1,91 @@ -import { useQueryClient } from '@tanstack/react-query'; -import { useEffect, useMemo } from 'react'; -import { getExplorerItemData, useSelector, type ExplorerItem } from '@sd/client'; +import { useCallback, useEffect, useMemo, useRef, useState } from 'react'; +import { subscribe } from 'valtio'; +import { + compareHumanizedSizes, + getExplorerItemData, + humanizeSize, + ThumbKey, + type ExplorerItem +} from '@sd/client'; +import { usePlatform } from '~/util/Platform'; import { explorerStore, flattenThumbnailKey } from './store'; -// This is where we intercept the state of the explorer item to determine if we should rerender -// This hook is used inside every thumbnail in the explorer +/** + * This is where we intercept the state of the explorer item to determine if we should rerender + * + * .. WARNING:: + * This hook is used inside every thumbnail in the explorer. + * Be careful with the performance of the code, make sure to always memoize any objects or functions to avoid unnecessary re-renders. + * + * @param explorerItem - The explorer item to get data from + * @returns The extracted data from the explorer item + */ export function useExplorerItemData(explorerItem: ExplorerItem) { - const newThumbnail = useSelector(explorerStore, (s) => { - const thumbnailKey = - explorerItem.type === 'Label' - ? // labels have .thumbnails, plural - explorerItem.thumbnails?.[0] - : // all other explorer items have .thumbnail singular - 'thumbnail' in explorerItem && explorerItem.thumbnail; + const platform = usePlatform(); + const cachedSize = useRef | null>(null); + const [newThumbnails, setNewThumbnails] = useState>(new Map()); - return !!(thumbnailKey && s.newThumbnails.has(flattenThumbnailKey(thumbnailKey))); - }); + let thumbnails: ThumbKey | ThumbKey[] | null = null; + switch (explorerItem.type) { + case 'Label': + thumbnails = explorerItem.thumbnails; + break; + case 'Path': + case 'Object': + case 'NonIndexedPath': + thumbnails = explorerItem.thumbnail; + break; + } + + useEffect(() => { + const thumbnailKeys = thumbnails + ? Array.isArray(thumbnails) + ? thumbnails + : [thumbnails] + : []; + + const updateThumbnails = () => + setNewThumbnails((oldThumbs) => { + const thumbs = thumbnailKeys.reduce>((acc, thumbKey) => { + const url = platform.getThumbnailUrlByThumbKey(thumbKey); + const thumbId = flattenThumbnailKey(thumbKey); + acc.set(url, explorerStore.newThumbnails.has(thumbId) ? thumbId : null); + return acc; + }, new Map()); + + // Avoid unnecessary re-renders + return oldThumbs.size !== thumbs.size || + Array.from(oldThumbs.keys()).some( + (key) => oldThumbs.get(key) !== thumbs.get(key) + ) + ? thumbs + : oldThumbs; + }); + + updateThumbnails(); + + return subscribe(explorerStore, updateThumbnails); + }, [thumbnails, platform]); return useMemo(() => { - const itemData = getExplorerItemData(explorerItem); + const explorerItemData = getExplorerItemData(explorerItem); - if (!itemData.hasLocalThumbnail) { - itemData.hasLocalThumbnail = newThumbnail; + // Avoid unecessary re-renders + if ( + cachedSize.current == null || + !compareHumanizedSizes(cachedSize.current, explorerItemData.size) + ) { + cachedSize.current = explorerItemData.size; } - return itemData; - // whatever goes here, is what can cause an atomic re-render of an explorer item - // this is used for when new thumbnails are generated, and files identified - }, [explorerItem, newThumbnail]); + return { + ...explorerItemData, + size: cachedSize.current, + thumbnails: newThumbnails, + hasLocalThumbnail: explorerItemData.hasLocalThumbnail || newThumbnails.size > 0 + }; + }, [explorerItem, newThumbnails]); } export type ExplorerItemData = ReturnType; diff --git a/interface/app/$libraryId/Explorer/useExplorerPreferences.ts b/interface/app/$libraryId/Explorer/useExplorerPreferences.ts index 6af94ec7b..60d4bc3d5 100644 --- a/interface/app/$libraryId/Explorer/useExplorerPreferences.ts +++ b/interface/app/$libraryId/Explorer/useExplorerPreferences.ts @@ -53,7 +53,7 @@ export function useExplorerPreferences({ try { await updatePreferences.mutateAsync(writeSettings(settings)); - rspc.queryClient.invalidateQueries(['preferences.get']); + rspc.queryClient.invalidateQueries({ queryKey: ['preferences.get'] }); } catch (e) { alert('An error has occurred while updating your preferences.'); } diff --git a/interface/app/$libraryId/Layout/CMDK/index.tsx b/interface/app/$libraryId/Layout/CMDK/index.tsx index 1055e3708..7cd205911 100644 --- a/interface/app/$libraryId/Layout/CMDK/index.tsx +++ b/interface/app/$libraryId/Layout/CMDK/index.tsx @@ -1,6 +1,7 @@ import './CMDK.css'; import './CMDK.scss'; +import { keepPreviousData } from '@tanstack/react-query'; import clsx from 'clsx'; import { useEffect, useState } from 'react'; import CommandPalette, { filterItems, getItemIndex } from 'react-cmdk'; @@ -50,7 +51,9 @@ const CMDK = () => { const [page, setPage] = useState<'root' | 'locations' | 'tags'>('root'); const [search, setSearch] = useState(''); - const locationsQuery = useLibraryQuery(['locations.list'], { keepPreviousData: true }); + const locationsQuery = useLibraryQuery(['locations.list'], { + placeholderData: keepPreviousData + }); const locations = locationsQuery.data; const onlineLocations = useOnlineLocations(); diff --git a/interface/app/$libraryId/Layout/CMDK/pages/CMDKLocations.tsx b/interface/app/$libraryId/Layout/CMDK/pages/CMDKLocations.tsx index f878dea0a..b0696c50b 100644 --- a/interface/app/$libraryId/Layout/CMDK/pages/CMDKLocations.tsx +++ b/interface/app/$libraryId/Layout/CMDK/pages/CMDKLocations.tsx @@ -1,3 +1,4 @@ +import { keepPreviousData } from '@tanstack/react-query'; import clsx from 'clsx'; import CommandPalette from 'react-cmdk'; import { useNavigate } from 'react-router'; @@ -5,7 +6,9 @@ import { arraysEqual, useLibraryQuery, useOnlineLocations } from '@sd/client'; import { Icon } from '~/components'; export default function CMDKLocations() { - const locationsQuery = useLibraryQuery(['locations.list'], { keepPreviousData: true }); + const locationsQuery = useLibraryQuery(['locations.list'], { + placeholderData: keepPreviousData + }); const locations = locationsQuery.data; const onlineLocations = useOnlineLocations(); diff --git a/interface/app/$libraryId/Layout/CMDK/pages/CMDKTags.tsx b/interface/app/$libraryId/Layout/CMDK/pages/CMDKTags.tsx index a3cc2c836..7ca898691 100644 --- a/interface/app/$libraryId/Layout/CMDK/pages/CMDKTags.tsx +++ b/interface/app/$libraryId/Layout/CMDK/pages/CMDKTags.tsx @@ -1,9 +1,10 @@ +import { keepPreviousData } from '@tanstack/react-query'; import CommandPalette from 'react-cmdk'; import { useNavigate } from 'react-router'; import { useLibraryQuery, type Tag } from '@sd/client'; export default function CMDKTags() { - const result = useLibraryQuery(['tags.list'], { keepPreviousData: true }); + const result = useLibraryQuery(['tags.list'], { placeholderData: keepPreviousData }); const tags = result.data || []; const navigate = useNavigate(); diff --git a/interface/app/$libraryId/Layout/Sidebar/DebugPopover.tsx b/interface/app/$libraryId/Layout/Sidebar/DebugPopover.tsx index 21eec1971..5cb5d97e5 100644 --- a/interface/app/$libraryId/Layout/Sidebar/DebugPopover.tsx +++ b/interface/app/$libraryId/Layout/Sidebar/DebugPopover.tsx @@ -154,15 +154,12 @@ export default () => { title="React Query Devtools" description="Configure the React Query devtools." > - + + (debugState.reactQueryDevtools = !debugState.reactQueryDevtools) + } + /> { - queryClient.invalidateQueries(['jobs.reports']); + queryClient.invalidateQueries({ queryKey: ['jobs.reports'] }); }) ); diff --git a/interface/app/$libraryId/Layout/Sidebar/JobManager/index.tsx b/interface/app/$libraryId/Layout/Sidebar/JobManager/index.tsx index 837dd0228..d317f4a31 100644 --- a/interface/app/$libraryId/Layout/Sidebar/JobManager/index.tsx +++ b/interface/app/$libraryId/Layout/Sidebar/JobManager/index.tsx @@ -97,7 +97,7 @@ export function JobManager() { title: t('success'), body: t('all_jobs_have_been_cleared') }); - queryClient.invalidateQueries(['jobs.reports']); + queryClient.invalidateQueries({ queryKey: ['jobs.reports'] }); } catch (error) { toast.error({ title: t('error'), diff --git a/interface/app/$libraryId/Layout/Sidebar/SidebarLayout/Footer.tsx b/interface/app/$libraryId/Layout/Sidebar/SidebarLayout/Footer.tsx index 2d7dc4835..b8eb0bffc 100644 --- a/interface/app/$libraryId/Layout/Sidebar/SidebarLayout/Footer.tsx +++ b/interface/app/$libraryId/Layout/Sidebar/SidebarLayout/Footer.tsx @@ -1,5 +1,5 @@ -import { inferSubscriptionResult } from '@oscartbeaumont-sd/rspc-client'; import { Gear } from '@phosphor-icons/react'; +import { inferSubscriptionResult } from '@spacedrive/rspc-client'; import { useState } from 'react'; import { useNavigate } from 'react-router'; import { diff --git a/interface/app/$libraryId/Layout/Sidebar/sections/Locations/index.tsx b/interface/app/$libraryId/Layout/Sidebar/sections/Locations/index.tsx index ef80dc378..703eddae0 100644 --- a/interface/app/$libraryId/Layout/Sidebar/sections/Locations/index.tsx +++ b/interface/app/$libraryId/Layout/Sidebar/sections/Locations/index.tsx @@ -1,3 +1,4 @@ +import { keepPreviousData } from '@tanstack/react-query'; import clsx from 'clsx'; import { Link, useMatch } from 'react-router-dom'; import { @@ -18,7 +19,9 @@ import { SeeMore } from '../../SidebarLayout/SeeMore'; import { ContextMenu } from './ContextMenu'; export default function Locations() { - const locationsQuery = useLibraryQuery(['locations.list'], { keepPreviousData: true }); + const locationsQuery = useLibraryQuery(['locations.list'], { + placeholderData: keepPreviousData + }); const locations = locationsQuery.data; const onlineLocations = useOnlineLocations(); diff --git a/interface/app/$libraryId/Layout/Sidebar/sections/Tags/index.tsx b/interface/app/$libraryId/Layout/Sidebar/sections/Tags/index.tsx index f251fc042..4131bbfe6 100644 --- a/interface/app/$libraryId/Layout/Sidebar/sections/Tags/index.tsx +++ b/interface/app/$libraryId/Layout/Sidebar/sections/Tags/index.tsx @@ -1,3 +1,4 @@ +import { keepPreviousData } from '@tanstack/react-query'; import clsx from 'clsx'; import { NavLink, useMatch } from 'react-router-dom'; import { useLibraryQuery, type Tag } from '@sd/client'; @@ -11,7 +12,7 @@ import { SeeMore } from '../../SidebarLayout/SeeMore'; import { ContextMenu } from './ContextMenu'; export default function TagsSection() { - const result = useLibraryQuery(['tags.list'], { keepPreviousData: true }); + const result = useLibraryQuery(['tags.list'], { placeholderData: keepPreviousData }); const tags = result.data; const { t } = useLocale(); diff --git a/interface/app/$libraryId/Spacedrop/index.tsx b/interface/app/$libraryId/Spacedrop/index.tsx index 4c9a81342..6803a5283 100644 --- a/interface/app/$libraryId/Spacedrop/index.tsx +++ b/interface/app/$libraryId/Spacedrop/index.tsx @@ -93,7 +93,7 @@ export function Spacedrop({ triggerClose }: { triggerClose: () => void }) { }); const onDropped = (id: string, files: string[]) => { - if (doSpacedrop.isLoading) { + if (doSpacedrop.isPending) { toast.warning(t('spacedrop_already_progress')); return; } diff --git a/interface/app/$libraryId/debug/actors.tsx b/interface/app/$libraryId/debug/actors.tsx index a744583a6..cb14658c1 100644 --- a/interface/app/$libraryId/debug/actors.tsx +++ b/interface/app/$libraryId/debug/actors.tsx @@ -1,10 +1,10 @@ -import { inferSubscriptionResult } from '@oscartbeaumont-sd/rspc-client'; +import { inferSubscriptionResult } from '@spacedrive/rspc-client'; import { useMemo, useState } from 'react'; import { Procedures, useLibraryMutation, useLibrarySubscription } from '@sd/client'; import { Button } from '@sd/ui'; import { useRouteTitle } from '~/hooks/useRouteTitle'; -// @million-ignore +// million-ignore export const Component = () => { useRouteTitle('Actors'); @@ -46,10 +46,10 @@ function StartButton({ name }: { name: string }) { return ( ); } @@ -60,10 +60,10 @@ function StopButton({ name }: { name: string }) { return ( ); } diff --git a/interface/app/$libraryId/debug/cloud.tsx b/interface/app/$libraryId/debug/cloud.tsx index c3bf6ae35..fcec4339f 100644 --- a/interface/app/$libraryId/debug/cloud.tsx +++ b/interface/app/$libraryId/debug/cloud.tsx @@ -87,13 +87,13 @@ function Authenticated() {
    @@ -125,8 +125,8 @@ function HostedLocationsPlayground() { /> {/* TODO: Cleanup this mess + styles */} - {locations.status === 'loading' ?
    Loading!
    : null} - {locations.status !== 'loading' && locations.data?.length === 0 ? ( + {locations.status === 'pending' ?
    Loading!
    : null} + {locations.status !== 'pending' && locations.data?.length === 0 ? (
    Looks like you don't have any!
    ) : (
    @@ -137,7 +137,7 @@ function HostedLocationsPlayground() { variant="accent" size="sm" onClick={() => removeLocation.mutate(location.id)} - disabled={isLoading} + disabled={isPending} > Delete @@ -152,7 +152,7 @@ function HostedLocationsPlayground() { className="grow" value={path} onInput={(e) => setPath(e.currentTarget.value)} - disabled={isLoading} + disabled={isPending} />
    diff --git a/interface/app/$libraryId/settings/client/backups.tsx b/interface/app/$libraryId/settings/client/backups.tsx index 69689b16f..0ebfe10e7 100644 --- a/interface/app/$libraryId/settings/client/backups.tsx +++ b/interface/app/$libraryId/settings/client/backups.tsx @@ -27,7 +27,7 @@ export const Component = () => { rightArea={
    diff --git a/interface/app/$libraryId/settings/library/general.tsx b/interface/app/$libraryId/settings/library/general.tsx index d79b29363..a8b850923 100644 --- a/interface/app/$libraryId/settings/library/general.tsx +++ b/interface/app/$libraryId/settings/library/general.tsx @@ -109,7 +109,7 @@ export const Component = () => {
    -// } -// {...form.register('masterPassword', { required: true })} -// /> - -// setShow((old) => ({ ...old, secretKey: !old.secretKey }))} -// size="icon" -// > -// -// -// } -// {...form.register('secretKey')} -// /> - -//
    -// -//
    -//
    -// ); -// }; diff --git a/interface/app/$libraryId/settings/library/keys/KeyViewerDialog.tsx b/interface/app/$libraryId/settings/library/keys/KeyViewerDialog.tsx deleted file mode 100644 index 9fe44cbac..000000000 --- a/interface/app/$libraryId/settings/library/keys/KeyViewerDialog.tsx +++ /dev/null @@ -1,160 +0,0 @@ -// import { Buffer } from 'buffer'; -// import { Clipboard } from '@phosphor-icons/react'; -// import { useState } from 'react'; -// import { slugFromHashingAlgo, useLibraryQuery } from '@sd/client'; -// import { Button, Dialog, Input, Select, SelectOption, UseDialogProps, useDialog } from '@sd/ui'; -// import { useZodForm } from '@sd/ui/src/forms'; -// import { KeyListSelectOptions } from '~/app/$libraryId/KeyManager/List'; - -// export const KeyUpdater = (props: { -// uuid: string; -// setKey: (value: string) => void; -// setEncryptionAlgo: (value: string) => void; -// setHashingAlgo: (value: string) => void; -// setContentSalt: (value: string) => void; -// }) => { -// useLibraryQuery(['keys.getKey', props.uuid], { -// onSuccess: (data) => { -// props.setKey(data); -// } -// }); - -// const keys = useLibraryQuery(['keys.list']); - -// const key = keys.data?.find((key) => key.uuid == props.uuid); - -// if (key) { -// props.setEncryptionAlgo(key?.algorithm); -// props.setHashingAlgo(slugFromHashingAlgo(key?.hashing_algorithm)); -// props.setContentSalt(Buffer.from(key.content_salt).toString('hex')); -// } - -// return <>; -// }; - -// export default (props: UseDialogProps) => { -// const keys = useLibraryQuery(['keys.list'], { -// onSuccess: (data) => { -// if (key === '' && data.length !== 0) { -// setKey(data[0]?.uuid ?? ''); -// } -// } -// }); - -// const [key, setKey] = useState(''); -// const [keyValue, setKeyValue] = useState(''); -// const [contentSalt, setContentSalt] = useState(''); -// const [encryptionAlgo, setEncryptionAlgo] = useState(''); -// const [hashingAlgo, setHashingAlgo] = useState(''); - -// return ( -// -// - -//
    -//
    -// Key -// -//
    -//
    -//
    -//
    -// Encryption -// -//
    -//
    -// Hashing -// -//
    -//
    -//
    -//
    -// Content Salt (hex) -// { -// navigator.clipboard.writeText(contentSalt); -// }} -// size="icon" -// > -// -// -// } -// /> -//
    -//
    -//
    -//
    -// Key Value -// { -// navigator.clipboard.writeText(keyValue); -// }} -// size="icon" -// > -// -// -// } -// /> -//
    -//
    -//     -// ); -// }; diff --git a/interface/app/$libraryId/settings/library/keys/MasterPasswordDialog.tsx b/interface/app/$libraryId/settings/library/keys/MasterPasswordDialog.tsx deleted file mode 100644 index 7b07a3135..000000000 --- a/interface/app/$libraryId/settings/library/keys/MasterPasswordDialog.tsx +++ /dev/null @@ -1,187 +0,0 @@ -// import { ArrowsClockwise, Clipboard, Eye, EyeSlash } from '@phosphor-icons/react'; -// import { useState } from 'react'; -// import { -// Algorithm, -// HASHING_ALGOS, -// HashingAlgoSlug, -// hashingAlgoSlugSchema, -// useLibraryMutation -// } from '@sd/client'; -// import { Button, Dialog, Input, Select, SelectOption, UseDialogProps, useDialog } from '@sd/ui'; -// import { useZodForm, z } from '@sd/ui/src/forms'; -// import { PasswordMeter, showAlertDialog } from '~/components'; -// import { generatePassword } from '~/util'; - -// const schema = z.object({ -// masterPassword: z.string(), -// masterPassword2: z.string(), -// encryptionAlgo: z.string(), -// hashingAlgo: hashingAlgoSlugSchema -// }); - -// export default (props: UseDialogProps) => { -// const changeMasterPassword = useLibraryMutation('keys.changeMasterPassword', { -// onSuccess: () => { -// showAlertDialog({ -// title: 'Success', -// value: 'Your master password was changed successfully' -// }); -// }, -// onError: () => { -// // this should never really happen -// showAlertDialog({ -// title: 'Master Password Change Error', -// value: 'There was an error while changing your master password.' -// }); -// } -// }); - -// const [show, setShow] = useState({ -// masterPassword: false, -// masterPassword2: false -// }); - -// const MP1CurrentEyeIcon = show.masterPassword ? EyeSlash : Eye; -// const MP2CurrentEyeIcon = show.masterPassword2 ? EyeSlash : Eye; - -// const form = useZodForm({ -// schema, -// defaultValues: { -// encryptionAlgo: 'XChaCha20Poly1305', -// hashingAlgo: 'Argon2id-s', -// masterPassword: '', -// masterPassword2: '' -// } -// }); - -// const onSubmit = form.handleSubmit((data) => { -// if (data.masterPassword !== data.masterPassword2) { -// showAlertDialog({ -// title: 'Error', -// value: 'Passwords are not the same, please try again.' -// }); -// } else { -// const hashing_algorithm = HASHING_ALGOS[data.hashingAlgo]; - -// return changeMasterPassword.mutateAsync({ -// algorithm: data.encryptionAlgo as Algorithm, -// hashing_algorithm, -// password: data.masterPassword -// }); -// } -// }); - -// return ( -// -// -// -// -// -// -// } -// /> - -// -// setShow((old) => ({ ...old, masterPassword2: !old.masterPassword2 })) -// } -// size="icon" -// type="button" -// > -// -// -// } -// /> - -// - -//
    -//
    -// Encryption -// -//
    -//
    -// Hashing -// -//
    -//
    -//     -// ); -// }; diff --git a/interface/app/$libraryId/settings/library/keys/index.tsx b/interface/app/$libraryId/settings/library/keys/index.tsx deleted file mode 100644 index e0c3bc314..000000000 --- a/interface/app/$libraryId/settings/library/keys/index.tsx +++ /dev/null @@ -1,293 +0,0 @@ -// import * as DropdownMenu from '@radix-ui/react-dropdown-menu'; -// import { animated, useTransition } from '@react-spring/web'; -// import clsx from 'clsx'; -// import { Lock, Plus } from '@phosphor-icons/react'; -// import { PropsWithChildren, ReactNode, useState } from 'react'; -// import QRCode from 'react-qr-code'; -// import { useLibraryMutation, useLibraryQuery } from '@sd/client'; -// import { Button, PasswordInput, dialogManager } from '@sd/ui'; -// import { showAlertDialog } from '~/components/AlertDialog'; -// import { usePlatform } from '~/util/Platform'; -// import KeyList from '../../../KeyManager/List'; -// import KeyMounter from '../../../KeyManager/Mounter'; -// import { Heading } from '../../Layout'; -// import BackupRestoreDialog from './BackupRestoreDialog'; -// import KeyViewerDialog from './KeyViewerDialog'; -// import MasterPasswordDialog from './MasterPasswordDialog'; - -// interface Props extends DropdownMenu.MenuContentProps { -// trigger: React.ReactNode; -// transformOrigin?: string; -// disabled?: boolean; -// } - -// export const KeyMounterDropdown = ({ -// trigger, -// children, -// transformOrigin, -// className -// }: PropsWithChildren) => { -// const [open, setOpen] = useState(false); - -// const transitions = useTransition(open, { -// from: { -// opacity: 0, -// transform: `scale(0.9)`, -// transformOrigin: transformOrigin || 'top' -// }, -// enter: { opacity: 1, transform: 'scale(1)' }, -// leave: { opacity: -0.5, transform: 'scale(0.95)' }, -// config: { mass: 0.4, tension: 200, friction: 10 } -// }); - -// return ( -// -// {trigger} -// {transitions( -// (styles, show) => -// show && ( -// -// -// -// {children} -// -// -// -// ) -// )} -// -// ); -// }; - -// export const Component = () => { -// const platform = usePlatform(); -// const isUnlocked = useLibraryQuery(['keys.isUnlocked']); -// const keyringSk = useLibraryQuery(['keys.getSecretKey'], { initialData: '' }); // assume true by default, as it will often be the case. need to fix this with an rspc subscription+such -// const unlockKeyManager = useLibraryMutation('keys.unlockKeyManager', { -// onError: () => { -// showAlertDialog({ -// title: 'Unlock Error', -// value: 'The information provided to the key manager was incorrect' -// }); -// } -// }); - -// const unmountAll = useLibraryMutation('keys.unmountAll'); -// const clearMasterPassword = useLibraryMutation('keys.clearMasterPassword'); -// const backupKeystore = useLibraryMutation('keys.backupKeystore'); -// const isKeyManagerUnlocking = useLibraryQuery(['keys.isKeyManagerUnlocking']); - -// const [masterPassword, setMasterPassword] = useState(''); -// const [secretKey, setSecretKey] = useState(''); // for the unlock form -// const [viewSecretKey, setViewSecretKey] = useState(false); // for the settings page - -// const keys = useLibraryQuery(['keys.list']); - -// const [enterSkManually, setEnterSkManually] = useState(keyringSk?.data === null); - -// if (!isUnlocked?.data) { -// return ( -//
    -// setMasterPassword(e.target.value)} -// autoFocus -// placeholder="Master Password" -// className="mb-2" -// /> - -// {enterSkManually && ( -// setSecretKey(e.target.value)} -// placeholder="Secret Key" -// className="mb-2" -// /> -// )} - -// -// {!enterSkManually && ( -//
    -//

    setEnterSkManually(true)}> -// or enter secret key manually -//

    -//
    -// )} -//
    -// ); -// } else { -// return ( -// <> -// -// -// -// -// -// } -// > -// -// -// -// } -// /> - -// {isUnlocked && ( -//
    -// -//
    -// )} - -// {keyringSk?.data && ( -// <> -// -// {!viewSecretKey && ( -//
    -// -//
    -// )} -// {viewSecretKey && ( -//
    { -// keyringSk.data && navigator.clipboard.writeText(keyringSk.data); -// }} -// > -// <> -// -//

    {keyringSk.data}

    -// -//
    -// )} -// -// )} - -// -//
    -// -// -//
    - -// -//
    -// -// -//
    -// -// ); -// } -// }; - -// interface SubheadingProps { -// title: string; -// rightArea?: ReactNode; -// } - -// const Subheading = (props: SubheadingProps) => ( -//
    -//
    -//

    {props.title}

    -//
    -// {props.rightArea} -//
    -// ); diff --git a/interface/app/$libraryId/settings/library/locations/$id.tsx b/interface/app/$libraryId/settings/library/locations/$id.tsx index 41fd59571..3bcec7b99 100644 --- a/interface/app/$libraryId/settings/library/locations/$id.tsx +++ b/interface/app/$libraryId/settings/library/locations/$id.tsx @@ -78,7 +78,7 @@ const EditLocationForm = () => { }, onSuccess: () => { form.reset(form.getValues()); - queryClient.invalidateQueries(['locations.list']); + queryClient.invalidateQueries({ queryKey: ['locations.list'] }); } }); diff --git a/interface/app/$libraryId/settings/library/saved-searches/index.tsx b/interface/app/$libraryId/settings/library/saved-searches/index.tsx index fa10daeb3..c3a6e19f7 100644 --- a/interface/app/$libraryId/settings/library/saved-searches/index.tsx +++ b/interface/app/$libraryId/settings/library/saved-searches/index.tsx @@ -108,7 +108,7 @@ function EditForm({ savedSearch, onDelete }: { savedSearch: SavedSearch; onDelet @@ -193,10 +193,10 @@ function StartButton({ name }: { name: string }) { return ( ); } @@ -208,10 +208,10 @@ function StopButton({ name }: { name: string }) { return ( ); } diff --git a/interface/app/$libraryId/settings/library/tags/CreateDialog.tsx b/interface/app/$libraryId/settings/library/tags/CreateDialog.tsx index 93032ecf4..60a54a93f 100644 --- a/interface/app/$libraryId/settings/library/tags/CreateDialog.tsx +++ b/interface/app/$libraryId/settings/library/tags/CreateDialog.tsx @@ -28,7 +28,7 @@ export function useAssignItemsToTag() { const mutation = useLibraryMutation(['tags.assign'], { onSuccess: () => { submitPlausibleEvent({ event: { type: 'tagAssign' } }); - rspc.queryClient.invalidateQueries(['search.paths']); + rspc.queryClient.invalidateQueries({ queryKey: ['search.paths'] }); } }); diff --git a/interface/app/$libraryId/settings/node/libraries/DeleteDialog.tsx b/interface/app/$libraryId/settings/node/libraries/DeleteDialog.tsx index 7425a0df0..075fb5280 100644 --- a/interface/app/$libraryId/settings/node/libraries/DeleteDialog.tsx +++ b/interface/app/$libraryId/settings/node/libraries/DeleteDialog.tsx @@ -25,7 +25,7 @@ export default function DeleteLibraryDialog(props: Props) { try { await deleteLib.mutateAsync(props.libraryUuid); - queryClient.invalidateQueries(['library.list']); + queryClient.invalidateQueries({ queryKey: ['library.list'] }); if (platform.refreshMenuBar) platform.refreshMenuBar(); diff --git a/interface/app/$libraryId/settings/resources/changelog.tsx b/interface/app/$libraryId/settings/resources/changelog.tsx index ed257162f..e6965feb9 100644 --- a/interface/app/$libraryId/settings/resources/changelog.tsx +++ b/interface/app/$libraryId/settings/resources/changelog.tsx @@ -9,9 +9,10 @@ import { Heading } from '../Layout'; export const Component = () => { const platform = usePlatform(); const isDark = useIsDark(); - const changelog = useQuery(['changelog'], () => - fetch(`${platform.landingApiOrigin}/api/releases`).then((r) => r.json()) - ); + const changelog = useQuery({ + queryKey: ['changelog'], + queryFn: () => fetch(`${platform.landingApiOrigin}/api/releases`).then((r) => r.json()) + }); const { t } = useLocale(); diff --git a/interface/app/$libraryId/settings/resources/dependencies.tsx b/interface/app/$libraryId/settings/resources/dependencies.tsx deleted file mode 100644 index 2024f9252..000000000 --- a/interface/app/$libraryId/settings/resources/dependencies.tsx +++ /dev/null @@ -1,51 +0,0 @@ -// import { useQuery } from '@tanstack/react-query'; -// import { ScreenHeading } from '@sd/ui'; -// import { usePlatform } from '~/util/Platform'; - -// export const Component = () => { -// const frontEnd = useQuery( -// ['frontend-deps'], -// () => import('@sd/assets/deps/frontend-deps.json') -// ); -// const backEnd = useQuery(['backend-deps'], () => import('@sd/assets/deps/backend-deps.json')); -// const platform = usePlatform(); - -// return ( -//
    -// Dependencies - -// {/* item has a LOT more data that we can display, i just went with the basics */} - -// Frontend Dependencies -//
    -// {frontEnd.data && -// frontEnd.data?.default.map((item) => { -// return ( -// platform.openLink(item.url ?? '')}> -//
    -//

    -// {item.title.trimEnd().substring(0, 24) + -// (item.title.length > 24 ? '...' : '')} -//

    -//
    -//     -// ); -// })} -//
    - -// Backend Dependencies -//
    -// {backEnd.data && -// backEnd.data?.default.map((item) => { -// return ( -// platform.openLink(item.url ?? '')}> -//
    -//

    {item.title.trimEnd()}

    -//
    -//     -// ); -// })} -//
    -//
    -// ); -// }; diff --git a/interface/app/index.tsx b/interface/app/index.tsx index 3216925a1..2f70f199a 100644 --- a/interface/app/index.tsx +++ b/interface/app/index.tsx @@ -1,4 +1,4 @@ -import { initRspc, wsBatchLink, type AlphaClient } from '@oscartbeaumont-sd/rspc-client/src/v2'; +import { initRspc, wsBatchLink, type AlphaClient } from '@spacedrive/rspc-client'; import { QueryClient, QueryClientProvider } from '@tanstack/react-query'; import { useEffect, useMemo, useState } from 'react'; import { diff --git a/interface/app/onboarding/join-library.tsx b/interface/app/onboarding/join-library.tsx index d4bbed977..1baf7f967 100644 --- a/interface/app/onboarding/join-library.tsx +++ b/interface/app/onboarding/join-library.tsx @@ -55,7 +55,7 @@ function CloudLibraries() { {cloudLibrary.name} diff --git a/interface/components/Devtools.tsx b/interface/components/Devtools.tsx index acb72e803..952019859 100644 --- a/interface/components/Devtools.tsx +++ b/interface/components/Devtools.tsx @@ -1,4 +1,3 @@ -import { defaultContext } from '@tanstack/react-query'; import { ReactQueryDevtools } from '@tanstack/react-query-devtools'; import { useDebugState } from '@sd/client'; @@ -7,18 +6,9 @@ export const Devtools = () => { return ( <> - {debugState.reactQueryDevtools !== 'disabled' ? ( - - ) : null} + {debugState.reactQueryDevtools && ( + + )} ); }; diff --git a/interface/components/Sparkles.tsx b/interface/components/Sparkles.tsx index ec6b73ffa..2b1dc612d 100644 --- a/interface/components/Sparkles.tsx +++ b/interface/components/Sparkles.tsx @@ -38,7 +38,8 @@ type SparklesProps = { children: React.ReactNode; }; -const Sparkles = ({ color = DEFAULT_COLOR, children, ...props }: SparklesProps) => { +// million-ignore +const Sparkles = ({ color = DEFAULT_COLOR, children }: SparklesProps) => { const [sparkles, setSparkles] = useState(() => { return range(3).map(() => generateSparkle(color)); }); @@ -60,7 +61,7 @@ const Sparkles = ({ color = DEFAULT_COLOR, children, ...props }: SparklesProps) ); return ( - + {sparkles.map((sparkle) => ( { + return useSuspenseQuery({ + queryKey: ['userDirs', 'home'], + queryFn: () => { if (platform.userHomeDir) return platform.userHomeDir(); else return null; - }, - { suspense: true } - ); + } + }); } diff --git a/interface/hooks/useOperatingSystem.ts b/interface/hooks/useOperatingSystem.ts index c5797661e..acba5a8c6 100644 --- a/interface/hooks/useOperatingSystem.ts +++ b/interface/hooks/useOperatingSystem.ts @@ -17,17 +17,15 @@ export function guessOperatingSystem(): OperatingSystem { // Setting `realOs` to true will return a best guess of the underlying operating system instead of 'browser'. export function useOperatingSystem(realOs?: boolean): OperatingSystem { const platform = usePlatform(); - const { data } = useQuery( - ['_tauri', 'platform'], - async () => { + const { data } = useQuery({ + queryKey: ['_tauri', 'platform'], + queryFn: async () => { return platform.getOs ? await platform.getOs() : guessOperatingSystem(); }, - { - // Here we guess the users operating system from the user agent for the first render. - initialData: guessOperatingSystem, - enabled: platform.getOs !== undefined - } - ); + // Here we guess the users operating system from the user agent for the first render. + initialData: guessOperatingSystem, + enabled: platform.getOs !== undefined + }); return platform.platform === 'web' && !realOs ? 'browser' : data; } diff --git a/interface/package.json b/interface/package.json index c3a528139..0b3b91acf 100644 --- a/interface/package.json +++ b/interface/package.json @@ -26,10 +26,10 @@ "@sd/client": "workspace:*", "@sd/ui": "workspace:*", "@sentry/browser": "^7.74.1", - "@tanstack/react-query": "^4.36.1", - "@tanstack/react-query-devtools": "^4.36.1", - "@tanstack/react-table": "^8.10.7", - "@tanstack/react-virtual": "3.0.0-beta.66", + "@tanstack/react-query": "^5.59", + "@tanstack/react-query-devtools": "^5.59", + "@tanstack/react-table": "^8.20.5", + "@tanstack/react-virtual": "3.10.8", "@total-typescript/ts-reset": "^0.5.1", "@virtual-grid/react": "^2.0.2", "class-variance-authority": "^0.7.0", @@ -69,7 +69,7 @@ "use-debounce": "^9.0.4", "use-resize-observer": "^9.1.0", "uuid": "^9.0.1", - "valtio": "^1.11.2" + "valtio": "^2.0" }, "devDependencies": { "@sd/config": "workspace:*", diff --git a/interface/util/useTraceUpdate.tsx b/interface/util/useTraceUpdate.tsx new file mode 100644 index 000000000..c40ebc326 --- /dev/null +++ b/interface/util/useTraceUpdate.tsx @@ -0,0 +1,28 @@ +import { useEffect, useRef } from 'react'; + +/** + * DO NOT DELETE THIS HOOK + * It probably isn't used in the codebase, but it's a useful debugging tool. + */ +export function useTraceUpdate(name: string, props: object | null) { + const prev = useRef<{ [key: string]: any } | null>(props); + useEffect(() => { + const { current } = prev; + if (props == null) { + console.log(`Change ${name} to null`); + } else if (current == null) { + console.log(`Change ${name} from null to`, props); + } else { + const changedProps = Object.entries(props).reduce((ps: any, [k, v]) => { + if (current[k] !== v) { + ps[k] = [current[k], v]; + } + return ps; + }, {}); + if (Object.keys(changedProps).length > 0) { + console.log(`Changed ${name}:`, changedProps); + } + } + prev.current = props; + }); +} diff --git a/package.json b/package.json index 80924fac7..393d7f33c 100644 --- a/package.json +++ b/package.json @@ -73,5 +73,5 @@ "eslintConfig": { "root": true }, - "packageManager": "pnpm@9.9.0" + "packageManager": "pnpm@9.12.1" } diff --git a/packages/client/package.json b/packages/client/package.json index 68f7f1a07..f2db671f6 100644 --- a/packages/client/package.json +++ b/packages/client/package.json @@ -11,11 +11,11 @@ "typecheck": "tsc -b" }, "dependencies": { - "@oscartbeaumont-sd/rspc-client": "github:spacedriveapp/rspc#path:packages/client&bc882f4724", - "@oscartbeaumont-sd/rspc-react": "github:spacedriveapp/rspc#path:packages/react&bc882f4724", + "@spacedrive/rspc-client": "github:spacedriveapp/rspc#path:packages/client&6a77167495", + "@spacedrive/rspc-react": "github:spacedriveapp/rspc#path:packages/react&6a77167495", "@solid-primitives/deep": "^0.2.4", - "@tanstack/react-query": "^4.36.1", - "@tanstack/solid-query": "^5.17.9", + "@tanstack/react-query": "^5.59", + "@tanstack/solid-query": "^5.59", "@zxcvbn-ts/core": "^3.0.4", "@zxcvbn-ts/language-common": "^3.0.4", "@zxcvbn-ts/language-en": "^3.0.2", @@ -23,7 +23,7 @@ "plausible-tracker": "^0.3.8", "react-hook-form": "^7.47.0", "solid-js": "^1.8.8", - "zod": "~3.22.4" + "zod": "^3.23" }, "devDependencies": { "@sd/config": "workspace:*", diff --git a/packages/client/src/core.ts b/packages/client/src/core.ts index 2935c0951..bc4c2fe8c 100644 --- a/packages/client/src/core.ts +++ b/packages/client/src/core.ts @@ -1,5 +1,5 @@ /* eslint-disable */ -// This file was generated by [rspc](https://github.com/oscartbeaumont/rspc). Do not edit this file manually. +// This file was generated by [rspc](https://github.com/spacedriveapp/rspc). Do not edit this file manually. export type Procedures = { queries: diff --git a/packages/client/src/explorer/index.ts b/packages/client/src/explorer/index.ts index 34dcf4dce..9703b13af 100644 --- a/packages/client/src/explorer/index.ts +++ b/packages/client/src/explorer/index.ts @@ -1,5 +1,4 @@ export * from './useExplorerInfiniteQuery'; -export * from './usePathsInfiniteQuery'; export * from './usePathsOffsetInfiniteQuery'; export * from './usePathsExplorerQuery'; export * from './useObjectsInfiniteQuery'; diff --git a/packages/client/src/explorer/useExplorerInfiniteQuery.ts b/packages/client/src/explorer/useExplorerInfiniteQuery.ts index 1b3c25ce5..80b4c5767 100644 --- a/packages/client/src/explorer/useExplorerInfiniteQuery.ts +++ b/packages/client/src/explorer/useExplorerInfiniteQuery.ts @@ -6,5 +6,4 @@ import { Ordering } from './index'; export type UseExplorerInfiniteQueryArgs = { arg: TArg; order: TOrder | null; - onSuccess?: () => void; -} & Pick>, 'enabled' | 'suspense'>; +}; diff --git a/packages/client/src/explorer/useExplorerQuery.ts b/packages/client/src/explorer/useExplorerQuery.ts index b41c9d247..ad7630b5d 100644 --- a/packages/client/src/explorer/useExplorerQuery.ts +++ b/packages/client/src/explorer/useExplorerQuery.ts @@ -1,13 +1,16 @@ -import { UseInfiniteQueryResult, UseQueryResult } from '@tanstack/react-query'; +import { InfiniteData, UseInfiniteQueryResult, UseQueryResult } from '@tanstack/react-query'; import { useCallback, useMemo } from 'react'; import { SearchData } from '../core'; export function useExplorerQuery( - query: UseInfiniteQueryResult>, + query: UseInfiniteQueryResult>>, count: UseQueryResult ) { - const items = useMemo(() => query.data?.pages.flatMap((d) => d.items) ?? null, [query.data]); + const items = useMemo( + () => query.data?.pages.flatMap((data) => data.items) ?? null, + [query.data] + ); const loadMore = useCallback(() => { if (query.hasNextPage && !query.isFetchingNextPage) { diff --git a/packages/client/src/explorer/useObjectsInfiniteQuery.ts b/packages/client/src/explorer/useObjectsInfiniteQuery.ts index 6e758e1ba..c8680a2b8 100644 --- a/packages/client/src/explorer/useObjectsInfiniteQuery.ts +++ b/packages/client/src/explorer/useObjectsInfiniteQuery.ts @@ -8,8 +8,7 @@ import { UseExplorerInfiniteQueryArgs } from './useExplorerInfiniteQuery'; export function useObjectsInfiniteQuery({ arg, - order, - ...args + order }: UseExplorerInfiniteQueryArgs) { const { library } = useLibraryContext(); const ctx = useRspcLibraryContext(); @@ -21,25 +20,23 @@ export function useObjectsInfiniteQuery({ const query = useInfiniteQuery({ queryKey: ['search.objects', { library_id: library.uuid, arg }] as const, queryFn: ({ pageParam, queryKey: [_, { arg }] }) => { - const cItem: Extract = pageParam; - let orderAndPagination: (typeof arg)['orderAndPagination']; - if (!cItem) { + if (!pageParam || pageParam.type !== 'Object') { if (order) orderAndPagination = { orderOnly: order }; } else { let cursor: ObjectCursor | undefined; if (!order) cursor = 'none'; - else if (cItem) { + else if (pageParam) { switch (order.field) { case 'kind': { - const data = cItem.item.kind; + const data = pageParam.item.kind; if (data !== null) cursor = { kind: { order: order.value, data } }; break; } case 'dateAccessed': { - const data = cItem.item.date_accessed; + const data = pageParam.item.date_accessed; if (data !== null) cursor = { dateAccessed: { order: order.value, data } }; break; @@ -47,18 +44,18 @@ export function useObjectsInfiniteQuery({ } } - if (cursor) orderAndPagination = { cursor: { cursor, id: cItem.item.id } }; + if (cursor) orderAndPagination = { cursor: { cursor, id: pageParam.item.id } }; } arg.orderAndPagination = orderAndPagination; return ctx.client.query(['search.objects', arg]); }, + initialPageParam: undefined as ExplorerItem | undefined, getNextPageParam: (lastPage) => { if (lastPage.items.length < arg.take) return undefined; else return lastPage.items[arg.take - 1]; - }, - ...args + } }); return query; diff --git a/packages/client/src/explorer/useObjectsOffsetInfiniteQuery.ts b/packages/client/src/explorer/useObjectsOffsetInfiniteQuery.ts index 7bb20c9e4..3580bcf67 100644 --- a/packages/client/src/explorer/useObjectsOffsetInfiniteQuery.ts +++ b/packages/client/src/explorer/useObjectsOffsetInfiniteQuery.ts @@ -8,8 +8,7 @@ import { UseExplorerInfiniteQueryArgs } from './useExplorerInfiniteQuery'; export function useObjectsOffsetInfiniteQuery({ arg, - order, - ...args + order }: UseExplorerInfiniteQueryArgs) { const { library } = useLibraryContext(); const ctx = useRspcLibraryContext(); @@ -40,10 +39,10 @@ export function useObjectsOffsetInfiniteQuery({ return { ...result, offset: pageParam, arg }; }, + initialPageParam: 0, getNextPageParam: ({ items, offset, arg }) => { if (items.length >= arg.take) return (offset ?? 0) + 1; - }, - ...args + } }); return query; diff --git a/packages/client/src/explorer/usePathsExplorerQuery.ts b/packages/client/src/explorer/usePathsExplorerQuery.ts index f64f56504..4894fbc74 100644 --- a/packages/client/src/explorer/usePathsExplorerQuery.ts +++ b/packages/client/src/explorer/usePathsExplorerQuery.ts @@ -8,8 +8,6 @@ export function usePathsExplorerQuery(props: { order: FilePathOrder | null; enabled?: boolean; suspense?: boolean; - /** This callback will fire any time the query successfully fetches new data. (NOTE: This will be removed on the next major version (react-query)) */ - onSuccess?: () => void; }) { const query = usePathsOffsetInfiniteQuery(props); diff --git a/packages/client/src/explorer/usePathsInfiniteQuery.ts b/packages/client/src/explorer/usePathsInfiniteQuery.ts deleted file mode 100644 index c28a44b8a..000000000 --- a/packages/client/src/explorer/usePathsInfiniteQuery.ts +++ /dev/null @@ -1,134 +0,0 @@ -import { useInfiniteQuery } from '@tanstack/react-query'; - -import { - ExplorerItem, - FilePathCursorVariant, - FilePathObjectCursor, - FilePathOrder, - FilePathSearchArgs -} from '../core'; -import { useLibraryContext } from '../hooks'; -import { useRspcLibraryContext } from '../rspc'; -import { UseExplorerInfiniteQueryArgs } from './useExplorerInfiniteQuery'; - -export function usePathsInfiniteQuery({ - arg, - order, - onSuccess, - ...args -}: UseExplorerInfiniteQueryArgs) { - const { library } = useLibraryContext(); - const ctx = useRspcLibraryContext(); - - if (order) { - arg.orderAndPagination = { orderOnly: order }; - if (arg.orderAndPagination.orderOnly.field === 'sizeInBytes') delete arg.take; - } - - const query = useInfiniteQuery({ - queryKey: ['search.paths', { library_id: library.uuid, arg }] as const, - queryFn: async ({ pageParam, queryKey: [_, { arg }] }) => { - const cItem: Extract = pageParam; - - let orderAndPagination: (typeof arg)['orderAndPagination']; - - if (!cItem) { - if (order) orderAndPagination = { orderOnly: order }; - } else { - let variant: FilePathCursorVariant | undefined; - - if (!order) variant = 'none'; - else if (cItem) { - switch (order.field) { - case 'name': { - const data = cItem.item.name; - if (data !== null) - variant = { - name: { order: order.value, data } - }; - break; - } - case 'sizeInBytes': { - variant = { sizeInBytes: order.value }; - break; - } - case 'dateCreated': { - const data = cItem.item.date_created; - if (data !== null) - variant = { - dateCreated: { order: order.value, data } - }; - break; - } - case 'dateModified': { - const data = cItem.item.date_modified; - if (data !== null) - variant = { - dateModified: { order: order.value, data } - }; - break; - } - case 'dateIndexed': { - const data = cItem.item.date_indexed; - if (data !== null) - variant = { - dateIndexed: { order: order.value, data } - }; - break; - } - case 'object': { - const object = cItem.item.object; - if (!object) break; - - let objectCursor: FilePathObjectCursor | undefined; - - switch (order.value.field) { - case 'dateAccessed': { - const data = object.date_accessed; - if (data !== null) - objectCursor = { - dateAccessed: { order: order.value.value, data } - }; - break; - } - case 'kind': { - const data = object.kind; - if (data !== null) - objectCursor = { - kind: { order: order.value.value, data } - }; - break; - } - } - - if (objectCursor) variant = { object: objectCursor }; - - break; - } - } - } - - if (cItem.item.is_dir === null) throw new Error(); - - if (variant) - orderAndPagination = { - cursor: { cursor: { variant, isDir: cItem.item.is_dir }, id: cItem.item.id } - }; - } - - arg.orderAndPagination = orderAndPagination; - - const result = await ctx.client.query(['search.paths', arg]); - return result; - }, - getNextPageParam: (lastPage) => { - if (arg.take === null || arg.take === undefined) return undefined; - if (lastPage.items.length < arg.take) return undefined; - else return lastPage.items[arg.take - 1]; - }, - onSuccess, - ...args - }); - - return query; -} diff --git a/packages/client/src/explorer/usePathsOffsetInfiniteQuery.ts b/packages/client/src/explorer/usePathsOffsetInfiniteQuery.ts index 52880b489..95d6fba3b 100644 --- a/packages/client/src/explorer/usePathsOffsetInfiniteQuery.ts +++ b/packages/client/src/explorer/usePathsOffsetInfiniteQuery.ts @@ -7,9 +7,7 @@ import { UseExplorerInfiniteQueryArgs } from './useExplorerInfiniteQuery'; export function usePathsOffsetInfiniteQuery({ arg, - order, - onSuccess, - ...args + order }: UseExplorerInfiniteQueryArgs) { const take = arg.take ?? 100; @@ -49,11 +47,10 @@ export function usePathsOffsetInfiniteQuery({ return { ...result, offset: pageParam, arg }; }, + initialPageParam: 0, getNextPageParam: ({ items, offset, arg }) => { if (items.length >= arg.take) return (offset ?? 0) + 1; - }, - onSuccess, - ...args + } }); return query; diff --git a/packages/client/src/hooks/useClientContext.tsx b/packages/client/src/hooks/useClientContext.tsx index 26c870c76..f66825a82 100644 --- a/packages/client/src/hooks/useClientContext.tsx +++ b/packages/client/src/hooks/useClientContext.tsx @@ -1,5 +1,6 @@ -import { AlphaClient } from '@oscartbeaumont-sd/rspc-client/src/v2'; -import { createContext, PropsWithChildren, useContext, useMemo } from 'react'; +import { AlphaClient } from '@spacedrive/rspc-client'; +import { keepPreviousData } from '@tanstack/react-query'; +import { createContext, PropsWithChildren, useContext, useEffect, useMemo } from 'react'; import { LibraryConfigWrapped, Procedures } from '../core'; import { valtioPersist } from '../lib'; @@ -9,8 +10,8 @@ import { useBridgeQuery } from '../rspc'; const libraryCacheLocalStorageKey = 'sd-library-list3'; // number is because the format of this underwent breaking changes export const useCachedLibraries = () => { - const result = useBridgeQuery(['library.list'], { - keepPreviousData: true, + const query = useBridgeQuery(['library.list'], { + placeholderData: keepPreviousData, initialData: () => { const cachedData = localStorage.getItem(libraryCacheLocalStorageKey); @@ -24,14 +25,15 @@ export const useCachedLibraries = () => { } return undefined; - }, - onSuccess: (data) => { - if (data.length > 0) - localStorage.setItem(libraryCacheLocalStorageKey, JSON.stringify(data)); } }); - return result; + useEffect(() => { + if ((query.data?.length ?? 0) > 0) + localStorage.setItem(libraryCacheLocalStorageKey, JSON.stringify(query.data)); + }, [query.data]); + + return query; }; export async function getCachedLibraries(client: AlphaClient) { diff --git a/packages/client/src/index.ts b/packages/client/src/index.ts index 4a96e699f..dc0c5d41a 100644 --- a/packages/client/src/index.ts +++ b/packages/client/src/index.ts @@ -1,4 +1,4 @@ -import { Link } from '@oscartbeaumont-sd/rspc-client/src/v2'; +import { Link } from '@spacedrive/rspc-client'; declare global { // eslint-disable-next-line diff --git a/packages/client/src/lib/humanizeSize.ts b/packages/client/src/lib/humanizeSize.ts index 45a2cd167..654fe000f 100644 --- a/packages/client/src/lib/humanizeSize.ts +++ b/packages/client/src/lib/humanizeSize.ts @@ -146,3 +146,10 @@ export const humanizeSize = ( } }; }; + +export const compareHumanizedSizes = ( + size1: ReturnType, + size2: ReturnType +): boolean => { + return size1.bytes === size2.bytes && size1.unit === size2.unit && size1.value === size2.value; +}; diff --git a/packages/client/src/rspc-cursed.ts b/packages/client/src/rspc-cursed.ts index 774dcfcfb..ff1c026ef 100644 --- a/packages/client/src/rspc-cursed.ts +++ b/packages/client/src/rspc-cursed.ts @@ -1,5 +1,5 @@ -import { _inferProcedureHandlerInput, inferProcedureResult } from '@oscartbeaumont-sd/rspc-client'; -import { useQuery, UseQueryOptions, UseQueryResult } from '@tanstack/react-query'; +import { _inferProcedureHandlerInput, inferProcedureResult } from '@spacedrive/rspc-client'; +import { UseQueryOptions, useSuspenseQuery, UseSuspenseQueryResult } from '@tanstack/react-query'; import { useRef } from 'react'; import { Procedures } from './core'; @@ -26,19 +26,20 @@ export function useUnsafeStreamedQuery< TData = inferProcedureResult >( keyAndInput: [K, ..._inferProcedureHandlerInput], - opts: UseQueryOptions & { + opts: Omit, 'queryKey'> & { onBatch(item: TData): void; } -): UseQueryResult & { streaming: TData[] } { +): UseSuspenseQueryResult & { streaming: TData[] } { const data = useRef([]); const rspc = useRspcContext(); // TODO: The normalised cache might cleanup nodes for this query before it's finished streaming. We need a global mutex on the cleanup routine. - const query = useQuery({ + const query = useSuspenseQuery({ + ...opts, queryKey: keyAndInput, queryFn: ({ signal }) => - new Promise((resolve) => { + new Promise((resolve) => { permits += 1; try { @@ -48,7 +49,7 @@ export function useUnsafeStreamedQuery< if (item === null || item === undefined) return; if (typeof item === 'object' && '__stream_complete' in item) { - resolve(data.current as any); + resolve(data.current); return; } @@ -60,8 +61,7 @@ export function useUnsafeStreamedQuery< } finally { permits -= 1; } - }), - ...opts + }) }); return { diff --git a/packages/client/src/rspc.tsx b/packages/client/src/rspc.tsx index 40ec57782..579eacd7f 100644 --- a/packages/client/src/rspc.tsx +++ b/packages/client/src/rspc.tsx @@ -1,6 +1,5 @@ -import { ProcedureDef } from '@oscartbeaumont-sd/rspc-client'; -import { AlphaRSPCError, initRspc } from '@oscartbeaumont-sd/rspc-client/src/v2'; -import { Context, createReactQueryHooks } from '@oscartbeaumont-sd/rspc-react/src/v2'; +import { initRspc, ProcedureDef, RSPCError } from '@spacedrive/rspc-client'; +import { Context, createReactQueryHooks } from '@spacedrive/rspc-react/src/v2'; import { QueryClient } from '@tanstack/react-query'; import { createContext, PropsWithChildren, useContext } from 'react'; import { match, P } from 'ts-pattern'; @@ -102,7 +101,7 @@ export function useInvalidateQuery() { for (const op of ops) { match(op) .with({ type: 'single', data: P.select() }, (op) => { - let key: any[] = [op.key]; + let key: unknown[] = [op.key]; if (op.arg !== null) { key = key.concat(op.arg); } @@ -110,7 +109,7 @@ export function useInvalidateQuery() { if (op.result !== null) { context.queryClient.setQueryData(key, op.result); } else { - context.queryClient.invalidateQueries(key); + context.queryClient.invalidateQueries({ queryKey: key }); } }) .with({ type: 'all' }, (op) => { @@ -124,6 +123,6 @@ export function useInvalidateQuery() { // TODO: Remove/fix this when rspc typesafe errors are working export function extractInfoRSPCError(error: unknown) { - if (!(error instanceof AlphaRSPCError)) return null; + if (!(error instanceof RSPCError)) return null; return error; } diff --git a/packages/client/src/solid/index.ts b/packages/client/src/solid/index.ts index dad153583..3a7aaf513 100644 --- a/packages/client/src/solid/index.ts +++ b/packages/client/src/solid/index.ts @@ -2,7 +2,6 @@ export * from './createPersistedMutable'; export * from './react'; export * from './solid.solid'; export * from './useObserver'; -export * from './useUniversalQuery'; export * from './useSolidStore'; export { InteropProviderReact } from './portals'; export { createSharedContext } from './context'; diff --git a/packages/client/src/solid/solid.solid.tsx b/packages/client/src/solid/solid.solid.tsx index 8036b6e07..dfe24b907 100644 --- a/packages/client/src/solid/solid.solid.tsx +++ b/packages/client/src/solid/solid.solid.tsx @@ -1,5 +1,4 @@ /** @jsxImportSource solid-js */ - import { trackDeep } from '@solid-primitives/deep'; import { createElement, StrictMode, type FunctionComponent } from 'react'; import { createPortal } from 'react-dom'; diff --git a/packages/client/src/solid/useUniversalQuery.ts b/packages/client/src/solid/useUniversalQuery.ts deleted file mode 100644 index 2e9435448..000000000 --- a/packages/client/src/solid/useUniversalQuery.ts +++ /dev/null @@ -1,14 +0,0 @@ -// import { useQuery } from '@tanstack/react-query'; -// import { createQuery } from '@tanstack/solid-query'; - -// import { insideReactRender } from './internal'; - -// export function useUniversalQuery() { -// if (insideReactRender()) { -// useQuery(); -// } else { -// createQuery(); -// } -// } - -export {}; // TODO diff --git a/packages/client/src/stores/auth.ts b/packages/client/src/stores/auth.ts index d261fc845..a494854c0 100644 --- a/packages/client/src/stores/auth.ts +++ b/packages/client/src/stores/auth.ts @@ -1,4 +1,4 @@ -import { RSPCError } from '@oscartbeaumont-sd/rspc-client'; +import { RSPCError } from '@spacedrive/rspc-client'; import { createMutable } from 'solid-js/store'; import { nonLibraryClient } from '../rspc'; diff --git a/packages/client/src/stores/debugState.ts b/packages/client/src/stores/debugState.ts index 23fd751f4..7bfc80855 100644 --- a/packages/client/src/stores/debugState.ts +++ b/packages/client/src/stores/debugState.ts @@ -6,7 +6,7 @@ import { createPersistedMutable, useSolidStore } from '../solid'; export interface DebugState { enabled: boolean; rspcLogger: boolean; - reactQueryDevtools: 'enabled' | 'disabled' | 'invisible'; + reactQueryDevtools: boolean; shareFullTelemetry: boolean; // used for sending telemetry even if the app is in debug mode telemetryLogging: boolean; } @@ -16,7 +16,7 @@ export const debugState = createPersistedMutable( createMutable({ enabled: globalThis.isDev, rspcLogger: false, - reactQueryDevtools: globalThis.isDev ? 'invisible' : 'enabled', + reactQueryDevtools: false, shareFullTelemetry: false, telemetryLogging: false }) diff --git a/packages/config/vite/narrowSolidPlugin.ts b/packages/config/vite/narrowSolidPlugin.ts index 425970ff5..1791ff0fd 100644 --- a/packages/config/vite/narrowSolidPlugin.ts +++ b/packages/config/vite/narrowSolidPlugin.ts @@ -13,10 +13,14 @@ export interface NarrowSolidPluginOptions extends Partial { export function narrowSolidPlugin({ include, exclude, ...rest }: NarrowSolidPluginOptions = {}) { const plugin = solidPlugin(rest); - const originalConfig = plugin.config!.bind(plugin); + const originalConfig = + typeof plugin.config == 'function' + ? (plugin.config.bind(plugin) as typeof plugin.config) + : plugin.config; const filter = createFilter(include, exclude); plugin.config = (...args) => { - const baseConfig = originalConfig(...args); + const baseConfig = + typeof originalConfig == 'function' ? originalConfig?.(...args) : originalConfig; return { ...baseConfig, esbuild: { diff --git a/packages/ui/package.json b/packages/ui/package.json index fc2a83e9b..9cdeef6fd 100644 --- a/packages/ui/package.json +++ b/packages/ui/package.json @@ -44,7 +44,7 @@ "react-router-dom": "=6.20.1", "sonner": "^1.0.3", "use-debounce": "^9.0.4", - "zod": "~3.22.4" + "zod": "^3.23" }, "devDependencies": { "@babel/core": "^7.24.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d975ad8f58f93b878dd6d3a87712a6509a5575aa..12c81db051136b40249728f97db4e5afbb05285c 100644 GIT binary patch delta 4170 zcmcInd2k!m86RmS%U-P{JC1B8abhKeaU6l}l~y56Y}t}^*t)E{O&r~lSJLWOosg@v z6OICgg*;l0#4{bB2arZ(mNE>9OPlEogky%8PMLx-+=Yz5DpBy`JunX*oiPkiq=T}*9DIbo;W5skOPK8x2SR2;qlv zKZ!W}Z4Dwerk1JHGK@O48--9AN);A*Im}ZJTyMku;0Y30Lp?5B&r(t6f7wgzo93`o zV39=DQVSv;kda77^Hbrxki%#g0r%Drar3~&NBG>mEv3|AS73ttb!#RquQv%7bG)zs$F*v>)f-CMUy&ei>Tqv=bNH?}`w}eRA8h_M>U3gH zQ?c+A1pae@-$T84)b*_p_)=j1AN|Q-#|P(P5TDwb?Y{(K$1;fEgHQPb&HRHo_7x76 zP#^V7Hs>CCw*QKW)U6LU>4%%Fi^pi4pMBqNyH-;9D(MD<9`?L!CBOH#L9N7#r8JS- zAyLRwG6l?x4Uesq1n{JXZCIVROf;_r0xvr~cr8W1wY{sKI{rR9w4zF0Eajs&Ml(aPM=lZuQnKAAA4*XxHTU((anm-6Z z;~>8m60lJ{#ro^EfPZQdD_c?ltvnVt%Sk*`aN48p`gm&6TPW#N`iS2~80tauERl?s zQh|gilFvJ09-|wT7J|C0sZz#lMw7MbQRHex%$;z0d?|%%bQCP@f%=eGyh0L0rqoIp z(DMp3;!B%iE~6(pjtn0wjzeYsV!U67m4`BKWp!%>nMRAQ1_ZebpJFt!ic&^}ZP zX>V{ckXJh#?nuxr#mshBO`1t)OggolNKa%ZqO+AyC0R47^p12Q>oVbrk|Eo=N-ysfHk02Y426@qgg@P+_q4(g%y9pZ7A z&ZY$2@3BU_zDOhDo0=_!GA^8~PZynNu4a^0jLMK(>Q&fH<9=tvI!#Qeq?0yhHKv-> z#e9|tkI!NW2Q$_}KpC1a=7N)vup^(HsYXiml!36USblUfwB_h4JP{b&0S$t&8z3%t zBL?+<5#e4{eZmnbS@m{v-ZpDXn5}BJ%Hu8vBQ?^iGB=V~v6hL~GU~E9tM*wcE|=0~ zR$_8zFy*1r%@{BS9;jAkK&|XW%q=`(~-Ql9y6BAU~?yLBXFGS zK4QGT>lkYteRxyXJ~K-V4nNp2>LkNa zFyI=1&0wDxo}wRJ4~xP~=zYOjc=Xwu;m3CYrV(#AGV>x#&%DULTYv42EM}_Z&HKMe z&;42|%`ixS72)ewu^zxW4I5WCP=<4(ms#{6IPo&v1x}sebH8E#{RWI4G=B4?JHpkD zAOXK1kzR(dx0*1`G~vI*+Q=?aU4!7`Fw)unQvO-A1-$b(A^<;(A~4WI5ZN`Pdcdg& zqFr`S78wHlQAE+ViZ?PhiXD>WD+Rc$nPxN>Snamo*&?wmXSDls@9O@{L|uF0;$GxrjO1bh(X_M(mcHzlE`~B3P9Pp|HiL*bElv&I9Yw}~B#W$Djz)XyJnK0Ov&Jq@b+t&R zkkc{~ymo;vyL@E9CGt7<8Gj@AB!~1Z>l6~mb)790ZQ~zFLIdCcfed!2THnj{ie~E7 ziPp{QG>Hs=ACSn7<##XhSg0}t>mw@z%OfPjqj#2(yC+w2rHke~huqKuI$lOl@bj0D zZo2#ua`V=<|Ni-RWP9s{mVAs5H_=S}GfC;)0m0cpdZaA4b1iFxT0FLu#_tgHvgz)- z1nX`Ej~*3tfUf&tEs*pGx%8oK;X_=o=cj^hAng-wfmAXj?d%h-`0Yp) z7HxUp1b+yS!=fW#4i$BQzO%dy^zm&XY_5HdrI(Nm?QrBWhk;?>BH(^ndzle@SKnP#v)&*pChlD7ow*W`;tt$t&a z$mYYvcq_?GUeHD9*Ar)|ANvT`S^f$+E~%ner?>wR3{nNov~`0j^Na7B2CgsJf2u`n@-%xIF*lyt<*ENGn3RuJ?Vmw zB~heoB~3ZQ8SLAA@BQ!pzyEvx_kZ=7{SUpf|EZ&cBUCLJuhtXkc%wv>>oc_!>%JQw zOe75kO-74qSePAUk4p67t0P))Zi8T)9oo0v+_xmY&z#HdfW}6r_pjL$JodK4eB&Ox z-Qe7uY%4p}%CR5qJ;;W{&#;ry0|Jdw%fd${*+-=lLah?fvlVPi!2Mvqacnnu>U)y= z*)P`z!9OlQ681gm6c1*HhbGuP8zexz0EvO`03=;@4CRM}+&a6#=_E7`*MBP$UTF+T zmoLn!wh3V+4711BG4|YGAK37QYy*4Fy_ubTa?^Ug$t=IR0HkY8yk$g**$?&};=;VZ z9+xU^655ji(emNK-^p(Bnv49W2OMG}N3H=x)J2LCo(6{_f^p&gRd5-GMXc*K73)7G zVNXi?*;kKD^59e79`83OVJ`A4uYt1OFDWHzrFtfjuE&*nJ^S=ukF9zhfhU$Pe90&3 z0q|r%x{Q1^A?#vx`7a-MfWO9*d)Vhw2ba%%?Tq06;IQ>l@c#uzy;GfgLJSJOk_@v) zAHvzYj*P8~psN|k#bJOEUu$yLbpEt8Y_!supea#Jx7%@FLFFSoI*r3RA0pG40%h_=8;dC_ zuGZ3ez}v~+4o-f|7Xu;)$p({DHQmi#J^S(xrxz^Fd9oPNDOz>R58D#9q}53=jJ8@X zmsDn|&+ZB^ z-@bdrVt@ysQJ@S9M0||_9NWm7UTEO4iqBSYhY2R)b>J~Q5=pt*S&hb9pKE8$7?H7* z@|HOg!6=)eOhnU|m+_FQMVr5Ft9o*kK-;cyRJEmO++GXE+9lGOkI=o{P8Bw>z9WNu zSC2-7qlJb6F}wVTFU^#r^L9g2h5L+Ibv20U7?>*Q985AyG)W@n)Vky4sNEeh#|*)g z&92L5Y89bwI4}SS3-wnP`u4C3^;$dZxURJ3GM`s zF7t=Lp|hd^0j%EzJ|>_k@ioQ=I}qM3q8+p8=W2y)kn!R)O;k}=ESqxMeAY-IS=MB7 zMA6G6G4p&9O_nW)-cHA`sxM)QmRqsBC)Gl$3cH=i*csAOPqh^lRl!z9an^Kx2jG0N z2k=7BW}b$v{B%0#v!*;)E2GI-GnTkMVoBJGF|VQG(9{5cp)l`>^*jPd(IxL? z^`_3yU_4GWn(%3Cup{hQ%-aG5CmC_!fkqPXRW$KNCQT(Qhyg_uTFT*3SpuzjM-x@x zHNPLc|Dtdw`0+u(mK%c>sn)y!6}bU8*FXpP#SNz(>vR=Zf$=|inz%;P2S zIb8_FN{V)PF7C@(vP_!{=E7B%K5s_jk=9(*;UKD!T-@89(>M3+1KBV%#nS-%clO+_-EO+_q0RfYS$WWq>yUCU^rSWr{ZFcq^eN9NKB3+$~U z8V7VM{@g3pcIHgsRNzX9xVx6S+s{#I@U^n42 zXEj94ort(ld#OaXJ&|0rkn-uZOwI`hd<0>S2W_pne5lFgjx`w!2J&ba4lcSA^Vni4 zYQWu?-<+iNdQY$wpKqxHv=2r><(K>&lHNvp<-7Lm>iqM&!dVr#<6+@+C>e3v^L}+w z8$>e2nA3&NH`-*Bs-T`?P*E*2nOHtUqdA2Gbr~^lM5itIjpbIDNhO+jT-$)jaKh)- zHjMgAHXU;ZDZCgnqs`;5Pxf*D{hik*MK2!(Km4SB4`EnrnA0AlXlPYZrG%e~rrn9E zDPqRL4cHj5Mw2>~O=HODD!w>na(G&q5=In?y7(gLQY;#&BEbZT$#j7!my8CJ-x#oz zvpz>h^O)$_(qKH93ZsPhf}D3B}&2Eh*&2DbtEuXr<^OCy8#&4CeF|1?3G6$S^T))rH#q@on6;6_sn z_2C*%m2la;3d(`EDln5SW$RjfzG-f{Gr0D78TXj zq9a`D{314Z<{6HCcs57@hGpUmSQ-+GfUn5g0349G*x?=@JbZ&j8Sf-1@#C|%RyI!n z`P-6pBnG&#F8*Vb>55DM@)yP1*D>buuO-&&3-3=m4xBFBD3o8X-5z;KyczucZfH07 z`EMoC+gFO|nR2|&vB%9y=|73}?`6g7iH}<{oYUA}2lRG{^ba(A9Sv~Hy%cclFT_*B zHwpo8GkZVnp7eo-4nTc8ctso>=fpv0^BKuEjn~P76aOW-lvtx|KPu-|Z1M<`)VksVo8@C!Miss$J3q1~U~HJDz( zFc=O)gIm{%BW~%gJW?9i`=nEAko(0^xxCh(dtA9!-X%*V0;Fvj3Vu&OBP(5+)vb!F zn{DhiyukTqC7Zy~8F4>&?{%3FY--E6mf7=SIkJcfTbUc z$G}sRxQ`VU#;z^fMRM2tKX}DT&qyx;4G6n!67br~V%Lfzz!?IP^~TNy3DErW*;)Pl)nbv8@O1r^Vyo`$10S4u>EyR}-InP<{3AgB#Y&<*Kk3 z#aqEr2%7CiPY39?BzJa&&X!&LvlYh9?PuarnD7wyO!vOB;JUXX2tKCNdy9o}^tmrSYj=T0AG z<;qcT?giPFHA{?wi$(Fq&i~9xFKpw$2X~%ckbY+joXts3v0wh?Fp%e^<6v0d*AJ4U z^f36~qGY1;0WE!mckPoe-Y?w(Q#x?qliZPdNAK!FwMJZSB1K!4N#!yEUG=Q)UElNOqiEx*r1PlpOxAA)MMM z8w5W)FWTCHq%y;y75|@q3EH|sO}F)Uqbi3i%0J!6CSsl!qtUbye)@)g`HqA;<5Pd*A z(fR1I?A7trUI!-y!;Nlu!N{LU$2;4F!<%{FyJuxGFw2*TIzQnJv%Jrnad>pK)5oqk Ro#XQTy{FEz6T^Pq{{mCd8HWG> diff --git a/scripts/list-dup-deps.sh b/scripts/list-dup-deps.sh deleted file mode 100755 index 97c4023ff..000000000 --- a/scripts/list-dup-deps.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -_root="$(CDPATH='' cd "$(dirname "$0")" && pwd -P)" - -grep -Po '^\s+"[\w-]+\s+\d(\.\d+)*[^"]*"' "${_root}/../Cargo.lock" \ - | xargs printf '%s\n' \ - | sort -u -k 1b,2V diff --git a/scripts/preprep.mjs b/scripts/preprep.mjs index 7ffa7a06a..7e0799e03 100755 --- a/scripts/preprep.mjs +++ b/scripts/preprep.mjs @@ -1,5 +1,4 @@ #!/usr/bin/env node - import * as fs from 'node:fs/promises' import * as path from 'node:path' import { env, exit, umask } from 'node:process' @@ -9,11 +8,11 @@ import { extractTo } from 'archive-wasm/src/fs.mjs' import * as _mustache from 'mustache' import { parse as parseTOML } from 'smol-toml' -import { getConst, NATIVE_DEPS_URL, NATIVE_DEPS_ASSETS } from './utils/consts.mjs' +import { getConst, NATIVE_DEPS_ASSETS, NATIVE_DEPS_URL } from './utils/consts.mjs' import { get } from './utils/fetch.mjs' import { getMachineId } from './utils/machineId.mjs' import { getRustTargetList } from './utils/rustup.mjs' -import { symlinkSharedLibsMacOS, symlinkSharedLibsLinux } from './utils/shared.mjs' +import { symlinkSharedLibsLinux, symlinkSharedLibsMacOS } from './utils/shared.mjs' import { spinTask } from './utils/spinner.mjs' import { which } from './utils/which.mjs' diff --git a/scripts/tauri.mjs b/scripts/tauri.mjs index 0eed0d73f..46e5f5640 100755 --- a/scripts/tauri.mjs +++ b/scripts/tauri.mjs @@ -1,8 +1,7 @@ #!/usr/bin/env node - import * as fs from 'node:fs/promises' import * as path from 'node:path' -import { env, exit, umask, platform } from 'node:process' +import { env, exit, platform, umask } from 'node:process' import { setTimeout } from 'node:timers/promises' import { fileURLToPath } from 'node:url' diff --git a/scripts/utils/fetch.mjs b/scripts/utils/fetch.mjs index cf8cd6c41..5907e418e 100644 --- a/scripts/utils/fetch.mjs +++ b/scripts/utils/fetch.mjs @@ -4,7 +4,7 @@ import { env } from 'node:process' import { fileURLToPath } from 'node:url' import { getSystemProxy } from 'os-proxy-config' -import { fetch, Headers, Agent, ProxyAgent } from 'undici' +import { Agent, fetch, Headers, ProxyAgent } from 'undici' const CONNECT_TIMEOUT = 5 * 60 * 1000 const __debug = env.NODE_ENV === 'debug' From 177763c536c4fdf49fcb214111dd1a795ac59346 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADtor=20Vasconcellos?= Date: Mon, 21 Oct 2024 14:49:02 -0300 Subject: [PATCH 11/21] Autoformat --- .../src/screens/settings/client/AccountSettings/Login.tsx | 2 +- interface/components/Authentication.tsx | 2 +- interface/components/Login.tsx | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/mobile/src/screens/settings/client/AccountSettings/Login.tsx b/apps/mobile/src/screens/settings/client/AccountSettings/Login.tsx index 464a701a5..fddd07041 100644 --- a/apps/mobile/src/screens/settings/client/AccountSettings/Login.tsx +++ b/apps/mobile/src/screens/settings/client/AccountSettings/Login.tsx @@ -1,6 +1,6 @@ -import { RSPCError } from '@spacedrive/rspc-client'; import AsyncStorage from '@react-native-async-storage/async-storage'; import { useNavigation } from '@react-navigation/native'; +import { RSPCError } from '@spacedrive/rspc-client'; import { UseMutationResult } from '@tanstack/react-query'; import { useState } from 'react'; import { Controller } from 'react-hook-form'; diff --git a/interface/components/Authentication.tsx b/interface/components/Authentication.tsx index 4669eb0c6..22c4c7269 100644 --- a/interface/components/Authentication.tsx +++ b/interface/components/Authentication.tsx @@ -1,6 +1,6 @@ -import { RSPCError } from '@spacedrive/rspc-client'; import { GoogleLogo, Icon } from '@phosphor-icons/react'; import { Apple, Github } from '@sd/assets/svgs/brands'; +import { RSPCError } from '@spacedrive/rspc-client'; import { UseMutationResult } from '@tanstack/react-query'; import { open } from '@tauri-apps/plugin-shell'; import clsx from 'clsx'; diff --git a/interface/components/Login.tsx b/interface/components/Login.tsx index 0a9a6d386..a4ac8bf63 100644 --- a/interface/components/Login.tsx +++ b/interface/components/Login.tsx @@ -1,5 +1,5 @@ -import { RSPCError } from '@spacedrive/rspc-client'; import { ArrowLeft } from '@phosphor-icons/react'; +import { RSPCError } from '@spacedrive/rspc-client'; import { UseMutationResult } from '@tanstack/react-query'; import clsx from 'clsx'; import { Dispatch, SetStateAction, useState } from 'react'; From 2fa90d64632accea4b2c00e05e84b60f44a42101 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADtor=20Vasconcellos?= Date: Mon, 21 Oct 2024 19:44:08 -0300 Subject: [PATCH 12/21] Update iroh and sd-cloud-schema - Fix usage of deprecated functions - Implement clippy beta suggestions --- Cargo.lock | Bin 332951 -> 331411 bytes Cargo.toml | 2 +- core/crates/cloud-services/Cargo.toml | 2 +- .../src/p2p/new_sync_messages_notifier.rs | 2 +- core/crates/cloud-services/src/p2p/runner.rs | 2 +- .../heavy-lifting/src/file_identifier/job.rs | 8 ++++---- .../heavy-lifting/src/file_identifier/mod.rs | 2 +- .../src/media_processor/shallow.rs | 2 +- core/crates/indexer-rules/src/serde_impl.rs | 2 +- core/crates/prisma-helpers/src/lib.rs | 2 +- core/src/api/utils/invalidate.rs | 1 + core/src/api/utils/library.rs | 5 ++++- core/src/util/mpscrr.rs | 2 +- crates/ai/Cargo.toml | 1 + crates/crypto/src/ct.rs | 2 +- crates/ffmpeg/src/dict.rs | 2 +- crates/images/src/consts.rs | 2 +- crates/media-metadata/src/exif/datetime.rs | 2 +- crates/p2p/src/smart_guards.rs | 6 +++--- 19 files changed, 26 insertions(+), 21 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b04ea2fcab529017e2a76465f9690ce71a16a464..082cbc3c36e6e6886d58a9c73488ea16165bf231 100644 GIT binary patch delta 1193 zcmYL|TZorM7{+tf_$QFBzm6YH87LSf-z z1ZrRHK&UMeM0c_aO@fD(y67q|f@E(Zy9om&3+vKGH}Y;6W}fHed46yFva>$ldFh<0 zZ*5o*f4Kr}ar4K}IXqiH3L(t*heIjo7n+_5SMUSTSzlAbb58D~@|6tHqu(Mm`75t=nx)Tkk? ziW`oW(`%Y~v%dZH>^RWg=qe8!*&olg72V_WE>C5Y zgSa*pIqk5;MtiALcFu+sHI|u53Zt-9oC?WBz5VHC{9zBwh$jYMVeGgMv*O)l1U`%~9$=v@}?{9W!yr-$Kfs5h@)5Wjf_x<@|951%^) zo61L}rgI;{tfF3Rcf_k_;AH)d>WLGl;f1(!tT8iPJqvST>fLW2!4627gG11<;@OVn zEA|{ap@|bhnu-}Bq*aOq>#)m} z1sdn=f@Yc=gXK@eOYOy+`o_xVTO_A0gcBYa>Pc|gupGA`*l=ciZj(`Msm%>*VyzDf zRh~(t9i=$qR8JgirqRo=ub7f-O>Ws}vvHsZ5#q5G8lzy5_DFN1Gw_&VBP=zFxC{#q z+$y1sRy6pEX3K|)anrTjhu!4>%Qk$!wo!Iq5-=g(3y9x`Q)^Smx5kCcfw zj3R<7Z!<4}T4Iz7EJ&>g6)1QrJf?Y*x@#n6q;1z=?|*6ExDM0P&)4D3+BoN@#?o~A zE^Oi_#L{zyl&ZWBj_HaZtb>W#LU&qy xr0gvgPT@2(bZE_>KCs(ArfZ74;iKizIm3c1SVfhqt1 delta 1802 zcmZ9NTWFVM9LIV7?|)OL(-zzQ&eZ*#P?_%Ub3VxQzd)gj4%3KOczTXn~LIO{QT4lzL` zB2)+E8S#R|^CP87{@XUfIS81137j8Ll_^idxLo%yv3^RuCscEz^mK*xB; z#jieymltIN>x35GI?JSzfon?A!8C)d@)&b&yh-zpx?lq4gd&L5ak>TY%;)yV^;t@ z4cl#VckbJ_rwVJhhH!rVr-gA#0PBmo2d{O^Mk`a4x#DGn_p2zi}ib4VEP zCAKit%p#LAObiQ3bM7%`nk!;_Jo8uzx%zdfWu~3Du?MU!&K?E7f^vi1_m%-vTp0!H zvl;xh7hT6dYpI;brV*?Jo7PDSC4tz>pb23n@o0pURtOdX<6dY;wC6MgnG!-}!#Nnd zH3E{f9EZQird)~r_0p_({W$1|RRr4N)xL7eDx7kVQp_jWnvs+LzI)AXcOY$|H6E9| z1vbP(CjiQ-JtJ}3YOo|e^)6_NUB|(V=IK2rMAn{PUOQ)6aYZ_#C*?P-&-S=sq+E?x zNqKf03JHQWpMlomo3}ybo?XR1?|{vqeM;g}&K%g|_cnAMyT2 zKpkYmg-rVdSpvtI3*-qlLRlpoM_xgU156TN-e9Ac;WW+J*mfYR z{lrSj6Ed8Oj;yzIy1=@i}nBX`EVQk Result<(), Error> { let client = Client::new(RpcClient::new(QuinnConnection::::from_connection( endpoint - .connect_by_node_id(*connection_id, CloudP2PALPN::LATEST) + .connect(*connection_id, CloudP2PALPN::LATEST) .await .map_err(Error::ConnectToCloudP2PNode)?, ))); diff --git a/core/crates/cloud-services/src/p2p/runner.rs b/core/crates/cloud-services/src/p2p/runner.rs index 298a6700d..3dfc33be2 100644 --- a/core/crates/cloud-services/src/p2p/runner.rs +++ b/core/crates/cloud-services/src/p2p/runner.rs @@ -601,7 +601,7 @@ async fn connect_to_first_available_client( ) -> Result, Service>, CloudP2PError> { for (device_pub_id, device_connection_id) in devices_in_group { if let Ok(connection) = endpoint - .connect_by_node_id(*device_connection_id, CloudP2PALPN::LATEST) + .connect(*device_connection_id, CloudP2PALPN::LATEST) .await .map_err( |e| error!(?e, %device_pub_id, "Failed to connect to authorizor device candidate"), diff --git a/core/crates/heavy-lifting/src/file_identifier/job.rs b/core/crates/heavy-lifting/src/file_identifier/job.rs index 249ea57f2..dc2d6866c 100644 --- a/core/crates/heavy-lifting/src/file_identifier/job.rs +++ b/core/crates/heavy-lifting/src/file_identifier/job.rs @@ -374,7 +374,7 @@ impl FileIdentifier { self.last_orphan_file_path_id = None; self.dispatch_deep_identifier_tasks( - &maybe_sub_iso_file_path, + maybe_sub_iso_file_path.as_ref(), ctx, device_id, dispatcher, @@ -419,7 +419,7 @@ impl FileIdentifier { self.last_orphan_file_path_id = None; self.dispatch_deep_identifier_tasks( - &maybe_sub_iso_file_path, + maybe_sub_iso_file_path.as_ref(), ctx, device_id, dispatcher, @@ -433,7 +433,7 @@ impl FileIdentifier { Phase::SearchingOrphans => { self.dispatch_deep_identifier_tasks( - &maybe_sub_iso_file_path, + maybe_sub_iso_file_path.as_ref(), ctx, device_id, dispatcher, @@ -752,7 +752,7 @@ impl FileIdentifier { async fn dispatch_deep_identifier_tasks( &mut self, - maybe_sub_iso_file_path: &Option>, + maybe_sub_iso_file_path: Option<&IsolatedFilePathData<'static>>, ctx: &impl JobContext, device_id: device::id::Type, dispatcher: &JobTaskDispatcher, diff --git a/core/crates/heavy-lifting/src/file_identifier/mod.rs b/core/crates/heavy-lifting/src/file_identifier/mod.rs index a44afbc40..f777c118d 100644 --- a/core/crates/heavy-lifting/src/file_identifier/mod.rs +++ b/core/crates/heavy-lifting/src/file_identifier/mod.rs @@ -176,7 +176,7 @@ fn orphan_path_filters_shallow( fn orphan_path_filters_deep( location_id: location::id::Type, file_path_id: Option, - maybe_sub_iso_file_path: &Option>, + maybe_sub_iso_file_path: Option<&IsolatedFilePathData<'_>>, ) -> Vec { sd_utils::chain_optional_iter( [ diff --git a/core/crates/heavy-lifting/src/media_processor/shallow.rs b/core/crates/heavy-lifting/src/media_processor/shallow.rs index fd7caac14..675dcd791 100644 --- a/core/crates/heavy-lifting/src/media_processor/shallow.rs +++ b/core/crates/heavy-lifting/src/media_processor/shallow.rs @@ -220,7 +220,7 @@ async fn dispatch_media_data_extractor_tasks( async fn dispatch_thumbnailer_tasks( parent_iso_file_path: &IsolatedFilePathData<'_>, should_regenerate: bool, - location_path: &PathBuf, + location_path: &Path, dispatcher: &BaseTaskDispatcher, ctx: &impl OuterContext, ) -> Result>, Error> { diff --git a/core/crates/indexer-rules/src/serde_impl.rs b/core/crates/indexer-rules/src/serde_impl.rs index a0b24dd23..461630669 100644 --- a/core/crates/indexer-rules/src/serde_impl.rs +++ b/core/crates/indexer-rules/src/serde_impl.rs @@ -60,7 +60,7 @@ impl<'de> Deserialize<'de> for RulePerKind { struct FieldsVisitor; - impl<'de> de::Visitor<'de> for FieldsVisitor { + impl de::Visitor<'_> for FieldsVisitor { type Value = Fields; fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { diff --git a/core/crates/prisma-helpers/src/lib.rs b/core/crates/prisma-helpers/src/lib.rs index ee8f11bb9..311d81947 100644 --- a/core/crates/prisma-helpers/src/lib.rs +++ b/core/crates/prisma-helpers/src/lib.rs @@ -338,7 +338,7 @@ impl Clone for CasId<'_> { } } -impl<'cas_id> CasId<'cas_id> { +impl CasId<'_> { #[must_use] pub fn as_str(&self) -> &str { self.0.as_ref() diff --git a/core/src/api/utils/invalidate.rs b/core/src/api/utils/invalidate.rs index 8df2eea6d..e888b08a2 100644 --- a/core/src/api/utils/invalidate.rs +++ b/core/src/api/utils/invalidate.rs @@ -121,6 +121,7 @@ impl InvalidRequests { } /// `invalidate_query` is a macro which stores a list of all of it's invocations so it can ensure all of the queries match the queries attached to the router. +/// /// This allows invalidate to be type-safe even when the router keys are stringly typed. /// ```ignore /// invalidate_query!( diff --git a/core/src/api/utils/library.rs b/core/src/api/utils/library.rs index effdb89ba..001943f55 100644 --- a/core/src/api/utils/library.rs +++ b/core/src/api/utils/library.rs @@ -22,7 +22,10 @@ pub(crate) struct LibraryArgs { pub(crate) struct LibraryArgsLike; impl MwArgMapper for LibraryArgsLike { - type Input = LibraryArgs where T: Type + DeserializeOwned + 'static; + type Input + = LibraryArgs + where + T: Type + DeserializeOwned + 'static; type State = Uuid; fn map( diff --git a/core/src/util/mpscrr.rs b/core/src/util/mpscrr.rs index 4c7826bea..72daf3441 100644 --- a/core/src/util/mpscrr.rs +++ b/core/src/util/mpscrr.rs @@ -230,7 +230,7 @@ impl<'a> Bomb<'a> { } } -impl<'a> Drop for Bomb<'a> { +impl Drop for Bomb<'_> { fn drop(&mut self) { self.0.store(false, Ordering::Relaxed); } diff --git a/crates/ai/Cargo.toml b/crates/ai/Cargo.toml index b521ee93d..35f9dc198 100644 --- a/crates/ai/Cargo.toml +++ b/crates/ai/Cargo.toml @@ -40,6 +40,7 @@ uuid = { workspace = true, features = ["serde", "v4"] } # Note: half and ndarray version must be the same as used in ort half = { version = "2.4", features = ['num-traits'] } ndarray = "0.15" +ort-sys = '=2.0.0-rc.0' # lock sys crate to the same version as ort url = '2.5' # Microsoft does not provide a release for osx-gpu. See: https://github.com/microsoft/onnxruntime/releases diff --git a/crates/crypto/src/ct.rs b/crates/crypto/src/ct.rs index e7edf6a89..8ce937ab9 100644 --- a/crates/crypto/src/ct.rs +++ b/crates/crypto/src/ct.rs @@ -87,7 +87,7 @@ impl ConstantTimeEq for String { } } -impl<'a> ConstantTimeEq for &'a str { +impl ConstantTimeEq for &str { fn ct_eq(&self, rhs: &Self) -> Choice { // Here we are just able to convert both values to bytes and use the // appropriate methods to compare the two in constant-time. diff --git a/crates/ffmpeg/src/dict.rs b/crates/ffmpeg/src/dict.rs index 7d1d5726b..feb84184f 100644 --- a/crates/ffmpeg/src/dict.rs +++ b/crates/ffmpeg/src/dict.rs @@ -87,7 +87,7 @@ pub struct FFmpegDictIter<'a> { _lifetime: std::marker::PhantomData<&'a ()>, } -impl<'a> Iterator for FFmpegDictIter<'a> { +impl Iterator for FFmpegDictIter<'_> { type Item = (String, Option); fn next(&mut self) -> Option<(String, Option)> { diff --git a/crates/images/src/consts.rs b/crates/images/src/consts.rs index cc844e19f..4afd8da84 100644 --- a/crates/images/src/consts.rs +++ b/crates/images/src/consts.rs @@ -159,7 +159,7 @@ impl serde::Serialize for ConvertibleExtension { struct ExtensionVisitor; #[cfg(feature = "serde")] -impl<'de> serde::de::Visitor<'de> for ExtensionVisitor { +impl serde::de::Visitor<'_> for ExtensionVisitor { type Value = ConvertibleExtension; fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { diff --git a/crates/media-metadata/src/exif/datetime.rs b/crates/media-metadata/src/exif/datetime.rs index 39c6a40b6..b238dcd4f 100644 --- a/crates/media-metadata/src/exif/datetime.rs +++ b/crates/media-metadata/src/exif/datetime.rs @@ -77,7 +77,7 @@ impl serde::Serialize for MediaDate { struct MediaDateVisitor; -impl<'de> Visitor<'de> for MediaDateVisitor { +impl Visitor<'_> for MediaDateVisitor { type Value = MediaDate; fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { diff --git a/crates/p2p/src/smart_guards.rs b/crates/p2p/src/smart_guards.rs index 6177ed930..a920508aa 100644 --- a/crates/p2p/src/smart_guards.rs +++ b/crates/p2p/src/smart_guards.rs @@ -28,7 +28,7 @@ impl<'a, T: Clone> SmartWriteGuard<'a, T> { } } -impl<'a, T> Deref for SmartWriteGuard<'a, T> { +impl Deref for SmartWriteGuard<'_, T> { type Target = T; fn deref(&self) -> &Self::Target { @@ -36,13 +36,13 @@ impl<'a, T> Deref for SmartWriteGuard<'a, T> { } } -impl<'a, T> DerefMut for SmartWriteGuard<'a, T> { +impl DerefMut for SmartWriteGuard<'_, T> { fn deref_mut(&mut self) -> &mut Self::Target { &mut self.lock } } -impl<'a, T> Drop for SmartWriteGuard<'a, T> { +impl Drop for SmartWriteGuard<'_, T> { fn drop(&mut self) { (self.save)( self.p2p, From f2f9f91e626b8cc324e059bd705fad80fd5d2a72 Mon Sep 17 00:00:00 2001 From: Ericson Soares Date: Mon, 21 Oct 2024 22:27:09 -0300 Subject: [PATCH 13/21] Optimize and fix attempt at cloud ingester --- core/crates/cloud-services/src/sync/ingest.rs | 85 +++----- core/crates/sync/src/ingest_utils.rs | 19 +- core/crates/sync/src/lib.rs | 4 +- core/crates/sync/src/manager.rs | 200 +++++++++++++----- core/src/location/mod.rs | 2 + crates/sync/src/compressed.rs | 2 +- 6 files changed, 199 insertions(+), 113 deletions(-) diff --git a/core/crates/cloud-services/src/sync/ingest.rs b/core/crates/cloud-services/src/sync/ingest.rs index 065ceb964..9592b64ab 100644 --- a/core/crates/cloud-services/src/sync/ingest.rs +++ b/core/crates/cloud-services/src/sync/ingest.rs @@ -1,10 +1,8 @@ use crate::Error; -use sd_core_sync::{from_cloud_crdt_ops, CompressedCRDTOperationsPerModelPerDevice, SyncManager}; +use sd_core_sync::SyncManager; use sd_actors::{Actor, Stopper}; -use sd_prisma::prisma::{cloud_crdt_operation, SortOrder}; -use sd_utils::timestamp_to_datetime; use std::{ future::IntoFuture, @@ -12,7 +10,6 @@ use std::{ atomic::{AtomicBool, Ordering}, Arc, }, - time::SystemTime, }; use futures::FutureExt; @@ -22,8 +19,6 @@ use tracing::{debug, error}; use super::{ReceiveAndIngestNotifiers, SyncActors, ONE_MINUTE}; -const BATCH_SIZE: i64 = 1000; - /// Responsible for taking sync operations received from the cloud, /// and applying them to the local database via the sync system's ingest actor. @@ -43,20 +38,14 @@ impl Actor for Ingester { Stopped, } - 'outer: loop { + loop { self.active.store(true, Ordering::Relaxed); self.active_notify.notify_waiters(); - loop { - match self.run_loop_iteration().await { - Ok(IngestStatus::Completed) => break, - Ok(IngestStatus::InProgress) => {} - Err(e) => { - error!(?e, "Error during cloud sync ingester actor iteration"); - sleep(ONE_MINUTE).await; - continue 'outer; - } - } + if let Err(e) = self.run_loop_iteration().await { + error!(?e, "Error during cloud sync ingester actor iteration"); + sleep(ONE_MINUTE).await; + continue; } self.active.store(false, Ordering::Relaxed); @@ -79,11 +68,6 @@ impl Actor for Ingester { } } -enum IngestStatus { - Completed, - InProgress, -} - impl Ingester { pub const fn new( sync: SyncManager, @@ -99,48 +83,33 @@ impl Ingester { } } - async fn run_loop_iteration(&self) -> Result { - let (ops_ids, ops) = self + async fn run_loop_iteration(&self) -> Result<(), Error> { + let operations_to_ingest_count = self .sync .db .cloud_crdt_operation() - .find_many(vec![]) - .take(BATCH_SIZE) - .order_by(cloud_crdt_operation::timestamp::order(SortOrder::Asc)) - .exec() - .await - .map_err(sd_core_sync::Error::from)? - .into_iter() - .map(from_cloud_crdt_ops) - .collect::, Vec<_>), _>>()?; - - if ops_ids.is_empty() { - return Ok(IngestStatus::Completed); - } - - debug!( - messages_count = ops.len(), - first_message = ?ops - .first() - .map_or_else(|| SystemTime::UNIX_EPOCH.into(), |op| timestamp_to_datetime(op.timestamp)), - last_message = ?ops - .last() - .map_or_else(|| SystemTime::UNIX_EPOCH.into(), |op| timestamp_to_datetime(op.timestamp)), - "Messages to ingest", - ); - - self.sync - .ingest_ops(CompressedCRDTOperationsPerModelPerDevice::new(ops)) - .await?; - - self.sync - .db - .cloud_crdt_operation() - .delete_many(vec![cloud_crdt_operation::id::in_vec(ops_ids)]) + .count(vec![]) .exec() .await .map_err(sd_core_sync::Error::from)?; - Ok(IngestStatus::InProgress) + if operations_to_ingest_count == 0 { + debug!("Nothing to ingest, early finishing ingester loop"); + return Ok(()); + } + + debug!( + operations_to_ingest_count, + "Starting sync messages cloud ingestion loop" + ); + + self.sync.ingest_ops().await?; + + debug!( + operations_to_ingest_count, + "Finished sync messages cloud ingestion loop" + ); + + Ok(()) } } diff --git a/core/crates/sync/src/ingest_utils.rs b/core/crates/sync/src/ingest_utils.rs index 6c77a96b7..e63f317ed 100644 --- a/core/crates/sync/src/ingest_utils.rs +++ b/core/crates/sync/src/ingest_utils.rs @@ -298,7 +298,24 @@ async fn handle_crdt_deletion( record_id: rmpv::Value, delete_op: &CompressedCRDTOperation, ) -> Result<(), Error> { - // deletes are the be all and end all, no need to check anything + // deletes are the be all and end all, except if we never created the object to begin with + // in this case we don't need to delete anything + + if db + .crdt_operation() + .count(vec![ + crdt_operation::model::equals(i32::from(model)), + crdt_operation::record_id::equals(rmp_serde::to_vec(&record_id)?), + ]) + .exec() + .await? + == 0 + { + // This means that in the other device this entry was created and deleted, before this + // device here could even take notice of it. So we don't need to do anything here. + return Ok(()); + } + let op = CRDTOperation { device_pub_id: device_pub_id.into(), model_id: model, diff --git a/core/crates/sync/src/lib.rs b/core/crates/sync/src/lib.rs index 56822509c..5b8d90efe 100644 --- a/core/crates/sync/src/lib.rs +++ b/core/crates/sync/src/lib.rs @@ -35,7 +35,7 @@ use sd_utils::uuid_to_bytes; use std::{collections::HashMap, sync::Arc}; -use tokio::sync::RwLock; +use tokio::{sync::RwLock, task::JoinError}; pub mod backfill; mod db_operation; @@ -77,6 +77,8 @@ pub enum Error { EmptyOperations, #[error("device not found: {0}")] DeviceNotFound(DevicePubId), + #[error("processes crdt task panicked")] + ProcessCrdtPanic(JoinError), } impl From for rspc::Error { diff --git a/core/crates/sync/src/manager.rs b/core/crates/sync/src/manager.rs index 382261b9d..a01b6716b 100644 --- a/core/crates/sync/src/manager.rs +++ b/core/crates/sync/src/manager.rs @@ -1,36 +1,39 @@ use sd_core_prisma_helpers::DevicePubId; use sd_prisma::{ - prisma::{crdt_operation, device, PrismaClient, SortOrder}, + prisma::{cloud_crdt_operation, crdt_operation, device, PrismaClient, SortOrder}, prisma_sync, }; -use sd_sync::{ - CRDTOperation, CompressedCRDTOperationsPerModel, CompressedCRDTOperationsPerModelPerDevice, - ModelId, OperationFactory, -}; +use sd_sync::{CRDTOperation, CompressedCRDTOperation, ModelId, OperationFactory, RecordId}; use sd_utils::timestamp_to_datetime; use std::{ - collections::BTreeMap, + collections::{hash_map::Entry, BTreeMap, HashMap}, fmt, num::NonZeroU128, sync::{ atomic::{self, AtomicBool}, Arc, }, + time::SystemTime, }; use async_stream::stream; use futures::Stream; -use futures_concurrency::future::TryJoin; -use tokio::sync::{broadcast, Mutex, Notify, RwLock}; +use futures_concurrency::future::{Join, TryJoin}; +use tokio::{ + spawn, + sync::{broadcast, Mutex, Notify, RwLock, Semaphore}, +}; use tracing::{debug, warn}; use uhlc::{HLCBuilder, HLC}; use uuid::Uuid; use super::{ - crdt_op_db, db_operation::from_crdt_ops, ingest_utils::process_crdt_operations, Error, - SyncEvent, TimestampPerDevice, NTP64, + crdt_op_db, + db_operation::{from_cloud_crdt_ops, from_crdt_ops}, + ingest_utils::process_crdt_operations, + Error, SyncEvent, TimestampPerDevice, NTP64, }; /// Wrapper that spawns the ingest actor and provides utilities for reading and writing sync operations. @@ -44,7 +47,8 @@ pub struct Manager { pub clock: Arc, pub active: Arc, pub active_notify: Arc, - pub sync_lock: Arc>, + pub(crate) sync_lock: Arc>, + pub(crate) available_parallelism: usize, } impl fmt::Debug for Manager { @@ -131,62 +135,154 @@ impl Manager { active: Arc::default(), active_notify: Arc::default(), sync_lock: Arc::new(Mutex::default()), + available_parallelism: std::thread::available_parallelism() + .map_or(1, std::num::NonZero::get), }, rx, )) } - pub async fn ingest_ops( + async fn fetch_cloud_crdt_ops( &self, - CompressedCRDTOperationsPerModelPerDevice(compressed_ops): CompressedCRDTOperationsPerModelPerDevice, - ) -> Result<(), Error> { - // WARN: this order here exists because sync messages MUST be processed in this exact order - // due to relationship dependencies between these tables. - const INGEST_ORDER: &[ModelId] = &[ - prisma_sync::device::MODEL_ID, - prisma_sync::storage_statistics::MODEL_ID, - prisma_sync::tag::MODEL_ID, - prisma_sync::location::MODEL_ID, - prisma_sync::object::MODEL_ID, - prisma_sync::exif_data::MODEL_ID, - prisma_sync::file_path::MODEL_ID, - prisma_sync::label::MODEL_ID, - prisma_sync::tag_on_object::MODEL_ID, - prisma_sync::label_on_object::MODEL_ID, - ]; + model_id: ModelId, + batch_size: i64, + ) -> Result<(Vec, Vec), Error> { + self.db + .cloud_crdt_operation() + .find_many(vec![cloud_crdt_operation::model::equals(i32::from( + model_id, + ))]) + .take(batch_size) + .order_by(cloud_crdt_operation::timestamp::order(SortOrder::Asc)) + .exec() + .await? + .into_iter() + .map(from_cloud_crdt_ops) + .collect::, Vec<_>), _>>() + } - let _lock_guard = self.sync_lock.lock().await; + async fn ingest_by_model(&self, model_id: ModelId) -> Result<(), Error> { + let (ops_ids, ops) = self.fetch_cloud_crdt_ops(model_id, 10_000).await?; + if ops_ids.is_empty() { + return Ok(()); + } - let mut ops_fut_by_model = INGEST_ORDER - .iter() - .map(|&model_id| (model_id, vec![])) - .collect::>(); + debug!( + messages_count = ops.len(), + first_message = ?ops + .first() + .map_or_else(|| SystemTime::UNIX_EPOCH.into(), |op| timestamp_to_datetime(op.timestamp)), + last_message = ?ops + .last() + .map_or_else(|| SystemTime::UNIX_EPOCH.into(), |op| timestamp_to_datetime(op.timestamp)), + model_id, + "Messages by model to ingest", + ); - for (device_pub_id, CompressedCRDTOperationsPerModel(ops_per_model)) in compressed_ops { - for (model_id, ops_per_record) in ops_per_model { - for (record_id, ops) in ops_per_record { - ops_fut_by_model - .get_mut(&model_id) - .ok_or(Error::InvalidModelId(model_id))? - .push(process_crdt_operations( - &self.clock, - &self.timestamp_per_device, - &self.db, - device_pub_id.into(), - model_id, - record_id, - ops, - )); + let mut compressed_map = + BTreeMap::, (RecordId, Vec)>>::new(); + + for CRDTOperation { + device_pub_id, + timestamp, + model_id: _, // Ignoring model_id as we know it already + record_id, + data, + } in ops + { + let records = compressed_map.entry(device_pub_id).or_default(); + + // Can't use RecordId as a key because rmpv::Value doesn't implement Hash + Eq. + // So we use it's serialized bytes as a key. + let record_id_bytes = + rmp_serde::to_vec_named(&record_id).expect("already serialized to Value"); + + match records.entry(record_id_bytes) { + Entry::Occupied(mut entry) => { + entry + .get_mut() + .1 + .push(CompressedCRDTOperation { timestamp, data }); + } + Entry::Vacant(entry) => { + entry.insert((record_id, vec![CompressedCRDTOperation { timestamp, data }])); } } } - for model_id in INGEST_ORDER { - if let Some(futs) = ops_fut_by_model.remove(model_id) { - futs.try_join().await?; - } + let _lock_guard = self.sync_lock.lock().await; + + let semaphore = &Arc::new(Semaphore::new(self.available_parallelism)); + + let handles = compressed_map + .into_iter() + .flat_map(|(device_pub_id, records)| { + records.into_values().map(move |(record_id, ops)| { + // We can process each record in parallel as they are independent + spawn({ + let clock = Arc::clone(&self.clock); + let timestamp_per_device = Arc::clone(&self.timestamp_per_device); + let db = Arc::clone(&self.db); + let device_pub_id = device_pub_id.into(); + let semaphore = Arc::clone(semaphore); + + async move { + let _permit = + semaphore.acquire().await.expect("semaphore never closes"); + + process_crdt_operations( + &clock, + ×tamp_per_device, + &db, + device_pub_id, + model_id, + record_id, + ops, + ) + .await + } + }) + }) + }) + .collect::>(); + + for res in handles.join().await { + res.map_err(Error::ProcessCrdtPanic)??; } + self.db + .cloud_crdt_operation() + .delete_many(vec![cloud_crdt_operation::id::in_vec(ops_ids)]) + .exec() + .await?; + + Ok(()) + } + + pub async fn ingest_ops(&self) -> Result<(), Error> { + // WARN: this order here exists because sync messages MUST be processed in this exact order + // due to relationship dependencies between these tables. + self.ingest_by_model(prisma_sync::device::MODEL_ID).await?; + + ( + self.ingest_by_model(prisma_sync::storage_statistics::MODEL_ID), + self.ingest_by_model(prisma_sync::tag::MODEL_ID), + self.ingest_by_model(prisma_sync::location::MODEL_ID), + self.ingest_by_model(prisma_sync::object::MODEL_ID), + self.ingest_by_model(prisma_sync::label::MODEL_ID), + ) + .try_join() + .await?; + + ( + self.ingest_by_model(prisma_sync::exif_data::MODEL_ID), + self.ingest_by_model(prisma_sync::file_path::MODEL_ID), + self.ingest_by_model(prisma_sync::tag_on_object::MODEL_ID), + self.ingest_by_model(prisma_sync::label_on_object::MODEL_ID), + ) + .try_join() + .await?; + if self.tx.send(SyncEvent::Ingested).is_err() { warn!("failed to send ingested message on `ingest_ops`"); } diff --git a/core/src/location/mod.rs b/core/src/location/mod.rs index a4a998995..d3baf8532 100644 --- a/core/src/location/mod.rs +++ b/core/src/location/mod.rs @@ -466,6 +466,7 @@ pub async fn scan_location( ) .await? } + ScanState::Indexed => { node.job_system .dispatch( @@ -478,6 +479,7 @@ pub async fn scan_location( ) .await? } + ScanState::FilesIdentified => { node.job_system .dispatch( diff --git a/crates/sync/src/compressed.rs b/crates/sync/src/compressed.rs index 47c38e7fe..a2e3a147d 100644 --- a/crates/sync/src/compressed.rs +++ b/crates/sync/src/compressed.rs @@ -46,7 +46,7 @@ impl CompressedCRDTOperationsPerModelPerDevice { // Can't use RecordId as a key because rmpv::Value doesn't implement Hash + Eq. // So we use it's serialized bytes as a key. let record_id_bytes = - rmp_serde::to_vec(&record_id).expect("already serialized to Value"); + rmp_serde::to_vec_named(&record_id).expect("already serialized to Value"); match records.entry(record_id_bytes) { Entry::Occupied(mut entry) => { From 3eba4bb31e30c3340a66cbddba5dacec36d9d0ab Mon Sep 17 00:00:00 2001 From: Ericson Soares Date: Mon, 21 Oct 2024 23:20:34 -0300 Subject: [PATCH 14/21] bruh --- core/crates/sync/src/ingest_utils.rs | 3 + core/crates/sync/src/manager.rs | 166 ++++++++++++++------------- 2 files changed, 92 insertions(+), 77 deletions(-) diff --git a/core/crates/sync/src/ingest_utils.rs b/core/crates/sync/src/ingest_utils.rs index e63f317ed..297dd3e5c 100644 --- a/core/crates/sync/src/ingest_utils.rs +++ b/core/crates/sync/src/ingest_utils.rs @@ -159,6 +159,7 @@ pub async fn process_crdt_operations( Ok(()) } +#[instrument(skip_all, err)] async fn handle_crdt_updates( db: &PrismaClient, device_pub_id: &DevicePubId, @@ -213,6 +214,7 @@ async fn handle_crdt_updates( .await } +#[instrument(skip_all, err)] async fn handle_crdt_create_and_updates( db: &PrismaClient, device_pub_id: &DevicePubId, @@ -291,6 +293,7 @@ async fn handle_crdt_create_and_updates( .await } +#[instrument(skip_all, err)] async fn handle_crdt_deletion( db: &PrismaClient, device_pub_id: &DevicePubId, diff --git a/core/crates/sync/src/manager.rs b/core/crates/sync/src/manager.rs index a01b6716b..d330852d4 100644 --- a/core/crates/sync/src/manager.rs +++ b/core/crates/sync/src/manager.rs @@ -25,7 +25,7 @@ use tokio::{ spawn, sync::{broadcast, Mutex, Notify, RwLock, Semaphore}, }; -use tracing::{debug, warn}; +use tracing::{debug, instrument, warn}; use uhlc::{HLCBuilder, HLC}; use uuid::Uuid; @@ -161,100 +161,112 @@ impl Manager { .collect::, Vec<_>), _>>() } + #[instrument(skip(self))] async fn ingest_by_model(&self, model_id: ModelId) -> Result<(), Error> { - let (ops_ids, ops) = self.fetch_cloud_crdt_ops(model_id, 10_000).await?; - if ops_ids.is_empty() { - return Ok(()); - } + let mut total_count = 0; - debug!( - messages_count = ops.len(), - first_message = ?ops - .first() - .map_or_else(|| SystemTime::UNIX_EPOCH.into(), |op| timestamp_to_datetime(op.timestamp)), - last_message = ?ops - .last() - .map_or_else(|| SystemTime::UNIX_EPOCH.into(), |op| timestamp_to_datetime(op.timestamp)), - model_id, - "Messages by model to ingest", - ); + loop { + let (ops_ids, ops) = self.fetch_cloud_crdt_ops(model_id, 10_000).await?; + if ops_ids.is_empty() { + break; + } - let mut compressed_map = - BTreeMap::, (RecordId, Vec)>>::new(); + debug!( + messages_count = ops.len(), + first_message = ?ops + .first() + .map_or_else(|| SystemTime::UNIX_EPOCH.into(), |op| timestamp_to_datetime(op.timestamp)), + last_message = ?ops + .last() + .map_or_else(|| SystemTime::UNIX_EPOCH.into(), |op| timestamp_to_datetime(op.timestamp)), + "Messages by model to ingest", + ); - for CRDTOperation { - device_pub_id, - timestamp, - model_id: _, // Ignoring model_id as we know it already - record_id, - data, - } in ops - { - let records = compressed_map.entry(device_pub_id).or_default(); + let mut compressed_map = + BTreeMap::, (RecordId, Vec)>>::new(); - // Can't use RecordId as a key because rmpv::Value doesn't implement Hash + Eq. - // So we use it's serialized bytes as a key. - let record_id_bytes = - rmp_serde::to_vec_named(&record_id).expect("already serialized to Value"); + for CRDTOperation { + device_pub_id, + timestamp, + model_id: _, // Ignoring model_id as we know it already + record_id, + data, + } in ops + { + let records = compressed_map.entry(device_pub_id).or_default(); - match records.entry(record_id_bytes) { - Entry::Occupied(mut entry) => { - entry - .get_mut() - .1 - .push(CompressedCRDTOperation { timestamp, data }); - } - Entry::Vacant(entry) => { - entry.insert((record_id, vec![CompressedCRDTOperation { timestamp, data }])); + // Can't use RecordId as a key because rmpv::Value doesn't implement Hash + Eq. + // So we use it's serialized bytes as a key. + let record_id_bytes = + rmp_serde::to_vec_named(&record_id).expect("already serialized to Value"); + + match records.entry(record_id_bytes) { + Entry::Occupied(mut entry) => { + entry + .get_mut() + .1 + .push(CompressedCRDTOperation { timestamp, data }); + } + Entry::Vacant(entry) => { + entry + .insert((record_id, vec![CompressedCRDTOperation { timestamp, data }])); + } } } - } - let _lock_guard = self.sync_lock.lock().await; + let _lock_guard = self.sync_lock.lock().await; - let semaphore = &Arc::new(Semaphore::new(self.available_parallelism)); + let semaphore = &Arc::new(Semaphore::new(self.available_parallelism)); - let handles = compressed_map - .into_iter() - .flat_map(|(device_pub_id, records)| { - records.into_values().map(move |(record_id, ops)| { - // We can process each record in parallel as they are independent - spawn({ - let clock = Arc::clone(&self.clock); - let timestamp_per_device = Arc::clone(&self.timestamp_per_device); - let db = Arc::clone(&self.db); - let device_pub_id = device_pub_id.into(); - let semaphore = Arc::clone(semaphore); + let handles = compressed_map + .into_iter() + .flat_map(|(device_pub_id, records)| { + records.into_values().map(move |(record_id, ops)| { + // We can process each record in parallel as they are independent + spawn({ + let clock = Arc::clone(&self.clock); + let timestamp_per_device = Arc::clone(&self.timestamp_per_device); + let db = Arc::clone(&self.db); + let device_pub_id = device_pub_id.into(); + let semaphore = Arc::clone(semaphore); - async move { - let _permit = - semaphore.acquire().await.expect("semaphore never closes"); + async move { + let _permit = + semaphore.acquire().await.expect("semaphore never closes"); - process_crdt_operations( - &clock, - ×tamp_per_device, - &db, - device_pub_id, - model_id, - record_id, - ops, - ) - .await - } + let count = ops.len(); + + process_crdt_operations( + &clock, + ×tamp_per_device, + &db, + device_pub_id, + model_id, + record_id, + ops, + ) + .await + .map(|()| count) + } + }) }) }) - }) - .collect::>(); + .collect::>(); - for res in handles.join().await { - res.map_err(Error::ProcessCrdtPanic)??; + for res in handles.join().await { + let count = res.map_err(Error::ProcessCrdtPanic)??; + debug!(count, "Ingested operations of model"); + total_count += count; + } + + self.db + .cloud_crdt_operation() + .delete_many(vec![cloud_crdt_operation::id::in_vec(ops_ids)]) + .exec() + .await?; } - self.db - .cloud_crdt_operation() - .delete_many(vec![cloud_crdt_operation::id::in_vec(ops_ids)]) - .exec() - .await?; + debug!(total_count, "Ingested all operations of this model"); Ok(()) } From 114d0df4bb67ebcf43dd7daf010520f8249a36e8 Mon Sep 17 00:00:00 2001 From: Ericson Soares Date: Tue, 22 Oct 2024 00:33:34 -0300 Subject: [PATCH 15/21] New ingester optimization attempt --- core/crates/sync/src/manager.rs | 93 +++++++++++++++++++++------------ 1 file changed, 59 insertions(+), 34 deletions(-) diff --git a/core/crates/sync/src/manager.rs b/core/crates/sync/src/manager.rs index d330852d4..108ade013 100644 --- a/core/crates/sync/src/manager.rs +++ b/core/crates/sync/src/manager.rs @@ -9,7 +9,7 @@ use sd_utils::timestamp_to_datetime; use std::{ collections::{hash_map::Entry, BTreeMap, HashMap}, - fmt, + fmt, mem, num::NonZeroU128, sync::{ atomic::{self, AtomicBool}, @@ -19,11 +19,11 @@ use std::{ }; use async_stream::stream; -use futures::Stream; -use futures_concurrency::future::{Join, TryJoin}; +use futures::{stream::FuturesUnordered, Stream, TryStreamExt}; +use futures_concurrency::future::TryJoin; use tokio::{ spawn, - sync::{broadcast, Mutex, Notify, RwLock, Semaphore}, + sync::{broadcast, Mutex, Notify, RwLock}, }; use tracing::{debug, instrument, warn}; use uhlc::{HLCBuilder, HLC}; @@ -36,6 +36,8 @@ use super::{ Error, SyncEvent, TimestampPerDevice, NTP64, }; +const INGESTION_BATCH_SIZE: i64 = 10_000; + /// Wrapper that spawns the ingest actor and provides utilities for reading and writing sync operations. #[derive(Clone)] pub struct Manager { @@ -165,14 +167,22 @@ impl Manager { async fn ingest_by_model(&self, model_id: ModelId) -> Result<(), Error> { let mut total_count = 0; + let mut buckets = (0..self.available_parallelism) + .map(|_| FuturesUnordered::new()) + .collect::>(); + loop { - let (ops_ids, ops) = self.fetch_cloud_crdt_ops(model_id, 10_000).await?; + let (ops_ids, ops) = self + .fetch_cloud_crdt_ops(model_id, INGESTION_BATCH_SIZE) + .await?; if ops_ids.is_empty() { break; } + let messages_count = ops.len(); + debug!( - messages_count = ops.len(), + messages_count, first_message = ?ops .first() .map_or_else(|| SystemTime::UNIX_EPOCH.into(), |op| timestamp_to_datetime(op.timestamp)), @@ -216,45 +226,60 @@ impl Manager { let _lock_guard = self.sync_lock.lock().await; - let semaphore = &Arc::new(Semaphore::new(self.available_parallelism)); - - let handles = compressed_map + compressed_map .into_iter() .flat_map(|(device_pub_id, records)| { records.into_values().map(move |(record_id, ops)| { // We can process each record in parallel as they are independent - spawn({ - let clock = Arc::clone(&self.clock); - let timestamp_per_device = Arc::clone(&self.timestamp_per_device); - let db = Arc::clone(&self.db); - let device_pub_id = device_pub_id.into(); - let semaphore = Arc::clone(semaphore); - async move { - let _permit = - semaphore.acquire().await.expect("semaphore never closes"); + let clock = Arc::clone(&self.clock); + let timestamp_per_device = Arc::clone(&self.timestamp_per_device); + let db = Arc::clone(&self.db); + let device_pub_id = device_pub_id.into(); - let count = ops.len(); + async move { + let count = ops.len(); - process_crdt_operations( - &clock, - ×tamp_per_device, - &db, - device_pub_id, - model_id, - record_id, - ops, - ) - .await - .map(|()| count) - } - }) + process_crdt_operations( + &clock, + ×tamp_per_device, + &db, + device_pub_id, + model_id, + record_id, + ops, + ) + .await + .map(|()| count) + } + }) + }) + .enumerate() + .for_each(|(idx, fut)| buckets[idx % self.available_parallelism].push(fut)); + + let handles = buckets + .iter_mut() + .enumerate() + .filter(|(_idx, bucket)| !bucket.is_empty()) + .map(|(idx, bucket)| { + let mut bucket = mem::take(bucket); + + spawn(async move { + let mut ops_count = 0; + while let Some(count) = bucket.try_next().await? { + ops_count += count; + } + + Ok::<_, Error>((ops_count, idx, bucket)) }) }) .collect::>(); - for res in handles.join().await { - let count = res.map_err(Error::ProcessCrdtPanic)??; + for res in handles.try_join().await.map_err(Error::ProcessCrdtPanic)? { + let (count, idx, bucket) = res?; + + buckets[idx] = bucket; + debug!(count, "Ingested operations of model"); total_count += count; } From 2313a7f2e3a0a9c0ebc71ade9c0cc5ec4118ac50 Mon Sep 17 00:00:00 2001 From: Ericson Soares Date: Tue, 22 Oct 2024 00:51:49 -0300 Subject: [PATCH 16/21] Small bootstrap fix --- core/src/api/cloud/mod.rs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/core/src/api/cloud/mod.rs b/core/src/api/cloud/mod.rs index dd2bde419..611fb0c31 100644 --- a/core/src/api/cloud/mod.rs +++ b/core/src/api/cloud/mod.rs @@ -215,7 +215,15 @@ pub(crate) fn mount() -> AlphaRouter { }| { let node = &node; - async move { initialize_cloud_sync(pub_id, library, node).await } + async move { + match initialize_cloud_sync(pub_id, library, node).await { + // If we don't have this library locally, we didn't joined this group yet + Ok(()) | Err(LibraryManagerError::LibraryNotFound) => { + Ok(()) + } + Err(e) => Err(e), + } + } }, ) .collect::>() From 86e4d21d1c5777751cc7c813cfb81034b981138a Mon Sep 17 00:00:00 2001 From: Ericson Soares Date: Tue, 22 Oct 2024 01:39:44 -0300 Subject: [PATCH 17/21] More optimization attempts and collecting metrics --- core/crates/sync/src/ingest_utils.rs | 40 +++++++++++++++++++---- core/crates/sync/src/manager.rs | 49 ++++++++++++++++++++++++---- 2 files changed, 75 insertions(+), 14 deletions(-) diff --git a/core/crates/sync/src/ingest_utils.rs b/core/crates/sync/src/ingest_utils.rs index 297dd3e5c..8fa2a89b0 100644 --- a/core/crates/sync/src/ingest_utils.rs +++ b/core/crates/sync/src/ingest_utils.rs @@ -8,9 +8,10 @@ use sd_sync::{ CRDTOperation, CRDTOperationData, CompressedCRDTOperation, ModelId, OperationKind, RecordId, }; -use std::{collections::BTreeMap, num::NonZeroU128}; +use std::{collections::BTreeMap, num::NonZeroU128, sync::Arc}; use futures_concurrency::future::TryJoin; +use tokio::sync::Mutex; use tracing::{debug, instrument, trace, warn}; use uhlc::{Timestamp, HLC, NTP64}; use uuid::Uuid; @@ -24,11 +25,11 @@ crdt_operation::select!(crdt_operation_id { id }); pub async fn process_crdt_operations( clock: &HLC, timestamp_per_device: &TimestampPerDevice, + sync_lock: Arc>, db: &PrismaClient, device_pub_id: DevicePubId, model_id: ModelId, - record_id: RecordId, - mut ops: Vec, + (record_id, mut ops): (RecordId, Vec), ) -> Result<(), Error> { ops.sort_by_key(|op| op.timestamp); @@ -52,7 +53,15 @@ pub async fn process_crdt_operations( .find(|op| matches!(op.data, CRDTOperationData::Delete)) { trace!("Deleting operation"); - handle_crdt_deletion(db, &device_pub_id, model_id, record_id, delete_op).await?; + handle_crdt_deletion( + db, + &sync_lock, + &device_pub_id, + model_id, + record_id, + delete_op, + ) + .await?; } // Create + > 0 Update - overwrites the create's data with the updates else if let Some(timestamp) = ops @@ -78,8 +87,16 @@ pub async fn process_crdt_operations( return Ok(()); } - handle_crdt_create_and_updates(db, &device_pub_id, model_id, record_id, ops, timestamp) - .await?; + handle_crdt_create_and_updates( + db, + &sync_lock, + &device_pub_id, + model_id, + record_id, + ops, + timestamp, + ) + .await?; } // > 0 Update - batches updates with a fake Create op else { @@ -138,7 +155,7 @@ pub async fn process_crdt_operations( } } - handle_crdt_updates(db, &device_pub_id, model_id, record_id, data).await?; + handle_crdt_updates(db, &sync_lock, &device_pub_id, model_id, record_id, data).await?; } // read the timestamp for the operation's device, or insert one if it doesn't exist @@ -162,6 +179,7 @@ pub async fn process_crdt_operations( #[instrument(skip_all, err)] async fn handle_crdt_updates( db: &PrismaClient, + sync_lock: &Mutex<()>, device_pub_id: &DevicePubId, model_id: ModelId, record_id: rmpv::Value, @@ -169,6 +187,8 @@ async fn handle_crdt_updates( ) -> Result<(), Error> { let device_pub_id = sd_sync::DevicePubId::from(device_pub_id); + let _lock_guard = sync_lock.lock().await; + db._transaction() .with_timeout(30 * 10000) .with_max_wait(30 * 10000) @@ -217,6 +237,7 @@ async fn handle_crdt_updates( #[instrument(skip_all, err)] async fn handle_crdt_create_and_updates( db: &PrismaClient, + sync_lock: &Mutex<()>, device_pub_id: &DevicePubId, model_id: ModelId, record_id: rmpv::Value, @@ -251,6 +272,8 @@ async fn handle_crdt_create_and_updates( } } + let _lock_guard = sync_lock.lock().await; + db._transaction() .with_timeout(30 * 10000) .with_max_wait(30 * 10000) @@ -296,6 +319,7 @@ async fn handle_crdt_create_and_updates( #[instrument(skip_all, err)] async fn handle_crdt_deletion( db: &PrismaClient, + sync_lock: &Mutex<()>, device_pub_id: &DevicePubId, model: u16, record_id: rmpv::Value, @@ -327,6 +351,8 @@ async fn handle_crdt_deletion( data: CRDTOperationData::Delete, }; + let _lock_guard = sync_lock.lock().await; + db._transaction() .with_timeout(30 * 10000) .with_max_wait(30 * 10000) diff --git a/core/crates/sync/src/manager.rs b/core/crates/sync/src/manager.rs index 108ade013..c70e31ba5 100644 --- a/core/crates/sync/src/manager.rs +++ b/core/crates/sync/src/manager.rs @@ -15,7 +15,7 @@ use std::{ atomic::{self, AtomicBool}, Arc, }, - time::SystemTime, + time::{Duration, SystemTime}, }; use async_stream::stream; @@ -24,6 +24,7 @@ use futures_concurrency::future::TryJoin; use tokio::{ spawn, sync::{broadcast, Mutex, Notify, RwLock}, + time::Instant, }; use tracing::{debug, instrument, warn}; use uhlc::{HLCBuilder, HLC}; @@ -171,7 +172,14 @@ impl Manager { .map(|_| FuturesUnordered::new()) .collect::>(); + let mut total_fetch_time = Duration::ZERO; + let mut total_compression_time = Duration::ZERO; + let mut total_work_distribution_time = Duration::ZERO; + let mut total_process_time = Duration::ZERO; + loop { + let fetching_start = Instant::now(); + let (ops_ids, ops) = self .fetch_cloud_crdt_ops(model_id, INGESTION_BATCH_SIZE) .await?; @@ -179,6 +187,8 @@ impl Manager { break; } + total_fetch_time += fetching_start.elapsed(); + let messages_count = ops.len(); debug!( @@ -192,6 +202,8 @@ impl Manager { "Messages by model to ingest", ); + let compression_start = Instant::now(); + let mut compressed_map = BTreeMap::, (RecordId, Vec)>>::new(); @@ -224,7 +236,9 @@ impl Manager { } } - let _lock_guard = self.sync_lock.lock().await; + total_compression_time += compression_start.elapsed(); + + let work_distribution_start = Instant::now(); compressed_map .into_iter() @@ -236,6 +250,7 @@ impl Manager { let timestamp_per_device = Arc::clone(&self.timestamp_per_device); let db = Arc::clone(&self.db); let device_pub_id = device_pub_id.into(); + let sync_lock = Arc::clone(&self.sync_lock); async move { let count = ops.len(); @@ -243,11 +258,11 @@ impl Manager { process_crdt_operations( &clock, ×tamp_per_device, + sync_lock, &db, device_pub_id, model_id, - record_id, - ops, + (record_id, ops), ) .await .map(|()| count) @@ -257,6 +272,10 @@ impl Manager { .enumerate() .for_each(|(idx, fut)| buckets[idx % self.available_parallelism].push(fut)); + total_work_distribution_time += work_distribution_start.elapsed(); + + let processing_start = Instant::now(); + let handles = buckets .iter_mut() .enumerate() @@ -266,21 +285,30 @@ impl Manager { spawn(async move { let mut ops_count = 0; + let processing_start = Instant::now(); while let Some(count) = bucket.try_next().await? { ops_count += count; } + debug!( + "Ingested {ops_count} operations in {:?}", + processing_start.elapsed() + ); + Ok::<_, Error>((ops_count, idx, bucket)) }) }) .collect::>(); - for res in handles.try_join().await.map_err(Error::ProcessCrdtPanic)? { + let results = handles.try_join().await.map_err(Error::ProcessCrdtPanic)?; + + total_process_time += processing_start.elapsed(); + + for res in results { let (count, idx, bucket) = res?; buckets[idx] = bucket; - debug!(count, "Ingested operations of model"); total_count += count; } @@ -291,7 +319,14 @@ impl Manager { .await?; } - debug!(total_count, "Ingested all operations of this model"); + debug!( + total_count, + ?total_fetch_time, + ?total_compression_time, + ?total_work_distribution_time, + ?total_process_time, + "Ingested all operations of this model" + ); Ok(()) } From b8d9a44da72bdadc6b67eaf5fb01f8c2ce257de1 Mon Sep 17 00:00:00 2001 From: Ericson Soares Date: Tue, 22 Oct 2024 12:52:23 -0300 Subject: [PATCH 18/21] Introducing partial indexes on crdt_operations and more optimization on ingestion --- core/crates/sync/src/ingest_utils.rs | 70 +++++++++++++++++++--------- core/src/library/manager/mod.rs | 66 ++++++++++++++++++++++++-- 2 files changed, 110 insertions(+), 26 deletions(-) diff --git a/core/crates/sync/src/ingest_utils.rs b/core/crates/sync/src/ingest_utils.rs index 8fa2a89b0..7cf7606a2 100644 --- a/core/crates/sync/src/ingest_utils.rs +++ b/core/crates/sync/src/ingest_utils.rs @@ -114,30 +114,40 @@ pub async fn process_crdt_operations( } } + let earlier_time = data.values().fold( + NTP64(u64::from(u32::MAX)), + |earlier_time, (_, timestamp)| { + if timestamp.0 < earlier_time.0 { + *timestamp + } else { + earlier_time + } + }, + ); + // conflict resolution - let (create, newer_updates_count) = db + let (create, possible_newer_updates_count) = db ._batch(( db.crdt_operation().count(vec![ crdt_operation::model::equals(i32::from(model_id)), crdt_operation::record_id::equals(rmp_serde::to_vec(&record_id)?), crdt_operation::kind::equals(OperationKind::Create.to_string()), ]), - data.iter() - .map(|(k, (_, timestamp))| { - Ok(db.crdt_operation().count(vec![ - crdt_operation::timestamp::gt({ - #[allow(clippy::cast_possible_wrap)] - // SAFETY: we had to store using i64 due to SQLite limitations - { - timestamp.as_u64() as i64 - } - }), - crdt_operation::model::equals(i32::from(model_id)), - crdt_operation::record_id::equals(rmp_serde::to_vec(&record_id)?), - crdt_operation::kind::contains(format!(":{k}:")), - ])) - }) - .collect::, Error>>()?, + // Fetching all update operations newer than our current earlier timestamp + db.crdt_operation() + .find_many(vec![ + crdt_operation::timestamp::gt({ + #[allow(clippy::cast_possible_wrap)] + // SAFETY: we had to store using i64 due to SQLite limitations + { + earlier_time.as_u64() as i64 + } + }), + crdt_operation::model::equals(i32::from(model_id)), + crdt_operation::record_id::equals(rmp_serde::to_vec(&record_id)?), + crdt_operation::kind::starts_with("u".to_string()), + ]) + .select(crdt_operation::select!({ kind timestamp })), )) .await?; @@ -146,12 +156,28 @@ pub async fn process_crdt_operations( return Ok(()); } - let keys = data.keys().cloned().collect::>(); + for candidate in possible_newer_updates_count { + // The first element is "u" meaning that this is an update, so we skip it + for key in candidate + .kind + .split(':') + .filter(|field| !field.is_empty()) + .skip(1) + { + // remove entries if we possess locally more recent updates for this field + if data.get(key).is_some_and(|(_, new_timestamp)| { + #[allow(clippy::cast_sign_loss)] + { + // we need to store as i64 due to SQLite limitations + *new_timestamp < NTP64(candidate.timestamp as u64) + } + }) { + data.remove(key); + } + } - // remove entries if we possess locally more recent updates for this field - for (update, key) in newer_updates_count.into_iter().zip(keys) { - if update > 0 { - data.remove(&key); + if data.is_empty() { + break; } } diff --git a/core/src/library/manager/mod.rs b/core/src/library/manager/mod.rs index 06aa0b8c9..2cd6652c3 100644 --- a/core/src/library/manager/mod.rs +++ b/core/src/library/manager/mod.rs @@ -9,8 +9,13 @@ use crate::{ }; use sd_core_sync::{SyncEvent, SyncManager}; + use sd_p2p::{Identity, RemoteIdentity}; -use sd_prisma::prisma::{device, instance, location}; +use sd_prisma::{ + prisma::{self, device, instance, location, PrismaClient}, + prisma_sync, +}; +use sd_sync::ModelId; use sd_utils::{ db, error::{FileIOError, NonUtf8PathError}, @@ -28,6 +33,7 @@ use std::{ use chrono::Utc; use futures_concurrency::future::{Join, TryJoin}; +use prisma_client_rust::Raw; use tokio::{ fs, io, spawn, sync::{broadcast, RwLock}, @@ -458,6 +464,10 @@ impl Libraries { ); let db = Arc::new(db::load_and_migrate(&db_url).await?); + // Configure database + configure_pragmas(&db).await?; + special_sync_indexes(&db).await?; + if let Some(create) = maybe_create_device { create.to_query(&db).exec().await?; } @@ -552,9 +562,6 @@ impl Libraries { ) .await?; - // Configure database - configure_pragmas(&db).await?; - let library = Library::new(id, config, instance_id, identity, db, node, sync).await; // This is an exception. Generally subscribe to this by `self.tx.subscribe`. @@ -635,3 +642,54 @@ async fn sync_rx_actor( } } } + +async fn special_sync_indexes(db: &PrismaClient) -> Result<(), LibraryManagerError> { + async fn create_index( + db: &PrismaClient, + model_id: ModelId, + model_name: &str, + ) -> Result<(), LibraryManagerError> { + db._execute_raw(Raw::new( + &format!( + "CREATE INDEX IF NOT EXISTS partial_index_model_{model_name} \ + ON crdt_operation(model,record_id,kind,timestamp) \ + WHERE model = {model_id} + " + ), + vec![], + )) + .exec() + .await?; + + debug!(model_name, "Created sync partial index"); + + Ok(()) + } + + for (model_id, model_name) in [ + (prisma_sync::device::MODEL_ID, prisma::device::NAME), + ( + prisma_sync::storage_statistics::MODEL_ID, + prisma::storage_statistics::NAME, + ), + (prisma_sync::tag::MODEL_ID, prisma::tag::NAME), + (prisma_sync::location::MODEL_ID, prisma::location::NAME), + (prisma_sync::object::MODEL_ID, prisma::object::NAME), + (prisma_sync::label::MODEL_ID, prisma::label::NAME), + (prisma_sync::exif_data::MODEL_ID, prisma::exif_data::NAME), + (prisma_sync::file_path::MODEL_ID, prisma::file_path::NAME), + ( + prisma_sync::tag_on_object::MODEL_ID, + prisma::tag_on_object::NAME, + ), + ( + prisma_sync::label_on_object::MODEL_ID, + prisma::label_on_object::NAME, + ), + ] { + // Creating indexes sequentially just in case + create_index(db, model_id, model_name).await?; + } + + Ok(()) +} From 3190a7b822497b0545140e5464db2f69b8b5440c Mon Sep 17 00:00:00 2001 From: Ericson Soares Date: Tue, 22 Oct 2024 13:26:55 -0300 Subject: [PATCH 19/21] Separate timestamp keepers per sync group --- core/crates/cloud-services/src/error.rs | 2 ++ core/crates/cloud-services/src/sync/receive.rs | 16 ++++++++++++---- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/core/crates/cloud-services/src/error.rs b/core/crates/cloud-services/src/error.rs index 5fb4691be..f90ee028e 100644 --- a/core/crates/cloud-services/src/error.rs +++ b/core/crates/cloud-services/src/error.rs @@ -123,6 +123,8 @@ pub enum Error { EndUpdatePushSyncMessages(io::Error), #[error("Unexpected end of stream while encrypting sync messages")] UnexpectedEndOfStream, + #[error("Failed to create directory to store timestamp keeper files")] + FailedToCreateTimestampKeepersDirectory(io::Error), #[error("Failed to read last timestamp keeper for pulling sync messages: {0}")] FailedToReadLastTimestampKeeper(io::Error), #[error("Failed to handle last timestamp keeper serialization: {0}")] diff --git a/core/crates/cloud-services/src/sync/receive.rs b/core/crates/cloud-services/src/sync/receive.rs index af915c9a9..f4db4b4c5 100644 --- a/core/crates/cloud-services/src/sync/receive.rs +++ b/core/crates/cloud-services/src/sync/receive.rs @@ -40,7 +40,7 @@ use uuid::Uuid; use super::{ReceiveAndIngestNotifiers, SyncActors, ONE_MINUTE}; -const CLOUD_SYNC_DATA_KEEPER_FILE: &str = "cloud_sync_data_keeper.bin"; +const CLOUD_SYNC_DATA_KEEPER_DIRECTORY: &str = "cloud_sync_data_keeper"; /// Responsible for downloading sync operations from the cloud to be processed by the ingester @@ -111,7 +111,7 @@ impl Receiver { active_notify: Arc, ) -> Result { let (keeper, cloud_client, key_manager) = ( - LastTimestampKeeper::load(data_dir.as_ref()), + LastTimestampKeeper::load(data_dir.as_ref(), sync_group_pub_id), cloud_services.client(), cloud_services.key_manager(), ) @@ -317,8 +317,16 @@ struct LastTimestampKeeper { } impl LastTimestampKeeper { - async fn load(data_dir: &Path) -> Result { - let file_path = data_dir.join(CLOUD_SYNC_DATA_KEEPER_FILE).into_boxed_path(); + async fn load(data_dir: &Path, sync_group_pub_id: groups::PubId) -> Result { + let cloud_sync_data_directory = data_dir.join(CLOUD_SYNC_DATA_KEEPER_DIRECTORY); + + fs::create_dir_all(&cloud_sync_data_directory) + .await + .map_err(Error::FailedToCreateTimestampKeepersDirectory)?; + + let file_path = cloud_sync_data_directory + .join(format!("{sync_group_pub_id}.bin")) + .into_boxed_path(); match fs::read(&file_path).await { Ok(bytes) => Ok(Self { From 9913b62ea267e8dc20902688c4f9aa3cf0f83adf Mon Sep 17 00:00:00 2001 From: Ericson Soares Date: Wed, 23 Oct 2024 03:01:04 -0300 Subject: [PATCH 20/21] Next attempt at ingestion optimization --- Cargo.lock | Bin 331411 -> 331432 bytes core/crates/sync/Cargo.toml | 1 + core/crates/sync/src/ingest_utils.rs | 170 +++++++++++++++++++++++---- core/crates/sync/src/manager.rs | 118 ++++++++++++++++++- 4 files changed, 262 insertions(+), 27 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 082cbc3c36e6e6886d58a9c73488ea16165bf231..5395ee416cca6d75bc05cc2f78838f10b66a7968 100644 GIT binary patch delta 39 vcmbO{S7gOpk%kt=7N#xC-)koy@fM$+F@f1)dQ}lKU%P!B^LG0>mXHYmJ_!#F delta 40 wcmZ26S7h>Bk%kt=7N#xC-)pD0, + sync_lock: Arc>, +) -> Result<(), Error> { + let latest_timestamp = ops.iter().fold(NTP64(0), |latest, (_, op)| { + if latest < op.timestamp { + op.timestamp + } else { + latest + } + }); + + update_clock(clock, latest_timestamp, &device_pub_id); + + let ops = ops + .into_iter() + .map(|(record_id, op)| { + rmp_serde::to_vec(&record_id) + .map(|serialized_record_id| (record_id, serialized_record_id, op)) + }) + .collect::, _>>()?; + + // conflict resolution + let delete_counts = db + ._batch( + ops.iter() + .map(|(_, serialized_record_id, _)| { + db.crdt_operation().count(vec![ + crdt_operation::model::equals(i32::from(model_id)), + crdt_operation::record_id::equals(serialized_record_id.clone()), + crdt_operation::kind::equals(OperationKind::Delete.to_string()), + ]) + }) + .collect::>(), + ) + .await?; + + let lock_guard = sync_lock.lock().await; + + db._transaction() + .with_timeout(30 * 10000) + .with_max_wait(30 * 10000) + .run(|db| { + let device_pub_id = device_pub_id.clone(); + + async move { + // complying with borrowck + let device_pub_id = &device_pub_id; + + let (crdt_creates, model_sync_data) = ops + .into_iter() + .zip(delete_counts) + .filter_map(|(data, delete_count)| (delete_count == 0).then_some(data)) + .map( + |( + record_id, + serialized_record_id, + CompressedCRDTOperation { timestamp, data }, + )| { + let crdt_create = crdt_operation::CreateUnchecked { + timestamp: { + #[allow(clippy::cast_possible_wrap)] + // SAFETY: we have to store using i64 due to SQLite limitations + { + timestamp.0 as i64 + } + }, + model: i32::from(model_id), + record_id: serialized_record_id, + kind: "c".to_string(), + data: rmp_serde::to_vec(&data)?, + device_pub_id: device_pub_id.to_db(), + _params: vec![], + }; + + // NOTE(@fogodev): I wish I could do a create many here instead of creating separately each + // entry, but it's not supported by PCR + let model_sync_data = ModelSyncData::from_op(CRDTOperation { + device_pub_id: Uuid::from(device_pub_id), + model_id, + record_id, + timestamp, + data, + })? + .exec(&db); + + Ok::<_, Error>((crdt_create, model_sync_data)) + }, + ) + .collect::, _>>()? + .into_iter() + .unzip::<_, _, Vec<_>, Vec<_>>(); + + model_sync_data.try_join().await?; + + db.crdt_operation().create_many(crdt_creates).exec().await?; + + Ok::<_, Error>(()) + } + }) + .await?; + + drop(lock_guard); + + update_timestamp_per_device(timestamp_per_device, device_pub_id, latest_timestamp).await; Ok(()) } @@ -389,3 +481,37 @@ async fn handle_crdt_deletion( }) .await } + +fn update_clock(clock: &HLC, latest_timestamp: NTP64, device_pub_id: &DevicePubId) { + // first, we update the HLC's timestamp with the incoming one. + // this involves a drift check + sets the last time of the clock + clock + .update_with_timestamp(&Timestamp::new( + latest_timestamp, + uhlc::ID::from( + NonZeroU128::new(Uuid::from(device_pub_id).to_u128_le()).expect("Non zero id"), + ), + )) + .expect("timestamp has too much drift!"); +} + +async fn update_timestamp_per_device( + timestamp_per_device: &TimestampPerDevice, + device_pub_id: DevicePubId, + latest_timestamp: NTP64, +) { + // read the timestamp for the operation's device, or insert one if it doesn't exist + let current_last_timestamp = timestamp_per_device + .read() + .await + .get(&device_pub_id) + .copied(); + + // update the stored timestamp for this device - will be derived from the crdt operations table on restart + let new_ts = NTP64::max(current_last_timestamp.unwrap_or_default(), latest_timestamp); + + timestamp_per_device + .write() + .await + .insert(device_pub_id, new_ts); +} diff --git a/core/crates/sync/src/manager.rs b/core/crates/sync/src/manager.rs index c70e31ba5..e4d830bcc 100644 --- a/core/crates/sync/src/manager.rs +++ b/core/crates/sync/src/manager.rs @@ -4,7 +4,9 @@ use sd_prisma::{ prisma::{cloud_crdt_operation, crdt_operation, device, PrismaClient, SortOrder}, prisma_sync, }; -use sd_sync::{CRDTOperation, CompressedCRDTOperation, ModelId, OperationFactory, RecordId}; +use sd_sync::{ + CRDTOperation, CRDTOperationData, CompressedCRDTOperation, ModelId, OperationFactory, RecordId, +}; use sd_utils::timestamp_to_datetime; use std::{ @@ -21,6 +23,7 @@ use std::{ use async_stream::stream; use futures::{stream::FuturesUnordered, Stream, TryStreamExt}; use futures_concurrency::future::TryJoin; +use itertools::Itertools; use tokio::{ spawn, sync::{broadcast, Mutex, Notify, RwLock}, @@ -33,7 +36,7 @@ use uuid::Uuid; use super::{ crdt_op_db, db_operation::{from_cloud_crdt_ops, from_crdt_ops}, - ingest_utils::process_crdt_operations, + ingest_utils::{bulk_ingest_create_only_ops, process_crdt_operations}, Error, SyncEvent, TimestampPerDevice, NTP64, }; @@ -236,6 +239,32 @@ impl Manager { } } + // Now that we separated all operations by their record_ids, we can do an optimization + // to process all records that only posses a single create operation, batching them together + let mut create_only_ops: BTreeMap> = + BTreeMap::new(); + for (device_pub_id, records) in &mut compressed_map { + for (record_id, ops) in records.values_mut() { + if ops.len() == 1 && matches!(ops[0].data, CRDTOperationData::Create(_)) { + create_only_ops + .entry(*device_pub_id) + .or_default() + .push((mem::replace(record_id, rmpv::Value::Nil), ops.remove(0))); + } + } + } + + bulk_process_of_create_only_ops( + self.available_parallelism, + Arc::clone(&self.clock), + Arc::clone(&self.timestamp_per_device), + Arc::clone(&self.db), + Arc::clone(&self.sync_lock), + model_id, + create_only_ops, + ) + .await?; + total_compression_time += compression_start.elapsed(); let work_distribution_start = Instant::now(); @@ -243,7 +272,11 @@ impl Manager { compressed_map .into_iter() .flat_map(|(device_pub_id, records)| { - records.into_values().map(move |(record_id, ops)| { + records.into_values().filter_map(move |(record_id, ops)| { + if record_id.is_nil() { + return None; + } + // We can process each record in parallel as they are independent let clock = Arc::clone(&self.clock); @@ -252,7 +285,7 @@ impl Manager { let device_pub_id = device_pub_id.into(); let sync_lock = Arc::clone(&self.sync_lock); - async move { + Some(async move { let count = ops.len(); process_crdt_operations( @@ -266,7 +299,7 @@ impl Manager { ) .await .map(|()| count) - } + }) }) }) .enumerate() @@ -618,6 +651,81 @@ impl Manager { // } } +async fn bulk_process_of_create_only_ops( + available_parallelism: usize, + clock: Arc, + timestamp_per_device: TimestampPerDevice, + db: Arc, + sync_lock: Arc>, + model_id: ModelId, + create_only_ops: BTreeMap>, +) -> Result { + let buckets = (0..available_parallelism) + .map(|_| FuturesUnordered::new()) + .collect::>(); + + let mut bucket_idx = 0; + + for (device_pub_id, records) in create_only_ops { + records + .into_iter() + .chunks(100) + .into_iter() + .for_each(|chunk| { + let ops = chunk.collect::>(); + + buckets[bucket_idx % available_parallelism].push({ + let clock = Arc::clone(&clock); + let timestamp_per_device = Arc::clone(×tamp_per_device); + let db = Arc::clone(&db); + let device_pub_id = device_pub_id.into(); + let sync_lock = Arc::clone(&sync_lock); + + async move { + let count = ops.len(); + bulk_ingest_create_only_ops( + &clock, + ×tamp_per_device, + &db, + device_pub_id, + model_id, + ops, + sync_lock, + ) + .await + .map(|()| count) + } + }); + + bucket_idx += 1; + }); + } + + let handles = buckets + .into_iter() + .map(|mut bucket| { + spawn(async move { + let mut total_count = 0; + + while let Some(count) = bucket.try_next().await? { + total_count += count; + } + + Ok::<_, Error>(total_count) + }) + }) + .collect::>(); + + Ok(handles + .try_join() + .await + .map_err(Error::ProcessCrdtPanic)? + .into_iter() + .collect::, _>>()? + .into_iter() + .sum()) +} + impl OperationFactory for Manager { fn get_clock(&self) -> &HLC { &self.clock From e4e6a116cca38f36fba3514349059f8418fd5030 Mon Sep 17 00:00:00 2001 From: Ericson Soares Date: Wed, 23 Oct 2024 13:14:18 -0300 Subject: [PATCH 21/21] More ingestion metrics --- core/crates/cloud-services/src/p2p/mod.rs | 39 +++++++++++++----- core/crates/cloud-services/src/sync/ingest.rs | 14 +++++-- core/crates/sync/src/manager.rs | 41 ++++++++++++------- 3 files changed, 65 insertions(+), 29 deletions(-) diff --git a/core/crates/cloud-services/src/p2p/mod.rs b/core/crates/cloud-services/src/p2p/mod.rs index 316b6f028..0f31f977c 100644 --- a/core/crates/cloud-services/src/p2p/mod.rs +++ b/core/crates/cloud-services/src/p2p/mod.rs @@ -9,20 +9,20 @@ use sd_cloud_schema::{ }; use sd_crypto::{CryptoRng, SeedableRng}; -use std::sync::Arc; +use std::{sync::Arc, time::Duration}; use iroh_net::{ discovery::{ dns::DnsDiscovery, local_swarm_discovery::LocalSwarmDiscovery, pkarr::dht::DhtDiscovery, - ConcurrentDiscovery, + ConcurrentDiscovery, Discovery, }, relay::{RelayMap, RelayMode, RelayUrl}, Endpoint, NodeId, }; use reqwest::Url; use serde::{Deserialize, Serialize}; -use tokio::{spawn, sync::oneshot}; -use tracing::error; +use tokio::{spawn, sync::oneshot, time::sleep}; +use tracing::{debug, error, warn}; mod new_sync_messages_notifier; mod runner; @@ -110,6 +110,12 @@ impl CloudP2P { dns_pkarr_url: Url, relay_url: RelayUrl, ) -> Result { + let dht_discovery = DhtDiscovery::builder() + .secret_key(iroh_secret_key.clone()) + .pkarr_relay(dns_pkarr_url) + .build() + .map_err(Error::DhtDiscoveryInit)?; + let endpoint = Endpoint::builder() .alpns(vec![CloudP2PALPN::LATEST.to_vec()]) .discovery(Box::new(ConcurrentDiscovery::from_services(vec![ @@ -118,13 +124,7 @@ impl CloudP2P { LocalSwarmDiscovery::new(iroh_secret_key.public()) .map_err(Error::LocalSwarmDiscoveryInit)?, ), - Box::new( - DhtDiscovery::builder() - .secret_key(iroh_secret_key.clone()) - .pkarr_relay(dns_pkarr_url) - .build() - .map_err(Error::DhtDiscoveryInit)?, - ), + Box::new(dht_discovery.clone()), ]))) .secret_key(iroh_secret_key) .relay_mode(RelayMode::Custom(RelayMap::from_url(relay_url))) @@ -132,6 +132,23 @@ impl CloudP2P { .await .map_err(Error::CreateCloudP2PEndpoint)?; + spawn({ + let endpoint = endpoint.clone(); + async move { + loop { + let Ok(node_addr) = endpoint.node_addr().await.map_err(|e| { + warn!(?e, "Failed to get direct addresses to force publish on DHT"); + }) else { + sleep(Duration::from_secs(5)).await; + continue; + }; + + debug!("Force publishing peer on DHT"); + return dht_discovery.publish(&node_addr.info); + } + } + }); + let (msgs_tx, msgs_rx) = flume::bounded(16); spawn({ diff --git a/core/crates/cloud-services/src/sync/ingest.rs b/core/crates/cloud-services/src/sync/ingest.rs index 9592b64ab..a7dd65af3 100644 --- a/core/crates/cloud-services/src/sync/ingest.rs +++ b/core/crates/cloud-services/src/sync/ingest.rs @@ -14,7 +14,10 @@ use std::{ use futures::FutureExt; use futures_concurrency::future::Race; -use tokio::{sync::Notify, time::sleep}; +use tokio::{ + sync::Notify, + time::{sleep, Instant}, +}; use tracing::{debug, error}; use super::{ReceiveAndIngestNotifiers, SyncActors, ONE_MINUTE}; @@ -84,6 +87,8 @@ impl Ingester { } async fn run_loop_iteration(&self) -> Result<(), Error> { + let start = Instant::now(); + let operations_to_ingest_count = self .sync .db @@ -103,11 +108,12 @@ impl Ingester { "Starting sync messages cloud ingestion loop" ); - self.sync.ingest_ops().await?; + let ingested_count = self.sync.ingest_ops().await?; debug!( - operations_to_ingest_count, - "Finished sync messages cloud ingestion loop" + ingested_count, + "Finished sync messages cloud ingestion loop in {:?}", + start.elapsed() ); Ok(()) diff --git a/core/crates/sync/src/manager.rs b/core/crates/sync/src/manager.rs index e4d830bcc..47460afac 100644 --- a/core/crates/sync/src/manager.rs +++ b/core/crates/sync/src/manager.rs @@ -168,7 +168,7 @@ impl Manager { } #[instrument(skip(self))] - async fn ingest_by_model(&self, model_id: ModelId) -> Result<(), Error> { + async fn ingest_by_model(&self, model_id: ModelId) -> Result { let mut total_count = 0; let mut buckets = (0..self.available_parallelism) @@ -254,7 +254,7 @@ impl Manager { } } - bulk_process_of_create_only_ops( + total_count += bulk_process_of_create_only_ops( self.available_parallelism, Arc::clone(&self.clock), Arc::clone(&self.timestamp_per_device), @@ -361,38 +361,44 @@ impl Manager { "Ingested all operations of this model" ); - Ok(()) + Ok(total_count) } - pub async fn ingest_ops(&self) -> Result<(), Error> { + pub async fn ingest_ops(&self) -> Result { + let mut total_count = 0; + // WARN: this order here exists because sync messages MUST be processed in this exact order // due to relationship dependencies between these tables. - self.ingest_by_model(prisma_sync::device::MODEL_ID).await?; + total_count += self.ingest_by_model(prisma_sync::device::MODEL_ID).await?; - ( + total_count += [ self.ingest_by_model(prisma_sync::storage_statistics::MODEL_ID), self.ingest_by_model(prisma_sync::tag::MODEL_ID), self.ingest_by_model(prisma_sync::location::MODEL_ID), self.ingest_by_model(prisma_sync::object::MODEL_ID), self.ingest_by_model(prisma_sync::label::MODEL_ID), - ) - .try_join() - .await?; + ] + .try_join() + .await? + .into_iter() + .sum::(); - ( + total_count += [ self.ingest_by_model(prisma_sync::exif_data::MODEL_ID), self.ingest_by_model(prisma_sync::file_path::MODEL_ID), self.ingest_by_model(prisma_sync::tag_on_object::MODEL_ID), self.ingest_by_model(prisma_sync::label_on_object::MODEL_ID), - ) - .try_join() - .await?; + ] + .try_join() + .await? + .into_iter() + .sum::(); if self.tx.send(SyncEvent::Ingested).is_err() { warn!("failed to send ingested message on `ingest_ops`"); } - Ok(()) + Ok(total_count) } #[must_use] @@ -707,10 +713,17 @@ async fn bulk_process_of_create_only_ops( spawn(async move { let mut total_count = 0; + let process_creates_batch_start = Instant::now(); + while let Some(count) = bucket.try_next().await? { total_count += count; } + debug!( + "Processed {total_count} creates in {:?}", + process_creates_batch_start.elapsed() + ); + Ok::<_, Error>(total_count) }) })