From c24063e892aac4e22495cfccc0304a9ee95ed13e Mon Sep 17 00:00:00 2001 From: Jamie Pine Date: Tue, 14 Oct 2025 22:39:16 -0700 Subject: [PATCH] (chore): CI fix --- .github/actions/setup-rust/action.yml | 35 +++ .github/actions/setup-system/action.yml | 23 +- .github/workflows/mobile-ci.yml | 207 ------------------ core/src/device/manager.rs | 10 + core/src/lib.rs | 40 ++-- core/src/library/manager.rs | 10 + core/src/ops/network/sync_setup/action.rs | 2 + core/src/service/network/core/event_loop.rs | 10 +- core/src/service/network/core/mod.rs | 22 +- .../src/service/network/protocol/messaging.rs | 2 + core/src/service/sync/peer.rs | 16 +- 11 files changed, 127 insertions(+), 250 deletions(-) create mode 100644 .github/actions/setup-rust/action.yml delete mode 100644 .github/workflows/mobile-ci.yml diff --git a/.github/actions/setup-rust/action.yml b/.github/actions/setup-rust/action.yml new file mode 100644 index 000000000..a691a27e0 --- /dev/null +++ b/.github/actions/setup-rust/action.yml @@ -0,0 +1,35 @@ +name: Setup Rust +description: Setup Rust toolchain and dependencies +inputs: + target: + description: toolchain target triple + required: false + save-cache: + description: Whether to save the Rust cache + required: false + default: 'false' + restore-cache: + description: Whether to restore the Rust cache + required: false + default: 'true' +runs: + using: 'composite' + steps: + - name: Install Rust + id: toolchain + uses: IronCoreLabs/rust-toolchain@v1 + with: + target: ${{ inputs.target }} + components: clippy, rustfmt + + - name: Cache Rust Dependencies + if: ${{ inputs.restore-cache == 'true' }} + uses: Swatinem/rust-cache@v2 + with: + key: ${{ inputs.target }} + save-if: ${{ inputs.save-cache }} + shared-key: stable-cache + + - name: Cargo config.toml + shell: bash + run: echo '{}' | npx -y mustache - .cargo/config.toml.mustache .cargo/config.toml diff --git a/.github/actions/setup-system/action.yml b/.github/actions/setup-system/action.yml index 6b934c9e0..801af6963 100644 --- a/.github/actions/setup-system/action.yml +++ b/.github/actions/setup-system/action.yml @@ -87,13 +87,16 @@ runs: if: ${{ runner.os == 'Windows' }} run: ./scripts/setup.ps1 - - name: Setup shared libraries - shell: bash - env: - TARGET_TRIPLE: ${{ inputs.target }} - GITHUB_TOKEN: ${{ inputs.token }} - run: | - pushd scripts - npm i --production - popd - env NODE_ENV=debug node scripts/preprep.mjs + # TODO: Re-enable this when preprep.mjs is ported to V2 + # This downloads native dependencies (FFmpeg, protoc, HEIF) needed for optional features + # Currently commented out to get basic builds working + # - name: Setup shared libraries + # shell: bash + # env: + # TARGET_TRIPLE: ${{ inputs.target }} + # GITHUB_TOKEN: ${{ inputs.token }} + # run: | + # pushd scripts + # npm i --production + # popd + # env NODE_ENV=debug node scripts/preprep.mjs diff --git a/.github/workflows/mobile-ci.yml b/.github/workflows/mobile-ci.yml deleted file mode 100644 index 6520db6e5..000000000 --- a/.github/workflows/mobile-ci.yml +++ /dev/null @@ -1,207 +0,0 @@ -name: Mobile CI - -on: - push: - branches: - - main - pull_request: - paths: - - 'apps/mobile/**' - - '.github/workflows/mobile-ci.yml' - - 'core/**' - - 'crates/**' - - 'packages/assets/**' - - 'packages/client/**' - - 'packages/config/**' - workflow_dispatch: - -env: - SPACEDRIVE_CUSTOM_APT_FLAGS: --no-install-recommends - SPACEDRIVE_CI: '1' - # From: https://github.com/rust-lang/rust-analyzer/blob/master/.github/workflows/ci.yaml - CARGO_INCREMENTAL: 0 - CARGO_NET_RETRY: 10 - RUST_BACKTRACE: short - RUSTUP_MAX_RETRIES: 10 - -# Cancel previous runs of the same workflow on the same branch. -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - js: - name: JS - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Setup Node.js, pnpm and dependencies - uses: ./.github/actions/setup-pnpm - with: - token: ${{ secrets.GITHUB_TOKEN }} - - - name: Build mobile JS - run: pnpm mobile export - - # Disabled until I can figure out why our app on x86_64 crashes on startup. - # android: - # name: Android - # runs-on: macos-12 - # steps: - # - name: Checkout repository - # uses: actions/checkout@v4 - # - # - name: Setup Java JDK - # uses: actions/setup-java@v3.10.0 - # with: - # java-version: '18' - # distribution: 'temurin' - # - # - name: Setup Node.js, pnpm and dependencies - # uses: ./.github/actions/setup-pnpm - # with: - # token: ${{ secrets.GITHUB_TOKEN }} - # - # - name: Setup System and Rust - # uses: ./.github/actions/setup-system - # with: - # token: ${{ secrets.GITHUB_TOKEN }} - # setup-arg: mobile - # - # - name: Setup Android SDK Tools - # uses: android-actions/setup-android@v2.0.2 - # - # - name: Cache NDK - # uses: actions/cache@v3 - # with: - # path: ${{ env.ANDROID_HOME }}/ndk/26.1.10909125 - # key: ndk-26.1.10909125 - # - # - name: Install NDK - # run: echo "y" | sudo ${ANDROID_HOME}/cmdline-tools/latest/bin/sdkmanager --install "ndk;26.1.10909125" - # - # - name: Cache Gradle - # uses: gradle/gradle-build-action@v2 - # - # - name: Build Android - # working-directory: ./apps/mobile/android - # run: chmod +x ./gradlew && ./gradlew assembleRelease -PreactNativeArchitectures=x86_64 --no-daemon - # - # - name: Cache AVD - # uses: actions/cache@v3 - # id: avd-cache - # with: - # path: | - # ~/.android/avd/* - # ~/.android/adb* - # key: avd-30 - # - # - name: Generate AVD Snapshot - # if: ${{ steps.avd-cache.outputs.cache-hit != 'true' }} - # uses: ReactiveCircus/android-emulator-runner@v2.28.0 - # with: - # arch: x86_64 - # api-level: 30 - # target: google_apis - # ndk: 26.1.10909125 - # ram-size: 4096M - # emulator-boot-timeout: 12000 - # force-avd-creation: false - # emulator-options: -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none - # disable-animations: false - # script: echo "Generated AVD snapshot." - # - # - name: Install Maestro - # run: | - # # workaround for https://github.com/mobile-dev-inc/maestro/issues/877 - # export MAESTRO_VERSION=1.21.3; curl -Ls "https://get.maestro.mobile.dev" | bash - # echo "$HOME/.maestro/bin" >> $GITHUB_PATH - # - # - name: Run Tests - # uses: ReactiveCircus/android-emulator-runner@v2.28.0 - # with: - # arch: x86_64 - # api-level: 30 - # target: google_apis - # ndk: 26.1.10909125 - # ram-size: 4096M - # emulator-boot-timeout: 12000 - # force-avd-creation: false - # emulator-options: -no-snapshot-save -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none - # disable-animations: true - # script: | - # adb install -r apps/mobile/android/app/build/outputs/apk/release/app-release.apk - # adb wait-for-device - # ./apps/mobile/scripts/run-maestro-tests.sh android - - ios: - name: iOS - runs-on: macos-14 - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - # - name: Install Xcode - # uses: maxim-lobanov/setup-xcode@v1 - # with: - # xcode-version: latest-stable - - - name: Setup System and Rust - uses: ./.github/actions/setup-system - with: - token: ${{ secrets.GITHUB_TOKEN }} - setup-arg: mobile - - - name: Setup Node.js, pnpm and dependencies - uses: ./.github/actions/setup-pnpm - with: - token: ${{ secrets.GITHUB_TOKEN }} - - - name: Generate iOS Project - working-directory: ./apps/mobile - run: pnpm expo prebuild --platform ios --no-install - - # Hermes doesn't work with Cocoapods 1.15.0 - # https://forums.developer.apple.com/forums/thread/745518 - - name: Setup Cocoapods - uses: maxim-lobanov/setup-cocoapods@v1 - with: - version: latest - - # - name: Cache Pods - # uses: actions/cache@v4 - # with: - # path: | - # ./apps/mobile/ios/Pods - # ~/Library/Caches/CocoaPods - # ~/.cocoapods - # key: pods-${{ hashFiles('./apps/mobile/ios/Podfile.lock') }} - # restore-keys: pods- - - - name: Install Pods - working-directory: ./apps/mobile/ios - run: pod install --repo-update - - - name: Build iOS - working-directory: ./apps/mobile/ios - run: xcodebuild -workspace ./Spacedrive.xcworkspace -scheme Spacedrive -configuration Release -sdk iphonesimulator -derivedDataPath build -arch "$(uname -m)" - - - name: Install Maestro - run: | - curl -Ls "https://get.maestro.mobile.dev" | bash - brew tap facebook/fb - brew install facebook/fb/idb-companion - echo "${HOME}/.maestro/bin" >> $GITHUB_PATH - - - name: Run Simulator - id: run_simulator - uses: futureware-tech/simulator-action@v3 - with: - model: 'iPhone 14' - os_version: 17.4 - erase_before_boot: false - - - name: Run Tests - run: ./apps/mobile/scripts/run-maestro-tests.sh ios ${{ steps.run_simulator.outputs.udid }} diff --git a/core/src/device/manager.rs b/core/src/device/manager.rs index 74f9d8871..94af2feab 100644 --- a/core/src/device/manager.rs +++ b/core/src/device/manager.rs @@ -105,10 +105,20 @@ impl DeviceManager { id: config.id, name: config.name.clone(), os: parse_os(&config.os), + os_version: None, hardware_model: config.hardware_model.clone(), network_addresses: vec![], + capabilities: serde_json::json!({ + "indexing": true, + "p2p": true, + "volume_detection": true + }), is_online: true, last_seen_at: chrono::Utc::now(), + sync_enabled: true, + last_sync_at: None, + last_state_watermark: None, + last_shared_watermark: None, created_at: chrono::Utc::now(), updated_at: chrono::Utc::now(), } diff --git a/core/src/lib.rs b/core/src/lib.rs index 3b12b4ebc..5bfb137c8 100644 --- a/core/src/lib.rs +++ b/core/src/lib.rs @@ -39,7 +39,7 @@ use crate::{ }; use std::{path::PathBuf, sync::Arc}; -use tokio::sync::{mpsc, RwLock}; +use tokio::sync::{broadcast, mpsc, RwLock}; use tracing::{error, info, warn}; use uuid::Uuid; @@ -267,6 +267,14 @@ impl Core { { Ok(()) => { info!("Sync service initialized for library {}", library.id()); + + // Wire up network event receiver to PeerSync for connection tracking + if let Some(sync_service) = library.sync_service() { + let peer_sync = sync_service.peer_sync(); + let network_events = networking.subscribe_events(); + peer_sync.set_network_events(network_events).await; + info!("Network event receiver wired to PeerSync for library {}", library.id()); + } } Err(e) => { warn!( @@ -397,13 +405,7 @@ impl Core { // Set up event bridge to integrate with core event system (only if not already done) if !already_initialized { let event_bridge = NetworkEventBridge::new( - networking_service - .subscribe_events() - .await - .unwrap_or_else(|| { - let (_, rx) = tokio::sync::mpsc::unbounded_channel(); - rx - }), + networking_service.subscribe_events(), self.events.clone(), ); tokio::spawn(event_bridge.run()); @@ -563,13 +565,13 @@ pub mod networking { /// Bridge between networking events and core events /// TODO: why? - james pub struct NetworkEventBridge { - network_events: mpsc::UnboundedReceiver, + network_events: broadcast::Receiver, core_events: Arc, } impl NetworkEventBridge { pub fn new( - network_events: mpsc::UnboundedReceiver, + network_events: broadcast::Receiver, core_events: Arc, ) -> Self { Self { @@ -579,9 +581,21 @@ impl NetworkEventBridge { } pub async fn run(mut self) { - while let Some(event) = self.network_events.recv().await { - if let Some(core_event) = self.translate_event(event) { - self.core_events.emit(core_event); + loop { + match self.network_events.recv().await { + Ok(event) => { + if let Some(core_event) = self.translate_event(event) { + self.core_events.emit(core_event); + } + } + Err(broadcast::error::RecvError::Lagged(skipped)) => { + warn!("NetworkEventBridge lagged, skipped {} events", skipped); + continue; + } + Err(broadcast::error::RecvError::Closed) => { + info!("NetworkEventBridge channel closed, stopping"); + break; + } } } } diff --git a/core/src/library/manager.rs b/core/src/library/manager.rs index 41136cd35..489d47074 100644 --- a/core/src/library/manager.rs +++ b/core/src/library/manager.rs @@ -289,6 +289,14 @@ impl LibraryManager { "Failed to initialize sync service for library {}: {}", config.id, e ); + } else { + // Wire up network event receiver to PeerSync for connection tracking + if let Some(sync_service) = library.sync_service() { + let peer_sync = sync_service.peer_sync(); + let network_events = networking.subscribe_events(); + peer_sync.set_network_events(network_events).await; + info!("Network event receiver wired to PeerSync for library {}", config.id); + } } } else { info!( @@ -581,6 +589,8 @@ impl LibraryManager { created_at: Set(device.created_at), sync_enabled: Set(true), // Enable sync by default for this device last_sync_at: Set(None), + last_state_watermark: Set(None), + last_shared_watermark: Set(None), updated_at: Set(Utc::now()), }; diff --git a/core/src/ops/network/sync_setup/action.rs b/core/src/ops/network/sync_setup/action.rs index e398812ae..c50478e31 100644 --- a/core/src/ops/network/sync_setup/action.rs +++ b/core/src/ops/network/sync_setup/action.rs @@ -196,6 +196,8 @@ impl LibrarySyncSetupAction { updated_at: Set(Utc::now()), sync_enabled: Set(true), // Enable sync for registered devices last_sync_at: Set(None), + last_state_watermark: Set(None), + last_shared_watermark: Set(None), }; device_model diff --git a/core/src/service/network/core/event_loop.rs b/core/src/service/network/core/event_loop.rs index 01880b0cb..08024eb3c 100644 --- a/core/src/service/network/core/event_loop.rs +++ b/core/src/service/network/core/event_loop.rs @@ -12,7 +12,7 @@ use iroh::NodeId; use iroh::{Endpoint, NodeAddr}; use std::sync::Arc; use tokio::io::AsyncWriteExt; -use tokio::sync::{mpsc, RwLock}; +use tokio::sync::{broadcast, mpsc, RwLock}; use uuid::Uuid; /// Commands that can be sent to the event loop @@ -60,8 +60,8 @@ pub struct NetworkingEventLoop { /// Device registry for managing device state device_registry: Arc>, - /// Event sender for broadcasting network events - event_sender: mpsc::UnboundedSender, + /// Event sender for broadcasting network events (broadcast channel allows multiple subscribers) + event_sender: broadcast::Sender, /// Command receiver command_rx: mpsc::UnboundedReceiver, @@ -91,7 +91,7 @@ impl NetworkingEventLoop { endpoint: Endpoint, protocol_registry: Arc>, device_registry: Arc>, - event_sender: mpsc::UnboundedSender, + event_sender: broadcast::Sender, identity: NetworkIdentity, active_connections: Arc>>, logger: Arc, @@ -294,7 +294,7 @@ impl NetworkingEventLoop { conn: Connection, protocol_registry: Arc>, device_registry: Arc>, - event_sender: mpsc::UnboundedSender, + event_sender: broadcast::Sender, command_sender: mpsc::UnboundedSender, remote_node_id: NodeId, logger: Arc, diff --git a/core/src/service/network/core/mod.rs b/core/src/service/network/core/mod.rs index b29cfee71..4560628e4 100644 --- a/core/src/service/network/core/mod.rs +++ b/core/src/service/network/core/mod.rs @@ -13,7 +13,7 @@ use iroh::discovery::{mdns::MdnsDiscovery, Discovery}; use iroh::endpoint::Connection; use iroh::{Endpoint, NodeAddr, NodeId, RelayMode, RelayUrl, Watcher}; use std::sync::Arc; -use tokio::sync::{mpsc, RwLock}; +use tokio::sync::{broadcast, mpsc, RwLock}; use uuid::Uuid; pub use event_loop::{EventLoopCommand, NetworkingEventLoop}; @@ -99,11 +99,8 @@ pub struct NetworkingService { /// Registry for device state and connections device_registry: Arc>, - /// Event sender for broadcasting network events - event_sender: mpsc::UnboundedSender, - - /// Event receiver for subscribers - event_receiver: Arc>>>, + /// Event sender for broadcasting network events (broadcast channel allows multiple subscribers) + event_sender: broadcast::Sender, /// Active connections tracker active_connections: Arc>>, @@ -130,8 +127,9 @@ impl NetworkingService { let secret_key = identity.to_iroh_secret_key()?; let node_id = secret_key.public(); - // Create event channel - let (event_sender, event_receiver) = mpsc::unbounded_channel(); + // Create event broadcast channel (capacity of 1000 events) + // Using broadcast allows multiple subscribers (NetworkEventBridge + PeerSync instances) + let (event_sender, _) = broadcast::channel(1000); // Create registries let protocol_registry = Arc::new(RwLock::new(ProtocolRegistry::new())); @@ -151,7 +149,6 @@ impl NetworkingService { protocol_registry, device_registry, event_sender, - event_receiver: Arc::new(RwLock::new(Some(event_receiver))), active_connections: Arc::new(RwLock::new(std::collections::HashMap::new())), logger, }) @@ -779,8 +776,11 @@ impl NetworkingService { } /// Subscribe to network events - pub async fn subscribe_events(&self) -> Option> { - self.event_receiver.write().await.take() + /// + /// Returns a new receiver that will receive all network events. + /// Can be called multiple times to create multiple subscribers. + pub fn subscribe_events(&self) -> broadcast::Receiver { + self.event_sender.subscribe() } /// Get our network identity diff --git a/core/src/service/network/protocol/messaging.rs b/core/src/service/network/protocol/messaging.rs index c7f6ffeb6..d66a52dc0 100644 --- a/core/src/service/network/protocol/messaging.rs +++ b/core/src/service/network/protocol/messaging.rs @@ -276,6 +276,8 @@ impl MessagingProtocolHandler { created_at: Set(Utc::now()), sync_enabled: Set(true), // Enable sync for registered devices last_sync_at: Set(None), + last_state_watermark: Set(None), + last_shared_watermark: Set(None), updated_at: Set(Utc::now()), }; diff --git a/core/src/service/sync/peer.rs b/core/src/service/sync/peer.rs index b9c076bc7..f509e3146 100644 --- a/core/src/service/sync/peer.rs +++ b/core/src/service/sync/peer.rs @@ -66,7 +66,7 @@ pub struct PeerSync { is_running: Arc, /// Network event receiver (optional - if provided, enables connection event handling) - network_events: Arc>>>, + network_events: Arc>>>, } impl PeerSync { @@ -102,7 +102,7 @@ impl PeerSync { } /// Set network event receiver for connection tracking - pub async fn set_network_events(&self, receiver: mpsc::UnboundedReceiver) { + pub async fn set_network_events(&self, receiver: broadcast::Receiver) { *self.network_events.lock().await = Some(receiver); } @@ -389,7 +389,7 @@ impl PeerSync { while is_running.load(Ordering::SeqCst) { match rx.recv().await { - Some(event) => { + Ok(event) => { use crate::service::network::core::NetworkEvent; match event { NetworkEvent::ConnectionEstablished { device_id, node_id } => { @@ -427,7 +427,15 @@ impl PeerSync { } } } - None => { + Err(broadcast::error::RecvError::Lagged(skipped)) => { + warn!( + skipped = skipped, + "PeerSync network event listener lagged, skipped {} events", + skipped + ); + continue; + } + Err(broadcast::error::RecvError::Closed) => { info!("Network event channel closed, stopping listener"); break; }