diff --git a/crates/matrix-sdk/src/sliding_sync/cache.rs b/crates/matrix-sdk/src/sliding_sync/cache.rs
index 885aae7db..0b0084767 100644
--- a/crates/matrix-sdk/src/sliding_sync/cache.rs
+++ b/crates/matrix-sdk/src/sliding_sync/cache.rs
@@ -166,7 +166,7 @@ pub(super) async fn restore_sliding_sync_state(
             trace!("Successfully read the `SlidingSync` from the cache");
             // Let's update the `SlidingSync`.
             if let Some(since) = to_device_since {
-                *to_device_token.get_or_insert_with(Default::default) = since;
+                *to_device_token = Some(since);
             }
             *delta_token = frozen_delta_token;
         }
diff --git a/crates/matrix-sdk/src/sliding_sync/mod.rs b/crates/matrix-sdk/src/sliding_sync/mod.rs
index ea6dac649..d00d71af5 100644
--- a/crates/matrix-sdk/src/sliding_sync/mod.rs
+++ b/crates/matrix-sdk/src/sliding_sync/mod.rs
@@ -116,7 +116,7 @@ pub(super) struct SlidingSyncInner {
     /// Number of times a Sliding Sync session has been reset.
     reset_counter: AtomicU8,
 
-    /// The intended state of the extensions being supplied to sliding /sync
+    /// The intended state of the extensions being supplied to Sliding Sync
     /// calls.
     extensions: Mutex<Option<ExtensionsConfig>>,
 
@@ -699,6 +699,7 @@ struct FrozenSlidingSync {
 impl From<&SlidingSync> for FrozenSlidingSync {
     fn from(sliding_sync: &SlidingSync) -> Self {
         let position = sliding_sync.inner.position.read().unwrap();
+
         FrozenSlidingSync {
             delta_token: position.delta_token.clone(),
             to_device_since: position.to_device_token.clone(),
@@ -834,6 +835,43 @@ mod tests {
         Ok(())
     }
 
+    #[tokio::test]
+    async fn test_to_device_token_properly_cached() -> Result<()> {
+        let (_server, sliding_sync) = new_sliding_sync(vec![SlidingSyncList::builder("foo")
+            .sync_mode(SlidingSyncMode::Selective)
+            .set_range(0..=10)])
+        .await?;
+
+        // When no to-device token is present, `prepare_extensions_config` doesn't fill
+        // the request with it.
+        let config = sliding_sync.prepare_extension_config(Some("pos"));
+        assert!(config.to_device.since.is_none());
+
+        let config = sliding_sync.prepare_extension_config(None);
+        assert!(config.to_device.since.is_none());
+
+        // When no to-device token is present, it's still not there after caching
+        // either.
+        let frozen = FrozenSlidingSync::from(&sliding_sync);
+        assert!(frozen.to_device_since.is_none());
+
+        // When a to-device token is present, `prepare_extensions_config` fills the
+        // request with it.
+        let since = String::from("my-to-device-since-token");
+        sliding_sync.update_to_device_since(since.clone());
+
+        let config = sliding_sync.prepare_extension_config(Some("pos"));
+        assert_eq!(config.to_device.since.as_ref(), Some(&since));
+
+        let config = sliding_sync.prepare_extension_config(None);
+        assert_eq!(config.to_device.since.as_ref(), Some(&since));
+
+        let frozen = FrozenSlidingSync::from(&sliding_sync);
+        assert_eq!(frozen.to_device_since, Some(since));
+
+        Ok(())
+    }
+
     #[tokio::test]
     async fn test_add_list() -> Result<()> {
         let (_server, sliding_sync) = new_sliding_sync(vec![SlidingSyncList::builder("foo")