Skip to main content

coven/sync/
sync_manager.rs

1//! High-level sync manager: lifecycle, membership, status.
2//!
3//! Owns the sync lifecycle — cloud home + sync loop — and starts/stops it when
4//! a provider is connected/disconnected, no app restart required. The host
5//! supplies the config snapshot, keys, encryption, database, clock, and blob
6//! handling; coven drives the rest.
7
8use std::collections::HashMap;
9use std::path::PathBuf;
10use std::sync::{Arc, RwLock};
11
12use tokio::sync::watch;
13use tracing::{error, info};
14
15use crate::blob::transition::{self, MakeLocalError, MakeRemoteError};
16use crate::blob::BlobTransitionObserver;
17use crate::clock::ClockRef;
18use crate::config::{Config, HomeStorage};
19use crate::coven::StoreOpenGuard;
20use crate::database::{Database, DbError};
21use crate::encryption::EncryptionService;
22use crate::keys::{DeviceIdentityCustody, KeyError, MasterKeyCustody, StoreKeys};
23use crate::storage::cloud::setup::{SetupError, StorageSetupError};
24use crate::storage::cloud::{CloudHome, CloudHomeError};
25#[cfg(any(test, feature = "test-utils"))]
26use crate::sync::cloud_storage::BlobPathScheme;
27use crate::sync::cloud_storage::CloudCipher;
28#[cfg(any(test, feature = "test-utils"))]
29use crate::sync::cloud_storage::CloudSyncStorage;
30use crate::sync::cycle::{InitSyncError, SyncComponents};
31/// `MemberInfo` lives next to `MemberRole` in the membership module; coven's
32/// public path reaches it through here (re-exported from `lib.rs`).
33pub(crate) use crate::sync::membership::MemberInfo;
34use crate::sync::membership::MemberRole;
35use crate::sync::storage::SyncStorage;
36use crate::sync::sync_loop::{SyncLoopError, SyncLoopHandle, SyncLoopStatus};
37
38/// Supplies the host's current config for building the next connection. Starting
39/// a loop captures one snapshot; commands on that running loop use its immutable
40/// store identity, representation, provider settings, and directory even if the
41/// host's next config changes meanwhile.
42pub(crate) type ConfigProvider = Arc<dyn Fn() -> Config + Send + Sync>;
43
44#[derive(Debug, thiserror::Error)]
45pub enum SyncError {
46    #[error("sync is not configured")]
47    NotConfigured,
48    #[error("sync loop is not running")]
49    LoopNotRunning,
50    #[error("sharing requires an encrypted cloud home")]
51    NotEncryptedHome,
52    #[error("no master key is established for this opaque store (locked, or never initialized)")]
53    MasterKeyNotEstablished,
54    #[error("failed to build cloud home: {0}")]
55    CloudHome(#[from] CloudHomeError),
56    #[error("failed to create sync storage: {0}")]
57    StorageSetup(StorageSetupError),
58    #[error("key error: {0}")]
59    Key(#[from] KeyError),
60    #[error("sync initialization error: {0}")]
61    Init(#[from] InitSyncError),
62    #[error("Store protocol state: {0}")]
63    Protocol(String),
64    #[error("{0}")]
65    Setup(#[from] SetupError),
66    #[error("membership error: {0}")]
67    Membership(Box<crate::sync::membership_ops::MembershipOpsError>),
68    #[error("circle operation: {0}")]
69    Circle(#[from] crate::sync::circle_ops::CircleOperationError),
70    #[error("device join: {0}")]
71    DeviceJoin(#[from] crate::DeviceJoinError),
72    #[error("{0}")]
73    Database(#[from] DbError),
74    #[error("blob upload drain failed: {0}")]
75    BlobUpload(DbError),
76    #[error("sync loop error: {0}")]
77    Loop(SyncLoopError),
78}
79
80impl From<crate::sync::membership_ops::MembershipOpsError> for SyncError {
81    fn from(error: crate::sync::membership_ops::MembershipOpsError) -> Self {
82        Self::Membership(Box::new(error))
83    }
84}
85
86/// High-level sync manager.
87///
88/// Holds the store's master-key custody. The at-rest cipher is resolved from
89/// it per [`start_sync`](Self::start_sync) call for an opaque home; a Merge
90/// store with scoped rows also loads generation 1 for stable row routing,
91/// independent of the home's storage representation.
92pub(crate) struct SyncManager {
93    config_provider: ConfigProvider,
94    key_service: StoreKeys,
95    custody: Arc<dyn MasterKeyCustody>,
96    identity_custody: Arc<dyn DeviceIdentityCustody>,
97    db: Database,
98    clock: ClockRef,
99    cloudkit_ops: Option<Arc<dyn crate::storage::cloud::cloudkit::CloudKitOps>>,
100    observer: Option<Arc<dyn BlobTransitionObserver>>,
101
102    /// The store-directory lock, cloned into every sync loop this manager
103    /// starts so the loop's thread keeps it alive for the whole of its final
104    /// cycle. The lock releases when the last writer — a running loop, else the
105    /// handle — is gone, never while a detached loop thread is still writing.
106    open_guard: Arc<StoreOpenGuard>,
107
108    /// The current status value the [`CovenHandle`](crate::CovenHandle) owns, cloned
109    /// into every sync loop this manager starts so a subscription outlives the
110    /// loop restarts a reconnect performs.
111    status_tx: tokio::sync::watch::Sender<SyncLoopStatus>,
112
113    // Mutable sync state — updated when providers are connected/disconnected
114    sync_loop_handle: RwLock<Option<Arc<SyncLoopHandle>>>,
115    cloud_home: RwLock<Option<Arc<dyn CloudHome>>>,
116
117    /// Serializes the membership operations that mint or rotate the store key —
118    /// invite (wraps the key to a new member) and remove (mints a fresh key and
119    /// re-wraps it to everyone remaining). Each clones the live cipher at entry
120    /// and builds a new keyring on top of it; without this, two rapid ops on one
121    /// device would both clone the SAME base generation and the second would
122    /// overwrite the first's wraps before the first's head guard could catch it,
123    /// forking the fleet. Held for the whole operation so the second waits and
124    /// builds on the first's committed state.
125    member_ops_lock: tokio::sync::Mutex<()>,
126}
127
128impl SyncManager {
129    fn require_configured_coordination(&self, config: &Config) -> Result<(), SyncError> {
130        crate::storage::cloud::setup::require_serial_coordination_config(
131            config,
132            self.db.write_policy(),
133        )
134        .map_err(SyncError::StorageSetup)
135    }
136
137    async fn storage_for_command(
138        &self,
139        config: &Config,
140        active_loop: Option<&Arc<SyncLoopHandle>>,
141    ) -> Result<Arc<crate::sync::cloud_storage::CloudSyncStorage>, SyncError> {
142        if let Some(active_loop) = active_loop {
143            return Ok(active_loop.storage().clone());
144        }
145        let storage = match self.cloud_home() {
146            Some(home) => crate::storage::cloud::setup::create_sync_storage_with_home(
147                config,
148                self.custody.as_ref(),
149                self.identity_custody.as_ref(),
150                home,
151                None,
152            ),
153            None => {
154                crate::storage::cloud::setup::create_sync_storage_with_cloudkit(
155                    config,
156                    &self.key_service,
157                    self.custody.as_ref(),
158                    self.identity_custody.as_ref(),
159                    None,
160                    self.clock.clone(),
161                    self.cloudkit_ops.clone(),
162                )
163                .await
164            }
165        }
166        .map_err(SyncError::StorageSetup)?;
167        Ok(Arc::new(storage))
168    }
169
170    /// Build the manager off the owned [`Database`]. Session initialization takes
171    /// the database's already-seeded register clock into [`SyncComponents`], and
172    /// every connected command reads that captured clock from the installed loop.
173    ///
174    /// Construction is infallible and synchronous: seeding already happened in
175    /// the open path. The manager is built lazily, only once a provider is
176    /// connected.
177    #[allow(clippy::too_many_arguments)]
178    pub(crate) fn new(
179        config_provider: ConfigProvider,
180        key_service: StoreKeys,
181        custody: Arc<dyn MasterKeyCustody>,
182        identity_custody: Arc<dyn DeviceIdentityCustody>,
183        db: Database,
184        clock: ClockRef,
185        cloudkit_ops: Option<Arc<dyn crate::storage::cloud::cloudkit::CloudKitOps>>,
186        observer: Option<Arc<dyn BlobTransitionObserver>>,
187        open_guard: Arc<StoreOpenGuard>,
188        status_tx: tokio::sync::watch::Sender<SyncLoopStatus>,
189    ) -> Self {
190        Self {
191            config_provider,
192            key_service,
193            custody,
194            identity_custody,
195            db,
196            clock,
197            cloudkit_ops,
198            observer,
199            open_guard,
200            status_tx,
201            sync_loop_handle: RwLock::new(None),
202            cloud_home: RwLock::new(None),
203            member_ops_lock: tokio::sync::Mutex::new(()),
204        }
205    }
206
207    pub(crate) fn cloud_home(&self) -> Option<Arc<dyn CloudHome>> {
208        self.cloud_home.read().unwrap().clone()
209    }
210
211    pub(crate) fn sync_loop_handle(&self) -> Option<Arc<SyncLoopHandle>> {
212        self.sync_loop_handle.read().unwrap().clone()
213    }
214
215    pub(crate) async fn prepare_serial_resolution(
216        &self,
217        branch_base: Option<crate::sync::store_commit::StoreBatchCommitRef>,
218        store_dir: &crate::store_dir::StoreDir,
219    ) -> Result<crate::sync::store_pull::SerialResolutionPlan, SyncError> {
220        let loop_handle = self.sync_loop_handle().ok_or(SyncError::LoopNotRunning)?;
221        let storage = loop_handle.storage();
222        let coordination = storage
223            .serial_coordination()
224            .map_err(|error| SyncError::Protocol(error.to_string()))?;
225        let store_root_hash = self
226            .db
227            .local_store_root_ref()
228            .await?
229            .ok_or_else(|| SyncError::Protocol("exact Store root authority is absent".to_string()))?
230            .store_root_hash;
231        let identity = crate::keys::require_identity(self.identity_custody.as_ref())?;
232        crate::sync::store_pull::prepare_serial_resolution(
233            &self.db,
234            &**storage,
235            coordination,
236            store_root_hash,
237            store_dir,
238            branch_base,
239            &identity,
240        )
241        .await
242        .map_err(|error| SyncError::Protocol(error.to_string()))
243    }
244
245    pub(crate) async fn cleanup_serial_candidates(
246        &self,
247        branch_id: coven_core::PendingBranchId,
248        plan: &crate::sync::store_pull::SerialResolutionPlan,
249    ) -> Result<(), SyncError> {
250        let loop_handle = self.sync_loop_handle().ok_or(SyncError::LoopNotRunning)?;
251        crate::sync::store_pull::cleanup_serial_candidates(
252            &self.db,
253            &**loop_handle.storage(),
254            branch_id,
255            plan,
256        )
257        .await
258        .map_err(|error| SyncError::Protocol(error.to_string()))
259    }
260
261    pub(crate) async fn abandon_serial_branch(
262        &self,
263        branch_id: coven_core::PendingBranchId,
264        store_dir: &crate::store_dir::StoreDir,
265    ) -> Result<coven_core::sync::store_outbound::SerialBranchAbandonment, SyncError> {
266        let loop_handle = self.sync_loop_handle().ok_or(SyncError::LoopNotRunning)?;
267        let storage = loop_handle.storage();
268        let coordination = storage
269            .serial_coordination()
270            .map_err(|error| SyncError::Protocol(error.to_string()))?;
271        let identity = crate::keys::require_identity(self.identity_custody.as_ref())?;
272        let device_id = self
273            .db
274            .get_protocol_state(coven_core::database::LOCAL_DEVICE_ID_STATE_KEY)
275            .await?
276            .ok_or_else(|| {
277                SyncError::Protocol("local Store device identity is absent".to_string())
278            })?;
279        crate::sync::store_outbound::abandon_serial_branch(
280            &self.db,
281            &**storage,
282            coordination,
283            &device_id,
284            &identity,
285            store_dir,
286            branch_id,
287        )
288        .await
289        .map_err(|error| SyncError::Protocol(error.to_string()))
290    }
291
292    pub(crate) async fn abandon_merge_candidate(
293        &self,
294        write_id: coven_core::WriteId,
295    ) -> Result<coven_core::sync::store_outbound::MergeCandidateAbandonment, SyncError> {
296        let loop_handle = self.sync_loop_handle().ok_or(SyncError::LoopNotRunning)?;
297        let identity = crate::keys::require_identity(self.identity_custody.as_ref())?;
298        let device_id = self
299            .db
300            .get_protocol_state(coven_core::database::LOCAL_DEVICE_ID_STATE_KEY)
301            .await?
302            .ok_or_else(|| {
303                SyncError::Protocol("local Store device identity is absent".to_string())
304            })?;
305        crate::sync::store_outbound::abandon_merge_candidate(
306            &self.db,
307            &**loop_handle.storage(),
308            &device_id,
309            &identity,
310            write_id,
311        )
312        .await
313        .map_err(|error| SyncError::Protocol(error.to_string()))
314    }
315
316    // =========================================================================
317    // Sync lifecycle
318    // =========================================================================
319
320    /// Resolve the home's at-rest cipher from `storage` and this manager's
321    /// custody. A browsable home never consults custody — it stores in the
322    /// clear regardless. An opaque home unlocks the master keyring; no key
323    /// established (a locked store, or a browsable/opaque storage mismatch)
324    /// is [`SyncError::MasterKeyNotEstablished`], surfaced here rather than
325    /// deep inside the home build. The single custody→cipher decision both
326    /// [`start_sync`](Self::start_sync) and the test-home connect path share.
327    fn resolve_cipher(&self, storage: HomeStorage) -> Result<CloudCipher, SyncError> {
328        if storage.is_browsable() {
329            Ok(CloudCipher::Plaintext)
330        } else {
331            let keyring = self.custody.unlock()?;
332            CloudCipher::for_storage(storage, keyring.map(Into::into))
333                .ok_or(SyncError::MasterKeyNotEstablished)
334        }
335    }
336
337    fn routing_encryption(&self) -> Result<Option<EncryptionService>, SyncError> {
338        (self.db.write_policy() == crate::WritePolicy::MergeConcurrent
339            && self.db.gates().has_scoped_graph())
340        .then(|| crate::handle::routing_encryption_from_custody(self.custody.as_ref()))
341        .transpose()
342        .map_err(SyncError::from)
343    }
344
345    /// Initialize cloud home and sync loop from current config.
346    /// Called at startup (if already configured) and after connecting a provider.
347    ///
348    /// Two outcomes are success: a configured provider whose home builds and whose
349    /// loop starts, and a store with no configured provider that legitimately
350    /// starts no loop — the latter is a logged `Ok(())` no-op. A cloud-home build
351    /// that *fails* (missing credentials, a bad provider config) is an `Err`, not
352    /// "no provider connected": the caller must not install a manager that reports
353    /// success with nothing started.
354    pub(crate) async fn start_sync(&self) -> Result<(), SyncError> {
355        let config = (self.config_provider)();
356
357        if config.cloud_home.provider.is_none() {
358            self.stop_sync()?;
359            // Not a failure: a store with no configured provider starts no
360            // cloud home or sync loop.
361            info!("start_sync: sync not configured; no loop started");
362            return Ok(());
363        }
364
365        self.require_configured_coordination(&config)?;
366        crate::storage::cloud::setup::require_exact_slot_capabilities_config(&config)
367            .map_err(SyncError::StorageSetup)?;
368
369        let routing_encryption = self.routing_encryption()?;
370
371        self.stop_current_connection()?;
372
373        // The home's at-rest cipher, resolved fresh on every start so a
374        // stop/start picks up whatever custody now holds. Built once here so
375        // the sync loop and storage share one instance — a member removal
376        // rotates the key in place through it.
377        let cipher = self.resolve_cipher(config.cloud_home.storage)?;
378
379        // Build the cloud home. A failure here is a real fault — surface it so the
380        // caller never installs a manager that started nothing.
381        let cloud_home = crate::storage::cloud::create_cloud_home_with_cloudkit(
382            &config,
383            &self.key_service,
384            self.clock.clone(),
385            self.cloudkit_ops.clone(),
386        )
387        .await
388        .map_err(SyncError::from)?;
389        let cloud_home: Arc<dyn CloudHome> = Arc::from(cloud_home);
390
391        // Initialize sync loop. The synced-table set is owned by the Database, so
392        // init_sync reads it from there rather than from a separately-held copy.
393        // Sync is enabled here, so `None` means a real startup failure (no synced
394        // tables, storage/keypair/auth/membership bootstrap) that init_sync already
395        // logged — surface it so the caller never installs a manager whose loop
396        // never started.
397        //
398        // Connect never mints a device identity: a locked agent with no
399        // identity established must fail here with `KeyError::NoDeviceIdentity`,
400        // not silently forge one.
401        let storage = crate::storage::cloud::setup::create_sync_storage_with_home(
402            &config,
403            self.custody.as_ref(),
404            self.identity_custody.as_ref(),
405            cloud_home.clone(),
406            Some(cipher.clone()),
407        )
408        .map_err(|error| match error {
409            crate::storage::cloud::setup::StorageSetupError::Key(error) => SyncError::Key(error),
410            error => SyncError::StorageSetup(error),
411        })?;
412
413        let initialization = self.store_initialization().await?;
414        let components = crate::sync::cycle::init_sync_over_storage(
415            &self.db,
416            storage,
417            initialization,
418            routing_encryption,
419        )
420        .await
421        .map_err(SyncError::from)?;
422
423        let _handle = self.install_sync_loop(components, config)?;
424        *self.cloud_home.write().unwrap() = Some(cloud_home);
425
426        Ok(())
427    }
428
429    /// Build the sync-loop handle off `components`, start it, and install it. The
430    /// shared install tail of [`start_sync`](Self::start_sync) and the test-only
431    /// [`start_sync_with_home`](Self::start_sync_with_home): both reach it only
432    /// after the bootstrap has produced [`SyncComponents`], so the loop handle is
433    /// installed whole, never on a half-built bootstrap.
434    fn install_sync_loop(
435        &self,
436        components: SyncComponents,
437        config: Config,
438    ) -> Result<Arc<SyncLoopHandle>, SyncError> {
439        let handle = Arc::new(SyncLoopHandle::new(
440            components,
441            self.custody.clone(),
442            self.clock.clone(),
443            config,
444            self.observer.clone(),
445            self.open_guard.clone(),
446            self.status_tx.clone(),
447        ));
448        handle.start().map_err(SyncError::Loop)?;
449
450        info!("Sync loop started");
451        *self.sync_loop_handle.write().unwrap() = Some(handle.clone());
452        Ok(handle)
453    }
454
455    /// Test-only: stand the sync loop over an injected `home`/`cipher` instead of
456    /// building the cloud home from config via `create_cloud_home`.
457    ///
458    /// The counterpart of [`start_sync`](Self::start_sync) for a host's
459    /// integration tests, which drive coven over a mock [`CloudHome`] no provider
460    /// match would ever produce. It skips the config-provider gate — the injected
461    /// home IS the enablement, there are no real credentials to check — installs
462    /// the home, builds a [`CloudSyncStorage`] over it under the supplied `cipher`
463    /// (and the config's blob-path scheme), runs the same bootstrap
464    /// [`init_sync`](crate::sync::cycle::init_sync) does via
465    /// [`init_sync_over_storage`](crate::sync::cycle::init_sync_over_storage), and
466    /// starts the loop. A bootstrap failure is an `Err`, the same fail-loud
467    /// discipline `start_sync` keeps — and commit-whole: the home and loop handle
468    /// are installed only after the keypair load and bootstrap both succeed, so a
469    /// failure leaves nothing installed.
470    ///
471    /// After this returns, the connected loop's storage is reachable via
472    /// [`sync_loop_handle`](Self::sync_loop_handle)`().storage()`, so the handle's
473    /// read path serves blobs over the same injected home with no separate hook.
474    ///
475    /// Like [`start_sync`](Self::start_sync), this never mints a device
476    /// identity: the caller must establish one under this manager's identity
477    /// custody first, or this fails with `KeyError::NoDeviceIdentity`.
478    #[cfg(any(test, feature = "test-utils"))]
479    pub(crate) async fn start_sync_with_home(
480        &self,
481        home: std::sync::Arc<dyn CloudHome>,
482        cipher: CloudCipher,
483    ) -> Result<(), SyncError> {
484        self.start_sync_with_home_parts(home, cipher, None).await
485    }
486
487    #[cfg(any(test, feature = "test-utils"))]
488    pub(crate) async fn start_sync_with_home_and_coordination(
489        &self,
490        home: std::sync::Arc<dyn CloudHome>,
491        coordination: std::sync::Arc<dyn crate::storage::cloud::CloudHeadStorage>,
492        cipher: CloudCipher,
493    ) -> Result<(), SyncError> {
494        self.start_sync_with_home_parts(home, cipher, Some(coordination))
495            .await
496    }
497
498    #[cfg(any(test, feature = "test-utils"))]
499    async fn start_sync_with_home_parts(
500        &self,
501        home: std::sync::Arc<dyn CloudHome>,
502        cipher: CloudCipher,
503        coordination: Option<std::sync::Arc<dyn crate::storage::cloud::CloudHeadStorage>>,
504    ) -> Result<(), SyncError> {
505        let config = (self.config_provider)();
506        crate::storage::cloud::setup::require_exact_slot_capabilities_home(
507            home.clone(),
508            config.cloud_home.provider.clone(),
509        )
510        .map_err(SyncError::StorageSetup)?;
511        let routing_encryption = self.routing_encryption()?;
512        self.stop_current_connection()?;
513
514        let keypair = crate::keys::require_identity(self.identity_custody.as_ref())?;
515        let blob_paths = if cipher.is_plaintext() {
516            BlobPathScheme::Plain
517        } else {
518            BlobPathScheme::Hashed
519        };
520        let mut storage = CloudSyncStorage::new(
521            home.clone(),
522            cipher.clone(),
523            blob_paths,
524            config.store_id.clone(),
525            keypair,
526        )?;
527        if let Some(coordination) = coordination {
528            storage = storage.with_test_serial_coordination(coordination);
529        }
530
531        let initialization = self.store_initialization().await?;
532        let components = crate::sync::cycle::init_sync_over_storage(
533            &self.db,
534            storage,
535            initialization,
536            routing_encryption,
537        )
538        .await
539        .map_err(SyncError::from)?;
540
541        let _handle = self.install_sync_loop(components, config)?;
542        *self.cloud_home.write().unwrap() = Some(home);
543
544        Ok(())
545    }
546
547    async fn store_initialization(
548        &self,
549    ) -> Result<crate::sync::cycle::StoreInitialization, SyncError> {
550        let Some(expected_store_root) = self.db.local_store_root_ref().await? else {
551            return Ok(crate::sync::cycle::StoreInitialization::CreateStore);
552        };
553        Ok(crate::sync::cycle::StoreInitialization::OpenStore {
554            expected_store_root,
555        })
556    }
557
558    /// Test-only: stand the sync loop over an injected `home` while resolving the
559    /// at-rest cipher from custody exactly as [`start_sync`](Self::start_sync)
560    /// does — the counterpart of [`start_sync_with_home`](Self::start_sync_with_home)
561    /// for proving the established master key is the one sealing traffic. Unlike
562    /// that method, which takes the cipher explicitly and never consults custody,
563    /// this drives the real [`resolve_cipher`](Self::resolve_cipher) path: an
564    /// opaque home with no key established fails [`SyncError::MasterKeyNotEstablished`]
565    /// before the loop starts.
566    #[cfg(any(test, feature = "test-utils"))]
567    pub(crate) async fn start_sync_with_test_home_custody(
568        &self,
569        home: std::sync::Arc<dyn CloudHome>,
570    ) -> Result<(), SyncError> {
571        let storage = (self.config_provider)().cloud_home.storage;
572        let cipher = self.resolve_cipher(storage)?;
573        self.start_sync_with_home(home, cipher).await
574    }
575
576    /// Tear down the sync loop and cloud home.
577    pub(crate) fn stop_sync(&self) -> Result<(), SyncError> {
578        let stop_result = self.stop_current_loop();
579        *self.sync_loop_handle.write().unwrap() = None;
580        *self.cloud_home.write().unwrap() = None;
581
582        match stop_result {
583            Ok(()) => {
584                info!("Sync loop stopped");
585                Ok(())
586            }
587            Err(error) => {
588                error!("Sync loop stop failed: {error}");
589                Err(error)
590            }
591        }
592    }
593
594    fn stop_current_loop(&self) -> Result<(), SyncError> {
595        let handle = self.sync_loop_handle.write().unwrap().take();
596        if let Some(handle) = handle {
597            handle.stop().map_err(SyncError::Loop)?;
598        }
599        Ok(())
600    }
601
602    fn stop_current_connection(&self) -> Result<(), SyncError> {
603        let stop_result = self.stop_current_loop();
604        *self.cloud_home.write().unwrap() = None;
605        stop_result
606    }
607
608    // =========================================================================
609    // Status / config queries
610    // =========================================================================
611
612    pub(crate) fn is_sync_ready(&self) -> bool {
613        self.sync_loop_handle
614            .read()
615            .unwrap()
616            .as_ref()
617            .is_some_and(|h| h.is_running())
618    }
619
620    pub(crate) fn trigger_sync(&self) {
621        if let Some(ref sync_loop) = *self.sync_loop_handle.read().unwrap() {
622            sync_loop.trigger();
623        }
624    }
625
626    // =========================================================================
627    // Blob locality transitions (make_remote / make_local / cancel_make_remote)
628    // =========================================================================
629
630    /// Make `(root_table, root_id)` Remote (Local → Remote): enqueue an upload per
631    /// user-provided blob from its external file and record the make_remote intent,
632    /// then return. The drain uploads each and flips the gate true on the last (see
633    /// [`crate::blob::transition::make_remote`]); the gate flip re-emits the subtree,
634    /// the cycle's inline push uploads the root's host-provided blobs, and
635    /// `on_root_made_remote` fires. `pin` keeps the uploaded blobs in coven's cache
636    /// as pinned (offline) copies.
637    pub(crate) async fn make_remote(
638        &self,
639        root_table: &str,
640        root_id: &str,
641        pin: bool,
642    ) -> Result<(), MakeRemoteError> {
643        if !self.is_sync_ready() {
644            return Err(MakeRemoteError::SyncNotReady);
645        }
646        let sync_loop = self
647            .sync_loop_handle()
648            .ok_or(MakeRemoteError::SyncNotReady)?;
649        transition::make_remote(
650            &self.db,
651            sync_loop.store_dir(),
652            sync_loop.hlc(),
653            root_table,
654            root_id,
655            pin,
656        )
657        .await?;
658        self.trigger_sync();
659        Ok(())
660    }
661
662    /// Cancel an in-flight make_remote of `(root_table, root_id)`: clear its intent
663    /// and pending uploads and tombstone any blob that already landed. The gate never
664    /// flips, so the root stays Local.
665    pub(crate) async fn cancel_make_remote(
666        &self,
667        root_table: &str,
668        root_id: &str,
669    ) -> Result<(), MakeRemoteError> {
670        if !self.is_sync_ready() {
671            return Err(MakeRemoteError::SyncNotReady);
672        }
673        transition::cancel_make_remote(&self.db, root_table, root_id).await?;
674        self.trigger_sync();
675        Ok(())
676    }
677
678    /// Make `(root_table, root_id)` Local (Remote → Local): bring each blob back to a
679    /// local file durability-first — a user-provided blob to the path named in `dest`
680    /// (blob id → destination path), a host-provided blob to coven's local store (no
681    /// dest) — then flip the gate false, register the user-provided external refs,
682    /// and enqueue the cloud deletes in one atomic commit. Awaitable; `cancel` aborts
683    /// before the commit (the root stays Remote). `dest` carries user-provided ids
684    /// only. Per-blob materialize progress and the completion event reach the
685    /// observer this manager was built with.
686    pub(crate) async fn make_local(
687        &self,
688        root_table: &str,
689        root_id: &str,
690        dest: &HashMap<String, PathBuf>,
691        cancel: &watch::Receiver<bool>,
692        routing_encryption: Option<EncryptionService>,
693    ) -> Result<(), MakeLocalError> {
694        if !self.is_sync_ready() {
695            return Err(MakeLocalError::SyncNotReady);
696        }
697        let sync_loop = self
698            .sync_loop_handle()
699            .ok_or(MakeLocalError::SyncNotReady)?;
700        let storage: &dyn SyncStorage = &**sync_loop.storage();
701        transition::make_local(
702            &self.db,
703            storage,
704            sync_loop.store_dir(),
705            sync_loop.hlc(),
706            routing_encryption,
707            self.observer.as_deref(),
708            root_table,
709            root_id,
710            dest,
711            cancel,
712        )
713        .await?;
714        self.trigger_sync();
715        Ok(())
716    }
717
718    // =========================================================================
719    // Keys / codes
720    // =========================================================================
721
722    // =========================================================================
723    // Membership
724    // =========================================================================
725
726    pub(crate) async fn get_members(&self) -> Result<Vec<MemberInfo>, SyncError> {
727        let active_loop = self.sync_loop_handle();
728        let config = active_loop
729            .as_ref()
730            .map(|handle| handle.config().clone())
731            .unwrap_or_else(|| (self.config_provider)());
732        if active_loop.is_none() && config.cloud_home.provider.is_none() {
733            info!("get_members: sync not configured; returning no members");
734            return Ok(Vec::new());
735        }
736        self.require_configured_coordination(&config)?;
737        let storage = self
738            .storage_for_command(&config, active_loop.as_ref())
739            .await?;
740
741        let user_pubkey = crate::keys::identity_public_key(self.identity_custody.as_ref())?;
742        crate::sync::membership_ops::get_members(
743            &*storage,
744            user_pubkey.as_ref().map(|k| k.as_slice()),
745            &self.db,
746        )
747        .await
748        .map_err(SyncError::from)
749    }
750
751    /// Build a restore code for this store: fetch the current membership-head
752    /// floor from the cloud and mint the code from it, so the restorer can seed
753    /// its watermark from mint-time
754    /// state rather than accepting any signed head as a fresh device would
755    /// otherwise have to. Requires a connected provider — unlike the old,
756    /// storage-free `generate_restore_code`, minting a trustworthy floor is a
757    /// network read, not a pure function of local config and keyring state.
758    pub(crate) async fn generate_restore_code(&self) -> Result<String, SyncError> {
759        let active_loop = self.sync_loop_handle();
760        let config = active_loop
761            .as_ref()
762            .map(|handle| handle.config().clone())
763            .unwrap_or_else(|| (self.config_provider)());
764        if active_loop.is_none() && config.cloud_home.provider.is_none() {
765            return Err(SyncError::NotConfigured);
766        }
767        self.require_configured_coordination(&config)?;
768        let storage = self
769            .storage_for_command(&config, active_loop.as_ref())
770            .await?;
771
772        let pinned_owner = self
773            .db
774            .get_protocol_state(crate::sync::membership_ops::OWNER_PUBKEY_STATE_KEY)
775            .await?;
776        let founder_pubkey = pinned_owner.clone().ok_or_else(|| {
777            SyncError::from(crate::sync::membership_ops::MembershipOpsError::NoFounderChain)
778        })?;
779        let store_root =
780            self.db.local_store_root_ref().await?.ok_or_else(|| {
781                SyncError::Protocol("store protocol root hash is absent".to_string())
782            })?;
783        let membership_floor = match self.db.write_policy() {
784            crate::WritePolicy::MergeConcurrent => {
785                crate::join_code::MembershipFloor::MergeConcurrent(
786                    crate::sync::membership_ops::current_membership_floor(
787                        &*storage,
788                        &store_root,
789                        pinned_owner.as_deref(),
790                        Some(&self.db),
791                    )
792                    .await
793                    .map_err(SyncError::from)?,
794                )
795            }
796            crate::WritePolicy::Serial => {
797                let coordination = storage.serial_coordination().map_err(|error| {
798                    SyncError::Protocol(format!("Serial coordination: {error}"))
799                })?;
800                let reference =
801                    crate::sync::store_outbound::current_serial_head_ref(&self.db, coordination)
802                        .await
803                        .map_err(|error| SyncError::Protocol(error.to_string()))?;
804                crate::join_code::MembershipFloor::Serial(reference)
805            }
806        };
807        let identity = crate::keys::require_identity(self.identity_custody.as_ref())?;
808        let authority = crate::sync::restore_code::RestoreAuthority::ActivatedContinuation(
809            self.db
810                .export_activated_device_continuation(&identity)
811                .await?,
812        );
813
814        crate::storage::cloud::setup::generate_restore_code(
815            &config,
816            &self.key_service,
817            self.custody.as_ref(),
818            store_root,
819            founder_pubkey,
820            membership_floor,
821            authority,
822        )
823        .map_err(SyncError::from)
824    }
825
826    pub(crate) fn invite_member<'a>(
827        &'a self,
828        public_key_hex: &'a str,
829        invitee_email: Option<&'a str>,
830        role: MemberRole,
831    ) -> std::pin::Pin<Box<dyn std::future::Future<Output = Result<String, SyncError>> + Send + 'a>>
832    {
833        Box::pin(async move {
834            // Serialize with any other key-minting/rotating member op on this device.
835            let _member_ops = self.member_ops_lock.lock().await;
836
837            let sync_loop = self
838                .sync_loop_handle
839                .read()
840                .unwrap()
841                .clone()
842                .ok_or(SyncError::LoopNotRunning)?;
843
844            // Inviting a member wraps the store key to them, which only an encrypted
845            // home has. Refuse before touching the membership chain.
846            if sync_loop.current_encryption().is_none() {
847                return Err(SyncError::NotEncryptedHome);
848            }
849            let store_name = sync_loop.config().store_name.clone();
850            let invite_code = sync_loop
851                .invite_member(public_key_hex, invitee_email, role, &store_name)
852                .await
853                .map_err(SyncError::from)?;
854
855            Ok(crate::join_code::encode(&invite_code))
856        })
857    }
858
859    pub(crate) fn remove_member<'a>(
860        &'a self,
861        public_key_hex: &'a str,
862    ) -> std::pin::Pin<Box<dyn std::future::Future<Output = Result<String, SyncError>> + Send + 'a>>
863    {
864        Box::pin(async move {
865            // Serialize with any other key-minting/rotating member op on this device,
866            // so a second removal builds on this one's committed state rather than
867            // cloning the same base cipher and overwriting its wraps.
868            let _member_ops = self.member_ops_lock.lock().await;
869
870            let sync_loop = self
871                .sync_loop_handle
872                .read()
873                .unwrap()
874                .clone()
875                .ok_or(SyncError::LoopNotRunning)?;
876
877            // Removing a member rotates the store key, which only an encrypted home
878            // has. Refuse up front so a plaintext home never mutates the membership
879            // chain or re-wraps keys before the rotation fails.
880            if sync_loop.current_encryption().is_none() {
881                return Err(SyncError::NotEncryptedHome);
882            }
883
884            // Removing a member commits the cloud key rotation and then adopts the
885            // rotated key into this device's keyring and live cipher. The host records
886            // the returned fingerprint and that a key is stored in its own config; an
887            // adoption failure surfaces as its own membership variant naming the
888            // half-applied state and its remedies — and, structurally, this device
889            // seals nothing new for the cloud until one of those remedies adopts it
890            // (`pending_rotation`, shared with the sync loop this same store runs).
891            let outcome = sync_loop.remove_member(public_key_hex).await;
892
893            // Durably record the marker's state whether adoption succeeded or failed:
894            // on a failed adoption the rotation is committed on the cloud but this
895            // device is still sealing under the superseded generation, so a restart
896            // must remember the pause; on success the marker is already cleared and
897            // this deletes the durable record.
898            sync_loop.persist_pending_rotation().await?;
899
900            let fingerprint = outcome.map_err(SyncError::from)?;
901            Ok(fingerprint)
902        })
903    }
904
905    pub(crate) async fn create_circle(&self, name: &str) -> Result<crate::CircleId, SyncError> {
906        let sync_loop = self.sync_loop_handle().ok_or(SyncError::LoopNotRunning)?;
907        sync_loop.create_circle(name).await.map_err(SyncError::from)
908    }
909}
910
911#[cfg(test)]
912mod tests {
913    use super::*;
914
915    use crate::clock::SystemClock;
916    use crate::config::CloudProvider;
917    use crate::coven::StoreOpenGuard;
918    use crate::encryption::MasterKeyring;
919    use crate::keys::{test_keyring, KeyError, StoreKeys};
920    use crate::storage::cloud::test_utils::InMemoryCloudHome;
921    use crate::storage::cloud::CloudHomeJoinInfo;
922    use crate::store_dir::StoreDir;
923    use std::sync::Arc;
924
925    struct NoImmutableCopyHome;
926
927    #[async_trait::async_trait]
928    impl CloudHome for NoImmutableCopyHome {
929        async fn put_object(&self, _key: &str, _data: Vec<u8>) -> Result<(), CloudHomeError> {
930            panic!("incapable home must be rejected before I/O")
931        }
932
933        async fn open_multipart<'a>(
934            &'a self,
935            _key: &str,
936            _total_len: u64,
937        ) -> Result<crate::storage::cloud::BoxPartSink<'a>, CloudHomeError> {
938            panic!("incapable home must be rejected before I/O")
939        }
940
941        fn multipart_threshold(&self) -> u64 {
942            panic!("incapable home must be rejected before I/O")
943        }
944
945        async fn read(&self, _key: &str) -> Result<Vec<u8>, CloudHomeError> {
946            panic!("incapable home must be rejected before I/O")
947        }
948
949        async fn read_range(
950            &self,
951            _key: &str,
952            _start: u64,
953            _end: u64,
954        ) -> Result<Vec<u8>, CloudHomeError> {
955            panic!("incapable home must be rejected before I/O")
956        }
957
958        async fn list(&self, _prefix: &str) -> Result<Vec<String>, CloudHomeError> {
959            panic!("incapable home must be rejected before I/O")
960        }
961
962        async fn delete(&self, _key: &str) -> Result<(), CloudHomeError> {
963            panic!("incapable home must be rejected before I/O")
964        }
965
966        async fn exists(&self, _key: &str) -> Result<bool, CloudHomeError> {
967            panic!("incapable home must be rejected before I/O")
968        }
969
970        async fn set_access(
971            &self,
972            _desired: crate::storage::cloud::CloudAccessState,
973        ) -> Result<crate::storage::cloud::CloudAccessOutcome, CloudHomeError> {
974            panic!("incapable home must be rejected before I/O")
975        }
976    }
977
978    /// A custody that never has a master key established — `unlock` always
979    /// returns `None`. For tests exercising a locked/unestablished store, or
980    /// a browsable home where custody is never consulted at all.
981    struct NoKeyCustody;
982
983    impl MasterKeyCustody for NoKeyCustody {
984        fn unlock(&self) -> Result<Option<MasterKeyring>, KeyError> {
985            Ok(None)
986        }
987        fn persist(&self, _keyring: &MasterKeyring) -> Result<(), KeyError> {
988            Ok(())
989        }
990        fn forget(&self) -> Result<(), KeyError> {
991            Ok(())
992        }
993    }
994
995    /// The identity sibling of [`NoKeyCustody`]: `unlock` always returns
996    /// `None`, for tests exercising a store with no identity established.
997    struct NoIdentityCustody;
998
999    impl DeviceIdentityCustody for NoIdentityCustody {
1000        fn unlock(&self) -> Result<Option<crate::keys::UserKeypair>, KeyError> {
1001            Ok(None)
1002        }
1003        fn persist(&self, _keypair: &crate::keys::UserKeypair) -> Result<(), KeyError> {
1004            Ok(())
1005        }
1006        fn forget(&self) -> Result<(), KeyError> {
1007            Ok(())
1008        }
1009    }
1010
1011    /// A ready-to-use, already-established identity custody for tests whose
1012    /// focus is elsewhere (blob transitions, membership, restore-code
1013    /// generation) — seeded in-memory so it needs no keyring registration.
1014    fn established_identity_custody() -> Arc<dyn DeviceIdentityCustody> {
1015        crate::identity_custody::IdentityCustody::InMemory(crate::keys::UserKeypair::generate())
1016            .resolve("unused-store-id", &StoreDir::new("unused-store-dir"))
1017    }
1018
1019    #[tokio::test]
1020    async fn get_members_surfaces_malformed_cloud_credentials() {
1021        test_keyring::install();
1022        let tmp = tempfile::tempdir().expect("temp dir");
1023        let store_dir = StoreDir::new(tmp.path());
1024        let store_id = "sync-enabled-malformed-credentials";
1025        let key_service = StoreKeys::new(store_id.to_string());
1026        key_service
1027            .cloud_home_credentials_entry_for_test()
1028            .expect("create credentials entry")
1029            .set_password("{")
1030            .expect("write malformed credentials");
1031        let join_info = CloudHomeJoinInfo::S3 {
1032            bucket: "bucket".to_string(),
1033            region: "region".to_string(),
1034            endpoint: None,
1035            access_key: "access".to_string(),
1036            secret_key: "secret".to_string(),
1037            key_prefix: None,
1038        };
1039        let config = crate::sync::join::build_config(
1040            store_id,
1041            "device",
1042            &store_dir,
1043            "store",
1044            &join_info,
1045            &CloudCipher::Plaintext,
1046            None,
1047        );
1048        let manager = SyncManager::new(
1049            Arc::new(move || config.clone()),
1050            key_service,
1051            Arc::new(NoKeyCustody),
1052            established_identity_custody(),
1053            crate::sync::test_helpers::open_test_db(),
1054            Arc::new(SystemClock),
1055            None,
1056            None,
1057            StoreOpenGuard::acquire_for_test(&store_dir),
1058            tokio::sync::watch::channel(SyncLoopStatus::Offline).0,
1059        );
1060
1061        let error = manager
1062            .get_members()
1063            .await
1064            .expect_err("malformed stored credentials must fail");
1065        // The typed CloudHomeError survives up through StorageSetup to the public
1066        // SyncError surface — not flattened into a string — so its retryability
1067        // verdict is still readable: malformed credentials are a configuration
1068        // fault the user must fix, not a transient retry.
1069        let SyncError::StorageSetup(StorageSetupError::CloudHome(cloud_home_error)) = &error else {
1070            panic!("expected StorageSetup(CloudHome(_)), got {error:?}");
1071        };
1072        assert!(matches!(cloud_home_error, CloudHomeError::Configuration(_)));
1073        assert!(!cloud_home_error.is_retryable());
1074        assert!(error
1075            .to_string()
1076            .contains("malformed cloud home credentials JSON"));
1077    }
1078
1079    #[tokio::test]
1080    async fn start_sync_rejects_an_opaque_home_without_a_master_key() {
1081        test_keyring::install();
1082        let (_tmp, store_dir) = crate::sync::test_helpers::temp_store_dir();
1083        let open_guard = StoreOpenGuard::acquire_for_test(&store_dir);
1084        let mut config = Config::with_defaults(
1085            "lib-opaque-no-encryption".to_string(),
1086            "test-device".to_string(),
1087            store_dir,
1088            "Test Store".to_string(),
1089        );
1090        // Opaque storage (the default) with a configured provider but no
1091        // established master key is a locked-store contradiction — custody's
1092        // `unlock` returns `None`.
1093        config.cloud_home.provider = Some(CloudProvider::S3);
1094        let manager = SyncManager::new(
1095            Arc::new(move || config.clone()),
1096            StoreKeys::new("lib-opaque-no-encryption".to_string()),
1097            Arc::new(NoKeyCustody),
1098            established_identity_custody(),
1099            crate::sync::test_helpers::open_test_db(),
1100            Arc::new(SystemClock),
1101            None,
1102            None,
1103            open_guard,
1104            tokio::sync::watch::channel(SyncLoopStatus::Offline).0,
1105        );
1106
1107        let error = manager
1108            .start_sync()
1109            .await
1110            .expect_err("opaque home without an established master key must fail");
1111        assert!(
1112            matches!(error, SyncError::MasterKeyNotEstablished),
1113            "expected MasterKeyNotEstablished, got {error:?}"
1114        );
1115    }
1116
1117    #[tokio::test]
1118    async fn start_sync_refuses_a_known_unsupported_serial_provider_before_provider_access() {
1119        let (_tmp, store_dir) = crate::sync::test_helpers::temp_store_dir();
1120        let open_guard = StoreOpenGuard::acquire_for_test(&store_dir);
1121        let mut config = Config::with_defaults(
1122            "serial-google-drive".to_string(),
1123            "test-device".to_string(),
1124            store_dir,
1125            "Serial Store".to_string(),
1126        );
1127        config.cloud_home.provider = Some(CloudProvider::GoogleDrive);
1128        let manager = SyncManager::new(
1129            Arc::new(move || config.clone()),
1130            StoreKeys::new("serial-google-drive".to_string()),
1131            Arc::new(NoKeyCustody),
1132            Arc::new(NoIdentityCustody),
1133            crate::sync::test_helpers::open_serial_test_db(),
1134            Arc::new(SystemClock),
1135            None,
1136            None,
1137            open_guard,
1138            tokio::sync::watch::channel(SyncLoopStatus::Offline).0,
1139        );
1140
1141        let error = manager
1142            .start_sync()
1143            .await
1144            .expect_err("Google Drive cannot coordinate a Serial Store");
1145        assert!(matches!(
1146            error,
1147            SyncError::StorageSetup(StorageSetupError::SerialCoordinationUnavailable {
1148                provider: CloudProvider::GoogleDrive,
1149            })
1150        ));
1151        assert!(manager.sync_loop_handle().is_none());
1152        assert!(manager.cloud_home().is_none());
1153    }
1154
1155    #[tokio::test]
1156    async fn immutable_copy_admission_refuses_before_stopping_the_active_loop() {
1157        let (_tmp, store_dir) = crate::sync::test_helpers::temp_store_dir();
1158        let open_guard = StoreOpenGuard::acquire_for_test(&store_dir);
1159        let config = Arc::new(RwLock::new(Config::with_defaults(
1160            "immutable-admission-before-stop".to_string(),
1161            "test-device".to_string(),
1162            store_dir,
1163            "Blob Store".to_string(),
1164        )));
1165        let db = crate::sync::test_helpers::open_test_db_with_blob(crate::BlobDecl::new(
1166            "photos",
1167            crate::Provenance::HostProvided,
1168            crate::CacheFill::CacheLazy,
1169        ));
1170        let manager = SyncManager::new(
1171            {
1172                let config = config.clone();
1173                Arc::new(move || config.read().expect("read config").clone())
1174            },
1175            StoreKeys::new("immutable-admission-before-stop".to_string()),
1176            Arc::new(NoKeyCustody),
1177            established_identity_custody(),
1178            db,
1179            Arc::new(SystemClock),
1180            None,
1181            None,
1182            open_guard,
1183            tokio::sync::watch::channel(SyncLoopStatus::Offline).0,
1184        );
1185        manager
1186            .start_sync_with_home(Arc::new(InMemoryCloudHome::new()), CloudCipher::Plaintext)
1187            .await
1188            .expect("install active loop");
1189        let active_loop = manager.sync_loop_handle().expect("active loop");
1190
1191        {
1192            let mut config = config.write().expect("write config");
1193            config.cloud_home.provider = Some(CloudProvider::S3);
1194            config.cloud_home.s3_endpoint = Some("https://objects.example".to_string());
1195            config.cloud_home.s3_exact_slots = None;
1196        }
1197        let error = manager
1198            .start_sync()
1199            .await
1200            .expect_err("unsupported immutable-copy provider is refused");
1201
1202        assert!(matches!(
1203            error,
1204            SyncError::StorageSetup(StorageSetupError::ExactSlotsUnavailable {
1205                provider: CloudProvider::S3,
1206            })
1207        ));
1208        assert!(active_loop.is_running());
1209        assert!(manager.cloud_home().is_some());
1210
1211        let error = manager
1212            .start_sync_with_home(Arc::new(NoImmutableCopyHome), CloudCipher::Plaintext)
1213            .await
1214            .expect_err("injected home without immutable-copy storage is refused");
1215        assert!(matches!(
1216            error,
1217            SyncError::StorageSetup(StorageSetupError::ExactSlotsUnavailable {
1218                provider: CloudProvider::S3,
1219            })
1220        ));
1221        assert!(active_loop.is_running());
1222        assert!(manager.cloud_home().is_some());
1223    }
1224
1225    #[tokio::test]
1226    async fn start_sync_with_home_stops_the_previous_loop_before_replacement() {
1227        test_keyring::install();
1228
1229        let (_tmp, store_dir) = crate::sync::test_helpers::temp_store_dir();
1230        let open_guard = StoreOpenGuard::acquire_for_test(&store_dir);
1231        let config = Config::with_defaults(
1232            "lib-manager-restart".to_string(),
1233            "test-device".to_string(),
1234            store_dir,
1235            "Test Store".to_string(),
1236        );
1237        let manager = SyncManager::new(
1238            Arc::new(move || config.clone()),
1239            StoreKeys::new("lib-manager-restart".to_string()),
1240            Arc::new(NoKeyCustody),
1241            established_identity_custody(),
1242            crate::sync::test_helpers::open_test_db(),
1243            Arc::new(SystemClock),
1244            None,
1245            None,
1246            open_guard,
1247            tokio::sync::watch::channel(SyncLoopStatus::Offline).0,
1248        );
1249
1250        let home = Arc::new(InMemoryCloudHome::new());
1251        manager
1252            .start_sync_with_home(home.clone(), CloudCipher::Plaintext)
1253            .await
1254            .expect("first test home starts");
1255        let first_loop = manager
1256            .sync_loop_handle()
1257            .expect("first loop handle installed");
1258        assert!(first_loop.is_running(), "first loop starts running");
1259
1260        manager
1261            .start_sync_with_home(home, CloudCipher::Plaintext)
1262            .await
1263            .expect("replacement test home starts");
1264        let replacement_loop = manager
1265            .sync_loop_handle()
1266            .expect("replacement loop handle installed");
1267
1268        assert!(
1269            !first_loop.is_running(),
1270            "starting sync again stops the previous loop before replacement",
1271        );
1272        assert!(
1273            replacement_loop.is_running(),
1274            "replacement loop remains running",
1275        );
1276    }
1277
1278    #[tokio::test]
1279    async fn failed_restart_leaves_no_stale_cloud_home() {
1280        test_keyring::install();
1281
1282        let (_tmp, store_dir) = crate::sync::test_helpers::temp_store_dir();
1283        let open_guard = StoreOpenGuard::acquire_for_test(&store_dir);
1284        let config = Arc::new(RwLock::new(Config::with_defaults(
1285            "lib-manager-failed-restart".to_string(),
1286            "test-device".to_string(),
1287            store_dir,
1288            "Test Store".to_string(),
1289        )));
1290        let manager = SyncManager::new(
1291            {
1292                let config = config.clone();
1293                Arc::new(move || config.read().unwrap().clone())
1294            },
1295            StoreKeys::new("lib-manager-failed-restart".to_string()),
1296            // An established master key so the opaque default storage passes
1297            // the cipher precondition and the restart fails at the home build
1298            // itself.
1299            crate::custody::KeyCustody::InMemory(MasterKeyring::generate()).resolve(
1300                "lib-manager-failed-restart",
1301                &StoreDir::new("unused-store-dir"),
1302            ),
1303            established_identity_custody(),
1304            crate::sync::test_helpers::open_test_db(),
1305            Arc::new(SystemClock),
1306            None,
1307            None,
1308            open_guard,
1309            tokio::sync::watch::channel(SyncLoopStatus::Offline).0,
1310        );
1311
1312        manager
1313            .start_sync_with_home(Arc::new(InMemoryCloudHome::new()), CloudCipher::Plaintext)
1314            .await
1315            .expect("injected home starts");
1316        assert!(manager.cloud_home().is_some(), "injected home is installed");
1317
1318        config.write().unwrap().cloud_home.provider = Some(CloudProvider::S3);
1319        let error = manager
1320            .start_sync()
1321            .await
1322            .expect_err("invalid configured provider fails restart");
1323        assert!(
1324            error.to_string().contains("failed to build cloud home"),
1325            "restart failure surfaces the provider setup error: {error}",
1326        );
1327        assert!(
1328            manager.sync_loop_handle().is_none(),
1329            "failed restart leaves no loop installed",
1330        );
1331        assert!(
1332            manager.cloud_home().is_none(),
1333            "failed restart must not leave the previous cloud home installed",
1334        );
1335    }
1336
1337    /// The `NoDeviceIdentity` sibling of
1338    /// `start_sync_rejects_an_opaque_home_without_a_master_key`: connecting
1339    /// with a configured home but no device identity established must fail
1340    /// typed, with nothing installed — never silently mint one. Browsable
1341    /// storage so the master-key precondition is out of the way and this
1342    /// isolates the identity check.
1343    #[tokio::test]
1344    async fn start_sync_rejects_a_connect_with_no_device_identity_established() {
1345        test_keyring::install();
1346
1347        let (_tmp, store_dir) = crate::sync::test_helpers::temp_store_dir();
1348        let open_guard = StoreOpenGuard::acquire_for_test(&store_dir);
1349        let store_id = "lib-no-device-identity".to_string();
1350        let key_service = StoreKeys::new(store_id.clone());
1351        key_service
1352            .set_cloud_home_credentials(&crate::keys::CloudHomeCredentials::S3 {
1353                access_key: "ak".to_string(),
1354                secret_key: "sk".to_string(),
1355            })
1356            .expect("seed S3 credentials");
1357
1358        let mut config = Config::with_defaults(
1359            store_id.clone(),
1360            "test-device".to_string(),
1361            store_dir,
1362            "Test Store".to_string(),
1363        );
1364        config.cloud_home.provider = Some(CloudProvider::S3);
1365        config.cloud_home.storage = HomeStorage::Browsable;
1366        config.cloud_home.s3_bucket = Some("bucket".to_string());
1367        config.cloud_home.s3_region = Some("us-east-1".to_string());
1368
1369        let manager = SyncManager::new(
1370            Arc::new(move || config.clone()),
1371            key_service,
1372            Arc::new(NoKeyCustody),
1373            Arc::new(NoIdentityCustody),
1374            crate::sync::test_helpers::open_test_db(),
1375            Arc::new(SystemClock),
1376            None,
1377            None,
1378            open_guard,
1379            tokio::sync::watch::channel(SyncLoopStatus::Offline).0,
1380        );
1381
1382        let error = manager
1383            .start_sync()
1384            .await
1385            .expect_err("no device identity established must fail the connect");
1386        assert!(
1387            matches!(error, SyncError::Key(KeyError::NoDeviceIdentity)),
1388            "got {error:?}"
1389        );
1390        assert!(
1391            manager.sync_loop_handle().is_none(),
1392            "a failed connect installs no loop",
1393        );
1394        assert!(
1395            manager.cloud_home().is_none(),
1396            "a failed connect installs no cloud home",
1397        );
1398    }
1399
1400    #[tokio::test]
1401    async fn browsable_test_home_with_a_foreign_founder_installs_nothing() {
1402        test_keyring::install();
1403
1404        let (_tmp, store_dir) = crate::sync::test_helpers::temp_store_dir();
1405        let store_id = "lib-foreign-browsable-founder";
1406        let mut config = Config::with_defaults(
1407            store_id.to_string(),
1408            "test-device".to_string(),
1409            store_dir.clone(),
1410            "Test Store".to_string(),
1411        );
1412        config.cloud_home.storage = HomeStorage::Browsable;
1413        let home = Arc::new(InMemoryCloudHome::new());
1414        let attacker = crate::keys::UserKeypair::generate();
1415        let attacker_storage = CloudSyncStorage::new(
1416            home.clone(),
1417            CloudCipher::Plaintext,
1418            BlobPathScheme::Plain,
1419            store_id,
1420            attacker.clone(),
1421        )
1422        .expect("build attacker storage");
1423        let attacker_db = crate::sync::test_helpers::open_test_db();
1424        crate::sync::store_protocol_root::create_store(
1425            &attacker_db,
1426            &attacker_storage,
1427            store_id,
1428            &attacker,
1429        )
1430        .await
1431        .expect("publish attacker Store root");
1432
1433        let victim = crate::keys::UserKeypair::generate();
1434        let db = crate::sync::test_helpers::open_test_db();
1435        let manager = SyncManager::new(
1436            Arc::new(move || config.clone()),
1437            StoreKeys::new(store_id.to_string()),
1438            Arc::new(NoKeyCustody),
1439            crate::identity_custody::IdentityCustody::InMemory(victim)
1440                .resolve(store_id, &store_dir),
1441            db.clone(),
1442            Arc::new(SystemClock),
1443            None,
1444            None,
1445            StoreOpenGuard::acquire_for_test(&store_dir),
1446            tokio::sync::watch::channel(SyncLoopStatus::Offline).0,
1447        );
1448
1449        let error = manager
1450            .start_sync_with_home(home, CloudCipher::Plaintext)
1451            .await
1452            .expect_err("foreign founder must prevent sync startup");
1453        assert!(
1454            matches!(error, SyncError::Init(InitSyncError::StoreProtocolRoot(_))),
1455            "unexpected startup error: {error:?}",
1456        );
1457        assert!(manager.sync_loop_handle().is_none());
1458        assert!(manager.cloud_home().is_none());
1459        assert_eq!(
1460            db.get_protocol_state(crate::sync::membership_ops::OWNER_PUBKEY_STATE_KEY)
1461                .await
1462                .unwrap(),
1463            None,
1464        );
1465    }
1466
1467    /// Key material a connect resolves from custody is never cached across
1468    /// connects — `start_sync` re-derives the cipher fresh every call via
1469    /// `resolve_cipher`, this manager's single
1470    /// custody→cipher decision. Persists key A, resolves, swaps what the SAME
1471    /// custody instance serves to key B (a rotation outside any manager call
1472    /// — the way a host's own key-rotation flow would), and resolves again:
1473    /// the second resolution reflects B, not a value cached from the first.
1474    ///
1475    /// This drives `resolve_cipher` directly rather than a full
1476    /// connect/disconnect/reconnect through an opaque home: an opaque store's
1477    /// membership chain is founded and pinned to the local device on first
1478    /// connect, so swapping its master key outright (rather than through the
1479    /// real in-place rotation `remove_member` performs, which also re-wraps
1480    /// existing membership content) would desync a live home — an unrelated
1481    /// concern to what this test pins. `resolve_cipher` is the exact
1482    /// mechanism `start_sync` and the custody-resolving test-home connect
1483    /// path share, so calling it twice with custody mutated in between is the
1484    /// real unit behind "reconnect uses new material," without wading into
1485    /// membership bootstrap.
1486    #[test]
1487    fn resolve_cipher_never_caches_reflects_whatever_custody_now_serves() {
1488        let (_tmp, store_dir) = crate::sync::test_helpers::temp_store_dir();
1489        let store_id = "lib-resolve-cipher-fresh";
1490        let custody = crate::custody::KeyCustody::Keyring.resolve(store_id, &store_dir);
1491        let key_a = MasterKeyring::generate();
1492        custody.persist(&key_a).expect("establish key A");
1493
1494        let config = Config::with_defaults(
1495            store_id.to_string(),
1496            "test-device".to_string(),
1497            store_dir.clone(),
1498            "Test Store".to_string(),
1499        );
1500        let manager = SyncManager::new(
1501            Arc::new(move || config.clone()),
1502            StoreKeys::new(store_id.to_string()),
1503            custody.clone(),
1504            established_identity_custody(),
1505            crate::sync::test_helpers::open_test_db(),
1506            Arc::new(SystemClock),
1507            None,
1508            None,
1509            StoreOpenGuard::acquire_for_test(&store_dir),
1510            tokio::sync::watch::channel(SyncLoopStatus::Offline).0,
1511        );
1512
1513        let fingerprint_a = match manager
1514            .resolve_cipher(crate::config::HomeStorage::Opaque)
1515            .expect("resolve the cipher custody serves for key A")
1516        {
1517            CloudCipher::Encrypted(enc) => enc.fingerprint(),
1518            CloudCipher::Plaintext => panic!("opaque storage must resolve an encrypted cipher"),
1519        };
1520        assert_eq!(fingerprint_a, key_a.fingerprint());
1521
1522        // Swap what the SAME custody instance serves — outside any manager
1523        // call, the way a host's own key-rotation flow would.
1524        let key_b = MasterKeyring::generate();
1525        custody
1526            .persist(&key_b)
1527            .expect("rotate custody's served key to B");
1528
1529        let fingerprint_b = match manager
1530            .resolve_cipher(crate::config::HomeStorage::Opaque)
1531            .expect("resolve the cipher custody serves for key B")
1532        {
1533            CloudCipher::Encrypted(enc) => enc.fingerprint(),
1534            CloudCipher::Plaintext => panic!("opaque storage must resolve an encrypted cipher"),
1535        };
1536        assert_eq!(
1537            fingerprint_b,
1538            key_b.fingerprint(),
1539            "the second resolution must reflect key B, not a value cached from the first call",
1540        );
1541        assert_ne!(
1542            fingerprint_a, fingerprint_b,
1543            "the two resolutions must differ — custody actually served different material",
1544        );
1545    }
1546}