Skip to main content

coven/
coven.rs

1//! Top-level API: open one handle and drive rows, blobs, sync, and
2//! membership through it.
3
4use std::num::NonZeroUsize;
5use std::path::{Path, PathBuf};
6use std::sync::Arc;
7
8use rusqlite::Connection;
9use tracing::{debug, warn};
10
11use crate::blob::local_files::LocalBlobError;
12use crate::blob::{BlobRef, BlobTransitionObserver};
13use crate::clock::{ClockRef, SystemClock};
14use crate::config::{Config, HomeStorage};
15use crate::custody::KeyCustody;
16use crate::database::{Database, DbError, OpenError};
17use crate::handle::CovenHandle;
18use crate::identity_custody::IdentityCustody;
19use crate::keys::StoreKeys;
20use crate::migration::{Migration, MigrationError};
21use crate::store_dir::PathTokenError;
22use crate::sync::hlc::UpdatedAtStamper;
23use crate::sync::session::SyncedTable;
24use crate::sync::sync_manager::ConfigProvider;
25use coven_core::{WriteId, WritePolicy, WriteReceipt};
26
27pub type CovenResult<T> = Result<T, CovenError>;
28
29const LOCAL_STAGE_MARKER: &str = ".coven-stage-";
30
31#[derive(Debug, thiserror::Error)]
32pub enum CovenError {
33    #[error("database error: {0}")]
34    Database(#[from] DbError),
35    #[error("migration error: {0}")]
36    Migration(MigrationError),
37    #[error("sqlite error: {0}")]
38    Sqlite(#[from] rusqlite::Error),
39    #[error("blob error: {0}")]
40    Blob(String),
41    #[error("unsafe blob path: {0}")]
42    UnsafeBlobPath(#[from] PathTokenError),
43    #[error("malformed local path: {0}")]
44    MalformedPath(String),
45    #[error("the write SQL closure panicked")]
46    WriteClosurePanicked,
47    #[error("synced_tables must be set before opening a coven store")]
48    MissingSyncedTables,
49    #[error("migrations must be set before opening a coven store")]
50    MissingMigrations,
51    #[error("write_policy must be set before opening a coven store")]
52    MissingWritePolicy,
53    #[error("Serial branch resolution failed: {0}")]
54    SerialResolution(String),
55    #[error("candidate resolution failed: {0}")]
56    CandidateResolution(String),
57    #[error("blob_tombstone_grace must be a positive duration")]
58    InvalidBlobTombstoneGrace,
59    #[error("browsable cloud storage cannot be used with scoped table {table:?}")]
60    BrowsableStorageWithScopedTable { table: String },
61    #[error("blob {namespace}/{id} is still referenced by a row after the write")]
62    BlobStillReferenced { namespace: String, id: String },
63    #[error("blob {namespace}/{id} is already referenced by a row")]
64    BlobAlreadyReferenced { namespace: String, id: String },
65    #[error("blob {namespace}/{id} is owned by an unpublished write")]
66    BlobOwnedByPendingWrite { namespace: String, id: String },
67    #[error("store is already open: {}", store_dir.display())]
68    AlreadyOpen { store_dir: PathBuf },
69    #[error("I/O error: {0}")]
70    Io(#[from] std::io::Error),
71}
72
73impl From<OpenError> for CovenError {
74    fn from(value: OpenError) -> Self {
75        match value {
76            OpenError::Migration(e) => CovenError::Migration(e),
77            OpenError::Db(e) => CovenError::Database(e),
78        }
79    }
80}
81
82impl From<LocalBlobError> for CovenError {
83    fn from(value: LocalBlobError) -> Self {
84        CovenError::Blob(value.to_string())
85    }
86}
87
88#[derive(Clone)]
89pub struct CovenConfig(ConfigProvider);
90
91impl CovenConfig {
92    fn current(&self) -> Config {
93        (self.0)()
94    }
95
96    fn provider(&self) -> ConfigProvider {
97        self.0.clone()
98    }
99}
100
101impl From<Config> for CovenConfig {
102    fn from(value: Config) -> Self {
103        let config = value;
104        Self(Arc::new(move || config.clone()))
105    }
106}
107
108impl<F> From<F> for CovenConfig
109where
110    F: Fn() -> Config + Send + Sync + 'static,
111{
112    fn from(value: F) -> Self {
113        Self(Arc::new(value))
114    }
115}
116
117pub struct Coven;
118
119impl Coven {
120    pub fn builder(config: impl Into<CovenConfig>) -> CovenBuilder {
121        let config = config.into();
122        let current = config.current();
123        CovenBuilder {
124            config,
125            synced_tables: None,
126            migrations: None,
127            write_policy: None,
128            blob_tombstone_grace: crate::blob::delete::BLOB_TOMBSTONE_GRACE,
129            max_concurrent_uploads: NonZeroUsize::MIN,
130            max_concurrent_downloads: NonZeroUsize::MIN,
131            clock: Arc::new(SystemClock),
132            key_service: StoreKeys::new(current.store_id),
133            key_custody: KeyCustody::Keyring,
134            identity_custody: IdentityCustody::Keyring,
135            cloudkit_ops: None,
136            observer: None,
137        }
138    }
139}
140
141pub struct CovenBuilder {
142    config: CovenConfig,
143    synced_tables: Option<Vec<SyncedTable>>,
144    migrations: Option<Vec<Migration>>,
145    write_policy: Option<WritePolicy>,
146    blob_tombstone_grace: chrono::Duration,
147    max_concurrent_uploads: NonZeroUsize,
148    max_concurrent_downloads: NonZeroUsize,
149    clock: ClockRef,
150    key_service: StoreKeys,
151    key_custody: KeyCustody,
152    identity_custody: IdentityCustody,
153    cloudkit_ops: Option<Arc<dyn crate::storage::cloud::cloudkit::CloudKitOps>>,
154    observer: Option<Arc<dyn BlobTransitionObserver>>,
155}
156
157/// The single-writer store lock: an exclusive advisory lock on
158/// `<store>/.coven-lock`, held for the life of a full [`open`](CovenBuilder::open)
159/// handle (and its running sync loop). A second full open of the same store is
160/// refused with [`CovenError::AlreadyOpen`] while the lock is held — the invariant
161/// that keeps two writers from racing the same db and blob store.
162///
163/// # Read-only opens take no lock
164///
165/// [`open_read_only`](CovenBuilder::open_read_only) deliberately does **not** touch
166/// this lock. The lock is exclusive, so a shared lock on the same file would block
167/// against a writer that already holds it (and vice versa) — a reader could never
168/// coexist with the writer it exists to read alongside. But a read-only open needs
169/// no lock at all: the lock guards against a second *writer*, and a read-only handle
170/// holds a `SQLITE_OPEN_READONLY` connection that cannot write. So a read-only open
171/// skips the guard entirely. Cross-process safety comes from WAL mode (a reader sees
172/// committed rows while the writer commits more), not from this lock; the blob cache
173/// a reader may populate is per-device scratch written atomically (temp + rename), so
174/// a reader and the writer touching the same cache file never tear it. This lets one
175/// writer and any number of read-only readers coexist on one store.
176pub(crate) struct StoreOpenGuard {
177    _file: std::fs::File,
178}
179
180impl StoreOpenGuard {
181    /// Acquire the guard for a test, panicking on refusal.
182    #[cfg(test)]
183    pub(crate) fn acquire_for_test(store_dir: &crate::store_dir::StoreDir) -> std::sync::Arc<Self> {
184        std::sync::Arc::new(Self::acquire(store_dir).expect("acquire store open guard"))
185    }
186
187    pub(crate) fn acquire(store_dir: &crate::store_dir::StoreDir) -> CovenResult<Self> {
188        let db_path = store_dir.db_path();
189        let Some(dir) = db_path.parent() else {
190            return Err(CovenError::MalformedPath(format!(
191                "store database path has no parent: {}",
192                db_path.display()
193            )));
194        };
195        std::fs::create_dir_all(dir)?;
196        let file = std::fs::OpenOptions::new()
197            .read(true)
198            .write(true)
199            .create(true)
200            .truncate(false)
201            .open(dir.join(".coven-lock"))?;
202        match file.try_lock() {
203            Ok(()) => Ok(Self { _file: file }),
204            Err(std::fs::TryLockError::WouldBlock) => Err(CovenError::AlreadyOpen {
205                store_dir: dir.to_path_buf(),
206            }),
207            Err(std::fs::TryLockError::Error(error)) => Err(CovenError::Io(error)),
208        }
209    }
210}
211
212impl CovenBuilder {
213    pub fn write_policy(mut self, policy: WritePolicy) -> Self {
214        self.write_policy = Some(policy);
215        self
216    }
217
218    pub fn synced_tables(mut self, tables: Vec<SyncedTable>) -> Self {
219        self.synced_tables = Some(tables);
220        self
221    }
222
223    /// How long a deleted blob is kept after its tombstone is written before the
224    /// tombstone GC erases it: the cross-device convergence window. Defaults to
225    /// [`crate::blob::delete::BLOB_TOMBSTONE_GRACE`]. Must be positive — a
226    /// zero-or-negative grace is refused at [`open`](Self::open), since it would
227    /// let the GC erase a blob a lagging peer still references.
228    pub fn blob_tombstone_grace(mut self, grace: chrono::Duration) -> Self {
229        self.blob_tombstone_grace = grace;
230        self
231    }
232
233    /// How many blob uploads the sync cycle's upload drain runs at once. Defaults
234    /// to one (fully serial). A [`NonZeroUsize`] so a zero — which would leave the
235    /// drain admitting nothing and never completing — cannot be set.
236    pub fn max_concurrent_uploads(mut self, n: NonZeroUsize) -> Self {
237        self.max_concurrent_uploads = n;
238        self
239    }
240
241    /// How many blob downloads a [`pin`](CovenHandle::pin) call fetches at once.
242    /// Defaults to one (fully serial). A [`NonZeroUsize`] so a zero — which would
243    /// leave the pin loop admitting nothing and never completing — cannot be set.
244    pub fn max_concurrent_downloads(mut self, n: NonZeroUsize) -> Self {
245        self.max_concurrent_downloads = n;
246        self
247    }
248
249    /// The host's synced-schema migration ladder, applied over `PRAGMA
250    /// user_version` at open. The top version is the wire `schema_version` every
251    /// changeset is stamped with.
252    pub fn migrations(mut self, migrations: Vec<Migration>) -> Self {
253        self.migrations = Some(migrations);
254        self
255    }
256
257    pub fn clock(mut self, clock: ClockRef) -> Self {
258        self.clock = clock;
259        self
260    }
261
262    pub fn key_service(mut self, key_service: StoreKeys) -> Self {
263        self.key_service = key_service;
264        self
265    }
266
267    /// How the store's master key is protected: the OS keyring (the
268    /// default), a passphrase-wrapped file, an in-memory session value, or a
269    /// host's own [`MasterKeyCustody`](crate::MasterKeyCustody) implementation.
270    /// coven builds every cipher internally from what this custody supplies —
271    /// the host never touches a crypto type.
272    pub fn key_custody(mut self, custody: KeyCustody) -> Self {
273        self.key_custody = custody;
274        self
275    }
276
277    /// How this store's device-signing identity is protected: the OS keyring
278    /// (the default), a passphrase-wrapped file, an in-memory session value,
279    /// or a host's own
280    /// [`DeviceIdentityCustody`](crate::DeviceIdentityCustody) implementation.
281    /// Selected next to [`key_custody`](Self::key_custody) — the identity is
282    /// scoped to this store, established as part of creating, joining, or
283    /// restoring it (see [`CovenHandle::initialize_identity`]).
284    pub fn identity_custody(mut self, custody: IdentityCustody) -> Self {
285        self.identity_custody = custody;
286        self
287    }
288
289    pub fn cloudkit_ops(
290        mut self,
291        ops: Arc<dyn crate::storage::cloud::cloudkit::CloudKitOps>,
292    ) -> Self {
293        self.cloudkit_ops = Some(ops);
294        self
295    }
296
297    pub fn apply_cloudkit_ops(
298        mut self,
299        ops: Option<Arc<dyn crate::storage::cloud::cloudkit::CloudKitOps>>,
300    ) -> Self {
301        self.cloudkit_ops = ops;
302        self
303    }
304
305    pub fn observer(mut self, observer: Arc<dyn BlobTransitionObserver>) -> Self {
306        self.observer = Some(observer);
307        self
308    }
309
310    /// Open the store, returning the [`CovenHandle`].
311    ///
312    /// Opening performs no keyring interaction: it opens the database, runs
313    /// migrations, and resolves the master-key custody selection to a value
314    /// (constructing the trait object, never calling its `unlock`) — a locked
315    /// agent (no OS keyring session, no established master key or device
316    /// identity) can `open()` a store and use it fully for rows and Local
317    /// blobs. The first read of any key happens lazily, at the specific call
318    /// that needs it ([`CovenHandle::connect_sync`],
319    /// [`CovenHandle::master_key_fingerprint`], and similar).
320    pub fn open(self) -> CovenResult<CovenHandle> {
321        let config = self.config.current();
322        let tables = self.synced_tables.ok_or(CovenError::MissingSyncedTables)?;
323        let migrations = self.migrations.ok_or(CovenError::MissingMigrations)?;
324        let write_policy = self.write_policy.ok_or(CovenError::MissingWritePolicy)?;
325        validate_storage_scope(&config, &tables)?;
326        if self.blob_tombstone_grace <= chrono::Duration::zero() {
327            return Err(CovenError::InvalidBlobTombstoneGrace);
328        }
329        let db_path = config.store_dir.db_path();
330        let provider = self.config.provider();
331        let store_dir = config.store_dir.clone();
332        let transfer_limits = crate::blob::TransferLimits {
333            uploads: self.max_concurrent_uploads,
334            downloads: self.max_concurrent_downloads,
335        };
336        let open_guard = Arc::new(StoreOpenGuard::acquire(&store_dir)?);
337        remove_orphaned_local_blob_temps(&store_dir, std::time::SystemTime::now())?;
338        let (db, stamper) = Database::open(
339            &db_path,
340            tables.clone(),
341            self.blob_tombstone_grace,
342            transfer_limits,
343            write_policy,
344            config.device_id.clone(),
345            &migrations,
346        )?;
347        // A second, read-only connection on the same WAL database, opened after the
348        // writer completed its migrations so the schema exists. It backs
349        // [`CovenHandle::sql_read`]: a pure read runs here on its own connection
350        // thread, concurrent with the writer rather than queued behind it, and
351        // attaches no changeset session. Opening it fails `open()` loudly — there is
352        // no full handle without its read path.
353        let read_db = Database::open_read_only(
354            &db_path,
355            tables,
356            self.blob_tombstone_grace,
357            transfer_limits,
358            write_policy,
359            config.device_id.clone(),
360            &migrations,
361        )?;
362        let key_custody = self.key_custody.resolve(&config.store_id, &store_dir);
363        let identity_custody = self.identity_custody.resolve(&config.store_id, &store_dir);
364        Ok(CovenHandle::new(
365            db,
366            read_db,
367            stamper,
368            store_dir,
369            provider,
370            self.key_service,
371            key_custody,
372            identity_custody,
373            self.clock,
374            self.cloudkit_ops,
375            self.observer,
376            open_guard,
377        ))
378    }
379
380    /// Open the store read-only for a same-store secondary reader: a separate
381    /// process (or a second handle) that must read rows and blobs while another
382    /// handle holds the full [`open`](Self::open). Returns a [`CovenReadHandle`],
383    /// whose surface is reads only — SQL queries and blob reads — with no write,
384    /// sync, migration, or stamp API by construction.
385    ///
386    /// Unlike [`open`](Self::open) this takes no store lock (see
387    /// [`StoreOpenGuard`]): it succeeds while a writer holds the exclusive lock,
388    /// and any number of read-only opens coexist. It opens a `SQLITE_OPEN_READONLY`
389    /// connection against the schema on disk, running no migration ladder — but it
390    /// refuses a db a newer binary migrated past what this binary supports
391    /// ([`CovenError::Migration`] with [`MigrationError::SchemaTooNew`]), the same
392    /// policy the writer enforces. It runs no orphan-temp cleanup either (that is a
393    /// write the lock-holding writer owns).
394    ///
395    /// Cross-process reads are safe because the writer opens the db in WAL mode; a
396    /// blob read that misses locally fetches from the cloud into the per-device
397    /// cache (files written atomically), which is device scratch and touches no
398    /// synced state.
399    pub fn open_read_only(self) -> CovenResult<crate::read_handle::CovenReadHandle> {
400        let config = self.config.current();
401        let tables = self.synced_tables.ok_or(CovenError::MissingSyncedTables)?;
402        let migrations = self.migrations.ok_or(CovenError::MissingMigrations)?;
403        let write_policy = self.write_policy.ok_or(CovenError::MissingWritePolicy)?;
404        validate_storage_scope(&config, &tables)?;
405        let db_path = config.store_dir.db_path();
406        let provider = self.config.provider();
407        let store_dir = config.store_dir.clone();
408        // No StoreOpenGuard and no orphan-temp cleanup: both are writer concerns
409        // (see StoreOpenGuard). A reader must not take the exclusive lock the
410        // writer holds, nor write the filesystem the writer owns.
411        let db = Database::open_read_only(
412            &db_path,
413            tables,
414            self.blob_tombstone_grace,
415            crate::blob::TransferLimits {
416                uploads: self.max_concurrent_uploads,
417                downloads: self.max_concurrent_downloads,
418            },
419            write_policy,
420            config.device_id.clone(),
421            &migrations,
422        )?;
423        let key_custody = self.key_custody.resolve(&config.store_id, &store_dir);
424        let identity_custody = self.identity_custody.resolve(&config.store_id, &store_dir);
425        Ok(crate::read_handle::CovenReadHandle::new(
426            db,
427            store_dir,
428            provider,
429            self.key_service,
430            key_custody,
431            identity_custody,
432            self.clock,
433            self.cloudkit_ops,
434        ))
435    }
436}
437
438fn validate_storage_scope(config: &Config, tables: &[SyncedTable]) -> CovenResult<()> {
439    if config.cloud_home.storage == HomeStorage::Browsable {
440        if let Some(table) = tables
441            .iter()
442            .find(|table| table.audience_column().is_some())
443        {
444            return Err(CovenError::BrowsableStorageWithScopedTable {
445                table: table.name().to_string(),
446            });
447        }
448    }
449    Ok(())
450}
451
452/// Host SQL inside one journaled write transaction.
453///
454/// The context exposes SQL operations, but never the underlying SQLite
455/// connection or transaction. In particular, a host cannot remove coven's SQL
456/// authorizer and address its attached gate baseline:
457///
458/// ```compile_fail
459/// # use coven::{CovenError, CovenHandle};
460/// # async fn cannot_remove_guard(handle: &CovenHandle) -> Result<(), CovenError> {
461/// handle.sql(|sql| {
462///     sql.tx().authorizer(
463///         None::<fn(coven::rusqlite::hooks::AuthContext<'_>)
464///             -> coven::rusqlite::hooks::Authorization>,
465///     )?;
466///     Ok(())
467/// }).await?;
468/// # Ok(())
469/// # }
470/// ```
471pub struct SqlContext<'ctx, 'conn> {
472    tx: &'ctx rusqlite::Transaction<'conn>,
473    stamper: UpdatedAtStamper,
474}
475
476impl<'ctx, 'conn> SqlContext<'ctx, 'conn> {
477    pub(crate) fn new(tx: &'ctx rusqlite::Transaction<'conn>, stamper: UpdatedAtStamper) -> Self {
478        Self { tx, stamper }
479    }
480
481    /// Execute one SQL statement with bound parameters.
482    pub fn execute<P>(&self, sql: &str, params: P) -> rusqlite::Result<usize>
483    where
484        P: rusqlite::Params,
485    {
486        self.tx.execute(sql, params)
487    }
488
489    /// Execute one or more semicolon-separated SQL statements.
490    pub fn execute_batch(&self, sql: &str) -> rusqlite::Result<()> {
491        self.tx.execute_batch(sql)
492    }
493
494    /// Query exactly one row and map it to a host value.
495    pub fn query_row<T, P, F>(&self, sql: &str, params: P, map: F) -> rusqlite::Result<T>
496    where
497        P: rusqlite::Params,
498        F: FnOnce(&rusqlite::Row<'_>) -> rusqlite::Result<T>,
499    {
500        self.tx.query_row(sql, params, map)
501    }
502
503    /// Query any number of rows and collect their mapped host values.
504    pub fn query<T, P, F>(&self, sql: &str, params: P, map: F) -> rusqlite::Result<Vec<T>>
505    where
506        P: rusqlite::Params,
507        F: FnMut(&rusqlite::Row<'_>) -> rusqlite::Result<T>,
508    {
509        let mut statement = self.tx.prepare(sql)?;
510        let values = statement.query_map(params, map)?.collect();
511        values
512    }
513
514    pub fn stamp(&self) -> String {
515        self.stamper.stamp()
516    }
517}
518
519type WriteSql<R> =
520    Box<dyn for<'ctx, 'conn> FnOnce(SqlContext<'ctx, 'conn>) -> CovenResult<R> + Send>;
521
522pub struct WriteBatch {
523    new_blobs: Vec<NewBlob>,
524    deleted_blobs: Vec<BlobRef>,
525}
526
527impl WriteBatch {
528    fn new() -> Self {
529        Self {
530            new_blobs: Vec::new(),
531            deleted_blobs: Vec::new(),
532        }
533    }
534
535    pub fn put_blob(
536        &mut self,
537        namespace: impl Into<String>,
538        id: impl Into<String>,
539        bytes: impl Into<Vec<u8>>,
540    ) {
541        self.new_blobs.push(NewBlob {
542            namespace: namespace.into(),
543            id: id.into(),
544            bytes: bytes.into(),
545        });
546    }
547
548    pub fn delete_blob(&mut self, blob: BlobRef) {
549        self.deleted_blobs.push(blob);
550    }
551}
552
553struct NewBlob {
554    namespace: String,
555    id: String,
556    bytes: Vec<u8>,
557}
558
559#[derive(Clone)]
560pub(crate) struct StagedBlob {
561    pub namespace: String,
562    pub id: String,
563    pub staged: PathBuf,
564    pub final_path: PathBuf,
565}
566
567impl CovenHandle {
568    async fn prepare_pending_branch_resolution(
569        &self,
570        branch_id: &coven_core::PendingBranchId,
571    ) -> CovenResult<crate::sync::store_pull::SerialResolutionPlan> {
572        let branch_base = self.db().conflicted_serial_branch_base(branch_id).await?;
573        let manager = self
574            .sync_manager()
575            .ok_or_else(|| CovenError::SerialResolution("sync is not connected".to_string()))?;
576        manager
577            .prepare_serial_resolution(branch_base, &self.store_dir())
578            .await
579            .map_err(|error| CovenError::SerialResolution(error.to_string()))
580    }
581
582    pub async fn sql<F, R>(&self, f: F) -> CovenResult<WriteReceipt<R>>
583    where
584        F: for<'ctx, 'conn> FnOnce(SqlContext<'ctx, 'conn>) -> CovenResult<R> + Send + 'static,
585        R: Send + 'static,
586    {
587        let stamper = self.stamper();
588        let tables = self.db().synced_tables().to_vec();
589        let gates = self.db().gates();
590        let blob_decls = self.db().blob_decls();
591        let write_policy = self.db().write_policy();
592        let routing_encryption = (write_policy == WritePolicy::MergeConcurrent
593            && gates.has_scoped_graph())
594        .then(|| self.routing_encryption())
595        .transpose()?;
596        let write_id = self.db().new_write_id();
597        let outcome = self
598            .db()
599            .call(move |conn| {
600                Ok(Database::run_store_write_transaction_on(
601                    conn,
602                    &tables,
603                    &gates,
604                    &blob_decls,
605                    write_policy,
606                    routing_encryption.as_ref(),
607                    write_id,
608                    |tx| f(SqlContext::new(tx, stamper)),
609                ))
610            })
611            .await
612            .map_err(CovenError::from)?;
613        outcome
614    }
615
616    pub async fn discard_pending_branch(
617        &self,
618        branch_id: coven_core::PendingBranchId,
619    ) -> CovenResult<()> {
620        match self.db().serial_branch_discard_state(&branch_id).await? {
621            coven_core::database::SerialBranchDiscardState::Local => {
622                self.db().discard_local_serial_branch(branch_id).await?;
623            }
624            coven_core::database::SerialBranchDiscardState::Abandonment => {
625                let outcome = self
626                    .sync_manager()
627                    .ok_or_else(|| {
628                        CovenError::SerialResolution("sync is not connected".to_string())
629                    })?
630                    .abandon_serial_branch(branch_id, &self.store_dir())
631                    .await
632                    .map_err(|error| CovenError::SerialResolution(error.to_string()))?;
633                if matches!(
634                    outcome,
635                    coven_core::sync::store_outbound::SerialBranchAbandonment::OriginalBranchActivated
636                ) {
637                    return Err(CovenError::SerialResolution(
638                        "the original Serial branch activated before abandonment and cannot be discarded"
639                            .to_string(),
640                    ));
641                }
642            }
643            coven_core::database::SerialBranchDiscardState::Conflict => {
644                let plan = self.prepare_pending_branch_resolution(&branch_id).await?;
645                self.sync_manager()
646                    .ok_or_else(|| {
647                        CovenError::SerialResolution("sync is not connected".to_string())
648                    })?
649                    .cleanup_serial_candidates(branch_id.clone(), &plan)
650                    .await
651                    .map_err(|error| CovenError::SerialResolution(error.to_string()))?;
652                self.db()
653                    .discard_pending_serial_branch(branch_id, plan)
654                    .await?;
655            }
656        }
657        self.sync_now();
658        Ok(())
659    }
660
661    pub async fn replace_pending_branch<F, R>(
662        &self,
663        branch_id: coven_core::PendingBranchId,
664        f: F,
665    ) -> CovenResult<WriteReceipt<R>>
666    where
667        F: for<'ctx, 'conn> FnOnce(SqlContext<'ctx, 'conn>) -> CovenResult<R> + Send + 'static,
668        R: Send + 'static,
669    {
670        let plan = self.prepare_pending_branch_resolution(&branch_id).await?;
671        self.sync_manager()
672            .ok_or_else(|| CovenError::SerialResolution("sync is not connected".to_string()))?
673            .cleanup_serial_candidates(branch_id.clone(), &plan)
674            .await
675            .map_err(|error| CovenError::SerialResolution(error.to_string()))?;
676        let write_id = self.db().new_write_id();
677        let stamper = self.stamper();
678        let receipt = self
679            .db()
680            .replace_pending_serial_branch(branch_id, plan, write_id, move |tx| {
681                f(SqlContext::new(tx, stamper))
682            })
683            .await?;
684        self.sync_now();
685        Ok(receipt)
686    }
687
688    /// Run a pure read against this handle's read-only companion connection and
689    /// await the result.
690    ///
691    /// Unlike [`sql`](Self::sql), this attaches no changeset session and opens no
692    /// journaled transaction: a read pays nothing for capture it would produce
693    /// nothing for, and runs on a separate `SQLITE_OPEN_READONLY` connection thread
694    /// (concurrent with the writer, not queued behind it). The connection is
695    /// read-only, so an `INSERT`/`UPDATE`/`DELETE`/DDL in the closure is refused by
696    /// SQLite — a read cannot silently escape the sync journal because it cannot
697    /// write at all. The closure receives the `&Connection` directly, so a host
698    /// closure written against
699    /// [`CovenReadHandle::sql_read`](crate::CovenReadHandle::sql_read) runs
700    /// unchanged here.
701    ///
702    /// Read-your-writes holds for committed writes: a `sql_read` issued after an
703    /// awaited [`sql`](Self::sql) or [`write`](Self::write) sees that data (the WAL
704    /// reader reads the last committed state). It may not see another task's write
705    /// that has not yet committed.
706    pub async fn sql_read<F, R>(&self, f: F) -> CovenResult<R>
707    where
708        F: FnOnce(&rusqlite::Connection) -> CovenResult<R> + Send + 'static,
709        R: Send + 'static,
710    {
711        let outcome = self
712            .read_db()
713            .call(move |conn| Ok(f(conn)))
714            .await
715            .map_err(CovenError::from)?;
716        outcome
717    }
718
719    pub async fn write<F, S, R>(&self, f: F, sql: S) -> CovenResult<WriteReceipt<R>>
720    where
721        F: FnOnce(&mut WriteBatch) -> CovenResult<()> + Send + 'static,
722        S: for<'ctx, 'conn> FnOnce(SqlContext<'ctx, 'conn>) -> CovenResult<R> + Send + 'static,
723        R: Send + 'static,
724    {
725        let mut batch = WriteBatch::new();
726        f(&mut batch)?;
727        let sql: WriteSql<R> = Box::new(sql);
728        let staged = self.stage_blobs(batch.new_blobs).await?;
729        let staged_paths = staged
730            .iter()
731            .map(|blob| blob.staged.clone())
732            .collect::<Vec<_>>();
733        let tables = self.db().synced_tables().to_vec();
734        let db = self.db().clone();
735        let stamper = self.stamper();
736        let gates = self.db().gates();
737        let blob_decls = self.db().blob_decls();
738        let write_policy = self.db().write_policy();
739        let routing_encryption = (write_policy == WritePolicy::MergeConcurrent
740            && gates.has_scoped_graph())
741        .then(|| self.routing_encryption())
742        .transpose()?;
743        let write_id = self.db().new_write_id();
744        let deleted = batch.deleted_blobs;
745        let store_dir = self.store_dir();
746        let outcome = match db
747            .call(move |conn| {
748                Ok(run_write_batch_on_connection(
749                    conn,
750                    stamper,
751                    store_dir,
752                    staged,
753                    deleted,
754                    tables,
755                    gates,
756                    blob_decls,
757                    write_policy,
758                    routing_encryption,
759                    write_id,
760                    sql,
761                ))
762            })
763            .await
764        {
765            Ok(outcome) => outcome,
766            Err(error) => {
767                remove_staged_paths(&staged_paths).await;
768                return Err(CovenError::from(error));
769            }
770        };
771        match outcome {
772            Ok(receipt) => {
773                if let Err(error) =
774                    crate::blob::local_cleanup::drain(self.db(), &self.store_dir()).await
775                {
776                    warn!(
777                        error = %error,
778                        "failed to drain local blob cleanup intents after write commit"
779                    );
780                }
781                Ok(receipt)
782            }
783            Err(error) => {
784                remove_staged_paths(&staged_paths).await;
785                Err(error)
786            }
787        }
788    }
789
790    async fn stage_blobs(&self, blobs: Vec<NewBlob>) -> CovenResult<Vec<StagedBlob>> {
791        let mut staged = Vec::new();
792        for blob in blobs {
793            let final_path = self
794                .store_dir()
795                .local_blob_path(&blob.namespace, &blob.id)?;
796            let staged_path = local_stage_temp_path(&final_path)?;
797            if let Err(e) = crate::local_blob::write_atomic(&staged_path, &blob.bytes).await {
798                remove_staged_files(&staged).await;
799                return Err(CovenError::Blob(e));
800            }
801            staged.push(StagedBlob {
802                namespace: blob.namespace,
803                id: blob.id,
804                staged: staged_path,
805                final_path,
806            });
807        }
808        Ok(staged)
809    }
810}
811
812fn local_stage_temp_path(final_path: &Path) -> CovenResult<PathBuf> {
813    let file_name = final_path
814        .file_name()
815        .and_then(|name| name.to_str())
816        .ok_or_else(|| {
817            CovenError::MalformedPath(format!(
818                "local blob path has no file name: {}",
819                final_path.display()
820            ))
821        })?;
822    Ok(final_path.with_file_name(format!(
823        "{file_name}{LOCAL_STAGE_MARKER}{}",
824        uuid::Uuid::new_v4()
825    )))
826}
827
828/// Remove temp debris a crashed prior process left in the blob folders, before the
829/// handle serves any read. Two temp conventions:
830///
831/// - Every atomic write (`write_atomic`/`copy_atomic`) in any blob folder — the local
832///   store, `cache/`, `pinned/` — leaves a `.tmp.<uuid>` sibling until its
833///   fsync-then-rename commits it. A crash orphans that temp. Only those older than
834///   this open (`process_start`) are crash leftovers; a temp a concurrent write is
835///   mid-producing is newer and is left in place so the sweep can never delete a live
836///   write's rename target.
837/// - `storage/local/` additionally holds write staging (`.coven-stage-<uuid>`). Open
838///   holds the exclusive store lock, so every staging temp here is a crash leftover
839///   — all are removed, no age check.
840fn remove_orphaned_local_blob_temps(
841    store_dir: &crate::store_dir::StoreDir,
842    process_start: std::time::SystemTime,
843) -> CovenResult<()> {
844    let storage = store_dir.storage_dir();
845    remove_orphaned_temps_in_dir(&storage.join("local"), &is_local_stage_temp, None)?;
846    for folder in ["local", "cache", "pinned"] {
847        remove_orphaned_temps_in_dir(
848            &storage.join(folder),
849            &crate::local_blob::is_temp_blob_path,
850            Some(process_start),
851        )?;
852    }
853    Ok(())
854}
855
856/// Recursively remove temp files under `dir` that `is_temp` recognizes. With
857/// `older_than = Some(cutoff)` a temp is removed only when its modification time is
858/// before `cutoff` — a temp at or after it is a live write, left untouched; `None`
859/// removes every recognized temp regardless of age.
860fn remove_orphaned_temps_in_dir(
861    dir: &Path,
862    is_temp: &dyn Fn(&Path) -> bool,
863    older_than: Option<std::time::SystemTime>,
864) -> CovenResult<()> {
865    let entries = match std::fs::read_dir(dir) {
866        Ok(entries) => entries,
867        Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
868            debug!(
869                path = %dir.display(),
870                "blob directory absent during orphaned temp cleanup"
871            );
872            return Ok(());
873        }
874        Err(error) => return Err(CovenError::Io(error)),
875    };
876    for entry in entries {
877        let entry = entry?;
878        let path = entry.path();
879        let file_type = entry.file_type()?;
880        if file_type.is_dir() {
881            remove_orphaned_temps_in_dir(&path, is_temp, older_than)?;
882        } else if file_type.is_file() && is_temp(&path) {
883            if let Some(cutoff) = older_than {
884                let modified = entry.metadata()?.modified()?;
885                if modified >= cutoff {
886                    debug!(
887                        path = %path.display(),
888                        "leaving fresh blob temp created at or after process start"
889                    );
890                    continue;
891                }
892            }
893            remove_file_if_present(&path)?;
894        } else if file_type.is_file() && path.file_name().and_then(|name| name.to_str()).is_none() {
895            debug!(
896                path = %path.display(),
897                "skipping blob path with non-utf8 file name during orphaned temp cleanup"
898            );
899        }
900    }
901    Ok(())
902}
903
904fn is_local_stage_temp(path: &Path) -> bool {
905    path.file_name()
906        .and_then(|name| name.to_str())
907        .is_some_and(|name| name.contains(LOCAL_STAGE_MARKER))
908}
909
910async fn remove_staged_files(staged: &[StagedBlob]) {
911    let paths = staged
912        .iter()
913        .map(|blob| blob.staged.clone())
914        .collect::<Vec<_>>();
915    remove_staged_paths(&paths).await;
916}
917
918async fn remove_staged_paths(paths: &[PathBuf]) {
919    for path in paths {
920        remove_staged_path(path).await;
921    }
922}
923
924async fn remove_staged_path(path: &Path) {
925    if let Err(error) = crate::local_blob::remove_file(path).await {
926        warn!(
927            path = %path.display(),
928            error = %error,
929            "failed to remove staged local blob"
930        );
931    }
932}
933
934fn remove_file_if_present(path: &Path) -> CovenResult<()> {
935    match std::fs::remove_file(path) {
936        Ok(()) => Ok(()),
937        Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
938            debug!(
939                path = %path.display(),
940                "file already absent during local blob cleanup"
941            );
942            Ok(())
943        }
944        Err(error) => Err(CovenError::Io(error)),
945    }
946}
947
948fn sync_parent_dir(path: &Path) -> CovenResult<()> {
949    let parent = path.parent().ok_or_else(|| {
950        CovenError::MalformedPath(format!("path has no parent directory: {}", path.display()))
951    })?;
952    std::fs::File::open(parent)?.sync_all()?;
953    Ok(())
954}
955
956fn run_write_batch_on_connection<R>(
957    conn: &Connection,
958    stamper: UpdatedAtStamper,
959    store_dir: crate::store_dir::StoreDir,
960    staged: Vec<StagedBlob>,
961    deleted: Vec<BlobRef>,
962    tables: Vec<SyncedTable>,
963    gates: Arc<crate::sync::gate::Gates>,
964    decls: Arc<crate::blob::decl::BlobDecls>,
965    write_policy: WritePolicy,
966    routing_encryption: Option<crate::encryption::EncryptionService>,
967    write_id: WriteId,
968    sql: WriteSql<R>,
969) -> CovenResult<WriteReceipt<R>> {
970    let mut moved = Vec::new();
971    let result = Database::run_store_write_transaction_on(
972        conn,
973        &tables,
974        &gates,
975        &decls,
976        write_policy,
977        routing_encryption.as_ref(),
978        write_id,
979        |tx| -> CovenResult<R> {
980            for blob in &staged {
981                match decls.row_for_blob_in_namespace(tx, &blob.namespace, &blob.id) {
982                    Ok(Some(_)) => {
983                        return Err(CovenError::BlobAlreadyReferenced {
984                            namespace: blob.namespace.clone(),
985                            id: blob.id.clone(),
986                        });
987                    }
988                    Ok(None) => {}
989                    Err(e) => return Err(CovenError::Blob(e.to_string())),
990                }
991                let leased = tx
992                    .query_row(
993                        "SELECT EXISTS(\
994                             SELECT 1 FROM store_write_blob_leases \
995                             WHERE namespace = ?1 AND blob_id = ?2\
996                         )",
997                        (&blob.namespace, &blob.id),
998                        |row| row.get::<_, bool>(0),
999                    )
1000                    .map_err(CovenError::from)?;
1001                if leased {
1002                    return Err(CovenError::BlobOwnedByPendingWrite {
1003                        namespace: blob.namespace.clone(),
1004                        id: blob.id.clone(),
1005                    });
1006                }
1007                if let Some(parent) = blob.final_path.parent() {
1008                    std::fs::create_dir_all(parent).map_err(|e| {
1009                        CovenError::Blob(format!(
1010                            "create local blob parent {}: {e}",
1011                            parent.display()
1012                        ))
1013                    })?;
1014                }
1015                std::fs::rename(&blob.staged, &blob.final_path).map_err(|e| {
1016                    CovenError::Blob(format!(
1017                        "install staged blob {} -> {}: {e}",
1018                        blob.staged.display(),
1019                        blob.final_path.display()
1020                    ))
1021                })?;
1022                moved.push(blob.clone());
1023                sync_parent_dir(&blob.final_path).map_err(|e| {
1024                    CovenError::Blob(format!(
1025                        "sync local blob parent after installing {}: {e}",
1026                        blob.final_path.display()
1027                    ))
1028                })?;
1029            }
1030
1031            match std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
1032                sql(SqlContext::new(tx, stamper))
1033            })) {
1034                Ok(Ok(value)) => {
1035                    for blob in &deleted {
1036                        let _ = store_dir.local_blob_path(&blob.namespace, &blob.id)?;
1037                        let _ = store_dir.pinned_blob_path(&blob.namespace, &blob.id)?;
1038                        let _ = store_dir.cache_blob_path(&blob.namespace, &blob.id)?;
1039                        let intent = crate::blob::local_cleanup::LocalBlobCleanupIntent::new(
1040                            &blob.namespace,
1041                            &blob.id,
1042                        );
1043                        if !crate::blob::local_cleanup::record_if_unreferenced_on(
1044                            tx, &decls, &intent,
1045                        )? {
1046                            return Err(CovenError::BlobStillReferenced {
1047                                namespace: blob.namespace.clone(),
1048                                id: blob.id.clone(),
1049                            });
1050                        }
1051                    }
1052                    Ok(value)
1053                }
1054                Ok(Err(error)) => Err(error),
1055                Err(_) => Err(CovenError::WriteClosurePanicked),
1056            }
1057        },
1058    );
1059    match result {
1060        Ok(value) => Ok(value),
1061        Err(error) => rollback_write_batch(error, moved),
1062    }
1063}
1064
1065fn rollback_write_batch<R>(error: CovenError, moved: Vec<StagedBlob>) -> CovenResult<R> {
1066    for blob in moved.iter().rev() {
1067        if let Err(e) = std::fs::remove_file(&blob.final_path) {
1068            warn!(
1069                namespace = %blob.namespace,
1070                blob_id = %blob.id,
1071                path = %blob.final_path.display(),
1072                error = %e,
1073                "failed to remove installed local blob after write rollback"
1074            );
1075        }
1076    }
1077    Err(error)
1078}
1079
1080#[cfg(test)]
1081mod tests {
1082    use super::*;
1083
1084    use crate::blob::{BlobScope, CacheFill, Provenance};
1085    use crate::config::Config;
1086    use crate::keys::test_keyring;
1087    use crate::storage::cloud::test_utils::InMemoryCloudHome;
1088    use crate::storage::cloud::{
1089        BlobBody, BoxPartSink, CloudAccessOutcome, CloudAccessState, CloudHome, CloudHomeError,
1090        ExactSlotStorage, ObjectSlot, UploadProgress,
1091    };
1092    use crate::store_dir::StoreDir;
1093    use crate::sync::cloud_storage::CloudCipher;
1094    use crate::sync::session::BlobDecl;
1095    use crate::sync::test_helpers::{pull_into, query_text, row_exists, TestStore};
1096    use async_trait::async_trait;
1097    use rusqlite::params;
1098    use std::sync::atomic::{AtomicBool, Ordering};
1099    use std::time::Duration;
1100    use tokio::sync::mpsc;
1101    use tokio::sync::Notify;
1102
1103    fn config(dir: StoreDir) -> Config {
1104        Config::with_defaults(
1105            "lib-test".to_string(),
1106            "device-test".to_string(),
1107            dir,
1108            "Test".to_string(),
1109        )
1110    }
1111
1112    fn media_files_decl() -> BlobDecl {
1113        BlobDecl::new(
1114            "media-files",
1115            Provenance::HostProvided,
1116            CacheFill::CacheLazy,
1117        )
1118        .with_id_column("blob_id")
1119    }
1120
1121    fn files_table() -> SyncedTable {
1122        SyncedTable::new("files", coven_core::RowIdentity::SharedKey)
1123            .carries_blob(media_files_decl())
1124    }
1125
1126    fn remote_root_files_table() -> SyncedTable {
1127        SyncedTable::new("files", coven_core::RowIdentity::SharedKey)
1128            .remote_root()
1129            .carries_blob(media_files_decl())
1130    }
1131
1132    fn files_migration() -> Migration {
1133        Migration::sql(
1134            1,
1135            "test-schema",
1136            "CREATE TABLE files (
1137                id TEXT PRIMARY KEY,
1138                blob_id TEXT,
1139                size INTEGER NOT NULL,
1140                hash TEXT,
1141                _updated_at TEXT NOT NULL
1142            ) STRICT;",
1143        )
1144    }
1145
1146    fn gated_roots_table() -> SyncedTable {
1147        SyncedTable::new("roots", coven_core::RowIdentity::SharedKey).gated_by("shared")
1148    }
1149
1150    fn gated_roots_migration() -> Migration {
1151        Migration::sql(
1152            1,
1153            "gated-roots",
1154            "CREATE TABLE roots (
1155                id TEXT PRIMARY KEY,
1156                title TEXT NOT NULL,
1157                shared INTEGER NOT NULL,
1158                _updated_at TEXT NOT NULL
1159            ) STRICT;",
1160        )
1161    }
1162
1163    fn open_gated_roots_handle() -> (tempfile::TempDir, CovenHandle) {
1164        let tmp = tempfile::tempdir().expect("temp dir");
1165        let handle = Coven::builder(config(StoreDir::new(tmp.path())))
1166            .write_policy(WritePolicy::MergeConcurrent)
1167            .synced_tables(vec![gated_roots_table()])
1168            .migrations(vec![gated_roots_migration()])
1169            .open()
1170            .expect("open gated handle");
1171        (tmp, handle)
1172    }
1173
1174    fn open_gated_roots_at(dir: StoreDir, write_policy: WritePolicy) -> CovenResult<CovenHandle> {
1175        Coven::builder(config(dir))
1176            .write_policy(write_policy)
1177            .synced_tables(vec![gated_roots_table()])
1178            .migrations(vec![gated_roots_migration()])
1179            .open()
1180    }
1181
1182    fn precreate_database(dir: &StoreDir, sql: &str) {
1183        let conn = rusqlite::Connection::open(dir.db_path()).expect("precreate store database");
1184        conn.execute_batch(sql)
1185            .expect("seed pre-existing database state");
1186    }
1187
1188    #[test]
1189    fn serial_store_opens_without_a_provider_or_coordination() {
1190        let tmp = tempfile::tempdir().expect("temp dir");
1191        let handle = Coven::builder(config(StoreDir::new(tmp.path())))
1192            .write_policy(WritePolicy::Serial)
1193            .synced_tables(vec![gated_roots_table()])
1194            .migrations(vec![gated_roots_migration()])
1195            .open()
1196            .expect("open local Serial store");
1197
1198        assert!(matches!(
1199            *handle.subscribe_sync_status().borrow(),
1200            crate::SyncLoopStatus::Offline
1201        ));
1202    }
1203
1204    #[tokio::test]
1205    async fn local_serial_branch_can_be_discarded_without_sync() {
1206        let tmp = tempfile::tempdir().expect("temp dir");
1207        let handle = Coven::builder(config(StoreDir::new(tmp.path())))
1208            .write_policy(WritePolicy::Serial)
1209            .synced_tables(vec![gated_roots_table()])
1210            .migrations(vec![gated_roots_migration()])
1211            .open()
1212            .expect("open local Serial store");
1213        let receipt = handle
1214            .sql(|ctx| {
1215                ctx.execute(
1216                    "INSERT INTO roots (id, title, shared, _updated_at)
1217                     VALUES ('discard-me', 'local', 1, '2026-01-01T00:00:00Z')",
1218                    [],
1219                )?;
1220                Ok(())
1221            })
1222            .await
1223            .expect("write local Serial branch");
1224        let branch_id = receipt
1225            .pending_branch_id
1226            .expect("local Serial write names its pending branch");
1227        let second = handle
1228            .sql(|ctx| {
1229                ctx.execute(
1230                    "INSERT INTO roots (id, title, shared, _updated_at)
1231                     VALUES ('discard-me-too', 'local', 1, '2026-01-01T00:00:01Z')",
1232                    [],
1233                )?;
1234                Ok(())
1235            })
1236            .await
1237            .expect("write another transaction on the local Serial branch");
1238        assert_eq!(second.pending_branch_id.as_ref(), Some(&branch_id));
1239
1240        handle
1241            .discard_pending_branch(branch_id)
1242            .await
1243            .expect("discard local Serial branch");
1244
1245        assert!(!row_exists(handle.db(), "SELECT 1 FROM roots WHERE id = 'discard-me'").await);
1246        assert!(
1247            !row_exists(
1248                handle.db(),
1249                "SELECT 1 FROM roots WHERE id = 'discard-me-too'"
1250            )
1251            .await
1252        );
1253        assert!(handle
1254            .db()
1255            .pending_branches()
1256            .await
1257            .expect("read discarded Serial branch")
1258            .is_none());
1259    }
1260
1261    #[test]
1262    fn precreated_empty_sqlite_file_initializes_coven_metadata() {
1263        let tmp = tempfile::tempdir().expect("temp dir");
1264        let dir = StoreDir::new(tmp.path());
1265        precreate_database(&dir, "");
1266
1267        let handle = open_gated_roots_at(dir, WritePolicy::Serial)
1268            .expect("initialize Coven in an empty SQLite database");
1269
1270        assert_eq!(handle.db().write_policy(), WritePolicy::Serial);
1271    }
1272
1273    #[test]
1274    fn existing_host_tables_without_a_coven_marker_initialize_coven_metadata() {
1275        let tmp = tempfile::tempdir().expect("temp dir");
1276        let dir = StoreDir::new(tmp.path());
1277        precreate_database(
1278            &dir,
1279            "CREATE TABLE roots (
1280                 id TEXT PRIMARY KEY,
1281                 title TEXT NOT NULL,
1282                 shared INTEGER NOT NULL,
1283                 _updated_at TEXT NOT NULL
1284             ) STRICT;
1285             PRAGMA user_version = 1;",
1286        );
1287
1288        let handle = open_gated_roots_at(dir, WritePolicy::Serial)
1289            .expect("initialize Coven beside an existing host schema");
1290
1291        assert_eq!(handle.db().write_policy(), WritePolicy::Serial);
1292    }
1293
1294    #[test]
1295    fn interrupted_coven_schema_without_a_marker_is_rejected() {
1296        let tmp = tempfile::tempdir().expect("temp dir");
1297        let dir = StoreDir::new(tmp.path());
1298        precreate_database(
1299            &dir,
1300            "CREATE TABLE protocol_state (
1301                 key TEXT PRIMARY KEY,
1302                 value TEXT NOT NULL
1303             ) STRICT;",
1304        );
1305
1306        let error = match open_gated_roots_at(dir, WritePolicy::Serial) {
1307            Ok(_) => panic!("partial Coven schema has no valid initialization commit"),
1308            Err(error) => error,
1309        };
1310
1311        assert!(matches!(
1312            error,
1313            CovenError::Database(DbError::Message(reason))
1314                if reason.contains("without the required initialization marker")
1315        ));
1316    }
1317
1318    #[tokio::test]
1319    async fn reopen_refuses_a_write_policy_that_differs_from_the_created_store() {
1320        let tmp = tempfile::tempdir().expect("temp dir");
1321        let dir = StoreDir::new(tmp.path());
1322        let serial = Coven::builder(config(dir.clone()))
1323            .write_policy(WritePolicy::Serial)
1324            .synced_tables(vec![gated_roots_table()])
1325            .migrations(vec![gated_roots_migration()])
1326            .open()
1327            .expect("create Serial store");
1328        drop(serial);
1329
1330        let wrong = Coven::builder(config(dir.clone()))
1331            .write_policy(WritePolicy::MergeConcurrent)
1332            .synced_tables(vec![gated_roots_table()])
1333            .migrations(vec![gated_roots_migration()])
1334            .open();
1335        if let Ok(wrong) = wrong {
1336            wrong
1337                .sql(|sql| {
1338                    sql.execute(
1339                        "INSERT INTO roots (id, title, shared, _updated_at) \
1340                         VALUES ('wrong-policy', 'wrong', 1, \
1341                                 '0000000001000-0000-device-test')",
1342                        [],
1343                    )?;
1344                    Ok(())
1345                })
1346                .await
1347                .expect("the mismatched open currently permits a host write");
1348            panic!("a store must reject a different write policy at open");
1349        }
1350
1351        let reopened = Coven::builder(config(dir))
1352            .write_policy(WritePolicy::Serial)
1353            .synced_tables(vec![gated_roots_table()])
1354            .migrations(vec![gated_roots_migration()])
1355            .open()
1356            .expect("the store reopens with its selected policy");
1357        assert!(reopened.db().pending_writes().await.unwrap().is_empty());
1358    }
1359
1360    #[tokio::test]
1361    async fn reopen_refuses_a_store_missing_required_write_policy_metadata() {
1362        let tmp = tempfile::tempdir().expect("temp dir");
1363        let dir = StoreDir::new(tmp.path());
1364        let handle = Coven::builder(config(dir.clone()))
1365            .write_policy(WritePolicy::Serial)
1366            .synced_tables(vec![gated_roots_table()])
1367            .migrations(vec![gated_roots_migration()])
1368            .open()
1369            .expect("create Serial store");
1370        handle
1371            .db()
1372            .call(|conn| {
1373                let marker: String = conn
1374                    .query_row(
1375                        "SELECT value FROM protocol_state WHERE key = 'coven_initialized'",
1376                        [],
1377                        |row| row.get(0),
1378                    )
1379                    .map_err(crate::database::DbError::from)?;
1380                assert_eq!(marker, "1");
1381                conn.execute("DELETE FROM protocol_state WHERE key = 'write_policy'", [])
1382                    .map_err(crate::database::DbError::from)
1383            })
1384            .await
1385            .expect("remove required policy metadata");
1386        drop(handle);
1387
1388        let reopened = Coven::builder(config(dir))
1389            .write_policy(WritePolicy::Serial)
1390            .synced_tables(vec![gated_roots_table()])
1391            .migrations(vec![gated_roots_migration()])
1392            .open();
1393        assert!(matches!(reopened, Err(CovenError::Database(_))));
1394    }
1395
1396    #[tokio::test]
1397    async fn host_sql_cannot_discover_or_mutate_the_gate_baseline() {
1398        let (_tmp, handle) = open_gated_roots_handle();
1399
1400        let discovery = handle
1401            .sql(|sql| {
1402                sql.query("PRAGMA database_list", [], |row| row.get::<_, String>(1))
1403                    .map_err(CovenError::from)
1404            })
1405            .await;
1406        assert!(
1407            discovery.is_err(),
1408            "host SQL must not enumerate coven's attached gate baseline",
1409        );
1410
1411        let mutation = handle
1412            .sql(|sql| {
1413                sql.execute(
1414                    "INSERT INTO coven_gate_empty.roots \
1415                     (id, title, shared, _updated_at) \
1416                     VALUES ('root-1', 'Private', 1, '0000000002000-0000-device-test')",
1417                    [],
1418                )?;
1419                Ok(())
1420            })
1421            .await;
1422        assert!(
1423            mutation.is_err(),
1424            "host SQL must not address coven's attached gate baseline",
1425        );
1426
1427        handle
1428            .sql(|sql| {
1429                sql.execute_batch(
1430                    "CREATE TABLE host_local (id TEXT PRIMARY KEY, value TEXT) STRICT; \
1431                     INSERT INTO host_local VALUES ('local-1', 'kept'); \
1432                     INSERT INTO roots (id, title, shared, _updated_at) \
1433                     VALUES ('root-1', 'Private', 0, \
1434                             '0000000001000-0000-device-test');",
1435                )?;
1436                Ok(())
1437            })
1438            .await
1439            .expect("arbitrary host-schema SQL remains available");
1440        let published = handle
1441            .sql(|sql| {
1442                sql.execute(
1443                    "UPDATE roots SET shared = 1, \
1444                     _updated_at = '0000000002000-0000-device-test' \
1445                     WHERE id = 'root-1'",
1446                    [],
1447                )?;
1448                Ok(())
1449            })
1450            .await
1451            .expect("flip root visible");
1452        let write_id = published.write_id.clone();
1453        let changeset = handle
1454            .db()
1455            .call(move |conn| {
1456                conn.query_row(
1457                    "SELECT changeset FROM store_writes WHERE write_id = ?1",
1458                    [write_id.as_str()],
1459                    |row| row.get::<_, Vec<u8>>(0),
1460                )
1461                .map_err(crate::database::DbError::from)
1462            })
1463            .await
1464            .expect("load gated changeset");
1465        let rows = coven_core::changeset::walk(&changeset).expect("walk gated changeset");
1466        assert_eq!(rows.len(), 1);
1467        assert_eq!(rows[0].op, coven_core::changeset::ChangeOp::Insert);
1468        assert_eq!(rows[0].pk(), Some("root-1"));
1469        assert_eq!(rows[0].col(1), Some("Private"));
1470        assert_eq!(rows[0].col(2), Some("1"));
1471        assert_eq!(rows[0].col(3), Some("0000000002000-0000-device-test"));
1472    }
1473
1474    fn open_files_handle() -> (tempfile::TempDir, CovenHandle) {
1475        let tmp = tempfile::tempdir().expect("temp dir");
1476        let dir = StoreDir::new(tmp.path());
1477        let handle = Coven::builder(config(dir))
1478            .write_policy(WritePolicy::MergeConcurrent)
1479            .synced_tables(vec![files_table()])
1480            .migrations(vec![files_migration()])
1481            .open()
1482            .expect("open handle");
1483        (tmp, handle)
1484    }
1485
1486    #[tokio::test]
1487    async fn second_open_of_one_store_is_refused_until_the_first_handle_drops() {
1488        let tmp = tempfile::tempdir().expect("temp dir");
1489        let dir = StoreDir::new(tmp.path());
1490        let first = Coven::builder(config(dir.clone()))
1491            .write_policy(WritePolicy::MergeConcurrent)
1492            .synced_tables(vec![files_table()])
1493            .migrations(vec![files_migration()])
1494            .open()
1495            .expect("first open succeeds");
1496        let clone = first.clone();
1497
1498        let second = Coven::builder(config(dir.clone()))
1499            .write_policy(WritePolicy::MergeConcurrent)
1500            .synced_tables(vec![files_table()])
1501            .migrations(vec![files_migration()])
1502            .open();
1503        assert!(matches!(
1504            second,
1505            Err(CovenError::AlreadyOpen { store_dir }) if store_dir == tmp.path()
1506        ));
1507
1508        drop(first);
1509
1510        let still_locked = Coven::builder(config(dir.clone()))
1511            .write_policy(WritePolicy::MergeConcurrent)
1512            .synced_tables(vec![files_table()])
1513            .migrations(vec![files_migration()])
1514            .open();
1515        assert!(matches!(
1516            still_locked,
1517            Err(CovenError::AlreadyOpen { store_dir }) if store_dir == tmp.path()
1518        ));
1519
1520        drop(clone);
1521
1522        Coven::builder(config(dir))
1523            .write_policy(WritePolicy::MergeConcurrent)
1524            .synced_tables(vec![files_table()])
1525            .migrations(vec![files_migration()])
1526            .open()
1527            .expect("open succeeds after the first handle drops");
1528    }
1529
1530    #[tokio::test]
1531    async fn a_zero_or_negative_blob_tombstone_grace_is_refused_at_open() {
1532        for grace in [chrono::Duration::zero(), chrono::Duration::seconds(-1)] {
1533            let tmp = tempfile::tempdir().expect("temp dir");
1534            let dir = StoreDir::new(tmp.path());
1535            let result = Coven::builder(config(dir))
1536                .write_policy(WritePolicy::MergeConcurrent)
1537                .synced_tables(vec![files_table()])
1538                .migrations(vec![files_migration()])
1539                .blob_tombstone_grace(grace)
1540                .open();
1541            assert!(
1542                matches!(result, Err(CovenError::InvalidBlobTombstoneGrace)),
1543                "grace {grace:?} must be refused at open",
1544            );
1545        }
1546    }
1547
1548    #[tokio::test]
1549    async fn a_positive_blob_tombstone_grace_opens() {
1550        let tmp = tempfile::tempdir().expect("temp dir");
1551        let dir = StoreDir::new(tmp.path());
1552        Coven::builder(config(dir))
1553            .write_policy(WritePolicy::MergeConcurrent)
1554            .synced_tables(vec![files_table()])
1555            .migrations(vec![files_migration()])
1556            .blob_tombstone_grace(chrono::Duration::hours(1))
1557            .open()
1558            .expect("a positive grace opens");
1559    }
1560
1561    fn open_remote_root_files_handle() -> (tempfile::TempDir, CovenHandle) {
1562        let tmp = tempfile::tempdir().expect("temp dir");
1563        let dir = StoreDir::new(tmp.path());
1564        let handle = Coven::builder(config(dir))
1565            .write_policy(WritePolicy::MergeConcurrent)
1566            .synced_tables(vec![remote_root_files_table()])
1567            .migrations(vec![files_migration()])
1568            .open()
1569            .expect("open handle");
1570        (tmp, handle)
1571    }
1572
1573    fn open_files_handle_in(dir: StoreDir) -> CovenHandle {
1574        try_open(&dir).expect("open handle")
1575    }
1576
1577    async fn run_test_cycle(storage: &TestStore, handle: &CovenHandle) {
1578        storage
1579            .publish_pending(handle.db(), &handle.store_dir())
1580            .await
1581            .expect("publish pending Store write");
1582    }
1583
1584    async fn merge_test_storage(
1585        handle: &CovenHandle,
1586        keypair: &crate::keys::UserKeypair,
1587    ) -> TestStore {
1588        TestStore::create(handle.db(), "lib-test", keypair.clone())
1589            .await
1590            .expect("create exact test Store")
1591    }
1592
1593    async fn publish_current_writes(handle: &CovenHandle) {
1594        let keypair = crate::keys::UserKeypair::generate();
1595        let storage = merge_test_storage(handle, &keypair).await;
1596        run_test_cycle(&storage, handle).await;
1597    }
1598
1599    #[tokio::test]
1600    async fn write_survives_reopen_before_sync_cycle() {
1601        let tmp = tempfile::tempdir().expect("temp dir");
1602        let dir = StoreDir::new(tmp.path());
1603        let handle = open_files_handle_in(dir.clone());
1604        handle
1605            .sql(|sql| {
1606                sql.execute(
1607                    "INSERT INTO files (id, blob_id, size, _updated_at) \
1608                     VALUES ('file-before-reopen', NULL, 0, ?1)",
1609                    [sql.stamp()],
1610                )?;
1611                Ok(())
1612            })
1613            .await
1614            .expect("write before reopen");
1615        drop(handle);
1616
1617        let reopened = open_files_handle_in(dir);
1618        let keypair = crate::keys::UserKeypair::generate();
1619        let storage = merge_test_storage(&reopened, &keypair).await;
1620        run_test_cycle(&storage, &reopened).await;
1621
1622        let (_peer_tmp, peer) = open_files_handle();
1623        pull_into(peer.db(), &storage, &peer.store_dir()).await;
1624        assert_eq!(
1625            query_text(
1626                peer.db(),
1627                "SELECT id FROM files WHERE id = 'file-before-reopen'"
1628            )
1629            .await,
1630            "file-before-reopen",
1631        );
1632    }
1633
1634    #[tokio::test]
1635    async fn separate_host_transactions_publish_as_separate_store_commits_after_restart() {
1636        let tmp = tempfile::tempdir().expect("temp dir");
1637        let dir = StoreDir::new(tmp.path());
1638        let handle = open_files_handle_in(dir.clone());
1639        let mut write_ids = Vec::new();
1640        for id in ["file-pending-a", "file-pending-b"] {
1641            let receipt = handle
1642                .sql(move |sql| {
1643                    sql.execute(
1644                        "INSERT INTO files (id, blob_id, size, _updated_at) VALUES (?1, NULL, 0, ?2)",
1645                        (id, sql.stamp()),
1646                    )?;
1647                    Ok(())
1648                })
1649                .await
1650                .expect("write before reopen");
1651            assert_eq!(receipt.status, coven_core::WriteStatus::Pending);
1652            write_ids.push(receipt.write_id);
1653        }
1654        drop(handle);
1655
1656        let reopened = open_files_handle_in(dir);
1657        let pending = reopened.pending_writes().await.expect("pending writes");
1658        assert_eq!(pending.len(), 2);
1659        assert_eq!(pending[0].write_id, write_ids[0]);
1660        assert_eq!(pending[1].write_id, write_ids[1]);
1661        let mut first_status = reopened
1662            .subscribe_write_status(&write_ids[0])
1663            .await
1664            .expect("subscribe after restart");
1665        assert_eq!(*first_status.borrow(), coven_core::WriteStatus::Pending);
1666        let keypair = crate::keys::UserKeypair::generate();
1667        let storage = merge_test_storage(&reopened, &keypair).await;
1668        run_test_cycle(&storage, &reopened).await;
1669
1670        first_status.changed().await.expect("published status");
1671        let first_sequence = match &*first_status.borrow() {
1672            coven_core::WriteStatus::Published(position) => position.commit().coord.sequence(),
1673            status => panic!("first host transaction is not published: {status:?}"),
1674        };
1675        assert_eq!(
1676            reopened
1677                .write_status(&write_ids[1])
1678                .await
1679                .expect("second status after first publication"),
1680            coven_core::WriteStatus::Pending,
1681        );
1682        run_test_cycle(&storage, &reopened).await;
1683        let second_sequence = match reopened
1684            .write_status(&write_ids[1])
1685            .await
1686            .expect("second status")
1687        {
1688            coven_core::WriteStatus::Published(position) => position.commit().coord.sequence(),
1689            status => panic!("second host transaction is not published: {status:?}"),
1690        };
1691        assert_eq!(second_sequence, first_sequence + 1);
1692        assert!(reopened
1693            .pending_writes()
1694            .await
1695            .expect("published writes are not pending")
1696            .is_empty());
1697
1698        let (_peer_tmp, peer) = open_files_handle();
1699        pull_into(peer.db(), &storage, &peer.store_dir()).await;
1700        assert!(row_exists(peer.db(), "SELECT 1 FROM files WHERE id = 'file-pending-a'").await);
1701        assert!(row_exists(peer.db(), "SELECT 1 FROM files WHERE id = 'file-pending-b'").await);
1702    }
1703
1704    #[tokio::test]
1705    async fn device_local_transaction_is_local_only_and_never_pending() {
1706        let (_tmp, handle) = open_files_handle();
1707        let receipt = handle
1708            .sql(|sql| {
1709                sql.execute_batch(
1710                    "CREATE TABLE local_notes (id TEXT PRIMARY KEY, body TEXT) STRICT;
1711                     INSERT INTO local_notes VALUES ('local-1', 'private');",
1712                )?;
1713                Ok("saved")
1714            })
1715            .await
1716            .expect("local transaction");
1717
1718        assert_eq!(receipt.value, "saved");
1719        assert_eq!(receipt.status, coven_core::WriteStatus::LocalOnly);
1720        assert!(receipt.pending_branch_id.is_none());
1721        assert_eq!(
1722            handle
1723                .write_status(&receipt.write_id)
1724                .await
1725                .expect("durable local status"),
1726            coven_core::WriteStatus::LocalOnly
1727        );
1728        assert!(handle
1729            .pending_writes()
1730            .await
1731            .expect("pending writes")
1732            .is_empty());
1733        let local_write_id = receipt.write_id.clone();
1734        let lease_count = handle
1735            .db()
1736            .call(move |conn| {
1737                conn.query_row(
1738                    "SELECT COUNT(*) FROM store_write_blob_leases WHERE write_id = ?1",
1739                    [local_write_id.as_str()],
1740                    |row| row.get::<_, i64>(0),
1741                )
1742                .map_err(crate::database::DbError::from)
1743            })
1744            .await
1745            .expect("count local-only blob leases");
1746        assert_eq!(lease_count, 0);
1747
1748        let keypair = crate::keys::UserKeypair::generate();
1749        let storage = merge_test_storage(&handle, &keypair).await;
1750        run_test_cycle(&storage, &handle).await;
1751        assert_eq!(
1752            handle
1753                .write_status(&receipt.write_id)
1754                .await
1755                .expect("local status after sync"),
1756            coven_core::WriteStatus::LocalOnly
1757        );
1758        assert!(handle
1759            .pending_writes()
1760            .await
1761            .expect("pending writes after sync")
1762            .is_empty());
1763    }
1764
1765    #[tokio::test]
1766    async fn mixed_transaction_tracks_and_publishes_only_shared_rows() {
1767        let (_tmp, handle) = open_files_handle();
1768        let receipt = handle
1769            .sql(|sql| {
1770                sql.execute_batch(
1771                    "CREATE TABLE local_notes (id TEXT PRIMARY KEY, body TEXT) STRICT;
1772                     INSERT INTO local_notes VALUES ('local-1', 'private');",
1773                )?;
1774                sql.execute(
1775                    "INSERT INTO files (id, blob_id, size, _updated_at)
1776                     VALUES ('shared-1', NULL, 0, ?1)",
1777                    [sql.stamp()],
1778                )?;
1779                Ok(())
1780            })
1781            .await
1782            .expect("mixed transaction");
1783
1784        assert_eq!(receipt.status, coven_core::WriteStatus::Pending);
1785        assert!(receipt.pending_branch_id.is_none());
1786        let pending = handle.pending_writes().await.expect("pending mixed write");
1787        assert_eq!(pending.len(), 1);
1788        assert_eq!(pending[0].write_id, receipt.write_id);
1789        assert_eq!(
1790            pending[0].affected_rows,
1791            vec![coven_core::AffectedRow {
1792                table: "files".to_string(),
1793                primary_key: "shared-1".to_string(),
1794            }]
1795        );
1796
1797        let keypair = crate::keys::UserKeypair::generate();
1798        let storage = merge_test_storage(&handle, &keypair).await;
1799        run_test_cycle(&storage, &handle).await;
1800        assert!(matches!(
1801            handle
1802                .write_status(&receipt.write_id)
1803                .await
1804                .expect("published mixed write"),
1805            coven_core::WriteStatus::Published(_)
1806        ));
1807
1808        let (_peer_tmp, peer) = open_files_handle();
1809        pull_into(peer.db(), &storage, &peer.store_dir()).await;
1810        assert!(row_exists(peer.db(), "SELECT 1 FROM files WHERE id = 'shared-1'").await);
1811        assert!(
1812            !row_exists(
1813                peer.db(),
1814                "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'local_notes'"
1815            )
1816            .await
1817        );
1818        assert!(
1819            row_exists(
1820                handle.db(),
1821                "SELECT 1 FROM local_notes WHERE id = 'local-1'"
1822            )
1823            .await
1824        );
1825    }
1826
1827    #[tokio::test]
1828    async fn delete_survives_reopen_before_sync_cycle() {
1829        let tmp = tempfile::tempdir().expect("temp dir");
1830        let dir = StoreDir::new(tmp.path());
1831        let handle = open_files_handle_in(dir.clone());
1832        handle
1833            .sql(|sql| {
1834                sql.execute(
1835                    "INSERT INTO files (id, blob_id, size, _updated_at) \
1836                     VALUES ('file-delete-reopen', NULL, 0, ?1)",
1837                    [sql.stamp()],
1838                )?;
1839                Ok(())
1840            })
1841            .await
1842            .expect("insert before first cycle");
1843        let keypair = crate::keys::UserKeypair::generate();
1844        let storage = merge_test_storage(&handle, &keypair).await;
1845        run_test_cycle(&storage, &handle).await;
1846
1847        let (_peer_tmp, peer) = open_files_handle();
1848        pull_into(peer.db(), &storage, &peer.store_dir()).await;
1849        assert!(
1850            row_exists(
1851                peer.db(),
1852                "SELECT 1 FROM files WHERE id = 'file-delete-reopen'"
1853            )
1854            .await,
1855            "the peer receives the insert before the delete",
1856        );
1857
1858        handle
1859            .sql(|sql| {
1860                sql.execute("DELETE FROM files WHERE id = 'file-delete-reopen'", [])?;
1861                Ok(())
1862            })
1863            .await
1864            .expect("delete before reopen");
1865        drop(handle);
1866
1867        let reopened = open_files_handle_in(dir);
1868        run_test_cycle(&storage, &reopened).await;
1869
1870        pull_into(peer.db(), &storage, &peer.store_dir()).await;
1871        assert!(
1872            !row_exists(
1873                peer.db(),
1874                "SELECT 1 FROM files WHERE id = 'file-delete-reopen'"
1875            )
1876            .await,
1877            "the delete changeset reaches the peer after reopening",
1878        );
1879    }
1880
1881    #[tokio::test]
1882    async fn pending_write_drains_only_after_changeset_push() {
1883        let (_tmp, handle) = open_files_handle();
1884        let receipt = handle
1885            .sql(|sql| {
1886                sql.execute(
1887                    "INSERT INTO files (id, blob_id, size, _updated_at) \
1888                     VALUES ('file-retry-publish', NULL, 0, ?1)",
1889                    [sql.stamp()],
1890                )?;
1891                Ok(())
1892            })
1893            .await
1894            .expect("write before failed push");
1895        let keypair = crate::keys::UserKeypair::generate();
1896        let storage = merge_test_storage(&handle, &keypair).await;
1897        storage.home.fail_exact_create_before_call(1);
1898
1899        let first = storage
1900            .publish_pending(handle.db(), &handle.store_dir())
1901            .await;
1902        assert!(
1903            first
1904                .as_ref()
1905                .is_err_and(|error| error.contains("forced failure before exact create")),
1906            "the first cycle must report the append failure while preserving its outbox: {first:?}",
1907        );
1908        assert_eq!(
1909            handle
1910                .write_status(&receipt.write_id)
1911                .await
1912                .expect("write status after failed append"),
1913            coven_core::WriteStatus::Publishing,
1914        );
1915
1916        run_test_cycle(&storage, &handle).await;
1917        assert!(
1918            matches!(
1919                handle
1920                    .write_status(&receipt.write_id)
1921                    .await
1922                    .expect("write status after retry"),
1923                coven_core::WriteStatus::Published(_)
1924            ),
1925            "the pending write is published as an immutable Store commit after retry",
1926        );
1927    }
1928
1929    async fn write_raw_file(path: &std::path::Path, bytes: &[u8]) {
1930        crate::local_blob::write_atomic(path, bytes)
1931            .await
1932            .expect("write file");
1933    }
1934
1935    async fn cleanup_intent_count(handle: &CovenHandle, namespace: &str, id: &str) -> i64 {
1936        let namespace = namespace.to_string();
1937        let id = id.to_string();
1938        handle
1939            .sql(move |sql| {
1940                sql.query_row(
1941                    "SELECT count(*) FROM local_cleanup_intents \
1942                         WHERE namespace = ?1 AND blob_id = ?2",
1943                    params![namespace, id],
1944                    |row| row.get(0),
1945                )
1946                .map_err(CovenError::from)
1947            })
1948            .await
1949            .expect("count cleanup intents")
1950            .value
1951    }
1952
1953    #[tokio::test]
1954    async fn builder_open_runs_coven_and_host_migrations() {
1955        let (_tmp, handle) = open_files_handle();
1956        let has_coven_table: i64 = handle
1957            .sql(|sql| {
1958                sql.query_row(
1959                    "SELECT count(*) FROM sqlite_master WHERE type = 'table' AND name = 'protocol_state'",
1960                    [],
1961                    |row| row.get(0),
1962                ).map_err(CovenError::from)
1963            })
1964            .await
1965            .expect("query coven table")
1966            .value;
1967        let has_host_table: i64 = handle
1968            .sql(|sql| {
1969                sql.query_row(
1970                    "SELECT count(*) FROM sqlite_master WHERE type = 'table' AND name = 'files'",
1971                    [],
1972                    |row| row.get(0),
1973                )
1974                .map_err(CovenError::from)
1975            })
1976            .await
1977            .expect("query host table")
1978            .value;
1979        assert_eq!(has_coven_table, 1);
1980        assert_eq!(has_host_table, 1);
1981    }
1982
1983    #[tokio::test]
1984    async fn open_of_a_too_new_db_yields_the_matchable_migration_variant() {
1985        let tmp = tempfile::tempdir().expect("temp dir");
1986        let dir = StoreDir::new(tmp.path());
1987
1988        // Open with a two-step ladder so the db lands at synced-schema version 2.
1989        let ahead = Coven::builder(config(dir.clone()))
1990            .write_policy(WritePolicy::MergeConcurrent)
1991            .synced_tables(vec![files_table()])
1992            .migrations(vec![
1993                files_migration(),
1994                Migration::sql(2, "add-extra", "CREATE TABLE extra (id TEXT PRIMARY KEY)"),
1995            ])
1996            .open()
1997            .expect("open at version 2");
1998        drop(ahead);
1999
2000        // Reopen with only the first step: an older binary meeting a db a newer one
2001        // already migrated. The remedy is "update the app", so the host must be able
2002        // to match the specific variant rather than string-scrape a DbError.
2003        let reopened = Coven::builder(config(dir))
2004            .write_policy(WritePolicy::MergeConcurrent)
2005            .synced_tables(vec![files_table()])
2006            .migrations(vec![files_migration()])
2007            .open();
2008        assert!(matches!(
2009            reopened,
2010            Err(CovenError::Migration(MigrationError::SchemaTooNew {
2011                current: 2,
2012                supported: 1
2013            }))
2014        ));
2015    }
2016
2017    #[tokio::test]
2018    async fn sql_reads_writes_and_stamps() {
2019        let (_tmp, handle) = open_files_handle();
2020        let id = "file-sql".to_string();
2021        handle
2022            .sql(move |sql| {
2023                sql.execute(
2024                    "INSERT INTO files (id, blob_id, size, _updated_at) VALUES (?1, NULL, 0, ?2)",
2025                    params![id, sql.stamp()],
2026                )?;
2027                Ok(())
2028            })
2029            .await
2030            .expect("insert through sql");
2031        let count: i64 = handle
2032            .sql(|sql| {
2033                sql.query_row("SELECT count(*) FROM files", [], |row| row.get(0))
2034                    .map_err(CovenError::from)
2035            })
2036            .await
2037            .expect("count rows")
2038            .value;
2039        assert_eq!(count, 1);
2040    }
2041
2042    #[tokio::test]
2043    async fn sql_surfaces_sqlite_constraint_typed() {
2044        let (_tmp, handle) = open_files_handle();
2045        handle
2046            .sql(|sql| {
2047                sql.execute(
2048                    "INSERT INTO files (id, blob_id, size, _updated_at) VALUES (?1, NULL, 0, ?2)",
2049                    params!["duplicate-id", sql.stamp()],
2050                )?;
2051                Ok(())
2052            })
2053            .await
2054            .expect("seed row");
2055
2056        let result: CovenResult<WriteReceipt<()>> = handle
2057            .sql(|sql| {
2058                sql.execute(
2059                    "INSERT INTO files (id, blob_id, size, _updated_at) VALUES (?1, NULL, 0, ?2)",
2060                    params!["duplicate-id", sql.stamp()],
2061                )?;
2062                Ok(())
2063            })
2064            .await;
2065
2066        assert!(matches!(result, Err(CovenError::Sqlite(_))));
2067    }
2068
2069    #[tokio::test]
2070    async fn sql_read_sees_a_committed_write() {
2071        let (_tmp, handle) = open_files_handle();
2072        handle
2073            .sql(|sql| {
2074                sql.execute(
2075                    "INSERT INTO files (id, blob_id, size, _updated_at) \
2076                     VALUES ('file-read-your-write', NULL, 0, ?1)",
2077                    [sql.stamp()],
2078                )?;
2079                Ok(())
2080            })
2081            .await
2082            .expect("insert through sql");
2083
2084        // The read runs on the separate read-only connection; it must observe the
2085        // just-committed write (WAL read-your-writes for committed data).
2086        let id: String = handle
2087            .sql_read(|conn| {
2088                conn.query_row(
2089                    "SELECT id FROM files WHERE id = 'file-read-your-write'",
2090                    [],
2091                    |row| row.get(0),
2092                )
2093                .map_err(CovenError::from)
2094            })
2095            .await
2096            .expect("read the committed row back through sql_read");
2097        assert_eq!(id, "file-read-your-write");
2098    }
2099
2100    #[tokio::test]
2101    async fn sql_read_cannot_write() {
2102        let (_tmp, handle) = open_files_handle();
2103        let result: CovenResult<()> = handle
2104            .sql_read(|conn| {
2105                conn.execute(
2106                    "INSERT INTO files (id, blob_id, size, _updated_at) \
2107                     VALUES ('file-read-only', NULL, 0, '0')",
2108                    [],
2109                )
2110                .map_err(CovenError::from)?;
2111                Ok(())
2112            })
2113            .await;
2114        // The companion connection is SQLITE_OPEN_READONLY, so the write is refused
2115        // at the SQLite layer with the readonly code — a read cannot escape the sync
2116        // journal because it cannot write at all.
2117        match result {
2118            Err(CovenError::Sqlite(rusqlite::Error::SqliteFailure(err, _))) => {
2119                assert_eq!(err.code, rusqlite::ErrorCode::ReadOnly);
2120            }
2121            other => panic!("expected a readonly sqlite failure, got {other:?}"),
2122        }
2123
2124        let count: i64 = handle
2125            .sql_read(|conn| {
2126                conn.query_row(
2127                    "SELECT count(*) FROM files WHERE id = 'file-read-only'",
2128                    [],
2129                    |row| row.get(0),
2130                )
2131                .map_err(CovenError::from)
2132            })
2133            .await
2134            .expect("count rows through sql_read");
2135        assert_eq!(count, 0, "the refused write left no row behind");
2136    }
2137
2138    #[tokio::test]
2139    async fn open_on_a_fresh_store_serves_sql_read() {
2140        // A fresh (empty) directory: `open` runs the writer's migrations, then opens
2141        // the read connection against the schema they created. A `sql_read` over the
2142        // host table then succeeds rather than failing on a missing table — proof the
2143        // read connection opened after, not before, the schema exists.
2144        let tmp = tempfile::tempdir().expect("temp dir");
2145        let dir = StoreDir::new(tmp.path());
2146        let handle = Coven::builder(config(dir))
2147            .write_policy(WritePolicy::MergeConcurrent)
2148            .synced_tables(vec![files_table()])
2149            .migrations(vec![files_migration()])
2150            .open()
2151            .expect("open on an empty directory");
2152        let count: i64 = handle
2153            .sql_read(|conn| {
2154                conn.query_row("SELECT count(*) FROM files", [], |row| row.get(0))
2155                    .map_err(CovenError::from)
2156            })
2157            .await
2158            .expect("sql_read on a fresh store");
2159        assert_eq!(count, 0);
2160    }
2161
2162    #[tokio::test]
2163    async fn open_removes_orphaned_local_blob_temps() {
2164        let tmp = tempfile::tempdir().expect("temp dir");
2165        let dir = StoreDir::new(tmp.path());
2166        let final_path = dir
2167            .local_blob_path("media-files", "tempaaaa")
2168            .expect("local path");
2169        let temp = local_stage_temp_path(&final_path).expect("stage temp path");
2170        write_raw_file(&temp, b"interrupted write").await;
2171
2172        let _handle = Coven::builder(config(dir.clone()))
2173            .write_policy(WritePolicy::MergeConcurrent)
2174            .synced_tables(vec![files_table()])
2175            .migrations(vec![files_migration()])
2176            .open()
2177            .expect("open handle");
2178
2179        assert!(
2180            !temp.exists(),
2181            "open removes orphaned local blob staging temps"
2182        );
2183        assert!(
2184            !dir.local_blob_path("media-files", "tempaaaa")
2185                .expect("local path")
2186                .exists(),
2187            "the interrupted blob has no committed final file"
2188        );
2189    }
2190
2191    #[tokio::test]
2192    async fn write_inserts_row_and_host_provided_blob() {
2193        let (_tmp, handle) = open_files_handle();
2194        let bytes = b"piece-bytes".to_vec();
2195        let hash = crate::blob::content_hash(&bytes);
2196        handle
2197            .write(
2198                {
2199                    let bytes = bytes.clone();
2200                    move |w| {
2201                        w.put_blob("media-files", "blobaaaa", bytes);
2202                        Ok(())
2203                    }
2204                },
2205                move |sql| {
2206                    sql.execute(
2207                        "INSERT INTO files (id, blob_id, size, hash, _updated_at) \
2208                         VALUES (?1, ?2, ?3, ?4, ?5)",
2209                        params!["file-1", "blobaaaa", bytes.len() as i64, hash, sql.stamp()],
2210                    )?;
2211                    Ok(())
2212                },
2213            )
2214            .await
2215            .expect("write row and blob");
2216        let path = handle
2217            .store_dir()
2218            .local_blob_path("media-files", "blobaaaa")
2219            .expect("local path");
2220        assert_eq!(
2221            std::fs::read(path).expect("read local blob"),
2222            b"piece-bytes"
2223        );
2224    }
2225
2226    #[tokio::test]
2227    async fn orphaned_final_blob_is_replaced_by_next_write() {
2228        let (_tmp, handle) = open_files_handle();
2229        let path = handle
2230            .store_dir()
2231            .local_blob_path("media-files", "orphaaaa")
2232            .expect("local path");
2233        write_raw_file(&path, b"orphaned bytes").await;
2234
2235        handle
2236            .write(
2237                |w| {
2238                    w.put_blob("media-files", "orphaaaa", b"committed bytes".to_vec());
2239                    Ok(())
2240                },
2241                |sql| {
2242                    sql.execute(
2243                        "INSERT INTO files (id, blob_id, size, hash, _updated_at) \
2244                         VALUES (?1, ?2, ?3, ?4, ?5)",
2245                        params![
2246                            "file-orphan",
2247                            "orphaaaa",
2248                            15i64,
2249                            crate::blob::content_hash(b"committed bytes"),
2250                            sql.stamp()
2251                        ],
2252                    )?;
2253                    Ok(())
2254                },
2255            )
2256            .await
2257            .expect("write replaces orphaned final blob");
2258
2259        assert_eq!(
2260            std::fs::read(path).expect("read committed blob"),
2261            b"committed bytes"
2262        );
2263    }
2264
2265    #[tokio::test]
2266    async fn put_blob_rejects_id_already_referenced_by_a_row() {
2267        let (_tmp, handle) = open_files_handle();
2268        handle
2269            .write(
2270                |w| {
2271                    w.put_blob("media-files", "dupeaaaa", b"original".to_vec());
2272                    Ok(())
2273                },
2274                |sql| {
2275                    sql.execute(
2276                        "INSERT INTO files (id, blob_id, size, hash, _updated_at) \
2277                         VALUES (?1, ?2, ?3, ?4, ?5)",
2278                        params![
2279                            "file-original",
2280                            "dupeaaaa",
2281                            8i64,
2282                            crate::blob::content_hash(b"original"),
2283                            sql.stamp()
2284                        ],
2285                    )?;
2286                    Ok(())
2287                },
2288            )
2289            .await
2290            .expect("seed original blob");
2291
2292        let result: CovenResult<WriteReceipt<()>> = handle
2293            .write(
2294                |w| {
2295                    w.put_blob("media-files", "dupeaaaa", b"replacement".to_vec());
2296                    Ok(())
2297                },
2298                |sql| {
2299                    sql.execute(
2300                        "INSERT INTO files (id, blob_id, size, hash, _updated_at) \
2301                         VALUES (?1, ?2, ?3, ?4, ?5)",
2302                        params![
2303                            "file-replacement",
2304                            "dupeaaaa",
2305                            11i64,
2306                            crate::blob::content_hash(b"replacement"),
2307                            sql.stamp()
2308                        ],
2309                    )?;
2310                    Ok(())
2311                },
2312            )
2313            .await;
2314
2315        assert!(matches!(
2316            result,
2317            Err(CovenError::BlobAlreadyReferenced { .. })
2318        ));
2319        let path = handle
2320            .store_dir()
2321            .local_blob_path("media-files", "dupeaaaa")
2322            .expect("dupe path");
2323        assert_eq!(
2324            std::fs::read(path).expect("read original blob"),
2325            b"original"
2326        );
2327        let replacement_rows: i64 = handle
2328            .sql(|sql| {
2329                sql.query_row(
2330                    "SELECT count(*) FROM files WHERE id = 'file-replacement'",
2331                    [],
2332                    |row| row.get(0),
2333                )
2334                .map_err(CovenError::from)
2335            })
2336            .await
2337            .expect("count replacement rows")
2338            .value;
2339        assert_eq!(replacement_rows, 0);
2340    }
2341
2342    #[tokio::test]
2343    async fn remote_root_host_provided_write_reads_staging_through_handle_before_upload() {
2344        let (_tmp, handle) = open_remote_root_files_handle();
2345        let expected = b"remote-root-host-provided-staging-bytes".to_vec();
2346        let bytes = expected.clone();
2347        let hash = crate::blob::content_hash(&bytes);
2348
2349        handle
2350            .write(
2351                {
2352                    let bytes = bytes.clone();
2353                    move |w| {
2354                        w.put_blob("media-files", "rrhpaaaa", bytes);
2355                        Ok(())
2356                    }
2357                },
2358                move |sql| {
2359                    sql.execute(
2360                        "INSERT INTO files (id, blob_id, size, hash, _updated_at) \
2361                         VALUES (?1, ?2, ?3, ?4, ?5)",
2362                        params![
2363                            "file-remote-root",
2364                            "rrhpaaaa",
2365                            bytes.len() as i64,
2366                            hash,
2367                            sql.stamp()
2368                        ],
2369                    )?;
2370                    Ok(())
2371                },
2372            )
2373            .await
2374            .expect("write remote-root row and host-provided blob");
2375        let blob = handle
2376            .row_blob_ref("files", "file-remote-root")
2377            .await
2378            .expect("capture remote-root blob row");
2379
2380        let whole = handle
2381            .read_blob(&blob)
2382            .await
2383            .expect("read_blob serves upload staging before sync upload");
2384        assert_eq!(
2385            whole, expected,
2386            "read_blob returns the bytes written through handle.write",
2387        );
2388
2389        let (offset, len) = (12u64, 19u64);
2390        let range = handle
2391            .open_blob_stream(&blob, offset, len)
2392            .await
2393            .expect("open_blob_stream serves upload staging before sync upload");
2394        assert_eq!(
2395            range,
2396            &expected[offset as usize..(offset + len) as usize],
2397            "open_blob_stream returns the requested slice of the staged bytes",
2398        );
2399    }
2400
2401    #[tokio::test]
2402    async fn sql_failure_removes_staged_blob() {
2403        let (_tmp, handle) = open_files_handle();
2404        let err = handle
2405            .write(
2406                |w| {
2407                    w.put_blob("media-files", "blobbbbb", b"staged".to_vec());
2408                    Ok(())
2409                },
2410                |_sql| Err::<(), CovenError>(CovenError::Blob("sql failed".to_string())),
2411            )
2412            .await
2413            .expect_err("write fails");
2414        assert!(err.to_string().contains("sql failed"));
2415        let path = handle
2416            .store_dir()
2417            .local_blob_path("media-files", "blobbbbb")
2418            .expect("local path");
2419        assert!(!path.exists());
2420    }
2421
2422    #[tokio::test]
2423    async fn blob_stage_failure_does_not_run_sql() {
2424        let (_tmp, handle) = open_files_handle();
2425        let result: CovenResult<WriteReceipt<()>> = handle
2426            .write(
2427                |w| {
2428                    w.put_blob("media-files", "..", b"bad".to_vec());
2429                    Ok(())
2430                },
2431                |sql| {
2432                    sql.execute(
2433                        "INSERT INTO files (id, blob_id, size, _updated_at) \
2434                         VALUES ('should-not-exist', NULL, 0, ?1)",
2435                        [sql.stamp()],
2436                    )?;
2437                    Ok(())
2438                },
2439            )
2440            .await;
2441        // A blob id that can't form a safe path is its own typed error, not the
2442        // generic blob catch-all.
2443        assert!(matches!(result, Err(CovenError::UnsafeBlobPath(_))));
2444        let count: i64 = handle
2445            .sql(|sql| {
2446                sql.query_row("SELECT count(*) FROM files", [], |row| row.get(0))
2447                    .map_err(CovenError::from)
2448            })
2449            .await
2450            .expect("count rows")
2451            .value;
2452        assert_eq!(count, 0);
2453    }
2454
2455    #[tokio::test]
2456    async fn replacement_deletes_old_blob_after_sql_drops_reference() {
2457        let (_tmp, handle) = open_files_handle();
2458        crate::blob::local_files::store(&handle.store_dir(), "media-files", "oldaaaa", b"old")
2459            .await
2460            .expect("store old");
2461        handle
2462            .sql(|sql| {
2463                sql.execute(
2464                    "INSERT INTO files (id, blob_id, size, hash, _updated_at) \
2465                     VALUES (?1, ?2, 3, ?3, ?4)",
2466                    params![
2467                        "file-1",
2468                        "oldaaaa",
2469                        crate::blob::content_hash(b"old"),
2470                        sql.stamp()
2471                    ],
2472                )?;
2473                Ok(())
2474            })
2475            .await
2476            .expect("seed row");
2477        publish_current_writes(&handle).await;
2478        crate::blob::local_files::store(&handle.store_dir(), "media-files", "oldaaaa", b"old")
2479            .await
2480            .expect("restore published blob locally");
2481        let old_ref = BlobRef {
2482            namespace: "media-files".to_string(),
2483            id: "oldaaaa".to_string(),
2484            scope: BlobScope::Master,
2485            cloud_path: None,
2486            provenance: Provenance::HostProvided,
2487            fill: CacheFill::CacheLazy,
2488        };
2489        handle
2490            .write(
2491                move |w| {
2492                    w.put_blob("media-files", "newaaaa", b"new".to_vec());
2493                    w.delete_blob(old_ref);
2494                    Ok(())
2495                },
2496                move |sql| {
2497                    sql.execute(
2498                        "UPDATE files SET blob_id = ?1, size = 3, hash = ?2, \
2499                         _updated_at = ?3 WHERE id = 'file-1'",
2500                        params!["newaaaa", crate::blob::content_hash(b"new"), sql.stamp()],
2501                    )?;
2502                    Ok(())
2503                },
2504            )
2505            .await
2506            .expect("replace blob");
2507        assert!(!handle
2508            .store_dir()
2509            .local_blob_path("media-files", "oldaaaa")
2510            .expect("old path")
2511            .exists());
2512        assert!(handle
2513            .store_dir()
2514            .local_blob_path("media-files", "newaaaa")
2515            .expect("new path")
2516            .exists());
2517    }
2518
2519    async fn queue_replacement_before_sync(
2520        handle: &CovenHandle,
2521    ) -> (WriteId, WriteId, std::path::PathBuf) {
2522        let first = handle
2523            .write(
2524                |batch| {
2525                    batch.put_blob("media-files", "ownedaaa", b"first".to_vec());
2526                    Ok(())
2527                },
2528                |sql| {
2529                    sql.execute(
2530                        "INSERT INTO files (id, blob_id, size, hash, _updated_at) \
2531                         VALUES ('owned-file', 'ownedaaa', 5, ?1, ?2)",
2532                        params![crate::blob::content_hash(b"first"), sql.stamp()],
2533                    )?;
2534                    Ok(())
2535                },
2536            )
2537            .await
2538            .expect("queue first blob write");
2539        let first_path = handle
2540            .store_dir()
2541            .local_blob_path("media-files", "ownedaaa")
2542            .expect("first blob path");
2543        let first_blob = BlobRef {
2544            namespace: "media-files".to_string(),
2545            id: "ownedaaa".to_string(),
2546            scope: BlobScope::Master,
2547            cloud_path: None,
2548            provenance: Provenance::HostProvided,
2549            fill: CacheFill::CacheLazy,
2550        };
2551        let second = handle
2552            .write(
2553                move |batch| {
2554                    batch.put_blob("media-files", "ownedbbb", b"second".to_vec());
2555                    batch.delete_blob(first_blob);
2556                    Ok(())
2557                },
2558                |sql| {
2559                    sql.execute(
2560                        "UPDATE files SET blob_id = 'ownedbbb', size = 6, hash = ?1, \
2561                         _updated_at = ?2 \
2562                         WHERE id = 'owned-file'",
2563                        params![crate::blob::content_hash(b"second"), sql.stamp()],
2564                    )?;
2565                    Ok(())
2566                },
2567            )
2568            .await
2569            .expect("queue replacement write");
2570        (first.write_id, second.write_id, first_path)
2571    }
2572
2573    async fn assert_replacement_publishes_in_order(
2574        handle: &CovenHandle,
2575        first_write: &WriteId,
2576        second_write: &WriteId,
2577        first_path: &std::path::Path,
2578    ) {
2579        let keypair = crate::keys::UserKeypair::generate();
2580        let storage = merge_test_storage(handle, &keypair).await;
2581        run_test_cycle(&storage, handle).await;
2582
2583        let first_sequence = match handle
2584            .write_status(first_write)
2585            .await
2586            .expect("first status")
2587        {
2588            coven_core::WriteStatus::Published(position) => position.commit().coord.sequence(),
2589            status => panic!("first replacement write is not published: {status:?}"),
2590        };
2591        assert_eq!(
2592            handle
2593                .write_status(second_write)
2594                .await
2595                .expect("second status after first publication"),
2596            coven_core::WriteStatus::Pending,
2597            "one Store publication consumes one pending host transaction",
2598        );
2599
2600        run_test_cycle(&storage, handle).await;
2601        let second_sequence = match handle
2602            .write_status(second_write)
2603            .await
2604            .expect("second status")
2605        {
2606            coven_core::WriteStatus::Published(position) => position.commit().coord.sequence(),
2607            status => panic!("second replacement write is not published: {status:?}"),
2608        };
2609        assert_eq!(
2610            second_sequence,
2611            first_sequence + 1,
2612            "replacement writes publish in their host transaction order"
2613        );
2614        let blob_objects = storage
2615            .home
2616            .keys()
2617            .into_iter()
2618            .filter(|key| key.starts_with("media-files/opaque/"))
2619            .count();
2620        assert_eq!(blob_objects, 2, "both row versions upload their blob bytes");
2621        assert!(
2622            !first_path.exists(),
2623            "the first write releases its local bytes after publication"
2624        );
2625    }
2626
2627    #[tokio::test]
2628    async fn pending_write_owns_blob_bytes_until_its_publication() {
2629        let local = tokio::task::LocalSet::new();
2630        local
2631            .run_until(async {
2632                tokio::task::spawn_local(run_pending_write_owns_blob_bytes_until_its_publication())
2633                    .await
2634                    .expect("pending-write blob ownership test task");
2635            })
2636            .await;
2637    }
2638
2639    async fn run_pending_write_owns_blob_bytes_until_its_publication() {
2640        let (_tmp, handle) = open_files_handle();
2641        let (first_write, second_write, first_path) = queue_replacement_before_sync(&handle).await;
2642
2643        assert_eq!(
2644            std::fs::read(&first_path).expect("first write still owns its bytes"),
2645            b"first"
2646        );
2647        let overwrite: CovenResult<WriteReceipt<()>> = handle
2648            .write(
2649                |batch| {
2650                    batch.put_blob("media-files", "ownedaaa", b"overwritten".to_vec());
2651                    Ok(())
2652                },
2653                |sql| {
2654                    sql.execute(
2655                        "INSERT INTO files (id, blob_id, size, hash, _updated_at) \
2656                         VALUES ('overwrite', 'ownedaaa', 11, ?1, ?2)",
2657                        params![crate::blob::content_hash(b"overwritten"), sql.stamp()],
2658                    )?;
2659                    Ok(())
2660                },
2661            )
2662            .await;
2663        assert!(matches!(
2664            overwrite,
2665            Err(CovenError::BlobOwnedByPendingWrite { .. })
2666        ));
2667        assert_eq!(
2668            std::fs::read(&first_path).expect("lease prevents overwrite"),
2669            b"first"
2670        );
2671        assert_replacement_publishes_in_order(&handle, &first_write, &second_write, &first_path)
2672            .await;
2673    }
2674
2675    #[tokio::test]
2676    async fn pending_write_blob_ownership_survives_restart() {
2677        let local = tokio::task::LocalSet::new();
2678        local
2679            .run_until(async {
2680                tokio::task::spawn_local(run_pending_write_blob_ownership_survives_restart())
2681                    .await
2682                    .expect("reopened pending-write blob ownership test task");
2683            })
2684            .await;
2685    }
2686
2687    async fn run_pending_write_blob_ownership_survives_restart() {
2688        let tmp = tempfile::tempdir().expect("temp dir");
2689        let dir = StoreDir::new(tmp.path());
2690        let handle = open_files_handle_in(dir.clone());
2691        let (first_write, second_write, first_path) = queue_replacement_before_sync(&handle).await;
2692        drop(handle);
2693
2694        let reopened = open_files_handle_in(dir);
2695        assert_eq!(
2696            std::fs::read(&first_path).expect("reopened first write still owns its bytes"),
2697            b"first"
2698        );
2699        assert_replacement_publishes_in_order(&reopened, &first_write, &second_write, &first_path)
2700            .await;
2701    }
2702
2703    #[tokio::test]
2704    async fn cache_eviction_cannot_remove_pending_write_blob_bytes() {
2705        let (_tmp, handle) = open_files_handle();
2706        let (_first_write, _second_write, first_path) =
2707            queue_replacement_before_sync(&handle).await;
2708        let first_blob = BlobRef {
2709            namespace: "media-files".to_string(),
2710            id: "ownedaaa".to_string(),
2711            scope: BlobScope::Master,
2712            cloud_path: None,
2713            provenance: Provenance::HostProvided,
2714            fill: CacheFill::CacheLazy,
2715        };
2716
2717        handle
2718            .evict_blob(&first_blob)
2719            .await
2720            .expect("evict only re-fetchable cache copies");
2721
2722        assert_eq!(
2723            std::fs::read(first_path).expect("pending write still owns local-store bytes"),
2724            b"first"
2725        );
2726    }
2727
2728    #[tokio::test]
2729    async fn author_delete_drops_all_local_blob_copies() {
2730        let (_tmp, handle) = open_files_handle();
2731        crate::blob::local_files::store(&handle.store_dir(), "media-files", "oldcccc", b"old")
2732            .await
2733            .expect("store old");
2734        let pinned = handle
2735            .store_dir()
2736            .pinned_blob_path("media-files", "oldcccc")
2737            .expect("pinned path");
2738        let cached = handle
2739            .store_dir()
2740            .cache_blob_path("media-files", "oldcccc")
2741            .expect("cache path");
2742        handle
2743            .sql(|sql| {
2744                sql.execute(
2745                    "INSERT INTO files (id, blob_id, size, hash, _updated_at) \
2746                     VALUES (?1, ?2, 3, ?3, ?4)",
2747                    params![
2748                        "file-1",
2749                        "oldcccc",
2750                        crate::blob::content_hash(b"old"),
2751                        sql.stamp()
2752                    ],
2753                )?;
2754                Ok(())
2755            })
2756            .await
2757            .expect("seed row");
2758        publish_current_writes(&handle).await;
2759        crate::blob::local_files::store(&handle.store_dir(), "media-files", "oldcccc", b"old")
2760            .await
2761            .expect("restore published blob locally");
2762        write_raw_file(&pinned, b"pinned").await;
2763        write_raw_file(&cached, b"cached").await;
2764        let old_ref = BlobRef {
2765            namespace: "media-files".to_string(),
2766            id: "oldcccc".to_string(),
2767            scope: BlobScope::Master,
2768            cloud_path: None,
2769            provenance: Provenance::HostProvided,
2770            fill: CacheFill::CacheLazy,
2771        };
2772
2773        handle
2774            .write(
2775                move |w| {
2776                    w.delete_blob(old_ref);
2777                    Ok(())
2778                },
2779                move |sql| {
2780                    sql.execute(
2781                        "UPDATE files SET blob_id = NULL, size = 0, hash = NULL, _updated_at = ?1 \
2782                         WHERE id = 'file-1'",
2783                        [sql.stamp()],
2784                    )?;
2785                    Ok(())
2786                },
2787            )
2788            .await
2789            .expect("delete blob");
2790
2791        assert!(!handle
2792            .store_dir()
2793            .local_blob_path("media-files", "oldcccc")
2794            .expect("local path")
2795            .exists());
2796        assert!(!pinned.exists(), "pinned copy is removed");
2797        assert!(!cached.exists(), "cache copy is removed");
2798    }
2799
2800    #[tokio::test]
2801    async fn failed_local_blob_cleanup_keeps_intent_for_later_drain() {
2802        let (_tmp, handle) = open_files_handle();
2803        crate::blob::local_files::store(&handle.store_dir(), "media-files", "oldddddd", b"old")
2804            .await
2805            .expect("store old");
2806        let pinned = handle
2807            .store_dir()
2808            .pinned_blob_path("media-files", "oldddddd")
2809            .expect("pinned path");
2810        handle
2811            .sql(|sql| {
2812                sql.execute(
2813                    "INSERT INTO files (id, blob_id, size, hash, _updated_at) \
2814                     VALUES (?1, ?2, 3, ?3, ?4)",
2815                    params![
2816                        "file-1",
2817                        "oldddddd",
2818                        crate::blob::content_hash(b"old"),
2819                        sql.stamp()
2820                    ],
2821                )?;
2822                Ok(())
2823            })
2824            .await
2825            .expect("seed row");
2826        publish_current_writes(&handle).await;
2827        crate::blob::local_files::store(&handle.store_dir(), "media-files", "oldddddd", b"old")
2828            .await
2829            .expect("restore published blob locally");
2830        std::fs::create_dir_all(&pinned).expect("create pinned blocker");
2831        let old_ref = BlobRef {
2832            namespace: "media-files".to_string(),
2833            id: "oldddddd".to_string(),
2834            scope: BlobScope::Master,
2835            cloud_path: None,
2836            provenance: Provenance::HostProvided,
2837            fill: CacheFill::CacheLazy,
2838        };
2839
2840        handle
2841            .write(
2842                move |w| {
2843                    w.delete_blob(old_ref);
2844                    Ok(())
2845                },
2846                |sql| {
2847                    sql.execute(
2848                        "UPDATE files SET blob_id = NULL, size = 0, hash = NULL, _updated_at = ?1 \
2849                         WHERE id = 'file-1'",
2850                        [sql.stamp()],
2851                    )?;
2852                    Ok(())
2853                },
2854            )
2855            .await
2856            .expect("row delete commits despite cleanup failure");
2857
2858        assert_eq!(
2859            cleanup_intent_count(&handle, "media-files", "oldddddd").await,
2860            1
2861        );
2862        assert!(handle
2863            .store_dir()
2864            .local_blob_path("media-files", "oldddddd")
2865            .expect("local path")
2866            .exists());
2867
2868        std::fs::remove_dir_all(&pinned).expect("remove pinned blocker");
2869        handle
2870            .write(
2871                |_| Ok(()),
2872                |sql| {
2873                    sql.execute(
2874                        "INSERT INTO files (id, blob_id, size, _updated_at) \
2875                         VALUES (?1, NULL, 0, ?2)",
2876                        params!["drain-trigger", sql.stamp()],
2877                    )?;
2878                    Ok(())
2879                },
2880            )
2881            .await
2882            .expect("later committed write drains pending cleanup");
2883
2884        assert_eq!(
2885            cleanup_intent_count(&handle, "media-files", "oldddddd").await,
2886            0
2887        );
2888        assert!(!handle
2889            .store_dir()
2890            .local_blob_path("media-files", "oldddddd")
2891            .expect("local path")
2892            .exists());
2893    }
2894
2895    #[tokio::test]
2896    async fn write_drain_preserves_a_blob_still_referenced_after_remote_delete() {
2897        let (_tmp, handle) = open_files_handle();
2898        let blob_id = "shared01";
2899        handle
2900            .sql(move |sql| {
2901                let hash = crate::blob::content_hash(b"live");
2902                for id in ["remote-deletes", "still-live"] {
2903                    sql.execute(
2904                        "INSERT INTO files (id, blob_id, size, hash, _updated_at) \
2905                         VALUES (?1, 'shared01', 4, ?2, \
2906                                 '0000000001000-0000-dev-remote')",
2907                        params![id, &hash],
2908                    )?;
2909                }
2910                Ok(())
2911            })
2912            .await
2913            .expect("seed two rows sharing the blob");
2914
2915        let local = handle
2916            .store_dir()
2917            .local_blob_path("media-files", blob_id)
2918            .expect("local path");
2919        let pinned = handle
2920            .store_dir()
2921            .pinned_blob_path("media-files", blob_id)
2922            .expect("pinned path");
2923        let cached = handle
2924            .store_dir()
2925            .cache_blob_path("media-files", blob_id)
2926            .expect("cache path");
2927        write_raw_file(&local, b"live").await;
2928        write_raw_file(&pinned, b"live").await;
2929        write_raw_file(&cached, b"live").await;
2930
2931        let (_source_tmp, source) = open_files_handle();
2932        let storage = Arc::new(
2933            TestStore::create(
2934                source.db(),
2935                "lib-test",
2936                crate::keys::UserKeypair::generate(),
2937            )
2938            .await
2939            .expect("create remote exact test Store"),
2940        );
2941        source
2942            .write(
2943                |batch| {
2944                    batch.put_blob("media-files", "shared01", b"live".to_vec());
2945                    Ok(())
2946                },
2947                |sql| {
2948                    sql.execute(
2949                        "INSERT INTO files (id, blob_id, size, hash, _updated_at) \
2950                         VALUES ('remote-deletes', 'shared01', 4, ?1, ?2)",
2951                        params![crate::blob::content_hash(b"live"), sql.stamp()],
2952                    )?;
2953                    Ok(())
2954                },
2955            )
2956            .await
2957            .expect("insert remote row");
2958        storage
2959            .publish_pending(source.db(), &source.store_dir())
2960            .await
2961            .expect("publish remote insert");
2962        let delete = source
2963            .sql(|sql| {
2964                sql.execute("DELETE FROM files WHERE id = 'remote-deletes'", [])?;
2965                Ok(())
2966            })
2967            .await
2968            .expect("delete remote row");
2969        storage
2970            .publish_pending(source.db(), &source.store_dir())
2971            .await
2972            .expect("publish remote delete");
2973
2974        assert!(matches!(
2975            source
2976                .write_status(&delete.write_id)
2977                .await
2978                .expect("remote delete status"),
2979            coven_core::WriteStatus::Published(_)
2980        ));
2981        let (device_id, sequence) = source
2982            .db()
2983            .call(|conn| {
2984                conn.query_row(
2985                    "SELECT device_id, seq FROM materialized_commits ORDER BY seq DESC LIMIT 1",
2986                    [],
2987                    |row| Ok((row.get::<_, String>(0)?, row.get::<_, i64>(1)?)),
2988                )
2989                .map_err(coven_core::database::DbError::from)
2990            })
2991            .await
2992            .expect("load remote delete stream coordinate");
2993        let sequence = u64::try_from(sequence).expect("remote delete sequence is non-negative");
2994
2995        let (commit_reached, resume_pull) = handle.db().arm_test_pause(
2996            coven_core::database::DatabaseTestPoint::PullAfterRemoteCommit {
2997                device_id,
2998                seq: sequence,
2999            },
3000        );
3001        let pull_handle = handle.clone();
3002        let pull_storage = storage.clone();
3003        let pull = tokio::spawn(async move {
3004            pull_into(
3005                pull_handle.db(),
3006                pull_storage.as_ref(),
3007                &pull_handle.store_dir(),
3008            )
3009            .await
3010        });
3011
3012        commit_reached.notified().await;
3013        handle
3014            .write(
3015                |_| Ok(()),
3016                |sql| {
3017                    sql.execute(
3018                        "INSERT INTO files (id, blob_id, size, _updated_at) \
3019                         VALUES ('drain-trigger', NULL, 0, ?1)",
3020                        [sql.stamp()],
3021                    )?;
3022                    Ok(())
3023                },
3024            )
3025            .await
3026            .expect("write drains cleanup queue");
3027        resume_pull.notify_one();
3028        pull.await.expect("pull task");
3029
3030        assert!(
3031            !row_exists(
3032                handle.db(),
3033                "SELECT 1 FROM files WHERE id = 'remote-deletes'"
3034            )
3035            .await
3036        );
3037        assert!(row_exists(handle.db(), "SELECT 1 FROM files WHERE id = 'still-live'").await);
3038        assert!(local.exists(), "the live blob's local copy survives");
3039        assert!(pinned.exists(), "the live blob's pinned copy survives");
3040        assert!(cached.exists(), "the live blob's cache copy survives");
3041        assert_eq!(
3042            cleanup_intent_count(&handle, "media-files", blob_id).await,
3043            0,
3044        );
3045    }
3046
3047    #[tokio::test]
3048    async fn replacement_is_rejected_while_sql_still_references_old_blob() {
3049        let (_tmp, handle) = open_files_handle();
3050        crate::blob::local_files::store(&handle.store_dir(), "media-files", "oldbbbb", b"old")
3051            .await
3052            .expect("store old");
3053        handle
3054            .sql(|sql| {
3055                sql.execute(
3056                    "INSERT INTO files (id, blob_id, size, hash, _updated_at) \
3057                     VALUES (?1, ?2, 3, ?3, ?4)",
3058                    params![
3059                        "file-1",
3060                        "oldbbbb",
3061                        crate::blob::content_hash(b"old"),
3062                        sql.stamp()
3063                    ],
3064                )?;
3065                Ok(())
3066            })
3067            .await
3068            .expect("seed row");
3069        let old_ref = BlobRef {
3070            namespace: "media-files".to_string(),
3071            id: "oldbbbb".to_string(),
3072            scope: BlobScope::Master,
3073            cloud_path: None,
3074            provenance: Provenance::HostProvided,
3075            fill: CacheFill::CacheLazy,
3076        };
3077        let result: CovenResult<WriteReceipt<()>> = handle
3078            .write(
3079                move |w| {
3080                    w.put_blob("media-files", "newbbbb", b"new".to_vec());
3081                    w.delete_blob(old_ref);
3082                    Ok(())
3083                },
3084                move |sql| {
3085                    sql.execute(
3086                        "UPDATE files SET _updated_at = ?1 WHERE id = 'file-1'",
3087                        [sql.stamp()],
3088                    )?;
3089                    Ok(())
3090                },
3091            )
3092            .await;
3093        assert!(matches!(
3094            result,
3095            Err(CovenError::BlobStillReferenced { .. })
3096        ));
3097        assert!(handle
3098            .store_dir()
3099            .local_blob_path("media-files", "oldbbbb")
3100            .expect("old path")
3101            .exists());
3102        assert!(!handle
3103            .store_dir()
3104            .local_blob_path("media-files", "newbbbb")
3105            .expect("new path")
3106            .exists());
3107    }
3108
3109    #[tokio::test]
3110    async fn sql_panic_removes_moved_blob() {
3111        let (_tmp, handle) = open_files_handle();
3112        let result: CovenResult<WriteReceipt<()>> = handle
3113            .write(
3114                |w| {
3115                    w.put_blob("media-files", "panicccc", b"new".to_vec());
3116                    Ok(())
3117                },
3118                |_sql| panic!("boom"),
3119            )
3120            .await;
3121        assert!(matches!(result, Err(CovenError::WriteClosurePanicked)));
3122        assert!(!handle
3123            .store_dir()
3124            .local_blob_path("media-files", "panicccc")
3125            .expect("panic path")
3126            .exists());
3127    }
3128
3129    #[tokio::test]
3130    async fn concurrent_duplicate_blob_write_does_not_delete_committed_blob() {
3131        let (_tmp, handle) = open_files_handle();
3132        let winner = handle.clone();
3133        let loser = handle.clone();
3134
3135        let write_winner = tokio::spawn(async move {
3136            winner
3137                .write(
3138                    move |w| {
3139                        w.put_blob("media-files", "raceblob", b"committed".to_vec());
3140                        Ok(())
3141                    },
3142                    move |sql| {
3143                        sql.execute(
3144                            "INSERT INTO files (id, blob_id, size, hash, _updated_at) \
3145                             VALUES (?1, ?2, ?3, ?4, ?5)",
3146                            params![
3147                                "winner",
3148                                "raceblob",
3149                                9i64,
3150                                crate::blob::content_hash(b"committed"),
3151                                sql.stamp()
3152                            ],
3153                        )?;
3154                        Ok(())
3155                    },
3156                )
3157                .await
3158        });
3159
3160        let write_loser = tokio::spawn(async move {
3161            loser
3162                .write(
3163                    move |w| {
3164                        w.put_blob("media-files", "raceblob", b"rolled-back".to_vec());
3165                        Ok(())
3166                    },
3167                    |_sql| Err::<(), CovenError>(CovenError::Blob("force rollback".to_string())),
3168                )
3169                .await
3170        });
3171
3172        let winner_result = write_winner.await.expect("winner task");
3173        let loser_result = write_loser.await.expect("loser task");
3174        assert!(winner_result.is_ok() || loser_result.is_ok());
3175        assert!(winner_result.is_err() || loser_result.is_err());
3176
3177        let path = handle
3178            .store_dir()
3179            .local_blob_path("media-files", "raceblob")
3180            .expect("race path");
3181        assert_eq!(std::fs::read(path).expect("read race blob"), b"committed");
3182        let rows: i64 = handle
3183            .sql(|sql| {
3184                sql.query_row(
3185                    "SELECT count(*) FROM files WHERE id = 'winner'",
3186                    [],
3187                    |row| row.get(0),
3188                )
3189                .map_err(CovenError::from)
3190            })
3191            .await
3192            .expect("count winner row")
3193            .value;
3194        assert_eq!(rows, 1);
3195    }
3196
3197    /// A [`CloudHome`] that blocks the sync loop inside a cycle on demand. It
3198    /// delegates every call to an inner [`InMemoryCloudHome`], but once `armed`,
3199    /// the first cloud operation of a cycle signals `entered` and parks on
3200    /// `release` until the test wakes it — holding the loop mid-cycle so the test
3201    /// can observe the store-directory lock while a cycle is in flight.
3202    ///
3203    /// Arming happens only after `connect_sync_with_test_home` returns, so the
3204    /// connect-time bootstrap runs unblocked and only a loop cycle is gated.
3205    struct GateCloudHome {
3206        inner: InMemoryCloudHome,
3207        armed: Arc<AtomicBool>,
3208        gated: AtomicBool,
3209        entered: mpsc::UnboundedSender<()>,
3210        release: Arc<Notify>,
3211    }
3212
3213    impl GateCloudHome {
3214        async fn gate(&self) {
3215            if self.armed.load(Ordering::Acquire) && !self.gated.swap(true, Ordering::AcqRel) {
3216                self.entered
3217                    .send(())
3218                    .expect("test observes the loop entering a cycle");
3219                self.release.notified().await;
3220            }
3221        }
3222    }
3223
3224    #[async_trait]
3225    impl CloudHome for GateCloudHome {
3226        fn exact_slot_storage(self: Arc<Self>) -> Option<Arc<dyn ExactSlotStorage>> {
3227            Some(self)
3228        }
3229
3230        async fn put_object(&self, key: &str, data: Vec<u8>) -> Result<(), CloudHomeError> {
3231            self.gate().await;
3232            self.inner.put_object(key, data).await
3233        }
3234
3235        async fn open_multipart<'a>(
3236            &'a self,
3237            key: &str,
3238            total_len: u64,
3239        ) -> Result<BoxPartSink<'a>, CloudHomeError> {
3240            self.gate().await;
3241            self.inner.open_multipart(key, total_len).await
3242        }
3243
3244        fn multipart_threshold(&self) -> u64 {
3245            self.inner.multipart_threshold()
3246        }
3247
3248        async fn read(&self, key: &str) -> Result<Vec<u8>, CloudHomeError> {
3249            self.gate().await;
3250            self.inner.read(key).await
3251        }
3252
3253        async fn read_range(
3254            &self,
3255            key: &str,
3256            start: u64,
3257            end: u64,
3258        ) -> Result<Vec<u8>, CloudHomeError> {
3259            self.gate().await;
3260            self.inner.read_range(key, start, end).await
3261        }
3262
3263        async fn list(&self, prefix: &str) -> Result<Vec<String>, CloudHomeError> {
3264            self.gate().await;
3265            self.inner.list(prefix).await
3266        }
3267
3268        async fn delete(&self, key: &str) -> Result<(), CloudHomeError> {
3269            self.gate().await;
3270            self.inner.delete(key).await
3271        }
3272
3273        async fn exists(&self, key: &str) -> Result<bool, CloudHomeError> {
3274            self.gate().await;
3275            self.inner.exists(key).await
3276        }
3277
3278        async fn set_access(
3279            &self,
3280            desired: CloudAccessState,
3281        ) -> Result<CloudAccessOutcome, CloudHomeError> {
3282            self.gate().await;
3283            self.inner.set_access(desired).await
3284        }
3285    }
3286
3287    #[async_trait]
3288    impl ExactSlotStorage for GateCloudHome {
3289        async fn provider_binding(
3290            &self,
3291        ) -> Result<crate::sync::storage::ResolvedProviderBinding, CloudHomeError> {
3292            self.gate().await;
3293            ExactSlotStorage::provider_binding(&self.inner).await
3294        }
3295
3296        async fn allocate_slot(&self, key: &str) -> Result<ObjectSlot, CloudHomeError> {
3297            self.gate().await;
3298            ExactSlotStorage::allocate_slot(&self.inner, key).await
3299        }
3300
3301        async fn create_at(
3302            &self,
3303            slot: &ObjectSlot,
3304            body: BlobBody,
3305            progress: &UploadProgress<'_>,
3306        ) -> Result<(), CloudHomeError> {
3307            self.gate().await;
3308            ExactSlotStorage::create_at(&self.inner, slot, body, progress).await
3309        }
3310
3311        async fn read_at(&self, slot: &ObjectSlot) -> Result<Vec<u8>, CloudHomeError> {
3312            self.gate().await;
3313            ExactSlotStorage::read_at(&self.inner, slot).await
3314        }
3315
3316        async fn read_range_at(
3317            &self,
3318            slot: &ObjectSlot,
3319            start: u64,
3320            end: u64,
3321        ) -> Result<Vec<u8>, CloudHomeError> {
3322            self.gate().await;
3323            ExactSlotStorage::read_range_at(&self.inner, slot, start, end).await
3324        }
3325
3326        async fn read_at_to_file(
3327            &self,
3328            slot: &ObjectSlot,
3329            destination: &std::path::Path,
3330        ) -> Result<(), crate::storage::cloud::CloudFileReadError> {
3331            self.gate().await;
3332            ExactSlotStorage::read_at_to_file(&self.inner, slot, destination).await
3333        }
3334
3335        async fn delete_at(&self, slot: &ObjectSlot) -> Result<(), CloudHomeError> {
3336            self.gate().await;
3337            ExactSlotStorage::delete_at(&self.inner, slot).await
3338        }
3339    }
3340
3341    fn try_open(dir: &StoreDir) -> CovenResult<CovenHandle> {
3342        Coven::builder(config(dir.clone()))
3343            .write_policy(WritePolicy::MergeConcurrent)
3344            .synced_tables(vec![files_table()])
3345            .migrations(vec![files_migration()])
3346            .open()
3347    }
3348
3349    /// Reopen `dir`, retrying while a still-running sync loop holds the lock.
3350    /// Retries only the lock refusal; any other open error fails immediately.
3351    /// Fails the calling test if the lock is not released within the budget.
3352    async fn open_when_lock_released(dir: &StoreDir) -> CovenHandle {
3353        for _ in 0..100 {
3354            match try_open(dir) {
3355                Ok(handle) => return handle,
3356                Err(CovenError::AlreadyOpen { .. }) => {
3357                    tokio::time::sleep(Duration::from_millis(100)).await;
3358                }
3359                Err(other) => panic!("reopen failed for a reason other than the lock: {other}"),
3360            }
3361        }
3362        panic!("store lock never released: reopen kept failing");
3363    }
3364
3365    #[tokio::test]
3366    async fn lock_is_held_until_the_sync_loop_exits_its_cycle() {
3367        test_keyring::install();
3368
3369        let tmp = tempfile::tempdir().expect("temp dir");
3370        let dir = StoreDir::new(tmp.path());
3371        // A store id distinct from every other test's — `try_open`/
3372        // `open_when_lock_released` below reopen the same directory (the
3373        // lock is path-scoped, not store-id-scoped) but this test's own
3374        // identity establishment must not collide with a concurrently
3375        // running test that also establishes one under the shared default id.
3376        let handle = Coven::builder(Config::with_defaults(
3377            "lock-held-until-cycle-exits".to_string(),
3378            "device-test".to_string(),
3379            dir.clone(),
3380            "Test".to_string(),
3381        ))
3382        .write_policy(WritePolicy::MergeConcurrent)
3383        .synced_tables(vec![files_table()])
3384        .migrations(vec![files_migration()])
3385        .open()
3386        .expect("open handle");
3387        handle
3388            .initialize_identity()
3389            .expect("establish this store's identity before connecting");
3390
3391        let armed = Arc::new(AtomicBool::new(false));
3392        let (entered_tx, mut entered_rx) = mpsc::unbounded_channel();
3393        let release = Arc::new(Notify::new());
3394        let home = Arc::new(GateCloudHome {
3395            inner: InMemoryCloudHome::new(),
3396            armed: armed.clone(),
3397            gated: AtomicBool::new(false),
3398            entered: entered_tx,
3399            release: release.clone(),
3400        });
3401        let connect_handle = handle.clone();
3402        tokio::spawn(async move {
3403            connect_handle
3404                .connect_sync_with_test_home(home, CloudCipher::Plaintext)
3405                .await
3406        })
3407        .await
3408        .expect("gating test connection task completes")
3409        .expect("connect over the gating test home");
3410
3411        // Bootstrap is done; gate the loop's next cycle and wait until it parks
3412        // mid-cycle inside the cloud home.
3413        armed.store(true, Ordering::Release);
3414        tokio::time::timeout(Duration::from_secs(30), entered_rx.recv())
3415            .await
3416            .expect("sync loop reached a cycle within the timeout")
3417            .expect("gating home signalled the cycle entry");
3418
3419        // Drop the last handle while the loop is still mid-cycle. The loop owns a
3420        // guard clone, so the lock must stay held: a concurrent open is refused.
3421        drop(handle);
3422        assert!(
3423            matches!(
3424                try_open(&dir),
3425                Err(CovenError::AlreadyOpen { store_dir }) if store_dir == tmp.path()
3426            ),
3427            "a concurrent open must be refused while the sync loop is mid-cycle",
3428        );
3429
3430        // Release the cycle; the loop finishes, sees its channels closed, exits,
3431        // and drops the last guard clone — the lock frees and a reopen succeeds.
3432        release.notify_one();
3433        let _reopened = open_when_lock_released(&dir).await;
3434    }
3435
3436    #[tokio::test]
3437    async fn normal_shutdown_releases_the_lock_for_reopen() {
3438        test_keyring::install();
3439
3440        let tmp = tempfile::tempdir().expect("temp dir");
3441        let dir = StoreDir::new(tmp.path());
3442        // A store id distinct from every other test's — see the identical
3443        // note in `lock_is_held_until_the_sync_loop_exits_its_cycle`.
3444        let handle = Coven::builder(Config::with_defaults(
3445            "normal-shutdown-releases-lock".to_string(),
3446            "device-test".to_string(),
3447            dir.clone(),
3448            "Test".to_string(),
3449        ))
3450        .write_policy(WritePolicy::MergeConcurrent)
3451        .synced_tables(vec![files_table()])
3452        .migrations(vec![files_migration()])
3453        .open()
3454        .expect("open handle");
3455        handle
3456            .initialize_identity()
3457            .expect("establish this store's identity before connecting");
3458        let connect_handle = handle.clone();
3459        tokio::spawn(async move {
3460            connect_handle
3461                .connect_sync_with_test_home(
3462                    Arc::new(InMemoryCloudHome::new()),
3463                    CloudCipher::Plaintext,
3464                )
3465                .await
3466        })
3467        .await
3468        .expect("shutdown test connection task completes")
3469        .expect("connect over the test home");
3470
3471        drop(handle);
3472
3473        let _reopened = open_when_lock_released(&dir).await;
3474    }
3475
3476    /// The open-time orphan sweep clears crash-left atomic-write temps (`.tmp.<uuid>`)
3477    /// under every blob folder — `cache/`, `pinned/`, and the local store — that
3478    /// predate this open, while leaving a temp a concurrent write is mid-producing
3479    /// (newer than process start) and every committed blob untouched. Local-store
3480    /// staging temps (`.coven-stage-<uuid>`) are still swept regardless of age.
3481    #[test]
3482    fn open_time_sweep_clears_stale_blob_temps_but_keeps_fresh_ones() {
3483        let tmp = tempfile::tempdir().expect("temp dir");
3484        let ld = StoreDir::new(tmp.path());
3485
3486        let process_start = std::time::SystemTime::now();
3487        let stale = process_start - Duration::from_secs(3600);
3488        let fresh = process_start + Duration::from_secs(3600);
3489
3490        let write_with_mtime = |path: &Path, mtime: std::time::SystemTime| {
3491            std::fs::create_dir_all(path.parent().unwrap()).expect("create dir");
3492            std::fs::write(path, b"x").expect("write file");
3493            std::fs::OpenOptions::new()
3494                .write(true)
3495                .open(path)
3496                .expect("open to set mtime")
3497                .set_modified(mtime)
3498                .expect("set mtime");
3499        };
3500        let temp_name = |marker: &str| format!("{marker}{}", uuid::Uuid::new_v4());
3501
3502        let cache_shard = ld
3503            .storage_dir()
3504            .join("cache")
3505            .join("release_files")
3506            .join("ab")
3507            .join("cd");
3508        let pinned_shard = ld
3509            .storage_dir()
3510            .join("pinned")
3511            .join("photos")
3512            .join("ef")
3513            .join("gh");
3514        let local_ns = ld.storage_dir().join("local").join("audio");
3515
3516        let stale_cache_temp = cache_shard.join(temp_name(crate::local_blob::TEMP_BLOB_PREFIX));
3517        let fresh_cache_temp = cache_shard.join(temp_name(crate::local_blob::TEMP_BLOB_PREFIX));
3518        // A committed blob with an OLD mtime: not a temp, so age is irrelevant — it must
3519        // survive, proving the sweep classifies by the temp prefix, not by age.
3520        let committed_cache = cache_shard.join("blob0aaa");
3521        let stale_pinned_temp = pinned_shard.join(temp_name(crate::local_blob::TEMP_BLOB_PREFIX));
3522        let fresh_pinned_temp = pinned_shard.join(temp_name(crate::local_blob::TEMP_BLOB_PREFIX));
3523        // The local store's own `write_atomic` leaves `.tmp.<uuid>` temps too; a crash
3524        // orphans them the same way. A committed local blob (old mtime) must survive.
3525        let stale_local_temp = local_ns.join(temp_name(crate::local_blob::TEMP_BLOB_PREFIX));
3526        let fresh_local_temp = local_ns.join(temp_name(crate::local_blob::TEMP_BLOB_PREFIX));
3527        let committed_local = local_ns.join("blob0bbb");
3528        let stale_local_stage = local_ns.join(temp_name(&format!("blob{LOCAL_STAGE_MARKER}")));
3529
3530        write_with_mtime(&stale_cache_temp, stale);
3531        write_with_mtime(&fresh_cache_temp, fresh);
3532        write_with_mtime(&committed_cache, stale);
3533        write_with_mtime(&stale_pinned_temp, stale);
3534        write_with_mtime(&fresh_pinned_temp, fresh);
3535        write_with_mtime(&stale_local_temp, stale);
3536        write_with_mtime(&fresh_local_temp, fresh);
3537        write_with_mtime(&committed_local, stale);
3538        write_with_mtime(&stale_local_stage, stale);
3539
3540        remove_orphaned_local_blob_temps(&ld, process_start).expect("open-time orphan sweep");
3541
3542        assert!(
3543            !stale_cache_temp.exists(),
3544            "a cache temp older than process start is a crash leftover, removed",
3545        );
3546        assert!(
3547            !stale_pinned_temp.exists(),
3548            "a pinned temp older than process start is a crash leftover, removed",
3549        );
3550        assert!(
3551            !stale_local_temp.exists(),
3552            "a local-store atomic-write temp older than process start is a crash leftover, removed",
3553        );
3554        assert!(
3555            fresh_cache_temp.exists(),
3556            "a cache temp newer than process start is a live write, left alone",
3557        );
3558        assert!(
3559            fresh_pinned_temp.exists(),
3560            "a pinned temp newer than process start is a live write, left alone",
3561        );
3562        assert!(
3563            fresh_local_temp.exists(),
3564            "a local-store temp newer than process start is a live write, left alone",
3565        );
3566        assert!(
3567            committed_cache.exists(),
3568            "a committed cache blob is never a temp — kept regardless of its mtime",
3569        );
3570        assert!(
3571            committed_local.exists(),
3572            "a committed local-store blob is never a temp — kept regardless of its mtime",
3573        );
3574        assert!(
3575            !stale_local_stage.exists(),
3576            "local-store staging temps are still swept at open",
3577        );
3578    }
3579
3580    // ========================================================================
3581    // Read-only opens (CovenReadHandle)
3582    // ========================================================================
3583
3584    fn try_open_read_only(dir: &StoreDir) -> CovenResult<crate::read_handle::CovenReadHandle> {
3585        Coven::builder(config(dir.clone()))
3586            .write_policy(WritePolicy::MergeConcurrent)
3587            .synced_tables(vec![files_table()])
3588            .migrations(vec![files_migration()])
3589            .open_read_only()
3590    }
3591
3592    async fn read_files_count(handle: &crate::read_handle::CovenReadHandle) -> i64 {
3593        handle
3594            .sql_read(|conn| {
3595                conn.query_row("SELECT count(*) FROM files", [], |row| row.get(0))
3596                    .map_err(CovenError::from)
3597            })
3598            .await
3599            .expect("count files through the read handle")
3600    }
3601
3602    /// The requirement's failing baseline made to pass: a second FULL open is still
3603    /// refused while the first holds the store, but a read-only open succeeds
3604    /// against the same held store — it takes no writer lock.
3605    #[tokio::test]
3606    async fn read_only_open_succeeds_while_a_full_open_holds_the_store() {
3607        let tmp = tempfile::tempdir().expect("temp dir");
3608        let dir = StoreDir::new(tmp.path());
3609        let writer = open_files_handle_in(dir.clone());
3610
3611        // A second full open is refused (the invariant the lock protects).
3612        assert!(matches!(
3613            try_open(&dir),
3614            Err(CovenError::AlreadyOpen { store_dir }) if store_dir == tmp.path()
3615        ));
3616
3617        // A read-only open succeeds against the very same held store.
3618        let reader = try_open_read_only(&dir).expect("read-only open succeeds under a full open");
3619        assert_eq!(read_files_count(&reader).await, 0);
3620
3621        drop(writer);
3622    }
3623
3624    /// Multiple read-only opens coexist with each other and with the writer.
3625    #[tokio::test]
3626    async fn multiple_read_only_opens_coexist_with_a_writer() {
3627        let tmp = tempfile::tempdir().expect("temp dir");
3628        let dir = StoreDir::new(tmp.path());
3629        let writer = open_files_handle_in(dir.clone());
3630
3631        let reader_a = try_open_read_only(&dir).expect("first read-only open");
3632        let reader_b = try_open_read_only(&dir).expect("second read-only open");
3633
3634        assert_eq!(read_files_count(&reader_a).await, 0);
3635        assert_eq!(read_files_count(&reader_b).await, 0);
3636        drop(writer);
3637    }
3638
3639    /// A read-only handle sees rows the writer committed — both before the reader
3640    /// opened and after (WAL cross-connection visibility on the one db file).
3641    #[tokio::test]
3642    async fn read_only_open_sees_committed_writer_data() {
3643        let tmp = tempfile::tempdir().expect("temp dir");
3644        let dir = StoreDir::new(tmp.path());
3645        let writer = open_files_handle_in(dir.clone());
3646
3647        writer
3648            .sql(|sql| {
3649                sql.execute(
3650                    "INSERT INTO files (id, blob_id, size, _updated_at) \
3651                     VALUES ('row-before-reader', NULL, 0, ?1)",
3652                    [sql.stamp()],
3653                )?;
3654                Ok(())
3655            })
3656            .await
3657            .expect("writer inserts before the reader opens");
3658
3659        let reader = try_open_read_only(&dir).expect("read-only open");
3660        assert_eq!(
3661            read_files_count(&reader).await,
3662            1,
3663            "the reader sees the row committed before it opened",
3664        );
3665
3666        // A commit after the reader is open is visible on its next read: coven runs
3667        // each read as its own transaction, so it never pins an old WAL snapshot.
3668        writer
3669            .sql(|sql| {
3670                sql.execute(
3671                    "INSERT INTO files (id, blob_id, size, _updated_at) \
3672                     VALUES ('row-after-reader', NULL, 0, ?1)",
3673                    [sql.stamp()],
3674                )?;
3675                Ok(())
3676            })
3677            .await
3678            .expect("writer inserts after the reader opened");
3679        assert_eq!(
3680            read_files_count(&reader).await,
3681            2,
3682            "the reader sees a row the writer committed after the reader opened",
3683        );
3684        drop(writer);
3685    }
3686
3687    /// A read-only handle has no write API at all — writes are absent by
3688    /// construction. The one place a raw statement could be run is the
3689    /// `sql_read` closure's `&Connection`; because the connection is
3690    /// `SQLITE_OPEN_READONLY`, SQLite refuses the write there rather than
3691    /// mutating the store.
3692    #[tokio::test]
3693    async fn read_only_handle_connection_refuses_writes() {
3694        let tmp = tempfile::tempdir().expect("temp dir");
3695        let dir = StoreDir::new(tmp.path());
3696        let _writer = open_files_handle_in(dir.clone());
3697        let reader = try_open_read_only(&dir).expect("read-only open");
3698
3699        let result: CovenResult<()> = reader
3700            .sql_read(|conn| {
3701                conn.execute(
3702                    "INSERT INTO files (id, blob_id, size, _updated_at) \
3703                     VALUES ('should-not-write', NULL, 0, 'x')",
3704                    [],
3705                )?;
3706                Ok(())
3707            })
3708            .await;
3709        assert!(
3710            matches!(result, Err(CovenError::Sqlite(_))),
3711            "a write through the read-only connection is refused by SQLite: {result:?}",
3712        );
3713    }
3714
3715    /// A read-only open runs no migration ladder, but refuses a db a newer binary
3716    /// migrated past what this binary knows — the writer's `SchemaTooNew` policy.
3717    #[tokio::test]
3718    async fn read_only_open_refuses_a_too_new_schema() {
3719        let tmp = tempfile::tempdir().expect("temp dir");
3720        let dir = StoreDir::new(tmp.path());
3721
3722        // A newer binary migrates the db to synced-schema version 2.
3723        let ahead = Coven::builder(config(dir.clone()))
3724            .write_policy(WritePolicy::MergeConcurrent)
3725            .synced_tables(vec![files_table()])
3726            .migrations(vec![
3727                files_migration(),
3728                Migration::sql(2, "add-extra", "CREATE TABLE extra (id TEXT PRIMARY KEY)"),
3729            ])
3730            .open()
3731            .expect("open at version 2");
3732        drop(ahead);
3733
3734        // An older binary opens the same db read-only with only the version-1 ladder:
3735        // it cannot understand the schema, so it refuses with the matchable variant.
3736        let reopened = Coven::builder(config(dir))
3737            .write_policy(WritePolicy::MergeConcurrent)
3738            .synced_tables(vec![files_table()])
3739            .migrations(vec![files_migration()])
3740            .open_read_only();
3741        assert!(matches!(
3742            reopened,
3743            Err(CovenError::Migration(MigrationError::SchemaTooNew {
3744                current: 2,
3745                supported: 1
3746            }))
3747        ));
3748    }
3749
3750    /// End-to-end blob read through the read handle: the writer stores a
3751    /// host-provided blob on a remote-root table (its bytes land in the local store
3752    /// as upload staging); the read-only handle resolves the blob's locality and
3753    /// serves those bytes — no cloud, no writer lock.
3754    #[tokio::test]
3755    async fn read_only_handle_reads_a_host_provided_blob() {
3756        let tmp = tempfile::tempdir().expect("temp dir");
3757        let dir = StoreDir::new(tmp.path());
3758        let writer = Coven::builder(config(dir.clone()))
3759            .write_policy(WritePolicy::MergeConcurrent)
3760            .synced_tables(vec![remote_root_files_table()])
3761            .migrations(vec![files_migration()])
3762            .open()
3763            .expect("open writer");
3764
3765        let bytes = b"read-only-handle-serves-these-blob-bytes".to_vec();
3766        let hash = crate::blob::content_hash(&bytes);
3767        writer
3768            .write(
3769                {
3770                    let bytes = bytes.clone();
3771                    move |w| {
3772                        w.put_blob("media-files", "roblob01", bytes);
3773                        Ok(())
3774                    }
3775                },
3776                {
3777                    let len = bytes.len() as i64;
3778                    move |sql| {
3779                        sql.execute(
3780                            "INSERT INTO files (id, blob_id, size, hash, _updated_at) \
3781                             VALUES (?1, ?2, ?3, ?4, ?5)",
3782                            params!["file-ro", "roblob01", len, hash, sql.stamp()],
3783                        )?;
3784                        Ok(())
3785                    }
3786                },
3787            )
3788            .await
3789            .expect("writer stores the row and blob");
3790
3791        let reader = Coven::builder(config(dir))
3792            .write_policy(WritePolicy::MergeConcurrent)
3793            .synced_tables(vec![remote_root_files_table()])
3794            .migrations(vec![files_migration()])
3795            .open_read_only()
3796            .expect("read-only open");
3797
3798        let blob = reader
3799            .row_blob_ref("files", "file-ro")
3800            .await
3801            .expect("capture read-only blob row");
3802        let read = reader
3803            .read_blob(&blob)
3804            .await
3805            .expect("read blob via read handle");
3806        assert_eq!(
3807            read, bytes,
3808            "the read handle serves the blob the writer stored"
3809        );
3810
3811        // A ranged read through the same handle serves the requested slice.
3812        let (offset, len) = (5u64, 10u64);
3813        let range = reader
3814            .open_blob_stream(&blob, offset, len)
3815            .await
3816            .expect("ranged read via read handle");
3817        assert_eq!(range, &bytes[offset as usize..(offset + len) as usize]);
3818        drop(writer);
3819    }
3820
3821    /// Concurrent same-path cache populate by two producers (the writer's and the
3822    /// reader's fetch both writing the same blob into `cache/`) is safe: the atomic
3823    /// temp-then-rename write means the destination is always one producer's whole
3824    /// file, never a torn interleave. This is the primitive requirement 6 rests on.
3825    #[tokio::test]
3826    async fn concurrent_same_blob_cache_writes_never_tear() {
3827        let tmp = tempfile::tempdir().expect("temp dir");
3828        let dir = StoreDir::new(tmp.path());
3829        let dest = dir
3830            .cache_blob_path("media-files", "raceblob")
3831            .expect("cache path");
3832        if let Some(parent) = dest.parent() {
3833            std::fs::create_dir_all(parent).expect("create cache shard dir");
3834        }
3835
3836        // Two full-length payloads that differ everywhere, so any interleave would be
3837        // detectable as a mixed file. In production both producers write the same
3838        // blob's bytes; the distinct payloads only make tearing observable.
3839        let a = vec![b'A'; 64 * 1024];
3840        let b = vec![b'B'; 64 * 1024];
3841        let (ra, rb) = tokio::join!(
3842            crate::local_blob::write_atomic(&dest, &a),
3843            crate::local_blob::write_atomic(&dest, &b),
3844        );
3845        ra.expect("first concurrent cache write");
3846        rb.expect("second concurrent cache write");
3847
3848        let final_bytes = crate::local_blob::read(&dest)
3849            .await
3850            .expect("read cache file");
3851        assert!(
3852            final_bytes == a || final_bytes == b,
3853            "the cache file is exactly one producer's whole payload, never a torn mix",
3854        );
3855    }
3856}