coven_foundation/files/
lock.rs

1//! Store locks live beside the directory so Windows can unpublish it.
2
3use std::fs::{File, OpenOptions, TryLockError};
4use std::path::{Path, PathBuf};
5
6use crate::files::file_lock::{FileLock, LockMode};
7use crate::files::{atomic_file, settings, FileError, SettingsError, StoreSettings};
8use crate::id_source::StoreId;
9
10use super::lock_registry as registry;
11use registry::LockKind;
12pub use registry::StoreClosingGuard;
13pub(super) use registry::{bootstrap_running, LockInstance, RegisteredLock};
14
15/// An exclusive writer lock. Dropping it unlocks and closes the OS handle.
16/// Readers use a separate shared lock so they can coexist with the writer.
17#[derive(Debug)]
18pub struct StoreLock {
19    store: LockedStore<RegisteredLock>,
20}
21
22#[derive(Debug)]
23struct LockedStore<L> {
24    _file: L,
25    directory: PathBuf,
26    id: StoreId,
27}
28
29impl StoreLock {
30    /// Mark this instance as closing when the caller's owning scope ends.
31    /// This guard does not retain the OS lock or any database connection.
32    pub fn closing_guard(&self) -> StoreClosingGuard {
33        self.store._file.closing_guard()
34    }
35
36    /// Read the settings while holding the store's writer lock.
37    pub fn settings(&self) -> Result<StoreSettings, SettingsError> {
38        settings::read(&self.store.directory, self.store.id)
39    }
40
41    /// The canonical directory of the store this lock holds.
42    pub(super) fn directory(&self) -> &Path {
43        &self.store.directory
44    }
45
46    pub(crate) fn verify(&self, directory: &Path, id: StoreId) -> Result<(), StoreLockError> {
47        let path = std::fs::canonicalize(directory)
48            .map_err(|source| FileError::at("resolve locked store", directory, source))?;
49        if path != self.store.directory || id != self.store.id {
50            return Err(StoreLockError::WrongDirectory(id));
51        }
52        Ok(())
53    }
54}
55
56/// A shared lock that prevents deletion while a reader holds store files open.
57#[derive(Debug)]
58pub struct StoreReadLock {
59    _file: RegisteredLock,
60}
61
62impl StoreReadLock {
63    /// Mark this reader as closing when its owner begins to close, so a
64    /// deletion in this process waits for its release instead of refusing.
65    /// This guard does not retain the OS lock.
66    pub fn closing_guard(&self) -> StoreClosingGuard {
67        self._file.closing_guard()
68    }
69}
70
71/// The writer lock, held while a closed store is read without opening it:
72/// holding it shows that no handle runs the store's sync. Registered in this
73/// process as closing from the start, so an open waits for its release.
74/// Dropping it unlocks.
75#[derive(Debug)]
76pub struct ClosedStoreLock {
77    _file: RegisteredLock,
78}
79
80/// A held lock on the cache's range writes: shared by a write in progress,
81/// exclusive while opening finds abandoned ones. Dropping it unlocks.
82#[derive(Debug)]
83pub struct CacheWriteLock {
84    _file: FileLock,
85}
86
87/// The cache-write lock file, in the store directory beside the cache folder,
88/// which holds only cache entries.
89const CACHE_WRITES: &str = ".coven-cache-writes.lock";
90
91/// Wait for the cache-write lock, shared or exclusive. Unlike the store
92/// locks it waits: a range write is short, and opening needs them finished.
93pub(super) fn cache_writes(store: &Path, shared: bool) -> Result<CacheWriteLock, FileError> {
94    let path = store.join(CACHE_WRITES);
95    let mode = if shared { LockMode::Shared } else { LockMode::Exclusive };
96    Ok(CacheWriteLock { _file: wait_for_lock(&path, mode, "lock cache writes")? })
97}
98
99/// Wait for the lock on the file at `path`. A test hears when the lock is
100/// held and the call is about to wait for it (`when_reached`).
101fn wait_for_lock(path: &Path, mode: LockMode, what: &'static str) -> Result<FileLock, FileError> {
102    let file = open_lock(path)?;
103    let attempt = file.try_clone().map_err(|source| FileError::at(what, path, source))?;
104    match FileLock::try_lock(attempt, mode) {
105        Ok(file) => return Ok(file),
106        Err(TryLockError::WouldBlock) => {
107            #[cfg(any(test, feature = "test-utils"))]
108            super::reached::reached(path);
109        }
110        Err(TryLockError::Error(source)) => return Err(FileError::at(what, path, source)),
111    }
112    FileLock::lock(file, mode).map_err(|source| FileError::at(what, path, source))
113}
114
115/// The cache-write lock file of the store at `store`, which a test waits on
116/// to know a range write or an open waits for it.
117#[cfg(any(test, feature = "test-utils"))]
118pub fn cache_writes_lock(store: &Path) -> std::path::PathBuf {
119    store.join(CACHE_WRITES)
120}
121
122/// Exclusive access to the writer, readers and bootstrap, authorizing
123/// deletion. Holding the bootstrap lock keeps every restore, join and
124/// replacement of the store out, including a `reset_device` or
125/// `restore_device` that holds only that lock between the steps of an
126/// unfinished swap (§19.3), and lets deletion remove its lock file too.
127#[derive(Debug)]
128pub struct StoreDeletionLock {
129    store: LockedStore<FileLock>,
130    _readers: FileLock,
131    bootstrap: HeldBootstrap,
132}
133
134/// How a deletion holds the store's bootstrap lock.
135#[derive(Debug)]
136enum HeldBootstrap {
137    /// Taken by the deletion itself.
138    Locked(FileLock),
139    /// Handed over by a bootstrap cancelling its own unpublished store.
140    Leased(RegisteredLock),
141}
142
143impl HeldBootstrap {
144    fn release(self) {
145        match self {
146            Self::Locked(file) => drop(file),
147            Self::Leased(lock) => drop(lock),
148        }
149    }
150}
151
152impl StoreDeletionLock {
153    /// Rename the store's directory to its deletion name, durably: the store
154    /// leaves every listing and open, and the directory under that name, or
155    /// the store's lock files without a directory, record that its deletion
156    /// is under way, for the next listing, restore or join of the store to
157    /// finish (§11.6, E1). Done already, it changes nothing.
158    pub fn unpublish(&mut self) -> Result<(), FileError> {
159        let _layout = lock_layout(self.store.directory.parent().expect("store parent"))?;
160        self.unpublish_locked()
161    }
162
163    /// `unpublish`, under the layout lock.
164    fn unpublish_locked(&mut self) -> Result<(), FileError> {
165        let destination = deletion_path(&self.store.directory, self.store.id);
166        if self.store.directory != destination && inspect_directory(&self.store.directory)? {
167            crate::files::atomic_file::rename_new(&self.store.directory, &destination).map_err(
168                |source| FileError::at("unpublish store", &self.store.directory, source),
169            )?;
170            self.store.directory = destination;
171            #[cfg(unix)]
172            sync_removal(&self.store.directory)?;
173        }
174        Ok(())
175    }
176
177    /// Unpublish the directory, as `unpublish` does, and remove its contents
178    /// before releasing and removing its sibling locks. Retry failures
179    /// through `lock_for_deletion`, including a failure after only the lock
180    /// files remain.
181    pub fn remove_directory(mut self) -> Result<(), FileError> {
182        let root = self.store.directory.parent().expect("store parent");
183        // Every lock acquisition and creation also takes this lock. None can
184        // open a lock file between its release here and the removal of its
185        // name, so no one holds an unlinked lock while another locks a new
186        // file at the same path.
187        let _layout = lock_layout(root)?;
188        let paths = deletion_lock_paths(&self.store.directory, self.store.id);
189        self.unpublish_locked()?;
190        if inspect_directory(&self.store.directory)? {
191            std::fs::remove_dir_all(&self.store.directory).map_err(|source| {
192                FileError::at("remove store directory", &self.store.directory, source)
193            })?;
194        }
195        #[cfg(unix)]
196        sync_removal(&self.store.directory)?;
197        // Release before removing, on every platform. Windows keeps a file
198        // deleted while a handle is open until that handle closes, and
199        // refuses to open its name meanwhile, which would refuse the next
200        // lock of this store. Under the layout lock, releasing first is as
201        // safe as removing a held file.
202        let Self { store, _readers, bootstrap } = self;
203        drop((store, _readers));
204        bootstrap.release();
205        for path in paths {
206            atomic_file::remove(&path)?;
207        }
208        Ok(())
209    }
210}
211
212#[cfg(unix)]
213fn sync_removal(directory: &Path) -> Result<(), FileError> {
214    crate::files::atomic_file::sync_directory(directory.parent().expect("store parent")).map_err(
215        |source| FileError::AfterRemove { path: directory.to_owned(), source: Box::new(source) },
216    )
217}
218
219/// A store in use is distinct from a filesystem failure.
220#[derive(Debug, thiserror::Error)]
221pub enum StoreLockError {
222    /// A live instance or another process prevents the requested access.
223    /// Async opens wait for a closing instance in this process to release it.
224    #[error("store {0} is already open")]
225    AlreadyOpen(StoreId),
226    /// An unfinished first installation must be resumed by the call that
227    /// began it: a restore or join with its code. A restore on a join's
228    /// installation, or a join on a recorded restore's, is refused too (§13.4).
229    #[error("store {0} requires restore or join to finish bootstrap")]
230    BootstrapPending(StoreId),
231    /// A replacement of this device's installation recorded itself, just
232    /// before its fresh device registered, and did not finish. Only the call
233    /// it names finishes it; the other is refused before anything changes.
234    /// Deletion removes it instead (§19.3).
235    #[error("store {store} requires {by} to finish replacing this device's installation")]
236    ReplacementUnfinished {
237        /// The store whose installation is being replaced.
238        store: StoreId,
239        /// The call that began the replacement.
240        by: super::ReplacedBy,
241    },
242    /// This device is deleting the store and leaving it, and its deletion
243    /// recorded itself, just before it journaled the device's removal, and
244    /// did not finish. Opening and replacing the store are refused before
245    /// anything changes; only finishing the deletion, or discarding the
246    /// store, ends it (E1).
247    #[error("store {0} is being deleted from this device")]
248    DeletionUnfinished(StoreId),
249    /// A supplied lock protects a different directory.
250    #[error("lock does not protect store {0}")]
251    WrongDirectory(StoreId),
252    /// Opening or locking the lock file failed for another reason, or did
253    /// not run because its runtime is shutting down (`FileError::Shutdown`).
254    #[error("store lock: {0}")]
255    File(FileError),
256    /// Taking the lock panicked on the blocking pool: a bug, which the
257    /// caller reports (E15).
258    #[error(transparent)]
259    Bug(#[from] crate::task::Bug),
260}
261
262impl From<FileError> for StoreLockError {
263    fn from(error: FileError) -> Self {
264        match error {
265            FileError::Bug(bug) => Self::Bug(bug),
266            error => Self::File(error),
267        }
268    }
269}
270
271impl crate::task::CarriesBug for StoreLockError {
272    fn bug(&self) -> Option<&crate::task::Bug> {
273        match self {
274            Self::Bug(bug) => Some(bug),
275            Self::File(error) => error.bug(),
276            _ => None,
277        }
278    }
279}
280
281fn open_lock(path: &Path) -> Result<File, FileError> {
282    OpenOptions::new()
283        .read(true)
284        .write(true)
285        .create(true)
286        .truncate(false)
287        .open(path)
288        .map_err(|source| FileError::at("open store lock", path, source))
289}
290
291pub(crate) fn try_lock(path: &Path, id: StoreId, shared: bool) -> Result<FileLock, StoreLockError> {
292    let mode = if shared { LockMode::Shared } else { LockMode::Exclusive };
293    match FileLock::try_lock(open_lock(path)?, mode) {
294        Ok(file) => Ok(file),
295        Err(TryLockError::WouldBlock) => Err(StoreLockError::AlreadyOpen(id)),
296        Err(TryLockError::Error(source)) => Err(FileError::at("lock store", path, source).into()),
297    }
298}
299
300// This file belongs to the layout and is never removed. It serializes lock
301// acquisition, creation, and lock-file removal across processes, so no caller
302// can hold an unlinked lock while another locks a new file at the same path.
303pub(crate) fn lock_layout(root: &Path) -> Result<FileLock, FileError> {
304    wait_for_lock(&layout_lock(root), LockMode::Exclusive, "lock store layout")
305}
306
307/// The layout's lock file under `root`, which a test waits on to know a
308/// lock, creation or removal waits for the layout.
309pub(crate) fn layout_lock(root: &Path) -> PathBuf {
310    root.join(".coven-stores.lock")
311}
312
313#[cfg(any(test, feature = "test-utils"))]
314pub(crate) fn acquire(directory: &Path, id: StoreId) -> Result<StoreLock, StoreLockError> {
315    let directory = resolve_directory(directory)?;
316    let file = registry::try_acquire(&directory, id, LockKind::Writer)?;
317    Ok(StoreLock { store: LockedStore { _file: file, directory, id } })
318}
319
320pub(crate) async fn acquire_for_open(
321    directory: &Path,
322    id: StoreId,
323    bugs: &crate::task::BugSinkRef,
324) -> Result<StoreLock, StoreLockError> {
325    let directory = resolve_directory(directory)?;
326    let file = registry::acquire_registered(&directory, id, LockKind::Writer, bugs).await?;
327    Ok(StoreLock { store: LockedStore { _file: file, directory, id } })
328}
329
330/// Take the writer lock of a closed store to read it, without waiting for
331/// another process or a live instance in this one; an instance closing in
332/// this process is waited for first.
333pub(crate) async fn acquire_closed(
334    directory: &Path,
335    id: StoreId,
336    bugs: &crate::task::BugSinkRef,
337) -> Result<ClosedStoreLock, StoreLockError> {
338    let directory = resolve_directory(directory)?;
339    Ok(ClosedStoreLock { _file: registry::acquire_closing_writer(&directory, id, bugs).await? })
340}
341
342/// Wait until no instance of the store at `directory` in this process is
343/// closing. An absent directory has none.
344pub(crate) async fn closed(directory: &Path, id: StoreId) -> Result<(), FileError> {
345    let directory = match std::fs::canonicalize(directory) {
346        Ok(directory) => directory,
347        Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(()),
348        Err(source) => return Err(FileError::at("resolve store directory", directory, source)),
349    };
350    registry::closed(&directory, id).await;
351    Ok(())
352}
353
354pub(super) async fn acquire_bootstrap(
355    directory: &Path,
356    id: StoreId,
357    bugs: &crate::task::BugSinkRef,
358) -> Result<RegisteredLock, StoreLockError> {
359    registry::acquire_registered(directory, id, LockKind::Bootstrap, bugs).await
360}
361
362fn resolve_directory(directory: &Path) -> Result<PathBuf, FileError> {
363    std::fs::canonicalize(directory)
364        .map_err(|source| FileError::at("resolve store directory", directory, source))
365}
366
367pub(super) fn exclude_readers(directory: &Path, id: StoreId) -> Result<FileLock, StoreLockError> {
368    let _layout = lock_layout(directory.parent().expect("store parent"))?;
369    try_lock(&lock_paths(directory, id)[1], id, false)
370}
371
372pub(crate) fn acquire_reader(
373    directory: &Path,
374    id: StoreId,
375) -> Result<StoreReadLock, StoreLockError> {
376    let directory = resolve_directory(directory)?;
377    Ok(StoreReadLock { _file: registry::acquire_reader(&directory, id)? })
378}
379
380pub(super) fn lock_existing(
381    directory: &Path,
382    id: StoreId,
383    shared: bool,
384) -> Result<(PathBuf, FileLock), StoreLockError> {
385    let directory = std::fs::canonicalize(directory)
386        .map_err(|source| FileError::at("resolve store directory", directory, source))?;
387    let _layout = lock_layout(directory.parent().expect("store parent"))?;
388    // Deletion may have completed between canonicalization and the layout lock.
389    std::fs::canonicalize(&directory)
390        .map_err(|source| FileError::at("resolve store directory", &directory, source))?;
391    let [writer, readers] = lock_paths(&directory, id);
392    let path = if shared { readers } else { writer };
393    let file = try_lock(&path, id, shared)?;
394    Ok((directory, file))
395}
396
397pub(crate) fn lock_paths(directory: &Path, id: StoreId) -> [PathBuf; 2] {
398    [
399        directory.with_file_name(format!(".{id}.lock")),
400        directory.with_file_name(format!(".{id}.readers.lock")),
401    ]
402}
403
404/// The lock a bootstrap or replacement holds for its whole call, beside the
405/// store's directory.
406fn bootstrap_lock_path(directory: &Path, id: StoreId) -> PathBuf {
407    directory.with_file_name(format!(".coven-bootstrap-{id}.lock"))
408}
409
410/// The writer lock file of the store at `directory`, which a test waits on
411/// (`when_reached`) to know an open finds the store's writer closing and
412/// waits for it.
413#[cfg(any(test, feature = "test-utils"))]
414pub fn writer_lock(directory: &Path, id: StoreId) -> PathBuf {
415    let [writer, _] = lock_paths(directory, id);
416    writer
417}
418
419/// The bootstrap lock file of the store at `directory`, which a test waits
420/// on to know an open finds the store's bootstrap closing and waits for it.
421#[cfg(any(test, feature = "test-utils"))]
422pub fn bootstrap_lock(directory: &Path, id: StoreId) -> PathBuf {
423    bootstrap_lock_path(directory, id)
424}
425
426/// The file `kind` of the store's locks is held on.
427#[cfg(any(test, feature = "test-utils"))]
428pub(super) fn kind_lock(directory: &Path, id: StoreId, kind: LockKind) -> PathBuf {
429    match kind {
430        LockKind::Writer => writer_lock(directory, id),
431        LockKind::Reader => lock_paths(directory, id)[1].clone(),
432        LockKind::Bootstrap => bootstrap_lock(directory, id),
433    }
434}
435
436/// Take the bootstrap lock without waiting. Under the layout lock, so a
437/// deletion never removes the file between this open and this lock.
438pub(super) fn lock_bootstrap(directory: &Path, id: StoreId) -> Result<FileLock, StoreLockError> {
439    let _layout = lock_layout(directory.parent().expect("store parent"))?;
440    try_lock(&bootstrap_lock_path(directory, id), id, false)
441}
442
443/// Every lock file named for a store, in the order deletion removes them.
444/// The bootstrap lock goes first: while the writer or reader lock file
445/// remains, creation refuses to reuse the id (`creation::create`).
446fn deletion_lock_paths(directory: &Path, id: StoreId) -> [PathBuf; 3] {
447    let [writer, readers] = lock_paths(directory, id);
448    [bootstrap_lock_path(directory, id), writer, readers]
449}
450
451pub(crate) fn deletion_path(directory: &Path, id: StoreId) -> PathBuf {
452    directory.with_file_name(format!(".coven-delete-{id}"))
453}
454
455fn inspect_directory(directory: &Path) -> Result<bool, FileError> {
456    match std::fs::symlink_metadata(directory) {
457        Ok(metadata) if metadata.file_type().is_dir() => Ok(true),
458        Ok(_) => Err(FileError::at(
459            "inspect deletion directory",
460            directory,
461            std::io::Error::new(std::io::ErrorKind::InvalidInput, "store is not a directory"),
462        )),
463        Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
464            tracing::debug!(path = %directory.display(), "store directory is absent");
465            Ok(false)
466        }
467        Err(source) => Err(FileError::at("inspect store directory", directory, source)),
468    }
469}
470
471/// Lock a store, or what an interrupted deletion left of it, for deletion.
472/// `lease` holds the bootstrap lock of a bootstrap deleting its own store,
473/// which the deletion lock takes from it once it holds the store's other
474/// locks; otherwise deletion takes that lock itself.
475pub(super) fn for_deletion(
476    directory: &Path,
477    id: StoreId,
478    lease: Option<&mut Option<RegisteredLock>>,
479) -> Result<Option<StoreDeletionLock>, StoreLockError> {
480    let root = directory.parent().expect("store parent");
481    let root = match std::fs::canonicalize(root) {
482        Ok(root) => root,
483        Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
484            tracing::debug!(path = %root.display(), "store layout is absent");
485            return Ok(None);
486        }
487        Err(source) => return Err(FileError::at("resolve store layout", root, source).into()),
488    };
489    let _layout = lock_layout(&root)?;
490    let published = root.join(id.to_string());
491    let directory =
492        if inspect_directory(&published)? { published } else { deletion_path(&published, id) };
493    let paths = deletion_lock_paths(&directory, id);
494    if !inspect_directory(&directory)? {
495        let mut remaining = false;
496        for path in &paths {
497            remaining |= atomic_file::exists("inspect store lock", path)?;
498        }
499        if !remaining {
500            // Retry the durability barrier even if the preceding attempt
501            // removed the last lock file before its directory sync failed.
502            #[cfg(unix)]
503            sync_removal(&directory)?;
504            tracing::debug!(store = %id, "store and its locks are absent");
505            return Ok(None);
506        }
507    }
508    let [bootstrap, writer, readers] = paths;
509    let others =
510        || Ok::<_, StoreLockError>((try_lock(&writer, id, false)?, try_lock(&readers, id, false)?));
511    let (bootstrap, (writer, readers)) = match lease {
512        // First, so a bootstrap that has not yet made the directory is
513        // refused before any other lock file is created for its id.
514        None => (HeldBootstrap::Locked(try_lock(&bootstrap, id, false)?), others()?),
515        // Last, so a refusal leaves the bootstrap holding its lock.
516        Some(lease) => {
517            let others = others()?;
518            let held = lease.take().expect("a bootstrap hands over the bootstrap lock it holds");
519            (HeldBootstrap::Leased(held), others)
520        }
521    };
522    Ok(Some(StoreDeletionLock {
523        store: LockedStore { _file: writer, directory, id },
524        _readers: readers,
525        bootstrap,
526    }))
527}
528
529/// Entries in a stores directory besides the layout lock, for tests.
530#[cfg(test)]
531pub(crate) fn store_entries(root: &Path) -> usize {
532    std::fs::read_dir(root)
533        .unwrap()
534        .filter(|entry| entry.as_ref().unwrap().file_name() != ".coven-stores.lock")
535        .count()
536}
537
538#[cfg(test)]
539#[path = "lock_tests.rs"]
540mod tests;
coven_foundation/files/lock.rs

coven_foundation/files/
lock.rs
