1use crate::file_location::FileLocation;
4use crate::insert_rows::{insert_rows, value};
5use crate::internal_schema::FILE_NAMED_SQL;
6use crate::{sqlite::DatabaseConnection, DbError};
7use coven_crypto::{ContentHash, SecretBytes};
8use coven_format::file::FileHeader;
9use coven_foundation::id_source::{DeviceId, FileId, ObjectId};
10use std::time::SystemTime;
11
12#[derive(Debug)]
17pub struct FileUpload {
18 pub id: i64,
20 pub location: FileLocation,
22 pub size: u64,
24 pub content_hash: ContentHash,
26 pub files: Vec<FileId>,
29 pub queued_at: SystemTime,
31 pub retry: Option<crate::RetryWait>,
33 pub of: crate::MoveId,
35 pub stored: bool,
37 pub holds: Option<u64>,
42 pub session: Option<SecretBytes>,
45 pub progress: Option<UploadProgress>,
48 pub bytes_total: u64,
51}
52
53#[derive(Clone, Copy, Debug, PartialEq, Eq)]
56pub struct UploadProgress {
57 pub bytes_confirmed: u64,
59 pub bytes_total: u64,
61}
62impl Clone for FileUpload {
63 fn clone(&self) -> Self {
64 Self {
65 id: self.id,
66 location: self.location.clone(),
67 size: self.size,
68 content_hash: self.content_hash,
69 files: self.files.clone(),
70 queued_at: self.queued_at,
71 retry: self.retry,
72 of: self.of,
73 stored: self.stored,
74 holds: self.holds,
75 session: self
76 .session
77 .as_ref()
78 .map(|session| SecretBytes::new(session.as_bytes().to_vec())),
79 progress: self.progress,
80 bytes_total: self.bytes_total,
81 }
82 }
83}
84impl PartialEq for FileUpload {
85 fn eq(&self, other: &Self) -> bool {
86 self.id == other.id
87 && self.location == other.location
88 && self.size == other.size
89 && self.content_hash == other.content_hash
90 && self.files == other.files
91 && self.queued_at == other.queued_at
92 && self.retry == other.retry
93 && self.of == other.of
94 && self.stored == other.stored
95 && self.holds == other.holds
96 && self.session.as_ref().map(SecretBytes::as_bytes)
97 == other.session.as_ref().map(SecretBytes::as_bytes)
98 && self.progress == other.progress
99 && self.bytes_total == other.bytes_total
100 }
101}
102
103#[derive(Clone, Debug, PartialEq)]
107pub struct UploadQueueState {
108 pub paused: bool,
110 pub uploads: Vec<FileUpload>,
112 pub sending: std::collections::BTreeSet<FileId>,
115}
116
117#[derive(Clone, Copy, Debug, PartialEq, Eq)]
119pub enum UploadCompletion {
120 Stored,
123 Retired,
126}
127
128#[derive(Clone, Copy, Debug, PartialEq, Eq)]
130pub struct UnsentWork {
131 pub writes: u64,
133 pub files: u64,
135}
136
137impl UnsentWork {
138 pub fn is_empty(&self) -> bool {
140 self.writes == 0 && self.files == 0
141 }
142}
143
144pub(crate) fn unsent(db: &DatabaseConnection) -> Result<UnsentWork, DbError> {
145 db.query_row(
146 "SELECT (SELECT count(*) FROM _coven_uploads),(SELECT count(*) FROM _coven_file_uploads)",
147 [],
148 |r| {
149 Ok(UnsentWork { writes: r.get::<_, i64>(0)? as u64, files: r.get::<_, i64>(1)? as u64 })
150 },
151 )
152}
153
154pub(crate) fn state(db: &DatabaseConnection) -> Result<UploadQueueState, DbError> {
155 Ok(UploadQueueState {
156 paused: paused(db)?,
157 uploads: read(db)?,
158 sending: db
159 .query("SELECT file FROM coven_sending", [], |r| r.get::<_, FileId>(0))?
160 .into_iter()
161 .collect(),
162 })
163}
164
165pub(crate) fn paused(db: &DatabaseConnection) -> Result<bool, DbError> {
166 db.query_row("SELECT EXISTS(SELECT 1 FROM _coven_upload_pause)", [], |r| r.get(0))
167}
168
169pub(crate) fn unmetered_only(db: &DatabaseConnection) -> Result<bool, DbError> {
171 db.query_row("SELECT EXISTS(SELECT 1 FROM _coven_unmetered_only)", [], |r| r.get(0))
172}
173
174pub(crate) fn set_unmetered_only(
175 db: &DatabaseConnection,
176 unmetered_only: bool,
177) -> Result<(), DbError> {
178 if unmetered_only {
179 db.internal_execute(
180 "INSERT INTO _coven_unmetered_only(unmetered_only) VALUES(1) ON CONFLICT DO NOTHING",
181 [],
182 )?;
183 } else {
184 db.internal_execute("DELETE FROM _coven_unmetered_only", [])?;
185 }
186 Ok(())
187}
188
189pub(crate) fn set_paused(db: &DatabaseConnection, paused: bool) -> Result<(), DbError> {
190 if paused {
191 db.internal_execute(
192 "INSERT INTO _coven_upload_pause(paused) VALUES(1) ON CONFLICT DO NOTHING",
193 [],
194 )?;
195 } else {
196 db.internal_execute("DELETE FROM _coven_upload_pause", [])?;
197 }
198 Ok(())
199}
200
201pub(crate) fn read(db: &DatabaseConnection) -> Result<Vec<FileUpload>, DbError> {
205 let rows = db.query(
206 "SELECT u.id,u.location,u.size,u.hash,u.queued_at,u.session,u.bytes_confirmed,
207 u.bytes_total,r.failures,r.not_before,r.wait,u.move,u.stored,u.holds
208 FROM _coven_file_uploads AS u LEFT JOIN _coven_retries AS r ON r.upload=u.id
209 WHERE EXISTS(SELECT 1 FROM _coven_upload_files AS l WHERE l.upload=u.id)
210 ORDER BY u.id",
211 [],
212 upload_row,
213 )?;
214 decode_uploads(db, rows)
215}
216
217pub(crate) fn abandoned(db: &DatabaseConnection) -> Result<Vec<FileUpload>, DbError> {
222 let rows = db.query(
223 "SELECT u.id,u.location,u.size,u.hash,u.queued_at,u.session,u.bytes_confirmed,
224 u.bytes_total,r.failures,r.not_before,r.wait,u.move,u.stored,u.holds
225 FROM _coven_file_uploads AS u LEFT JOIN _coven_retries AS r ON r.upload=u.id
226 WHERE u.session IS NOT NULL
227 AND NOT EXISTS(SELECT 1 FROM _coven_upload_files AS l WHERE l.upload=u.id)
228 ORDER BY u.id",
229 [],
230 upload_row,
231 )?;
232 decode_uploads(db, rows)
233}
234
235#[cfg(any(test, feature = "test-utils"))]
237pub(crate) fn read_all(db: &DatabaseConnection) -> Result<Vec<FileUpload>, DbError> {
238 let rows = db.query(
239 "SELECT u.id,u.location,u.size,u.hash,u.queued_at,u.session,u.bytes_confirmed,
240 u.bytes_total,r.failures,r.not_before,r.wait,u.move,u.stored,u.holds
241 FROM _coven_file_uploads AS u LEFT JOIN _coven_retries AS r ON r.upload=u.id ORDER BY u.id",
242 [],
243 upload_row,
244 )?;
245 decode_uploads(db, rows)
246}
247
248type UploadRow = (
249 (i64, String, i64, Vec<u8>, Vec<u8>),
250 (Option<Vec<u8>>, Option<i64>, Option<i64>),
251 (Option<i64>, Option<Vec<u8>>, Option<i64>),
252 (crate::MoveId, bool, Option<Vec<u8>>),
253);
254
255fn upload_row(r: &rusqlite::Row<'_>) -> rusqlite::Result<UploadRow> {
256 Ok((
257 (r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?, r.get(4)?),
258 (r.get(5)?, r.get(6)?, r.get(7)?),
259 (r.get(8)?, r.get(9)?, r.get(10)?),
260 (r.get(11)?, r.get(12)?, r.get(13)?),
261 ))
262}
263
264#[expect(
265 clippy::map_err_ignore,
266 reason = "a stored value that won't convert or decode is DamagedDatabase"
267)]
268fn decode_uploads(
269 db: &DatabaseConnection,
270 rows: Vec<UploadRow>,
271) -> Result<Vec<FileUpload>, DbError> {
272 rows.into_iter()
273 .map(|((id, location, size, hash, queued), session, retry, (of, stored, holds))| {
274 let retry = match retry {
275 (None, None, None) => None,
276 (Some(failures), Some(not_before), Some(wait)) => {
277 Some(crate::RetryWait::stored(failures, ¬_before, wait)?)
278 }
279 _ => return Err(DbError::DamagedDatabase),
280 };
281 let (session, progress) = match session {
282 (None, None, None) => (None, None),
283 (Some(session), Some(confirmed), Some(total)) => (
284 Some(SecretBytes::new(session)),
285 Some(UploadProgress {
286 bytes_confirmed: u64::try_from(confirmed)
287 .map_err(|_| DbError::DamagedDatabase)?,
288 bytes_total: u64::try_from(total).map_err(|_| DbError::DamagedDatabase)?,
289 }),
290 ),
291 _ => return Err(DbError::DamagedDatabase),
292 };
293 let size = u64::try_from(size).map_err(|_| DbError::DamagedDatabase)?;
294 let bytes_total =
296 FileHeader::new(size).encrypted_size().map_err(|_| DbError::DamagedDatabase)?;
297 Ok(FileUpload {
298 id,
299 location: FileLocation::decode(&location)?,
300 size,
301 content_hash: crate::file_hashes::stored_hash(hash)?,
302 files: db.query(
303 "SELECT file FROM _coven_upload_files WHERE upload=?1 ORDER BY file",
304 [id],
305 |r| r.get::<_, FileId>(0),
306 )?,
307 queued_at: crate::user_file::decode_time(&queued)?,
308 retry,
309 of,
310 stored,
311 holds: holds.map(crate::write_encoding::counter).transpose()?,
312 session,
313 progress,
314 bytes_total,
315 })
316 })
317 .collect()
318}
319
320pub(crate) fn first_held(db: &DatabaseConnection) -> Result<Option<u64>, DbError> {
324 let holds = db.query_row(
325 "SELECT min(u.holds) FROM _coven_file_uploads AS u WHERE u.holds IS NOT NULL
326 AND EXISTS(SELECT 1 FROM _coven_upload_files AS l WHERE l.upload=u.id)",
327 [],
328 |r| r.get::<_, Option<Vec<u8>>>(0),
329 )?;
330 Ok(holds.map(crate::write_encoding::counter).transpose()?)
331}
332
333pub(crate) fn upload_id(
335 db: &DatabaseConnection,
336 uploader: DeviceId,
337 object: ObjectId,
338) -> Result<Option<i64>, DbError> {
339 Ok(db
340 .query(
341 "SELECT id FROM _coven_file_uploads WHERE device=?1 AND object=?2",
342 (uploader, object),
343 |row| row.get(0),
344 )?
345 .pop())
346}
347
348pub(crate) fn upload_of(db: &DatabaseConnection, file: FileId) -> Result<Option<i64>, DbError> {
350 Ok(db
351 .query("SELECT upload FROM _coven_upload_files WHERE file=?1", [file], |row| row.get(0))?
352 .pop())
353}
354
355pub(crate) fn bound(db: &DatabaseConnection) -> Result<std::collections::BTreeSet<i64>, DbError> {
357 Ok(db
358 .query("SELECT id FROM _coven_file_uploads WHERE holds IS NOT NULL", [], |r| r.get(0))?
359 .into_iter()
360 .collect())
361}
362
363pub(crate) fn held_upload_of(db: &DatabaseConnection, file: FileId) -> Result<bool, DbError> {
366 db.query_row(
367 "SELECT EXISTS(SELECT 1 FROM _coven_upload_files AS l
368 JOIN _coven_file_uploads AS u ON u.id=l.upload
369 WHERE l.file=?1 AND u.holds IS NOT NULL)",
370 [file],
371 |row| row.get(0),
372 )
373}
374
375#[expect(
378 clippy::map_err_ignore,
379 reason = "a stored value that won't convert is damage, a computed one a broken invariant"
380)]
381pub(crate) fn chunk_hash(
382 db: &DatabaseConnection,
383 location: &FileLocation,
384 index: u64,
385) -> Result<ContentHash, DbError> {
386 let index = i64::try_from(index)
387 .map_err(|_| DbError::Invariant { what: "a chunk index fits SQLite" })?;
388 let hash = db
389 .query_row(
390 "SELECT hash FROM _coven_file_chunks WHERE device=?1 AND object=?2 AND chunk=?3",
391 (location.uploader(), location.object(), index),
392 |r| r.get::<_, Vec<u8>>(0),
393 )
394 .map_err(|error| match error {
395 DbError::Sqlite(rusqlite::Error::QueryReturnedNoRows) => DbError::DamagedDatabase,
396 error => error,
397 })?;
398 crate::file_hashes::stored_hash(hash)
399}
400
401pub(crate) fn retirable(db: &DatabaseConnection) -> Result<Vec<(i64, FileLocation)>, DbError> {
404 db.query(
405 "SELECT u.id,u.location FROM _coven_file_uploads AS u
406 WHERE u.session IS NULL
407 AND NOT EXISTS(SELECT 1 FROM _coven_upload_files AS l WHERE l.upload=u.id)
408 ORDER BY u.id",
409 [],
410 |r| Ok((r.get::<_, i64>(0)?, r.get::<_, String>(1)?)),
411 )?
412 .into_iter()
413 .map(|(id, location)| Ok((id, FileLocation::decode(&location)?)))
414 .collect()
415}
416
417pub(crate) fn switched(db: &DatabaseConnection, id: i64) -> Result<(), DbError> {
422 for file in
425 db.query("SELECT file FROM _coven_upload_files WHERE upload=?1 AND keep=1", [id], |r| {
426 r.get::<_, FileId>(0)
427 })?
428 {
429 if crate::file_table::Record::read(db, file)?
430 .is_some_and(|record| matches!(record.place, crate::file_location::Place::Cloud(_)))
431 {
432 crate::file_pins::request(db, file)?;
433 }
434 }
435 for path in db.query(
436 "DELETE FROM _coven_device_files
437 WHERE file IN (SELECT file FROM _coven_upload_files WHERE upload=?1) RETURNING path",
438 [id],
439 |r| r.get::<_, String>(0),
440 )? {
441 release_copy(db, &path)?;
442 }
443 crate::user_file::uploaded(db, id)?;
444 retire(db, id)
445}
446
447pub(crate) fn retire(db: &DatabaseConnection, id: i64) -> Result<(), DbError> {
453 let copy: Option<String> = db
454 .query_row("SELECT copy FROM _coven_file_uploads WHERE id=?1", [id], |r| r.get(0))
455 .map_err(|error| match error {
456 DbError::Sqlite(rusqlite::Error::QueryReturnedNoRows) => {
457 DbError::Invariant { what: "a retired upload is queued" }
458 }
459 error => error,
460 })?;
461 db.internal_execute("DELETE FROM _coven_upload_files WHERE upload=?1", [id])?;
462 db.internal_execute("DELETE FROM _coven_file_uploads WHERE id=?1", [id])?;
463 if let Some(copy) = copy {
464 release_copy(db, ©)?;
465 }
466 Ok(())
467}
468
469pub(crate) fn release_copy(db: &DatabaseConnection, path: &str) -> Result<(), DbError> {
473 let named: bool = db.query_row(&format!("SELECT {FILE_NAMED_SQL}"), [path], |r| r.get(0))?;
474 if named {
475 return Ok(());
476 }
477 db.internal_execute("DELETE FROM _coven_copy_chunks WHERE path=?1", [path])?;
478 crate::file_removals::record_released(db, path)?;
479 Ok(())
480}
481
482pub(crate) struct Queued<'a> {
487 pub(crate) files: &'a [(FileId, bool)],
488 pub(crate) size: u64,
489 pub(crate) source: &'a crate::file_write::StagedSource,
490 pub(crate) hashes: &'a crate::file_hashes::FileHashes,
491 pub(crate) of: crate::MoveId,
492 pub(crate) holds: Option<u64>,
493}
494
495pub(crate) fn enqueue(
499 db: &DatabaseConnection,
500 queued: Queued<'_>,
501 location: &FileLocation,
502 now: std::time::SystemTime,
503) -> Result<(), DbError> {
504 enqueue_all(db, &[(queued, location)], now)
505}
506
507pub(crate) fn enqueue_all(
511 db: &DatabaseConnection,
512 uploads: &[(Queued<'_>, &FileLocation)],
513 now: std::time::SystemTime,
514) -> Result<(), DbError> {
515 let mut queue = Vec::new();
516 let mut chunks = Vec::new();
517 for (queued, location) in uploads {
518 let Queued { size, source, hashes, of, holds, .. } = queued;
519 let header = FileHeader::new(*size);
520 if hashes.chunks().len() as u64 != header.chunk_count() {
521 return Err(DbError::Invariant { what: "a staged file hashes every plaintext chunk" });
522 }
523 header.encrypted_size().map_err(|_overflow| DbError::Invariant {
525 what: "a queued file's encrypted layout fits in its offsets",
526 })?;
527 let size = crate::sqlite::integer("file size", *size)?;
528 let (original, modified_at, copy) = match source {
529 crate::file_write::StagedSource::Original(original) => (
530 Some(coven_foundation::files::path_bytes(&original.path)),
531 Some(crate::user_file::encode_time(original.modified_at)),
532 None,
533 ),
534 crate::file_write::StagedSource::Copy(name) => (None, None, Some(name.as_str())),
535 };
536 let (device, object) = (location.uploader(), location.object());
537 queue.push(vec![
538 value(&location.text())?,
539 value(&device)?,
540 value(&object)?,
541 value(&size)?,
542 value(&hashes.content.as_bytes().as_slice())?,
543 value(&original)?,
544 value(&modified_at)?,
545 value(©)?,
546 value(of)?,
547 value(&crate::user_file::encode_time(now))?,
548 value(&holds.map(u64::to_be_bytes))?,
549 ]);
550 for (index, hash) in hashes.chunks().iter().enumerate() {
551 chunks.push(vec![
552 value(&device)?,
553 value(&object)?,
554 value(&(index as i64))?,
555 value(&hash.as_bytes().as_slice())?,
556 ]);
557 }
558 }
559 let mut ids = std::collections::BTreeMap::new();
560 for (object, id) in insert_rows(
561 db,
562 11,
563 &queue,
564 |values| {
565 format!("INSERT INTO _coven_file_uploads(location,device,object,size,hash,original,modified_at,copy,move,queued_at,holds) VALUES{values} RETURNING object,id")
566 },
567 |r| Ok((r.get::<_, ObjectId>(0)?, r.get::<_, i64>(1)?)),
568 )? {
569 ids.insert(object, id);
570 }
571 insert_rows(
572 db,
573 4,
574 &chunks,
575 |values| format!("INSERT INTO _coven_file_chunks(device,object,chunk,hash) VALUES{values}"),
576 |_| Ok(()),
577 )?;
578 let mut linked = Vec::new();
579 let mut listed = Vec::new();
580 for (queued, location) in uploads {
581 let upload = ids
582 .get(&location.object())
583 .ok_or(DbError::Invariant { what: "an enqueued upload returns its id" })?;
584 for (file, keep) in queued.files {
585 linked.push(vec![value(file)?, value(upload)?, value(keep)?]);
586 listed.push((queued.of, *file));
587 }
588 }
589 insert_rows(
590 db,
591 3,
592 &linked,
593 |values| format!("INSERT INTO _coven_upload_files(file,upload,keep) VALUES{values}"),
594 |_| Ok(()),
595 )?;
596 crate::file_moves::list_all(db, &listed)?;
597 Ok(())
598}
599
600pub(crate) fn keep_copy(
604 db: &DatabaseConnection,
605 file: FileId,
606 name: &str,
607 hashes: &crate::file_hashes::FileHashes,
608) -> Result<(), DbError> {
609 keep_copies(db, &[(file, name, hashes)])
610}
611
612pub(crate) fn keep_copies(
615 db: &DatabaseConnection,
616 copies: &[(FileId, &str, &crate::file_hashes::FileHashes)],
617) -> Result<(), DbError> {
618 let mut files = Vec::new();
619 let mut chunks = Vec::new();
620 for (file, name, hashes) in copies {
621 files.push(vec![value(file)?, value(name)?, value(&hashes.content.as_bytes().as_slice())?]);
622 for (index, hash) in hashes.chunks().iter().enumerate() {
623 chunks.push(vec![
624 value(name)?,
625 value(&(index as i64))?,
626 value(&hash.as_bytes().as_slice())?,
627 ]);
628 }
629 }
630 insert_rows(
631 db,
632 3,
633 &files,
634 |values| format!("INSERT INTO _coven_device_files(file,path,hash) VALUES{values}"),
635 |_| Ok(()),
636 )?;
637 insert_rows(
638 db,
639 3,
640 &chunks,
641 |values| format!("INSERT INTO _coven_copy_chunks(path,chunk,hash) VALUES{values}"),
642 |_| Ok(()),
643 )?;
644 Ok(())
645}
646
647pub(crate) fn share(db: &DatabaseConnection, from: FileId, to: FileId) -> Result<(), DbError> {
650 db.internal_execute(
651 "INSERT INTO _coven_upload_files(file,upload,keep)
652 SELECT ?2,upload,keep FROM _coven_upload_files WHERE file=?1",
653 rusqlite::params![from, to],
654 )?;
655 db.internal_execute(
656 "INSERT INTO _coven_device_files(file,path,hash)
657 SELECT ?2,path,hash FROM _coven_device_files WHERE file=?1",
658 rusqlite::params![from, to],
659 )?;
660 Ok(())
661}
662
663pub(crate) fn forget(db: &DatabaseConnection, file: FileId) -> Result<(), DbError> {
667 db.internal_execute("DELETE FROM _coven_upload_files WHERE file=?1", [file])?;
668 for copy in
669 db.query("DELETE FROM _coven_device_files WHERE file=?1 RETURNING path", [file], |r| {
670 r.get::<_, String>(0)
671 })?
672 {
673 release_copy(db, ©)?;
674 }
675 Ok(())
676}
677
678pub(crate) fn forget_session(db: &DatabaseConnection, id: i64) -> Result<(), DbError> {
681 db.internal_execute(
682 "UPDATE _coven_file_uploads SET session=NULL,bytes_confirmed=NULL,bytes_total=NULL
683 WHERE id=?1
684 AND NOT EXISTS(SELECT 1 FROM _coven_upload_files WHERE upload=?1)",
685 [id],
686 )?;
687 Ok(())
688}
689
690pub(crate) fn record_session(
693 db: &DatabaseConnection,
694 id: i64,
695 session: &[u8],
696 confirmed: i64,
697 total: i64,
698) -> Result<usize, DbError> {
699 db.internal_execute(
700 "UPDATE _coven_file_uploads SET session=?2,bytes_confirmed=?3,bytes_total=?4
701 WHERE id=?1",
702 (id, session, confirmed, total),
703 )
704}
705
706pub(crate) fn stored(db: &DatabaseConnection, upload: i64) -> Result<(), DbError> {
708 db.internal_execute("UPDATE _coven_file_uploads SET stored=1 WHERE id=?1", [upload])?;
709 Ok(())
710}
711
712pub(crate) fn unswitch_move(db: &DatabaseConnection, of: crate::MoveId) -> Result<(), DbError> {
715 db.internal_execute(
716 "DELETE FROM _coven_upload_files
717 WHERE upload IN (SELECT id FROM _coven_file_uploads WHERE move=?1)",
718 [of],
719 )?;
720 Ok(())
721}
722
723#[cfg(test)]
724#[path = "file_queue_tests.rs"]
725pub(crate) mod tests;