reading: owner archive / restore / delete / merge for books (⋯ menu on cards, rows and detail pages), archived filter, tombstones so merged ids redirect and the ingest never recreates deleted works, books-quarantine.py for deleted files

scripts/books-ingest.py

@@ -244,6 +244,7 @@ class Work:
     shelves: set = field(default_factory=set)
     origins: set = field(default_factory=set)
     db_id: int | None = None
+    existing_id: int | None = None  # rerun: attach new files to this live work
 
     def add_ref(self, k, v):
         self.refs.setdefault(k, [])
@@ -844,6 +845,54 @@ def main():
             used.add(rel)
             fr.dest_rel = rel
 
+    # ---- 5b. reruns: never recreate what is already there, archived, deleted or merged away.
+    # Works that match a live row (lists id, md5, ISBN-13 or file hash) only contribute new
+    # files to it; works that match a tombstone (books.deleted_works) are dropped entirely.
+    skipped_existing = skipped_deleted = 0
+    if not args.reset:
+        live, dead = {}, {}
+        with conn.cursor() as cur:
+            cur.execute("""select w.id, w.isbn13, w.annas_md5, coalesce(w.source_refs->'lists_books_id', '[]'::jsonb),
+                                  coalesce((select array_agg(f.sha256) from books.files f where f.work_id = w.id), '{}')
+                           from books.works w""")
+            rows = [(live, r) for r in cur.fetchall()]
+            cur.execute("select to_regclass('books.deleted_works')")
+            if cur.fetchone()[0] is not None:
+                cur.execute("""select id, isbn13, annas_md5, coalesce(source_refs->'lists_books_id', '[]'::jsonb), file_sha256
+                               from books.deleted_works""")
+                rows += [(dead, r) for r in cur.fetchall()]
+        for index, (wid, i13, md5s, lists_ids, shas) in rows:
+            keys = [("isbn", i13)] if i13 else []
+            keys += [("md5", m) for m in md5s or []] + [("sha", h) for h in shas or []]
+            keys += [("lists", int(x)) for x in (lists_ids if isinstance(lists_ids, list) else [lists_ids])]
+            for k in keys:
+                index.setdefault(k, wid)
+
+        def keys_of(w):
+            ks = [("lists", int(x)) for x in w.refs.get("lists_books_id", [])]
+            ks += [("md5", m) for m in w.md5s] + [("sha", f.sha256) for f in w.files]
+            i13 = next((i for i in w.isbns if i.startswith("978") or i.startswith("979")), None)
+            return ks + ([("isbn", i13)] if i13 else [])
+
+        kept = []
+        for w in works:
+            ks = keys_of(w)
+            gone = next((dead[k] for k in ks if k in dead and k not in live), None)
+            if gone is not None:
+                log(f"skip (deleted/merged #{gone}): {w.title}" + (f" — {len(w.files)} file(s) left in inbox" if w.files else ""))
+                skipped_deleted += 1
+                continue
+            hit = next((live[k] for k in ks if k in live), None)
+            if hit is not None:
+                skipped_existing += 1
+                if w.files:
+                    w.existing_id = hit
+                    kept.append(w)
+                continue
+            kept.append(w)
+        works = kept
+        log(f"reruns: {skipped_existing} already in the library, {skipped_deleted} deleted or merged away")
+
     plan = [{"title": w.title, "authors": w.authors, "isbns": w.isbns, "status": w.status, "origins": sorted(w.origins),
              "files": [{"from": f.rel_inbox, "to": f.dest_rel, "sha256": f.sha256} for f in w.files], "refs": w.refs,
              "cover": w.cover_url} for w in works]
@@ -866,6 +915,12 @@ def main():
         cur.execute("select slug, id from books.shelves")
         shelves = dict(cur.fetchall())
         for w in works:
+            if w.existing_id:
+                w.db_id = w.existing_id
+                for fr in w.files:
+                    cur.execute("insert into books.files (work_id, path, format, size_bytes, sha256, md5, source) values (%s,%s,%s,%s,%s,%s,%s) on conflict do nothing",
+                                (w.db_id, fr.dest_rel, fr.fmt, fr.size, fr.sha256, fr.md5, args.source))
+                continue
             i13 = isbn13(w)
             if i13 in seen13:
                 w.refs["isbn13_shared_with_other_work"] = i13
@@ -899,7 +954,7 @@ def main():
     got = 0
     if not args.no_network:
         for w in works:
-            if not w.cover_url:
+            if not w.cover_url or w.existing_id:
                 continue
             dest = covers / (hashlib.sha1(w.cover_url.encode()).hexdigest()[:16] + ".jpg")
             if dest.exists() or http.download(w.cover_url, dest):
@@ -942,6 +997,7 @@ def main():
         "obsidian_overlapping_lists": obs_overlap, "placeholders": n_placeholders,
         "files_total": len(files), "files_unique": len(uniq), "sha256_duplicates": len(dups),
         "placeholders_with_files": matched_existing, "works_total": len(works),
+        "skipped_existing": skipped_existing, "skipped_deleted": skipped_deleted,
         "works_with_files": sum(1 for w in works if w.files), "metadata": stats, "covers": got,
         "papers": sum(1 for w in works if "paper" in w.tags), "http_calls": http.calls, "manifest": str(manifest),
     }

scripts/books-quarantine.py

@@ -0,0 +1,61 @@
+#!/usr/bin/env python3
+"""Move the files of deleted books to quarantine.
+
+Deleting a book on the site removes its rows but leaves the files in place (the app sees the
+library through a read-only mount). Each delete leaves a tombstone in books.deleted_works with
+the file paths; this moves those files to $BOOKS_ROOT/quarantine/deleted/<work id>/<path> and
+stamps quarantined_at. Files that another work still references are left alone.
+
+Usage (dry run by default):
+  DB_URL=postgresql://... BOOKS_ROOT=~/books python scripts/books-quarantine.py [--apply]
+Undo: move the files back from quarantine/deleted/<id>/ into library/.
+"""
+from __future__ import annotations
+
+import argparse, os, shutil, sys
+from pathlib import Path
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--apply", action="store_true", help="actually move files (default: dry run)")
+    args = ap.parse_args()
+    db_url = os.environ.get("DB_URL") or sys.exit("DB_URL required")
+    root = Path(os.path.expanduser(os.environ.get("BOOKS_ROOT", "/srv/books")))
+    library, quarantine = root / "library", root / "quarantine" / "deleted"
+
+    import psycopg
+    with psycopg.connect(db_url) as conn, conn.cursor() as cur:
+        cur.execute("""select d.id, d.title, d.file_paths from books.deleted_works d
+                       where d.quarantined_at is null and cardinality(d.file_paths) > 0 order by d.id""")
+        rows = cur.fetchall()
+        moved = kept = missing = 0
+        for wid, title, paths in rows:
+            for rel in paths:
+                src = (library / rel).resolve()
+                if not str(src).startswith(str(library.resolve()) + os.sep):
+                    print(f"#{wid}: unsafe path, skipped: {rel}")
+                    continue
+                cur.execute("select work_id from books.files where path = %s", (rel,))
+                if cur.fetchone():
+                    kept += 1
+                    print(f"#{wid}: still used by another work, kept: {rel}")
+                    continue
+                if not src.exists():
+                    missing += 1
+                    continue
+                dest = quarantine / str(wid) / rel
+                print(f"#{wid} {title}: {rel} -> {dest.relative_to(root)}")
+                if args.apply:
+                    dest.parent.mkdir(parents=True, exist_ok=True)
+                    shutil.move(str(src), dest)
+                moved += 1
+            if args.apply:
+                cur.execute("update books.deleted_works set quarantined_at = now() where id = %s", (wid,))
+        if args.apply:
+            conn.commit()
+        print(f"{'moved' if args.apply else 'would move'} {moved}, kept {kept}, already gone {missing}")
+
+
+if __name__ == "__main__":
+    main()