diff --git a/scripts/books-ingest.py b/scripts/books-ingest.py index a029768..91d8345 100644 --- a/scripts/books-ingest.py +++ b/scripts/books-ingest.py @@ -7,7 +7,7 @@ Builds books.works / books.files from: * ebook files dropped in $BOOKS_ROOT/inbox (sha256-deduped) -> works + files Metadata: embedded (EPUB OPF / PDF info + ISBN scan) -> Open Library -> Google Books -(-> arXiv for anything tagged paper). Covers go to $BOOKS_ROOT/covers/.jpg. +(-> arXiv for anything tagged paper). Covers go to $BOOKS_ROOT/covers/.jpg (stable across dev/prod). Files are moved to $BOOKS_ROOT/library///. Usage: @@ -24,6 +24,9 @@ from dataclasses import dataclass, field from pathlib import Path from xml.etree import ElementTree as ET +def log(*a): + print(*a, flush=True) + UA = "stan-books-ingest/0.1 (personal library catalogue; low volume)" LANG = {"eng": "en", "dut": "nl", "nld": "nl", "ger": "de", "deu": "de", "fre": "fr", "fra": "fr", "spa": "es", "ita": "it", "por": "pt", "rus": "ru", "jpn": "ja", "kor": "ko", "chi": "zh", "zho": "zh", "lat": "la", "gre": "el", "swe": "sv"} @@ -541,6 +544,7 @@ def main(): ap.add_argument("--extra", action="append", default=[], help='extra placeholder "Title|Author|isbn13|origin"') ap.add_argument("--reset", action="store_true", help="truncate books.works/files first (scratch DBs only)") ap.add_argument("--dry-run", action="store_true", help="no DB writes, no file moves; write plan json") + ap.add_argument("--no-move", action="store_true", help="write DB rows (and covers) but leave files in the inbox; for a dev rehearsal before the prod run") ap.add_argument("--no-network", action="store_true") ap.add_argument("--allow-prod", action="store_true") ap.add_argument("--source", default="gdrive") @@ -562,7 +566,7 @@ def main(): http = Http(root / ".cache" / "http", delay=0.0 if args.no_network else 1.0) en = Enricher(http) works: list[Work] = [] - log = lambda *a: print(*a, flush=True) # noqa + # ---- 1. placeholders: lists.books with conn.cursor() as cur: @@ -889,7 +893,7 @@ def main(): if s in shelves: cur.execute("insert into books.shelf_works (shelf_id, work_id) values (%s,%s) on conflict do nothing", (shelves[s], w.db_id)) if "done" in w.shelves and w.refs.get("done_at"): - cur.execute("insert into books.progress (work_id, percent, finished_at) values (%s, 100, %s) on conflict do nothing", (w.db_id, w.refs["done_at"])) + cur.execute("insert into books.progress (work_id, percent, finished_at, status) values (%s, 100, %s, 'finished') on conflict do nothing", (w.db_id, w.refs["done_at"])) # ---- 7. covers got = 0 @@ -897,7 +901,7 @@ def main(): for w in works: if not w.cover_url: continue - dest = covers / f"{w.db_id}.jpg" + dest = covers / (hashlib.sha1(w.cover_url.encode()).hexdigest()[:16] + ".jpg") if dest.exists() or http.download(w.cover_url, dest): got += 1 with conn.cursor() as cur: @@ -905,6 +909,9 @@ def main(): conn.commit() # ---- 8. move files (inbox -> library); write undo manifest + if args.no_move: + log("--no-move: DB rows written, files left in inbox") + return manifest = root / ".cache" / f"moves-{time.strftime('%Y%m%d-%H%M%S')}.tsv" dupdir = root / "duplicates" qdir = root / "quarantine"