stan/website
1
0
Fork 0
Code Issues 25 Pull Requests Actions Packages Projects Releases Wiki Activity

books-ingest: --no-move rehearsal mode, stable cover filenames, finished status, module-level log

Browse Source
This commit is contained in:
Stan Runge
2026-10-24 01:31:35 +00:00
parent 178fa8ddce
commit 7914b0140b
1 changed files with 11 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
scripts/books-ingest.py
View File

@@ -7,7 +7,7 @@ Builds books.works / books.files from:
* ebook files dropped in $BOOKS_ROOT/inbox (sha256-deduped) -> works + files * ebook files dropped in $BOOKS_ROOT/inbox (sha256-deduped) -> works + files
Metadata: embedded (EPUB OPF / PDF info + ISBN scan) -> Open Library -> Google Books Metadata: embedded (EPUB OPF / PDF info + ISBN scan) -> Open Library -> Google Books
(-> arXiv for anything tagged paper). Covers go to $BOOKS_ROOT/covers/<work_id>.jpg. (-> arXiv for anything tagged paper). Covers go to $BOOKS_ROOT/covers/<sha1(cover_url)[:16]>.jpg (stable across dev/prod).
Files are moved to $BOOKS_ROOT/library/<Author>/<Title>/. Files are moved to $BOOKS_ROOT/library/<Author>/<Title>/.
Usage: Usage:
@@ -24,6 +24,9 @@ from dataclasses import dataclass, field
from pathlib import Path from pathlib import Path
from xml.etree import ElementTree as ET from xml.etree import ElementTree as ET
def log(*a):
print(*a, flush=True)
UA = "stan-books-ingest/0.1 (personal library catalogue; low volume)" UA = "stan-books-ingest/0.1 (personal library catalogue; low volume)"
LANG = {"eng": "en", "dut": "nl", "nld": "nl", "ger": "de", "deu": "de", "fre": "fr", "fra": "fr", "spa": "es", "ita": "it", LANG = {"eng": "en", "dut": "nl", "nld": "nl", "ger": "de", "deu": "de", "fre": "fr", "fra": "fr", "spa": "es", "ita": "it",
"por": "pt", "rus": "ru", "jpn": "ja", "kor": "ko", "chi": "zh", "zho": "zh", "lat": "la", "gre": "el", "swe": "sv"} "por": "pt", "rus": "ru", "jpn": "ja", "kor": "ko", "chi": "zh", "zho": "zh", "lat": "la", "gre": "el", "swe": "sv"}
@@ -541,6 +544,7 @@ def main():
ap.add_argument("--extra", action="append", default=[], help='extra placeholder "Title|Author|isbn13|origin"') ap.add_argument("--extra", action="append", default=[], help='extra placeholder "Title|Author|isbn13|origin"')
ap.add_argument("--reset", action="store_true", help="truncate books.works/files first (scratch DBs only)") ap.add_argument("--reset", action="store_true", help="truncate books.works/files first (scratch DBs only)")
ap.add_argument("--dry-run", action="store_true", help="no DB writes, no file moves; write plan json") ap.add_argument("--dry-run", action="store_true", help="no DB writes, no file moves; write plan json")
ap.add_argument("--no-move", action="store_true", help="write DB rows (and covers) but leave files in the inbox; for a dev rehearsal before the prod run")
ap.add_argument("--no-network", action="store_true") ap.add_argument("--no-network", action="store_true")
ap.add_argument("--allow-prod", action="store_true") ap.add_argument("--allow-prod", action="store_true")
ap.add_argument("--source", default="gdrive") ap.add_argument("--source", default="gdrive")
@@ -562,7 +566,7 @@ def main():
http = Http(root / ".cache" / "http", delay=0.0 if args.no_network else 1.0) http = Http(root / ".cache" / "http", delay=0.0 if args.no_network else 1.0)
en = Enricher(http) en = Enricher(http)
works: list[Work] = [] works: list[Work] = []
log = lambda *a: print(*a, flush=True) # noqa
# ---- 1. placeholders: lists.books # ---- 1. placeholders: lists.books
with conn.cursor() as cur: with conn.cursor() as cur:
@@ -889,7 +893,7 @@ def main():
if s in shelves: if s in shelves:
cur.execute("insert into books.shelf_works (shelf_id, work_id) values (%s,%s) on conflict do nothing", (shelves[s], w.db_id)) cur.execute("insert into books.shelf_works (shelf_id, work_id) values (%s,%s) on conflict do nothing", (shelves[s], w.db_id))
if "done" in w.shelves and w.refs.get("done_at"): if "done" in w.shelves and w.refs.get("done_at"):
cur.execute("insert into books.progress (work_id, percent, finished_at) values (%s, 100, %s) on conflict do nothing", (w.db_id, w.refs["done_at"])) cur.execute("insert into books.progress (work_id, percent, finished_at, status) values (%s, 100, %s, 'finished') on conflict do nothing", (w.db_id, w.refs["done_at"]))
# ---- 7. covers # ---- 7. covers
got = 0 got = 0
@@ -897,7 +901,7 @@ def main():
for w in works: for w in works:
if not w.cover_url: if not w.cover_url:
continue continue
dest = covers / f"{w.db_id}.jpg" dest = covers / (hashlib.sha1(w.cover_url.encode()).hexdigest()[:16] + ".jpg")
if dest.exists() or http.download(w.cover_url, dest): if dest.exists() or http.download(w.cover_url, dest):
got += 1 got += 1
with conn.cursor() as cur: with conn.cursor() as cur:
@@ -905,6 +909,9 @@ def main():
conn.commit() conn.commit()
# ---- 8. move files (inbox -> library); write undo manifest # ---- 8. move files (inbox -> library); write undo manifest
if args.no_move:
log("--no-move: DB rows written, files left in inbox")
return
manifest = root / ".cache" / f"moves-{time.strftime('%Y%m%d-%H%M%S')}.tsv" manifest = root / ".cache" / f"moves-{time.strftime('%Y%m%d-%H%M%S')}.tsv"
dupdir = root / "duplicates" dupdir = root / "duplicates"
qdir = root / "quarantine" qdir = root / "quarantine"