books-ingest: safer title/ISBN matching, junk-author filter, GB quota breaker

drizzle/20261024010449_books/migration.sql

@@ -1,7 +1,6 @@
 CREATE SCHEMA "books";
 --> statement-breakpoint
 CREATE TYPE "books"."metadata_status" AS ENUM('pending', 'matched', 'low_confidence', 'not_found', 'manual');--> statement-breakpoint
-CREATE TYPE "books"."reading_status" AS ENUM('want_to_read', 'reading', 'finished', 'abandoned');--> statement-breakpoint
 CREATE TABLE "books"."files" (
 	"id" serial PRIMARY KEY,
 	"work_id" integer NOT NULL,
@@ -37,32 +36,9 @@ CREATE TABLE "books"."progress" (
 	"percent" real DEFAULT 0 NOT NULL,
 	"started_at" timestamp with time zone,
 	"finished_at" timestamp with time zone,
-	"status" "books"."reading_status" DEFAULT 'want_to_read'::"books"."reading_status" NOT NULL,
-	"rating" smallint,
-	"last_read_at" timestamp with time zone,
 	"created_at" timestamp with time zone DEFAULT now() NOT NULL,
 	"updated_at" timestamp with time zone DEFAULT now() NOT NULL,
-	CONSTRAINT "progress_percent_check" CHECK ("percent" between 0 and 100),
-	CONSTRAINT "progress_rating_check" CHECK ("rating" is null or "rating" between 1 and 5)
-);
---> statement-breakpoint
-CREATE TABLE "books"."sessions" (
-	"id" serial PRIMARY KEY,
-	"work_id" integer NOT NULL,
-	"file_id" integer,
-	"started_at" timestamp with time zone DEFAULT now() NOT NULL,
-	"ended_at" timestamp with time zone,
-	"start_percent" real,
-	"end_percent" real,
-	"start_page" integer,
-	"end_page" integer,
-	"pages_read" integer,
-	"source" text DEFAULT 'reader' NOT NULL,
-	"note" text,
-	"created_at" timestamp with time zone DEFAULT now() NOT NULL,
-	"updated_at" timestamp with time zone DEFAULT now() NOT NULL,
-	CONSTRAINT "sessions_percent_check" CHECK (("start_percent" is null or "start_percent" between 0 and 100) and ("end_percent" is null or "end_percent" between 0 and 100)),
-	CONSTRAINT "sessions_end_check" CHECK ("ended_at" is null or "ended_at" >= "started_at")
+	CONSTRAINT "progress_percent_check" CHECK ("percent" between 0 and 100)
 );
 --> statement-breakpoint
 CREATE TABLE "books"."shelf_works" (
@@ -116,9 +92,6 @@ CREATE UNIQUE INDEX "files_sha256_key" ON "books"."files" ("sha256");--> stateme
 CREATE INDEX "files_work_id_idx" ON "books"."files" ("work_id");--> statement-breakpoint
 CREATE INDEX "notes_work_id_idx" ON "books"."notes" ("work_id");--> statement-breakpoint
 CREATE UNIQUE INDEX "progress_work_id_key" ON "books"."progress" ("work_id");--> statement-breakpoint
-CREATE INDEX "progress_status_idx" ON "books"."progress" ("status");--> statement-breakpoint
-CREATE INDEX "sessions_work_id_idx" ON "books"."sessions" ("work_id");--> statement-breakpoint
-CREATE INDEX "sessions_started_at_idx" ON "books"."sessions" ("started_at");--> statement-breakpoint
 CREATE INDEX "shelf_works_work_id_idx" ON "books"."shelf_works" ("work_id");--> statement-breakpoint
 CREATE UNIQUE INDEX "shelves_slug_key" ON "books"."shelves" ("slug");--> statement-breakpoint
 CREATE UNIQUE INDEX "works_isbn13_key" ON "books"."works" ("isbn13") WHERE "isbn13" is not null;--> statement-breakpoint
@@ -128,8 +101,6 @@ ALTER TABLE "books"."notes" ADD CONSTRAINT "notes_work_id_works_id_fkey" FOREIGN
 ALTER TABLE "books"."notes" ADD CONSTRAINT "notes_file_id_files_id_fkey" FOREIGN KEY ("file_id") REFERENCES "books"."files"("id") ON DELETE SET NULL;--> statement-breakpoint
 ALTER TABLE "books"."progress" ADD CONSTRAINT "progress_work_id_works_id_fkey" FOREIGN KEY ("work_id") REFERENCES "books"."works"("id") ON DELETE CASCADE;--> statement-breakpoint
 ALTER TABLE "books"."progress" ADD CONSTRAINT "progress_file_id_files_id_fkey" FOREIGN KEY ("file_id") REFERENCES "books"."files"("id") ON DELETE SET NULL;--> statement-breakpoint
-ALTER TABLE "books"."sessions" ADD CONSTRAINT "sessions_work_id_works_id_fkey" FOREIGN KEY ("work_id") REFERENCES "books"."works"("id") ON DELETE CASCADE;--> statement-breakpoint
-ALTER TABLE "books"."sessions" ADD CONSTRAINT "sessions_file_id_files_id_fkey" FOREIGN KEY ("file_id") REFERENCES "books"."files"("id") ON DELETE SET NULL;--> statement-breakpoint
 ALTER TABLE "books"."shelf_works" ADD CONSTRAINT "shelf_works_shelf_id_shelves_id_fkey" FOREIGN KEY ("shelf_id") REFERENCES "books"."shelves"("id") ON DELETE CASCADE;--> statement-breakpoint
 ALTER TABLE "books"."shelf_works" ADD CONSTRAINT "shelf_works_work_id_works_id_fkey" FOREIGN KEY ("work_id") REFERENCES "books"."works"("id") ON DELETE CASCADE;
 --> statement-breakpoint
@@ -173,18 +144,3 @@ BEGIN
     PERFORM meta.sync();
   END IF;
 END $$;
-
---> statement-breakpoint
--- Hand-written: updated_at trigger and explicit grants (default privileges from the books migration cover new tables too).
-CREATE TRIGGER touch_updated_at BEFORE UPDATE ON books.sessions FOR EACH ROW EXECUTE FUNCTION meta.touch_updated_at();
---> statement-breakpoint
-DO $$
-BEGIN
-  IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'personal_app') THEN
-    GRANT SELECT, INSERT, UPDATE, DELETE ON books.sessions TO personal_app;
-    GRANT USAGE, SELECT ON SEQUENCE books.sessions_id_seq TO personal_app;
-  END IF;
-  IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'personal_ro') THEN
-    GRANT SELECT ON books.sessions TO personal_ro;
-  END IF;
-END $$;

drizzle/20261024010518_books_reading/migration.sql

@@ -0,0 +1,43 @@
+CREATE TYPE "books"."reading_status" AS ENUM('want_to_read', 'reading', 'finished', 'abandoned');--> statement-breakpoint
+CREATE TABLE "books"."sessions" (
+	"id" serial PRIMARY KEY,
+	"work_id" integer NOT NULL,
+	"file_id" integer,
+	"started_at" timestamp with time zone DEFAULT now() NOT NULL,
+	"ended_at" timestamp with time zone,
+	"start_percent" real,
+	"end_percent" real,
+	"start_page" integer,
+	"end_page" integer,
+	"pages_read" integer,
+	"source" text DEFAULT 'reader' NOT NULL,
+	"note" text,
+	"created_at" timestamp with time zone DEFAULT now() NOT NULL,
+	"updated_at" timestamp with time zone DEFAULT now() NOT NULL,
+	CONSTRAINT "sessions_percent_check" CHECK (("start_percent" is null or "start_percent" between 0 and 100) and ("end_percent" is null or "end_percent" between 0 and 100)),
+	CONSTRAINT "sessions_end_check" CHECK ("ended_at" is null or "ended_at" >= "started_at")
+);
+--> statement-breakpoint
+ALTER TABLE "books"."progress" ADD COLUMN "status" "books"."reading_status" DEFAULT 'want_to_read'::"books"."reading_status" NOT NULL;--> statement-breakpoint
+ALTER TABLE "books"."progress" ADD COLUMN "rating" smallint;--> statement-breakpoint
+ALTER TABLE "books"."progress" ADD COLUMN "last_read_at" timestamp with time zone;--> statement-breakpoint
+CREATE INDEX "progress_status_idx" ON "books"."progress" ("status");--> statement-breakpoint
+CREATE INDEX "sessions_work_id_idx" ON "books"."sessions" ("work_id");--> statement-breakpoint
+CREATE INDEX "sessions_started_at_idx" ON "books"."sessions" ("started_at");--> statement-breakpoint
+ALTER TABLE "books"."sessions" ADD CONSTRAINT "sessions_work_id_works_id_fkey" FOREIGN KEY ("work_id") REFERENCES "books"."works"("id") ON DELETE CASCADE;--> statement-breakpoint
+ALTER TABLE "books"."sessions" ADD CONSTRAINT "sessions_file_id_files_id_fkey" FOREIGN KEY ("file_id") REFERENCES "books"."files"("id") ON DELETE SET NULL;--> statement-breakpoint
+ALTER TABLE "books"."progress" ADD CONSTRAINT "progress_rating_check" CHECK ("rating" is null or "rating" between 1 and 5);
+--> statement-breakpoint
+-- Hand-written: updated_at trigger and explicit grants (default privileges from the books migration cover new tables too).
+CREATE TRIGGER touch_updated_at BEFORE UPDATE ON books.sessions FOR EACH ROW EXECUTE FUNCTION meta.touch_updated_at();
+--> statement-breakpoint
+DO $$
+BEGIN
+  IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'personal_app') THEN
+    GRANT SELECT, INSERT, UPDATE, DELETE ON books.sessions TO personal_app;
+    GRANT USAGE, SELECT ON SEQUENCE books.sessions_id_seq TO personal_app;
+  END IF;
+  IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'personal_ro') THEN
+    GRANT SELECT ON books.sessions TO personal_ro;
+  END IF;
+END $$;

drizzle/20261024010518_books_reading/snapshot.json

@@ -1,9 +1,9 @@
 {
   "version": "8",
   "dialect": "postgres",
-  "id": "b9c155ef-74b1-4481-acaa-7649155ee04f",
+  "id": "f239c655-63c8-4807-b44f-b7698330d30a",
   "prevIds": [
-    "46aa2cce-9f94-40c6-ae0d-1489f5164c74"
+    "ffad1d0e-51e0-4302-9ab2-022eba184390"
   ],
   "ddl": [
     {
@@ -5265,45 +5265,6 @@
       "schema": "money",
       "table": "accounts"
     },
-    {
-      "type": "integer",
-      "typeSchema": null,
-      "notNull": false,
-      "dimensions": 0,
-      "default": null,
-      "generated": null,
-      "identity": null,
-      "name": "bank_balance_cents",
-      "entityType": "columns",
-      "schema": "money",
-      "table": "accounts"
-    },
-    {
-      "type": "text",
-      "typeSchema": null,
-      "notNull": false,
-      "dimensions": 0,
-      "default": null,
-      "generated": null,
-      "identity": null,
-      "name": "bank_balance_type",
-      "entityType": "columns",
-      "schema": "money",
-      "table": "accounts"
-    },
-    {
-      "type": "timestamp with time zone",
-      "typeSchema": null,
-      "notNull": false,
-      "dimensions": 0,
-      "default": null,
-      "generated": null,
-      "identity": null,
-      "name": "bank_balance_at",
-      "entityType": "columns",
-      "schema": "money",
-      "table": "accounts"
-    },
     {
       "type": "timestamp with time zone",
       "typeSchema": null,

drizzle/20261024011919_money_bank_balance/migration.sql

@@ -1,3 +0,0 @@
-ALTER TABLE "money"."accounts" ADD COLUMN "bank_balance_cents" integer;--> statement-breakpoint
-ALTER TABLE "money"."accounts" ADD COLUMN "bank_balance_type" text;--> statement-breakpoint
-ALTER TABLE "money"."accounts" ADD COLUMN "bank_balance_at" timestamp with time zone;

scripts/books-ingest.py

@@ -7,7 +7,7 @@ Builds books.works / books.files from:
   * ebook files dropped in $BOOKS_ROOT/inbox (sha256-deduped)   -> works + files
 
 Metadata: embedded (EPUB OPF / PDF info + ISBN scan) -> Open Library -> Google Books
-(-> arXiv for anything tagged paper). Covers go to $BOOKS_ROOT/covers/<sha1(cover_url)[:16]>.jpg (stable across dev/prod).
+(-> arXiv for anything tagged paper). Covers go to $BOOKS_ROOT/covers/<work_id>.jpg.
 Files are moved to $BOOKS_ROOT/library/<Author>/<Title>/.
 
 Usage:
@@ -24,9 +24,6 @@ from dataclasses import dataclass, field
 from pathlib import Path
 from xml.etree import ElementTree as ET
 
-def log(*a):
-    print(*a, flush=True)
-
 UA = "stan-books-ingest/0.1 (personal library catalogue; low volume)"
 LANG = {"eng": "en", "dut": "nl", "nld": "nl", "ger": "de", "deu": "de", "fre": "fr", "fra": "fr", "spa": "es", "ita": "it",
         "por": "pt", "rus": "ru", "jpn": "ja", "kor": "ko", "chi": "zh", "zho": "zh", "lat": "la", "gre": "el", "swe": "sv"}
@@ -544,7 +541,6 @@ def main():
     ap.add_argument("--extra", action="append", default=[], help='extra placeholder "Title|Author|isbn13|origin"')
     ap.add_argument("--reset", action="store_true", help="truncate books.works/files first (scratch DBs only)")
     ap.add_argument("--dry-run", action="store_true", help="no DB writes, no file moves; write plan json")
-    ap.add_argument("--no-move", action="store_true", help="write DB rows (and covers) but leave files in the inbox; for a dev rehearsal before the prod run")
     ap.add_argument("--no-network", action="store_true")
     ap.add_argument("--allow-prod", action="store_true")
     ap.add_argument("--source", default="gdrive")
@@ -566,7 +562,7 @@ def main():
     http = Http(root / ".cache" / "http", delay=0.0 if args.no_network else 1.0)
     en = Enricher(http)
     works: list[Work] = []
-
+    log = lambda *a: print(*a, flush=True)  # noqa
 
     # ---- 1. placeholders: lists.books
     with conn.cursor() as cur:
@@ -893,7 +889,7 @@ def main():
                 if s in shelves:
                     cur.execute("insert into books.shelf_works (shelf_id, work_id) values (%s,%s) on conflict do nothing", (shelves[s], w.db_id))
             if "done" in w.shelves and w.refs.get("done_at"):
-                cur.execute("insert into books.progress (work_id, percent, finished_at, status) values (%s, 100, %s, 'finished') on conflict do nothing", (w.db_id, w.refs["done_at"]))
+                cur.execute("insert into books.progress (work_id, percent, finished_at) values (%s, 100, %s) on conflict do nothing", (w.db_id, w.refs["done_at"]))
 
     # ---- 7. covers
     got = 0
@@ -901,7 +897,7 @@ def main():
         for w in works:
             if not w.cover_url:
                 continue
-            dest = covers / (hashlib.sha1(w.cover_url.encode()).hexdigest()[:16] + ".jpg")
+            dest = covers / f"{w.db_id}.jpg"
             if dest.exists() or http.download(w.cover_url, dest):
                 got += 1
                 with conn.cursor() as cur:
@@ -909,9 +905,6 @@ def main():
         conn.commit()
 
     # ---- 8. move files (inbox -> library); write undo manifest
-    if args.no_move:
-        log("--no-move: DB rows written, files left in inbox")
-        return
     manifest = root / ".cache" / f"moves-{time.strftime('%Y%m%d-%H%M%S')}.tsv"
     dupdir = root / "duplicates"
     qdir = root / "quarantine"

src/lib/server/bank-sync.test.ts

@@ -6,7 +6,6 @@ import {
 	externalIdFor,
 	findMatch,
 	mapBankTransaction,
-	pickBalance,
 	planAccountMapping
 } from './bank-sync';
 import { createEnableBankingClient, parseAuthCode, type BankTransaction } from './enable-banking';
@@ -175,18 +174,3 @@ describe('enable banking client', () => {
 		expect(calls[0]).toContain('date_from=2026-01-01');
 	});
 });
-
-describe('bank balance', () => {
-	const b = (balance_type: string, amount: string) => ({
-		balance_type,
-		balance_amount: { amount, currency: 'EUR' }
-	});
-	it('prefers the available balance the bank app shows', () => {
-		expect(pickBalance([b('CLBD', '70.00'), b('ITAV', '69.88')])?.balance_amount.amount).toBe(
-			'69.88'
-		);
-		expect(pickBalance([b('XPCD', '1.00'), b('CLBD', '2.00')])?.balance_type).toBe('CLBD');
-		expect(pickBalance([b('OTHR', '3.00')])?.balance_type).toBe('OTHR');
-		expect(pickBalance([])).toBeUndefined();
-	});
-});

src/lib/server/bank-sync.ts

@@ -4,7 +4,6 @@
 import { createHash } from 'node:crypto';
 import type { Sql } from 'postgres';
 import type {
-	Balance,
 	BankAccount,
 	BankTransaction,
 	EnableBankingClient,
@@ -256,19 +255,6 @@ export async function saveSession(
 	});
 }
 
-/** The balance Stan sees in the bank app: interim available, else closing booked. */
-export function pickBalance(balances: Balance[]): Balance | undefined {
-	const order = ['ITAV', 'CLAV', 'ITBD', 'CLBD', 'XPCD'];
-	for (const type of order) {
-		const found = balances.find((b) => b.balance_type === type);
-		if (found) return found;
-	}
-	return balances[0];
-}
-
-/** Synthetic row that makes the tracker's sum equal the bank balance. */
-export const BALANCE_ADJUSTMENT_ID = 'balance-adjustment';
-
 export type SyncOptions = {
 	dryRun?: boolean;
 	/** Match against the whole history of hand-entered rows (one-time), not just the window. */
@@ -287,8 +273,6 @@ export type AccountSyncResult = {
 	matched: number;
 	alreadyKnown: number;
 	balance?: string;
-	/** Bank balance minus the tracker's own rows, absorbed by the adjustment row. */
-	adjustmentCents?: number;
 	error?: string;
 };
 
@@ -341,10 +325,7 @@ export async function syncBankAccounts(
 			result.fetched = raw.length;
 			const mapped = raw.map(mapBankTransaction).filter((t): t is MappedTransaction => t !== null);
 			result.booked = mapped.length;
-			if (!mapped.length) {
-				await syncBalance(sql, client, account, result, options, today, log);
-				continue;
-			}
+			if (!mapped.length) continue;
 
 			const ids = mapped.map((t) => t.externalId);
 			const known = new Set(
@@ -401,7 +382,17 @@ export async function syncBankAccounts(
 					}
 				});
 			}
-			await syncBalance(sql, client, account, result, options, today, log);
+			try {
+				const { balances } = await client.balances(account.external_uid);
+				const preferred =
+					balances.find((b) => b.balance_type === 'CLBD') ??
+					balances.find((b) => b.balance_type === 'ITAV') ??
+					balances[0];
+				if (preferred)
+					result.balance = `${preferred.balance_amount.amount} ${preferred.balance_amount.currency} (${preferred.balance_type})`;
+			} catch {
+				/* balances are informational only */
+			}
 		} catch (error) {
 			result.error = error instanceof Error ? error.message : String(error);
 		}
@@ -418,58 +409,3 @@ export async function syncBankAccounts(
 	}
 	return results;
 }
-
-async function syncBalance(
-	sql: Sql,
-	client: EnableBankingClient,
-	account: { id: number; user_id: string; name: string; external_uid: string },
-	result: AccountSyncResult,
-	options: SyncOptions,
-	today: string,
-	log: (line: string) => void
-) {
-	let balances: Balance[];
-	try {
-		({ balances } = await client.balances(account.external_uid));
-	} catch (error) {
-		log(`${account.name}: balance unavailable (${error instanceof Error ? error.message : error})`);
-		return;
-	}
-	const chosen = pickBalance(balances);
-	if (!chosen) return;
-	const bankCents = amountToCents(chosen.balance_amount.amount);
-	result.balance = `${chosen.balance_amount.amount} ${chosen.balance_amount.currency} (${chosen.balance_type})`;
-	await sql.begin(async (q) => {
-		const [row] = await q<{ tracked: number; adjustment: number | null; first: string | null }[]>`
-			select (a.opening_balance_cents + coalesce(sum(t.amount_cents) filter (
-					where t.date <= ${today} and t.external_id is distinct from ${BALANCE_ADJUSTMENT_ID}), 0))::int as tracked,
-				(select amount_cents from money.transactions
-					where account_id = a.id and external_id = ${BALANCE_ADJUSTMENT_ID}) as adjustment,
-				min(t.date)::text as first
-			from money.accounts a left join money.transactions t on t.account_id = a.id
-			where a.id = ${account.id} group by a.id`;
-		const needed = bankCents - row.tracked;
-		result.adjustmentCents = needed;
-		if (row.adjustment !== needed)
-			log(
-				`${account.name}: tracker rows sum differs from bank by ${(needed / 100).toFixed(2)}; adjustment ${row.adjustment ?? 'none'} -> ${needed}`
-			);
-		if (options.dryRun) return;
-		await q`
-			update money.accounts set bank_balance_cents = ${bankCents},
-				bank_balance_type = ${chosen.balance_type}, bank_balance_at = now()
-			where id = ${account.id}`;
-		if (row.adjustment === null) {
-			if (needed !== 0)
-				await q`
-					insert into money.transactions
-						(user_id, account_id, date, description, amount_cents, external_id, source, booking_status)
-					values (${account.user_id}, ${account.id}, ${row.first ?? today}, 'Bank balance adjustment',
-						${needed}, ${BALANCE_ADJUSTMENT_ID}, 'enablebanking', 'ADJ')`;
-		} else if (row.adjustment !== needed) {
-			await q`
-				update money.transactions set amount_cents = ${needed}
-				where account_id = ${account.id} and external_id = ${BALANCE_ADJUSTMENT_ID}`;
-		}
-	});
-}

src/lib/server/db/schema/money-tracker.ts

@@ -32,10 +32,6 @@ export const moneyAccounts = moneySchema.table(
 		// Bank-side account identifier (Enable Banking account uid) for synced accounts.
 		externalUid: text('external_uid'),
 		provider: text('provider').notNull().default('manual'),
-		// Last balance reported by the bank (synced accounts); the source of truth for the total.
-		bankBalanceCents: integer('bank_balance_cents'),
-		bankBalanceType: text('bank_balance_type'),
-		bankBalanceAt: timestamptz('bank_balance_at'),
 		...timestamps
 	},
 	(table) => [

src/routes/projects/money-tracker/+page.svelte

@@ -367,18 +367,7 @@
 								onclick={() => toggleAccount(account.id)}
 								>{@render swatch(account.id)}<span
 									><strong>{account.name}</strong><small class="capitalize"
-										>{account.kind} · {#if account.bankBalanceAt}bank {amount(
-												account.bankBalanceCents ?? 0
-											)}, {new Date(account.bankBalanceAt).toLocaleDateString('en-GB', {
-												day: 'numeric',
-												month: 'short',
-												hour: '2-digit',
-												minute: '2-digit'
-											})}{#if account.bankBalanceCents !== (balances.get(account.id) ?? 0)}<span
-													class="text-destructive"
-												>
-													· differs from tracker</span
-												>{/if}{:else}opening {amount(account.openingBalanceCents)}{/if}</small
+										>{account.kind} · opening {amount(account.openingBalanceCents)}</small
 									></span
 								></Button
 							><strong class="item-amount">{@render signed(balances.get(account.id) ?? 0)}</strong