Add movies & TV tracker: movies schema, Netflix/Letterboxd/Plex/JSON/recs importers, TMDB matching, stats, owner visibility

2026-10-26 00:31:23 +00:00
parent 75571bbfe0
commit 71e015d1d0
28 changed files with 42754 additions and 14 deletions
--- a/scripts/deploy-preflight.ts
+++ b/scripts/deploy-preflight.ts
@@ -211,6 +211,30 @@ try {
 		ai.public_usage
 	)
 		throw new Error(`ai schema tables/grants are wrong: ${JSON.stringify(ai)}`);
+	// Movies & TV tracker (public pages, owner-only edits in the app): app role read/write,
+	// personal_ro none. 5 = titles, episodes, plays, recs, settings.
+	const MOVIES_TABLES = 5;
+	const [movies] = await client`
+		select
+			count(*) filter (where has_table_privilege(c.oid, 'select, insert, update, delete')) as app_rw,
+			count(*) filter (
+				where exists (
+					select 1 from aclexplode(c.relacl) a join pg_roles r on r.oid = a.grantee
+					where r.rolname = 'personal_ro'
+				)
+			) as ro_read,
+			count(*) as tables,
+			has_schema_privilege('public', 'movies', 'usage') as public_usage
+		from pg_class c join pg_namespace n on n.oid = c.relnamespace
+		where n.nspname = 'movies' and c.relkind = 'r'
+	`;
+	if (
+		Number(movies.tables) !== MOVIES_TABLES ||
+		Number(movies.app_rw) !== MOVIES_TABLES ||
+		Number(movies.ro_read) !== 0 ||
+		movies.public_usage
+	)
+		throw new Error(`movies schema tables/grants are wrong: ${JSON.stringify(movies)}`);
 	// Energy tracker reads the ned-collector tables (SELECT only).
 	const [energy] = await client`
 		select count(*) as hours, max(valid_from) as latest from energy.ned_utilization
@@ -220,7 +244,7 @@ try {
 	if (!Number(energy.hours))
 		console.warn('Warning: energy.ned_utilization has no hourly data from the last 7 days');
 	console.log(
-		`health: ${health.tables} tables, tasks: ${tasks.tables} tables, groceries: ${groceries.tables} tables, stats: ${stats.tables} tables, ai: ${ai.tables} tables, app read/write, personal_ro none; energy: ${energy.hours} hourly rows, latest ${energy.latest ? new Date(energy.latest).toISOString() : 'none'}`
+		`health: ${health.tables} tables, tasks: ${tasks.tables} tables, groceries: ${groceries.tables} tables, stats: ${stats.tables} tables, ai: ${ai.tables} tables, movies: ${movies.tables} tables, app read/write, personal_ro none; energy: ${energy.hours} hourly rows, latest ${energy.latest ? new Date(energy.latest).toISOString() : 'none'}`
 	);
 	const [catalog] = await client`
 		select
--- a/scripts/movies-import.ts
+++ b/scripts/movies-import.ts
@@ -0,0 +1,200 @@
+/**
+ * Import watch history into the movies & TV tracker (movies schema, /trackers/movies).
+ * Idempotent: re-running a file adds nothing new. Plays are keyed by source (source_key), titles
+ * by (type, tmdb_id) or normalised title + year, episodes by season + number (or name). Titles are
+ * deduped across sources by TMDB id: when TMDB_API_KEY is set, the import ends with a matching
+ * pass (scripts/movies-tmdb.ts) that merges rows resolving to the same TMDB title.
+ *
+ * On stan-ovh (DB_URL and the TMDB key are loaded for you, nothing printed):
+ *   scripts/movies-run.sh import --netflix ~/imports/ViewingActivity.csv --profile-map "Stan=stan,Juhee=juhee"
+ *   scripts/movies-run.sh import --netflix ~/imports/NetflixViewingHistory.csv --who both
+ *   scripts/movies-run.sh import --letterboxd ~/imports/letterboxd-export.zip
+ *   scripts/movies-run.sh import --plex ~/imports/tautulli-history.json --user-map "stan=stan,juhee=juhee"
+ *   scripts/movies-run.sh import --json ~/imports/ju-movies.json
+ * Add --dry-run to parse and print a summary without touching the database.
+ *
+ * --netflix <csv>
+ *   ViewingActivity.csv (netflix.com/account/getmyinfo → CONTENT_INTERACTION/ViewingActivity.csv):
+ *     Profile Name,Start Time,Duration,Attributes,Title,Supplemental Video Type,Device Type,…
+ *     Start Time is UTC ("2024-01-31 20:15:00"), Duration "HH:MM:SS". Trailers/hooks (any
+ *     Supplemental Video Type) are skipped; sessions of the same title on the same Amsterdam day are
+ *     summed into one play and dropped when under --min-minutes (default 5).
+ *     --profile-map "Profile=stan|juhee|both,…" (unmapped profiles use --who), --profiles "Stan,Juhee"
+ *     to import only those profiles.
+ *   NetflixViewingHistory.csv (account → Viewing activity → Download all): Title,Date. Dates may be
+ *     ISO, D-M-YYYY (Dutch site) or slash dates; D/M vs M/D is guessed from the file, else
+ *     --date-order dmy|mdy (default dmy). All rows count for --who (default stan).
+ *   Titles: "Show: Season 2: Episode" (also Seizoen/Part/Limited Series/…) become show episodes
+ *   (season number + name; scripts/movies-tmdb.ts fills episode numbers from TMDB). A title whose
+ *   first ": " segment appears with 3+ different endings in the file is a show too; everything
+ *   else is a movie. Platform netflix.
+ *
+ * --letterboxd <zip|csv|dir>
+ *   The Letterboxd export zip (Settings → Data → Export), its unzipped folder, or one of its CSVs.
+ *   diary.csv: one play per entry on its Watched Date (rating from the entry); ratings.csv:
+ *   rating; watched.csv: status watched (no play, its date is the logging date); watchlist.csv:
+ *   status watchlist. All movies, who = --who (default stan), platform --platform (default none).
+ *
+ * --plex <json|csv>   (assumptions: Plex has no official history export, so this reads)
+ *   - Tautulli get_history JSON ({"response":{"data":{"data":[…]}}} or a plain array) or a CSV with
+ *     the same column names: media_type (movie|episode; others skipped), title, year,
+ *     grandparent_title (show), parent_media_index (season), media_index (episode), started or date
+ *     (epoch seconds), play_duration or duration (seconds), friendly_name or user, percent_complete
+ *     (plays under --min-percent, default 50, skipped), id/row_id (play id), rating_key, guids.
+ *   - Plex Media Server /status/sessions/history/all as JSON ({"MediaContainer":{"Metadata":[…]}}):
+ *     type, title, grandparentTitle, parentIndex, index, viewedAt, accountID, historyKey.
+ *   A tmdb:// or themoviedb:// guid gives the TMDB id directly. --user-map "name=who,…" maps
+ *   Plex/Tautulli users (unmapped → --who), --users "a,b" imports only those. Platform plex.
+ *
+ * --json <file>   manual entries (Ju or Daisy). An array of:
+ *   {
+ *     "type": "movie" | "show",            // "tv" is accepted as "show"
+ *     "title": "Past Lives", "year": 2023,  // year optional
+ *     "tmdb_id": 666277,                    // optional; skips matching when given
+ *     "status": "watched",                  // watching | watched | watchlist | dropped (optional)
+ *     "rating": 4.5,                        // 0.5–5 in steps of 0.5 (optional)
+ *     "rewatch_count": 1, "first_watched": "2023-09-01", "last_watched": "2024-02-11",
+ *     "who": "both",                        // stan | juhee | both (default stan for plays)
+ *     "platform": "cinema",                 // netflix | plex | cinema | other
+ *     "notes": "…", "source": "ju",         // "manual" (default) or "ju"
+ *     "watched_on": ["2023-09-01"],         // movie plays, one per date
+ *     "episodes": [{ "season": 1, "episode": 3, "name": "…", "watched_on": "2024-01-02" }]
+ *   }
+ *   Fields given here overwrite the stored ones; omitted fields are left alone.
+ *
+ * Uses DB_URL (app role). TMDB_API_KEY is optional; without it everything imports unmatched
+ * (no posters) and a later `scripts/movies-run.sh tmdb` matches it. --no-tmdb skips the pass.
+ */
+import { readFile, readdir, stat } from 'node:fs/promises';
+import { execFileSync } from 'node:child_process';
+import { join } from 'node:path';
+import postgres from 'postgres';
+import {
+	fromManual,
+	manualFile,
+	parseLetterboxd,
+	parseNetflix,
+	parsePlex,
+	parseWhoMap,
+	plexRows,
+	normTitle,
+	type ImportItem
+} from '../src/lib/server/movies/parse';
+import { importItems } from '../src/lib/server/movies/store';
+import { Tmdb, runTmdb } from '../src/lib/server/movies/tmdb';
+import {
+	MOVIE_PLATFORMS,
+	MOVIE_WHO,
+	type MoviePlatform,
+	type MovieWho
+} from '../src/lib/server/db/schema/movies';
+
+const argv = process.argv.slice(2);
+const flag = (name: string) => argv.includes(`--${name}`);
+const opt = (name: string) => {
+	const i = argv.indexOf(`--${name}`);
+	return i >= 0 ? argv[i + 1] : undefined;
+};
+const usage = () => {
+	console.error(
+		'usage: bun scripts/movies-import.ts (--netflix <csv> | --letterboxd <zip|csv|dir> | --plex <json|csv> | --json <file>) [--dry-run] [--no-tmdb] [--who stan|juhee|both] …'
+	);
+	process.exit(2);
+};
+const who = (opt('who') ?? 'stan') as MovieWho;
+if (!MOVIE_WHO.includes(who)) usage();
+const platform = opt('platform') as MoviePlatform | undefined;
+if (platform && !MOVIE_PLATFORMS.includes(platform)) usage();
+const list = (s?: string) =>
+	new Set(
+		(s ?? '')
+			.split(',')
+			.map((x) => normTitle(x))
+			.filter(Boolean)
+	);
+
+let items: ImportItem[] = [];
+let skipped = 0;
+if (opt('netflix')) {
+	const order = opt('date-order');
+	if (order && order !== 'dmy' && order !== 'mdy') usage();
+	const r = parseNetflix(await readFile(opt('netflix')!, 'utf8'), {
+		who,
+		profiles: parseWhoMap(opt('profile-map')),
+		onlyProfiles: list(opt('profiles')),
+		dateOrder: order as 'dmy' | 'mdy' | undefined,
+		minMinutes: opt('min-minutes') ? Number(opt('min-minutes')) : undefined
+	});
+	items = r.items;
+	skipped = r.skipped;
+} else if (opt('letterboxd')) {
+	const path = opt('letterboxd')!;
+	const files: Record<string, string> = {};
+	if ((await stat(path)).isDirectory()) {
+		for (const f of await readdir(path))
+			if (f.endsWith('.csv')) files[f] = await readFile(join(path, f), 'utf8');
+	} else if (path.toLowerCase().endsWith('.zip')) {
+		const names = execFileSync('unzip', ['-Z1', path], { encoding: 'utf8' })
+			.split('\n')
+			.filter((n) => /(^|\/)[a-z]+\.csv$/i.test(n) && !n.includes('deleted/'));
+		for (const n of names)
+			files[n] = execFileSync('unzip', ['-p', path, n], { encoding: 'utf8', maxBuffer: 64 << 20 });
+	} else files[path] = await readFile(path, 'utf8');
+	items = parseLetterboxd(files, { who, platform: platform ?? null });
+} else if (opt('plex')) {
+	const path = opt('plex')!;
+	const r = parsePlex(
+		plexRows(await readFile(path, 'utf8'), path.toLowerCase().endsWith('.csv') ? 'csv' : 'json'),
+		{
+			who,
+			users: parseWhoMap(opt('user-map')),
+			onlyUsers: list(opt('users')),
+			minPercent: opt('min-percent') ? Number(opt('min-percent')) : undefined
+		}
+	);
+	items = r.items;
+	skipped = r.skipped;
+} else if (opt('json')) {
+	const parsed = manualFile.safeParse(JSON.parse(await readFile(opt('json')!, 'utf8')));
+	if (!parsed.success) {
+		for (const issue of parsed.error.issues.slice(0, 50))
+			console.error(`${issue.path.join('.')}: ${issue.message}`);
+		process.exit(1);
+	}
+	items = parsed.data.map(fromManual);
+} else usage();
+
+const movies = items.filter((i) => i.type === 'movie').length;
+const plays = items.reduce((n, i) => n + i.plays.length, 0);
+console.log(
+	`Parsed ${items.length} titles (${movies} movies, ${items.length - movies} shows), ${plays} plays, ${skipped} rows skipped`
+);
+if (flag('dry-run')) {
+	for (const i of items.slice(0, 15))
+		console.log(
+			`  ${i.type.padEnd(5)} ${i.title}${i.year ? ` (${i.year})` : ''}: ${i.plays.length} plays${i.episodes?.length ? `, ${i.episodes.length} episodes` : ''}`
+		);
+	process.exit(0);
+}
+if (!process.env.DB_URL)
+	throw new Error('DB_URL is required (use scripts/movies-run.sh on stan-ovh)');
+const sql = postgres(process.env.DB_URL, { max: 1, onnotice: () => {} });
+try {
+	const s = await importItems(sql, items);
+	console.log(
+		`Imported: ${s.titles} titles (${s.newTitles} new), ${s.newPlays} new plays of ${s.plays}, ${s.episodes} episodes seen`
+	);
+	if (process.env.TMDB_API_KEY && !flag('no-tmdb')) {
+		const m = await runTmdb(sql, new Tmdb(process.env.TMDB_API_KEY), {
+			log: (l) => console.log(`  ${l}`)
+		});
+		console.log(
+			`TMDB: ${m.matched} matched, ${m.merged} merged, ${m.notFound} not found, ${m.refreshed} refreshed, ${m.errors} errors`
+		);
+	} else if (!process.env.TMDB_API_KEY)
+		console.log(
+			'No TMDB_API_KEY: titles stay unmatched (no posters) until scripts/movies-run.sh tmdb runs with a key'
+		);
+} finally {
+	await sql.end();
+}
--- a/scripts/movies-recs-import.ts
+++ b/scripts/movies-recs-import.ts
@@ -0,0 +1,47 @@
+/**
+ * Import Ju's recommendations into movies.recs (the "Ju recommends" feed on /trackers/movies).
+ * Idempotent: upserts on (type, tmdb_id); title, for, reason and added_at are overwritten.
+ *
+ *   scripts/movies-run.sh recs ~/imports/ju-recs.json            # upsert
+ *   scripts/movies-run.sh recs ~/imports/ju-recs.json --prune    # also delete recs not in the file
+ *   bun scripts/movies-recs-import.ts --dry-run ju-recs.json      # validate only (no DB)
+ *
+ * JSON: an array of
+ *   { "tmdb_id": 496243, "type": "movie", "title": "Parasite", "for": "both",
+ *     "reason": "one line on why", "added_at": "2026-10-26T10:00:00+01:00" }
+ * type: movie | show ("tv" accepted); for: stan | juhee | both; reason and added_at optional
+ * (added_at defaults to now). Posters and year come from TMDB when TMDB_API_KEY is set.
+ */
+import { readFile } from 'node:fs/promises';
+import postgres from 'postgres';
+import { recsFile } from '../src/lib/server/movies/parse';
+import { upsertRecs } from '../src/lib/server/movies/store';
+import { Tmdb, runTmdb } from '../src/lib/server/movies/tmdb';
+
+const args = process.argv.slice(2);
+const files = args.filter((a) => !a.startsWith('--'));
+if (files.length !== 1) {
+	console.error('usage: bun scripts/movies-recs-import.ts [--dry-run] [--prune] <file.json>');
+	process.exit(2);
+}
+const parsed = recsFile.safeParse(JSON.parse(await readFile(files[0], 'utf8')));
+if (!parsed.success) {
+	for (const issue of parsed.error.issues.slice(0, 50))
+		console.error(`${issue.path.join('.')}: ${issue.message}`);
+	process.exit(1);
+}
+console.log(`Valid: ${parsed.data.length} recommendations`);
+if (args.includes('--dry-run')) process.exit(0);
+if (!process.env.DB_URL)
+	throw new Error('DB_URL is required (use scripts/movies-run.sh on stan-ovh)');
+const sql = postgres(process.env.DB_URL, { max: 1, onnotice: () => {} });
+try {
+	const r = await upsertRecs(sql, parsed.data, args.includes('--prune'));
+	console.log(`Upserted ${r.upserted} recommendations${r.pruned ? `, pruned ${r.pruned}` : ''}`);
+	if (process.env.TMDB_API_KEY) {
+		const m = await runTmdb(sql, new Tmdb(process.env.TMDB_API_KEY), { limit: 0 });
+		console.log(`TMDB: ${m.errors} errors`);
+	}
+} finally {
+	await sql.end();
+}
--- a/scripts/movies-run.sh
+++ b/scripts/movies-run.sh
@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+# Run a movies tracker script on stan-ovh against prod, without printing secrets.
+#   scripts/movies-run.sh import --netflix ~/imports/ViewingActivity.csv [--dry-run] …
+#   scripts/movies-run.sh recs ~/imports/ju-recs.json [--prune]
+#   scripts/movies-run.sh tmdb [--retry-not-found]
+# DB_URL (app role) comes from the running app container (db:5432 → 127.0.0.1:5440).
+# TMDB_API_KEY comes from ~/.config/movies/tmdb.env (mode 600, one line TMDB_API_KEY=…), if present.
+# Runs from this checkout; use a checkout of main (e.g. ~/dev/personal-website after a pull).
+set -euo pipefail
+cd "$(dirname "$0")/.."
+cmd=${1:?usage: movies-run.sh import|recs|tmdb [args]}
+shift
+case "$cmd" in
+	import) script=scripts/movies-import.ts ;;
+	recs) script=scripts/movies-recs-import.ts ;;
+	tmdb) script=scripts/movies-tmdb.ts ;;
+	*) echo "unknown command $cmd (import, recs or tmdb)" >&2; exit 2 ;;
+esac
+DB_URL=$(sudo -n docker inspect personal-website-app-1 --format '{{range .Config.Env}}{{println .}}{{end}}' \
+	| sed -n 's/^DB_URL=//p' | sed 's/@db:5432/@127.0.0.1:5440/')
+[ -n "$DB_URL" ] || { echo "no DB_URL from the app container" >&2; exit 1; }
+export DB_URL
+env_file="${MOVIES_ENV_FILE:-$HOME/.config/movies/tmdb.env}"
+if [ -r "$env_file" ]; then
+	set -a
+	. "$env_file"
+	set +a
+fi
+exec bun "$script" "$@"
--- a/scripts/movies-tmdb.ts
+++ b/scripts/movies-tmdb.ts
@@ -0,0 +1,38 @@
+/**
+ * TMDB pass for the movies & TV tracker: match unmatched titles to TMDB ids, merge rows that turn
+ * out to be the same TMDB title, and cache posters, genres, year, runtime, cast and episode numbers.
+ * Needs TMDB_API_KEY (v3 key or v4 read token); scripts/movies-run.sh loads it from
+ * ~/.config/movies/tmdb.env. Without a key it exits 0 and changes nothing.
+ *
+ *   scripts/movies-run.sh tmdb                    # pending titles + missing/old metadata
+ *   scripts/movies-run.sh tmdb --retry-not-found  # also retry titles TMDB didn't find before
+ *   scripts/movies-run.sh tmdb --refresh-days 30  # refetch metadata older than 30 days (default 90)
+ */
+import postgres from 'postgres';
+import { Tmdb, runTmdb } from '../src/lib/server/movies/tmdb';
+
+const args = process.argv.slice(2);
+const opt = (n: string) => {
+	const i = args.indexOf(`--${n}`);
+	return i >= 0 ? args[i + 1] : undefined;
+};
+if (!process.env.TMDB_API_KEY) {
+	console.log('No TMDB_API_KEY set: nothing to do');
+	process.exit(0);
+}
+if (!process.env.DB_URL)
+	throw new Error('DB_URL is required (use scripts/movies-run.sh on stan-ovh)');
+const sql = postgres(process.env.DB_URL, { max: 1, onnotice: () => {} });
+try {
+	const m = await runTmdb(sql, new Tmdb(process.env.TMDB_API_KEY), {
+		retryNotFound: args.includes('--retry-not-found'),
+		refreshDays: opt('refresh-days') ? Number(opt('refresh-days')) : undefined,
+		limit: opt('limit') ? Number(opt('limit')) : undefined,
+		log: (l) => console.log(`  ${l}`)
+	});
+	console.log(
+		`TMDB: tried ${m.tried}, ${m.matched} matched, ${m.merged} merged, ${m.notFound} not found, ${m.refreshed} refreshed, ${m.errors} errors`
+	);
+} finally {
+	await sql.end();
+}