Add movies & TV tracker: movies schema, Netflix/Letterboxd/Plex/JSON/recs importers, TMDB matching, stats, owner visibility
This commit is contained in:
200
scripts/movies-import.ts
Normal file
200
scripts/movies-import.ts
Normal file
@@ -0,0 +1,200 @@
|
||||
/**
|
||||
* Import watch history into the movies & TV tracker (movies schema, /trackers/movies).
|
||||
* Idempotent: re-running a file adds nothing new. Plays are keyed by source (source_key), titles
|
||||
* by (type, tmdb_id) or normalised title + year, episodes by season + number (or name). Titles are
|
||||
* deduped across sources by TMDB id: when TMDB_API_KEY is set, the import ends with a matching
|
||||
* pass (scripts/movies-tmdb.ts) that merges rows resolving to the same TMDB title.
|
||||
*
|
||||
* On stan-ovh (DB_URL and the TMDB key are loaded for you, nothing printed):
|
||||
* scripts/movies-run.sh import --netflix ~/imports/ViewingActivity.csv --profile-map "Stan=stan,Juhee=juhee"
|
||||
* scripts/movies-run.sh import --netflix ~/imports/NetflixViewingHistory.csv --who both
|
||||
* scripts/movies-run.sh import --letterboxd ~/imports/letterboxd-export.zip
|
||||
* scripts/movies-run.sh import --plex ~/imports/tautulli-history.json --user-map "stan=stan,juhee=juhee"
|
||||
* scripts/movies-run.sh import --json ~/imports/ju-movies.json
|
||||
* Add --dry-run to parse and print a summary without touching the database.
|
||||
*
|
||||
* --netflix <csv>
|
||||
* ViewingActivity.csv (netflix.com/account/getmyinfo → CONTENT_INTERACTION/ViewingActivity.csv):
|
||||
* Profile Name,Start Time,Duration,Attributes,Title,Supplemental Video Type,Device Type,…
|
||||
* Start Time is UTC ("2024-01-31 20:15:00"), Duration "HH:MM:SS". Trailers/hooks (any
|
||||
* Supplemental Video Type) are skipped; sessions of the same title on the same Amsterdam day are
|
||||
* summed into one play and dropped when under --min-minutes (default 5).
|
||||
* --profile-map "Profile=stan|juhee|both,…" (unmapped profiles use --who), --profiles "Stan,Juhee"
|
||||
* to import only those profiles.
|
||||
* NetflixViewingHistory.csv (account → Viewing activity → Download all): Title,Date. Dates may be
|
||||
* ISO, D-M-YYYY (Dutch site) or slash dates; D/M vs M/D is guessed from the file, else
|
||||
* --date-order dmy|mdy (default dmy). All rows count for --who (default stan).
|
||||
* Titles: "Show: Season 2: Episode" (also Seizoen/Part/Limited Series/…) become show episodes
|
||||
* (season number + name; scripts/movies-tmdb.ts fills episode numbers from TMDB). A title whose
|
||||
* first ": " segment appears with 3+ different endings in the file is a show too; everything
|
||||
* else is a movie. Platform netflix.
|
||||
*
|
||||
* --letterboxd <zip|csv|dir>
|
||||
* The Letterboxd export zip (Settings → Data → Export), its unzipped folder, or one of its CSVs.
|
||||
* diary.csv: one play per entry on its Watched Date (rating from the entry); ratings.csv:
|
||||
* rating; watched.csv: status watched (no play, its date is the logging date); watchlist.csv:
|
||||
* status watchlist. All movies, who = --who (default stan), platform --platform (default none).
|
||||
*
|
||||
* --plex <json|csv> (assumptions: Plex has no official history export, so this reads)
|
||||
* - Tautulli get_history JSON ({"response":{"data":{"data":[…]}}} or a plain array) or a CSV with
|
||||
* the same column names: media_type (movie|episode; others skipped), title, year,
|
||||
* grandparent_title (show), parent_media_index (season), media_index (episode), started or date
|
||||
* (epoch seconds), play_duration or duration (seconds), friendly_name or user, percent_complete
|
||||
* (plays under --min-percent, default 50, skipped), id/row_id (play id), rating_key, guids.
|
||||
* - Plex Media Server /status/sessions/history/all as JSON ({"MediaContainer":{"Metadata":[…]}}):
|
||||
* type, title, grandparentTitle, parentIndex, index, viewedAt, accountID, historyKey.
|
||||
* A tmdb:// or themoviedb:// guid gives the TMDB id directly. --user-map "name=who,…" maps
|
||||
* Plex/Tautulli users (unmapped → --who), --users "a,b" imports only those. Platform plex.
|
||||
*
|
||||
* --json <file> manual entries (Ju or Daisy). An array of:
|
||||
* {
|
||||
* "type": "movie" | "show", // "tv" is accepted as "show"
|
||||
* "title": "Past Lives", "year": 2023, // year optional
|
||||
* "tmdb_id": 666277, // optional; skips matching when given
|
||||
* "status": "watched", // watching | watched | watchlist | dropped (optional)
|
||||
* "rating": 4.5, // 0.5–5 in steps of 0.5 (optional)
|
||||
* "rewatch_count": 1, "first_watched": "2023-09-01", "last_watched": "2024-02-11",
|
||||
* "who": "both", // stan | juhee | both (default stan for plays)
|
||||
* "platform": "cinema", // netflix | plex | cinema | other
|
||||
* "notes": "…", "source": "ju", // "manual" (default) or "ju"
|
||||
* "watched_on": ["2023-09-01"], // movie plays, one per date
|
||||
* "episodes": [{ "season": 1, "episode": 3, "name": "…", "watched_on": "2024-01-02" }]
|
||||
* }
|
||||
* Fields given here overwrite the stored ones; omitted fields are left alone.
|
||||
*
|
||||
* Uses DB_URL (app role). TMDB_API_KEY is optional; without it everything imports unmatched
|
||||
* (no posters) and a later `scripts/movies-run.sh tmdb` matches it. --no-tmdb skips the pass.
|
||||
*/
|
||||
import { readFile, readdir, stat } from 'node:fs/promises';
|
||||
import { execFileSync } from 'node:child_process';
|
||||
import { join } from 'node:path';
|
||||
import postgres from 'postgres';
|
||||
import {
|
||||
fromManual,
|
||||
manualFile,
|
||||
parseLetterboxd,
|
||||
parseNetflix,
|
||||
parsePlex,
|
||||
parseWhoMap,
|
||||
plexRows,
|
||||
normTitle,
|
||||
type ImportItem
|
||||
} from '../src/lib/server/movies/parse';
|
||||
import { importItems } from '../src/lib/server/movies/store';
|
||||
import { Tmdb, runTmdb } from '../src/lib/server/movies/tmdb';
|
||||
import {
|
||||
MOVIE_PLATFORMS,
|
||||
MOVIE_WHO,
|
||||
type MoviePlatform,
|
||||
type MovieWho
|
||||
} from '../src/lib/server/db/schema/movies';
|
||||
|
||||
const argv = process.argv.slice(2);
|
||||
const flag = (name: string) => argv.includes(`--${name}`);
|
||||
const opt = (name: string) => {
|
||||
const i = argv.indexOf(`--${name}`);
|
||||
return i >= 0 ? argv[i + 1] : undefined;
|
||||
};
|
||||
const usage = () => {
|
||||
console.error(
|
||||
'usage: bun scripts/movies-import.ts (--netflix <csv> | --letterboxd <zip|csv|dir> | --plex <json|csv> | --json <file>) [--dry-run] [--no-tmdb] [--who stan|juhee|both] …'
|
||||
);
|
||||
process.exit(2);
|
||||
};
|
||||
const who = (opt('who') ?? 'stan') as MovieWho;
|
||||
if (!MOVIE_WHO.includes(who)) usage();
|
||||
const platform = opt('platform') as MoviePlatform | undefined;
|
||||
if (platform && !MOVIE_PLATFORMS.includes(platform)) usage();
|
||||
const list = (s?: string) =>
|
||||
new Set(
|
||||
(s ?? '')
|
||||
.split(',')
|
||||
.map((x) => normTitle(x))
|
||||
.filter(Boolean)
|
||||
);
|
||||
|
||||
let items: ImportItem[] = [];
|
||||
let skipped = 0;
|
||||
if (opt('netflix')) {
|
||||
const order = opt('date-order');
|
||||
if (order && order !== 'dmy' && order !== 'mdy') usage();
|
||||
const r = parseNetflix(await readFile(opt('netflix')!, 'utf8'), {
|
||||
who,
|
||||
profiles: parseWhoMap(opt('profile-map')),
|
||||
onlyProfiles: list(opt('profiles')),
|
||||
dateOrder: order as 'dmy' | 'mdy' | undefined,
|
||||
minMinutes: opt('min-minutes') ? Number(opt('min-minutes')) : undefined
|
||||
});
|
||||
items = r.items;
|
||||
skipped = r.skipped;
|
||||
} else if (opt('letterboxd')) {
|
||||
const path = opt('letterboxd')!;
|
||||
const files: Record<string, string> = {};
|
||||
if ((await stat(path)).isDirectory()) {
|
||||
for (const f of await readdir(path))
|
||||
if (f.endsWith('.csv')) files[f] = await readFile(join(path, f), 'utf8');
|
||||
} else if (path.toLowerCase().endsWith('.zip')) {
|
||||
const names = execFileSync('unzip', ['-Z1', path], { encoding: 'utf8' })
|
||||
.split('\n')
|
||||
.filter((n) => /(^|\/)[a-z]+\.csv$/i.test(n) && !n.includes('deleted/'));
|
||||
for (const n of names)
|
||||
files[n] = execFileSync('unzip', ['-p', path, n], { encoding: 'utf8', maxBuffer: 64 << 20 });
|
||||
} else files[path] = await readFile(path, 'utf8');
|
||||
items = parseLetterboxd(files, { who, platform: platform ?? null });
|
||||
} else if (opt('plex')) {
|
||||
const path = opt('plex')!;
|
||||
const r = parsePlex(
|
||||
plexRows(await readFile(path, 'utf8'), path.toLowerCase().endsWith('.csv') ? 'csv' : 'json'),
|
||||
{
|
||||
who,
|
||||
users: parseWhoMap(opt('user-map')),
|
||||
onlyUsers: list(opt('users')),
|
||||
minPercent: opt('min-percent') ? Number(opt('min-percent')) : undefined
|
||||
}
|
||||
);
|
||||
items = r.items;
|
||||
skipped = r.skipped;
|
||||
} else if (opt('json')) {
|
||||
const parsed = manualFile.safeParse(JSON.parse(await readFile(opt('json')!, 'utf8')));
|
||||
if (!parsed.success) {
|
||||
for (const issue of parsed.error.issues.slice(0, 50))
|
||||
console.error(`${issue.path.join('.')}: ${issue.message}`);
|
||||
process.exit(1);
|
||||
}
|
||||
items = parsed.data.map(fromManual);
|
||||
} else usage();
|
||||
|
||||
const movies = items.filter((i) => i.type === 'movie').length;
|
||||
const plays = items.reduce((n, i) => n + i.plays.length, 0);
|
||||
console.log(
|
||||
`Parsed ${items.length} titles (${movies} movies, ${items.length - movies} shows), ${plays} plays, ${skipped} rows skipped`
|
||||
);
|
||||
if (flag('dry-run')) {
|
||||
for (const i of items.slice(0, 15))
|
||||
console.log(
|
||||
` ${i.type.padEnd(5)} ${i.title}${i.year ? ` (${i.year})` : ''}: ${i.plays.length} plays${i.episodes?.length ? `, ${i.episodes.length} episodes` : ''}`
|
||||
);
|
||||
process.exit(0);
|
||||
}
|
||||
if (!process.env.DB_URL)
|
||||
throw new Error('DB_URL is required (use scripts/movies-run.sh on stan-ovh)');
|
||||
const sql = postgres(process.env.DB_URL, { max: 1, onnotice: () => {} });
|
||||
try {
|
||||
const s = await importItems(sql, items);
|
||||
console.log(
|
||||
`Imported: ${s.titles} titles (${s.newTitles} new), ${s.newPlays} new plays of ${s.plays}, ${s.episodes} episodes seen`
|
||||
);
|
||||
if (process.env.TMDB_API_KEY && !flag('no-tmdb')) {
|
||||
const m = await runTmdb(sql, new Tmdb(process.env.TMDB_API_KEY), {
|
||||
log: (l) => console.log(` ${l}`)
|
||||
});
|
||||
console.log(
|
||||
`TMDB: ${m.matched} matched, ${m.merged} merged, ${m.notFound} not found, ${m.refreshed} refreshed, ${m.errors} errors`
|
||||
);
|
||||
} else if (!process.env.TMDB_API_KEY)
|
||||
console.log(
|
||||
'No TMDB_API_KEY: titles stay unmatched (no posters) until scripts/movies-run.sh tmdb runs with a key'
|
||||
);
|
||||
} finally {
|
||||
await sql.end();
|
||||
}
|
||||
Reference in New Issue
Block a user