reading: split the book tracker into books, papers and articles (kind column + migration classifying 20 papers), segmented tabs with counts, article cards that open the link, owner + add link with server-side unfurl, kind switch on detail pages, /trackers/reading alias

src/lib/server/books/unfurl.ts

@@ -0,0 +1,259 @@
+// Server-side link preview for the reading list: fetches a public web page and pulls the
+// title, description, image, site, author and publish date from its <meta>/JSON-LD tags.
+// Only http(s) to public addresses (no localhost / private ranges, checked again on every
+// redirect), a short timeout and a size cap, so the owner's "+ add link" can't be pointed
+// at internal services.
+import { lookup } from 'node:dns/promises';
+import { isIP } from 'node:net';
+
+export type LinkMeta = {
+	url: string;
+	title: string | null;
+	description: string | null;
+	image: string | null;
+	siteName: string | null;
+	author: string | null;
+	publishedAt: string | null;
+};
+
+const TIMEOUT_MS = 8000;
+const MAX_BYTES = 1_500_000;
+const MAX_REDIRECTS = 5;
+const UA = 'Mozilla/5.0 (compatible; stanrunge.dev reading list; +https://stanrunge.dev)';
+
+/** Strips tracking parameters and the fragment-less noise, keeps the rest of the URL. */
+export function cleanUrl(input: string): string | null {
+	let url: URL;
+	try {
+		url = new URL(input.trim());
+	} catch {
+		return null;
+	}
+	if (url.protocol !== 'http:' && url.protocol !== 'https:') return null;
+	if (url.username || url.password) return null;
+	for (const key of [...url.searchParams.keys()])
+		if (/^(utm_|fbclid$|gclid$|mc_|ref_src$)/i.test(key)) url.searchParams.delete(key);
+	return url.toString();
+}
+
+/** arXiv, DOI and similar scholarly hosts are papers rather than articles. */
+export function kindForUrl(url: string): 'paper' | 'article' {
+	const host = new URL(url).hostname.replace(/^www\./, '');
+	return /(^|\.)(arxiv\.org|doi\.org|openreview\.net|aclanthology\.org|proceedings\.neurips\.cc|proceedings\.mlr\.press|semanticscholar\.org|biorxiv\.org|ssrn\.com)$/.test(
+		host
+	) || /\.pdf$/i.test(new URL(url).pathname)
+		? 'paper'
+		: 'article';
+}
+
+export function isPrivateAddress(ip: string): boolean {
+	if (isIP(ip) === 4) {
+		const [a, b] = ip.split('.').map(Number);
+		return (
+			a === 0 ||
+			a === 10 ||
+			a === 127 ||
+			(a === 100 && b >= 64 && b <= 127) ||
+			(a === 169 && b === 254) ||
+			(a === 172 && b >= 16 && b <= 31) ||
+			(a === 192 && b === 168) ||
+			(a === 198 && (b === 18 || b === 19)) ||
+			a >= 224
+		);
+	}
+	const v6 = ip.toLowerCase();
+	if (v6.startsWith('::ffff:')) return isPrivateAddress(v6.slice(7));
+	return v6 === '::' || v6 === '::1' || /^(fc|fd|fe[89ab])/.test(v6);
+}
+
+async function assertPublic(url: URL) {
+	const host = url.hostname.replace(/^\[|\]$/g, '');
+	if (host === 'localhost' || host.endsWith('.localhost') || host.endsWith('.internal'))
+		throw new Error('private host');
+	const addresses = isIP(host) ? [{ address: host }] : await lookup(host, { all: true });
+	if (!addresses.length || addresses.some((a) => isPrivateAddress(a.address)))
+		throw new Error('private host');
+}
+
+const entities: Record<string, string> = {
+	amp: '&',
+	lt: '<',
+	gt: '>',
+	quot: '"',
+	apos: "'",
+	nbsp: ' ',
+	ndash: '–',
+	mdash: '—',
+	hellip: '…',
+	rsquo: '’',
+	lsquo: '‘',
+	rdquo: '”',
+	ldquo: '“'
+};
+export function decodeEntities(s: string): string {
+	return s.replace(/&(#x[0-9a-f]+|#\d+|[a-z]+);/gi, (m, e: string) => {
+		if (e[0] === '#') {
+			const code = e[1] === 'x' || e[1] === 'X' ? parseInt(e.slice(2), 16) : Number(e.slice(1));
+			return Number.isFinite(code) && code > 0 && code < 0x110000 ? String.fromCodePoint(code) : m;
+		}
+		return entities[e.toLowerCase()] ?? m;
+	});
+}
+const clean = (s: string | null | undefined, max = 500) => {
+	const v = s ? decodeEntities(s).replace(/\s+/g, ' ').trim() : '';
+	return v ? (v.length > max ? `${v.slice(0, max - 1).trimEnd()}…` : v) : null;
+};
+
+function attrs(tag: string): Record<string, string> {
+	const out: Record<string, string> = {};
+	for (const m of tag.matchAll(/([a-zA-Z:_-]+)\s*=\s*("([^"]*)"|'([^']*)'|([^\s>]+))/g))
+		out[m[1].toLowerCase()] = m[3] ?? m[4] ?? m[5] ?? '';
+	return out;
+}
+
+/** Parses the <head> of an HTML page. Pure; `base` resolves relative image URLs. */
+export function parseLinkMeta(html: string, base: string): LinkMeta {
+	const head = html.slice(0, 400_000);
+	const meta: Record<string, string> = {};
+	for (const m of head.matchAll(/<meta\b[^>]*>/gi)) {
+		const a = attrs(m[0]);
+		const key = (a.property ?? a.name ?? a.itemprop ?? '').toLowerCase();
+		if (key && a.content != null && !(key in meta)) meta[key] = a.content;
+	}
+	let canonical: string | null = null;
+	for (const m of head.matchAll(/<link\b[^>]*>/gi)) {
+		const a = attrs(m[0]);
+		if (a.rel?.toLowerCase().split(/\s+/).includes('canonical') && a.href) canonical = a.href;
+	}
+	// JSON-LD (Article / BlogPosting): author and dates when the meta tags lack them.
+	let ld: { author?: unknown; datePublished?: string; headline?: string } = {};
+	for (const m of head.matchAll(
+		/<script[^>]+type=["']application\/ld\+json["'][^>]*>([\s\S]*?)<\/script>/gi
+	)) {
+		try {
+			const parsed = JSON.parse(m[1]);
+			const nodes = [
+				parsed,
+				...(Array.isArray(parsed) ? parsed : []),
+				...(parsed?.['@graph'] ?? [])
+			];
+			const article = nodes.find(
+				(n) => n && typeof n === 'object' && /Article|BlogPosting|Report/.test(String(n['@type']))
+			);
+			if (article) {
+				ld = article;
+				break;
+			}
+		} catch {
+			// Broken JSON-LD is common; ignore it.
+		}
+	}
+	const ldAuthor = (() => {
+		const a = ld.author;
+		const list = Array.isArray(a) ? a : a ? [a] : [];
+		const names = list
+			.map((x) => (typeof x === 'string' ? x : (x as { name?: string })?.name))
+			.filter(Boolean);
+		return names.length ? names.join(', ') : null;
+	})();
+	const titleTag = /<title[^>]*>([\s\S]*?)<\/title>/i.exec(head)?.[1];
+	const abs = (u: string | undefined) => {
+		if (!u) return null;
+		try {
+			const url = new URL(decodeEntities(u.trim()), base);
+			return url.protocol === 'https:' || url.protocol === 'http:' ? url.toString() : null;
+		} catch {
+			return null;
+		}
+	};
+	const date = (s: string | undefined) => {
+		if (!s) return null;
+		const d = new Date(s);
+		return Number.isNaN(d.getTime()) ? null : d.toISOString();
+	};
+	const author =
+		meta['author'] ?? meta['article:author'] ?? meta['twitter:creator'] ?? ldAuthor ?? undefined;
+	return {
+		url: abs(canonical ?? meta['og:url']) ?? base,
+		title: clean(meta['og:title'] ?? meta['twitter:title'] ?? ld.headline ?? titleTag, 300),
+		description: clean(
+			meta['og:description'] ?? meta['description'] ?? meta['twitter:description'],
+			600
+		),
+		image: abs(meta['og:image'] ?? meta['og:image:url'] ?? meta['twitter:image']),
+		siteName: clean(meta['og:site_name'] ?? meta['application-name'], 120),
+		author: author && !/^https?:\/\//.test(author) ? clean(author, 200) : null,
+		publishedAt: date(
+			meta['article:published_time'] ??
+				meta['date'] ??
+				meta['dc.date'] ??
+				meta['citation_publication_date'] ??
+				ld.datePublished
+		)
+	};
+}
+
+/** Fetches `input` and returns its preview; falls back to just the URL when the page is unreachable. */
+export async function fetchLinkMeta(input: string): Promise<LinkMeta & { fetched: boolean }> {
+	const empty = (url: string) => ({
+		url,
+		title: null,
+		description: null,
+		image: null,
+		siteName: null,
+		author: null,
+		publishedAt: null,
+		fetched: false
+	});
+	const start = cleanUrl(input);
+	if (!start) throw new Error('Not an http(s) URL');
+	let url = new URL(start);
+	const signal = AbortSignal.timeout(TIMEOUT_MS);
+	try {
+		for (let i = 0; ; i++) {
+			await assertPublic(url);
+			const res = await fetch(url, {
+				redirect: 'manual',
+				signal,
+				headers: { 'user-agent': UA, accept: 'text/html,application/xhtml+xml;q=0.9,*/*;q=0.5' }
+			});
+			const location = res.headers.get('location');
+			if (res.status >= 300 && res.status < 400 && location) {
+				await res.body?.cancel();
+				if (i >= MAX_REDIRECTS) return empty(start);
+				url = new URL(location, url);
+				if (url.protocol !== 'http:' && url.protocol !== 'https:') return empty(start);
+				continue;
+			}
+			if (!res.ok || !/html/i.test(res.headers.get('content-type') ?? '')) {
+				await res.body?.cancel();
+				return empty(cleanUrl(url.toString()) ?? start);
+			}
+			const reader = res.body!.getReader();
+			const chunks: Uint8Array[] = [];
+			let size = 0;
+			while (size < MAX_BYTES) {
+				const { done, value } = await reader.read();
+				if (done) break;
+				chunks.push(value);
+				size += value.length;
+				// The head is all we need.
+				if (new TextDecoder().decode(value).includes('</head>')) break;
+			}
+			await reader.cancel().catch(() => {});
+			const html = new TextDecoder().decode(Buffer.concat(chunks));
+			const final = cleanUrl(url.toString()) ?? start;
+			const meta = parseLinkMeta(html, final);
+			// Only trust a canonical URL on the same site (some pages point at their homepage).
+			const canonical = cleanUrl(meta.url);
+			const sameSite = canonical && new URL(canonical).hostname === new URL(final).hostname;
+			const keep = sameSite && new URL(canonical).pathname.length > 1 ? canonical : final;
+			return { ...meta, url: keep, fetched: true };
+		}
+	} catch (err) {
+		if (err instanceof Error && err.message === 'private host') throw err;
+		return empty(start);
+	}
+}
+
+export const siteFromUrl = (url: string) => new URL(url).hostname.replace(/^www\./, '');