Stan Runge
e31e1be73f
All checks were successful
Deploy production / deploy (push) Successful in 1m8s
260 lines
8.9 KiB
TypeScript
260 lines
8.9 KiB
TypeScript
// Server-side link preview for the reading list: fetches a public web page and pulls the
|
||
// title, description, image, site, author and publish date from its <meta>/JSON-LD tags.
|
||
// Only http(s) to public addresses (no localhost / private ranges, checked again on every
|
||
// redirect), a short timeout and a size cap, so the owner's "+ add link" can't be pointed
|
||
// at internal services.
|
||
import { lookup } from 'node:dns/promises';
|
||
import { isIP } from 'node:net';
|
||
|
||
export type LinkMeta = {
|
||
url: string;
|
||
title: string | null;
|
||
description: string | null;
|
||
image: string | null;
|
||
siteName: string | null;
|
||
author: string | null;
|
||
publishedAt: string | null;
|
||
};
|
||
|
||
const TIMEOUT_MS = 8000;
|
||
const MAX_BYTES = 1_500_000;
|
||
const MAX_REDIRECTS = 5;
|
||
const UA = 'Mozilla/5.0 (compatible; stanrunge.dev reading list; +https://stanrunge.dev)';
|
||
|
||
/** Strips tracking parameters and the fragment-less noise, keeps the rest of the URL. */
|
||
export function cleanUrl(input: string): string | null {
|
||
let url: URL;
|
||
try {
|
||
url = new URL(input.trim());
|
||
} catch {
|
||
return null;
|
||
}
|
||
if (url.protocol !== 'http:' && url.protocol !== 'https:') return null;
|
||
if (url.username || url.password) return null;
|
||
for (const key of [...url.searchParams.keys()])
|
||
if (/^(utm_|fbclid$|gclid$|mc_|ref_src$)/i.test(key)) url.searchParams.delete(key);
|
||
return url.toString();
|
||
}
|
||
|
||
/** arXiv, DOI and similar scholarly hosts are papers rather than articles. */
|
||
export function kindForUrl(url: string): 'paper' | 'article' {
|
||
const host = new URL(url).hostname.replace(/^www\./, '');
|
||
return /(^|\.)(arxiv\.org|doi\.org|openreview\.net|aclanthology\.org|proceedings\.neurips\.cc|proceedings\.mlr\.press|semanticscholar\.org|biorxiv\.org|ssrn\.com)$/.test(
|
||
host
|
||
) || /\.pdf$/i.test(new URL(url).pathname)
|
||
? 'paper'
|
||
: 'article';
|
||
}
|
||
|
||
export function isPrivateAddress(ip: string): boolean {
|
||
if (isIP(ip) === 4) {
|
||
const [a, b] = ip.split('.').map(Number);
|
||
return (
|
||
a === 0 ||
|
||
a === 10 ||
|
||
a === 127 ||
|
||
(a === 100 && b >= 64 && b <= 127) ||
|
||
(a === 169 && b === 254) ||
|
||
(a === 172 && b >= 16 && b <= 31) ||
|
||
(a === 192 && b === 168) ||
|
||
(a === 198 && (b === 18 || b === 19)) ||
|
||
a >= 224
|
||
);
|
||
}
|
||
const v6 = ip.toLowerCase();
|
||
if (v6.startsWith('::ffff:')) return isPrivateAddress(v6.slice(7));
|
||
return v6 === '::' || v6 === '::1' || /^(fc|fd|fe[89ab])/.test(v6);
|
||
}
|
||
|
||
async function assertPublic(url: URL) {
|
||
const host = url.hostname.replace(/^\[|\]$/g, '');
|
||
if (host === 'localhost' || host.endsWith('.localhost') || host.endsWith('.internal'))
|
||
throw new Error('private host');
|
||
const addresses = isIP(host) ? [{ address: host }] : await lookup(host, { all: true });
|
||
if (!addresses.length || addresses.some((a) => isPrivateAddress(a.address)))
|
||
throw new Error('private host');
|
||
}
|
||
|
||
const entities: Record<string, string> = {
|
||
amp: '&',
|
||
lt: '<',
|
||
gt: '>',
|
||
quot: '"',
|
||
apos: "'",
|
||
nbsp: ' ',
|
||
ndash: '–',
|
||
mdash: '—',
|
||
hellip: '…',
|
||
rsquo: '’',
|
||
lsquo: '‘',
|
||
rdquo: '”',
|
||
ldquo: '“'
|
||
};
|
||
export function decodeEntities(s: string): string {
|
||
return s.replace(/&(#x[0-9a-f]+|#\d+|[a-z]+);/gi, (m, e: string) => {
|
||
if (e[0] === '#') {
|
||
const code = e[1] === 'x' || e[1] === 'X' ? parseInt(e.slice(2), 16) : Number(e.slice(1));
|
||
return Number.isFinite(code) && code > 0 && code < 0x110000 ? String.fromCodePoint(code) : m;
|
||
}
|
||
return entities[e.toLowerCase()] ?? m;
|
||
});
|
||
}
|
||
const clean = (s: string | null | undefined, max = 500) => {
|
||
const v = s ? decodeEntities(s).replace(/\s+/g, ' ').trim() : '';
|
||
return v ? (v.length > max ? `${v.slice(0, max - 1).trimEnd()}…` : v) : null;
|
||
};
|
||
|
||
function attrs(tag: string): Record<string, string> {
|
||
const out: Record<string, string> = {};
|
||
for (const m of tag.matchAll(/([a-zA-Z:_-]+)\s*=\s*("([^"]*)"|'([^']*)'|([^\s>]+))/g))
|
||
out[m[1].toLowerCase()] = m[3] ?? m[4] ?? m[5] ?? '';
|
||
return out;
|
||
}
|
||
|
||
/** Parses the <head> of an HTML page. Pure; `base` resolves relative image URLs. */
|
||
export function parseLinkMeta(html: string, base: string): LinkMeta {
|
||
const head = html.slice(0, 400_000);
|
||
const meta: Record<string, string> = {};
|
||
for (const m of head.matchAll(/<meta\b[^>]*>/gi)) {
|
||
const a = attrs(m[0]);
|
||
const key = (a.property ?? a.name ?? a.itemprop ?? '').toLowerCase();
|
||
if (key && a.content != null && !(key in meta)) meta[key] = a.content;
|
||
}
|
||
let canonical: string | null = null;
|
||
for (const m of head.matchAll(/<link\b[^>]*>/gi)) {
|
||
const a = attrs(m[0]);
|
||
if (a.rel?.toLowerCase().split(/\s+/).includes('canonical') && a.href) canonical = a.href;
|
||
}
|
||
// JSON-LD (Article / BlogPosting): author and dates when the meta tags lack them.
|
||
let ld: { author?: unknown; datePublished?: string; headline?: string } = {};
|
||
for (const m of head.matchAll(
|
||
/<script[^>]+type=["']application\/ld\+json["'][^>]*>([\s\S]*?)<\/script>/gi
|
||
)) {
|
||
try {
|
||
const parsed = JSON.parse(m[1]);
|
||
const nodes = [
|
||
parsed,
|
||
...(Array.isArray(parsed) ? parsed : []),
|
||
...(parsed?.['@graph'] ?? [])
|
||
];
|
||
const article = nodes.find(
|
||
(n) => n && typeof n === 'object' && /Article|BlogPosting|Report/.test(String(n['@type']))
|
||
);
|
||
if (article) {
|
||
ld = article;
|
||
break;
|
||
}
|
||
} catch {
|
||
// Broken JSON-LD is common; ignore it.
|
||
}
|
||
}
|
||
const ldAuthor = (() => {
|
||
const a = ld.author;
|
||
const list = Array.isArray(a) ? a : a ? [a] : [];
|
||
const names = list
|
||
.map((x) => (typeof x === 'string' ? x : (x as { name?: string })?.name))
|
||
.filter(Boolean);
|
||
return names.length ? names.join(', ') : null;
|
||
})();
|
||
const titleTag = /<title[^>]*>([\s\S]*?)<\/title>/i.exec(head)?.[1];
|
||
const abs = (u: string | undefined) => {
|
||
if (!u) return null;
|
||
try {
|
||
const url = new URL(decodeEntities(u.trim()), base);
|
||
return url.protocol === 'https:' || url.protocol === 'http:' ? url.toString() : null;
|
||
} catch {
|
||
return null;
|
||
}
|
||
};
|
||
const date = (s: string | undefined) => {
|
||
if (!s) return null;
|
||
const d = new Date(s);
|
||
return Number.isNaN(d.getTime()) ? null : d.toISOString();
|
||
};
|
||
const author =
|
||
meta['author'] ?? meta['article:author'] ?? meta['twitter:creator'] ?? ldAuthor ?? undefined;
|
||
return {
|
||
url: abs(canonical ?? meta['og:url']) ?? base,
|
||
title: clean(meta['og:title'] ?? meta['twitter:title'] ?? ld.headline ?? titleTag, 300),
|
||
description: clean(
|
||
meta['og:description'] ?? meta['description'] ?? meta['twitter:description'],
|
||
600
|
||
),
|
||
image: abs(meta['og:image'] ?? meta['og:image:url'] ?? meta['twitter:image']),
|
||
siteName: clean(meta['og:site_name'] ?? meta['application-name'], 120),
|
||
author: author && !/^https?:\/\//.test(author) ? clean(author, 200) : null,
|
||
publishedAt: date(
|
||
meta['article:published_time'] ??
|
||
meta['date'] ??
|
||
meta['dc.date'] ??
|
||
meta['citation_publication_date'] ??
|
||
ld.datePublished
|
||
)
|
||
};
|
||
}
|
||
|
||
/** Fetches `input` and returns its preview; falls back to just the URL when the page is unreachable. */
|
||
export async function fetchLinkMeta(input: string): Promise<LinkMeta & { fetched: boolean }> {
|
||
const empty = (url: string) => ({
|
||
url,
|
||
title: null,
|
||
description: null,
|
||
image: null,
|
||
siteName: null,
|
||
author: null,
|
||
publishedAt: null,
|
||
fetched: false
|
||
});
|
||
const start = cleanUrl(input);
|
||
if (!start) throw new Error('Not an http(s) URL');
|
||
let url = new URL(start);
|
||
const signal = AbortSignal.timeout(TIMEOUT_MS);
|
||
try {
|
||
for (let i = 0; ; i++) {
|
||
await assertPublic(url);
|
||
const res = await fetch(url, {
|
||
redirect: 'manual',
|
||
signal,
|
||
headers: { 'user-agent': UA, accept: 'text/html,application/xhtml+xml;q=0.9,*/*;q=0.5' }
|
||
});
|
||
const location = res.headers.get('location');
|
||
if (res.status >= 300 && res.status < 400 && location) {
|
||
await res.body?.cancel();
|
||
if (i >= MAX_REDIRECTS) return empty(start);
|
||
url = new URL(location, url);
|
||
if (url.protocol !== 'http:' && url.protocol !== 'https:') return empty(start);
|
||
continue;
|
||
}
|
||
if (!res.ok || !/html/i.test(res.headers.get('content-type') ?? '')) {
|
||
await res.body?.cancel();
|
||
return empty(cleanUrl(url.toString()) ?? start);
|
||
}
|
||
const reader = res.body!.getReader();
|
||
const chunks: Uint8Array[] = [];
|
||
let size = 0;
|
||
while (size < MAX_BYTES) {
|
||
const { done, value } = await reader.read();
|
||
if (done) break;
|
||
chunks.push(value);
|
||
size += value.length;
|
||
// The head is all we need.
|
||
if (new TextDecoder().decode(value).includes('</head>')) break;
|
||
}
|
||
await reader.cancel().catch(() => {});
|
||
const html = new TextDecoder().decode(Buffer.concat(chunks));
|
||
const final = cleanUrl(url.toString()) ?? start;
|
||
const meta = parseLinkMeta(html, final);
|
||
// Only trust a canonical URL on the same site (some pages point at their homepage).
|
||
const canonical = cleanUrl(meta.url);
|
||
const sameSite = canonical && new URL(canonical).hostname === new URL(final).hostname;
|
||
const keep = sameSite && new URL(canonical).pathname.length > 1 ? canonical : final;
|
||
return { ...meta, url: keep, fetched: true };
|
||
}
|
||
} catch (err) {
|
||
if (err instanceof Error && err.message === 'private host') throw err;
|
||
return empty(start);
|
||
}
|
||
}
|
||
|
||
export const siteFromUrl = (url: string) => new URL(url).hostname.replace(/^www\./, '');
|