reading: split the book tracker into books, papers and articles (kind column + migration classifying 20 papers), segmented tabs with counts, article cards that open the link, owner + add link with server-side unfurl, kind switch on detail pages, /trackers/reading alias
All checks were successful
Deploy production / deploy (push) Successful in 1m8s

This commit is contained in:
2026-10-25 13:42:51 +00:00
parent 7b5faf6e74
commit e31e1be73f
20 changed files with 27592 additions and 247 deletions

View File

@@ -1,4 +1,18 @@
// Shared (client + server) types and helpers for the book tracker.
export const bookKinds = ['book', 'paper', 'article'] as const;
export type BookKind = (typeof bookKinds)[number];
export const isBookKind = (v: unknown): v is BookKind =>
typeof v === 'string' && (bookKinds as readonly string[]).includes(v);
/** Tab labels and the `?kind=` value for each (plural, as in the URL). */
export const kindTabs: Record<BookKind, { label: string; param: string }> = {
book: { label: 'books', param: 'books' },
paper: { label: 'papers', param: 'papers' },
article: { label: 'articles', param: 'articles' }
};
export function kindFromParam(param: string | null): BookKind {
const hit = bookKinds.find((k) => kindTabs[k].param === param || k === param);
return hit ?? 'book';
}
export const readingStatuses = ['want_to_read', 'reading', 'finished', 'abandoned'] as const;
export type ReadingStatus = (typeof readingStatuses)[number];
@@ -24,6 +38,11 @@ export const formatPreference = ['epub', 'azw3', 'mobi', 'azw', 'fb2', 'pdf', 'c
/** One card in the library. Public fields only, unless noted. */
export type BookCard = {
id: number;
kind: BookKind;
url: string | null; // articles/papers on the web
site: string | null;
publishedAt: string | null;
excerpt: string | null; // articles only (short description)
title: string;
subtitle: string | null;
authors: string[];

View File

@@ -2,18 +2,18 @@ import { describe, expect, it } from 'vitest';
import { visibleTrackers } from './projects';
describe('trackers', () => {
it('lists tasks, money, books first for the owner', () => {
it('lists tasks, money, reading first for the owner', () => {
expect(
visibleTrackers(true)
.map((t) => t.name)
.slice(0, 4)
).toEqual(['tasks', 'money', 'books', 'spotting']);
).toEqual(['tasks', 'money', 'reading', 'spotting']);
});
it('hides private trackers when logged out', () => {
const names = visibleTrackers(false).map((t) => t.name);
expect(names).not.toContain('tasks');
expect(names).not.toContain('health');
expect(names.slice(0, 3)).toEqual(['money', 'books', 'spotting']);
expect(names.slice(0, 3)).toEqual(['money', 'reading', 'spotting']);
});
it('uses short single-word names', () => {
for (const t of visibleTrackers(true)) expect(t.name).toMatch(/^[a-z]+$/);

View File

@@ -92,8 +92,8 @@ const allTrackers: readonly (Tracker & { private: boolean })[] = [
private: false
},
{
name: 'books',
subtitle: 'my library: what I am reading, have read and want to read',
name: 'reading',
subtitle: 'books, scientific papers and articles I am reading, have read and want to read',
href: '/trackers/books',
external: false,
private: false

View File

@@ -42,7 +42,8 @@ const publicPages: Record<string, { title: string; description: string }> = {
},
'/trackers/books': {
title: 'Books',
description: 'Stan Runge’s library: books being read, finished and on the to-read pile.'
description:
'Stan Runge’s reading: books, scientific papers and articles being read, finished and on the to-read pile.'
},
'/trackers/spotting': {
title: 'Spotting',

View File

@@ -51,6 +51,13 @@ export async function libraryCards(owner: boolean): Promise<BookCard[]> {
const rows = await db
.select({
id: bookWorks.id,
kind: bookWorks.kind,
url: bookWorks.url,
siteName: bookWorks.siteName,
publishedAt: bookWorks.publishedAt,
excerpt: sql<
string | null
>`case when ${bookWorks.kind} = 'article' then left(${bookWorks.description}, 320) end`,
title: bookWorks.title,
subtitle: bookWorks.subtitle,
authors: bookWorks.authors,
@@ -74,6 +81,11 @@ export async function libraryCards(owner: boolean): Promise<BookCard[]> {
.orderBy(asc(bookWorks.title));
return rows.map((r) => ({
id: r.id,
kind: r.kind,
url: r.url,
site: r.siteName ?? (r.url ? new URL(r.url).hostname.replace(/^www\./, '') : null),
publishedAt: iso(r.publishedAt),
excerpt: r.excerpt,
title: r.title,
subtitle: r.subtitle,
authors: r.authors,
@@ -98,7 +110,7 @@ export async function libraryStats(): Promise<LibraryStats> {
const [[counts], [pages]] = await Promise.all([
db
.select({
total: sql<number>`(select count(*)::int from ${bookWorks})`,
total: sql<number>`(select count(*)::int from ${bookWorks} where ${bookWorks.kind} <> 'article')`,
finished: sql<number>`count(*) filter (where ${bookProgress.status} = 'finished' and ${bookProgress.finishedAt} >= ${yearStart})::int`,
reading: sql<number>`count(*) filter (where ${bookProgress.status} = 'reading')::int`
})

View File

@@ -0,0 +1,83 @@
// Reading-list links (articles, and papers added by URL). A link is a books.works row with
// kind 'article' or 'paper', its URL in `url`, the og:image in `cover_url` and the page
// description in `description`; status and notes work as for books.
import { eq } from 'drizzle-orm';
import { db } from '#lib/server/db/index.js';
import { bookProgress, bookWorks } from '#lib/server/db/schema/index.js';
import type { ReadingStatus } from '#lib/books/shared.js';
import { cleanUrl, fetchLinkMeta, kindForUrl, siteFromUrl, type LinkMeta } from './unfurl.js';
export type NewLink = {
url: string;
title?: string | null;
status?: ReadingStatus;
addedAt?: Date;
tags?: string[];
source?: string;
};
/** Builds the row for a link from its fetched preview. Pure (exported for tests). */
export function linkRow(meta: Omit<LinkMeta, 'url'> & { url: string }, input: NewLink) {
const kind = kindForUrl(meta.url);
const title = input.title?.trim() || meta.title || siteFromUrl(meta.url);
return {
kind,
title: title.slice(0, 300),
url: meta.url,
siteName: meta.siteName ?? siteFromUrl(meta.url),
authors: meta.author ? [meta.author] : [],
description: meta.description,
coverUrl: meta.image?.startsWith('https://') ? meta.image : null,
publishedAt: meta.publishedAt ? new Date(meta.publishedAt) : null,
year: meta.publishedAt ? new Date(meta.publishedAt).getUTCFullYear() : null,
tags: input.tags ?? [],
metadataStatus: (meta.title ? 'matched' : 'not_found') as 'matched' | 'not_found',
metadataSource: 'web',
metadataFetchedAt: new Date(),
sourceRefs: { origins: [input.source ?? 'link'] }
};
}
/**
* Adds a link to the reading list (fetching its title, description and image), or returns the
* existing work when the URL is already there. `created` tells the two apart.
*/
export async function addLink(input: NewLink): Promise<{ id: number; created: boolean }> {
const url = cleanUrl(input.url);
if (!url) throw new Error('Not an http(s) URL');
const existing = await findByUrl(url);
if (existing) return { id: existing, created: false };
const meta = await fetchLinkMeta(url);
const again = meta.url !== url ? await findByUrl(meta.url) : null;
if (again) return { id: again, created: false };
const row = linkRow(meta, input);
return db.transaction(async (tx) => {
const [work] = await tx
.insert(bookWorks)
.values({ ...row, ...(input.addedAt ? { createdAt: input.addedAt } : {}) })
.onConflictDoNothing()
.returning({ id: bookWorks.id });
if (!work) return { id: (await findByUrl(row.url))!, created: false };
await tx.insert(bookProgress).values({
workId: work.id,
status: input.status ?? 'want_to_read',
startedAt: input.status === 'reading' || input.status === 'finished' ? new Date() : null,
finishedAt: input.status === 'finished' ? new Date() : null,
percent: input.status === 'finished' ? 100 : 0
});
return { id: work.id, created: true };
});
}
async function findByUrl(url: string) {
const [row] = await db
.select({ id: bookWorks.id })
.from(bookWorks)
.where(eq(bookWorks.url, url))
.limit(1);
return row?.id ?? null;
}
export async function setKind(workId: number, kind: 'book' | 'paper' | 'article') {
await db.update(bookWorks).set({ kind }).where(eq(bookWorks.id, workId));
}

View File

@@ -0,0 +1,75 @@
import { describe, expect, it } from 'vitest';
import { cleanUrl, decodeEntities, isPrivateAddress, kindForUrl, parseLinkMeta } from './unfurl';
describe('cleanUrl', () => {
it('drops tracking params and rejects non-http', () => {
expect(cleanUrl('https://github.com/x/y?utm_source=tldrnewsletter')).toBe(
'https://github.com/x/y'
);
expect(cleanUrl('javascript:alert(1)')).toBeNull();
expect(cleanUrl('https://user:pw@example.com/')).toBeNull();
expect(cleanUrl('not a url')).toBeNull();
});
});
describe('kindForUrl', () => {
it('treats scholarly links as papers', () => {
expect(kindForUrl('https://arxiv.org/abs/1706.03762')).toBe('paper');
expect(kindForUrl('https://doi.org/10.1145/1')).toBe('paper');
expect(kindForUrl('https://mll.sh/humiliating-iis-servers-for-fun-and-jail-time/')).toBe(
'article'
);
});
});
describe('isPrivateAddress', () => {
it('blocks loopback, private and link-local ranges', () => {
for (const ip of [
'127.0.0.1',
'10.1.2.3',
'172.20.0.1',
'192.168.1.1',
'169.254.169.254',
'::1',
'fd00::1',
'::ffff:127.0.0.1'
])
expect(isPrivateAddress(ip), ip).toBe(true);
for (const ip of ['1.1.1.1', '37.187.157.191', '2606:4700::1111'])
expect(isPrivateAddress(ip), ip).toBe(false);
});
});
describe('parseLinkMeta', () => {
const html = `<!doctype html><html><head>
<title>Fallback title</title>
<meta property="og:title" content="Humiliating IIS servers &amp; jail time">
<meta name="description" content=" A story
about servers. ">
<meta property="og:image" content="/img/cover.png">
<meta property="og:site_name" content="mll.sh">
<meta property="article:published_time" content="2026-10-01T10:00:00Z">
<link rel="canonical" href="https://mll.sh/post/">
<script type="application/ld+json">{"@type":"BlogPosting","author":{"name":"M L"}}</script>
</head><body></body></html>`;
it('reads og, meta, canonical and JSON-LD', () => {
expect(parseLinkMeta(html, 'https://mll.sh/post/?x=1')).toEqual({
url: 'https://mll.sh/post/',
title: 'Humiliating IIS servers & jail time',
description: 'A story about servers.',
image: 'https://mll.sh/img/cover.png',
siteName: 'mll.sh',
author: 'M L',
publishedAt: '2026-10-01T10:00:00.000Z'
});
});
it('falls back to the <title> and handles pages without metadata', () => {
const meta = parseLinkMeta('<title>Just a title</title>', 'https://example.com/a');
expect(meta.title).toBe('Just a title');
expect(meta.image).toBeNull();
expect(meta.url).toBe('https://example.com/a');
});
it('decodes numeric entities', () => {
expect(decodeEntities('It&#39;s &#x2014; ok')).toBe("It's — ok");
});
});

View File

@@ -0,0 +1,259 @@
// Server-side link preview for the reading list: fetches a public web page and pulls the
// title, description, image, site, author and publish date from its <meta>/JSON-LD tags.
// Only http(s) to public addresses (no localhost / private ranges, checked again on every
// redirect), a short timeout and a size cap, so the owner's "+ add link" can't be pointed
// at internal services.
import { lookup } from 'node:dns/promises';
import { isIP } from 'node:net';
export type LinkMeta = {
url: string;
title: string | null;
description: string | null;
image: string | null;
siteName: string | null;
author: string | null;
publishedAt: string | null;
};
const TIMEOUT_MS = 8000;
const MAX_BYTES = 1_500_000;
const MAX_REDIRECTS = 5;
const UA = 'Mozilla/5.0 (compatible; stanrunge.dev reading list; +https://stanrunge.dev)';
/** Strips tracking parameters and the fragment-less noise, keeps the rest of the URL. */
export function cleanUrl(input: string): string | null {
let url: URL;
try {
url = new URL(input.trim());
} catch {
return null;
}
if (url.protocol !== 'http:' && url.protocol !== 'https:') return null;
if (url.username || url.password) return null;
for (const key of [...url.searchParams.keys()])
if (/^(utm_|fbclid$|gclid$|mc_|ref_src$)/i.test(key)) url.searchParams.delete(key);
return url.toString();
}
/** arXiv, DOI and similar scholarly hosts are papers rather than articles. */
export function kindForUrl(url: string): 'paper' | 'article' {
const host = new URL(url).hostname.replace(/^www\./, '');
return /(^|\.)(arxiv\.org|doi\.org|openreview\.net|aclanthology\.org|proceedings\.neurips\.cc|proceedings\.mlr\.press|semanticscholar\.org|biorxiv\.org|ssrn\.com)$/.test(
host
) || /\.pdf$/i.test(new URL(url).pathname)
? 'paper'
: 'article';
}
export function isPrivateAddress(ip: string): boolean {
if (isIP(ip) === 4) {
const [a, b] = ip.split('.').map(Number);
return (
a === 0 ||
a === 10 ||
a === 127 ||
(a === 100 && b >= 64 && b <= 127) ||
(a === 169 && b === 254) ||
(a === 172 && b >= 16 && b <= 31) ||
(a === 192 && b === 168) ||
(a === 198 && (b === 18 || b === 19)) ||
a >= 224
);
}
const v6 = ip.toLowerCase();
if (v6.startsWith('::ffff:')) return isPrivateAddress(v6.slice(7));
return v6 === '::' || v6 === '::1' || /^(fc|fd|fe[89ab])/.test(v6);
}
async function assertPublic(url: URL) {
const host = url.hostname.replace(/^\[|\]$/g, '');
if (host === 'localhost' || host.endsWith('.localhost') || host.endsWith('.internal'))
throw new Error('private host');
const addresses = isIP(host) ? [{ address: host }] : await lookup(host, { all: true });
if (!addresses.length || addresses.some((a) => isPrivateAddress(a.address)))
throw new Error('private host');
}
const entities: Record<string, string> = {
amp: '&',
lt: '<',
gt: '>',
quot: '"',
apos: "'",
nbsp: ' ',
ndash: '–',
mdash: '—',
hellip: '…',
rsquo: '’',
lsquo: '‘',
rdquo: '”',
ldquo: '“'
};
export function decodeEntities(s: string): string {
return s.replace(/&(#x[0-9a-f]+|#\d+|[a-z]+);/gi, (m, e: string) => {
if (e[0] === '#') {
const code = e[1] === 'x' || e[1] === 'X' ? parseInt(e.slice(2), 16) : Number(e.slice(1));
return Number.isFinite(code) && code > 0 && code < 0x110000 ? String.fromCodePoint(code) : m;
}
return entities[e.toLowerCase()] ?? m;
});
}
const clean = (s: string | null | undefined, max = 500) => {
const v = s ? decodeEntities(s).replace(/\s+/g, ' ').trim() : '';
return v ? (v.length > max ? `${v.slice(0, max - 1).trimEnd()}…` : v) : null;
};
function attrs(tag: string): Record<string, string> {
const out: Record<string, string> = {};
for (const m of tag.matchAll(/([a-zA-Z:_-]+)\s*=\s*("([^"]*)"|'([^']*)'|([^\s>]+))/g))
out[m[1].toLowerCase()] = m[3] ?? m[4] ?? m[5] ?? '';
return out;
}
/** Parses the <head> of an HTML page. Pure; `base` resolves relative image URLs. */
export function parseLinkMeta(html: string, base: string): LinkMeta {
const head = html.slice(0, 400_000);
const meta: Record<string, string> = {};
for (const m of head.matchAll(/<meta\b[^>]*>/gi)) {
const a = attrs(m[0]);
const key = (a.property ?? a.name ?? a.itemprop ?? '').toLowerCase();
if (key && a.content != null && !(key in meta)) meta[key] = a.content;
}
let canonical: string | null = null;
for (const m of head.matchAll(/<link\b[^>]*>/gi)) {
const a = attrs(m[0]);
if (a.rel?.toLowerCase().split(/\s+/).includes('canonical') && a.href) canonical = a.href;
}
// JSON-LD (Article / BlogPosting): author and dates when the meta tags lack them.
let ld: { author?: unknown; datePublished?: string; headline?: string } = {};
for (const m of head.matchAll(
/<script[^>]+type=["']application\/ld\+json["'][^>]*>([\s\S]*?)<\/script>/gi
)) {
try {
const parsed = JSON.parse(m[1]);
const nodes = [
parsed,
...(Array.isArray(parsed) ? parsed : []),
...(parsed?.['@graph'] ?? [])
];
const article = nodes.find(
(n) => n && typeof n === 'object' && /Article|BlogPosting|Report/.test(String(n['@type']))
);
if (article) {
ld = article;
break;
}
} catch {
// Broken JSON-LD is common; ignore it.
}
}
const ldAuthor = (() => {
const a = ld.author;
const list = Array.isArray(a) ? a : a ? [a] : [];
const names = list
.map((x) => (typeof x === 'string' ? x : (x as { name?: string })?.name))
.filter(Boolean);
return names.length ? names.join(', ') : null;
})();
const titleTag = /<title[^>]*>([\s\S]*?)<\/title>/i.exec(head)?.[1];
const abs = (u: string | undefined) => {
if (!u) return null;
try {
const url = new URL(decodeEntities(u.trim()), base);
return url.protocol === 'https:' || url.protocol === 'http:' ? url.toString() : null;
} catch {
return null;
}
};
const date = (s: string | undefined) => {
if (!s) return null;
const d = new Date(s);
return Number.isNaN(d.getTime()) ? null : d.toISOString();
};
const author =
meta['author'] ?? meta['article:author'] ?? meta['twitter:creator'] ?? ldAuthor ?? undefined;
return {
url: abs(canonical ?? meta['og:url']) ?? base,
title: clean(meta['og:title'] ?? meta['twitter:title'] ?? ld.headline ?? titleTag, 300),
description: clean(
meta['og:description'] ?? meta['description'] ?? meta['twitter:description'],
600
),
image: abs(meta['og:image'] ?? meta['og:image:url'] ?? meta['twitter:image']),
siteName: clean(meta['og:site_name'] ?? meta['application-name'], 120),
author: author && !/^https?:\/\//.test(author) ? clean(author, 200) : null,
publishedAt: date(
meta['article:published_time'] ??
meta['date'] ??
meta['dc.date'] ??
meta['citation_publication_date'] ??
ld.datePublished
)
};
}
/** Fetches `input` and returns its preview; falls back to just the URL when the page is unreachable. */
export async function fetchLinkMeta(input: string): Promise<LinkMeta & { fetched: boolean }> {
const empty = (url: string) => ({
url,
title: null,
description: null,
image: null,
siteName: null,
author: null,
publishedAt: null,
fetched: false
});
const start = cleanUrl(input);
if (!start) throw new Error('Not an http(s) URL');
let url = new URL(start);
const signal = AbortSignal.timeout(TIMEOUT_MS);
try {
for (let i = 0; ; i++) {
await assertPublic(url);
const res = await fetch(url, {
redirect: 'manual',
signal,
headers: { 'user-agent': UA, accept: 'text/html,application/xhtml+xml;q=0.9,*/*;q=0.5' }
});
const location = res.headers.get('location');
if (res.status >= 300 && res.status < 400 && location) {
await res.body?.cancel();
if (i >= MAX_REDIRECTS) return empty(start);
url = new URL(location, url);
if (url.protocol !== 'http:' && url.protocol !== 'https:') return empty(start);
continue;
}
if (!res.ok || !/html/i.test(res.headers.get('content-type') ?? '')) {
await res.body?.cancel();
return empty(cleanUrl(url.toString()) ?? start);
}
const reader = res.body!.getReader();
const chunks: Uint8Array[] = [];
let size = 0;
while (size < MAX_BYTES) {
const { done, value } = await reader.read();
if (done) break;
chunks.push(value);
size += value.length;
// The head is all we need.
if (new TextDecoder().decode(value).includes('</head>')) break;
}
await reader.cancel().catch(() => {});
const html = new TextDecoder().decode(Buffer.concat(chunks));
const final = cleanUrl(url.toString()) ?? start;
const meta = parseLinkMeta(html, final);
// Only trust a canonical URL on the same site (some pages point at their homepage).
const canonical = cleanUrl(meta.url);
const sameSite = canonical && new URL(canonical).hostname === new URL(final).hostname;
const keep = sameSite && new URL(canonical).pathname.length > 1 ? canonical : final;
return { ...meta, url: keep, fetched: true };
}
} catch (err) {
if (err instanceof Error && err.message === 'private host') throw err;
return empty(start);
}
}
export const siteFromUrl = (url: string) => new URL(url).hostname.replace(/^www\./, '');

View File

@@ -34,10 +34,15 @@ export const bookReadingStatus = booksSchema.enum('reading_status', [
'abandoned'
]);
// What a work is: a book, a scientific paper, or a web article / link on the reading list.
// Articles have no files; they live at `url` and are opened externally.
export const bookKind = booksSchema.enum('kind', ['book', 'paper', 'article']);
export const bookWorks = booksSchema.table(
'works',
{
id: id(),
kind: bookKind('kind').notNull().default('book'),
title: text('title').notNull(),
subtitle: text('subtitle'),
authors: text('authors').array().notNull().default([]),
@@ -61,9 +66,18 @@ export const bookWorks = booksSchema.table(
metadataSource: text('metadata_source'), // 'openlibrary' | 'googlebooks' | 'libgen' | 'manual'
metadataFetchedAt: timestamptz('metadata_fetched_at'),
notes: text('notes'),
// Articles and papers on the web: canonical link, site name and publish date. The excerpt
// goes in `description` and the og:image in `cover_url`.
url: text('url'),
siteName: text('site_name'),
publishedAt: timestamptz('published_at'),
...timestamps
},
(t) => [
uniqueIndex('works_url_key')
.on(t.url)
.where(sql`${t.url} is not null`),
index('works_kind_idx').on(t.kind),
uniqueIndex('works_isbn13_key')
.on(t.isbn13)
.where(sql`${t.isbn13} is not null`),
@@ -206,4 +220,5 @@ export type BookShelf = typeof bookShelves.$inferSelect;
export type BookProgress = typeof bookProgress.$inferSelect;
export type BookNote = typeof bookNotes.$inferSelect;
export type BookSession = typeof bookSessions.$inferSelect;
export type BookKind = (typeof bookKind.enumValues)[number];
export type BookReadingStatus = (typeof bookReadingStatus.enumValues)[number];

View File

@@ -18,7 +18,7 @@ export function getTitle(path: string, environment: SiteEnvironment = null): str
'/': 'Home',
'/projects': 'Projects',
'/trackers': 'Trackers',
'/trackers/books': 'Books',
'/trackers/books': 'Reading',
'/trackers/spotting': 'Spotting',
'/trackers/cars': 'Cars',
'/trackers/planes': 'Planes',

View File

@@ -10,11 +10,11 @@ describe('tracker icons', () => {
expect(new Set(colours).size).toBe(colours.length);
});
it('keeps tasks, money and books first', () => {
it('keeps tasks, money and reading first', () => {
expect(
visibleTrackers(true)
.slice(0, 3)
.map((t) => t.name)
).toEqual(['tasks', 'money', 'books']);
).toEqual(['tasks', 'money', 'reading']);
});
});

View File

@@ -30,7 +30,7 @@ export const trackerIcons: Record<string, TrackerIcon> = {
ink: WHITE,
paths: [c(12, 12, 9), 'M15.5 8.6a4 4 0 1 0 0 6.8', 'M7 10.8h6M7 13.2h6']
},
books: {
reading: {
from: '#2dd4bf',
to: '#0d9488',
ink: WHITE,