reading: split the book tracker into books, papers and articles (kind column + migration classifying 20 papers), segmented tabs with counts, article cards that open the link, owner + add link with server-side unfurl, kind switch on detail pages, /trackers/reading alias
All checks were successful
Deploy production / deploy (push) Successful in 1m8s
All checks were successful
Deploy production / deploy (push) Successful in 1m8s
This commit is contained in:
@@ -1,4 +1,18 @@
|
||||
// Shared (client + server) types and helpers for the book tracker.
|
||||
export const bookKinds = ['book', 'paper', 'article'] as const;
|
||||
export type BookKind = (typeof bookKinds)[number];
|
||||
export const isBookKind = (v: unknown): v is BookKind =>
|
||||
typeof v === 'string' && (bookKinds as readonly string[]).includes(v);
|
||||
/** Tab labels and the `?kind=` value for each (plural, as in the URL). */
|
||||
export const kindTabs: Record<BookKind, { label: string; param: string }> = {
|
||||
book: { label: 'books', param: 'books' },
|
||||
paper: { label: 'papers', param: 'papers' },
|
||||
article: { label: 'articles', param: 'articles' }
|
||||
};
|
||||
export function kindFromParam(param: string | null): BookKind {
|
||||
const hit = bookKinds.find((k) => kindTabs[k].param === param || k === param);
|
||||
return hit ?? 'book';
|
||||
}
|
||||
export const readingStatuses = ['want_to_read', 'reading', 'finished', 'abandoned'] as const;
|
||||
export type ReadingStatus = (typeof readingStatuses)[number];
|
||||
|
||||
@@ -24,6 +38,11 @@ export const formatPreference = ['epub', 'azw3', 'mobi', 'azw', 'fb2', 'pdf', 'c
|
||||
/** One card in the library. Public fields only, unless noted. */
|
||||
export type BookCard = {
|
||||
id: number;
|
||||
kind: BookKind;
|
||||
url: string | null; // articles/papers on the web
|
||||
site: string | null;
|
||||
publishedAt: string | null;
|
||||
excerpt: string | null; // articles only (short description)
|
||||
title: string;
|
||||
subtitle: string | null;
|
||||
authors: string[];
|
||||
|
||||
@@ -2,18 +2,18 @@ import { describe, expect, it } from 'vitest';
|
||||
import { visibleTrackers } from './projects';
|
||||
|
||||
describe('trackers', () => {
|
||||
it('lists tasks, money, books first for the owner', () => {
|
||||
it('lists tasks, money, reading first for the owner', () => {
|
||||
expect(
|
||||
visibleTrackers(true)
|
||||
.map((t) => t.name)
|
||||
.slice(0, 4)
|
||||
).toEqual(['tasks', 'money', 'books', 'spotting']);
|
||||
).toEqual(['tasks', 'money', 'reading', 'spotting']);
|
||||
});
|
||||
it('hides private trackers when logged out', () => {
|
||||
const names = visibleTrackers(false).map((t) => t.name);
|
||||
expect(names).not.toContain('tasks');
|
||||
expect(names).not.toContain('health');
|
||||
expect(names.slice(0, 3)).toEqual(['money', 'books', 'spotting']);
|
||||
expect(names.slice(0, 3)).toEqual(['money', 'reading', 'spotting']);
|
||||
});
|
||||
it('uses short single-word names', () => {
|
||||
for (const t of visibleTrackers(true)) expect(t.name).toMatch(/^[a-z]+$/);
|
||||
|
||||
@@ -92,8 +92,8 @@ const allTrackers: readonly (Tracker & { private: boolean })[] = [
|
||||
private: false
|
||||
},
|
||||
{
|
||||
name: 'books',
|
||||
subtitle: 'my library: what I am reading, have read and want to read',
|
||||
name: 'reading',
|
||||
subtitle: 'books, scientific papers and articles I am reading, have read and want to read',
|
||||
href: '/trackers/books',
|
||||
external: false,
|
||||
private: false
|
||||
|
||||
@@ -42,7 +42,8 @@ const publicPages: Record<string, { title: string; description: string }> = {
|
||||
},
|
||||
'/trackers/books': {
|
||||
title: 'Books',
|
||||
description: 'Stan Runge’s library: books being read, finished and on the to-read pile.'
|
||||
description:
|
||||
'Stan Runge’s reading: books, scientific papers and articles being read, finished and on the to-read pile.'
|
||||
},
|
||||
'/trackers/spotting': {
|
||||
title: 'Spotting',
|
||||
|
||||
@@ -51,6 +51,13 @@ export async function libraryCards(owner: boolean): Promise<BookCard[]> {
|
||||
const rows = await db
|
||||
.select({
|
||||
id: bookWorks.id,
|
||||
kind: bookWorks.kind,
|
||||
url: bookWorks.url,
|
||||
siteName: bookWorks.siteName,
|
||||
publishedAt: bookWorks.publishedAt,
|
||||
excerpt: sql<
|
||||
string | null
|
||||
>`case when ${bookWorks.kind} = 'article' then left(${bookWorks.description}, 320) end`,
|
||||
title: bookWorks.title,
|
||||
subtitle: bookWorks.subtitle,
|
||||
authors: bookWorks.authors,
|
||||
@@ -74,6 +81,11 @@ export async function libraryCards(owner: boolean): Promise<BookCard[]> {
|
||||
.orderBy(asc(bookWorks.title));
|
||||
return rows.map((r) => ({
|
||||
id: r.id,
|
||||
kind: r.kind,
|
||||
url: r.url,
|
||||
site: r.siteName ?? (r.url ? new URL(r.url).hostname.replace(/^www\./, '') : null),
|
||||
publishedAt: iso(r.publishedAt),
|
||||
excerpt: r.excerpt,
|
||||
title: r.title,
|
||||
subtitle: r.subtitle,
|
||||
authors: r.authors,
|
||||
@@ -98,7 +110,7 @@ export async function libraryStats(): Promise<LibraryStats> {
|
||||
const [[counts], [pages]] = await Promise.all([
|
||||
db
|
||||
.select({
|
||||
total: sql<number>`(select count(*)::int from ${bookWorks})`,
|
||||
total: sql<number>`(select count(*)::int from ${bookWorks} where ${bookWorks.kind} <> 'article')`,
|
||||
finished: sql<number>`count(*) filter (where ${bookProgress.status} = 'finished' and ${bookProgress.finishedAt} >= ${yearStart})::int`,
|
||||
reading: sql<number>`count(*) filter (where ${bookProgress.status} = 'reading')::int`
|
||||
})
|
||||
|
||||
83
src/lib/server/books/links.ts
Normal file
83
src/lib/server/books/links.ts
Normal file
@@ -0,0 +1,83 @@
|
||||
// Reading-list links (articles, and papers added by URL). A link is a books.works row with
|
||||
// kind 'article' or 'paper', its URL in `url`, the og:image in `cover_url` and the page
|
||||
// description in `description`; status and notes work as for books.
|
||||
import { eq } from 'drizzle-orm';
|
||||
import { db } from '#lib/server/db/index.js';
|
||||
import { bookProgress, bookWorks } from '#lib/server/db/schema/index.js';
|
||||
import type { ReadingStatus } from '#lib/books/shared.js';
|
||||
import { cleanUrl, fetchLinkMeta, kindForUrl, siteFromUrl, type LinkMeta } from './unfurl.js';
|
||||
|
||||
export type NewLink = {
|
||||
url: string;
|
||||
title?: string | null;
|
||||
status?: ReadingStatus;
|
||||
addedAt?: Date;
|
||||
tags?: string[];
|
||||
source?: string;
|
||||
};
|
||||
|
||||
/** Builds the row for a link from its fetched preview. Pure (exported for tests). */
|
||||
export function linkRow(meta: Omit<LinkMeta, 'url'> & { url: string }, input: NewLink) {
|
||||
const kind = kindForUrl(meta.url);
|
||||
const title = input.title?.trim() || meta.title || siteFromUrl(meta.url);
|
||||
return {
|
||||
kind,
|
||||
title: title.slice(0, 300),
|
||||
url: meta.url,
|
||||
siteName: meta.siteName ?? siteFromUrl(meta.url),
|
||||
authors: meta.author ? [meta.author] : [],
|
||||
description: meta.description,
|
||||
coverUrl: meta.image?.startsWith('https://') ? meta.image : null,
|
||||
publishedAt: meta.publishedAt ? new Date(meta.publishedAt) : null,
|
||||
year: meta.publishedAt ? new Date(meta.publishedAt).getUTCFullYear() : null,
|
||||
tags: input.tags ?? [],
|
||||
metadataStatus: (meta.title ? 'matched' : 'not_found') as 'matched' | 'not_found',
|
||||
metadataSource: 'web',
|
||||
metadataFetchedAt: new Date(),
|
||||
sourceRefs: { origins: [input.source ?? 'link'] }
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Adds a link to the reading list (fetching its title, description and image), or returns the
|
||||
* existing work when the URL is already there. `created` tells the two apart.
|
||||
*/
|
||||
export async function addLink(input: NewLink): Promise<{ id: number; created: boolean }> {
|
||||
const url = cleanUrl(input.url);
|
||||
if (!url) throw new Error('Not an http(s) URL');
|
||||
const existing = await findByUrl(url);
|
||||
if (existing) return { id: existing, created: false };
|
||||
const meta = await fetchLinkMeta(url);
|
||||
const again = meta.url !== url ? await findByUrl(meta.url) : null;
|
||||
if (again) return { id: again, created: false };
|
||||
const row = linkRow(meta, input);
|
||||
return db.transaction(async (tx) => {
|
||||
const [work] = await tx
|
||||
.insert(bookWorks)
|
||||
.values({ ...row, ...(input.addedAt ? { createdAt: input.addedAt } : {}) })
|
||||
.onConflictDoNothing()
|
||||
.returning({ id: bookWorks.id });
|
||||
if (!work) return { id: (await findByUrl(row.url))!, created: false };
|
||||
await tx.insert(bookProgress).values({
|
||||
workId: work.id,
|
||||
status: input.status ?? 'want_to_read',
|
||||
startedAt: input.status === 'reading' || input.status === 'finished' ? new Date() : null,
|
||||
finishedAt: input.status === 'finished' ? new Date() : null,
|
||||
percent: input.status === 'finished' ? 100 : 0
|
||||
});
|
||||
return { id: work.id, created: true };
|
||||
});
|
||||
}
|
||||
|
||||
async function findByUrl(url: string) {
|
||||
const [row] = await db
|
||||
.select({ id: bookWorks.id })
|
||||
.from(bookWorks)
|
||||
.where(eq(bookWorks.url, url))
|
||||
.limit(1);
|
||||
return row?.id ?? null;
|
||||
}
|
||||
|
||||
export async function setKind(workId: number, kind: 'book' | 'paper' | 'article') {
|
||||
await db.update(bookWorks).set({ kind }).where(eq(bookWorks.id, workId));
|
||||
}
|
||||
75
src/lib/server/books/unfurl.test.ts
Normal file
75
src/lib/server/books/unfurl.test.ts
Normal file
@@ -0,0 +1,75 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { cleanUrl, decodeEntities, isPrivateAddress, kindForUrl, parseLinkMeta } from './unfurl';
|
||||
|
||||
describe('cleanUrl', () => {
|
||||
it('drops tracking params and rejects non-http', () => {
|
||||
expect(cleanUrl('https://github.com/x/y?utm_source=tldrnewsletter')).toBe(
|
||||
'https://github.com/x/y'
|
||||
);
|
||||
expect(cleanUrl('javascript:alert(1)')).toBeNull();
|
||||
expect(cleanUrl('https://user:pw@example.com/')).toBeNull();
|
||||
expect(cleanUrl('not a url')).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe('kindForUrl', () => {
|
||||
it('treats scholarly links as papers', () => {
|
||||
expect(kindForUrl('https://arxiv.org/abs/1706.03762')).toBe('paper');
|
||||
expect(kindForUrl('https://doi.org/10.1145/1')).toBe('paper');
|
||||
expect(kindForUrl('https://mll.sh/humiliating-iis-servers-for-fun-and-jail-time/')).toBe(
|
||||
'article'
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('isPrivateAddress', () => {
|
||||
it('blocks loopback, private and link-local ranges', () => {
|
||||
for (const ip of [
|
||||
'127.0.0.1',
|
||||
'10.1.2.3',
|
||||
'172.20.0.1',
|
||||
'192.168.1.1',
|
||||
'169.254.169.254',
|
||||
'::1',
|
||||
'fd00::1',
|
||||
'::ffff:127.0.0.1'
|
||||
])
|
||||
expect(isPrivateAddress(ip), ip).toBe(true);
|
||||
for (const ip of ['1.1.1.1', '37.187.157.191', '2606:4700::1111'])
|
||||
expect(isPrivateAddress(ip), ip).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('parseLinkMeta', () => {
|
||||
const html = `<!doctype html><html><head>
|
||||
<title>Fallback title</title>
|
||||
<meta property="og:title" content="Humiliating IIS servers & jail time">
|
||||
<meta name="description" content=" A story
|
||||
about servers. ">
|
||||
<meta property="og:image" content="/img/cover.png">
|
||||
<meta property="og:site_name" content="mll.sh">
|
||||
<meta property="article:published_time" content="2026-10-01T10:00:00Z">
|
||||
<link rel="canonical" href="https://mll.sh/post/">
|
||||
<script type="application/ld+json">{"@type":"BlogPosting","author":{"name":"M L"}}</script>
|
||||
</head><body></body></html>`;
|
||||
it('reads og, meta, canonical and JSON-LD', () => {
|
||||
expect(parseLinkMeta(html, 'https://mll.sh/post/?x=1')).toEqual({
|
||||
url: 'https://mll.sh/post/',
|
||||
title: 'Humiliating IIS servers & jail time',
|
||||
description: 'A story about servers.',
|
||||
image: 'https://mll.sh/img/cover.png',
|
||||
siteName: 'mll.sh',
|
||||
author: 'M L',
|
||||
publishedAt: '2026-10-01T10:00:00.000Z'
|
||||
});
|
||||
});
|
||||
it('falls back to the <title> and handles pages without metadata', () => {
|
||||
const meta = parseLinkMeta('<title>Just a title</title>', 'https://example.com/a');
|
||||
expect(meta.title).toBe('Just a title');
|
||||
expect(meta.image).toBeNull();
|
||||
expect(meta.url).toBe('https://example.com/a');
|
||||
});
|
||||
it('decodes numeric entities', () => {
|
||||
expect(decodeEntities('It's — ok')).toBe("It's — ok");
|
||||
});
|
||||
});
|
||||
259
src/lib/server/books/unfurl.ts
Normal file
259
src/lib/server/books/unfurl.ts
Normal file
@@ -0,0 +1,259 @@
|
||||
// Server-side link preview for the reading list: fetches a public web page and pulls the
|
||||
// title, description, image, site, author and publish date from its <meta>/JSON-LD tags.
|
||||
// Only http(s) to public addresses (no localhost / private ranges, checked again on every
|
||||
// redirect), a short timeout and a size cap, so the owner's "+ add link" can't be pointed
|
||||
// at internal services.
|
||||
import { lookup } from 'node:dns/promises';
|
||||
import { isIP } from 'node:net';
|
||||
|
||||
export type LinkMeta = {
|
||||
url: string;
|
||||
title: string | null;
|
||||
description: string | null;
|
||||
image: string | null;
|
||||
siteName: string | null;
|
||||
author: string | null;
|
||||
publishedAt: string | null;
|
||||
};
|
||||
|
||||
const TIMEOUT_MS = 8000;
|
||||
const MAX_BYTES = 1_500_000;
|
||||
const MAX_REDIRECTS = 5;
|
||||
const UA = 'Mozilla/5.0 (compatible; stanrunge.dev reading list; +https://stanrunge.dev)';
|
||||
|
||||
/** Strips tracking parameters and the fragment-less noise, keeps the rest of the URL. */
|
||||
export function cleanUrl(input: string): string | null {
|
||||
let url: URL;
|
||||
try {
|
||||
url = new URL(input.trim());
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
if (url.protocol !== 'http:' && url.protocol !== 'https:') return null;
|
||||
if (url.username || url.password) return null;
|
||||
for (const key of [...url.searchParams.keys()])
|
||||
if (/^(utm_|fbclid$|gclid$|mc_|ref_src$)/i.test(key)) url.searchParams.delete(key);
|
||||
return url.toString();
|
||||
}
|
||||
|
||||
/** arXiv, DOI and similar scholarly hosts are papers rather than articles. */
|
||||
export function kindForUrl(url: string): 'paper' | 'article' {
|
||||
const host = new URL(url).hostname.replace(/^www\./, '');
|
||||
return /(^|\.)(arxiv\.org|doi\.org|openreview\.net|aclanthology\.org|proceedings\.neurips\.cc|proceedings\.mlr\.press|semanticscholar\.org|biorxiv\.org|ssrn\.com)$/.test(
|
||||
host
|
||||
) || /\.pdf$/i.test(new URL(url).pathname)
|
||||
? 'paper'
|
||||
: 'article';
|
||||
}
|
||||
|
||||
export function isPrivateAddress(ip: string): boolean {
|
||||
if (isIP(ip) === 4) {
|
||||
const [a, b] = ip.split('.').map(Number);
|
||||
return (
|
||||
a === 0 ||
|
||||
a === 10 ||
|
||||
a === 127 ||
|
||||
(a === 100 && b >= 64 && b <= 127) ||
|
||||
(a === 169 && b === 254) ||
|
||||
(a === 172 && b >= 16 && b <= 31) ||
|
||||
(a === 192 && b === 168) ||
|
||||
(a === 198 && (b === 18 || b === 19)) ||
|
||||
a >= 224
|
||||
);
|
||||
}
|
||||
const v6 = ip.toLowerCase();
|
||||
if (v6.startsWith('::ffff:')) return isPrivateAddress(v6.slice(7));
|
||||
return v6 === '::' || v6 === '::1' || /^(fc|fd|fe[89ab])/.test(v6);
|
||||
}
|
||||
|
||||
async function assertPublic(url: URL) {
|
||||
const host = url.hostname.replace(/^\[|\]$/g, '');
|
||||
if (host === 'localhost' || host.endsWith('.localhost') || host.endsWith('.internal'))
|
||||
throw new Error('private host');
|
||||
const addresses = isIP(host) ? [{ address: host }] : await lookup(host, { all: true });
|
||||
if (!addresses.length || addresses.some((a) => isPrivateAddress(a.address)))
|
||||
throw new Error('private host');
|
||||
}
|
||||
|
||||
const entities: Record<string, string> = {
|
||||
amp: '&',
|
||||
lt: '<',
|
||||
gt: '>',
|
||||
quot: '"',
|
||||
apos: "'",
|
||||
nbsp: ' ',
|
||||
ndash: '–',
|
||||
mdash: '—',
|
||||
hellip: '…',
|
||||
rsquo: '’',
|
||||
lsquo: '‘',
|
||||
rdquo: '”',
|
||||
ldquo: '“'
|
||||
};
|
||||
export function decodeEntities(s: string): string {
|
||||
return s.replace(/&(#x[0-9a-f]+|#\d+|[a-z]+);/gi, (m, e: string) => {
|
||||
if (e[0] === '#') {
|
||||
const code = e[1] === 'x' || e[1] === 'X' ? parseInt(e.slice(2), 16) : Number(e.slice(1));
|
||||
return Number.isFinite(code) && code > 0 && code < 0x110000 ? String.fromCodePoint(code) : m;
|
||||
}
|
||||
return entities[e.toLowerCase()] ?? m;
|
||||
});
|
||||
}
|
||||
const clean = (s: string | null | undefined, max = 500) => {
|
||||
const v = s ? decodeEntities(s).replace(/\s+/g, ' ').trim() : '';
|
||||
return v ? (v.length > max ? `${v.slice(0, max - 1).trimEnd()}…` : v) : null;
|
||||
};
|
||||
|
||||
function attrs(tag: string): Record<string, string> {
|
||||
const out: Record<string, string> = {};
|
||||
for (const m of tag.matchAll(/([a-zA-Z:_-]+)\s*=\s*("([^"]*)"|'([^']*)'|([^\s>]+))/g))
|
||||
out[m[1].toLowerCase()] = m[3] ?? m[4] ?? m[5] ?? '';
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Parses the <head> of an HTML page. Pure; `base` resolves relative image URLs. */
|
||||
export function parseLinkMeta(html: string, base: string): LinkMeta {
|
||||
const head = html.slice(0, 400_000);
|
||||
const meta: Record<string, string> = {};
|
||||
for (const m of head.matchAll(/<meta\b[^>]*>/gi)) {
|
||||
const a = attrs(m[0]);
|
||||
const key = (a.property ?? a.name ?? a.itemprop ?? '').toLowerCase();
|
||||
if (key && a.content != null && !(key in meta)) meta[key] = a.content;
|
||||
}
|
||||
let canonical: string | null = null;
|
||||
for (const m of head.matchAll(/<link\b[^>]*>/gi)) {
|
||||
const a = attrs(m[0]);
|
||||
if (a.rel?.toLowerCase().split(/\s+/).includes('canonical') && a.href) canonical = a.href;
|
||||
}
|
||||
// JSON-LD (Article / BlogPosting): author and dates when the meta tags lack them.
|
||||
let ld: { author?: unknown; datePublished?: string; headline?: string } = {};
|
||||
for (const m of head.matchAll(
|
||||
/<script[^>]+type=["']application\/ld\+json["'][^>]*>([\s\S]*?)<\/script>/gi
|
||||
)) {
|
||||
try {
|
||||
const parsed = JSON.parse(m[1]);
|
||||
const nodes = [
|
||||
parsed,
|
||||
...(Array.isArray(parsed) ? parsed : []),
|
||||
...(parsed?.['@graph'] ?? [])
|
||||
];
|
||||
const article = nodes.find(
|
||||
(n) => n && typeof n === 'object' && /Article|BlogPosting|Report/.test(String(n['@type']))
|
||||
);
|
||||
if (article) {
|
||||
ld = article;
|
||||
break;
|
||||
}
|
||||
} catch {
|
||||
// Broken JSON-LD is common; ignore it.
|
||||
}
|
||||
}
|
||||
const ldAuthor = (() => {
|
||||
const a = ld.author;
|
||||
const list = Array.isArray(a) ? a : a ? [a] : [];
|
||||
const names = list
|
||||
.map((x) => (typeof x === 'string' ? x : (x as { name?: string })?.name))
|
||||
.filter(Boolean);
|
||||
return names.length ? names.join(', ') : null;
|
||||
})();
|
||||
const titleTag = /<title[^>]*>([\s\S]*?)<\/title>/i.exec(head)?.[1];
|
||||
const abs = (u: string | undefined) => {
|
||||
if (!u) return null;
|
||||
try {
|
||||
const url = new URL(decodeEntities(u.trim()), base);
|
||||
return url.protocol === 'https:' || url.protocol === 'http:' ? url.toString() : null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
};
|
||||
const date = (s: string | undefined) => {
|
||||
if (!s) return null;
|
||||
const d = new Date(s);
|
||||
return Number.isNaN(d.getTime()) ? null : d.toISOString();
|
||||
};
|
||||
const author =
|
||||
meta['author'] ?? meta['article:author'] ?? meta['twitter:creator'] ?? ldAuthor ?? undefined;
|
||||
return {
|
||||
url: abs(canonical ?? meta['og:url']) ?? base,
|
||||
title: clean(meta['og:title'] ?? meta['twitter:title'] ?? ld.headline ?? titleTag, 300),
|
||||
description: clean(
|
||||
meta['og:description'] ?? meta['description'] ?? meta['twitter:description'],
|
||||
600
|
||||
),
|
||||
image: abs(meta['og:image'] ?? meta['og:image:url'] ?? meta['twitter:image']),
|
||||
siteName: clean(meta['og:site_name'] ?? meta['application-name'], 120),
|
||||
author: author && !/^https?:\/\//.test(author) ? clean(author, 200) : null,
|
||||
publishedAt: date(
|
||||
meta['article:published_time'] ??
|
||||
meta['date'] ??
|
||||
meta['dc.date'] ??
|
||||
meta['citation_publication_date'] ??
|
||||
ld.datePublished
|
||||
)
|
||||
};
|
||||
}
|
||||
|
||||
/** Fetches `input` and returns its preview; falls back to just the URL when the page is unreachable. */
|
||||
export async function fetchLinkMeta(input: string): Promise<LinkMeta & { fetched: boolean }> {
|
||||
const empty = (url: string) => ({
|
||||
url,
|
||||
title: null,
|
||||
description: null,
|
||||
image: null,
|
||||
siteName: null,
|
||||
author: null,
|
||||
publishedAt: null,
|
||||
fetched: false
|
||||
});
|
||||
const start = cleanUrl(input);
|
||||
if (!start) throw new Error('Not an http(s) URL');
|
||||
let url = new URL(start);
|
||||
const signal = AbortSignal.timeout(TIMEOUT_MS);
|
||||
try {
|
||||
for (let i = 0; ; i++) {
|
||||
await assertPublic(url);
|
||||
const res = await fetch(url, {
|
||||
redirect: 'manual',
|
||||
signal,
|
||||
headers: { 'user-agent': UA, accept: 'text/html,application/xhtml+xml;q=0.9,*/*;q=0.5' }
|
||||
});
|
||||
const location = res.headers.get('location');
|
||||
if (res.status >= 300 && res.status < 400 && location) {
|
||||
await res.body?.cancel();
|
||||
if (i >= MAX_REDIRECTS) return empty(start);
|
||||
url = new URL(location, url);
|
||||
if (url.protocol !== 'http:' && url.protocol !== 'https:') return empty(start);
|
||||
continue;
|
||||
}
|
||||
if (!res.ok || !/html/i.test(res.headers.get('content-type') ?? '')) {
|
||||
await res.body?.cancel();
|
||||
return empty(cleanUrl(url.toString()) ?? start);
|
||||
}
|
||||
const reader = res.body!.getReader();
|
||||
const chunks: Uint8Array[] = [];
|
||||
let size = 0;
|
||||
while (size < MAX_BYTES) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
chunks.push(value);
|
||||
size += value.length;
|
||||
// The head is all we need.
|
||||
if (new TextDecoder().decode(value).includes('</head>')) break;
|
||||
}
|
||||
await reader.cancel().catch(() => {});
|
||||
const html = new TextDecoder().decode(Buffer.concat(chunks));
|
||||
const final = cleanUrl(url.toString()) ?? start;
|
||||
const meta = parseLinkMeta(html, final);
|
||||
// Only trust a canonical URL on the same site (some pages point at their homepage).
|
||||
const canonical = cleanUrl(meta.url);
|
||||
const sameSite = canonical && new URL(canonical).hostname === new URL(final).hostname;
|
||||
const keep = sameSite && new URL(canonical).pathname.length > 1 ? canonical : final;
|
||||
return { ...meta, url: keep, fetched: true };
|
||||
}
|
||||
} catch (err) {
|
||||
if (err instanceof Error && err.message === 'private host') throw err;
|
||||
return empty(start);
|
||||
}
|
||||
}
|
||||
|
||||
export const siteFromUrl = (url: string) => new URL(url).hostname.replace(/^www\./, '');
|
||||
@@ -34,10 +34,15 @@ export const bookReadingStatus = booksSchema.enum('reading_status', [
|
||||
'abandoned'
|
||||
]);
|
||||
|
||||
// What a work is: a book, a scientific paper, or a web article / link on the reading list.
|
||||
// Articles have no files; they live at `url` and are opened externally.
|
||||
export const bookKind = booksSchema.enum('kind', ['book', 'paper', 'article']);
|
||||
|
||||
export const bookWorks = booksSchema.table(
|
||||
'works',
|
||||
{
|
||||
id: id(),
|
||||
kind: bookKind('kind').notNull().default('book'),
|
||||
title: text('title').notNull(),
|
||||
subtitle: text('subtitle'),
|
||||
authors: text('authors').array().notNull().default([]),
|
||||
@@ -61,9 +66,18 @@ export const bookWorks = booksSchema.table(
|
||||
metadataSource: text('metadata_source'), // 'openlibrary' | 'googlebooks' | 'libgen' | 'manual'
|
||||
metadataFetchedAt: timestamptz('metadata_fetched_at'),
|
||||
notes: text('notes'),
|
||||
// Articles and papers on the web: canonical link, site name and publish date. The excerpt
|
||||
// goes in `description` and the og:image in `cover_url`.
|
||||
url: text('url'),
|
||||
siteName: text('site_name'),
|
||||
publishedAt: timestamptz('published_at'),
|
||||
...timestamps
|
||||
},
|
||||
(t) => [
|
||||
uniqueIndex('works_url_key')
|
||||
.on(t.url)
|
||||
.where(sql`${t.url} is not null`),
|
||||
index('works_kind_idx').on(t.kind),
|
||||
uniqueIndex('works_isbn13_key')
|
||||
.on(t.isbn13)
|
||||
.where(sql`${t.isbn13} is not null`),
|
||||
@@ -206,4 +220,5 @@ export type BookShelf = typeof bookShelves.$inferSelect;
|
||||
export type BookProgress = typeof bookProgress.$inferSelect;
|
||||
export type BookNote = typeof bookNotes.$inferSelect;
|
||||
export type BookSession = typeof bookSessions.$inferSelect;
|
||||
export type BookKind = (typeof bookKind.enumValues)[number];
|
||||
export type BookReadingStatus = (typeof bookReadingStatus.enumValues)[number];
|
||||
|
||||
@@ -18,7 +18,7 @@ export function getTitle(path: string, environment: SiteEnvironment = null): str
|
||||
'/': 'Home',
|
||||
'/projects': 'Projects',
|
||||
'/trackers': 'Trackers',
|
||||
'/trackers/books': 'Books',
|
||||
'/trackers/books': 'Reading',
|
||||
'/trackers/spotting': 'Spotting',
|
||||
'/trackers/cars': 'Cars',
|
||||
'/trackers/planes': 'Planes',
|
||||
|
||||
@@ -10,11 +10,11 @@ describe('tracker icons', () => {
|
||||
expect(new Set(colours).size).toBe(colours.length);
|
||||
});
|
||||
|
||||
it('keeps tasks, money and books first', () => {
|
||||
it('keeps tasks, money and reading first', () => {
|
||||
expect(
|
||||
visibleTrackers(true)
|
||||
.slice(0, 3)
|
||||
.map((t) => t.name)
|
||||
).toEqual(['tasks', 'money', 'books']);
|
||||
).toEqual(['tasks', 'money', 'reading']);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -30,7 +30,7 @@ export const trackerIcons: Record<string, TrackerIcon> = {
|
||||
ink: WHITE,
|
||||
paths: [c(12, 12, 9), 'M15.5 8.6a4 4 0 1 0 0 6.8', 'M7 10.8h6M7 13.2h6']
|
||||
},
|
||||
books: {
|
||||
reading: {
|
||||
from: '#2dd4bf',
|
||||
to: '#0d9488',
|
||||
ink: WHITE,
|
||||
|
||||
Reference in New Issue
Block a user