spotters: most-active-in sort with country/region selector, collection tracking (seen marks, progress, filters), refresh scripts score activity and report new NL entries

src/lib/server/cars.ts

@@ -4,6 +4,8 @@ import { db } from '#lib/server/db/index.js';
 import { carBrands, carModels, carPhotos, carVariants } from '#lib/server/db/schema/index.js';
 import { and, arrayContains, asc, desc, eq, gt, gte, ilike, lte, sql, type SQL } from 'drizzle-orm';
 import { BODIES, DRIVES, FUELS, SORTS, type CarCard, type CarSort } from '#lib/cars/shared.js';
+import { parseArea, parseGot, type Got } from '#lib/spotting/areas.js';
+import { activitySql, gotSql, seenSql } from '#lib/server/spotting.js';
 
 export const PAGE_SIZE = 60;
 
@@ -17,6 +19,9 @@ export type CarFilters = {
 	from: number | null;
 	to: number | null;
 	sort: CarSort;
+	/** Area for the "most active" sort (?in=, default NL). */
+	area: string;
+	got: Got;
 	page: number;
 };
 
@@ -38,12 +43,16 @@ export function parseFilters(params: URLSearchParams): CarFilters {
 		market: pick(params.get('market'), ['nl', 'us'] as const),
 		from: year(params.get('from')),
 		to: year(params.get('to')),
-		sort: sort && sort in SORTS ? (sort as CarSort) : 'popular',
+		sort: sort && sort in SORTS ? (sort as CarSort) : 'active',
+		area: parseArea(params.get('in')),
+		got: parseGot(params.get('got')),
 		page: Math.min(Math.max(1, Math.floor(Number(params.get('page')) || 1)), 1000)
 	};
 }
 
 const cardColumns = {
+	id: carModels.id,
+	seen: seenSql('cars', carModels.id),
 	dex: carModels.dex,
 	slug: carModels.slug,
 	name: carModels.name,
@@ -74,7 +83,15 @@ export async function searchCars(f: CarFilters): Promise<{ cars: CarCard[]; tota
 	if (f.market === 'us') where.push(gt(carModels.variantCount, 0));
 	if (f.from) where.push(gte(carModels.lastYear, f.from));
 	if (f.to) where.push(lte(carModels.firstYear, f.to));
+	const got = gotSql('cars', carModels.id, f.got);
+	if (got) where.push(got);
 	const order = {
+		active: [
+			sql`${activitySql('cars', carModels.id, f.area)} desc`,
+			desc(carModels.nlCount),
+			desc(carModels.variantCount),
+			asc(carModels.dex)
+		],
 		dex: [asc(carModels.dex)],
 		popular: [desc(carModels.nlCount), asc(carModels.dex)],
 		name: [asc(carModels.name), asc(carBrands.name)],
@@ -104,6 +121,7 @@ export async function searchCars(f: CarFilters): Promise<{ cars: CarCard[]; tota
 }
 
 export type BrandTile = {
+	id: number;
 	slug: string;
 	name: string;
 	models: number;
@@ -125,6 +143,7 @@ export async function carOverview(): Promise<Overview> {
 	const [brands, [counts]] = await Promise.all([
 		db
 			.select({
+				id: carBrands.id,
 				slug: carBrands.slug,
 				name: carBrands.name,
 				models: carBrands.modelCount,

src/lib/server/planes.ts

@@ -12,6 +12,8 @@ import {
 	type PlaneEra,
 	type PlaneSort
 } from '#lib/planes/shared.js';
+import { parseArea, parseGot, type Got } from '#lib/spotting/areas.js';
+import { activitySql, gotSql, seenSql } from '#lib/server/spotting.js';
 
 export const PAGE_SIZE = 60;
 
@@ -23,6 +25,9 @@ export type PlaneFilters = {
 	era: PlaneEra | '';
 	service: '' | 'yes' | 'no';
 	sort: PlaneSort;
+	/** Area for the "most active" sort (?in=, default NL). */
+	area: string;
+	got: Got;
 	page: number;
 };
 
@@ -38,12 +43,16 @@ export function parseFilters(params: URLSearchParams): PlaneFilters {
 		engine: pick(params.get('engine'), ENGINES),
 		era: pick(params.get('era'), Object.keys(ERAS) as PlaneEra[]),
 		service: pick(params.get('service'), ['yes', 'no'] as const),
-		sort: sort && sort in SORTS ? (sort as PlaneSort) : 'built',
+		sort: sort && sort in SORTS ? (sort as PlaneSort) : 'active',
+		area: parseArea(params.get('in')),
+		got: parseGot(params.get('got')),
 		page: Math.min(Math.max(1, Math.floor(Number(params.get('page')) || 1)), 1000)
 	};
 }
 
 const cardColumns = {
+	id: planeTypes.id,
+	seen: seenSql('planes', planeTypes.id),
 	dex: planeTypes.dex,
 	slug: planeTypes.slug,
 	name: planeTypes.name,
@@ -89,7 +98,14 @@ export async function searchPlanes(
 	}
 	if (f.service === 'yes') where.push(eq(planeTypes.inService, true));
 	if (f.service === 'no') where.push(eq(planeTypes.inService, false));
+	const got = gotSql('planes', planeTypes.id, f.got);
+	if (got) where.push(got);
 	const order = {
+		active: [
+			sql`${activitySql('planes', planeTypes.id, f.area)} desc`,
+			sql`${planeTypes.numberBuilt} desc nulls last`,
+			asc(planeTypes.dex)
+		],
 		built: [sql`${planeTypes.numberBuilt} desc nulls last`, asc(planeTypes.dex)],
 		dex: [asc(planeTypes.dex)],
 		name: [asc(planeTypes.fullName)],
@@ -120,6 +136,7 @@ export async function searchPlanes(
 }
 
 export type MakerTile = {
+	id: number;
 	slug: string;
 	name: string;
 	types: number;
@@ -141,6 +158,7 @@ export async function planeOverview(): Promise<Overview> {
 	const [makers, [counts]] = await Promise.all([
 		db
 			.select({
+				id: planeMakers.id,
 				slug: planeMakers.slug,
 				name: planeMakers.name,
 				types: planeMakers.typeCount,

src/lib/server/spotting.ts

@@ -0,0 +1,164 @@
+// Shared server helpers for the four spotting pokedexes (cars, planes, trains, wildlife):
+// Stan's collection (<schema>.seen) and the per-country activity areas (<schema>.activity,
+// filled by scripts/spotters-activity.ts). Only the owner writes `seen`; everyone can read.
+import { client } from '#lib/server/db/index.js';
+import { isRegion } from '#lib/spotting/areas.js';
+
+export type Tracker = 'cars' | 'planes' | 'trains' | 'wildlife';
+
+/** Constant identifiers only (never user input), so building SQL text from them is safe. */
+const T: Record<Tracker, { items: string; col: string; group: string }> = {
+	cars: { items: 'cars.models', col: 'model_id', group: 'brand_id' },
+	planes: { items: 'planes.types', col: 'type_id', group: 'maker_id' },
+	trains: { items: 'trains.types', col: 'type_id', group: 'maker_id' },
+	wildlife: { items: 'wildlife.species', col: 'species_id', group: 'group_id' }
+};
+export const isTracker = (t: string): t is Tracker => t in T;
+
+export type SeenEntry = { seenOn: string | null; note: string | null; at: string };
+
+/** Overall progress: seen / total. */
+export async function progress(tracker: Tracker): Promise<{ seen: number; total: number }> {
+	const { items, col } = T[tracker];
+	const schema = tracker;
+	const [r] = await client.unsafe<{ seen: number; total: number }[]>(
+		`select (select count(*)::int from ${schema}.seen s join ${items} i on i.id = s.${col}) as seen,
+		        (select count(*)::int from ${items}) as total`
+	);
+	return r;
+}
+
+/** Seen count per group (brand / maker / animal group id). */
+export async function seenByGroup(tracker: Tracker): Promise<Map<number, number>> {
+	const { items, col, group } = T[tracker];
+	const rows = await client.unsafe<{ g: number; n: number }[]>(
+		`select i.${group} as g, count(*)::int as n from ${tracker}.seen s
+		 join ${items} i on i.id = s.${col} group by 1`
+	);
+	return new Map(rows.map((r) => [r.g, r.n]));
+}
+
+/** Seen / total for one group. */
+export async function groupProgress(tracker: Tracker, groupId: number) {
+	const { items, col, group } = T[tracker];
+	const [r] = await client.unsafe<{ seen: number; total: number }[]>(
+		`select count(s.${col})::int as seen, count(*)::int as total
+		 from ${items} i left join ${tracker}.seen s on s.${col} = i.id where i.${group} = $1`,
+		[groupId]
+	);
+	return r;
+}
+
+/** Seen / total for an arbitrary set of item ids (e.g. a train operator's fleet). */
+export async function idsProgress(tracker: Tracker, ids: number[]) {
+	if (!ids.length) return { seen: 0, total: 0 };
+	const { col } = T[tracker];
+	const [r] = await client.unsafe<{ seen: number }[]>(
+		`select count(*)::int as seen from ${tracker}.seen where ${col} = any($1::int[])`,
+		[ids]
+	);
+	return { seen: r.seen, total: new Set(ids).size };
+}
+
+export async function getSeen(tracker: Tracker, id: number): Promise<SeenEntry | null> {
+	const { col } = T[tracker];
+	const [r] = await client.unsafe<{ seen_on: string | null; note: string | null; at: Date }[]>(
+		`select seen_on::text, note, created_at as at from ${tracker}.seen where ${col} = $1`,
+		[id]
+	);
+	return r ? { seenOn: r.seen_on, note: r.note, at: r.at.toISOString() } : null;
+}
+
+/** Marks (or unmarks) an item as seen. Returns false when the item does not exist. */
+export async function setSeen(
+	tracker: Tracker,
+	id: number,
+	seen: boolean,
+	seenOn: string | null = null,
+	note: string | null = null
+): Promise<boolean> {
+	const { items, col } = T[tracker];
+	const [exists] = await client.unsafe(`select 1 from ${items} where id = $1`, [id]);
+	if (!exists) return false;
+	if (!seen) {
+		await client.unsafe(`delete from ${tracker}.seen where ${col} = $1`, [id]);
+		return true;
+	}
+	await client.unsafe(
+		`insert into ${tracker}.seen (${col}, seen_on, note) values ($1, $2::date, $3)
+		 on conflict (${col}) do update set seen_on = excluded.seen_on, note = excluded.note`,
+		[id, seenOn, note]
+	);
+	return true;
+}
+
+const areaCache = new Map<Tracker, { at: number; value: Set<string> }>();
+/** Areas that have any activity rows (cached 10 min), to grey out empty choices. */
+export async function areasWithData(tracker: Tracker): Promise<string[]> {
+	const hit = areaCache.get(tracker);
+	if (hit && Date.now() - hit.at < 600_000) return [...hit.value];
+	const rows = await client.unsafe<{ area: string }[]>(
+		`select distinct area from ${tracker}.activity`
+	);
+	const value = new Set(rows.map((r) => r.area));
+	if (value.size) areaCache.set(tracker, { at: Date.now(), value });
+	return [...value];
+}
+
+/** Validates a POSTed seen change. */
+export function parseSeenBody(body: unknown) {
+	const b = (body ?? {}) as Record<string, unknown>;
+	const id = Number(b.id);
+	if (!Number.isInteger(id) || id <= 0) return null;
+	const seen = b.seen !== false;
+	const seenOn =
+		typeof b.seenOn === 'string' && /^\d{4}-\d{2}-\d{2}$/.test(b.seenOn) ? b.seenOn : null;
+	const note = typeof b.note === 'string' && b.note.trim() ? b.note.trim().slice(0, 500) : null;
+	return { id, seen, seenOn, note };
+}
+
+export { isRegion };
+
+// ---- SQL fragments for the drizzle list queries ----
+import { sql, type AnyColumn, type SQL } from 'drizzle-orm';
+
+/** `exists(seen)` for a card column. */
+export const seenSql = (tracker: Tracker, idCol: AnyColumn) =>
+	sql<boolean>`exists(select 1 from ${sql.raw(`${tracker}.seen`)} s where s.${sql.raw(T[tracker].col)} = ${idCol})`;
+
+/** Activity score of an item in an area (0 when unknown), for the "most active" sort. */
+export const activitySql = (tracker: Tracker, idCol: AnyColumn, area: string) =>
+	sql<number>`coalesce((select a.score from ${sql.raw(`${tracker}.activity`)} a where a.${sql.raw(T[tracker].col)} = ${idCol} and a.area = ${area}), 0)`;
+
+/** ?got=yes|no filter. */
+export function gotSql(tracker: Tracker, idCol: AnyColumn, got: string): SQL | null {
+	if (got === 'yes') return seenSql(tracker, idCol);
+	if (got === 'no') return sql`not ${seenSql(tracker, idCol)}`;
+	return null;
+}
+
+const groupActCache = new Map<string, { at: number; value: Map<number, number> }>();
+/** Summed activity of each group's entries in an area (cached 10 min): ranks brand/maker tiles. */
+export async function groupActivity(tracker: Tracker, area: string): Promise<Map<number, number>> {
+	const k = `${tracker}:${area}`;
+	const hit = groupActCache.get(k);
+	if (hit && Date.now() - hit.at < 600_000) return hit.value;
+	const { items, col, group } = T[tracker];
+	const rows = await client.unsafe<{ g: number; s: number }[]>(
+		`select i.${group} as g, sum(a.score)::float8 as s from ${tracker}.activity a
+		 join ${items} i on i.id = a.${col} where a.area = $1 group by 1`,
+		[area]
+	);
+	const value = new Map(rows.map((r) => [r.g, r.s]));
+	groupActCache.set(k, { at: Date.now(), value });
+	return value;
+}
+
+/** Orders tiles by area activity (then their existing order). */
+export function rankByActivity<T extends { id: number }>(tiles: T[], act: Map<number, number>) {
+	if (!act.size) return tiles;
+	return tiles
+		.map((t, i) => ({ t, i, s: act.get(t.id) ?? 0 }))
+		.sort((a, b) => b.s - a.s || a.i - b.i)
+		.map((x) => x.t);
+}

src/lib/server/trains.ts

@@ -19,6 +19,8 @@ import {
 	type TrainEra,
 	type TrainSort
 } from '#lib/trains/shared.js';
+import { parseArea, parseGot, type Got } from '#lib/spotting/areas.js';
+import { activitySql, gotSql, seenSql } from '#lib/server/spotting.js';
 
 export const PAGE_SIZE = 60;
 
@@ -31,6 +33,9 @@ export type TrainFilters = {
 	era: TrainEra | '';
 	service: '' | 'yes' | 'no';
 	sort: TrainSort;
+	/** Area for the "most active" sort (?in=, default NL). */
+	area: string;
+	got: Got;
 	page: number;
 };
 
@@ -47,12 +52,16 @@ export function parseFilters(params: URLSearchParams): TrainFilters {
 		country: (params.get('country') ?? '').trim().slice(0, 80),
 		era: pick(params.get('era'), Object.keys(ERAS) as TrainEra[]),
 		service: pick(params.get('service'), ['yes', 'no'] as const),
-		sort: sort && sort in SORTS ? (sort as TrainSort) : 'built',
+		sort: sort && sort in SORTS ? (sort as TrainSort) : 'active',
+		area: parseArea(params.get('in')),
+		got: parseGot(params.get('got')),
 		page: Math.min(Math.max(1, Math.floor(Number(params.get('page')) || 1)), 1000)
 	};
 }
 
 const cardColumns = {
+	id: trainTypes.id,
+	seen: seenSql('trains', trainTypes.id),
 	dex: trainTypes.dex,
 	slug: trainTypes.slug,
 	name: trainTypes.name,
@@ -73,8 +82,13 @@ const cardColumns = {
 };
 
 const escapeLike = (s: string) => s.replace(/[\\%_]/g, (c) => `\\${c}`);
-const order = (sort: TrainSort) =>
+const order = (sort: TrainSort, area = 'nl') =>
 	({
+		active: [
+			sql`${activitySql('trains', trainTypes.id, area)} desc`,
+			sql`${trainTypes.numberBuilt} desc nulls last`,
+			asc(trainTypes.dex)
+		],
 		built: [sql`${trainTypes.numberBuilt} desc nulls last`, asc(trainTypes.dex)],
 		dex: [asc(trainTypes.dex)],
 		name: [asc(trainTypes.fullName)],
@@ -107,6 +121,8 @@ export async function searchTrains(
 	}
 	if (f.service === 'yes') where.push(eq(trainTypes.inService, true));
 	if (f.service === 'no') where.push(eq(trainTypes.inService, false));
+	const got = gotSql('trains', trainTypes.id, f.got);
+	if (got) where.push(got);
 	const condition = where.length ? and(...where) : undefined;
 	const [trains, [{ total }]] = await Promise.all([
 		db
@@ -114,7 +130,7 @@ export async function searchTrains(
 			.from(trainTypes)
 			.innerJoin(trainMakers, eq(trainMakers.id, trainTypes.makerId))
 			.where(condition)
-			.orderBy(...order(f.sort))
+			.orderBy(...order(f.sort, f.area))
 			.limit(PAGE_SIZE)
 			.offset((f.page - 1) * PAGE_SIZE),
 		db
@@ -127,6 +143,7 @@ export async function searchTrains(
 }
 
 export type MakerTile = {
+	id: number;
 	slug: string;
 	name: string;
 	types: number;
@@ -165,6 +182,7 @@ export async function trainOverview(): Promise<Overview> {
 	const [makers, operators, countries, [counts]] = await Promise.all([
 		db
 			.select({
+				id: trainMakers.id,
 				slug: trainMakers.slug,
 				name: trainMakers.name,
 				types: trainMakers.typeCount,

src/lib/server/wildlife.ts

@@ -11,6 +11,8 @@ import {
 	type WildSort,
 	type WildWhere
 } from '#lib/wildlife/shared.js';
+import { parseArea, parseGot, type Got } from '#lib/spotting/areas.js';
+import { activitySql, gotSql, seenSql } from '#lib/server/spotting.js';
 
 export const PAGE_SIZE = 60;
 
@@ -21,6 +23,9 @@ export type WildFilters = {
 	where: WildWhere | '';
 	photos: '' | 'yes';
 	sort: WildSort;
+	/** Area for the "most active" sort (?in=, default NL). */
+	area: string;
+	got: Got;
 	page: number;
 };
 
@@ -36,12 +41,16 @@ export function parseFilters(params: URLSearchParams, group = ''): WildFilters {
 		status: status === 'threatened' ? status : pick(status, STATUSES),
 		where: pick(params.get('where'), Object.keys(WHERE) as WildWhere[]),
 		photos: pick(params.get('photos'), ['yes'] as const),
-		sort: sort && sort in SORTS ? (sort as WildSort) : 'nl',
+		sort: sort && sort in SORTS ? (sort as WildSort) : 'active',
+		area: parseArea(params.get('in')),
+		got: parseGot(params.get('got')),
 		page: Math.min(Math.max(1, Math.floor(Number(params.get('page')) || 1)), 1000)
 	};
 }
 
 const cardColumns = {
+	id: wildlifeSpecies.id,
+	seen: seenSql('wildlife', wildlifeSpecies.id),
 	dex: wildlifeSpecies.dex,
 	slug: wildlifeSpecies.slug,
 	groupSlug: wildlifeGroups.slug,
@@ -77,7 +86,15 @@ export async function searchSpecies(
 	if (f.where === 'world')
 		where.push(and(eq(wildlifeSpecies.nlObs, 0), eq(wildlifeSpecies.europeObs, 0))!);
 	if (f.photos) where.push(isNotNull(wildlifeSpecies.photo));
+	const got = gotSql('wildlife', wildlifeSpecies.id, f.got);
+	if (got) where.push(got);
 	const order = {
+		active: [
+			sql`${activitySql('wildlife', wildlifeSpecies.id, f.area)} desc`,
+			sql`${wildlifeSpecies.nlObs} desc`,
+			sql`${wildlifeSpecies.worldObs} desc`,
+			asc(wildlifeSpecies.dex)
+		],
 		nl: [
 			sql`${wildlifeSpecies.nlObs} desc`,
 			sql`${wildlifeSpecies.worldObs} desc`,
@@ -114,6 +131,7 @@ export async function searchSpecies(
 
 type Overview = {
 	groups: {
+		id: number;
 		slug: string;
 		name: string;
 		nameNl: string;
@@ -132,6 +150,7 @@ export async function wildlifeOverview(): Promise<Overview> {
 	const [groups, [counts]] = await Promise.all([
 		db
 			.select({
+				id: wildlifeGroups.id,
 				slug: wildlifeGroups.slug,
 				name: wildlifeGroups.name,
 				nameNl: wildlifeGroups.nameNl,