stats: new public stats schema (source/series/obs/region/fetch_run + raw CBS, Eurostat, FRED, BIS tables) and the stats collector (scripts/stats-fetch.ts); preflight checks the stats tables

src/lib/server/db/schema/stats.ts

@@ -0,0 +1,204 @@
+import {
+	bigint,
+	boolean,
+	char,
+	date,
+	doublePrecision,
+	index,
+	integer,
+	jsonb,
+	primaryKey,
+	text
+} from 'drizzle-orm/pg-core';
+import { statsSchema } from './schemas';
+import { timestamptz } from './helpers';
+
+// Public /stats section: open statistics from official sources (CBS, Eurostat, FRED, BIS, …).
+// Two layers (see docs/stats.md):
+//  - raw tables per source (`cbs_*`, `eurostat_*`, `fred_*`, `bis_*`), keeping the source's own codes;
+//  - a normalised layer (`source`, `series`, `obs`, `region`) that every chart reads.
+// Written only by the stats collector (scripts/stats-fetch.ts, systemd user timers on the host);
+// every run is logged in `fetch_run`. Everything here is open data, so personal_ro may read it.
+
+export const statsSource = statsSchema.table('source', {
+	sourceId: text('source_id').primaryKey(),
+	name: text('name').notNull(),
+	baseUrl: text('base_url').notNull(),
+	licence: text('licence').notNull(),
+	keyRequired: boolean('key_required').notNull().default(false),
+	/** Footer line under every chart that uses this source. */
+	attribution: text('attribution').notNull()
+});
+
+export const statsRegion = statsSchema.table('region', {
+	/** ISO3 for countries, 'EU27' / 'EA' / 'WLD' aggregates, CBS codes (GM0513, CR28) below that. */
+	regionCode: text('region_code').primaryKey(),
+	name: text('name').notNull(),
+	/** country, municipality, corop, aggregate */
+	kind: text('kind').notNull(),
+	parent: text('parent')
+});
+
+export const statsSeries = statsSchema.table(
+	'series',
+	{
+		/** Stable id, e.g. `cbs:86141NED:cpi-yoy`, `fred:DCOILBRENTEU`, `bis:XM`. */
+		seriesId: text('series_id').primaryKey(),
+		sourceId: text('source_id')
+			.notNull()
+			.references(() => statsSource.sourceId),
+		/** The source's own code (table + measure + dimension keys, FRED id, …). */
+		sourceCode: text('source_code').notNull(),
+		title: text('title').notNull(),
+		unit: text('unit').notNull(),
+		/** D, M, Q, S (semester) or A */
+		freq: char('freq', { length: 1 }).notNull(),
+		regionCode: text('region_code').references(() => statsRegion.regionCode),
+		/** economy, energy, gouda, budget, countries, … */
+		theme: text('theme').notNull(),
+		meta: jsonb('meta').$type<Record<string, unknown>>().notNull().default({})
+	},
+	(t) => [index('series_theme_idx').on(t.theme), index('series_source_idx').on(t.sourceId)]
+);
+
+export const statsObs = statsSchema.table(
+	'obs',
+	{
+		seriesId: text('series_id')
+			.notNull()
+			.references(() => statsSeries.seriesId, { onDelete: 'cascade' }),
+		/** Period start date: 2026-S1 → 2026-01-01, 2026Q3 → 2026-07-01, 2026 → 2026-01-01. */
+		period: date('period', { mode: 'string' }).notNull(),
+		value: doublePrecision('value'),
+		/** Source status flag (provisional, estimated, …) where given. */
+		status: text('status'),
+		fetchedAt: timestamptz('fetched_at').notNull().defaultNow()
+	},
+	(t) => [primaryKey({ columns: [t.seriesId, t.period] })]
+);
+
+export const statsFetchRun = statsSchema.table(
+	'fetch_run',
+	{
+		runId: integer('run_id').primaryKey().generatedAlwaysAsIdentity(),
+		sourceId: text('source_id')
+			.notNull()
+			.references(() => statsSource.sourceId),
+		/** What the run did: 'refresh', 'backfill', or a single table/dataset code. */
+		job: text('job').notNull().default('refresh'),
+		startedAt: timestamptz('started_at').notNull().defaultNow(),
+		finishedAt: timestamptz('finished_at'),
+		rows: integer('rows'),
+		ok: boolean('ok'),
+		error: text('error')
+	},
+	(t) => [index('fetch_run_source_idx').on(t.sourceId, t.startedAt)]
+);
+
+// ── A1 CBS StatLine (OData v4, long format) ───────────────────────────────────────────────
+
+export const statsCbsTable = statsSchema.table('cbs_table', {
+	tableId: text('table_id').primaryKey(),
+	title: text('title').notNull(),
+	/** `Modified` from the table's Properties; the nightly run re-pulls only when it changes. */
+	modified: timestamptz('modified'),
+	status: text('status'),
+	/** OData $filter used for big tables (regions of interest only); null = full table. */
+	filter: text('filter'),
+	obsCount: integer('obs_count'),
+	fetchedAt: timestamptz('fetched_at')
+});
+
+export const statsCbsCode = statsSchema.table(
+	'cbs_code',
+	{
+		tableId: text('table_id').notNull(),
+		/** 'Measure' for MeasureCodes, otherwise the dimension identifier (RegioS, Perioden, …). */
+		dimension: text('dimension').notNull(),
+		code: text('code').notNull(),
+		title: text('title').notNull(),
+		unit: text('unit'),
+		decimals: integer('decimals'),
+		groupId: text('group_id'),
+		sortIndex: integer('sort_index')
+	},
+	(t) => [primaryKey({ columns: [t.tableId, t.dimension, t.code] })]
+);
+
+export const statsCbsObs = statsSchema.table(
+	'cbs_obs',
+	{
+		tableId: text('table_id').notNull(),
+		/** CBS observation Id (unique per table). */
+		obsId: bigint('obs_id', { mode: 'number' }).notNull(),
+		measure: text('measure').notNull(),
+		/** Every non-time dimension, trimmed: {"RegioS":"GM0513","Geslacht":"T001038"}. */
+		dims: jsonb('dims').$type<Record<string, string>>().notNull(),
+		/** CBS period code as-is: 2026MM09, 2026KW03, 2026JJ00. */
+		period: text('period').notNull(),
+		value: doublePrecision('value'),
+		valueAttribute: text('value_attribute')
+	},
+	(t) => [
+		primaryKey({ columns: [t.tableId, t.obsId] }),
+		index('cbs_obs_table_measure_idx').on(t.tableId, t.measure)
+	]
+);
+
+// ── A2 Eurostat dissemination API (JSON-stat) ─────────────────────────────────────────────
+
+export const statsEurostatDataset = statsSchema.table('eurostat_dataset', {
+	code: text('code').primaryKey(),
+	label: text('label').notNull(),
+	/** `updated` from the JSON-stat payload. */
+	updated: timestamptz('updated'),
+	/** Query used (filters), so a refresh repeats it exactly. */
+	query: text('query').notNull(),
+	/** Dimension ids in order plus their category labels. */
+	dims: jsonb('dims').$type<Record<string, unknown>>().notNull().default({}),
+	obsCount: integer('obs_count'),
+	fetchedAt: timestamptz('fetched_at')
+});
+
+export const statsEurostatObs = statsSchema.table(
+	'eurostat_obs',
+	{
+		dataset: text('dataset').notNull(),
+		/** Non-time dimension codes joined with '.', in the dataset's dimension order. */
+		key: text('key').notNull(),
+		geo: text('geo').notNull(),
+		/** Eurostat time code as-is: 2026-S1, 2026-09, 2026-Q2, 2025. */
+		period: text('period').notNull(),
+		value: doublePrecision('value'),
+		status: text('status')
+	},
+	(t) => [
+		primaryKey({ columns: [t.dataset, t.key, t.period] }),
+		index('eurostat_obs_dataset_geo_idx').on(t.dataset, t.geo)
+	]
+);
+
+// ── A7 FRED graph CSV, A5 BIS policy rates ────────────────────────────────────────────────
+
+export const statsFredObs = statsSchema.table(
+	'fred_obs',
+	{
+		seriesId: text('series_id').notNull(),
+		period: date('period', { mode: 'string' }).notNull(),
+		value: doublePrecision('value')
+	},
+	(t) => [primaryKey({ columns: [t.seriesId, t.period] })]
+);
+
+export const statsBisPolicyRate = statsSchema.table(
+	'bis_policy_rate',
+	{
+		/** BIS area code: US, XM (euro area), KR, CN, GB, JP. */
+		refArea: text('ref_area').notNull(),
+		/** D or M */
+		freq: char('freq', { length: 1 }).notNull(),
+		period: date('period', { mode: 'string' }).notNull(),
+		rate: doublePrecision('rate')
+	},
+	(t) => [primaryKey({ columns: [t.refArea, t.freq, t.period] })]
+);