stan/website
1
0
Fork 0
Code Issues 13 Pull Requests Actions Packages Projects Releases Wiki Activity

stats: new public stats schema (source/series/obs/region/fetch_run + raw CBS, Eurostat, FRED, BIS tables) and the stats collector (scripts/stats-fetch.ts); preflight checks the stats tables
Some checks failed
Deploy production / deploy (push) Failing after 54s
Details

Browse Source
This commit is contained in:
Stan Runge
2026-10-25 20:27:04 +00:00
parent 902b1cadd5
commit 8ef291f586
14 changed files with 33128 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

145
drizzle/20261025202715_stats_core/migration.sql Normal file
View File

@@ -0,0 +1,145 @@
CREATE SCHEMA "stats";
--> statement-breakpoint
CREATE TABLE "stats"."bis_policy_rate" (
"ref_area" text,
"freq" char(1),
"period" date,
"rate" double precision,
CONSTRAINT "bis_policy_rate_pkey" PRIMARY KEY("ref_area","freq","period")
);
--> statement-breakpoint
CREATE TABLE "stats"."cbs_code" (
"table_id" text,
"dimension" text,
"code" text,
"title" text NOT NULL,
"unit" text,
"decimals" integer,
"group_id" text,
"sort_index" integer,
CONSTRAINT "cbs_code_pkey" PRIMARY KEY("table_id","dimension","code")
);
--> statement-breakpoint
CREATE TABLE "stats"."cbs_obs" (
"table_id" text,
"obs_id" bigint,
"measure" text NOT NULL,
"dims" jsonb NOT NULL,
"period" text NOT NULL,
"value" double precision,
"value_attribute" text,
CONSTRAINT "cbs_obs_pkey" PRIMARY KEY("table_id","obs_id")
);
--> statement-breakpoint
CREATE TABLE "stats"."cbs_table" (
"table_id" text PRIMARY KEY,
"title" text NOT NULL,
"modified" timestamp with time zone,
"status" text,
"filter" text,
"obs_count" integer,
"fetched_at" timestamp with time zone
);
--> statement-breakpoint
CREATE TABLE "stats"."eurostat_dataset" (
"code" text PRIMARY KEY,
"label" text NOT NULL,
"updated" timestamp with time zone,
"query" text NOT NULL,
"dims" jsonb DEFAULT '{}' NOT NULL,
"obs_count" integer,
"fetched_at" timestamp with time zone
);
--> statement-breakpoint
CREATE TABLE "stats"."eurostat_obs" (
"dataset" text,
"key" text,
"geo" text NOT NULL,
"period" text,
"value" double precision,
"status" text,
CONSTRAINT "eurostat_obs_pkey" PRIMARY KEY("dataset","key","period")
);
--> statement-breakpoint
CREATE TABLE "stats"."fetch_run" (
"run_id" integer PRIMARY KEY GENERATED ALWAYS AS IDENTITY (sequence name "stats"."fetch_run_run_id_seq" INCREMENT BY 1 MINVALUE 1 MAXVALUE 2147483647 START WITH 1 CACHE 1),
"source_id" text NOT NULL,
"job" text DEFAULT 'refresh' NOT NULL,
"started_at" timestamp with time zone DEFAULT now() NOT NULL,
"finished_at" timestamp with time zone,
"rows" integer,
"ok" boolean,
"error" text
);
--> statement-breakpoint
CREATE TABLE "stats"."fred_obs" (
"series_id" text,
"period" date,
"value" double precision,
CONSTRAINT "fred_obs_pkey" PRIMARY KEY("series_id","period")
);
--> statement-breakpoint
CREATE TABLE "stats"."obs" (
"series_id" text,
"period" date,
"value" double precision,
"status" text,
"fetched_at" timestamp with time zone DEFAULT now() NOT NULL,
CONSTRAINT "obs_pkey" PRIMARY KEY("series_id","period")
);
--> statement-breakpoint
CREATE TABLE "stats"."region" (
"region_code" text PRIMARY KEY,
"name" text NOT NULL,
"kind" text NOT NULL,
"parent" text
);
--> statement-breakpoint
CREATE TABLE "stats"."series" (
"series_id" text PRIMARY KEY,
"source_id" text NOT NULL,
"source_code" text NOT NULL,
"title" text NOT NULL,
"unit" text NOT NULL,
"freq" char(1) NOT NULL,
"region_code" text,
"theme" text NOT NULL,
"meta" jsonb DEFAULT '{}' NOT NULL
);
--> statement-breakpoint
CREATE TABLE "stats"."source" (
"source_id" text PRIMARY KEY,
"name" text NOT NULL,
"base_url" text NOT NULL,
"licence" text NOT NULL,
"key_required" boolean DEFAULT false NOT NULL,
"attribution" text NOT NULL
);
--> statement-breakpoint
CREATE INDEX "cbs_obs_table_measure_idx" ON "stats"."cbs_obs" ("table_id","measure");--> statement-breakpoint
CREATE INDEX "eurostat_obs_dataset_geo_idx" ON "stats"."eurostat_obs" ("dataset","geo");--> statement-breakpoint
CREATE INDEX "fetch_run_source_idx" ON "stats"."fetch_run" ("source_id","started_at");--> statement-breakpoint
CREATE INDEX "series_theme_idx" ON "stats"."series" ("theme");--> statement-breakpoint
CREATE INDEX "series_source_idx" ON "stats"."series" ("source_id");--> statement-breakpoint
ALTER TABLE "stats"."fetch_run" ADD CONSTRAINT "fetch_run_source_id_source_source_id_fkey" FOREIGN KEY ("source_id") REFERENCES "stats"."source"("source_id");--> statement-breakpoint
ALTER TABLE "stats"."obs" ADD CONSTRAINT "obs_series_id_series_series_id_fkey" FOREIGN KEY ("series_id") REFERENCES "stats"."series"("series_id") ON DELETE CASCADE;--> statement-breakpoint
ALTER TABLE "stats"."series" ADD CONSTRAINT "series_source_id_source_source_id_fkey" FOREIGN KEY ("source_id") REFERENCES "stats"."source"("source_id");--> statement-breakpoint
ALTER TABLE "stats"."series" ADD CONSTRAINT "series_region_code_region_region_code_fkey" FOREIGN KEY ("region_code") REFERENCES "stats"."region"("region_code");--> statement-breakpoint
-- Hand-written below this line (not generated by drizzle-kit).
-- Public open data: the app role (also used by the host-side stats collector) reads and writes,
-- personal_ro may read. Not registered in meta.tables (no generic table GUI for it).
DO $$
BEGIN
IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = personal_app) THEN
GRANT USAGE ON SCHEMA stats TO personal_app;
GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA stats TO personal_app;
GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA stats TO personal_app;
ALTER DEFAULT PRIVILEGES IN SCHEMA stats GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO personal_app;
ALTER DEFAULT PRIVILEGES IN SCHEMA stats GRANT USAGE, SELECT ON SEQUENCES TO personal_app;
END IF;
IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = personal_ro) THEN
GRANT USAGE ON SCHEMA stats TO personal_ro;
GRANT SELECT ON ALL TABLES IN SCHEMA stats TO personal_ro;
ALTER DEFAULT PRIVILEGES IN SCHEMA stats GRANT SELECT ON TABLES TO personal_ro;
END IF;
END $$;