Stan Runge
64fb8df262
- pgSchema tables for auth, money, house, ref, games, cs, animaldex, site, meta (lists excluded via schemaFilter; runtime list tables are managed by meta.*) - drizzle/ baseline matches the reorged DB exactly (catalog diff = 0 on rehearsal) - db/reorg: one-time SQL that moves the flat public schema (rehearsed on a prod copy) - scripts/db-migrate.ts: migrator with --init to record the baseline and a guard against running the baseline on a non-empty, unrecorded database
177 lines
11 KiB
PL/PgSQL
177 lines
11 KiB
PL/PgSQL
-- 05_lists.sql: the 22 legacy lists become real tables in schema `lists`.
|
||
-- legacy.* stays untouched (rollback reference); typo fixes + dedupes are applied to a temp working copy.
|
||
-- Runs as superuser (inserts into personal_ddl-owned tables); tables are created via meta.create_list.
|
||
|
||
CREATE TEMP TABLE src ON COMMIT DROP AS
|
||
SELECT i.id, i.list_id, btrim(i.name) AS name, nullif(btrim(i.description), '') AS notes, i.picture_url,
|
||
c.created_at AS done_at, i.created_at, i.updated_at
|
||
FROM legacy.list_items i
|
||
LEFT JOIN legacy.list_item_completions c ON c.item_id = i.id;
|
||
|
||
CREATE TEMP TABLE fix_log (item_id integer, list_id integer, action text, before text, after text) ON COMMIT DROP;
|
||
|
||
-- ---- dedupes: keep the lower id, carry over done_at / picture / notes, delete the other ----
|
||
CREATE TEMP TABLE dedupe (keep_id integer, drop_id integer, new_name text) ON COMMIT DROP;
|
||
INSERT INTO dedupe VALUES
|
||
(165, 323, NULL), -- places: New York x2
|
||
(168, 331, NULL), -- places: France x2
|
||
(156, 271, 'Advanced Programming in the UNIX Environment'), -- books: "...UNIX® Environment" dup
|
||
(139, 287, 'Building Knowledge Graphs: A Practitioner''s Guide'); -- books: short + full title
|
||
UPDATE src k SET done_at = coalesce(k.done_at, d.done_at), picture_url = coalesce(k.picture_url, d.picture_url),
|
||
notes = coalesce(k.notes, d.notes), name = coalesce(x.new_name, k.name)
|
||
FROM dedupe x JOIN src d ON d.id = x.drop_id WHERE k.id = x.keep_id;
|
||
INSERT INTO fix_log SELECT s.id, s.list_id, 'dedupe-drop', s.name, 'merged into ' || x.keep_id FROM src s JOIN dedupe x ON x.drop_id = s.id;
|
||
DELETE FROM src WHERE id IN (SELECT drop_id FROM dedupe);
|
||
|
||
-- ---- typo / naming fixes ----
|
||
CREATE TEMP TABLE renames (id integer PRIMARY KEY, new_name text NOT NULL) ON COMMIT DROP;
|
||
INSERT INTO renames VALUES
|
||
-- vinyls (kept as "Artist - Album" here; split below)
|
||
(343, 'Pearl Jam - Ten'), (346, 'Amy Winehouse - Back to Black'),
|
||
(27, 'Slipknot - Vol. 3: (The Subliminal Verses)'), (6, 'Them Crooked Vultures - Them Crooked Vultures'),
|
||
(39, 'Various Artists - Katamari Damacy (OST)'),
|
||
(40, 'Gojira - The Way of All Flesh'), (370, 'Gojira - The Link'),
|
||
(80, 'Deftones - Private Music'), (308, 'Kendrick Lamar - DAMN.'),
|
||
-- movies/shows
|
||
(98, 'Kim''s Convenience'), (394, 'Azuki-chan'), (111, 'A Minecraft Movie'),
|
||
-- books
|
||
(150, 'Domain-Driven Design: Tackling Complexity in the Heart of Software'),
|
||
(285, 'Molecular Biology of the Cell'), (288, 'Building Microservices'),
|
||
(147, 'Scrum: The Art of Doing Twice the Work in Half the Time'),
|
||
-- places
|
||
(325, 'Curaçao'),
|
||
-- cars
|
||
(322, 'Mazda MX-5 Miata'),
|
||
-- leetcode
|
||
(217, 'Two Sum'),
|
||
-- foods
|
||
(221, 'Sotteok Sotteok'), (230, 'Bibim Naengmyeon'), (240, 'Doenjang Jjigae'), (241, 'Kimchi Jjigae'),
|
||
-- papers
|
||
(190, 'Order Matters: Sequence to Sequence for Sets'),
|
||
(200, 'GPipe: Easy Scaling with Micro-Batch Pipeline Parallelism'),
|
||
(203, 'Attention Is All You Need'),
|
||
(201, 'Deep Speech 2: End-to-End Speech Recognition in English and Mandarin'),
|
||
-- games
|
||
(385, 'Chrono Trigger');
|
||
DO $$ BEGIN
|
||
IF (SELECT count(*) FROM renames r JOIN src s ON s.id = r.id) <> (SELECT count(*) FROM renames) THEN
|
||
RAISE EXCEPTION 'rename list references missing item ids'; END IF;
|
||
END $$;
|
||
INSERT INTO fix_log SELECT s.id, s.list_id, 'rename', s.name, r.new_name FROM src s JOIN renames r ON r.id = s.id WHERE s.name <> r.new_name;
|
||
UPDATE src s SET name = r.new_name FROM renames r WHERE r.id = s.id;
|
||
|
||
-- helper: copy standard columns for one list into lists.<t> and return the row count
|
||
CREATE FUNCTION pg_temp.copy_std(p_list integer, p_table text, p_has_done boolean) RETURNS bigint LANGUAGE plpgsql AS $$
|
||
DECLARE n bigint;
|
||
BEGIN
|
||
EXECUTE format('INSERT INTO lists.%I (id, name, notes, picture_url, %s sort_order, created_at, updated_at)
|
||
SELECT id, name, notes, picture_url, %s row_number() OVER (ORDER BY id), created_at, updated_at
|
||
FROM src WHERE list_id = %s', p_table,
|
||
CASE WHEN p_has_done THEN 'done_at,' ELSE '' END,
|
||
CASE WHEN p_has_done THEN 'done_at,' ELSE '' END, p_list);
|
||
GET DIAGNOSTICS n = ROW_COUNT;
|
||
EXECUTE format('SELECT setval(pg_get_serial_sequence(%L, ''id''), (SELECT max(id) FROM lists.%I))', 'lists.' || p_table, p_table);
|
||
RETURN n;
|
||
END $$;
|
||
|
||
CREATE TEMP TABLE list_map (list_id integer PRIMARY KEY, tbl text NOT NULL, has_done boolean NOT NULL, cols jsonb NOT NULL) ON COMMIT DROP;
|
||
INSERT INTO list_map VALUES
|
||
(1, 'vinyls', true, '[{"name":"artist","ui_type":"text"}]'),
|
||
(2, 'watchlist', true, '[{"name":"kind","ui_type":"select","options":{"values":["movie","show","anime","documentary"]}}]'),
|
||
(3, 'books', true, '[{"name":"author","ui_type":"text"}]'),
|
||
(4, 'side_projects', false, '[{"name":"status","ui_type":"select","default":"idea","options":{"values":["idea","active","paused","done","dropped"]}}]'),
|
||
(5, 'artists_to_see', true, '[]'),
|
||
(6, 'osu_maps', true, '[{"name":"mods","ui_type":"text"},{"name":"goal","ui_type":"select","options":{"values":["pass","fc"]}}]'),
|
||
(7, 'animals_to_spot', true, '[]'),
|
||
(8, 'vehicles_to_spot', true, '[{"name":"kind","ui_type":"select","default":"car","options":{"values":["car","train","plane"]}}]'),
|
||
(9, 'leetcode', true, '[{"name":"kind","ui_type":"select","default":"problem","options":{"values":["problem","topic"]}},{"name":"url","ui_type":"url"}]'),
|
||
(10, 'restaurants', true, '[{"name":"city","ui_type":"text"}]'),
|
||
(11, 'recipes', true, '[{"name":"cuisine","ui_type":"text"},{"name":"recipe_url","ui_type":"url"}]'),
|
||
(12, 'places', true, '[{"name":"kind","ui_type":"select","options":{"values":["country","region","city","island"]}}]'),
|
||
(13, 'papers', false, '[{"name":"authors","ui_type":"text"},{"name":"year","ui_type":"number"},{"name":"url","ui_type":"url"}]'),
|
||
(14, 'piano_songs', true, '[{"name":"artist","ui_type":"text"},{"name":"reference_url","ui_type":"url"}]'),
|
||
(15, 'games_to_complete', true, '[{"name":"platform","ui_type":"text"}]'),
|
||
(22, 'things_to_learn', true, '[]'),
|
||
(23, 'build_from_scratch', true, '[]'),
|
||
(24, 'tools_to_try', true, '[{"name":"url","ui_type":"url"}]'),
|
||
(25, 'setup_upgrades', true, '[]'),
|
||
(26, 'bucket_list', true, '[]'),
|
||
(27, 'docs_to_read', true, '[{"name":"rfc_number","ui_type":"number"},{"name":"topic","ui_type":"text"},{"name":"starter_pack","ui_type":"bool","default":"false"},{"name":"url","ui_type":"url"}]'),
|
||
(28, 'banger_songs', false, '[{"name":"artist","ui_type":"text"},{"name":"kind","ui_type":"select","default":"song","options":{"values":["song","playlist"]}},{"name":"url","ui_type":"url"}]');
|
||
|
||
DO $$ BEGIN
|
||
IF EXISTS (SELECT 1 FROM legacy.lists l WHERE NOT EXISTS (SELECT 1 FROM list_map m WHERE m.list_id = l.id))
|
||
OR EXISTS (SELECT 1 FROM list_map m JOIN legacy.lists l ON l.id = m.list_id WHERE l.is_progress <> m.has_done AND m.list_id NOT IN (4)) THEN
|
||
RAISE EXCEPTION 'list_map does not cover legacy.lists / has_done mismatch';
|
||
END IF;
|
||
END $$;
|
||
|
||
-- create tables (as personal_ddl via SECURITY DEFINER) and copy standard columns
|
||
DO $$
|
||
DECLARE m record; l record; n bigint;
|
||
BEGIN
|
||
FOR m IN SELECT * FROM list_map ORDER BY list_id LOOP
|
||
SELECT * INTO l FROM legacy.lists WHERE id = m.list_id;
|
||
PERFORM meta.create_list(l.slug, m.tbl, l.name, m.has_done, m.cols, l.picture_url, NULL, NULL, m.list_id);
|
||
UPDATE meta.tables SET created_at = l.created_at WHERE slug = l.slug;
|
||
n := pg_temp.copy_std(m.list_id, m.tbl, m.has_done);
|
||
RAISE NOTICE 'lists.%: % rows', m.tbl, n;
|
||
END LOOP;
|
||
END $$;
|
||
|
||
-- parsing must not bump updated_at: suppress user triggers for these UPDATEs (superuser only)
|
||
SET LOCAL session_replication_role = replica;
|
||
-- ---- per-list parsing into the extra columns (only where it parses cleanly) ----
|
||
-- vinyls: "Artist - Album"
|
||
UPDATE lists.vinyls SET artist = split_part(name, ' - ', 1), name = substr(name, strpos(name, ' - ') + 3)
|
||
WHERE strpos(name, ' - ') > 0;
|
||
-- books: "Author - Title" only for the 3 personal-name ones (others contain ':' subtitles, never ' - ')
|
||
UPDATE lists.books SET author = split_part(name, ' - ', 1), name = substr(name, strpos(name, ' - ') + 3)
|
||
WHERE id IN (28, 29, 306);
|
||
-- side projects: everything starts as an idea
|
||
UPDATE lists.side_projects SET status = 'idea';
|
||
-- osu: trailing mods
|
||
UPDATE lists.osu_maps SET name = 'Lonely Go', mods = 'HR' WHERE id = 310;
|
||
UPDATE lists.osu_maps SET name = 'Imagination', mods = 'HR' WHERE id = 317;
|
||
-- vehicles: all current entries are cars
|
||
UPDATE lists.vehicles_to_spot SET kind = 'car';
|
||
-- leetcode
|
||
UPDATE lists.leetcode SET kind = CASE WHEN name IN ('Two Sum','3Sum') THEN 'problem' ELSE 'topic' END;
|
||
UPDATE lists.leetcode SET url = 'https://leetcode.com/problems/two-sum/' WHERE name = 'Two Sum';
|
||
UPDATE lists.leetcode SET url = 'https://leetcode.com/problems/3sum/' WHERE name = '3Sum';
|
||
-- restaurants: "Name (City)"
|
||
UPDATE lists.restaurants SET city = substring(name from '\(([^)]+)\)\s*$'), name = btrim(regexp_replace(name, '\s*\([^)]+\)\s*$', ''))
|
||
WHERE name ~ '\([^)]+\)\s*$';
|
||
-- recipes
|
||
UPDATE lists.recipes SET cuisine = CASE WHEN name = 'Yogurt, honey & cruesli' THEN NULL ELSE 'korean' END;
|
||
-- places
|
||
UPDATE lists.places SET kind = CASE
|
||
WHEN name IN ('London','Seattle','Miami','Dubai','New York') THEN 'city'
|
||
WHEN name IN ('California','Texas') THEN 'region'
|
||
WHEN name IN ('Curaçao','Aruba','Bonaire') THEN 'island'
|
||
ELSE 'country' END;
|
||
-- piano: "artist - title" + URL in notes
|
||
UPDATE lists.piano_songs SET artist = 'ryo (supercell)', name = 'odds & ends', reference_url = notes, notes = NULL
|
||
WHERE name = 'ryo/supercell - odds & ends';
|
||
-- tools: notes that are a bare URL become url
|
||
UPDATE lists.tools_to_try SET url = notes, notes = NULL WHERE notes ~ '^https?://\S+$';
|
||
-- docs/RFCs: "RFC n – Title", notes "Topic · starter pack"
|
||
UPDATE lists.docs_to_read SET
|
||
rfc_number = substring(name from '^RFC (\d+)')::integer,
|
||
topic = btrim(split_part(notes, '·', 1)),
|
||
starter_pack = notes LIKE '%starter pack%',
|
||
url = 'https://www.rfc-editor.org/rfc/rfc' || substring(name from '^RFC (\d+)'),
|
||
notes = NULL
|
||
WHERE name ~ '^RFC \d+';
|
||
-- banger songs: "Artist - Title" + URL notes; the 2 YT mixes are playlists
|
||
UPDATE lists.banger_songs SET url = notes, notes = NULL WHERE notes ~ '^https?://\S+$';
|
||
UPDATE lists.banger_songs SET artist = split_part(name, ' - ', 1), name = substr(name, strpos(name, ' - ') + 3), kind = 'song'
|
||
WHERE strpos(name, ' - ') > 0;
|
||
UPDATE lists.banger_songs SET kind = 'playlist' WHERE artist IS NULL;
|
||
|
||
SET LOCAL session_replication_role = origin;
|
||
|
||
-- persist the fix log for review (legacy schema, dropped with it)
|
||
CREATE TABLE legacy.reorg_list_fixes AS SELECT * FROM fix_log;
|
||
ALTER TABLE legacy.reorg_list_fixes OWNER TO personal; -- so `personal` can pg_dump everything
|