Tasks: import TickTick backup CSV (csv: keys, matched against API rows, rekeyed when the API returns them)

2026-10-24 03:28:14 +00:00
parent 1754ea9316
commit b77c3f8cf5
3 changed files with 312 additions and 11 deletions
--- a/scripts/ticktick-import.ts
+++ b/scripts/ticktick-import.ts
@@ -1,9 +1,11 @@
 /**
- * Sync a TickTick export (scripts/ticktick-export.ts) into the private `tasks`
- * schema. Idempotent; rerun after a new export to pull in changes.
+ * Sync TickTick into the private `tasks` schema. Idempotent; rerun after a new export.
 *
- *   bun scripts/ticktick-import.ts [--dry-run] ~/.cache/ticktick-export/all.json
+ *   bun scripts/ticktick-import.ts [--dry-run] [--csv backup.csv] ~/.cache/ticktick-export/all.json
 *
+ * all.json comes from scripts/ticktick-export.ts (Open API). --csv adds the tasks from a
+ * TickTick backup (Settings → Backup & Import) that the API no longer returns (closed lists,
+ * old won't-do tasks); rows the API also has are matched and skipped (see mergeCsv).
 * Links listed in data/tasks/ticktick-merges.json (plus exact title matches, see
 * syncTickTick) attach a TickTick task to an existing task instead of duplicating it.
 * Uses DB_URL (the app role on prod, personal_dev on dev).
@@ -11,16 +13,31 @@
 import { readFile } from 'node:fs/promises';
 import { drizzle } from 'drizzle-orm/postgres-js';
 import postgres from 'postgres';
-import { syncTickTick, type TtExport, type TtMerge } from '../src/lib/server/ticktick';
+import {
+	mergeCsv,
+	parseTickTickCsv,
+	syncTickTick,
+	type TtExport,
+	type TtMerge
+} from '../src/lib/server/ticktick';

 const args = process.argv.slice(2);
 const dryRun = args.includes('--dry-run');
-const files = args.filter((a) => !a.startsWith('--'));
-if (files.length !== 1) {
-	console.error('usage: bun scripts/ticktick-import.ts [--dry-run] <export.json>');
+const csvAt = args.indexOf('--csv');
+const csvPath = csvAt >= 0 ? args[csvAt + 1] : undefined;
+const files = args.filter((a, i) => !a.startsWith('--') && (csvAt < 0 || i !== csvAt + 1));
+if (files.length !== 1 || (csvAt >= 0 && !csvPath)) {
+	console.error(
+		'usage: bun scripts/ticktick-import.ts [--dry-run] [--csv backup.csv] <export.json>'
+	);
 	process.exit(2);
 }
-const data = JSON.parse(await readFile(files[0], 'utf8')) as TtExport;
+let data = JSON.parse(await readFile(files[0], 'utf8')) as TtExport;
+if (csvPath) {
+	const merged = mergeCsv(data, parseTickTickCsv(await readFile(csvPath, 'utf8')));
+	data = merged.data;
+	console.log(JSON.stringify({ csv: merged.report }));
+}
 const merges = JSON.parse(
 	await readFile(new URL('../data/tasks/ticktick-merges.json', import.meta.url), 'utf8')
 ) as TtMerge[];
@@ -29,9 +46,8 @@ const client = postgres(process.env.DB_URL, { max: 1, onnotice: () => {} });
 class DryRun extends Error {}
 try {
 	const db = drizzle({ client });
-	let report;
 	try {
-		report = await db.transaction(async (tx) => {
+		const report = await db.transaction(async (tx) => {
 			const r = await syncTickTick(tx, data, { merges });
 			if (dryRun) {
 				console.log(JSON.stringify(r, null, 2));
--- a/src/lib/server/ticktick.test.ts
+++ b/src/lib/server/ticktick.test.ts
@@ -1,6 +1,8 @@
 import { describe, expect, it } from 'vitest';
 import {
 	mapTask,
+	mergeCsv,
+	parseTickTickCsv,
 	moreComplete,
 	normTitle,
 	slugify,
@@ -81,3 +83,58 @@ describe('ticktick mapping', () => {
 		expect(normTitle('Book  haircut!')).toBe('book haircut');
 	});
 });
+
+describe('ticktick backup csv', () => {
+	const csv = [
+		'"Date: 2026-10-24+0000"',
+		'"Status: ',
+		'0 Normal',
+		'2 Completed"',
+		'"Folder Name","List Name","Title","Kind","Tags","Content","Is Check list","Start Date","Due Date","Reminder","Repeat","Priority","Status","Created Time","Completed Time","Order","Timezone","Is All Day","Is Floating","Column Name","Column Order","View Mode","taskId","parentId","projectKind"',
+		'"","Inbox","Known","TEXT","","","N","","","","","0","2","2025-01-01T10:00:00+0000","2025-01-02T10:00:00+0000","0","Europe/Amsterdam","false","false",,,"list","1","","TASK"',
+		'"","Inbox","Old won\'t do","TEXT","a, b","line ""one""\nline two","N","","","","","5","-1","2024-01-01T10:00:00+0000","2024-02-01T10:00:00+0000","0","Europe/Amsterdam","false","false",,,"list","2","1","TASK"',
+		'"Old","Trips","Pack","CHECKLIST","","▫Tea\n▪Jerky\n","Y","","","","","0","0","2023-01-01T10:00:00+0000","","0","Europe/Amsterdam","false","false",,,"list","3","","TASK"',
+		''
+	].join('\n');
+	const api = {
+		projects: [],
+		tasks: [
+			{
+				id: 'api1',
+				projectId: 'inbox9',
+				title: 'Known',
+				status: 2,
+				createdTime: '2025-01-01T10:00:00.000+0000'
+			}
+		]
+	};
+	it('parses the backup, including multi-line quoted fields and checklists', () => {
+		const t = parseTickTickCsv(csv);
+		expect(t).toHaveLength(3);
+		expect(t[1]).toMatchObject({ content: 'line "one"\nline two', tags: ['a', 'b'], status: -1 });
+		expect(t[2].items).toEqual([
+			{ id: 'item1', title: 'Tea', status: 0 },
+			{ id: 'item2', title: 'Jerky', status: 1 }
+		]);
+	});
+	it('skips rows the API already has and keys the rest stably', () => {
+		const a = mergeCsv(api, parseTickTickCsv(csv));
+		const b = mergeCsv(api, parseTickTickCsv(csv));
+		expect(a.report).toMatchObject({
+			csvRows: 3,
+			matchedListTitleCreated: 1,
+			added: 2,
+			newLists: ['Trips']
+		});
+		const added = a.data.tasks.slice(1);
+		expect(added.map((t) => t.id)).toEqual(b.data.tasks.slice(1).map((t) => t.id));
+		expect(added[0].id).toMatch(/^csv:/);
+		expect(added[0]).toMatchObject({ projectId: 'inbox9', parentId: 'api1' });
+		expect(added[1].projectId).toBe('csvlist:Trips');
+		expect(a.data.projects).toContainEqual({
+			id: 'csvlist:Trips',
+			name: 'Old / Trips',
+			closed: true
+		});
+	});
+});
--- a/src/lib/server/ticktick.ts
+++ b/src/lib/server/ticktick.ts
@@ -1,6 +1,7 @@
 // TickTick → tasks schema sync. Pure mapping + an idempotent sync, shared by
 // scripts/ticktick-import.ts and its tests; must not import anything from SvelteKit.
 // Input is the JSON written by scripts/ticktick-export.ts (Open API v1 task objects).
+import { createHash } from 'node:crypto';
 import { and, eq, isNotNull, isNull, sql } from 'drizzle-orm';
 import type { PgAsyncDatabase } from 'drizzle-orm/pg-core';
 import { todoActivity, todoAreas, todoTasks, type TaskStatus } from './db/schema/tasks';
@@ -230,6 +231,9 @@ export type SyncReport = {
 	areas: { inserted: string[]; existing: string[] };
 	tasks: { inserted: number; updated: number; unchanged: number; subtasks: number };
 	merged: { ticktick: string; task: string; status: TaskStatus }[];
+	/** Rows inserted by this run. */
+	newByStatus: Partial<Record<TaskStatus, number>>;
+	newByArea: Record<string, number>;
 	byStatus: Record<TaskStatus, number>;
 	byArea: Record<string, number>;
 	range: { created: [string, string] | null; completed: [string, string] | null };
@@ -253,6 +257,8 @@ export async function syncTickTick(
 		areas: { inserted: [], existing: [] },
 		tasks: { inserted: 0, updated: 0, unchanged: 0, subtasks: 0 },
 		merged: [],
+		newByStatus: {},
+		newByArea: {},
 		byStatus: { todo: 0, in_progress: 0, done: 0, dropped: 0 },
 		byArea: {},
 		range: { created: null, completed: null }
@@ -349,7 +355,23 @@ export async function syncTickTick(
 		const activity: (typeof todoActivity.$inferInsert)[] = [];

 		for (const r of rows) {
-			const prev = existing.get(r.externalId);
+			let prev = existing.get(r.externalId);
+			// A task first imported from the backup CSV (csv:… key) that the API now returns:
+			// move it onto the API id instead of inserting a duplicate.
+			if (!prev && !r.externalId.startsWith('csv:') && !r.parentExternalId) {
+				const alias = existing.get(
+					csvKey(listNameOf(data.projects, r.projectId), r.title, r.createdAt)
+				);
+				if (alias && alias.source === TICKTICK_SOURCE) {
+					await tx
+						.update(todoTasks)
+						.set({ externalId: r.externalId })
+						.where(eq(todoTasks.id, alias.id));
+					existing.delete(alias.externalId!);
+					prev = { ...alias, externalId: r.externalId };
+					existing.set(r.externalId, prev);
+				}
+			}
 			const fields = {
 				title: r.title,
 				notes: r.notes,
@@ -445,6 +467,9 @@ export async function syncTickTick(
 				createdAt: r.createdAt
 			});
 			report.tasks.inserted++;
+			report.newByStatus[r.status] = (report.newByStatus[r.status] ?? 0) + 1;
+			const an = listNameOf(data.projects, r.projectId);
+			report.newByArea[an] = (report.newByArea[an] ?? 0) + 1;
 		}

 		// Parents (TickTick subtasks and checklist items), set after every row exists.
@@ -493,3 +518,206 @@ export async function syncTickTick(
 		return report;
 	});
 }
+
+// --- TickTick backup CSV (Settings → Backup & Import) --------------------------------
+// The backup has local row numbers as taskId, so CSV-only tasks get a stable key:
+// "csv:" + hash(list, title, created second) (+ ":n" for repeats). Rows that also came
+// through the API are dropped in favour of the API copy (matched on list + title +
+// created time, falling back to title + created time).
+
+/** Minimal RFC 4180 parser (quoted fields may contain commas, quotes and newlines). */
+export function parseCsv(text: string): string[][] {
+	const out: string[][] = [];
+	let row: string[] = [];
+	let field = '';
+	let quoted = false;
+	for (let i = 0; i < text.length; i++) {
+		const c = text[i];
+		if (quoted) {
+			if (c === '"') {
+				if (text[i + 1] === '"') {
+					field += '"';
+					i++;
+				} else quoted = false;
+			} else field += c;
+		} else if (c === '"') quoted = true;
+		else if (c === ',') {
+			row.push(field);
+			field = '';
+		} else if (c === '\n' || c === '\r') {
+			if (c === '\r' && text[i + 1] === '\n') i++;
+			row.push(field);
+			out.push(row);
+			row = [];
+			field = '';
+		} else field += c;
+	}
+	if (field || row.length) {
+		row.push(field);
+		out.push(row);
+	}
+	return out;
+}
+
+export const csvKey = (list: string, title: string, created: Date | null) =>
+	'csv:' +
+	createHash('sha1')
+		.update(
+			`${list}\u0000${title.trim()}\u0000${created ? Math.floor(created.getTime() / 1000) : ''}`
+		)
+		.digest('hex')
+		.slice(0, 20);
+
+export type CsvTask = TtTask & {
+	csvId: string;
+	csvParent: string;
+	listName: string;
+	folder: string;
+};
+
+/** Backup CSV → API-shaped tasks (ids still unset; see mergeCsv). */
+export function parseTickTickCsv(text: string): CsvTask[] {
+	const records = parseCsv(text.replace(/^\uFEFF/, ''));
+	const h = records.findIndex((r) => r[0] === 'Folder Name');
+	if (h < 0) throw new Error('Not a TickTick backup: no "Folder Name" header row');
+	const cols = records[h];
+	const tasks: CsvTask[] = [];
+	for (const rec of records.slice(h + 1)) {
+		if (rec.length < cols.length - 1) continue;
+		const r = Object.fromEntries(cols.map((c, i) => [c, rec[i] ?? '']));
+		const isChecklist = r['Kind'] === 'CHECKLIST' || r['Is Check list'] === 'Y';
+		const lines = (r['Content'] ?? '').split('\n');
+		const items: TtItem[] = [];
+		const noteLines: string[] = [];
+		if (isChecklist)
+			for (const l of lines) {
+				const m = l.match(/^\s*([▫▪])\s*(.*)$/u);
+				if (m)
+					items.push({ id: `item${items.length + 1}`, title: m[2], status: m[1] === '▪' ? 1 : 0 });
+				else noteLines.push(l);
+			}
+		const num = (s: string) => (s === '' || Number.isNaN(Number(s)) ? undefined : Number(s));
+		tasks.push({
+			id: '',
+			csvId: r['taskId'],
+			csvParent: r['parentId'],
+			listName: r['List Name'],
+			folder: r['Folder Name'],
+			projectId: '',
+			title: r['Title'],
+			content: isChecklist ? noteLines.join('\n') : r['Content'],
+			status: num(r['Status']),
+			priority: num(r['Priority']),
+			tags: r['Tags']
+				? r['Tags']
+						.split(',')
+						.map((t) => t.trim())
+						.filter(Boolean)
+				: [],
+			startDate: r['Start Date'] || undefined,
+			dueDate: r['Due Date'] || undefined,
+			isAllDay: r['Is All Day'] === 'true',
+			timeZone: r['Timezone'] || undefined,
+			repeatFlag: r['Repeat'] || undefined,
+			reminders: r['Reminder']
+				? r['Reminder']
+						.split(',')
+						.map((x) => (x.startsWith('TRIGGER:') ? x : `TRIGGER:${x.trim()}`))
+				: [],
+			items,
+			sortOrder: num(r['Order']),
+			createdTime: r['Created Time'] || undefined,
+			modifiedTime: r['Completed Time'] || r['Created Time'] || undefined,
+			completedTime: r['Completed Time'] || undefined
+		});
+	}
+	return tasks;
+}
+
+export const listNameOf = (projects: TtProject[], projectId: string) =>
+	projectId.startsWith('inbox')
+		? 'Inbox'
+		: (projects.find((p) => p.id === projectId)?.name ?? projectId);
+
+export type CsvMergeReport = {
+	csvRows: number;
+	matchedListTitleCreated: number;
+	matchedTitleCreated: number;
+	added: number;
+	newLists: string[];
+};
+
+/** Add the CSV-only tasks to an API export (returns a new export + what matched). */
+export function mergeCsv(
+	data: TtExport,
+	csv: CsvTask[]
+): { data: TtExport; report: CsvMergeReport } {
+	const sec = (v: string | undefined) => {
+		const d = ttDate(v);
+		return d ? String(Math.floor(d.getTime() / 1000)) : '';
+	};
+	const k1 = new Map<string, string[]>();
+	const k2 = new Map<string, string[]>();
+	for (const t of data.tasks) {
+		const title = (t.title ?? '').trim();
+		const a = `${listNameOf(data.projects, t.projectId)}\u0000${title}\u0000${sec(t.createdTime)}`;
+		const b = `${title}\u0000${sec(t.createdTime)}`;
+		k1.set(a, [...(k1.get(a) ?? []), t.id]);
+		k2.set(b, [...(k2.get(b) ?? []), t.id]);
+	}
+	const used = new Set<string>();
+	const take = (m: Map<string, string[]>, k: string) => {
+		const id = m.get(k)?.find((x) => !used.has(x));
+		if (id) used.add(id);
+		return id;
+	};
+	const report: CsvMergeReport = {
+		csvRows: csv.length,
+		matchedListTitleCreated: 0,
+		matchedTitleCreated: 0,
+		added: 0,
+		newLists: []
+	};
+	const projects = [...data.projects];
+	const inboxId =
+		data.inboxId ?? data.tasks.find((t) => t.projectId.startsWith('inbox'))?.projectId ?? 'inbox';
+	const projectFor = (list: string, folder: string) => {
+		if (list === 'Inbox') return inboxId;
+		const p = projects.find((x) => x.name === list);
+		if (p) return p.id;
+		const id = `csvlist:${list}`;
+		projects.push({ id, name: folder ? `${folder} / ${list}` : list, closed: true });
+		report.newLists.push(list);
+		return id;
+	};
+	const idOf = new Map<string, string>(); // csv taskId → API id or csv key
+	const keyCount = new Map<string, number>();
+	const added: CsvTask[] = [];
+	for (const t of csv) {
+		const title = (t.title ?? '').trim();
+		let id = take(k1, `${t.listName}\u0000${title}\u0000${sec(t.createdTime)}`);
+		if (id) report.matchedListTitleCreated++;
+		else if ((id = take(k2, `${title}\u0000${sec(t.createdTime)}`))) report.matchedTitleCreated++;
+		if (!id) {
+			const base = csvKey(t.listName, title, ttDate(t.createdTime));
+			const n = (keyCount.get(base) ?? 0) + 1;
+			keyCount.set(base, n);
+			id = n === 1 ? base : `${base}:${n}`;
+			added.push(t);
+		}
+		idOf.set(t.csvId, id);
+		t.id = id;
+	}
+	const tasks: TtTask[] = [...data.tasks];
+	for (const t of added) {
+		const { csvId, csvParent, listName, folder, ...task } = t;
+		void csvId;
+		tasks.push({
+			...task,
+			projectId: projectFor(listName, folder),
+			parentId: (csvParent && idOf.get(csvParent)) || undefined
+		});
+	}
+	report.added = added.length;
+	return { data: { ...data, projects, tasks }, report };
+}