diff --git a/scripts/ticktick-import.ts b/scripts/ticktick-import.ts index 294c83c..b0cd4eb 100644 --- a/scripts/ticktick-import.ts +++ b/scripts/ticktick-import.ts @@ -1,9 +1,11 @@ /** - * Sync a TickTick export (scripts/ticktick-export.ts) into the private `tasks` - * schema. Idempotent; rerun after a new export to pull in changes. + * Sync TickTick into the private `tasks` schema. Idempotent; rerun after a new export. * - * bun scripts/ticktick-import.ts [--dry-run] ~/.cache/ticktick-export/all.json + * bun scripts/ticktick-import.ts [--dry-run] [--csv backup.csv] ~/.cache/ticktick-export/all.json * + * all.json comes from scripts/ticktick-export.ts (Open API). --csv adds the tasks from a + * TickTick backup (Settings → Backup & Import) that the API no longer returns (closed lists, + * old won't-do tasks); rows the API also has are matched and skipped (see mergeCsv). * Links listed in data/tasks/ticktick-merges.json (plus exact title matches, see * syncTickTick) attach a TickTick task to an existing task instead of duplicating it. * Uses DB_URL (the app role on prod, personal_dev on dev). @@ -11,16 +13,31 @@ import { readFile } from 'node:fs/promises'; import { drizzle } from 'drizzle-orm/postgres-js'; import postgres from 'postgres'; -import { syncTickTick, type TtExport, type TtMerge } from '../src/lib/server/ticktick'; +import { + mergeCsv, + parseTickTickCsv, + syncTickTick, + type TtExport, + type TtMerge +} from '../src/lib/server/ticktick'; const args = process.argv.slice(2); const dryRun = args.includes('--dry-run'); -const files = args.filter((a) => !a.startsWith('--')); -if (files.length !== 1) { - console.error('usage: bun scripts/ticktick-import.ts [--dry-run] '); +const csvAt = args.indexOf('--csv'); +const csvPath = csvAt >= 0 ? args[csvAt + 1] : undefined; +const files = args.filter((a, i) => !a.startsWith('--') && (csvAt < 0 || i !== csvAt + 1)); +if (files.length !== 1 || (csvAt >= 0 && !csvPath)) { + console.error( + 'usage: bun scripts/ticktick-import.ts [--dry-run] [--csv backup.csv] ' + ); process.exit(2); } -const data = JSON.parse(await readFile(files[0], 'utf8')) as TtExport; +let data = JSON.parse(await readFile(files[0], 'utf8')) as TtExport; +if (csvPath) { + const merged = mergeCsv(data, parseTickTickCsv(await readFile(csvPath, 'utf8'))); + data = merged.data; + console.log(JSON.stringify({ csv: merged.report })); +} const merges = JSON.parse( await readFile(new URL('../data/tasks/ticktick-merges.json', import.meta.url), 'utf8') ) as TtMerge[]; @@ -29,9 +46,8 @@ const client = postgres(process.env.DB_URL, { max: 1, onnotice: () => {} }); class DryRun extends Error {} try { const db = drizzle({ client }); - let report; try { - report = await db.transaction(async (tx) => { + const report = await db.transaction(async (tx) => { const r = await syncTickTick(tx, data, { merges }); if (dryRun) { console.log(JSON.stringify(r, null, 2)); diff --git a/src/lib/server/ticktick.test.ts b/src/lib/server/ticktick.test.ts index 1b3c4fc..fa3f830 100644 --- a/src/lib/server/ticktick.test.ts +++ b/src/lib/server/ticktick.test.ts @@ -1,6 +1,8 @@ import { describe, expect, it } from 'vitest'; import { mapTask, + mergeCsv, + parseTickTickCsv, moreComplete, normTitle, slugify, @@ -81,3 +83,58 @@ describe('ticktick mapping', () => { expect(normTitle('Book haircut!')).toBe('book haircut'); }); }); + +describe('ticktick backup csv', () => { + const csv = [ + '"Date: 2026-10-24+0000"', + '"Status: ', + '0 Normal', + '2 Completed"', + '"Folder Name","List Name","Title","Kind","Tags","Content","Is Check list","Start Date","Due Date","Reminder","Repeat","Priority","Status","Created Time","Completed Time","Order","Timezone","Is All Day","Is Floating","Column Name","Column Order","View Mode","taskId","parentId","projectKind"', + '"","Inbox","Known","TEXT","","","N","","","","","0","2","2025-01-01T10:00:00+0000","2025-01-02T10:00:00+0000","0","Europe/Amsterdam","false","false",,,"list","1","","TASK"', + '"","Inbox","Old won\'t do","TEXT","a, b","line ""one""\nline two","N","","","","","5","-1","2024-01-01T10:00:00+0000","2024-02-01T10:00:00+0000","0","Europe/Amsterdam","false","false",,,"list","2","1","TASK"', + '"Old","Trips","Pack","CHECKLIST","","▫Tea\n▪Jerky\n","Y","","","","","0","0","2023-01-01T10:00:00+0000","","0","Europe/Amsterdam","false","false",,,"list","3","","TASK"', + '' + ].join('\n'); + const api = { + projects: [], + tasks: [ + { + id: 'api1', + projectId: 'inbox9', + title: 'Known', + status: 2, + createdTime: '2025-01-01T10:00:00.000+0000' + } + ] + }; + it('parses the backup, including multi-line quoted fields and checklists', () => { + const t = parseTickTickCsv(csv); + expect(t).toHaveLength(3); + expect(t[1]).toMatchObject({ content: 'line "one"\nline two', tags: ['a', 'b'], status: -1 }); + expect(t[2].items).toEqual([ + { id: 'item1', title: 'Tea', status: 0 }, + { id: 'item2', title: 'Jerky', status: 1 } + ]); + }); + it('skips rows the API already has and keys the rest stably', () => { + const a = mergeCsv(api, parseTickTickCsv(csv)); + const b = mergeCsv(api, parseTickTickCsv(csv)); + expect(a.report).toMatchObject({ + csvRows: 3, + matchedListTitleCreated: 1, + added: 2, + newLists: ['Trips'] + }); + const added = a.data.tasks.slice(1); + expect(added.map((t) => t.id)).toEqual(b.data.tasks.slice(1).map((t) => t.id)); + expect(added[0].id).toMatch(/^csv:/); + expect(added[0]).toMatchObject({ projectId: 'inbox9', parentId: 'api1' }); + expect(added[1].projectId).toBe('csvlist:Trips'); + expect(a.data.projects).toContainEqual({ + id: 'csvlist:Trips', + name: 'Old / Trips', + closed: true + }); + }); +}); diff --git a/src/lib/server/ticktick.ts b/src/lib/server/ticktick.ts index fb1b26a..345a097 100644 --- a/src/lib/server/ticktick.ts +++ b/src/lib/server/ticktick.ts @@ -1,6 +1,7 @@ // TickTick → tasks schema sync. Pure mapping + an idempotent sync, shared by // scripts/ticktick-import.ts and its tests; must not import anything from SvelteKit. // Input is the JSON written by scripts/ticktick-export.ts (Open API v1 task objects). +import { createHash } from 'node:crypto'; import { and, eq, isNotNull, isNull, sql } from 'drizzle-orm'; import type { PgAsyncDatabase } from 'drizzle-orm/pg-core'; import { todoActivity, todoAreas, todoTasks, type TaskStatus } from './db/schema/tasks'; @@ -230,6 +231,9 @@ export type SyncReport = { areas: { inserted: string[]; existing: string[] }; tasks: { inserted: number; updated: number; unchanged: number; subtasks: number }; merged: { ticktick: string; task: string; status: TaskStatus }[]; + /** Rows inserted by this run. */ + newByStatus: Partial>; + newByArea: Record; byStatus: Record; byArea: Record; range: { created: [string, string] | null; completed: [string, string] | null }; @@ -253,6 +257,8 @@ export async function syncTickTick( areas: { inserted: [], existing: [] }, tasks: { inserted: 0, updated: 0, unchanged: 0, subtasks: 0 }, merged: [], + newByStatus: {}, + newByArea: {}, byStatus: { todo: 0, in_progress: 0, done: 0, dropped: 0 }, byArea: {}, range: { created: null, completed: null } @@ -349,7 +355,23 @@ export async function syncTickTick( const activity: (typeof todoActivity.$inferInsert)[] = []; for (const r of rows) { - const prev = existing.get(r.externalId); + let prev = existing.get(r.externalId); + // A task first imported from the backup CSV (csv:… key) that the API now returns: + // move it onto the API id instead of inserting a duplicate. + if (!prev && !r.externalId.startsWith('csv:') && !r.parentExternalId) { + const alias = existing.get( + csvKey(listNameOf(data.projects, r.projectId), r.title, r.createdAt) + ); + if (alias && alias.source === TICKTICK_SOURCE) { + await tx + .update(todoTasks) + .set({ externalId: r.externalId }) + .where(eq(todoTasks.id, alias.id)); + existing.delete(alias.externalId!); + prev = { ...alias, externalId: r.externalId }; + existing.set(r.externalId, prev); + } + } const fields = { title: r.title, notes: r.notes, @@ -445,6 +467,9 @@ export async function syncTickTick( createdAt: r.createdAt }); report.tasks.inserted++; + report.newByStatus[r.status] = (report.newByStatus[r.status] ?? 0) + 1; + const an = listNameOf(data.projects, r.projectId); + report.newByArea[an] = (report.newByArea[an] ?? 0) + 1; } // Parents (TickTick subtasks and checklist items), set after every row exists. @@ -493,3 +518,206 @@ export async function syncTickTick( return report; }); } + +// --- TickTick backup CSV (Settings → Backup & Import) -------------------------------- +// The backup has local row numbers as taskId, so CSV-only tasks get a stable key: +// "csv:" + hash(list, title, created second) (+ ":n" for repeats). Rows that also came +// through the API are dropped in favour of the API copy (matched on list + title + +// created time, falling back to title + created time). + +/** Minimal RFC 4180 parser (quoted fields may contain commas, quotes and newlines). */ +export function parseCsv(text: string): string[][] { + const out: string[][] = []; + let row: string[] = []; + let field = ''; + let quoted = false; + for (let i = 0; i < text.length; i++) { + const c = text[i]; + if (quoted) { + if (c === '"') { + if (text[i + 1] === '"') { + field += '"'; + i++; + } else quoted = false; + } else field += c; + } else if (c === '"') quoted = true; + else if (c === ',') { + row.push(field); + field = ''; + } else if (c === '\n' || c === '\r') { + if (c === '\r' && text[i + 1] === '\n') i++; + row.push(field); + out.push(row); + row = []; + field = ''; + } else field += c; + } + if (field || row.length) { + row.push(field); + out.push(row); + } + return out; +} + +export const csvKey = (list: string, title: string, created: Date | null) => + 'csv:' + + createHash('sha1') + .update( + `${list}\u0000${title.trim()}\u0000${created ? Math.floor(created.getTime() / 1000) : ''}` + ) + .digest('hex') + .slice(0, 20); + +export type CsvTask = TtTask & { + csvId: string; + csvParent: string; + listName: string; + folder: string; +}; + +/** Backup CSV → API-shaped tasks (ids still unset; see mergeCsv). */ +export function parseTickTickCsv(text: string): CsvTask[] { + const records = parseCsv(text.replace(/^\uFEFF/, '')); + const h = records.findIndex((r) => r[0] === 'Folder Name'); + if (h < 0) throw new Error('Not a TickTick backup: no "Folder Name" header row'); + const cols = records[h]; + const tasks: CsvTask[] = []; + for (const rec of records.slice(h + 1)) { + if (rec.length < cols.length - 1) continue; + const r = Object.fromEntries(cols.map((c, i) => [c, rec[i] ?? ''])); + const isChecklist = r['Kind'] === 'CHECKLIST' || r['Is Check list'] === 'Y'; + const lines = (r['Content'] ?? '').split('\n'); + const items: TtItem[] = []; + const noteLines: string[] = []; + if (isChecklist) + for (const l of lines) { + const m = l.match(/^\s*([▫▪])\s*(.*)$/u); + if (m) + items.push({ id: `item${items.length + 1}`, title: m[2], status: m[1] === '▪' ? 1 : 0 }); + else noteLines.push(l); + } + const num = (s: string) => (s === '' || Number.isNaN(Number(s)) ? undefined : Number(s)); + tasks.push({ + id: '', + csvId: r['taskId'], + csvParent: r['parentId'], + listName: r['List Name'], + folder: r['Folder Name'], + projectId: '', + title: r['Title'], + content: isChecklist ? noteLines.join('\n') : r['Content'], + status: num(r['Status']), + priority: num(r['Priority']), + tags: r['Tags'] + ? r['Tags'] + .split(',') + .map((t) => t.trim()) + .filter(Boolean) + : [], + startDate: r['Start Date'] || undefined, + dueDate: r['Due Date'] || undefined, + isAllDay: r['Is All Day'] === 'true', + timeZone: r['Timezone'] || undefined, + repeatFlag: r['Repeat'] || undefined, + reminders: r['Reminder'] + ? r['Reminder'] + .split(',') + .map((x) => (x.startsWith('TRIGGER:') ? x : `TRIGGER:${x.trim()}`)) + : [], + items, + sortOrder: num(r['Order']), + createdTime: r['Created Time'] || undefined, + modifiedTime: r['Completed Time'] || r['Created Time'] || undefined, + completedTime: r['Completed Time'] || undefined + }); + } + return tasks; +} + +export const listNameOf = (projects: TtProject[], projectId: string) => + projectId.startsWith('inbox') + ? 'Inbox' + : (projects.find((p) => p.id === projectId)?.name ?? projectId); + +export type CsvMergeReport = { + csvRows: number; + matchedListTitleCreated: number; + matchedTitleCreated: number; + added: number; + newLists: string[]; +}; + +/** Add the CSV-only tasks to an API export (returns a new export + what matched). */ +export function mergeCsv( + data: TtExport, + csv: CsvTask[] +): { data: TtExport; report: CsvMergeReport } { + const sec = (v: string | undefined) => { + const d = ttDate(v); + return d ? String(Math.floor(d.getTime() / 1000)) : ''; + }; + const k1 = new Map(); + const k2 = new Map(); + for (const t of data.tasks) { + const title = (t.title ?? '').trim(); + const a = `${listNameOf(data.projects, t.projectId)}\u0000${title}\u0000${sec(t.createdTime)}`; + const b = `${title}\u0000${sec(t.createdTime)}`; + k1.set(a, [...(k1.get(a) ?? []), t.id]); + k2.set(b, [...(k2.get(b) ?? []), t.id]); + } + const used = new Set(); + const take = (m: Map, k: string) => { + const id = m.get(k)?.find((x) => !used.has(x)); + if (id) used.add(id); + return id; + }; + const report: CsvMergeReport = { + csvRows: csv.length, + matchedListTitleCreated: 0, + matchedTitleCreated: 0, + added: 0, + newLists: [] + }; + const projects = [...data.projects]; + const inboxId = + data.inboxId ?? data.tasks.find((t) => t.projectId.startsWith('inbox'))?.projectId ?? 'inbox'; + const projectFor = (list: string, folder: string) => { + if (list === 'Inbox') return inboxId; + const p = projects.find((x) => x.name === list); + if (p) return p.id; + const id = `csvlist:${list}`; + projects.push({ id, name: folder ? `${folder} / ${list}` : list, closed: true }); + report.newLists.push(list); + return id; + }; + const idOf = new Map(); // csv taskId → API id or csv key + const keyCount = new Map(); + const added: CsvTask[] = []; + for (const t of csv) { + const title = (t.title ?? '').trim(); + let id = take(k1, `${t.listName}\u0000${title}\u0000${sec(t.createdTime)}`); + if (id) report.matchedListTitleCreated++; + else if ((id = take(k2, `${title}\u0000${sec(t.createdTime)}`))) report.matchedTitleCreated++; + if (!id) { + const base = csvKey(t.listName, title, ttDate(t.createdTime)); + const n = (keyCount.get(base) ?? 0) + 1; + keyCount.set(base, n); + id = n === 1 ? base : `${base}:${n}`; + added.push(t); + } + idOf.set(t.csvId, id); + t.id = id; + } + const tasks: TtTask[] = [...data.tasks]; + for (const t of added) { + const { csvId, csvParent, listName, folder, ...task } = t; + void csvId; + tasks.push({ + ...task, + projectId: projectFor(listName, folder), + parentId: (csvParent && idOf.get(csvParent)) || undefined + }); + } + report.added = added.length; + return { data: { ...data, projects, tasks }, report }; +}