stan/website
1
0
Fork 0
Code Issues 21 Pull Requests Actions Packages Projects Releases Wiki Activity

wildlife: photos script without response cache (it outgrew JSON.stringify)
All checks were successful
Deploy production / deploy (push) Successful in 1m5s
Details

Browse Source
This commit is contained in:
Stan Runge
2026-10-24 22:42:26 +00:00
parent 602ea9c8e1
commit 8515cf0e9c
1 changed files with 3 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
scripts/wildlife-photos.ts
View File

@@ -5,14 +5,13 @@
// DB_URL=... bun scripts/wildlife-photos.ts (re)do photos for every species // DB_URL=... bun scripts/wildlife-photos.ts (re)do photos for every species
// bun scripts/wildlife-photos.ts --missing only species never checked / without photos // bun scripts/wildlife-photos.ts --missing only species never checked / without photos
// bun scripts/wildlife-photos.ts --limit 100 at most 100 species this run // bun scripts/wildlife-photos.ts --limit 100 at most 100 species this run
// bun scripts/wildlife-photos.ts --cache /tmp/wildlife reuse API responses
// bun scripts/wildlife-photos.ts --missing --enrich report.json print the import report with photos // bun scripts/wildlife-photos.ts --missing --enrich report.json print the import report with photos
// //
// Per species: the curated iNaturalist taxon photos with a CC licence, topped up with the most // Per species: the curated iNaturalist taxon photos with a CC licence, topped up with the most
// faved research-grade observation photos (CC licence only) until there are MAX photos. // faved research-grade observation photos (CC licence only) until there are MAX photos.
import postgres from 'postgres'; import postgres from 'postgres';
import { readFile } from 'node:fs/promises'; import { readFile } from 'node:fs/promises';
import { chunks, log, saveCache, useCache } from './planes-wiki'; import { chunks, log } from './planes-wiki';
import { import {
author, author,
inat, inat,
@@ -63,7 +62,8 @@ const toPhoto = (p: InatPhoto, kind: string, page: string, place: string | null)
type Obs = { id: number; uri?: string; place_guess?: string | null; photos: InatPhoto[] }; type Obs = { id: number; uri?: string; place_guess?: string | null; photos: InatPhoto[] };
async function main() { async function main() {
await useCache(opt('--cache'), 'wildlife-photos-api.json'); // No response cache: --missing already resumes, and caching every observation page grew
// past what JSON.stringify can write (RangeError: Out of memory).
const url = process.env.DB_URL; const url = process.env.DB_URL;
if (!url) throw new Error('Set DB_URL'); if (!url) throw new Error('Set DB_URL');
const sql = postgres(url, { max: 2, connect_timeout: 10, onnotice: () => {} }); const sql = postgres(url, { max: 2, connect_timeout: 10, onnotice: () => {} });
@@ -133,7 +133,6 @@ async function main() {
total += photos.length; total += photos.length;
done++; done++;
} }
await saveCache();
if (done % 150 < 30) log(` ${done}/${todo.length} species, ${total} photos`); if (done % 150 < 30) log(` ${done}/${todo.length} species, ${total} photos`);
} }
await sql` await sql`
@@ -164,7 +163,6 @@ async function main() {
console.log(JSON.stringify(report, null, 1)); console.log(JSON.stringify(report, null, 1));
} }
} finally { } finally {
await saveCache(true);
await sql.end(); await sql.end();
} }
} }