wildlife: photos script without response cache (it outgrew JSON.stringify)
All checks were successful
Deploy production / deploy (push) Successful in 1m5s
All checks were successful
Deploy production / deploy (push) Successful in 1m5s
This commit is contained in:
@@ -5,14 +5,13 @@
|
||||
// DB_URL=... bun scripts/wildlife-photos.ts (re)do photos for every species
|
||||
// bun scripts/wildlife-photos.ts --missing only species never checked / without photos
|
||||
// bun scripts/wildlife-photos.ts --limit 100 at most 100 species this run
|
||||
// bun scripts/wildlife-photos.ts --cache /tmp/wildlife reuse API responses
|
||||
// bun scripts/wildlife-photos.ts --missing --enrich report.json print the import report with photos
|
||||
//
|
||||
// Per species: the curated iNaturalist taxon photos with a CC licence, topped up with the most
|
||||
// faved research-grade observation photos (CC licence only) until there are MAX photos.
|
||||
import postgres from 'postgres';
|
||||
import { readFile } from 'node:fs/promises';
|
||||
import { chunks, log, saveCache, useCache } from './planes-wiki';
|
||||
import { chunks, log } from './planes-wiki';
|
||||
import {
|
||||
author,
|
||||
inat,
|
||||
@@ -63,7 +62,8 @@ const toPhoto = (p: InatPhoto, kind: string, page: string, place: string | null)
|
||||
type Obs = { id: number; uri?: string; place_guess?: string | null; photos: InatPhoto[] };
|
||||
|
||||
async function main() {
|
||||
await useCache(opt('--cache'), 'wildlife-photos-api.json');
|
||||
// No response cache: --missing already resumes, and caching every observation page grew
|
||||
// past what JSON.stringify can write (RangeError: Out of memory).
|
||||
const url = process.env.DB_URL;
|
||||
if (!url) throw new Error('Set DB_URL');
|
||||
const sql = postgres(url, { max: 2, connect_timeout: 10, onnotice: () => {} });
|
||||
@@ -133,7 +133,6 @@ async function main() {
|
||||
total += photos.length;
|
||||
done++;
|
||||
}
|
||||
await saveCache();
|
||||
if (done % 150 < 30) log(` ${done}/${todo.length} species, ${total} photos`);
|
||||
}
|
||||
await sql`
|
||||
@@ -164,7 +163,6 @@ async function main() {
|
||||
console.log(JSON.stringify(report, null, 1));
|
||||
}
|
||||
} finally {
|
||||
await saveCache(true);
|
||||
await sql.end();
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user