From d246552951a6b49a35a8bfbac0a38ecfce1fbfb3 Mon Sep 17 00:00:00 2001 From: Stan Runge Date: Sat, 24 Oct 2026 01:03:24 +0000 Subject: [PATCH] books-ingest: safer title/ISBN matching, junk-author filter, GB quota breaker --- scripts/books-ingest.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/scripts/books-ingest.py b/scripts/books-ingest.py index 8f28d20..a029768 100644 --- a/scripts/books-ingest.py +++ b/scripts/books-ingest.py @@ -60,8 +60,6 @@ def sim(a: str, b: str) -> float: r = difflib.SequenceMatcher(None, a, b).ratio() if na and na == nb and len(na) >= 15: r = max(r, 0.93) - elif na and nb and (na.startswith(nb) or nb.startswith(na)) and min(len(na), len(nb)) >= 15: - r = max(r, 0.9) return r def surnames(authors: list[str]) -> set[str]: @@ -151,8 +149,12 @@ class Http: self.cache, self.delay, self.last = cache, delay, 0.0 cache.mkdir(parents=True, exist_ok=True) self.calls = 0 + self.dead_hosts: set[str] = set() def _get(self, url: str, binary=False): + host = urllib.parse.urlparse(url).netloc + if host in self.dead_hosts: # daily quota hit (Google Books): stop hammering it this run + return 429, b"" wait = self.delay - (time.time() - self.last) if wait > 0: time.sleep(wait) @@ -164,6 +166,10 @@ class Http: with urllib.request.urlopen(req, timeout=30) as r: return r.status, r.read() except urllib.error.HTTPError as e: + if e.code == 429 and "googleapis" in host: + self.dead_hosts.add(host) + log(f" {host}: quota exhausted (429), skipping it for the rest of this run") + return 429, b"" if e.code in (429, 503) and attempt < 2: time.sleep(5 * (attempt + 1)) continue @@ -580,7 +586,7 @@ def main(): for w in works: if md5 and md5 in w.md5s: return w - if isbns and set(isbns) & set(w.isbns) and (not title or not w.title or sim(title, w.title) >= 0.6): + if isbns and set(isbns) & set(w.isbns) and (not title or not w.title or sim(title, w.title) >= 0.85): return w best, bs = None, 0.0 for w in works: