stan/website
1
0
Fork 0
Code Issues 25 Pull Requests Actions Packages Projects Releases Wiki Activity

books-ingest: safer title/ISBN matching, junk-author filter, GB quota breaker

Browse Source
This commit is contained in:
Stan Runge
2026-10-24 01:03:24 +00:00
parent 2dee93f7b9
commit d246552951
1 changed files with 9 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
scripts/books-ingest.py
View File

@@ -60,8 +60,6 @@ def sim(a: str, b: str) -> float:
r = difflib.SequenceMatcher(None, a, b).ratio() r = difflib.SequenceMatcher(None, a, b).ratio()
if na and na == nb and len(na) >= 15: if na and na == nb and len(na) >= 15:
r = max(r, 0.93) r = max(r, 0.93)
elif na and nb and (na.startswith(nb) or nb.startswith(na)) and min(len(na), len(nb)) >= 15:
r = max(r, 0.9)
return r return r
def surnames(authors: list[str]) -> set[str]: def surnames(authors: list[str]) -> set[str]:
@@ -151,8 +149,12 @@ class Http:
self.cache, self.delay, self.last = cache, delay, 0.0 self.cache, self.delay, self.last = cache, delay, 0.0
cache.mkdir(parents=True, exist_ok=True) cache.mkdir(parents=True, exist_ok=True)
self.calls = 0 self.calls = 0
self.dead_hosts: set[str] = set()
def _get(self, url: str, binary=False): def _get(self, url: str, binary=False):
host = urllib.parse.urlparse(url).netloc
if host in self.dead_hosts: # daily quota hit (Google Books): stop hammering it this run
return 429, b""
wait = self.delay - (time.time() - self.last) wait = self.delay - (time.time() - self.last)
if wait > 0: if wait > 0:
time.sleep(wait) time.sleep(wait)
@@ -164,6 +166,10 @@ class Http:
with urllib.request.urlopen(req, timeout=30) as r: with urllib.request.urlopen(req, timeout=30) as r:
return r.status, r.read() return r.status, r.read()
except urllib.error.HTTPError as e: except urllib.error.HTTPError as e:
if e.code == 429 and "googleapis" in host:
self.dead_hosts.add(host)
log(f" {host}: quota exhausted (429), skipping it for the rest of this run")
return 429, b""
if e.code in (429, 503) and attempt < 2: if e.code in (429, 503) and attempt < 2:
time.sleep(5 * (attempt + 1)) time.sleep(5 * (attempt + 1))
continue continue
@@ -580,7 +586,7 @@ def main():
for w in works: for w in works:
if md5 and md5 in w.md5s: if md5 and md5 in w.md5s:
return w return w
if isbns and set(isbns) & set(w.isbns) and (not title or not w.title or sim(title, w.title) >= 0.6): if isbns and set(isbns) & set(w.isbns) and (not title or not w.title or sim(title, w.title) >= 0.85):
return w return w
best, bs = None, 0.0 best, bs = None, 0.0
for w in works: for w in works: