Expand website and game trackers

scripts/build-cod-challenges.py

@@ -0,0 +1,91 @@
+"""Import attributable Call of Duty Wiki table snapshots into a factual challenge catalog.
+
+Input is one or more rendered, line-numbered source snapshots per game, exported
+by the research tool. Ambiguous or incomplete rows are skipped. No thresholds
+are invented. Keys depend on category + name, never page line numbers.
+
+Usage: python3 scripts/build-cod-challenges.py --snapshots /tmp/cod-wiki
+The checked-in JSON is the deployable snapshot; this command does not need DB access.
+"""
+import argparse
+import json
+from pathlib import Path
+import re
+
+TITLES={'black-ops':'Call_of_Duty:_Black_Ops','black-ops-2':'Call_of_Duty:_Black_Ops_II','ghosts':'Operations_(challenges)/Call_of_Duty:_Ghosts','advanced-warfare':'Call_of_Duty:_Advanced_Warfare','black-ops-3':'Call_of_Duty:_Black_Ops_III','infinite-warfare':'Call_of_Duty:_Infinite_Warfare','wwii':'Call_of_Duty:_WWII'}
+VERBS=r'(?:Get|Earn|Kill|Win|Complete|Destroy|Perform|Play|Achieve|Reach|Use|Capture|Defend|Survive|Assist|Shoot|Recover|Hack|Call in|Obtain|Unlock|Steal|Equip|Finish|Plant|Defuse|Deny|Pick up|Revive|Take|Absorb|Damage|Land|Hold|Score|Purchase|Make|Eliminate|Return|Collect|Prevent|Detonate|End|Resupply)'
+
+def slug(s):return re.sub(r'[^a-z0-9]+','-',s.lower()).strip('-')
+def clean(s):return re.sub(r'\s+',' ',s).strip().replace('[]','')
+
+def lines_for(directory,game):
+ lines={}
+ for p in directory.glob(game+'*.txt'):
+  if p.name.split('--')[0].split('.txt')[0]!=game:continue
+  raw=p.read_text();raw=re.sub(r'cite[^†]+†([^†]+)(?:†[^]+)?',r'\1',raw)
+  for m in re.finditer(r'L(\d+):\s*(.*?)(?=L\d+:|$)',raw,re.S):
+   number=int(m[1]);text=m[2].split('-------------------------')[0].strip()
+   if number not in lines or len(text)>len(lines[number]):lines[number]=text
+ return sorted(lines.items())
+
+def parse(directory,game):
+ rows=lines_for(directory,game);heads={};items={};previous='';previous_line=0
+ source='https://callofduty.fandom.com/wiki/'+('' if game=='ghosts' else 'Challenges/')+TITLES[game]
+ for number,line in rows:
+  line=line.replace('\xa0','  ')
+  heading=re.match(r'^(#{2,6})\s+(.+)',line)
+  if heading:
+   level=len(heading[1]);heads={k:v for k,v in heads.items() if k<level};heads[level]=clean(heading[2]);previous='';continue
+  if game=='ghosts':
+   # The source omits many tier thresholds. Only standalone, explicit operations
+   # are imported; unknown '#' counts are deliberately left out.
+   bullet=re.match(rf'^\s*\*\s+(.+?):\s*((?:{VERBS}|Fire|Force|Crush|Have|Be|Successfully|Infect)\b.+)$',line)
+   if bullet and heads and '#' not in bullet[2] and any('Operations' in h for h in heads.values()) and not any('References' in h or 'Trivia' in h for h in heads.values()):
+    name=clean(bullet[1]);requirement=clean(bullet[2]);category=' / '.join(heads[k] for k in sorted(heads))
+    key=slug(category)+':'+slug(name)
+    items[key]=dict(key=key,name=name,category=category,requirement=requirement,reward=None,target=None,icon=None,sourceUrl=source+'#'+heads[max(heads)].replace(' ','_'))
+   continue
+  if not heads or not line or line.startswith(('*','Note:','Challenge','Name ','Description','XP Reward','Image','This ','The ','After ','All ','I:','II:','III:','IV:','V:','VI:')):continue
+  if any(re.search(r'Contents|Trivia|References|Gallery|Navigation|Explore|Follow|Community|Advertise',h) for h in heads.values()):continue
+  # BO2 campaign objectives lack stable named table rows; exclude those.
+  if game=='black-ops-2' and number<705:continue
+  if game=='black-ops' and number<490:continue
+  match=re.match(rf'^(.{{2,100}}?)\s+({VERBS}\b.+)$',line)
+  if match:
+   name,rest=match.groups()
+   if re.match(rf'^{VERBS}\b',name) or 'Challenge How' in name:continue
+  elif game=='advanced-warfare' and previous and number-previous_line<5 and re.match(rf'^{VERBS}\b',line):
+   name,rest=previous,line
+  else:
+   if game=='advanced-warfare' and len(line)<70 and not re.search(r'\d|XP|Playercard|Emblem|Reticle',line):previous=clean(line);previous_line=number
+   continue
+  fields=re.split(r'\s{2,}',rest)
+  requirement=clean(fields[0]);rewards=fields[1:]
+  if len(fields)==1:
+   match_xp=re.search(r'\s+(\d[\d,]*)(?:\s*XP)?$',requirement)
+   if match_xp:rewards=[match_xp[1]+' XP'];requirement=requirement[:match_xp.start()]
+  reward=clean(' · '.join(r for r in rewards if r not in ['None','Image'] and not re.fullmatch(r'\d+',r)))
+  reward=re.sub(r'Image:\s*','',reward).replace('Image ','').strip(' ·')
+  if not reward and any('Image' in r for r in rewards):reward='Calling card (see source)'
+  if not requirement or len(requirement)>1000 or re.search(r'\bL\d+:',requirement):continue
+  # Numeric progress only when the requirement starts with an explicit count.
+  count=re.match(r'^(?:Get|Earn|Kill|Win|Destroy|Play|Capture|Complete|Absorb)\s+(\d[\d,]*)\s+',requirement)
+  target=int(count[1].replace(',','')) if count else None
+  category=' / '.join(heads[k] for k in sorted(heads))
+  if game=='black-ops':category='Multiplayer / '+category
+  name=clean(name).strip('* ')
+  if re.search(r'Image:|\bXP\b',name):continue  # merged table cells, not a valid challenge name
+  key=slug(category)+':'+slug(name)
+  if key in items:continue
+  items[key]=dict(key=key,name=name,category=category,requirement=requirement,reward=reward or None,target=target,icon=None,sourceUrl=source+'#'+heads[max(heads)].replace(' ','_'))
+ return list(items.values())
+
+
+def main():
+ parser=argparse.ArgumentParser();parser.add_argument('--snapshots',type=Path,required=True);args=parser.parse_args()
+ games={game:parse(args.snapshots,game) for game in TITLES}
+ result=dict(source=dict(name='Call of Duty Wiki contributors',license='CC BY-SA',licenseUrl='https://www.fandom.com/licensing',attribution='https://callofduty.fandom.com/wiki/Call_of_Duty_Wiki:Copyrights',retrieved='2026-09-25',scope='Named challenge rows with explicit requirements. Incomplete and ambiguous source rows are excluded. Source coverage varies; these are not complete game inventories. Numeric targets are recorded only when explicit at the start of a requirement.'),games=games)
+ Path('data/cod-challenges.json').write_text(json.dumps(result,ensure_ascii=False,indent=2)+'\n')
+ print({game:len(rows) for game,rows in games.items()})
+
+if __name__=='__main__':main()