personal-website/scripts/build-cod-challenges.py

"""Import attributable Call of Duty Wiki table snapshots into a factual challenge catalog.

Input is one or more rendered, line-numbered source snapshots per game, exported
by the research tool. Ambiguous or incomplete rows are skipped. No thresholds
are invented. Keys depend on category + name, never page line numbers.

Usage: python3 scripts/build-cod-challenges.py --snapshots /tmp/cod-wiki
The checked-in JSON is the deployable snapshot; this command does not need DB access.
"""
import argparse
import json
from pathlib import Path
import re

TITLES={'black-ops':'Call_of_Duty:_Black_Ops','black-ops-2':'Call_of_Duty:_Black_Ops_II','ghosts':'Operations_(challenges)/Call_of_Duty:_Ghosts','advanced-warfare':'Call_of_Duty:_Advanced_Warfare','black-ops-3':'Call_of_Duty:_Black_Ops_III','infinite-warfare':'Call_of_Duty:_Infinite_Warfare','wwii':'Call_of_Duty:_WWII'}
VERBS=r'(?:Get|Earn|Kill|Win|Complete|Destroy|Perform|Play|Achieve|Reach|Use|Capture|Defend|Survive|Assist|Shoot|Recover|Hack|Call in|Obtain|Unlock|Steal|Equip|Finish|Plant|Defuse|Deny|Pick up|Revive|Take|Absorb|Damage|Land|Hold|Score|Purchase|Make|Eliminate|Return|Collect|Prevent|Detonate|End|Resupply)'

def slug(s):return re.sub(r'[^a-z0-9]+','-',s.lower()).strip('-')
def clean(s):return re.sub(r'\s+',' ',s).strip().replace('[]','')

def lines_for(directory,game):
 lines={}
 for p in directory.glob(game+'*.txt'):
  if p.name.split('--')[0].split('.txt')[0]!=game:continue
  raw=p.read_text();raw=re.sub(r'cite[^†]+†([^†]+)(?:†[^]+)?',r'\1',raw)
  for m in re.finditer(r'L(\d+):\s*(.*?)(?=L\d+:|$)',raw,re.S):
   number=int(m[1]);text=m[2].split('-------------------------')[0].strip()
   if number not in lines or len(text)>len(lines[number]):lines[number]=text
 return sorted(lines.items())

def parse(directory,game):
 rows=lines_for(directory,game);heads={};items={};previous='';previous_line=0
 source='https://callofduty.fandom.com/wiki/'+('' if game=='ghosts' else 'Challenges/')+TITLES[game]
 for number,line in rows:
  line=line.replace('\xa0','  ')
  heading=re.match(r'^(#{2,6})\s+(.+)',line)
  if heading:
   level=len(heading[1]);heads={k:v for k,v in heads.items() if k<level};heads[level]=clean(heading[2]);previous='';continue
  if game=='ghosts':
   # The source omits many tier thresholds. Only standalone, explicit operations
   # are imported; unknown '#' counts are deliberately left out.
   bullet=re.match(rf'^\s*\*\s+(.+?):\s*((?:{VERBS}|Fire|Force|Crush|Have|Be|Successfully|Infect)\b.+)$',line)
   if bullet and heads and '#' not in bullet[2] and any('Operations' in h for h in heads.values()) and not any('References' in h or 'Trivia' in h for h in heads.values()):
    name=clean(bullet[1]);requirement=clean(bullet[2]);category=' / '.join(heads[k] for k in sorted(heads))
    key=slug(category)+':'+slug(name)
    items[key]=dict(key=key,name=name,category=category,requirement=requirement,reward=None,target=None,icon=None,sourceUrl=source+'#'+heads[max(heads)].replace(' ','_'))
   continue
  if not heads or not line or line.startswith(('*','Note:','Challenge','Name ','Description','XP Reward','Image','This ','The ','After ','All ','I:','II:','III:','IV:','V:','VI:')):continue
  if any(re.search(r'Contents|Trivia|References|Gallery|Navigation|Explore|Follow|Community|Advertise',h) for h in heads.values()):continue
  # BO2 campaign objectives lack stable named table rows; exclude those.
  if game=='black-ops-2' and number<705:continue
  if game=='black-ops' and number<490:continue
  match=re.match(rf'^(.{{2,100}}?)\s+({VERBS}\b.+)$',line)
  if match:
   name,rest=match.groups()
   if re.match(rf'^{VERBS}\b',name) or 'Challenge How' in name:continue
  elif game=='advanced-warfare' and previous and number-previous_line<5 and re.match(rf'^{VERBS}\b',line):
   name,rest=previous,line
  else:
   if game=='advanced-warfare' and len(line)<70 and not re.search(r'\d|XP|Playercard|Emblem|Reticle',line):previous=clean(line);previous_line=number
   continue
  fields=re.split(r'\s{2,}',rest)
  requirement=clean(fields[0]);rewards=fields[1:]
  if len(fields)==1:
   match_xp=re.search(r'\s+(\d[\d,]*)(?:\s*XP)?$',requirement)
   if match_xp:rewards=[match_xp[1]+' XP'];requirement=requirement[:match_xp.start()]
  reward=clean(' · '.join(r for r in rewards if r not in ['None','Image'] and not re.fullmatch(r'\d+',r)))
  reward=re.sub(r'Image:\s*','',reward).replace('Image ','').strip(' ·')
  if not reward and any('Image' in r for r in rewards):reward='Calling card (see source)'
  if not requirement or len(requirement)>1000 or re.search(r'\bL\d+:',requirement):continue
  # Numeric progress only when the requirement starts with an explicit count.
  count=re.match(r'^(?:Get|Earn|Kill|Win|Destroy|Play|Capture|Complete|Absorb)\s+(\d[\d,]*)\s+',requirement)
  target=int(count[1].replace(',','')) if count else None
  category=' / '.join(heads[k] for k in sorted(heads))
  if game=='black-ops':category='Multiplayer / '+category
  name=clean(name).strip('* ')
  if re.search(r'Image:|\bXP\b',name):continue  # merged table cells, not a valid challenge name
  key=slug(category)+':'+slug(name)
  if key in items:continue
  items[key]=dict(key=key,name=name,category=category,requirement=requirement,reward=reward or None,target=target,icon=None,sourceUrl=source+'#'+heads[max(heads)].replace(' ','_'))
 return list(items.values())


def main():
 parser=argparse.ArgumentParser();parser.add_argument('--snapshots',type=Path,required=True);args=parser.parse_args()
 games={game:parse(args.snapshots,game) for game in TITLES}
 result=dict(source=dict(name='Call of Duty Wiki contributors',license='CC BY-SA',licenseUrl='https://www.fandom.com/licensing',attribution='https://callofduty.fandom.com/wiki/Call_of_Duty_Wiki:Copyrights',retrieved='2026-09-25',scope='Named challenge rows with explicit requirements. Incomplete and ambiguous source rows are excluded. Source coverage varies; these are not complete game inventories. Numeric targets are recorded only when explicit at the start of a requirement.'),games=games)
 Path('data/cod-challenges.json').write_text(json.dumps(result,ensure_ascii=False,indent=2)+'\n')
 print({game:len(rows) for game,rows in games.items()})

if __name__=='__main__':main()