Expand website and game trackers

scripts/build-pokemon-research.py

@@ -0,0 +1,119 @@
+"""Reproducible factual PokéAPI import: regional dexes, learnsets and species biology.
+
+Uses the same pinned revision as the encounter catalog. Reuses cached CSV files.
+Never infers encounter absence from missing data.
+"""
+import argparse
+import concurrent.futures
+import csv
+import json
+from pathlib import Path
+import urllib.request
+
+REVISION = '575291cdb197a7e3a320297be276c9de4ef8401a'
+TABLES = ['pokemon', 'pokemon_species', 'pokemon_abilities', 'ability_names', 'pokemon_egg_groups',
+          'egg_group_prose', 'growth_rates', 'pokemon_stats', 'pokemon_dex_numbers', 'pokedex_prose',
+          'pokedexes', 'pokedex_version_groups', 'versions', 'version_group_regions', 'regions',
+          'moves', 'move_names', 'type_names', 'move_damage_class_prose', 'move_targets',
+          'pokemon_moves', 'pokemon_move_method_prose']
+
+
+def build(directory):
+    def rows(name):
+        with (directory / f'{name}.csv').open() as f:
+            yield from csv.DictReader(f)
+
+    def names(table, key):
+        return {int(r[key]): r['name'] for r in rows(table) if r['local_language_id'] == '9'}
+
+    def integer(v):
+        return int(v) if v else None
+
+    defaults = {int(r['id']): r for r in rows('pokemon') if r['is_default'] == '1'}
+    species_for = {pid: int(r['species_id']) for pid, r in defaults.items()}
+    abilities, eggs, evs = {}, {}, {}
+    ability_names = names('ability_names', 'ability_id')
+    egg_names = names('egg_group_prose', 'egg_group_id')
+    growth = {int(r['id']): r['identifier'].replace('-', ' ') for r in rows('growth_rates')}
+    for r in rows('pokemon_abilities'):
+        if int(r['pokemon_id']) in species_for:
+            abilities.setdefault(species_for[int(r['pokemon_id'])], []).append(dict(name=ability_names[int(r['ability_id'])], hidden=r['is_hidden'] == '1'))
+    for r in rows('pokemon_egg_groups'):
+        eggs.setdefault(int(r['species_id']), []).append(egg_names[int(r['egg_group_id'])])
+    for r in rows('pokemon_stats'):
+        pid = int(r['pokemon_id'])
+        if pid in species_for:
+            evs.setdefault(species_for[pid], [0]*6)[int(r['stat_id'])-1] = int(r['effort'])
+    by_species = {int(r['species_id']): r for r in defaults.values()}
+    facts = []
+    for s in rows('pokemon_species'):
+        number = int(s['id'])
+        p = by_species[number]
+        facts.append(dict(number=number, facts=dict(height=int(p['height']), weight=int(p['weight']),
+            baseExperience=integer(p['base_experience']), captureRate=int(s['capture_rate']),
+            baseHappiness=integer(s['base_happiness']), growthRate=growth[int(s['growth_rate_id'])],
+            genderRate=int(s['gender_rate']), hatchCounter=integer(s['hatch_counter']),
+            abilities=abilities.get(number, []), eggGroups=eggs.get(number, []), evYield=evs.get(number, []))))
+
+    versions = list(rows('versions'))
+    dex_names = names('pokedex_prose', 'pokedex_id')
+    dex_slugs = {int(r['id']): r['identifier'] for r in rows('pokedexes')}
+    dex_groups = {}
+    for r in rows('pokedex_version_groups'):
+        dex_groups.setdefault(int(r['pokedex_id']), []).append(int(r['version_group_id']))
+    dexes = []
+    for r in rows('pokemon_dex_numbers'):
+        dex = int(r['pokedex_id'])
+        if dex == 1:
+            continue
+        for v in versions:
+            if int(v['version_group_id']) in dex_groups.get(dex, []):
+                dexes.append([int(r['species_id']), int(v['id']), dex_slugs[dex], dex_names[dex], int(r['pokedex_number'])])
+    regions = {int(r['id']): r['identifier'] for r in rows('regions')}
+    group_regions = {}
+    for r in rows('version_group_regions'):
+        group_regions.setdefault(int(r['version_group_id']), []).append(regions[int(r['region_id'])])
+    games = [dict(sourceId=int(v['id']), versionGroup=int(v['version_group_id']), regions=group_regions.get(int(v['version_group_id']), [])) for v in versions]
+    move_names = names('move_names', 'move_id')
+    type_names = names('type_names', 'type_id')
+    damage = names('move_damage_class_prose', 'move_damage_class_id')
+    targets = {int(r['id']): r['identifier'].replace('-', ' ') for r in rows('move_targets')}
+    moves = [dict(id=int(r['id']), name=move_names.get(int(r['id']), r['identifier']), type=type_names[int(r['type_id'])],
+        category=damage[int(r['damage_class_id'])], power=integer(r['power']), accuracy=integer(r['accuracy']),
+        pp=integer(r['pp']), priority=int(r['priority']), target=targets[int(r['target_id'])]) for r in rows('moves')]
+    method_names = names('pokemon_move_method_prose', 'pokemon_move_method_id')
+    learnsets = {}
+    for r in rows('pokemon_moves'):
+        pid = int(r['pokemon_id'])
+        if pid not in species_for:
+            continue
+        key = (species_for[pid], int(r['version_group_id']))
+        learnsets.setdefault(key, set()).add((int(r['move_id']), method_names[int(r['pokemon_move_method_id'])], int(r['level'])))
+    return dict(source=dict(repository='https://github.com/PokeAPI/pokeapi', revision=REVISION,
+        scope='Default forms; regional dex numbers, species biology and version-specific learnsets. Move battle values are current snapshot values, not historical per-generation values.'),
+        facts=facts, dexes=dexes, games=games, moves=moves,
+        learnsets=[[species, group, sorted(entries)] for (species, group), entries in sorted(learnsets.items())])
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--cache', type=Path, default=Path('/tmp/pokemon-research-' + REVISION))
+    args = parser.parse_args()
+    args.cache.mkdir(parents=True, exist_ok=True)
+    def download(table):
+        path = args.cache / f'{table}.csv'
+        if path.exists():
+            return
+        with urllib.request.urlopen(f'https://raw.githubusercontent.com/PokeAPI/pokeapi/{REVISION}/data/v2/csv/{table}.csv', timeout=120) as response:
+            content = response.read()
+        path.write_bytes(content)
+    with concurrent.futures.ThreadPoolExecutor(max_workers=4) as executor:
+        list(executor.map(download, TABLES))
+    result = build(args.cache)
+    target = Path(__file__).resolve().parent.parent / 'data/pokemon-research.json'
+    target.write_text(json.dumps(result, ensure_ascii=False, separators=(',', ':')) + '\n')
+    print({k: len(v) for k, v in result.items() if isinstance(v, list)})
+
+
+if __name__ == '__main__':
+    main()