Stan Runge
c2ba8febb4
Some checks failed
Test and deploy staging / staging (push) Failing after 5m29s
120 lines
6.2 KiB
Python
120 lines
6.2 KiB
Python
"""Reproducible factual PokéAPI import: regional dexes, learnsets and species biology.
|
|
|
|
Uses the same pinned revision as the encounter catalog. Reuses cached CSV files.
|
|
Never infers encounter absence from missing data.
|
|
"""
|
|
import argparse
|
|
import concurrent.futures
|
|
import csv
|
|
import json
|
|
from pathlib import Path
|
|
import urllib.request
|
|
|
|
REVISION = '575291cdb197a7e3a320297be276c9de4ef8401a'
|
|
TABLES = ['pokemon', 'pokemon_species', 'pokemon_abilities', 'ability_names', 'pokemon_egg_groups',
|
|
'egg_group_prose', 'growth_rates', 'pokemon_stats', 'pokemon_dex_numbers', 'pokedex_prose',
|
|
'pokedexes', 'pokedex_version_groups', 'versions', 'version_group_regions', 'regions',
|
|
'moves', 'move_names', 'type_names', 'move_damage_class_prose', 'move_targets',
|
|
'pokemon_moves', 'pokemon_move_method_prose']
|
|
|
|
|
|
def build(directory):
|
|
def rows(name):
|
|
with (directory / f'{name}.csv').open() as f:
|
|
yield from csv.DictReader(f)
|
|
|
|
def names(table, key):
|
|
return {int(r[key]): r['name'] for r in rows(table) if r['local_language_id'] == '9'}
|
|
|
|
def integer(v):
|
|
return int(v) if v else None
|
|
|
|
defaults = {int(r['id']): r for r in rows('pokemon') if r['is_default'] == '1'}
|
|
species_for = {pid: int(r['species_id']) for pid, r in defaults.items()}
|
|
abilities, eggs, evs = {}, {}, {}
|
|
ability_names = names('ability_names', 'ability_id')
|
|
egg_names = names('egg_group_prose', 'egg_group_id')
|
|
growth = {int(r['id']): r['identifier'].replace('-', ' ') for r in rows('growth_rates')}
|
|
for r in rows('pokemon_abilities'):
|
|
if int(r['pokemon_id']) in species_for:
|
|
abilities.setdefault(species_for[int(r['pokemon_id'])], []).append(dict(name=ability_names[int(r['ability_id'])], hidden=r['is_hidden'] == '1'))
|
|
for r in rows('pokemon_egg_groups'):
|
|
eggs.setdefault(int(r['species_id']), []).append(egg_names[int(r['egg_group_id'])])
|
|
for r in rows('pokemon_stats'):
|
|
pid = int(r['pokemon_id'])
|
|
if pid in species_for:
|
|
evs.setdefault(species_for[pid], [0]*6)[int(r['stat_id'])-1] = int(r['effort'])
|
|
by_species = {int(r['species_id']): r for r in defaults.values()}
|
|
facts = []
|
|
for s in rows('pokemon_species'):
|
|
number = int(s['id'])
|
|
p = by_species[number]
|
|
facts.append(dict(number=number, facts=dict(height=int(p['height']), weight=int(p['weight']),
|
|
baseExperience=integer(p['base_experience']), captureRate=int(s['capture_rate']),
|
|
baseHappiness=integer(s['base_happiness']), growthRate=growth[int(s['growth_rate_id'])],
|
|
genderRate=int(s['gender_rate']), hatchCounter=integer(s['hatch_counter']),
|
|
abilities=abilities.get(number, []), eggGroups=eggs.get(number, []), evYield=evs.get(number, []))))
|
|
|
|
versions = list(rows('versions'))
|
|
dex_names = names('pokedex_prose', 'pokedex_id')
|
|
dex_slugs = {int(r['id']): r['identifier'] for r in rows('pokedexes')}
|
|
dex_groups = {}
|
|
for r in rows('pokedex_version_groups'):
|
|
dex_groups.setdefault(int(r['pokedex_id']), []).append(int(r['version_group_id']))
|
|
dexes = []
|
|
for r in rows('pokemon_dex_numbers'):
|
|
dex = int(r['pokedex_id'])
|
|
if dex == 1:
|
|
continue
|
|
for v in versions:
|
|
if int(v['version_group_id']) in dex_groups.get(dex, []):
|
|
dexes.append([int(r['species_id']), int(v['id']), dex_slugs[dex], dex_names[dex], int(r['pokedex_number'])])
|
|
regions = {int(r['id']): r['identifier'] for r in rows('regions')}
|
|
group_regions = {}
|
|
for r in rows('version_group_regions'):
|
|
group_regions.setdefault(int(r['version_group_id']), []).append(regions[int(r['region_id'])])
|
|
games = [dict(sourceId=int(v['id']), versionGroup=int(v['version_group_id']), regions=group_regions.get(int(v['version_group_id']), [])) for v in versions]
|
|
move_names = names('move_names', 'move_id')
|
|
type_names = names('type_names', 'type_id')
|
|
damage = names('move_damage_class_prose', 'move_damage_class_id')
|
|
targets = {int(r['id']): r['identifier'].replace('-', ' ') for r in rows('move_targets')}
|
|
moves = [dict(id=int(r['id']), name=move_names.get(int(r['id']), r['identifier']), type=type_names[int(r['type_id'])],
|
|
category=damage[int(r['damage_class_id'])], power=integer(r['power']), accuracy=integer(r['accuracy']),
|
|
pp=integer(r['pp']), priority=int(r['priority']), target=targets[int(r['target_id'])]) for r in rows('moves')]
|
|
method_names = names('pokemon_move_method_prose', 'pokemon_move_method_id')
|
|
learnsets = {}
|
|
for r in rows('pokemon_moves'):
|
|
pid = int(r['pokemon_id'])
|
|
if pid not in species_for:
|
|
continue
|
|
key = (species_for[pid], int(r['version_group_id']))
|
|
learnsets.setdefault(key, set()).add((int(r['move_id']), method_names[int(r['pokemon_move_method_id'])], int(r['level'])))
|
|
return dict(source=dict(repository='https://github.com/PokeAPI/pokeapi', revision=REVISION,
|
|
scope='Default forms; regional dex numbers, species biology and version-specific learnsets. Move battle values are current snapshot values, not historical per-generation values.'),
|
|
facts=facts, dexes=dexes, games=games, moves=moves,
|
|
learnsets=[[species, group, sorted(entries)] for (species, group), entries in sorted(learnsets.items())])
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument('--cache', type=Path, default=Path('/tmp/pokemon-research-' + REVISION))
|
|
args = parser.parse_args()
|
|
args.cache.mkdir(parents=True, exist_ok=True)
|
|
def download(table):
|
|
path = args.cache / f'{table}.csv'
|
|
if path.exists():
|
|
return
|
|
with urllib.request.urlopen(f'https://raw.githubusercontent.com/PokeAPI/pokeapi/{REVISION}/data/v2/csv/{table}.csv', timeout=120) as response:
|
|
content = response.read()
|
|
path.write_bytes(content)
|
|
with concurrent.futures.ThreadPoolExecutor(max_workers=4) as executor:
|
|
list(executor.map(download, TABLES))
|
|
result = build(args.cache)
|
|
target = Path(__file__).resolve().parent.parent / 'data/pokemon-research.json'
|
|
target.write_text(json.dumps(result, ensure_ascii=False, separators=(',', ':')) + '\n')
|
|
print({k: len(v) for k, v in result.items() if isinstance(v, list)})
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|