From d414e7190e05c8994a0930f7c609749d691b2302 Mon Sep 17 00:00:00 2001 From: Stan Runge Date: Sun, 25 Oct 2026 16:15:13 +0000 Subject: [PATCH] games ingest: send dates as ISO strings (the app pool uses drizzle's date serializers), pause Leetify on 429 and resume next poll, FACEIT 403 from the server is a skip (history imported by backfill), weekly FACEIT --- src/lib/games-ingest/faceit.ts | 33 ++++++++++++++++++++++++++------- src/lib/games-ingest/leetify.ts | 8 +++++++- src/lib/games-ingest/run.ts | 2 +- src/lib/games-ingest/store.ts | 11 +++++++---- src/lib/games-ingest/types.ts | 2 +- 5 files changed, 42 insertions(+), 14 deletions(-) diff --git a/src/lib/games-ingest/faceit.ts b/src/lib/games-ingest/faceit.ts index 6b876fa..bf3d96e 100644 --- a/src/lib/games-ingest/faceit.ts +++ b/src/lib/games-ingest/faceit.ts @@ -1,6 +1,15 @@ import { fetchJson } from './http'; import { addSnapshot, hoursSince, lastSnapshot, upsertMatch } from './store'; -import { int, num, sleep, type Account, type Db, type MatchInput, type RunOptions } from './types'; +import { + int, + num, + sleep, + SourceError, + type Account, + type Db, + type MatchInput, + type RunOptions +} from './types'; // FACEIT match history from the stats service faceit.com itself uses (no key needed): // /api/stats/v1/stats/time/users/{playerId}/games/{game}?size&page lists every FACEIT @@ -90,12 +99,22 @@ export async function pollFaceit(sql: Db, account: Account, opts: RunOptions) { for (const game of games) { const size = opts.backfill ? 100 : 20; for (let page = 0; page < (opts.backfill ? 50 : 1); page++) { - const { body } = await fetchJson( - `${BASE}/time/users/${id}/games/${game}?size=${size}&page=${page}`, - { - headers: { 'User-Agent': 'Mozilla/5.0 (stanrunge.dev games tracker)' } - } - ); + let body: Row[] | null; + try { + ({ body } = await fetchJson( + `${BASE}/time/users/${id}/games/${game}?size=${size}&page=${page}`, + { headers: { 'User-Agent': 'Mozilla/5.0 (stanrunge.dev games tracker)' } } + )); + } catch (e) { + // faceit.com (Cloudflare) answers 403 to datacenter IPs such as the server; the + // history is closed and was imported from elsewhere, so this is not an error. + if (e instanceof SourceError && e.status === 403) + return { + skipped: true, + note: 'faceit.com blocks the server IP (403); history imported by backfill' + }; + throw e; + } const rows = Array.isArray(body) ? body : []; for (const r of rows) { const m = parseFaceitRow(r); diff --git a/src/lib/games-ingest/leetify.ts b/src/lib/games-ingest/leetify.ts index 7b4f332..d914d5c 100644 --- a/src/lib/games-ingest/leetify.ts +++ b/src/lib/games-ingest/leetify.ts @@ -4,6 +4,7 @@ import { int, num, sleep, + SourceError, type Account, type Db, type GameEnv, @@ -131,10 +132,15 @@ export async function pollLeetify(sql: Db, account: Account, env: GameEnv, opts: await fetchJson(`${BASE}/v2/matches/${encodeURIComponent(m.id)}`, { headers }) ).body; } catch (e) { + // Rate limited (no key): stop here; oldest-first, so the next poll resumes at this match. + if (e instanceof SourceError && e.status === 429) { + opts.log?.(`[games] leetify rate limited after ${added} new matches; resuming next poll`); + break; + } opts.log?.(`[games] leetify details ${m.id}: ${(e as Error).message}`); } if (await upsertMatch(sql, parseLeetifyMatch(m, steamId, details))) added++; - await sleep(env.LEETIFY_API_KEY ? 250 : 1100); + await sleep(env.LEETIFY_API_KEY ? 250 : 2500); } // Profile (ranks, ratings, totals) once a day, as a dated snapshot. const last = await lastSnapshot(sql, 'cs2', 'leetify', 'profile'); diff --git a/src/lib/games-ingest/run.ts b/src/lib/games-ingest/run.ts index d36e3f5..9518ca0 100644 --- a/src/lib/games-ingest/run.ts +++ b/src/lib/games-ingest/run.ts @@ -12,7 +12,7 @@ export const SOURCE_ORDER = ['leetify', 'faceit', 'steam', 'pubg', 'ubisoft'] as /** Minimum hours between polls per source (the poller ticks every 20 minutes). */ const DUE_HOURS: Record = { leetify: 0.3, - faceit: 6, + faceit: 24 * 7, // FACEIT history is closed (last match 2023); weekly is plenty steam: 0.3, pubg: 0.3, ubisoft: 0.3 diff --git a/src/lib/games-ingest/store.ts b/src/lib/games-ingest/store.ts index 06a1bd6..67582e2 100644 --- a/src/lib/games-ingest/store.ts +++ b/src/lib/games-ingest/store.ts @@ -8,7 +8,7 @@ export async function upsertMatch(sql: Db, m: MatchInput): Promise { score_opp, placement, team_count, kills, deaths, assists, damage, headshots, rating, rating_label, match_count, share_code, extra, raw ) values ( - ${m.game}, ${m.source}, ${m.externalId}, ${m.playedAt}, ${m.durationS ?? null}, + ${m.game}, ${m.source}, ${m.externalId}, ${m.playedAt.toISOString()}, ${m.durationS ?? null}, ${m.mode ?? null}, ${m.map ?? null}, ${m.result ?? null}, ${m.scoreOwn ?? null}, ${m.scoreOpp ?? null}, ${m.placement ?? null}, ${m.teamCount ?? null}, ${m.kills ?? null}, ${m.deaths ?? null}, ${m.assists ?? null}, ${m.damage ?? null}, ${m.headshots ?? null}, @@ -103,7 +103,7 @@ export async function addSnapshot( insert into games.snapshots (game, source, kind, data) values (${game}, ${source}, ${kind}, ${sql.json(data as never)}) returning id, taken_at as "takenAt"`; - return row; + return { ...row, takenAt: new Date(row.takenAt) }; } export async function lastSnapshot(sql: Db, game: string, source: string, kind: string) { @@ -111,7 +111,7 @@ export async function lastSnapshot(sql: Db, game: string, source: string, kind: select id, taken_at as "takenAt", data from games.snapshots where game = ${game} and source = ${source} and kind = ${kind} order by taken_at desc limit 1`; - return row ?? null; + return row ? { ...row, takenAt: new Date(row.takenAt) } : null; } export async function saveState(sql: Db, account: Account, patch: Record) { @@ -119,4 +119,7 @@ export async function saveState(sql: Db, account: Account, patch: Record (d ? (Date.now() - d.getTime()) / 3.6e6 : Infinity); +// Timestamps come back as strings on the app pool (drizzle swaps the date parsers) and as +// Dates on a plain postgres.js client; Date params are sent as ISO strings for the same reason. +export const hoursSince = (d: Date | string | null) => + d ? (Date.now() - new Date(d).getTime()) / 3.6e6 : Infinity; diff --git a/src/lib/games-ingest/types.ts b/src/lib/games-ingest/types.ts index 83574cd..281e5a7 100644 --- a/src/lib/games-ingest/types.ts +++ b/src/lib/games-ingest/types.ts @@ -22,7 +22,7 @@ export type Account = { externalId: string; name: string | null; state: Record; - lastSuccessAt: Date | null; + lastSuccessAt: Date | string | null; }; export type PlayerInput = {