games ingest: send dates as ISO strings (the app pool uses drizzle's date serializers), pause Leetify on 429 and resume next poll, FACEIT 403 from the server is a skip (history imported by backfill), weekly FACEIT

src/lib/games-ingest/faceit.ts

@@ -1,6 +1,15 @@
 import { fetchJson } from './http';
 import { addSnapshot, hoursSince, lastSnapshot, upsertMatch } from './store';
-import { int, num, sleep, type Account, type Db, type MatchInput, type RunOptions } from './types';
+import {
+	int,
+	num,
+	sleep,
+	SourceError,
+	type Account,
+	type Db,
+	type MatchInput,
+	type RunOptions
+} from './types';
 
 // FACEIT match history from the stats service faceit.com itself uses (no key needed):
 // /api/stats/v1/stats/time/users/{playerId}/games/{game}?size&page lists every FACEIT
@@ -90,12 +99,22 @@ export async function pollFaceit(sql: Db, account: Account, opts: RunOptions) {
 	for (const game of games) {
 		const size = opts.backfill ? 100 : 20;
 		for (let page = 0; page < (opts.backfill ? 50 : 1); page++) {
-			const { body } = await fetchJson<Row[]>(
+			let body: Row[] | null;
+			try {
+				({ body } = await fetchJson<Row[]>(
 					`${BASE}/time/users/${id}/games/${game}?size=${size}&page=${page}`,
-				{
+					{ headers: { 'User-Agent': 'Mozilla/5.0 (stanrunge.dev games tracker)' } }
-					headers: { 'User-Agent': 'Mozilla/5.0 (stanrunge.dev games tracker)' }
+				));
+			} catch (e) {
+				// faceit.com (Cloudflare) answers 403 to datacenter IPs such as the server; the
+				// history is closed and was imported from elsewhere, so this is not an error.
+				if (e instanceof SourceError && e.status === 403)
+					return {
+						skipped: true,
+						note: 'faceit.com blocks the server IP (403); history imported by backfill'
+					};
+				throw e;
 			}
-			);
 			const rows = Array.isArray(body) ? body : [];
 			for (const r of rows) {
 				const m = parseFaceitRow(r);

src/lib/games-ingest/leetify.ts

@@ -4,6 +4,7 @@ import {
 	int,
 	num,
 	sleep,
+	SourceError,
 	type Account,
 	type Db,
 	type GameEnv,
@@ -131,10 +132,15 @@ export async function pollLeetify(sql: Db, account: Account, env: GameEnv, opts:
 				await fetchJson<LeetifyMatch>(`${BASE}/v2/matches/${encodeURIComponent(m.id)}`, { headers })
 			).body;
 		} catch (e) {
+			// Rate limited (no key): stop here; oldest-first, so the next poll resumes at this match.
+			if (e instanceof SourceError && e.status === 429) {
+				opts.log?.(`[games] leetify rate limited after ${added} new matches; resuming next poll`);
+				break;
+			}
 			opts.log?.(`[games] leetify details ${m.id}: ${(e as Error).message}`);
 		}
 		if (await upsertMatch(sql, parseLeetifyMatch(m, steamId, details))) added++;
-		await sleep(env.LEETIFY_API_KEY ? 250 : 1100);
+		await sleep(env.LEETIFY_API_KEY ? 250 : 2500);
 	}
 	// Profile (ranks, ratings, totals) once a day, as a dated snapshot.
 	const last = await lastSnapshot(sql, 'cs2', 'leetify', 'profile');

src/lib/games-ingest/run.ts

@@ -12,7 +12,7 @@ export const SOURCE_ORDER = ['leetify', 'faceit', 'steam', 'pubg', 'ubisoft'] as
 /** Minimum hours between polls per source (the poller ticks every 20 minutes). */
 const DUE_HOURS: Record<string, number> = {
 	leetify: 0.3,
-	faceit: 6,
+	faceit: 24 * 7, // FACEIT history is closed (last match 2023); weekly is plenty
 	steam: 0.3,
 	pubg: 0.3,
 	ubisoft: 0.3

src/lib/games-ingest/store.ts

@@ -8,7 +8,7 @@ export async function upsertMatch(sql: Db, m: MatchInput): Promise<boolean> {
 			score_opp, placement, team_count, kills, deaths, assists, damage, headshots, rating,
 			rating_label, match_count, share_code, extra, raw
 		) values (
-			${m.game}, ${m.source}, ${m.externalId}, ${m.playedAt}, ${m.durationS ?? null},
+			${m.game}, ${m.source}, ${m.externalId}, ${m.playedAt.toISOString()}, ${m.durationS ?? null},
 			${m.mode ?? null}, ${m.map ?? null}, ${m.result ?? null}, ${m.scoreOwn ?? null},
 			${m.scoreOpp ?? null}, ${m.placement ?? null}, ${m.teamCount ?? null}, ${m.kills ?? null},
 			${m.deaths ?? null}, ${m.assists ?? null}, ${m.damage ?? null}, ${m.headshots ?? null},
@@ -103,7 +103,7 @@ export async function addSnapshot(
 		insert into games.snapshots (game, source, kind, data)
 		values (${game}, ${source}, ${kind}, ${sql.json(data as never)})
 		returning id, taken_at as "takenAt"`;
-	return row;
+	return { ...row, takenAt: new Date(row.takenAt) };
 }
 
 export async function lastSnapshot(sql: Db, game: string, source: string, kind: string) {
@@ -111,7 +111,7 @@ export async function lastSnapshot(sql: Db, game: string, source: string, kind:
 		select id, taken_at as "takenAt", data from games.snapshots
 		where game = ${game} and source = ${source} and kind = ${kind}
 		order by taken_at desc limit 1`;
-	return row ?? null;
+	return row ? { ...row, takenAt: new Date(row.takenAt) } : null;
 }
 
 export async function saveState(sql: Db, account: Account, patch: Record<string, unknown>) {
@@ -119,4 +119,7 @@ export async function saveState(sql: Db, account: Account, patch: Record<string,
 	await sql`update games.accounts set state = ${sql.json(account.state as never)} where id = ${account.id}`;
 }
 
-export const hoursSince = (d: Date | null) => (d ? (Date.now() - d.getTime()) / 3.6e6 : Infinity);
+// Timestamps come back as strings on the app pool (drizzle swaps the date parsers) and as
+// Dates on a plain postgres.js client; Date params are sent as ISO strings for the same reason.
+export const hoursSince = (d: Date | string | null) =>
+	d ? (Date.now() - new Date(d).getTime()) / 3.6e6 : Infinity;

src/lib/games-ingest/types.ts

@@ -22,7 +22,7 @@ export type Account = {
 	externalId: string;
 	name: string | null;
 	state: Record<string, unknown>;
-	lastSuccessAt: Date | null;
+	lastSuccessAt: Date | string | null;
 };
 
 export type PlayerInput = {