cs-prices: full ByMykel catalog, every config, daily multi-source prices, item pages with working charts
All checks were successful
Deploy production / deploy (push) Successful in 1m18s
All checks were successful
Deploy production / deploy (push) Successful in 1m18s
- cs.skins catalog (weapons, knives, gloves, stickers, cases, agents, charms, …) refreshed weekly - every market config: wears in float range, StatTrak, Souvenir, Doppler/Gamma phases - cs.daily_prices (one row per config/stream/day, idempotent) from Skinport, Steam, CSFloat, Buff163 - cs.price_windows for 24h/7d/30d/90d sale aggregates, cs.fx_rates (ECB) - scripts/cs-prices.ts daily|catalog|prices; legacy ingest fixed for the new unique index - skin browser + /projects/cs-prices/[slug] with variant/wear/phase switcher and real charts - charts show points for short histories and pad sparse time axes
This commit is contained in:
624
scripts/cs-prices.ts
Normal file
624
scripts/cs-prices.ts
Normal file
@@ -0,0 +1,624 @@
|
||||
/**
|
||||
* CS2 catalog + daily prices (all configs) into cs.*. Runs inside the app container:
|
||||
*
|
||||
* bun scripts/cs-prices.ts daily catalog if older than 6 days, then today's prices
|
||||
* bun scripts/cs-prices.ts catalog refresh the full catalog from ByMykel/CSGO-API
|
||||
* bun scripts/cs-prices.ts prices today's prices from every bulk source
|
||||
*
|
||||
* One row per config per stream per UTC day (cs.daily_prices, idempotent upsert), so a rerun on
|
||||
* the same day just refreshes that day. Bulk endpoints only, no per-item Steam scraping:
|
||||
* - Skinport /v1/items (EUR, lowest listing + quantity, Doppler phases via `version`)
|
||||
* - Skinport /v1/sales/history (EUR, 24h/7d/30d/90d sale aggregates)
|
||||
* - prices.csgotrader.app (USD): Steam Market average sales (24h, else 7d), CSFloat and
|
||||
* Buff163 lowest listings (with Doppler phases)
|
||||
* USD→EUR comes from the ECB rate (Frankfurter) and is stored per day in cs.fx_rates.
|
||||
*/
|
||||
import postgres from 'postgres';
|
||||
import { dimensionsFromName } from '../src/lib/cs-prices/taxonomy';
|
||||
import { parseConfig, slugify } from '../src/lib/cs-prices/catalog';
|
||||
|
||||
if (!process.env.DB_URL) throw new Error('DB_URL is required');
|
||||
const sql = postgres(process.env.DB_URL, { max: 2, onnotice: () => {} });
|
||||
const mode = process.argv[2] ?? 'daily';
|
||||
const day = new Date().toISOString().slice(0, 10);
|
||||
const ua = { 'User-Agent': 'stanrunge.dev cs-prices/2.0 (daily personal price archive)' };
|
||||
const bymykel = 'https://raw.githubusercontent.com/ByMykel/CSGO-API/main/public/api/en';
|
||||
|
||||
type Skin = {
|
||||
id: string;
|
||||
slug: string;
|
||||
name: string;
|
||||
kind: string;
|
||||
weapon: string | null;
|
||||
pattern: string | null;
|
||||
category: string | null;
|
||||
rarity: string | null;
|
||||
rarity_color: string | null;
|
||||
image: string | null;
|
||||
min_float: number | null;
|
||||
max_float: number | null;
|
||||
collections: string[] | null;
|
||||
source: string;
|
||||
};
|
||||
type Item = {
|
||||
market_hash_name: string;
|
||||
phase: string | null;
|
||||
skin_id: string;
|
||||
name: string;
|
||||
icon_url: string | null;
|
||||
item_type: string | null;
|
||||
wear: string | null;
|
||||
stattrak: boolean;
|
||||
souvenir: boolean;
|
||||
};
|
||||
|
||||
async function getJson<T>(
|
||||
url: string,
|
||||
headers: Record<string, string> = {},
|
||||
tries = 4
|
||||
): Promise<T> {
|
||||
let last: unknown;
|
||||
for (let attempt = 0; attempt < tries; attempt++) {
|
||||
try {
|
||||
const response = await fetch(url, {
|
||||
headers: { ...ua, ...headers },
|
||||
signal: AbortSignal.timeout(180_000)
|
||||
});
|
||||
if (response.ok) return (await response.json()) as T;
|
||||
last = new Error(`HTTP ${response.status} for ${new URL(url).host}${new URL(url).pathname}`);
|
||||
if (response.status !== 429 && response.status < 500) break;
|
||||
} catch (cause) {
|
||||
last = cause;
|
||||
}
|
||||
await new Promise((resolve) => setTimeout(resolve, 5000 * 2 ** attempt));
|
||||
}
|
||||
throw last;
|
||||
}
|
||||
|
||||
const configKey = (hash: string, phase: string | null) => `${hash}\u0000${phase ?? ''}`;
|
||||
const cents = (value: unknown) =>
|
||||
typeof value === 'number' && Number.isFinite(value) && value >= 0 && value < 21_000_000
|
||||
? Math.round(value * 100)
|
||||
: null;
|
||||
|
||||
function kindFor(file: string, category?: string): string {
|
||||
if (file === 'skins_not_grouped') {
|
||||
if (category === 'Knives') return 'knife';
|
||||
if (category === 'Gloves') return 'gloves';
|
||||
return 'weapon';
|
||||
}
|
||||
return (
|
||||
{
|
||||
stickers: 'sticker',
|
||||
crates: 'container',
|
||||
agents: 'agent',
|
||||
keychains: 'charm',
|
||||
collectibles: 'collectible',
|
||||
graffiti: 'graffiti',
|
||||
patches: 'patch',
|
||||
music_kits: 'music-kit',
|
||||
keys: 'key'
|
||||
}[file] ?? 'other'
|
||||
);
|
||||
}
|
||||
|
||||
/** Feed-only names (e.g. sticker slabs, highlight charms) still get a skin page. */
|
||||
function kindFromName(hash: string) {
|
||||
const { base, starred } = parseConfig(hash);
|
||||
const head = base.split(' | ')[0];
|
||||
if (starred) return /Gloves|Wraps/.test(head) ? 'gloves' : 'knife';
|
||||
if (/^Sticker Slab$|^Charm$|Highlight/.test(head) || /^Souvenir Charm/.test(hash)) return 'charm';
|
||||
if (/^Sticker$/.test(head)) return 'sticker';
|
||||
if (/^(Sealed )?Graffiti$/.test(head)) return 'graffiti';
|
||||
if (/^Patch$/.test(head)) return 'patch';
|
||||
if (/^Music Kit$/.test(head)) return 'music-kit';
|
||||
if (/Case|Capsule|Package|Container|Collection/.test(base)) return 'container';
|
||||
if (dimensionsFromName(hash).category === 'weapon') return 'weapon';
|
||||
return 'other';
|
||||
}
|
||||
|
||||
async function upsertSkins(skins: Skin[]) {
|
||||
for (let offset = 0; offset < skins.length; offset += 2000) {
|
||||
const chunk = skins.slice(offset, offset + 2000);
|
||||
await sql`
|
||||
INSERT INTO cs.skins (id, slug, name, kind, weapon, pattern, category, rarity, rarity_color,
|
||||
image, min_float, max_float, collections, source, updated_at)
|
||||
SELECT id, slug, name, kind, weapon, pattern, category, rarity, rarity_color, image,
|
||||
min_float, max_float, collections, source, now()
|
||||
FROM jsonb_to_recordset(${sql.json(chunk as never)}::jsonb) AS x(id text, slug text, name text,
|
||||
kind text, weapon text, pattern text, category text, rarity text, rarity_color text,
|
||||
image text, min_float real, max_float real, collections text[], source text)
|
||||
ON CONFLICT (id) DO UPDATE SET name = EXCLUDED.name, kind = EXCLUDED.kind,
|
||||
weapon = coalesce(EXCLUDED.weapon, cs.skins.weapon),
|
||||
pattern = coalesce(EXCLUDED.pattern, cs.skins.pattern),
|
||||
category = coalesce(EXCLUDED.category, cs.skins.category),
|
||||
rarity = coalesce(EXCLUDED.rarity, cs.skins.rarity),
|
||||
rarity_color = coalesce(EXCLUDED.rarity_color, cs.skins.rarity_color),
|
||||
image = coalesce(EXCLUDED.image, cs.skins.image),
|
||||
min_float = coalesce(EXCLUDED.min_float, cs.skins.min_float),
|
||||
max_float = coalesce(EXCLUDED.max_float, cs.skins.max_float),
|
||||
collections = coalesce(EXCLUDED.collections, cs.skins.collections),
|
||||
source = CASE WHEN EXCLUDED.source = 'bymykel' THEN 'bymykel' ELSE cs.skins.source END,
|
||||
updated_at = CASE WHEN EXCLUDED.source = 'bymykel' THEN now() ELSE cs.skins.updated_at END`;
|
||||
}
|
||||
}
|
||||
|
||||
async function upsertItems(items: Item[]) {
|
||||
const ids = new Map<string, number>();
|
||||
for (let offset = 0; offset < items.length; offset += 2000) {
|
||||
const chunk = items.slice(offset, offset + 2000);
|
||||
const rows = await sql<{ id: number; market_hash_name: string; phase: string | null }[]>`
|
||||
INSERT INTO cs.items (market_hash_name, phase, skin_id, name, icon_url, item_type, wear,
|
||||
stattrak, souvenir)
|
||||
SELECT market_hash_name, phase, skin_id, name, icon_url, item_type, wear,
|
||||
coalesce(stattrak, false), coalesce(souvenir, false)
|
||||
FROM jsonb_to_recordset(${sql.json(chunk as never)}::jsonb) AS x(market_hash_name text,
|
||||
phase text, skin_id text, name text, icon_url text, item_type text, wear text,
|
||||
stattrak boolean, souvenir boolean)
|
||||
ON CONFLICT (market_hash_name, (coalesce(phase, ''))) DO UPDATE SET
|
||||
skin_id = EXCLUDED.skin_id,
|
||||
name = EXCLUDED.name,
|
||||
icon_url = coalesce(EXCLUDED.icon_url, cs.items.icon_url),
|
||||
item_type = coalesce(EXCLUDED.item_type, cs.items.item_type),
|
||||
wear = EXCLUDED.wear, stattrak = EXCLUDED.stattrak, souvenir = EXCLUDED.souvenir,
|
||||
last_seen_at = now()
|
||||
RETURNING id, market_hash_name, phase`;
|
||||
for (const row of rows) ids.set(configKey(row.market_hash_name, row.phase), row.id);
|
||||
// Facets for the /projects/cs-prices compare explorer.
|
||||
const dims: { item_id: number; key: string; value: string }[] = [];
|
||||
const byKey = new Map(
|
||||
chunk.map((item) => [configKey(item.market_hash_name, item.phase), item])
|
||||
);
|
||||
for (const row of rows) {
|
||||
const item = byKey.get(configKey(row.market_hash_name, row.phase));
|
||||
const facets = dimensionsFromName(row.market_hash_name);
|
||||
if (row.phase) facets.phase = row.phase;
|
||||
if (item?.item_type) facets.steam_type = item.item_type;
|
||||
for (const [key, value] of Object.entries(facets))
|
||||
if (value) dims.push({ item_id: row.id, key, value });
|
||||
}
|
||||
if (dims.length)
|
||||
await sql`
|
||||
INSERT INTO cs.item_dimensions (item_id, key, value)
|
||||
SELECT item_id, key, value FROM jsonb_to_recordset(${sql.json(dims as never)}::jsonb)
|
||||
AS x(item_id int, key text, value text)
|
||||
ON CONFLICT (item_id, key) DO UPDATE SET value = EXCLUDED.value`;
|
||||
}
|
||||
return ids;
|
||||
}
|
||||
|
||||
type Raw = Record<string, unknown> & {
|
||||
id: string;
|
||||
name: string;
|
||||
market_hash_name?: string | null;
|
||||
image?: string | null;
|
||||
phase?: string | null;
|
||||
skin_id?: string;
|
||||
weapon?: { name?: string } | null;
|
||||
pattern?: { name?: string } | null;
|
||||
category?: { name?: string } | null;
|
||||
rarity?: { name?: string; color?: string } | null;
|
||||
min_float?: number | null;
|
||||
max_float?: number | null;
|
||||
collections?: { name?: string }[] | null;
|
||||
type?: string | null;
|
||||
};
|
||||
|
||||
async function catalog() {
|
||||
const files = [
|
||||
'skins_not_grouped',
|
||||
'stickers',
|
||||
'crates',
|
||||
'agents',
|
||||
'keychains',
|
||||
'collectibles',
|
||||
'graffiti',
|
||||
'patches',
|
||||
'music_kits',
|
||||
'keys'
|
||||
];
|
||||
const skins = new Map<string, Skin>();
|
||||
const items = new Map<string, Item>();
|
||||
const slugOwner = new Map<string, string>();
|
||||
const existing = await sql<{ id: string; slug: string }[]>`SELECT id, slug FROM cs.skins`;
|
||||
for (const row of existing) slugOwner.set(row.slug, row.id);
|
||||
const skinFor = (base: string) => {
|
||||
// The grouping key is the base name; the id stays stable once assigned.
|
||||
const slug = slugify(base);
|
||||
return slugOwner.get(slug) ?? slug;
|
||||
};
|
||||
for (const file of files) {
|
||||
const rows = await getJson<Raw[]>(`${bymykel}/${file}.json`);
|
||||
console.log(`ByMykel ${file}: ${rows.length}`);
|
||||
for (const raw of rows) {
|
||||
if (!raw.market_hash_name) continue;
|
||||
const config = parseConfig(raw.market_hash_name);
|
||||
const kind = kindFor(file, raw.category?.name);
|
||||
const id = skinFor(config.base);
|
||||
slugOwner.set(slugify(config.base), id);
|
||||
const current = skins.get(id);
|
||||
const preferImage = !raw.phase || raw.phase === 'Phase 2';
|
||||
if (!current)
|
||||
skins.set(id, {
|
||||
id,
|
||||
slug: slugify(config.base),
|
||||
name: config.display,
|
||||
kind,
|
||||
weapon: raw.weapon?.name ?? null,
|
||||
pattern: raw.pattern?.name ?? null,
|
||||
category: raw.category?.name ?? raw.type ?? null,
|
||||
rarity: raw.rarity?.name ?? null,
|
||||
rarity_color: raw.rarity?.color ?? null,
|
||||
image: preferImage ? (raw.image ?? null) : null,
|
||||
min_float: raw.min_float ?? null,
|
||||
max_float: raw.max_float ?? null,
|
||||
collections: raw.collections?.map((c) => c.name ?? '').filter(Boolean) ?? null,
|
||||
source: 'bymykel'
|
||||
});
|
||||
else if (!current.image && preferImage && raw.image) current.image = raw.image;
|
||||
const add = (phase: string | null) =>
|
||||
items.set(configKey(raw.market_hash_name!, phase), {
|
||||
market_hash_name: raw.market_hash_name!,
|
||||
phase,
|
||||
skin_id: id,
|
||||
name: phase ? `${raw.market_hash_name} - ${phase}` : raw.market_hash_name!,
|
||||
icon_url: raw.image ?? null,
|
||||
item_type: raw.category?.name ?? raw.type ?? kind,
|
||||
wear: config.wear,
|
||||
stattrak: config.stattrak,
|
||||
souvenir: config.souvenir
|
||||
});
|
||||
add(raw.phase ?? null);
|
||||
// The phase-less listing carries Steam's and other phase-blind prices.
|
||||
if (raw.phase && !items.has(configKey(raw.market_hash_name, null))) add(null);
|
||||
}
|
||||
}
|
||||
const itemList = [...items.values()];
|
||||
for (const skin of skins.values())
|
||||
if (!skin.image)
|
||||
skin.image =
|
||||
itemList.find((item) => item.skin_id === skin.id && item.icon_url)?.icon_url ?? null;
|
||||
// Phase-less rows of Doppler listings should not show one phase's image.
|
||||
for (const item of items.values())
|
||||
if (!item.phase && /Doppler/.test(item.name)) item.icon_url = null;
|
||||
await upsertSkins([...skins.values()]);
|
||||
const ids = await upsertItems([...items.values()]);
|
||||
// Older Steam-only rows: link them to a skin by name.
|
||||
await sql`UPDATE cs.items i SET skin_id = s.id FROM cs.skins s
|
||||
WHERE i.skin_id IS NULL AND s.slug = regexp_replace(regexp_replace(lower(regexp_replace(
|
||||
regexp_replace(regexp_replace(i.market_hash_name, '^★ ', ''), '^(StatTrak™ |Souvenir )', ''),
|
||||
' \\((Factory New|Minimal Wear|Field-Tested|Well-Worn|Battle-Scarred)\\)$', '')),
|
||||
'[^a-z0-9]+', '-', 'g'), '^-+|-+$', '', 'g')`;
|
||||
console.log(`Catalog: ${skins.size} skins, ${ids.size} configs`);
|
||||
}
|
||||
|
||||
type Row = { hash: string; phase: string | null; cents: number; volume: number | null };
|
||||
|
||||
async function ensureItems(rows: Row[]) {
|
||||
const known = new Map<string, number>();
|
||||
for (const row of await sql<{ id: number; market_hash_name: string; phase: string | null }[]>`
|
||||
SELECT id, market_hash_name, phase FROM cs.items`)
|
||||
known.set(configKey(row.market_hash_name, row.phase), row.id);
|
||||
const missing = new Map<string, Item>();
|
||||
const newSkins = new Map<string, Skin>();
|
||||
const slugs = new Map(
|
||||
(await sql<{ id: string; slug: string }[]>`SELECT id, slug FROM cs.skins`).map((r) => [
|
||||
r.slug,
|
||||
r.id
|
||||
])
|
||||
);
|
||||
for (const row of rows) {
|
||||
const key = configKey(row.hash, row.phase);
|
||||
if (known.has(key) || missing.has(key)) continue;
|
||||
// Untranslated tokens and truncated names in the feeds are not real listings.
|
||||
if (/^#|\|\s*$|^\s*$/.test(row.hash) || row.hash.length > 200) continue;
|
||||
const config = parseConfig(row.hash);
|
||||
const slug = slugify(config.base);
|
||||
let skinId = slugs.get(slug);
|
||||
if (!skinId) {
|
||||
skinId = slug;
|
||||
slugs.set(slug, skinId);
|
||||
newSkins.set(skinId, {
|
||||
id: skinId,
|
||||
slug,
|
||||
name: config.display,
|
||||
kind: kindFromName(row.hash),
|
||||
weapon: null,
|
||||
pattern: null,
|
||||
category: null,
|
||||
rarity: null,
|
||||
rarity_color: null,
|
||||
image: null,
|
||||
min_float: null,
|
||||
max_float: null,
|
||||
collections: null,
|
||||
source: 'market'
|
||||
});
|
||||
}
|
||||
missing.set(key, {
|
||||
market_hash_name: row.hash,
|
||||
phase: row.phase,
|
||||
skin_id: skinId,
|
||||
name: row.phase ? `${row.hash} - ${row.phase}` : row.hash,
|
||||
icon_url: null,
|
||||
item_type: null,
|
||||
wear: config.wear,
|
||||
stattrak: config.stattrak,
|
||||
souvenir: config.souvenir
|
||||
});
|
||||
}
|
||||
if (newSkins.size) await upsertSkins([...newSkins.values()]);
|
||||
if (missing.size) {
|
||||
for (const [key, id] of await upsertItems([...missing.values()])) known.set(key, id);
|
||||
console.log(`New configs from price feeds: ${missing.size} (${newSkins.size} new skins)`);
|
||||
}
|
||||
return known;
|
||||
}
|
||||
|
||||
async function streamId(source: string, currency: string, kind: string) {
|
||||
const [row] = await sql<{ id: number }[]>`
|
||||
INSERT INTO cs.price_streams (source, currency, kind) VALUES (${source}, ${currency}, ${kind})
|
||||
ON CONFLICT (source, currency, kind) DO UPDATE SET source = EXCLUDED.source RETURNING id`;
|
||||
return row.id;
|
||||
}
|
||||
|
||||
async function saveDaily(source: string, currency: string, kind: string, rows: Row[]) {
|
||||
const stream = await streamId(source, currency, kind);
|
||||
const ids = await ensureItems(rows);
|
||||
const dedup = new Map<number, Row>();
|
||||
for (const row of rows) {
|
||||
const id = ids.get(configKey(row.hash, row.phase));
|
||||
if (id) dedup.set(id, row);
|
||||
}
|
||||
const entries = [...dedup.entries()];
|
||||
for (let offset = 0; offset < entries.length; offset += 5000) {
|
||||
const chunk = entries.slice(offset, offset + 5000);
|
||||
await sql`
|
||||
INSERT INTO cs.daily_prices (item_id, stream_id, day, price_cents, volume)
|
||||
SELECT item_id, ${stream}::smallint, ${day}::date, price_cents, volume
|
||||
FROM unnest(${chunk.map(([id]) => id)}::int[], ${chunk.map(([, r]) => r.cents)}::int[],
|
||||
${chunk.map(([, r]) => r.volume)}::int[]) AS x(item_id, price_cents, volume)
|
||||
ON CONFLICT (item_id, stream_id, day) DO UPDATE SET
|
||||
price_cents = EXCLUDED.price_cents, volume = EXCLUDED.volume`;
|
||||
}
|
||||
await sql`UPDATE cs.price_streams SET last_fetched_at = now(), last_rows = ${entries.length},
|
||||
last_error = NULL WHERE id = ${stream}`;
|
||||
console.log(`${source} ${kind}: ${entries.length} configs for ${day}`);
|
||||
return ids;
|
||||
}
|
||||
|
||||
type Window = {
|
||||
item: number;
|
||||
span: string;
|
||||
median: number | null;
|
||||
avg: number | null;
|
||||
min: number | null;
|
||||
max: number | null;
|
||||
volume: number | null;
|
||||
};
|
||||
async function saveWindows(source: string, currency: string, windows: Window[]) {
|
||||
for (let offset = 0; offset < windows.length; offset += 5000) {
|
||||
const chunk = windows.slice(offset, offset + 5000);
|
||||
await sql`
|
||||
INSERT INTO cs.price_windows (item_id, source, currency, span, median_cents, avg_cents,
|
||||
min_cents, max_cents, volume, updated_at)
|
||||
SELECT item_id, ${source}, ${currency}, span, median_cents, avg_cents, min_cents, max_cents,
|
||||
volume, now()
|
||||
FROM unnest(${chunk.map((w) => w.item)}::int[], ${chunk.map((w) => w.span)}::text[],
|
||||
${chunk.map((w) => w.median)}::int[], ${chunk.map((w) => w.avg)}::int[],
|
||||
${chunk.map((w) => w.min)}::int[], ${chunk.map((w) => w.max)}::int[],
|
||||
${chunk.map((w) => w.volume)}::int[])
|
||||
AS x(item_id, span, median_cents, avg_cents, min_cents, max_cents, volume)
|
||||
ON CONFLICT (item_id, source, span) DO UPDATE SET median_cents = EXCLUDED.median_cents,
|
||||
avg_cents = EXCLUDED.avg_cents, min_cents = EXCLUDED.min_cents,
|
||||
max_cents = EXCLUDED.max_cents, volume = EXCLUDED.volume, updated_at = now()`;
|
||||
}
|
||||
}
|
||||
|
||||
async function fx() {
|
||||
for (const url of [
|
||||
'https://api.frankfurter.dev/v1/latest?base=USD&symbols=EUR',
|
||||
'https://api.frankfurter.app/latest?from=USD&to=EUR'
|
||||
]) {
|
||||
try {
|
||||
const data = await getJson<{ rates?: { EUR?: number } }>(url, {}, 2);
|
||||
if (data.rates?.EUR) {
|
||||
await sql`INSERT INTO cs.fx_rates (day, usd_eur) VALUES (${day}, ${data.rates.EUR})
|
||||
ON CONFLICT (day) DO UPDATE SET usd_eur = EXCLUDED.usd_eur`;
|
||||
console.log(`FX USD→EUR ${data.rates.EUR}`);
|
||||
return;
|
||||
}
|
||||
} catch (cause) {
|
||||
console.warn(`FX ${new URL(url).host}: ${String(cause)}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function guarded(
|
||||
source: string,
|
||||
currency: string,
|
||||
kinds: string[],
|
||||
run: () => Promise<void>
|
||||
) {
|
||||
try {
|
||||
await run();
|
||||
return true;
|
||||
} catch (cause) {
|
||||
console.error(`${source} failed: ${String(cause)}`);
|
||||
for (const kind of kinds)
|
||||
await sql`UPDATE cs.price_streams SET last_error = ${String(cause).slice(0, 500)}
|
||||
WHERE source = ${source} AND currency = ${currency} AND kind = ${kind}`;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
const spans = { last_24_hours: '24h', last_7_days: '7d', last_30_days: '30d', last_90_days: '90d' };
|
||||
|
||||
async function prices() {
|
||||
await fx();
|
||||
const results: boolean[] = [];
|
||||
const br = { 'Accept-Encoding': 'br' };
|
||||
results.push(
|
||||
await guarded('skinport', 'EUR', ['lowest-listing'], async () => {
|
||||
const items = await getJson<
|
||||
{
|
||||
market_hash_name: string;
|
||||
version: string | null;
|
||||
min_price: number | null;
|
||||
quantity: number;
|
||||
}[]
|
||||
>('https://api.skinport.com/v1/items?app_id=730¤cy=EUR', br);
|
||||
await saveDaily(
|
||||
'skinport',
|
||||
'EUR',
|
||||
'lowest-listing',
|
||||
items.flatMap((item) => {
|
||||
const value = cents(item.min_price);
|
||||
return value == null
|
||||
? []
|
||||
: [
|
||||
{
|
||||
hash: item.market_hash_name,
|
||||
phase: item.version || null,
|
||||
cents: value,
|
||||
volume: item.quantity ?? null
|
||||
}
|
||||
];
|
||||
})
|
||||
);
|
||||
})
|
||||
);
|
||||
results.push(
|
||||
await guarded('skinport', 'EUR', ['sale-median-24h'], async () => {
|
||||
type Agg = {
|
||||
min: number | null;
|
||||
max: number | null;
|
||||
avg: number | null;
|
||||
median: number | null;
|
||||
volume: number;
|
||||
};
|
||||
const history = await getJson<
|
||||
(Record<string, Agg> & { market_hash_name: string; version: string | null })[]
|
||||
>('https://api.skinport.com/v1/sales/history?app_id=730¤cy=EUR', br);
|
||||
const rows: Row[] = [];
|
||||
for (const entry of history) {
|
||||
const day24 = entry.last_24_hours;
|
||||
const value = cents(day24?.median);
|
||||
if (value != null)
|
||||
rows.push({
|
||||
hash: entry.market_hash_name,
|
||||
phase: entry.version || null,
|
||||
cents: value,
|
||||
volume: day24.volume
|
||||
});
|
||||
}
|
||||
const ids = await saveDaily('skinport', 'EUR', 'sale-median-24h', rows);
|
||||
const windows: Window[] = [];
|
||||
const allIds = await ensureItems(
|
||||
history.map((e) => ({
|
||||
hash: e.market_hash_name,
|
||||
phase: e.version || null,
|
||||
cents: 0,
|
||||
volume: 0
|
||||
}))
|
||||
);
|
||||
for (const entry of history) {
|
||||
const item = (ids.get(configKey(entry.market_hash_name, entry.version || null)) ??
|
||||
allIds.get(configKey(entry.market_hash_name, entry.version || null)))!;
|
||||
if (!item) continue;
|
||||
for (const [field, span] of Object.entries(spans)) {
|
||||
const agg = entry[field];
|
||||
if (!agg || !agg.volume) continue;
|
||||
windows.push({
|
||||
item,
|
||||
span,
|
||||
median: cents(agg.median),
|
||||
avg: cents(agg.avg),
|
||||
min: cents(agg.min),
|
||||
max: cents(agg.max),
|
||||
volume: agg.volume
|
||||
});
|
||||
}
|
||||
}
|
||||
await saveWindows('skinport', 'EUR', windows);
|
||||
console.log(`skinport windows: ${windows.length}`);
|
||||
})
|
||||
);
|
||||
results.push(
|
||||
await guarded('steam-market', 'USD', ['sale-avg'], async () => {
|
||||
const steam = await getJson<Record<string, Record<string, number | null>>>(
|
||||
'https://prices.csgotrader.app/latest/steam.json'
|
||||
);
|
||||
const rows: Row[] = [];
|
||||
for (const [hash, value] of Object.entries(steam)) {
|
||||
// Latest average sale price: the last 24 hours, else the last 7 days.
|
||||
const price = cents(value?.last_24h ?? value?.last_7d);
|
||||
if (price != null) rows.push({ hash, phase: null, cents: price, volume: null });
|
||||
}
|
||||
const ids = await saveDaily('steam-market', 'USD', 'sale-avg', rows);
|
||||
const all = await ensureItems(
|
||||
Object.keys(steam).map((hash) => ({ hash, phase: null, cents: 0, volume: 0 }))
|
||||
);
|
||||
const windows: Window[] = [];
|
||||
for (const [hash, value] of Object.entries(steam)) {
|
||||
const item = ids.get(configKey(hash, null)) ?? all.get(configKey(hash, null));
|
||||
if (!item || !value) continue;
|
||||
for (const span of ['24h', '7d', '30d', '90d']) {
|
||||
const avg = cents(value[`last_${span}`]);
|
||||
if (avg != null)
|
||||
windows.push({ item, span, median: null, avg, min: null, max: null, volume: null });
|
||||
}
|
||||
}
|
||||
await saveWindows('steam-market', 'USD', windows);
|
||||
})
|
||||
);
|
||||
for (const [source, file] of [
|
||||
['csfloat', 'csfloat'],
|
||||
['buff163', 'buff163']
|
||||
] as const) {
|
||||
results.push(
|
||||
await guarded(source, 'USD', ['lowest-listing'], async () => {
|
||||
type Entry = {
|
||||
price?: number;
|
||||
starting_at?: { price?: number | null; doppler?: Record<string, number> } | null;
|
||||
doppler?: Record<string, number>;
|
||||
};
|
||||
const feed = await getJson<Record<string, Entry>>(
|
||||
`https://prices.csgotrader.app/latest/${file}.json`
|
||||
);
|
||||
const rows: Row[] = [];
|
||||
for (const [hash, entry] of Object.entries(feed)) {
|
||||
const base = cents(entry?.starting_at?.price ?? entry?.price);
|
||||
if (base != null) rows.push({ hash, phase: null, cents: base, volume: null });
|
||||
const phases = entry?.starting_at?.doppler ?? entry?.doppler;
|
||||
for (const [phase, value] of Object.entries(phases ?? {})) {
|
||||
const price = cents(value);
|
||||
if (price != null) rows.push({ hash, phase, cents: price, volume: null });
|
||||
}
|
||||
}
|
||||
await saveDaily(source, 'USD', 'lowest-listing', rows);
|
||||
})
|
||||
);
|
||||
}
|
||||
if (!results.some(Boolean)) throw new Error('Every price source failed');
|
||||
const [summary] = await sql<{ rows: number; items: number }[]>`
|
||||
SELECT count(*)::int AS rows, count(DISTINCT item_id)::int AS items
|
||||
FROM cs.daily_prices WHERE day = ${day}`;
|
||||
console.log(`Today (${day}): ${summary.rows} price rows across ${summary.items} configs`);
|
||||
}
|
||||
|
||||
try {
|
||||
if (mode === 'catalog') await catalog();
|
||||
else if (mode === 'prices') await prices();
|
||||
else if (mode === 'daily') {
|
||||
const [state] = await sql<{ fresh: boolean }[]>`
|
||||
SELECT coalesce(max(updated_at) > now() - interval '6 days', false) AS fresh
|
||||
FROM cs.skins WHERE source = 'bymykel'`;
|
||||
if (!state.fresh) await catalog();
|
||||
await prices();
|
||||
} else throw new Error('Usage: cs-prices.ts daily|catalog|prices');
|
||||
await sql`ANALYZE cs.items`;
|
||||
await sql`ANALYZE cs.daily_prices`;
|
||||
} finally {
|
||||
await sql.end();
|
||||
}
|
||||
@@ -1,5 +1,6 @@
|
||||
/**
|
||||
* Steam Community Market ingestion. Run after db:migrate:cs-prices.
|
||||
* Legacy Steam Community Market ingestion (per item, rate limited). The daily archive is
|
||||
* scripts/cs-prices.ts; this stays for one-off Steam history imports with a logged-in session.
|
||||
*
|
||||
* bun scripts/ingest-cs-prices.ts catalog [--limit=500] [--start=0]
|
||||
* STEAM_SESSIONID=... STEAM_LOGIN_SECURE=... bun scripts/ingest-cs-prices.ts history [--limit=100] [--start=0]
|
||||
@@ -87,10 +88,10 @@ async function itemId(
|
||||
const [row] = await sql<{ id: number }[]>`
|
||||
INSERT INTO cs.items (market_hash_name, name, icon_url, item_type)
|
||||
VALUES (${name}, ${displayName}, ${iconUrl ?? null}, ${itemType ?? null})
|
||||
ON CONFLICT (market_hash_name) DO UPDATE SET
|
||||
ON CONFLICT (market_hash_name, (coalesce(phase, ''))) DO UPDATE SET
|
||||
name = EXCLUDED.name,
|
||||
icon_url = COALESCE(EXCLUDED.icon_url, cs_market_items.icon_url),
|
||||
item_type = COALESCE(EXCLUDED.item_type, cs_market_items.item_type),
|
||||
icon_url = COALESCE(EXCLUDED.icon_url, cs.items.icon_url),
|
||||
item_type = COALESCE(EXCLUDED.item_type, cs.items.item_type),
|
||||
last_seen_at = now()
|
||||
RETURNING id`;
|
||||
const inferred = dimensionsFromName(name);
|
||||
@@ -281,7 +282,7 @@ try {
|
||||
else if (mode === 'history') await history();
|
||||
else if (mode === 'import' && process.argv[3]) await importFile(process.argv[3]);
|
||||
else throw new Error('Usage: ingest-cs-prices.ts catalog|history|import <file>');
|
||||
await sql`ANALYZE cs_market_items, cs_item_dimensions, cs_price_observations`;
|
||||
await sql`ANALYZE cs.items, cs.item_dimensions, cs.price_observations`;
|
||||
} finally {
|
||||
await sql.end();
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user