Stan Runge
42c9351ebd
All checks were successful
Deploy production / deploy (push) Successful in 1m12s
57 lines
2.0 KiB
TypeScript
57 lines
2.0 KiB
TypeScript
// A16 Google Trends "trending now" RSS (daily; terms of service: display with attribution).
|
|
// Keeps each day's list per country; a re-run on the same day refreshes rank and traffic.
|
|
import { getText, insertChunks, run, sql } from './common';
|
|
import { decode } from './htmltable';
|
|
|
|
const GEOS = ['NL', 'US', 'KR'];
|
|
|
|
/** "200+", "2K+", "1M+" → lower bound as a number. */
|
|
const traffic = (s: string | null) => {
|
|
const m = s?.replace(/,/g, '').match(/([\d.]+)\s*([KM])?/i);
|
|
if (!m) return null;
|
|
return Math.round(Number(m[1]) * (m[2]?.toUpperCase() === 'M' ? 1e6 : m[2] ? 1e3 : 1));
|
|
};
|
|
|
|
const tag = (xml: string, name: string) => {
|
|
const m = xml.match(new RegExp(`<${name}>([\\s\\S]*?)</${name}>`));
|
|
return m ? decode(m[1].replace(/^<!\[CDATA\[|\]\]>$/g, '').trim()) : null;
|
|
};
|
|
|
|
export async function gtrends(opts: { job: string }) {
|
|
await run('gtrends', opts.job, async () => {
|
|
let rows = 0;
|
|
for (const geo of GEOS) {
|
|
const xml = await getText(`https://trends.google.com/trending/rss?geo=${geo}`);
|
|
const today = new Intl.DateTimeFormat('en-CA', { timeZone: 'Europe/Amsterdam' }).format(
|
|
new Date()
|
|
);
|
|
const items = [...xml.matchAll(/<item>([\s\S]*?)<\/item>/g)].map((m, i) => {
|
|
const news = [...m[1].matchAll(/<ht:news_item>([\s\S]*?)<\/ht:news_item>/g)]
|
|
.map((n) => ({
|
|
title: tag(n[1], 'ht:news_item_title'),
|
|
source: tag(n[1], 'ht:news_item_source'),
|
|
url: tag(n[1], 'ht:news_item_url')
|
|
}))
|
|
.slice(0, 3);
|
|
const pub = tag(m[1], 'pubDate');
|
|
return {
|
|
geo,
|
|
date: today,
|
|
rank: i + 1,
|
|
query: tag(m[1], 'title') ?? '',
|
|
approx_traffic: traffic(tag(m[1], 'ht:approx_traffic')),
|
|
pub_date: pub ? new Date(pub) : null,
|
|
news: JSON.stringify(news)
|
|
};
|
|
});
|
|
const unique = [...new Map(items.filter((x) => x.query).map((x) => [x.query, x])).values()];
|
|
await sql.begin(async (tx) => {
|
|
await tx`delete from stats.gt_trending where geo = ${geo} and date = ${today}`;
|
|
await insertChunks(tx, 'gt_trending', unique);
|
|
});
|
|
rows += unique.length;
|
|
}
|
|
return rows;
|
|
});
|
|
}
|