stats step 3: elections, party positions and opinion feeds (schema, collectors, timers); world-opinions data moved into stats.wo_*

scripts/stats/remotezip.ts

@@ -0,0 +1,123 @@
+// Read single entries from a remote zip with HTTP range requests, so a 25 MB archive costs only
+// its central directory plus the files we need. Falls back to one full download when the server
+// ignores ranges. Handles stored and deflated entries (no zip64: the archives we read are small).
+import { inflateRawSync } from 'node:zlib';
+import { http } from './common';
+
+export type ZipEntry = {
+	name: string;
+	method: number;
+	compSize: number;
+	size: number;
+	offset: number;
+};
+
+export class RemoteZip {
+	private full?: Uint8Array;
+	private constructor(
+		readonly url: string,
+		readonly size: number,
+		readonly entries: ZipEntry[]
+	) {}
+
+	private static async range(url: string, start: number, end: number, size: number) {
+		const res = await http(url, {
+			headers: { range: `bytes=${start}-${end}` },
+			signal: AbortSignal.timeout(1_200_000)
+		});
+		const buf = new Uint8Array(await res.arrayBuffer());
+		if (res.status === 206) return { buf, whole: null as Uint8Array | null };
+		if (buf.length !== size) throw new Error(`short download of ${url}: ${buf.length}/${size}`);
+		return { buf: buf.subarray(start, end + 1), whole: buf };
+	}
+
+	static async open(url: string): Promise<RemoteZip> {
+		const head = await http(url, { method: 'HEAD' });
+		const size = Number(head.headers.get('content-length'));
+		if (!size) throw new Error(`no content-length for ${url}`);
+		const tailLen = Math.min(size, 66000);
+		const tail = await RemoteZip.range(url, size - tailLen, size - 1, size);
+		const t = tail.buf;
+		const v = new DataView(t.buffer, t.byteOffset, t.byteLength);
+		let eocd = -1;
+		for (let i = t.length - 22; i >= 0; i--)
+			if (v.getUint32(i, true) === 0x06054b50) {
+				eocd = i;
+				break;
+			}
+		if (eocd < 0) throw new Error(`not a zip: ${url}`);
+		const cdSize = v.getUint32(eocd + 12, true);
+		const cdOffset = v.getUint32(eocd + 16, true);
+		const cd = tail.whole
+			? tail.whole.subarray(cdOffset, cdOffset + cdSize)
+			: (await RemoteZip.range(url, cdOffset, cdOffset + cdSize - 1, size)).buf;
+		const c = new DataView(cd.buffer, cd.byteOffset, cd.byteLength);
+		const entries: ZipEntry[] = [];
+		const dec = new TextDecoder();
+		for (let p = 0; p + 46 <= cd.length && c.getUint32(p, true) === 0x02014b50;) {
+			const nameLen = c.getUint16(p + 28, true);
+			const extraLen = c.getUint16(p + 30, true);
+			const commentLen = c.getUint16(p + 32, true);
+			entries.push({
+				method: c.getUint16(p + 10, true),
+				compSize: c.getUint32(p + 20, true),
+				size: c.getUint32(p + 24, true),
+				offset: c.getUint32(p + 42, true),
+				name: dec.decode(cd.subarray(p + 46, p + 46 + nameLen))
+			});
+			p += 46 + nameLen + extraLen + commentLen;
+		}
+		const zip = new RemoteZip(url, size, entries);
+		if (tail.whole) zip.full = tail.whole;
+		return zip;
+	}
+
+	find(pattern: RegExp) {
+		return this.entries.filter((e) => pattern.test(e.name) && !e.name.endsWith('/'));
+	}
+
+	async read(entry: ZipEntry): Promise<Uint8Array> {
+		let local: Uint8Array;
+		if (this.full)
+			local = this.full.subarray(entry.offset, entry.offset + 30 + 65535 * 2 + entry.compSize);
+		else
+			local = (
+				await RemoteZip.range(
+					this.url,
+					entry.offset,
+					Math.min(
+						this.size - 1,
+						entry.offset + 30 + 1024 + entry.name.length * 4 + entry.compSize
+					),
+					this.size
+				)
+			).buf;
+		const v = new DataView(local.buffer, local.byteOffset, local.byteLength);
+		if (v.getUint32(0, true) !== 0x04034b50) throw new Error(`bad local header for ${entry.name}`);
+		const start = 30 + v.getUint16(26, true) + v.getUint16(28, true);
+		if (start + entry.compSize > local.length) {
+			// Unusually long local extra field: fetch exactly.
+			const exact = await RemoteZip.range(
+				this.url,
+				entry.offset + start,
+				entry.offset + start + entry.compSize - 1,
+				this.size
+			);
+			return this.inflate(entry, exact.buf);
+		}
+		return this.inflate(entry, local.subarray(start, start + entry.compSize));
+	}
+
+	private inflate(entry: ZipEntry, data: Uint8Array) {
+		if (entry.method === 0) return data;
+		if (entry.method !== 8)
+			throw new Error(`unsupported zip method ${entry.method} (${entry.name})`);
+		const out = new Uint8Array(inflateRawSync(data));
+		if (out.length !== entry.size) throw new Error(`size mismatch for ${entry.name}`);
+		return out;
+	}
+
+	async text(entry: ZipEntry, encoding = 'utf-8') {
+		return new TextDecoder(encoding).decode(await this.read(entry)).replace(/^\uFEFF/, '');
+	}
+}