website/scripts/stats/remotezip.ts

// Read single entries from a remote zip with HTTP range requests, so a 25 MB archive costs only
// its central directory plus the files we need. Falls back to one full download when the server
// ignores ranges. Handles stored and deflated entries (no zip64: the archives we read are small).
import { inflateRawSync } from 'node:zlib';
import { http } from './common';

export type ZipEntry = {
	name: string;
	method: number;
	compSize: number;
	size: number;
	offset: number;
};

export class RemoteZip {
	private full?: Uint8Array;
	private constructor(
		readonly url: string,
		readonly size: number,
		readonly entries: ZipEntry[]
	) {}

	private static async range(url: string, start: number, end: number, size: number) {
		const res = await http(url, {
			headers: { range: `bytes=${start}-${end}` },
			signal: AbortSignal.timeout(1_200_000)
		});
		const buf = new Uint8Array(await res.arrayBuffer());
		if (res.status === 206) return { buf, whole: null as Uint8Array | null };
		if (buf.length !== size) throw new Error(`short download of ${url}: ${buf.length}/${size}`);
		return { buf: buf.subarray(start, end + 1), whole: buf };
	}

	static async open(url: string): Promise<RemoteZip> {
		const head = await http(url, { method: 'HEAD' });
		const size = Number(head.headers.get('content-length'));
		if (!size) throw new Error(`no content-length for ${url}`);
		const tailLen = Math.min(size, 66000);
		const tail = await RemoteZip.range(url, size - tailLen, size - 1, size);
		const t = tail.buf;
		const v = new DataView(t.buffer, t.byteOffset, t.byteLength);
		let eocd = -1;
		for (let i = t.length - 22; i >= 0; i--)
			if (v.getUint32(i, true) === 0x06054b50) {
				eocd = i;
				break;
			}
		if (eocd < 0) throw new Error(`not a zip: ${url}`);
		const cdSize = v.getUint32(eocd + 12, true);
		const cdOffset = v.getUint32(eocd + 16, true);
		const cd = tail.whole
			? tail.whole.subarray(cdOffset, cdOffset + cdSize)
			: (await RemoteZip.range(url, cdOffset, cdOffset + cdSize - 1, size)).buf;
		const c = new DataView(cd.buffer, cd.byteOffset, cd.byteLength);
		const entries: ZipEntry[] = [];
		const dec = new TextDecoder();
		for (let p = 0; p + 46 <= cd.length && c.getUint32(p, true) === 0x02014b50;) {
			const nameLen = c.getUint16(p + 28, true);
			const extraLen = c.getUint16(p + 30, true);
			const commentLen = c.getUint16(p + 32, true);
			entries.push({
				method: c.getUint16(p + 10, true),
				compSize: c.getUint32(p + 20, true),
				size: c.getUint32(p + 24, true),
				offset: c.getUint32(p + 42, true),
				name: dec.decode(cd.subarray(p + 46, p + 46 + nameLen))
			});
			p += 46 + nameLen + extraLen + commentLen;
		}
		const zip = new RemoteZip(url, size, entries);
		if (tail.whole) zip.full = tail.whole;
		return zip;
	}

	find(pattern: RegExp) {
		return this.entries.filter((e) => pattern.test(e.name) && !e.name.endsWith('/'));
	}

	async read(entry: ZipEntry): Promise<Uint8Array> {
		let local: Uint8Array;
		if (this.full)
			local = this.full.subarray(entry.offset, entry.offset + 30 + 65535 * 2 + entry.compSize);
		else
			local = (
				await RemoteZip.range(
					this.url,
					entry.offset,
					Math.min(
						this.size - 1,
						entry.offset + 30 + 1024 + entry.name.length * 4 + entry.compSize
					),
					this.size
				)
			).buf;
		const v = new DataView(local.buffer, local.byteOffset, local.byteLength);
		if (v.getUint32(0, true) !== 0x04034b50) throw new Error(`bad local header for ${entry.name}`);
		const start = 30 + v.getUint16(26, true) + v.getUint16(28, true);
		if (start + entry.compSize > local.length) {
			// Unusually long local extra field: fetch exactly.
			const exact = await RemoteZip.range(
				this.url,
				entry.offset + start,
				entry.offset + start + entry.compSize - 1,
				this.size
			);
			return this.inflate(entry, exact.buf);
		}
		return this.inflate(entry, local.subarray(start, start + entry.compSize));
	}

	private inflate(entry: ZipEntry, data: Uint8Array) {
		if (entry.method === 0) return data;
		if (entry.method !== 8)
			throw new Error(`unsupported zip method ${entry.method} (${entry.name})`);
		const out = new Uint8Array(inflateRawSync(data));
		if (out.length !== entry.size) throw new Error(`size mismatch for ${entry.name}`);
		return out;
	}

	async text(entry: ZipEntry, encoding = 'utf-8') {
		return new TextDecoder(encoding).decode(await this.read(entry)).replace(/^\uFEFF/, '');
	}
}