import { readFile } from 'node:fs/promises'; import { pathToFileURL } from 'node:url'; // One labeled field per row, including omissions and rule failures. // eligible means your non-score checks passed, not that the value is correct. type Row = { confidence: number | null; eligible: boolean; correct: boolean }; export function summarize(input: unknown, threshold: number) { if (!Number.isFinite(threshold) || threshold < 0 || threshold > 1) { throw new Error('Threshold must be in [0, 1]'); } if (!Array.isArray(input) || input.length === 0) throw new Error('Expected labeled rows'); const rows: Row[] = input.map((row: unknown) => { if (typeof row !== 'object' || row === null) throw new Error('Invalid row'); const r = row as Record; if ( typeof r.correct !== 'boolean' || typeof r.eligible !== 'boolean' || (r.confidence !== null && (typeof r.confidence !== 'number' || !Number.isFinite(r.confidence) || r.confidence < 0 || r.confidence > 1)) ) { throw new Error('Each row needs correct, eligible and confidence (number or null)'); } return { correct: r.correct, eligible: r.eligible, confidence: r.confidence }; }); const accepted = rows.filter( (r) => r.eligible && r.confidence !== null && r.confidence >= threshold, ); const errors = accepted.filter((r) => !r.correct).length; return { threshold, total: rows.length, accepted: accepted.length, errors, coverage: accepted.length / rows.length, empiricalRisk: accepted.length === 0 ? null : errors / accepted.length, }; } if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { const path = process.argv[2]; if (!path) throw new Error('Usage: npx tsx thresholds.ts labeled-fields.json'); const rows: unknown = JSON.parse(await readFile(path, 'utf8')); // Candidate thresholds for exploration only; none is a recommended policy. console.table([0.5, 0.7, 0.9, 0.95].map((t) => summarize(rows, t))); }