import { createReadStream } from 'node:fs'; import { writeFile } from 'node:fs/promises'; import { createInterface } from 'node:readline'; import { createHash } from 'node:crypto'; // Input is a Mongo JSON-lines export. Never copy credentials or account assets. const [input, asOf] = process.argv.slice(2); if (!input || !asOf || !Number.isFinite(Date.parse(asOf))) { throw new Error('Usage: node select-cloud-rise-profiles.mjs '); } const cutoff = new Date(asOf), day = cutoff.getUTCDate(); cutoff.setUTCDate(1); cutoff.setUTCMonth(cutoff.getUTCMonth() - 2); cutoff.setUTCDate(Math.min(day, new Date(Date.UTC(cutoff.getUTCFullYear(), cutoff.getUTCMonth() + 1, 0)).getUTCDate())); const groups = { real: { count: 0, rows: [] }, game: { count: 0, rows: [] } }; const seen = new Set(), hash = createHash('sha256'); let total = 0, inactive = 0, invalidProfile = 0, duplicates = 0, seed = 20260924; function random() { seed ^= seed << 13; seed ^= seed >>> 17; seed ^= seed << 5; return (seed >>> 0) / 4294967296; } function avatarKind(row) { const icon = String(row.useravatarIcon || ''), avatar = String(row.useravatar || ''); // Match CloudRisePanel.avatar: an HTTPS icon takes precedence over useravatar. const displayed = /^https:\/\//.test(icon) ? icon : avatar; if (/^https:\/\//.test(displayed)) { try { if (new URL(displayed).hostname) return 'real'; } catch {} } if (/^(?:icon_)?\d+$/.test(avatar) && !/^https?:/.test(icon)) return 'game'; return null; } const stream = createReadStream(input); stream.on('data', chunk => hash.update(chunk)); for await (const line of createInterface({ input: stream, crlfDelay: Infinity })) { if (!line.trim()) continue; const row = JSON.parse(line.replace(/^\uFEFF/, '')); total++; if (!Number.isSafeInteger(row.taskTime) || row.taskTime <= 0 || row.taskTime >= cutoff.getTime()) continue; inactive++; const kind = avatarKind(row); if (!kind || !Number.isSafeInteger(row.onlyId) || row.onlyId <= 0 || typeof row.username !== 'string' || !row.username.trim()) { invalidProfile++; continue; } if (seen.has(row.onlyId)) { duplicates++; continue; } seen.add(row.onlyId); const profile = { onlyId: row.onlyId, taskTime: row.taskTime, username: row.username, useravatar: String(row.useravatar || ''), useravatarIcon: String(row.useravatarIcon || 'icon_0') }; const group = groups[kind]; group.count++; // Keep enough unbiased candidates to fill all 5,000 slots if one category is scarce. if (group.rows.length < 5000) group.rows.push(profile); else { const index = Math.floor(random() * group.count); if (index < 5000) group.rows[index] = profile; } } if (groups.real.count + groups.game.count < 5000) throw new Error('Fewer than 5,000 eligible unique players'); // Shuffle the reservoirs before taking a smaller sample from either category. for (const group of Object.values(groups)) { for (let i = group.rows.length - 1; i > 0; i--) { const j = Math.floor(random() * (i + 1)); [group.rows[i], group.rows[j]] = [group.rows[j], group.rows[i]]; } } let realCount = Math.min(4000, groups.real.count), gameCount = Math.min(1000, groups.game.count); realCount += Math.min(5000 - realCount - gameCount, groups.real.count - realCount); gameCount += 5000 - realCount - gameCount; const profiles = [...groups.real.rows.slice(0, realCount), ...groups.game.rows.slice(0, gameCount)] .sort((a, b) => a.onlyId - b.onlyId); const metadata = { sampledAt: new Date(asOf).toISOString(), inactiveBefore: cutoff.toISOString(), sourceSha256: hash.digest('hex'), total, inactive, invalidProfile, duplicates, eligible: { real: groups.real.count, game: groups.game.count }, selected: { real: realCount, game: gameCount }, seed: 20260924 }; await writeFile(new URL('../data/cloudRiseProfiles.json', import.meta.url), JSON.stringify({ ...metadata, profiles }, null, 2) + '\n'); console.log(JSON.stringify(metadata, null, 2));