#!/usr/bin/env node // Load on a vcs host over HTTP. Agents land file edits with no workspace // (one call each, its own file, on the latest version), as fast as they can, // across projects; readers read files meanwhile. Prints lands a second, // latencies and errors, then how long each project's git view took to catch up. // Usage: VCS_TOKEN=... node tests/load.mjs --host https://vcs.coline.dev \ // --projects o/a,o/b [--agents 50] [--seconds 60] [--readers 0] [--create] [--git on|off] // --agents is per project; --create makes missing projects; --git sets each one's git view first. const args = Object.fromEntries( process.argv.slice(2).reduce((out, a, i, all) => { if (a.startsWith("--")) out.push([a.slice(2), all[i + 1]?.startsWith("--") || all[i + 1] === undefined ? "1" : all[i + 1]]); return out; }, []), ); const HOST = (args.host ?? "https://vcs.coline.dev").replace(/\/+$/, ""); const PROJECTS = (args.projects ?? "").split(",").filter(Boolean); const AGENTS = Number(args.agents ?? 50); const SECONDS = Number(args.seconds ?? 60); const READERS = Number(args.readers ?? 0); const TOKEN = process.env.VCS_TOKEN; const RUN = `${Date.now().toString(36)}${process.pid.toString(36)}`; if (!TOKEN || PROJECTS.length === 0) { console.error("usage: VCS_TOKEN=... node tests/load.mjs --host URL --projects o/a,o/b [--agents N] [--seconds S] [--readers N] [--create] [--git on|off]"); process.exit(2); } const headers = { authorization: `Bearer ${TOKEN}`, "content-type": "application/json", "user-agent": "vcs-load" }; async function call(method, path, body) { const started = performance.now(); try { const response = await fetch(`${HOST}/v1${path}`, { method, headers, body: body === undefined ? undefined : JSON.stringify(body) }); const text = await response.text(); let json = null; try { json = JSON.parse(text); } catch { // Not JSON (a proxy's page). } return { status: response.status, json, ms: performance.now() - started }; } catch (error) { return { status: 0, json: { error: { code: "network", message: String(error?.cause?.code ?? error) } }, ms: performance.now() - started }; } } function pct(sorted, p) { return sorted.length ? sorted[Math.min(sorted.length - 1, Math.floor((p / 100) * sorted.length))] : 0; } for (const p of PROJECTS) { const [owner, name] = p.split("/"); let r = await call("GET", `/projects/${p}`); if (r.status === 404 && args.create) { r = await call("POST", "/projects", { owner, name }); if (r.status !== 200) throw new Error(`creating ${p}: ${r.status} ${JSON.stringify(r.json)}`); console.log(`created ${p}`); } else if (r.status !== 200) throw new Error(`${p}: ${r.status} ${JSON.stringify(r.json)}`); if (args.git) { const s = await call("PUT", `/projects/${p}/settings`, { git: args.git }); if (s.status !== 200) throw new Error(`${p} settings: ${s.status} ${JSON.stringify(s.json)}`); } } const lands = []; const reads = []; const errors = new Map(); const perProject = new Map(PROJECTS.map((p) => [p, 0])); const deadline = Date.now() + SECONDS * 1000; const samples = new Map(); const count = (key, message) => { errors.set(key, (errors.get(key) ?? 0) + 1); if (message && !samples.has(key)) samples.set(key, message); }; async function agent(p, a) { for (let i = 1; Date.now() < deadline; i++) { const r = await call("POST", `/projects/${p}/edits`, { title: `Load ${RUN} agent ${a} edit ${i}`, files: { [`load/${RUN}/agent-${a}.txt`]: `${a} ${i}\n` }, }); if (r.status === 200 && r.json?.outcome === "landed") { lands.push(r.ms); perProject.set(p, perProject.get(p) + 1); } else if (r.status === 200) count(r.json?.outcome ?? "?"); else { count(`${r.status} ${r.json?.error?.code ?? ""}`.trim(), r.json?.error?.message); // Back off a little on errors, like a client would. await new Promise((resolve) => setTimeout(resolve, 200)); } } } async function reader(p, k) { while (Date.now() < deadline) { const r = await call("GET", `/projects/${p}/file?path=${encodeURIComponent(`load/${RUN}/agent-${k % AGENTS}.txt`)}`); if (r.status === 200 || r.status === 404) reads.push(r.ms); else { count(`read ${r.status} ${r.json?.error?.code ?? ""}`.trim()); await new Promise((resolve) => setTimeout(resolve, 200)); } } } console.log(`${PROJECTS.length} projects × ${AGENTS} agents${READERS ? ` + ${READERS} readers` : ""} for ${SECONDS}s against ${HOST} (run ${RUN})`); const started = Date.now(); const work = []; for (const p of PROJECTS) { for (let a = 0; a < AGENTS; a++) work.push(agent(p, a)); for (let k = 0; k < READERS; k++) work.push(reader(p, k)); } const tick = setInterval(() => { const s = (Date.now() - started) / 1000; process.stdout.write(` ${s.toFixed(0)}s: ${lands.length} landed (${(lands.length / s).toFixed(0)}/s), ${[...errors.values()].reduce((x, y) => x + y, 0)} not\n`); }, 10_000); await Promise.all(work); clearInterval(tick); const elapsed = (Date.now() - started) / 1000; lands.sort((x, y) => x - y); reads.sort((x, y) => x - y); const ms = (v) => `${v.toFixed(0)} ms`; console.log(); console.log(`landed ${lands.length} in ${elapsed.toFixed(1)}s = ${(lands.length / elapsed).toFixed(0)} lands/s`); console.log(`per project ${(lands.length / elapsed / PROJECTS.length).toFixed(1)} lands/s (busiest ${(Math.max(...perProject.values()) / elapsed).toFixed(1)})`); console.log(`land latency p50 ${ms(pct(lands, 50))} p90 ${ms(pct(lands, 90))} p99 ${ms(pct(lands, 99))} max ${ms(lands.at(-1) ?? 0)}`); if (reads.length) console.log(`read latency p50 ${ms(pct(reads, 50))} p90 ${ms(pct(reads, 90))} p99 ${ms(pct(reads, 99))} (${(reads.length / elapsed).toFixed(0)}/s)`); console.log(`not landed ${errors.size ? [...errors].map(([k, v]) => `${k}: ${v}`).join(", ") : "none"}`); for (const [k, m] of samples) console.log(` ${k}: ${m}`); // The git view catching up. const behind = new Map(); const waitStart = Date.now(); for (;;) { const views = await Promise.all(PROJECTS.map((p) => call("GET", `/projects/${p}`))); const lagging = views.filter((v, i) => { if (v.json?.git_stopped) behind.set(PROJECTS[i], `paused: ${v.json.git_stopped}`); return v.json?.git && !v.json?.git_stopped && v.json?.git_behind > 0; }); if (lagging.length === 0 || Date.now() - waitStart > 180_000) { const off = views.filter((v) => !v.json?.git).length; console.log( `git view ${off === views.length ? "off" : `caught up ${((Date.now() - waitStart) / 1000).toFixed(1)}s after the run${lagging.length ? `; still behind: ${lagging.map((v) => `${v.json.owner}/${v.json.name} ${v.json.git_behind}`).join(", ")}` : ""}`}`, ); for (const [p, why] of behind) console.log(` ${p} ${why}`); break; } await new Promise((resolve) => setTimeout(resolve, 1000)); }