158 lines · 7 KBtext/plain; charset=utf-8
Onset: version control where nobody waits and nothing gets lost. The on command, its core and onset-server.
| 6 | # Usage: tests/bench.sh [path/to/target/release] |
| 7 | set -uo pipefail |
| 8 |
| 9 | ROOT=$(cd "$(dirname "$0")/.." && pwd) |
| 10 | BIN=${1:-$ROOT/target/release} |
| 11 | VCS=$BIN/on |
| 12 | WORK=$(mktemp -d) |
| 13 | export ONSET_SAVER=0 ONSET_HOME=$WORK/home ONSET_NAME=Bench ONSET_EMAIL=bench@example.com |
| 14 | export GIT_AUTHOR_NAME=Bench GIT_AUTHOR_EMAIL=bench@example.com GIT_COMMITTER_NAME=Bench GIT_COMMITTER_EMAIL=bench@example.com |
| 15 | PORT=$((20000 + RANDOM % 20000)) |
| 16 | HOST="127.0.0.1:$PORT" |
| 17 | SERVER_PID= |
| 18 | cleanup() { |
| 19 | [ -n "$SERVER_PID" ] && kill "$SERVER_PID" 2>/dev/null |
| 20 | rm -rf "$WORK" |
| 21 | } |
| 22 | trap cleanup EXIT |
| 23 | cd "$WORK" |
| 24 | g() { git -c init.defaultBranch=main -c commit.gpgsign=false -c advice.detachedHead=false "$@"; } |
| 25 | now() { python3 -c 'import time; print(time.time())'; } |
| 26 | since() { python3 -c "import time; print(f'{time.time() - $1:.2f}')"; } |
| 27 | best() { # fastest of 5 runs, in ms |
| 28 | python3 - "$@" <<'EOF' |
| 29 | import subprocess, sys, time |
| 30 | best = 1e9 |
| 31 | for _ in range(5): |
| 32 | t = time.time(); subprocess.run(sys.argv[1:], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL); best = min(best, time.time() - t) |
| 33 | print(f"{best * 1000:.0f}") |
| 34 | EOF |
| 35 | } |
| 36 | files() { # folder, count |
| 37 | python3 - "$1" "$2" <<'EOF' |
| 38 | import os, sys |
| 39 | root, n = sys.argv[1], int(sys.argv[2]) |
| 40 | for i in range(n): |
| 41 | d = os.path.join(root, "src", f"m{i // 20:04d}") |
| 42 | os.makedirs(d, exist_ok=True) |
| 43 | with open(os.path.join(d, f"f{i:05d}.ts"), "w") as f: |
| 44 | f.write(f"export const v{i} = {i};\n" * 30) |
| 45 | EOF |
| 46 | } |
| 47 | history() { # git folder, commits: each changes three lines of hot.ts (500 lines) and all of count.ts |
| 48 | python3 - "$2" <<'EOF' | (cd "$1" && g fast-import --quiet) |
| 49 | import random, sys |
| 50 | n = int(sys.argv[1]) |
| 51 | random.seed(1) |
| 52 | lines = [f"export const line{i} = {i};" for i in range(500)] |
| 53 | out = sys.stdout.buffer |
| 54 | def data(b): |
| 55 | out.write(b"data %d\n" % len(b) + b + b"\n") |
| 56 | for c in range(n): |
| 57 | for _ in range(3): |
| 58 | i = random.randrange(len(lines)) |
| 59 | lines[i] = f"export const line{i} = {c}; // commit {c}" |
| 60 | out.write(b"commit refs/heads/main\ncommitter Bench <bench@example.com> %d +0000\n" % (1700000000 + c * 60)) |
| 61 | data(f"Commit {c}".encode()) |
| 62 | out.write(b"M 100644 inline hot.ts\n") |
| 63 | data(("\n".join(lines) + "\n").encode()) |
| 64 | out.write(b"M 100644 inline count.ts\n") |
| 65 | data(f"export const n = {c};\n".encode()) |
| 66 | EOF |
| 67 | } |
| 68 | row() { printf '%-52s %12s %12s\n' "$1" "$2" "$3"; } |
| 69 |
| 70 | "$BIN/onset-server" user add bench --name Bench --email bench@example.com --data "$WORK/server" > token |
| 71 | "$BIN/onset-server" serve --data "$WORK/server" --listen "$HOST" --threads 32 2>/dev/null & |
| 72 | SERVER_PID=$! |
| 73 | for _ in $(seq 50); do curl -s "http://$HOST/.well-known/onset.json" >/dev/null && break; sleep 0.1; done |
| 74 | "$VCS" login "$HOST" --token "$(cat token)" >/dev/null |
| 75 |
| 76 | echo "Onset $("$VCS" --version | cut -d' ' -f2) vs $(git --version) ($(nproc) cores, $(uname -s))" |
| 77 | echo |
| 78 | row "" "git" "Onset" |
| 79 |
| 80 | # The same 20,000-file project in both. |
| 81 | mkdir gseed && files gseed 20000 && (cd gseed && g init -q && g add -A && g commit -qm "20k files") |
| 82 | g clone -q --bare gseed big.git |
| 83 | "$VCS" init vseed >/dev/null && files vseed 20000 && "$VCS" -C vseed land -m "20k files" >/dev/null |
| 84 | "$VCS" -C vseed publish "$HOST/bench/big" >/dev/null |
| 85 | g clone -q big.git gbig |
| 86 | "$VCS" get "$HOST/bench/big" vbig >/dev/null |
| 87 | sleep 1.5 |
| 88 |
| 89 | row "status, 20,000 files, nothing changed (ms)" "$(best git -C gbig status)" "$(best "$VCS" -C vbig status)" |
| 90 | printf 'edit\n' >> gbig/src/m0500/f10000.ts |
| 91 | printf 'edit\n' >> vbig/src/m0500/f10000.ts |
| 92 | row "status, 20,000 files, one file changed (ms)" "$(best git -C gbig status)" "$(best "$VCS" -C vbig status)" |
| 93 | git -C gbig checkout -q -- . && "$VCS" -C vbig restore src/m0500/f10000.ts >/dev/null |
| 94 |
| 95 | t=$(now); g clone -q --no-local "file://$WORK/big.git" gfresh; tg=$(since "$t") |
| 96 | t=$(now); ONSET_HOME=$WORK/fresh "$VCS" login "$HOST" --token "$(cat token)" >/dev/null; ONSET_HOME=$WORK/fresh "$VCS" get "$HOST/bench/big" vfresh >/dev/null; tv=$(since "$t") |
| 97 | row "first copy on a new machine, 20,000 files (s)" "$tg" "$tv" |
| 98 | t=$(now); git -C gbig worktree add -q ../gwt >/dev/null 2>&1; tg=$(since "$t") |
| 99 | t=$(now); "$VCS" get "$HOST/bench/big" vbig2 >/dev/null; tv=$(since "$t") |
| 100 | row "second working copy, same machine (s)" "$tg" "$tv" |
| 101 |
| 102 | # One small change, start to finish, ten times: commit + push vs land. |
| 103 | gl=() |
| 104 | vl=() |
| 105 | for i in $(seq 1 10); do |
| 106 | printf 'change %s\n' "$i" >> gbig/README.md |
| 107 | t=$(now); (cd gbig && g add -A && g commit -qm "Change $i" && g push -q origin HEAD:main); gl+=("$(since "$t")") |
| 108 | printf 'change %s\n' "$i" >> vbig/README.md |
| 109 | t=$(now); "$VCS" -C vbig land -m "Change $i" >/dev/null; vl+=("$(since "$t")") |
| 110 | done |
| 111 | med() { printf '%s\n' "$@" | sort -n | sed -n "$(( ($# + 1) / 2 ))p"; } |
| 112 | row "one change: commit + push vs land, median (s)" "$(med "${gl[@]}")" "$(med "${vl[@]}")" |
| 113 |
| 114 | # A long history: the same 5,000 commits in both (Onset's imported from git's, |
| 115 | # then published and fetched into a fresh workspace, as a teammate's would be). |
| 116 | mkdir glong && (cd glong && g init -q) && history glong 5000 && git -C glong reset -q --hard main |
| 117 | "$VCS" import glong vlocal >/dev/null && "$VCS" -C vlocal publish "$HOST/bench/long" >/dev/null |
| 118 | "$VCS" get "$HOST/bench/long" vlong >/dev/null |
| 119 | "$VCS" -C vlong log -n 1 >/dev/null |
| 120 | row "log, newest 100 of 5,000 versions (ms)" "$(best git -C glong log -n 100)" "$(best "$VCS" -C vlong log -n 100)" |
| 121 | row "log of one file changed in all 5,000 (ms)" "$(best git -C glong log -n 100 -- hot.ts)" "$(best "$VCS" -C vlong log -n 100 hot.ts)" |
| 122 | row "blame, 500 lines changed over 5,000 versions (ms)" "$(best git -C glong blame hot.ts)" "$(best "$VCS" -C vlong blame hot.ts)" |
| 123 |
| 124 | # Many agents, one branch, at once. Each makes one change to its own file. |
| 125 | race() { # agents |
| 126 | local n=$1 |
| 127 | rm -rf race && mkdir race && cd race |
| 128 | mkdir seed && files seed 1500 && (cd seed && g init -q && g add -A && g commit -qm base) |
| 129 | g clone -q --bare seed server.git |
| 130 | for i in $(seq 1 "$n"); do g clone -q server.git "g$i" && printf '%s\n' "$i" > "g$i/agent-$i.txt" && (cd "g$i" && g add -A && g commit -qm "Agent $i"); done |
| 131 | "$VCS" init vseed >/dev/null && files vseed 1500 && "$VCS" -C vseed land -m base >/dev/null && "$VCS" -C vseed publish "$HOST/bench/race$n" >/dev/null |
| 132 | for i in $(seq 1 "$n"); do "$VCS" get "$HOST/bench/race$n" "v$i" >/dev/null && printf '%s\n' "$i" > "v$i/agent-$i.txt"; done |
| 133 | sync |
| 134 | local t pids=() |
| 135 | t=$(now) |
| 136 | for i in $(seq 1 "$n"); do |
| 137 | (cd "g$i"; tries=0; until g push -q origin HEAD:main 2>/dev/null; do tries=$((tries + 1)); g pull -q --rebase origin main 2>/dev/null; done; echo "$tries" > "../tries$i") & |
| 138 | pids+=($!) |
| 139 | done |
| 140 | wait "${pids[@]}" |
| 141 | local tg refused |
| 142 | tg=$(since "$t") |
| 143 | refused=$(cat tries* | python3 -c "import sys; print(sum(int(l) for l in sys.stdin))") |
| 144 | pids=() |
| 145 | t=$(now) |
| 146 | for i in $(seq 1 "$n"); do ("$VCS" -C "v$i" land -m "Agent $i" >/dev/null 2>&1; echo $? > "vcode$i") & pids+=($!); done |
| 147 | wait "${pids[@]}" |
| 148 | local tv landed |
| 149 | tv=$(since "$t") |
| 150 | landed=$(grep -l '^0$' vcode* | wc -l | tr -d ' ') |
| 151 | row "$n agents land one change each on one branch (s)" "$tg" "$tv" |
| 152 | row " refused pushes / changes that didn't land" "$refused" "$((n - landed))" |
| 153 | cd "$WORK" |
| 154 | } |
| 155 | race 8 |
| 156 | race 32 |
| 157 | race 64 |
| 158 |