#!/usr/bin/env bash # Checks the end state of an agent run (tests/agent/README.md), then stops # the sandbox's server. The agent's report covers what this can't see (what # it told Alice, which flags it used). # Usage: tests/agent/grade.sh [path/to/target/debug] set -uo pipefail SB=$(cd "$1" && pwd) ROOT=$(cd "$(dirname "$0")/../.." && pwd) BIN=${2:-$ROOT/target/debug} . "$SB/facts" export VCS_HOME="$SB/grader" VCS_SAVER=0 "$BIN/vcs" login "$H" --token "$(cat "$SB/alice.token")" >/dev/null rm -rf "$SB/check" "$BIN/vcs" get "$H/acme/shop" "$SB/check" >/dev/null M=$SB/check TITLES=$("$BIN/vcs" -C "$M" log -n 40) pass=0 fail=0 check() { local name=$1 shift if "$@" >/dev/null 2>&1; then pass=$((pass + 1)); echo "pass $name"; else fail=$((fail + 1)); echo "FAIL $name"; fi } has() { grep -q -- "$2" "$M/$1"; } lacks() { [ ! -e "$M/$1" ]; } check "T1 typo fixed on main" has README.md "Welcome to the shop" check "T2 landed with 50 and Sam's comment" sh -c "grep -q 50 '$M/src/limits.ts' && grep -q 'Sam' '$M/src/limits.ts' && ! grep -q '<<<<<<<\|>>>>>>>\|=======' '$M/src/limits.ts'" check "T3 cart.ts is the working version" grep -q "skips free items" "$SB/home/shop-cart/src/cart.ts" check "T3 other files kept" grep -q "keep me" "$SB/home/shop-cart/notes.txt" check "T3 nothing landed from it" sh -c "! grep -q 'BROKEN' '$M/src/cart.ts'" check "T4 notes landed then taken back" sh -c "echo '$TITLES' | grep -q 'Add notes' && [ ! -e '$M/notes.txt' ]" check "T5 search landed" has search.txt "search" check "T5 didn't touch the other agent's folder" sh -c "[ ! -e '$SB/home/shop/search.txt' ]" check "T6 draft continued and landed" sh -c "grep -q draft '$M/draft.txt' && grep -q done '$M/draft.txt'" check "T7 parser waits for Sam" sh -c "curl -s -H 'authorization: Bearer $(cat "$SB/sam.token")' http://$H/v1/projects/sam/parser | python3 -c 'import json,sys; assert json.load(sys.stdin)[\"main\"][\"n\"] == 1'" check "T8 landed without the key" sh -c "grep -q fallback '$M/src/config.ts' || echo '$TITLES' | grep -q 'fallback key'; ! grep -q sk_live '$M/src/config.ts'" check "T8 key isn't on main" sh -c "! grep -rq sk_live '$M'" check "T9 banner taken out" lacks banner.txt check "T9 later versions kept" has footer.txt "footer" check "T10 change still saved locally" grep -q "faster" "$SB/home/search/search.txt" check "T11 legacy has 3 versions" sh -c "curl -s -H 'authorization: Bearer $(cat "$SB/alice.token")' http://$H/v1/projects/acme/legacy | python3 -c 'import json,sys; assert json.load(sys.stdin)[\"main\"][\"n\"] == 3'" check "T12 docs landed" has docs.txt "How to run" check "T13 junk never landed" lacks junk.txt check "T13 junk gone from the workspace" sh -c "[ ! -e '$SB/home/shop-junk/junk.txt' ]" check "T14 shop-old has Sam's change" grep -q "Sam" "$SB/home/shop-old/src/limits.ts" kill "$(cat "$SB/server.pid")" 2>/dev/null echo "agent run: $pass passed, $fail failed" [ "$fail" = 0 ]