handle-regression

Handle Regression Protocol

Use this when a test262 run shows fewer passes than the baseline or previous run.

Step 1: Confirm the regression is real

# Compare current vs archived results
python3 -c "
import json, os
current = {}
with open('benchmarks/results/test262-results.jsonl') as f:
    for line in f:
        r = json.loads(line)
        current[r['file']] = r

archives = sorted([f for f in os.listdir('benchmarks/results')
                   if f.startswith('test262-results-') and f.endswith('.jsonl')])
if not archives:
    print('No archived results to compare. Run was the first.')
    exit()

prev = {}
with open(f'benchmarks/results/{archives[-1]}') as f:
    for line in f:
        r = json.loads(line)
        prev[r['file']] = r

regressions = [(t, prev[t]['status'], current[t]['status'], current[t].get('error','')[:60])
               for t in prev if prev[t]['status'] == 'pass' and current.get(t,{}).get('status') != 'pass']
fixes = [t for t in current if current[t]['status'] == 'pass' and prev.get(t,{}).get('status') != 'pass']

print(f'Regressions: {len(regressions)}, Fixes: {len(fixes)}, Net: {len(fixes) - len(regressions):+d}')

from collections import Counter
patterns = Counter()
for t, old, new, err in regressions:
    if 'SyntaxError' in err: patterns['expected SyntaxError but compiled'] += 1
    elif 'null' in err.lower(): patterns['null deref'] += 1
    elif new == 'compile_error': patterns['new compile error'] += 1
    elif 'returned' in err: patterns['wrong return value'] += 1
    else: patterns[err[:30] or new] += 1

print()
print('Regression patterns:')
for p, c in patterns.most_common(10):
    print(f'  {c:4d}  {p}')

print()
print('Sample regressions (1 per pattern):')
shown = set()
for t, old, new, err in sorted(regressions):
    for p in patterns:
        if p not in shown and p[:15] in (err + new)[:60]:
            print(f'  {t}')
            shown.add(p)
            break
    if len(shown) >= 5: break
"

If regressions < 10 and could be flaky tests, re-run test262 to confirm.