Real evaluation with Inspect AI

Open in Colab Run this notebook in Colab

Real evaluation with Inspect AI#

Track 08 - Production · Notebook 08 · Runtime: ~2 min LIVE, <1s replay

Build an Inspect AI task: 10 math word problems, a custom numeric scorer, two models head-to-head. Same shape used by AISI for the public benchmarks they publish.

from llm_systems_cookbook.nb import bootstrap
from llm_systems_cookbook._utils import repo_root

import json
import os
import re
from pathlib import Path

s = bootstrap("08_production_08_inspect_ai_eval_harness")

try:
    import inspect_ai  # noqa: F401
    HAS_INSPECT = True
except ImportError:
    HAS_INSPECT = False

LIVE = HAS_INSPECT and bool(os.environ.get("ANTHROPIC_API_KEY")) and bool(os.environ.get("INSPECT_LIVE"))
FIXTURE = json.loads((Path(repo_root()) / "notebooks/08_production/_fixtures/08_inspect.json").read_text())
print(f"mode={'LIVE' if LIVE else 'REPLAY'}  inspect_installed={HAS_INSPECT}")

Dataset — 10 math word problems#

Each sample is (input, target). Inspect AI represents these as Sample objects; we serialise to a list of dicts in the fixture so we can replay scoring without spinning up the framework.

samples = FIXTURE["samples"]
for sm in samples[:3]:
    print(f"  [{sm['id']}] target={sm['target']:>5}  {sm['input']}")

Task definition#

A task wires three things: dataset, solver (generate() is the default — just call the model), and scorer. The custom scorer here extracts the final integer from the model output and matches it against target with a 0.05 tolerance for floats.

TASK_SOURCE = """
from inspect_ai import Task, task, eval
from inspect_ai.dataset import Sample
from inspect_ai.solver import generate
from inspect_ai.scorer import scorer, Score, Scorer, Target

@scorer(metrics=[\"accuracy\", \"stderr\"])
def numeric_match(tolerance: float = 0.05) -> Scorer:
    async def score(state, target: Target) -> Score:
        text = state.output.completion.strip()
        # Extract the last number in the answer.
        m = re.findall(r\"-?\\d+(?:\\.\\d+)?\", text)
        if not m:
            return Score(value=0, answer=text, explanation=\"no number\")
        got = float(m[-1])
        want = float(target.text)
        ok = abs(got - want) / max(abs(want), 1) <= tolerance
        return Score(value=int(ok), answer=str(got), explanation=f\"want {want}, got {got}\")
    return score

@task
def math_word_problems():
    return Task(
        dataset=[Sample(input=s[\"input\"], target=s[\"target\"], id=s[\"id\"])
                 for s in samples],
        solver=generate(),
        scorer=numeric_match(),
    )
"""
print(TASK_SOURCE)

Run on two models#

In LIVE mode eval(math_word_problems(), model="anthropic/claude-haiku-4-5") returns an EvalLog with per-sample scores, latency, and cost. We replay the same shape from the fixture below.

MODELS = ["claude_haiku_4_5", "claude_sonnet_4_6"]

def run_eval(model_key: str) -> dict:
    if not LIVE:
        return FIXTURE["aggregates"][model_key]

    # LIVE: this is what actually runs the eval — see the source above.
    # from inspect_ai import eval
    # logs = eval(math_word_problems(), model=f"anthropic/{model_key.replace('_', '-')}")
    # ... extract aggregates from logs[0] ...
    raise NotImplementedError("LIVE branch — install inspect-ai and set INSPECT_LIVE=1")


results = {m: run_eval(m) for m in MODELS}
print(f"{'model':<22} {'accuracy':>10} {'$ total':>10} {'mean lat':>10}")
for m, r in results.items():
    print(f"{m:<22} {r['accuracy']:>10.1%} ${r['total_cost_usd']:>8.5f}  {r['mean_latency_s']:>8.2f}s")

Per-sample diff#

The most useful Inspect AI affordance is per-sample inspection. Below: every problem, both models’ answers, and where the smaller model failed.

print(f"{'id':<6} {'haiku':>9} {'sonnet':>9}  problem")
print("-" * 80)
for sm in samples:
    h = sm["claude_haiku_4_5"];  hk = "✓" if h["correct"] else "✗"
    so = sm["claude_sonnet_4_6"]; sk = "✓" if so["correct"] else "✗"
    diff = "  ←" if h["correct"] != so["correct"] else ""
    print(f"{sm['id']:<6} {hk:>9} {sk:>9}  {sm['input'][:55]}{diff}")

Visualisation#

import matplotlib.pyplot as plt

fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(9.5, 3.4))
colors = ["tab:orange", "tab:green"]

ax1.bar([m.replace("_", "-") for m in MODELS],
        [results[m]["accuracy"] * 100 for m in MODELS], color=colors)
ax1.set_ylabel("accuracy (%)"); ax1.set_ylim(0, 105)
ax1.set_title(f"{len(samples)}-problem accuracy")
for i, m in enumerate(MODELS):
    ax1.text(i, results[m]["accuracy"] * 100 + 1,
             f"{results[m]['correct']}/{results[m]['total']}", ha="center")

ax2.bar([m.replace("_", "-") for m in MODELS],
        [results[m]["total_cost_usd"] * 1000 for m in MODELS], color=colors)
ax2.set_ylabel("total cost (×1e-3 $)")
ax2.set_title("eval cost across both models")
for i, m in enumerate(MODELS):
    ax2.text(i, results[m]["total_cost_usd"] * 1000,
             f"${results[m]['total_cost_usd']:.4f}", ha="center", va="bottom")

fig.tight_layout(); plt.show()

Checks#

haiku  = results["claude_haiku_4_5"]
sonnet = results["claude_sonnet_4_6"]

s.check(
    "fixture_samples_match_aggregate",
    lambda: sum(s["claude_haiku_4_5"]["correct"] for s in samples) == haiku["correct"]
            and sum(s["claude_sonnet_4_6"]["correct"] for s in samples) == sonnet["correct"],
    msg=f"haiku per-sample agrees with aggregate ({haiku['correct']}); "
         f"sonnet too ({sonnet['correct']})",
)
s.check(
    "sonnet_at_least_as_accurate_as_haiku",
    lambda: sonnet["accuracy"] >= haiku["accuracy"],
    msg=f"haiku={haiku['accuracy']:.1%}  sonnet={sonnet['accuracy']:.1%}",
)
s.check(
    "sonnet_more_expensive_than_haiku",
    lambda: sonnet["total_cost_usd"] > haiku["total_cost_usd"],
    msg=f"haiku=${haiku['total_cost_usd']:.5f}  sonnet=${sonnet['total_cost_usd']:.5f}",
)
# Numeric extraction — verify the scorer behaves on representative outputs.
def _extract(text: str) -> float | None:
    m = re.findall(r"-?\d+(?:\.\d+)?", text)
    return float(m[-1]) if m else None
s.check(
    "scorer_extracts_last_number_from_text",
    lambda: _extract("3 * 7 = 21") == 21 and _extract("$4/12 per egg * 30 eggs = $10.") == 10,
)
s.check(
    "more_than_half_correct_on_smaller_model",
    lambda: haiku["accuracy"] >= 0.5,
    msg=f"haiku acc = {haiku['accuracy']:.1%}",
)

Notes for production#

  • inspect eval CLI: inspect eval task.py --model anthropic/claude-sonnet-4-6 --limit 50 runs the file and saves a log to ./logs/. inspect view opens a browser UI to inspect every sample, retry, and tool trace.

  • Built-in datasets: inspect_ai.dataset.hf_dataset(...), plus first-class loaders for HumanEval, GSM8K, MMLU, IFEval, MATH. Building on those is one line.

  • Model-graded scorers: model_graded_qa(), model_graded_fact() use a judge LLM with a rubric. Cheaper than human eval, more flexible than substring match. Pin the judge model and rubric — drift will invalidate week-over-week comparisons.

  • Solvers compose. chain(prompt_template(), generate(), self_critique()) runs prompt → completion → second-pass critique. Useful for measuring how much “agent scaffolding” actually buys you on a fixed benchmark.

  • Cost-aware sweeps: eval_set([task1, task2], models=[...], max_samples=50) runs the matrix and writes one log per (task, model). Pin a budget with --max-tasks and --max-samples.

s.summary()
s.save()