Agent Evaluation and Testing Harness.

golden_set = [
    {
        "id": "refund-happy-path",
        "input": "I want a refund for order 42, it arrived damaged.",
        "expected_tools": ["verify_customer", "lookup_order", "process_refund"],
        "expected_stop_reason": "end_turn",
    },
    {
        "id": "refund-over-cap",
        "input": "I need a $900 refund on order 55.",
        "expected_tools": ["verify_customer", "lookup_order", "escalate_to_human"],
        "expected_stop_reason": "end_turn",
    },
]

def submit_eval_batch(golden_set, model="claude-sonnet-5"):
    return client.messages.batches.create(requests=[
        {"custom_id": case["id"], "params": {
            "model": model, "max_tokens": 4096, "tools": tools,
            "messages": [{"role": "user", "content": case["input"]}],
        }} for case in golden_set
    ])

def score_tool_sequence(response, expected_tools: list[str]) -> bool:
    actual = [b.name for b in response.content if b.type == "tool_use"]
    return actual == expected_tools

JUDGE_SCHEMA = {
    "type": "object",
    "properties": {
        "correct": {"type": "boolean"},
        "reasoning": {"type": "string"},
    },
    "required": ["correct", "reasoning"],
    "additionalProperties": False,
}

def judge_output(case_input: str, agent_output: str, expected_behavior: str):
    return client.messages.parse(
        model="claude-opus-4-8",
        max_tokens=1024,
        output_config={"format": {"type": "json_schema", "schema": JUDGE_SCHEMA}},
        messages=[{"role": "user", "content":
            f"Input: {case_input}\nExpected: {expected_behavior}\nActual: {agent_output}\n"
            "Does the actual output satisfy the expected behavior?"}],
    ).parsed_output

def check_regression_gate(new_pass_rate: float, baseline_pass_rate: float, tolerance=0.02):
    if new_pass_rate < baseline_pass_rate - tolerance:
        raise SystemExit(
            f"Regression: {new_pass_rate:.2%} vs baseline {baseline_pass_rate:.2%}"
        )
    print(f"OK: {new_pass_rate:.2%} (baseline {baseline_pass_rate:.2%})")

def debug_failure(case_id: str, response):
    print(f"--- {case_id} ---")
    print(f"stop_reason: {response.stop_reason}")
    for block in response.content:
        if block.type == "tool_use":
            print(f"  tool_use: {block.name}({block.input})")
        elif block.type == "text":
            print(f"  text: {block.text[:200]}")