Testing the Agent with Examples › Running the Tests

def run_test(agent, test_case):
    """
    Run a single test case and return the results.
    """
    print(f"\nRunning test: {test_case['id']}")
    print(f"Input: {test_case['input']}")

    # Run the agent with the test input
    response = agent.process(test_case["input"])

    # Record what happened
    result = {
        "test_id": test_case["id"],
        "input": test_case["input"],
        "output": response["text"],
        "behavior": response.get("actions_taken", []),
        "passed": False,
        "notes": [],
    }

    # Check if the agent's behavior matches expectations
    if test_case["expected_behavior"] in str(response.get("actions_taken", [])):
        result["notes"].append(
            f"✓ Correct behavior: {test_case['expected_behavior']}"
        )
    else:
        result["notes"].append(
            f"✗ Expected {test_case['expected_behavior']}, got {response.get('actions_taken', [])}"
        )

    # Check if the output matches expectations
    if test_case["expected_output"].lower() in response["text"].lower():
        result["notes"].append("✓ Correct output")
        result["passed"] = True
    else:
        result["notes"].append(
            f"✗ Output doesn't match expected: {test_case['expected_output']}"
        )

    return result

def run_test_suite(agent, test_suite):
    """
    Run all tests and generate a report.
    """
    results = []

    for test_case in test_suite:
        # Check if this test has dependencies
        if "requires" in test_case:
            # Make sure required tests ran first
            for required_id in test_case["requires"]:
                required_result = next(
                    (r for r in results if r["test_id"] == required_id), None
                )
                if not required_result or not required_result["passed"]:
                    print(
                        f"Skipping {test_case['id']} - dependency {required_id} failed"
                    )
                    continue

        result = run_test(agent, test_case)
        results.append(result)

        # Print immediate feedback
        status = "PASS" if result["passed"] else "FAIL"
        print(f"Result: {status}")
        for note in result["notes"]:
            print(f"  {note}")

    return results