Continuous Feedback and Improvement › Using AI to Evaluate AI

## Using GPT-5 as an evaluator for our agent's responses
from openai import OpenAI

evaluator = OpenAI(api_key=os.getenv("OPENAI_API_KEY"))

def evaluate_response(user_query, agent_response, expected_behavior):
    """Use an AI model to evaluate the agent's response quality."""
    
    evaluation_prompt = f"""You are evaluating an AI assistant's response for quality and accuracy.

User Query: {user_query}

Agent Response: {agent_response}

Expected Behavior: {expected_behavior}

Please evaluate the response on these criteria:

1. Did it address the user's actual question?
2. Was it accurate and helpful?
3. Did it follow the expected behavior?
4. Were there any mistakes or issues?

Provide a score from 1-5 (5 being excellent) and explain your reasoning."""

    evaluation = evaluator.chat.completions.create(
        model="gpt-5",
        messages=[
            {"role": "system", "content": "You are a careful evaluator of AI assistant responses."},
            {"role": "user", "content": evaluation_prompt}
        ]
    )
    
    return evaluation.choices[0].message.content

## Example usage
query = "What's the weather like for my trip next week?"
response = "I checked your calendar and found you have a trip to Seattle from November 15-17. The forecast shows rain with temperatures around 52°F."
expected = "Should check calendar first, then check weather for specific dates found"

evaluation = evaluate_response(query, response, expected)
print(evaluation)