Multimodal Evaluation › Code Implementation: Loading and Evaluating Multimodal Data

# Simulate VQA v2 style annotations
vqa_data = [
    {
        "question_id": 1,
        "question": "What color is the fire hydrant?",
        "image_id": 101,
        "answers": [
            {"answer": "red", "count": 7},
            {"answer": "crimson", "count": 1},
            {"answer": "scarlet", "count": 1},
            {"answer": "orange-red", "count": 1},
        ],
        "answer_type": "other",
        "question_type": "what color",
    },
    {
        "question_id": 2,
        "question": "How many dogs are in the picture?",
        "image_id": 102,
        "answers": [{"answer": "2", "count": 8}, {"answer": "two", "count": 2}],
        "answer_type": "number",
        "question_type": "how many",
    },
    {
        "question_id": 3,
        "question": "Is this a stop sign?",
        "image_id": 103,
        "answers": [{"answer": "yes", "count": 10}],
        "answer_type": "yes/no",
        "question_type": "is this",
    },
]

# Simulate model predictions - intentionally wrong on counting and yes/no
model_predictions = {
    1: "red",  # Correct and consensus
    2: "3",  # Wrong number (model overcounts)
    3: "no",  # Wrong boolean (model fails to recognize stop sign)
}

from collections import defaultdict
from typing import Dict, List


def compute_vqa_accuracy(
    question_data: List[Dict], predictions: Dict[int, str]
) -> Dict[str, float]:
    """
    Compute VQA accuracy with consensus scoring.

    Args:
        question_data: List of question annotations with answer distributions
        predictions: Dict mapping question_id to predicted answer

    Returns:
        Dict with overall accuracy and breakdown by question type
    """
    scores_by_type = defaultdict(list)
    all_scores = []

    for item in question_data:
        qid = item["question_id"]
        if qid not in predictions:
            continue

        pred = predictions[qid].lower().strip()

        # Build answer list with multiplicity
        answer_list = []
        for ans_dict in item["answers"]:
            answer_list.extend([ans_dict["answer"].lower()] * ans_dict["count"])

        # Count matches
        matching = sum(1 for ans in answer_list if ans == pred)

        # VQA accuracy formula: min(matching / 3, 1)
        score = min(matching / 3.0, 1.0)

        all_scores.append(score)
        scores_by_type[item["answer_type"]].append(score)

    # Calculate statistics
    results = {
        "overall_accuracy": np.mean(all_scores) if all_scores else 0.0,
        "total_questions": len(all_scores),
    }

    for ans_type, scores in scores_by_type.items():
        results[f"{ans_type}_accuracy"] = np.mean(scores)
        results[f"{ans_type}_count"] = len(scores)

    return results

results = compute_vqa_accuracy(vqa_data, model_predictions)

import math


def get_ngrams(tokens: List[str], n: int) -> Counter:
    """Extract n-grams from token list."""
    ngrams = []
    for i in range(len(tokens) - n + 1):
        ngram = tuple(tokens[i : i + n])
        ngrams.append(ngram)
    return Counter(ngrams)


def compute_bleu(
    candidate: str, references: List[str], max_n: int = 4
) -> float:
    """
    Simplified BLEU score calculation for demonstration.
    (Full implementation would include brevity penalty)
    """
    candidate_tokens = candidate.lower().split()

    scores = []
    for n in range(1, max_n + 1):
        candidate_ngrams = get_ngrams(candidate_tokens, n)

        # Count clipped n-grams (max count in any reference)
        clipped_counts = Counter()
        for ref in references:
            ref_tokens = ref.lower().split()
            ref_ngrams = get_ngrams(ref_tokens, n)
            for ngram in candidate_ngrams:
                if ngram in ref_ngrams:
                    clipped_counts[ngram] = max(
                        clipped_counts[ngram], ref_ngrams[ngram]
                    )

        # Calculate precision
        total = sum(candidate_ngrams.values())
        matching = sum(
            min(candidate_ngrams[g], clipped_counts[g])
            for g in candidate_ngrams
        )

        if total > 0:
            scores.append(matching / total)
        else:
            scores.append(0.0)

    # Geometric mean of n-gram precisions
    if all(s > 0 for s in scores):
        return math.exp(sum(math.log(s) for s in scores) / len(scores))
    return 0.0

# Example caption evaluation
candidate_caption = "a dog is playing with a ball in the park"
reference_captions = [
    "a dog plays with a ball in the park",
    "a puppy is playing with a toy in a grassy field",
    "dog running with ball in park",
]

bleu_score = compute_bleu(candidate_caption, reference_captions)

import numpy as np


# Simulated CLIP embeddings (in practice, use transformers CLIPModel)
# These stand for normalized embeddings in a shared space
def simulate_clip_score(
    text_embedding: np.ndarray, image_embedding: np.ndarray
) -> float:
    """
    Compute cosine similarity between text and image embeddings.
    In practice, these come from CLIP's vision and text encoders.
    """
    # Cosine similarity of normalized vectors
    similarity = np.dot(text_embedding, image_embedding)
    # Scale to 0-100 range commonly used in papers
    return float((similarity + 1) / 2 * 100)


# Simulate embeddings for "a red fire hydrant"
# High alignment scenario
text_emb_aligned = np.array([0.8, 0.3, 0.5])  # Concept: red, fire, hydrant
text_emb_aligned = text_emb_aligned / np.linalg.norm(text_emb_aligned)

image_emb = np.array([0.75, 0.35, 0.55])  # Image of red fire hydrant
image_emb = image_emb / np.linalg.norm(image_emb)

# Misaligned text: "a blue car"
text_emb_misaligned = np.array([0.2, 0.1, 0.9])  # Concept: blue, car
text_emb_misaligned = text_emb_misaligned / np.linalg.norm(text_emb_misaligned)

score_aligned = simulate_clip_score(text_emb_aligned, image_emb)
score_misaligned = simulate_clip_score(text_emb_misaligned, image_emb)