Context Length Challenges › Evaluation Challenges

def create_needle_haystack_prompt(
    needle_position, total_length, context_tokens=100
):
    """
    Create a needle-in-a-haystack test prompt.

    The 'needle' is a specific fact. The 'haystack' is filler text.
    We place the needle at a specific position and ask about it at the end.
    """
    # The needle: a specific retrievable fact
    needle = "The secret code is ALPHA-7392."

    # Filler text (in practice, this would be coherent text)
    filler_per_segment = "This is background text that provides context but is not relevant to the specific question. "

    # Build the document
    segments_before = needle_position // context_tokens
    segments_after = (
        total_length - needle_position - len(needle.split())
    ) // context_tokens

    document = filler_per_segment * segments_before
    document += needle + " "
    document += filler_per_segment * segments_after

    question = "\nQuestion: What is the secret code mentioned in the document?"

    return {
        "document": document[
            : total_length * 5
        ],  # Approximate token to char ratio
        "question": question,
        "answer": "ALPHA-7392",
        "needle_position": needle_position,
        "total_length": total_length,
    }


# Test at different positions
positions = [100, 1000, 5000, 10000, 50000]
total_length = 100000
test_cases = [
    create_needle_haystack_prompt(pos, total_length) for pos in positions
]

def multi_hop_test_design(num_hops, positions, total_length):
    """
    Design a multi-hop reasoning test over long context.

    Each hop provides a clue that points to the next location.
    The model must chain together all clues to answer.
    """
    clues = [
        ("Alice lives in", "France"),
        ("The capital of France is", "Paris"),
        ("The main airport in Paris is", "Charles de Gaulle"),
        ("The airline code for Charles de Gaulle is", "CDG"),
    ]

    test = {
        "num_hops": num_hops,
        "positions": positions[:num_hops],
        "total_length": total_length,
        "clues": clues[:num_hops],
        "question": "What is the airline code for the main airport in the capital of the country where Alice lives?",
        "answer": clues[num_hops - 1][1] if num_hops <= len(clues) else "N/A",
    }

    # Compute the total "distance" the model must span
    if len(positions) >= 2:
        test["max_span"] = max(positions) - min(positions)
    else:
        test["max_span"] = 0

    return test


# Different multi-hop configurations
configs = [
    (2, [1000, 50000]),
    (3, [1000, 25000, 50000]),
    (4, [1000, 20000, 40000, 60000]),
]

multi_hop_tests = [multi_hop_test_design(n, pos, 100000) for n, pos in configs]