Reasoning Foundations › Code Exploration: Reasoning Patterns and Failure Modes

import numpy as np

def chain_accuracy(step_accuracy: float, n_steps: int) -> float:
    """
    Compute the probability that a multi-step reasoning chain is fully correct,
    assuming each step is independently correct with probability step_accuracy.
    """
    return step_accuracy**n_steps


# Evaluate across different per-step accuracies and chain lengths
step_accuracies = [0.99, 0.95, 0.90, 0.80]
chain_lengths = list(range(1, 21))

# Store results for plotting
results = {}
for acc in step_accuracies:
    results[acc] = [chain_accuracy(acc, n) for n in chain_lengths]

import re


def check_syllogism(premise1: str, premise2: str, conclusion: str) -> dict:
    """
    Check a simple categorical syllogism of the form:
      All A are B. X is A. Therefore X is B.
    Returns a dict with 'valid' (bool) and 'explanation' (str).
    """
    # Parse premise1: "All [subject] are [predicate]"
    # Parse premise2: "[entity] is [category]"
    # Parse conclusion: "[entity] is [predicate]"

    p1_match = re.match(r"All (\w+) are (\w+)\.", premise1)
    p2_match = re.match(r"(\w+) are (\w+)\.", premise2)
    c_match = re.match(r"(\w+) are (\w+)\.", conclusion)

    if not (p1_match and p2_match and c_match):
        return {
            "valid": False,
            "explanation": "Could not parse syllogism form.",
        }

    all_subject, all_predicate = p1_match.group(1), p1_match.group(2)
    entity, entity_category = p2_match.group(1), p2_match.group(2)
    concl_entity, concl_predicate = c_match.group(1), c_match.group(2)

    # Valid if: entity_category == all_subject AND concl_entity == entity
    # AND concl_predicate == all_predicate
    is_valid = (
        entity_category == all_subject
        and concl_entity == entity
        and concl_predicate == all_predicate
    )

    if is_valid:
        explanation = (
            f"Valid. '{entity}' are {all_subject}, and all {all_subject}s are "
            f"{all_predicate}, so '{entity}' must be {all_predicate}."
        )
    else:
        explanation = (
            f"Invalid. The conclusion '{conclusion}' does not follow necessarily "
            f"from the given premises."
        )

    return {"valid": is_valid, "explanation": explanation}


# Test cases: valid syllogisms, an invalid middle-term mismatch, and a reversal
syllogisms = [
    (
        "All mammals are warm-blooded.",
        "Whales are mammals.",
        "Whales are warm-blooded.",
    ),
    (
        "All birds are feathered.",
        "Penguins are birds.",
        "Penguins are feathered.",
    ),
    (
        "All mammals are warm-blooded.",
        "Salmon are fish.",
        "Salmon are warm-blooded.",
    ),
    (
        "All mammals are warm-blooded.",
        "Whales are mammals.",
        "Mammals are warm-blooded.",  # Reversal: concludes general from specific
    ),
]

syllogism_results = [check_syllogism(*s) for s in syllogisms]

def naive_classifier(problem_text: str) -> str:
    """
    A deliberately naive classifier that answers math word problems
    by looking for surface keywords. This caricatures how a model
    relying on spurious correlations might behave.
    """
    text_lower = problem_text.lower()

    # Heuristic: "more" -> subtraction, "fewer/less" -> subtraction,
    # "times" -> multiplication, "split/each" -> division
    if "how many more" in text_lower or "how many fewer" in text_lower:
        nums = [int(x) for x in re.findall(r"\b\d+\b", problem_text)]
        if len(nums) >= 2:
            return str(abs(nums[0] - nums[1]))
    if "times as many" in text_lower or "times more" in text_lower:
        nums = [int(x) for x in re.findall(r"\b\d+\b", problem_text)]
        if len(nums) >= 2:
            return str(nums[0] * nums[1])
    if "split equally" in text_lower or "each person gets" in text_lower:
        nums = [int(x) for x in re.findall(r"\b\d+\b", problem_text)]
        if len(nums) >= 2 and nums[1] != 0:
            return str(nums[0] // nums[1])
    # Default: sum all numbers
    nums = [int(x) for x in re.findall(r"\b\d+\b", problem_text)]
    return str(sum(nums)) if nums else "unknown"


# Logically equivalent problems with different phrasings
problem_pairs = [
    {
        "problem_a": "Alice has 12 apples. Bob has 5 apples. How many more apples does Alice have than Bob?",
        "problem_b": "Alice has 12 apples. Bob has 5 apples. What is the difference in the number of apples they have?",
        "correct_answer": "7",
    },
    {
        "problem_a": "There are 24 cookies split equally among 6 children. Each person gets how many cookies?",
        "problem_b": "A baker divides 24 cookies into 6 equal groups. How large is each group?",
        "correct_answer": "4",
    },
    {
        "problem_a": "A cat has 3 times as many lives as a fish. The fish has 2 lives. How many lives does the cat have?",
        "problem_b": "A fish has 2 lives. A cat has triple that number. How many lives does the cat have?",
        "correct_answer": "6",
    },
]

pair_results = []
for pair in problem_pairs:
    ans_a = naive_classifier(pair["problem_a"])
    ans_b = naive_classifier(pair["problem_b"])
    pair_results.append(
        {
            "ans_a": ans_a,
            "ans_b": ans_b,
            "correct": pair["correct_answer"],
            "a_correct": ans_a == pair["correct_answer"],
            "b_correct": ans_b == pair["correct_answer"],
            "consistent": ans_a == ans_b,
        }
    )

# Characterize reasoning types across several dimensions
reasoning_types = [
    "Deductive",
    "Inductive",
    "Abductive",
    "Analogical",
    "Causal",
    "Multi-step\nArithmetic",
    "Commonsense",
]

# Illustrative composite of approximate results from heterogeneous benchmarks
llm_accuracy = [0.82, 0.74, 0.68, 0.85, 0.51, 0.77, 0.79]

# Brittleness to phrasing changes (1=robust, 5=brittle) -- qualitative expert assessment
phrasing_brittleness = [3.5, 3.0, 3.2, 2.0, 4.0, 4.5, 3.0]

# Simulate model performance on a benchmark across "model generations"
# using a logistic growth curve that saturates near ceiling


def model_accuracy_over_time(
    generations: np.ndarray,
    baseline: float = 0.45,
    ceiling: float = 0.95,
    growth_rate: float = 0.5,
    inflection: float = 5.0,
) -> np.ndarray:
    """
    Logistic growth model for benchmark accuracy.
    generations: array of model generation indices (e.g., 0, 1, 2, ...)
    baseline: accuracy at generation 0
    ceiling: maximum achievable accuracy
    growth_rate: how fast accuracy increases
    inflection: generation at which growth is fastest
    """
    return baseline + (ceiling - baseline) / (
        1.0 + np.exp(-growth_rate * (generations - inflection))
    )


generations = np.linspace(0, 12, 50)

# Three benchmarks: easy saturates first, medium later, hard not yet
benchmark_params = {
    "Easy benchmark": {"ceiling": 0.97, "growth_rate": 0.7, "inflection": 3.0},
    "Medium benchmark": {
        "ceiling": 0.95,
        "growth_rate": 0.5,
        "inflection": 6.0,
    },
    "Hard benchmark": {"ceiling": 0.90, "growth_rate": 0.35, "inflection": 9.0},
}

benchmark_curves = {
    name: model_accuracy_over_time(generations, **params)
    for name, params in benchmark_params.items()
}

# Define "saturated" as accuracy > 0.88
saturation_threshold = 0.88
saturation_points = {}
for name, curve in benchmark_curves.items():
    saturated_at = generations[curve >= saturation_threshold]
    saturation_points[name] = saturated_at[0] if len(saturated_at) > 0 else None