Summarization › Code Implementation

import re

import numpy as np


def preprocess(text):
    """Tokenize and clean a sentence."""
    words = re.findall(r"\b[a-z]+\b", text.lower())
    stopwords = {
        "the",
        "a",
        "an",
        "is",
        "are",
        "was",
        "were",
        "be",
        "been",
        "have",
        "has",
        "had",
        "do",
        "does",
        "did",
        "will",
        "would",
        "can",
        "could",
        "may",
        "might",
        "shall",
        "should",
        "must",
        "in",
        "on",
        "at",
        "to",
        "for",
        "of",
        "and",
        "or",
        "but",
        "that",
        "this",
        "it",
        "its",
        "with",
        "by",
        "from",
        "not",
    }
    return [w for w in words if w not in stopwords]


def sentence_similarity(s1_words, s2_words):
    """Compute overlap-based sentence similarity."""
    set1, set2 = set(s1_words), set(s2_words)
    intersection = set1 & set2
    if len(set1) == 0 or len(set2) == 0:
        return 0.0
    denominator = np.log(len(set1) + 1) + np.log(len(set2) + 1)
    return len(intersection) / denominator if denominator > 0 else 0.0


def textrank(sentences, top_k=3, damping=0.85, max_iter=50):
    """Run TextRank and return top-k sentences by score."""
    n = len(sentences)
    tokenized = [preprocess(s) for s in sentences]

    # Build similarity matrix
    sim_matrix = np.zeros((n, n))
    for i in range(n):
        for j in range(n):
            if i != j:
                sim_matrix[i][j] = sentence_similarity(
                    tokenized[i], tokenized[j]
                )

    # Row-normalize
    row_sums = sim_matrix.sum(axis=1, keepdims=True)
    row_sums[row_sums == 0] = 1  # Avoid division by zero
    sim_matrix = sim_matrix / row_sums

    # Power iteration
    scores = np.ones(n) / n
    for _ in range(max_iter):
        new_scores = (1 - damping) / n + damping * sim_matrix.T.dot(scores)
        if np.max(np.abs(new_scores - scores)) < 1e-6:
            break
        scores = new_scores

    # Select top-k sentences by original order
    ranked_idx = np.argsort(scores)[::-1][:top_k]
    selected_idx = sorted(ranked_idx)
    return [sentences[i] for i in selected_idx], scores

document = """
Global average temperatures have risen by approximately 1.1 degrees Celsius since the pre-industrial era.
The primary driver is the increase in greenhouse gases, particularly carbon dioxide from burning fossil fuels.
Sea levels are rising at an accelerating rate, threatening coastal communities around the world.
The past decade has been the warmest on record, with 2023 ranking as the hottest year ever measured.
Extreme weather events including heatwaves, floods, and wildfires have become more frequent and severe.
International agreements like the Paris Accord aim to limit warming to 1.5 degrees Celsius above pre-industrial levels.
Scientists warn that exceeding 2 degrees of warming could trigger irreversible tipping points in Earth's climate system.
Renewable energy adoption has accelerated dramatically, with solar and wind power now cheaper than fossil fuels in most markets.
Carbon capture technologies are being developed, but currently operate at a fraction of the scale needed.
Adaptation measures, including improved infrastructure and early warning systems, are being deployed in vulnerable regions.
""".strip().split("\n")

document = [s.strip() for s in document if s.strip()]

summary_sentences, scores = textrank(document, top_k=3)

def word_overlap_similarity(s1, s2):
    """Simple cosine-style similarity for MMR."""
    w1 = set(preprocess(s1))
    w2 = set(preprocess(s2))
    if not w1 or not w2:
        return 0.0
    return len(w1 & w2) / (len(w1) * len(w2)) ** 0.5


def mmr_summary(sentences, scores, top_k=3, lambda_param=0.6):
    """
    Select sentences using Maximum Marginal Relevance.
    lambda_param controls relevance vs. diversity tradeoff.
    """
    selected = []
    remaining = list(range(len(sentences)))

    for _ in range(top_k):
        best_idx = None
        best_score = -np.inf

        for i in remaining:
            relevance = scores[i]
            if selected:
                redundancy = max(
                    word_overlap_similarity(sentences[i], sentences[j])
                    for j in selected
                )
            else:
                redundancy = 0.0

            mmr_score = (
                lambda_param * relevance - (1 - lambda_param) * redundancy
            )

            if mmr_score > best_score:
                best_score = mmr_score
                best_idx = i

        selected.append(best_idx)
        remaining.remove(best_idx)

    selected_sorted = sorted(selected)
    return [sentences[i] for i in selected_sorted]

# Uncomment to install if needed:
# !uv pip install transformers torch

from transformers import BartForConditionalGeneration, BartTokenizer

# Load BART fine-tuned on CNN/DailyMail
bart_model_name = "facebook/bart-large-cnn"
bart_tokenizer = BartTokenizer.from_pretrained(bart_model_name)
bart_model = BartForConditionalGeneration.from_pretrained(bart_model_name)

import torch

# Summarize the climate change document
article = " ".join(document)
inputs = bart_tokenizer(
    article, return_tensors="pt", max_length=1024, truncation=True
)
with torch.no_grad():
    summary_ids = bart_model.generate(
        inputs["input_ids"],
        max_length=80,
        min_length=30,
        num_beams=4,
        early_stopping=True,
    )
bart_summary = bart_tokenizer.decode(summary_ids[0], skip_special_tokens=True)