Chunking › Chunking as Preprocessing

def extract_svo_triples(sentence, chunker_obj):
    """
    Extract subject-verb-object triples using chunk sequences.
    Looks for NP-VP-NP patterns in the chunk sequence.
    """
    tokens = word_tokenize(sentence)
    tagged = pos_tag(tokens)
    tree = chunker_obj.parse(tagged)

    ordered_chunks = []
    for subtree in tree:
        if hasattr(subtree, "label"):
            text = " ".join(w for w, _ in subtree.leaves())
            ordered_chunks.append((subtree.label(), text))

    triples = []
    for i in range(len(ordered_chunks) - 2):
        if (
            ordered_chunks[i][0] == "NP"
            and ordered_chunks[i + 1][0] == "VP"
            and ordered_chunks[i + 2][0] == "NP"
        ):
            triples.append(
                {
                    "subject": ordered_chunks[i][1],
                    "verb": ordered_chunks[i + 1][1],
                    "object": ordered_chunks[i + 2][1],
                }
            )

    return triples


# Test on news-style sentences
ie_sentences = [
    "The researchers published a landmark study.",
    "The company acquired its largest competitor.",
    "Scientists identified a new protein in the brain.",
    "The president signed the bill into law.",
]

ie_results = [(s, extract_svo_triples(s, chunker)) for s in ie_sentences]

from collections import Counter


def extract_keywords_via_np_chunking(text, top_n=10):
    """
    Extract noun phrase keywords from text using chunking.
    Returns the most frequent multi-word NPs.
    """
    sentences = nltk.sent_tokenize(text)
    all_nps = []

    for sent in sentences:
        tokens = word_tokenize(sent)
        tagged = pos_tag(tokens)
        tree = chunker.parse(tagged)
        for subtree in tree:
            if hasattr(subtree, "label") and subtree.label() == "NP":
                np_text = " ".join(w.lower() for w, _ in subtree.leaves())
                # Only keep multi-word NPs (more specific)
                if len(subtree.leaves()) > 1:
                    all_nps.append(np_text)

    return Counter(all_nps).most_common(top_n)


# Apply to a sample technology text
tech_text = """
Machine learning has transformed natural language processing research.
Deep learning models achieve remarkable performance on many language tasks.
Large language models like GPT and BERT have redefined the state of the art.
Transformer architectures use attention mechanisms to capture long-range dependencies.
Researchers continue to explore new training methods and model architectures.
Language models learn rich text representations from massive training corpora.
"""

keywords = extract_keywords_via_np_chunking(tech_text)

def find_answer_candidates(question, context_sentence, chunker_obj):
    """
    Identify likely answer spans based on question type and chunking.
    """
    q_lower = question.lower()

    # Map question words to likely answer chunk types
    if q_lower.startswith("who"):
        target_type = "NP"  # Named person = noun phrase
    elif q_lower.startswith("where"):
        target_type = "PP"  # Location = prepositional phrase
    elif q_lower.startswith("what"):
        target_type = "NP"  # Thing = noun phrase
    elif q_lower.startswith("when"):
        target_type = "NP"  # Time expressions are often NPs
    else:
        target_type = "NP"

    tokens = word_tokenize(context_sentence)
    tagged = pos_tag(tokens)
    tree = chunker_obj.parse(tagged)

    candidates = []
    for subtree in tree:
        if hasattr(subtree, "label") and subtree.label() == target_type:
            candidates.append(" ".join(w for w, _ in subtree.leaves()))
    return candidates


# Example QA preprocessing
qa_examples = [
    (
        "Who wrote the novel?",
        "The famous author Ernest Hemingway wrote the novel in 1929.",
    ),
    (
        "What did scientists discover?",
        "Scientists announced a new vaccine for the disease.",
    ),
    (
        "Where did the event occur?",
        "The summit took place in the heart of Geneva.",
    ),
]

qa_results = [
    (q, c, find_answer_candidates(q, c, chunker)) for q, c in qa_examples
]