Chunking › Regex-Based Chunking with NLTK

from nltk import RegexpParser

# Define a grammar for NP, VP, and PP chunking
np_grammar = r"""
    NP: {<DT|PRP\$>?<JJ>*<NN.*>+}
    VP: {<MD>?<VB.*>+}
    PP: {<IN>}
"""

# Create the parser
chunker = RegexpParser(np_grammar)


# A helper to extract chunks from NLTK trees
def tree_to_chunks(tree):
    """Extract (chunk_type, text) pairs from an NLTK parse tree."""
    chunks = []
    for subtree in tree:
        if hasattr(subtree, "label"):
            chunk_type = subtree.label()
            words = " ".join(word for word, _ in subtree.leaves())
            chunks.append((chunk_type, words))
    return chunks


# Test on several sentences
test_sentences = [
    "The quick brown fox jumps over the lazy dog.",
    "A beautiful sunset illuminated the entire valley.",
    "My old computer finally crashed yesterday.",
    "Scientists discovered a new species in the deep ocean.",
]

regex_results = []
for sent in test_sentences:
    tokens = word_tokenize(sent)
    tagged = pos_tag(tokens)
    tree = chunker.parse(tagged)
    chunks = tree_to_chunks(tree)
    regex_results.append((sent, tagged, chunks))

# Breakdown of the NP pattern components
np_pattern_breakdown = {
    "<DT|PRP\\$>?": "Optional: determiner (the, a, an) OR possessive pronoun (my, your, its)",
    "<JJ>*": "Zero or more adjectives (modifiers before the noun)",
    "<NN.*>+": "One or more nouns: NN (singular), NNS (plural), NNP (proper), NNPS (proper plural)",
}

regex_quantifiers = {
    "?": "zero or one (optional element)",
    "*": "zero or more (any number)",
    "+": "one or more (at least one required)",
    "|": "alternation (either/or)",
}

# Chinking grammar:
# Step 1: {<.*>+} chunks every token into one big NP
# Step 2: }<VB.*|IN|CC|\\.>{ removes (chinks) verbs, prepositions,
#         conjunctions, and punctuation from chunks
chink_grammar = r"""
    NP: {<.*>+}
        }<VB.*|IN|CC|\.>{
"""

chink_chunker = RegexpParser(chink_grammar)

chink_sentences = [
    "The cat and the dog sat on the mat.",
    "Alice ran quickly through the dark forest.",
]

chink_results = []
for sent in chink_sentences:
    tokens = word_tokenize(sent)
    tagged = pos_tag(tokens)
    tree = chink_chunker.parse(tagged)
    chunks = tree_to_chunks(tree)
    chink_results.append((sent, tagged, chunks))

# Cases where regex chunking struggles
hard_cases = [
    {
        "sentence": "The man I saw yesterday left.",
        "issue": "Relative clause ('I saw yesterday') interrupts the NP",
        "expected_np": "The man",
        "regex_np": "The man I saw",  # incorrectly groups relative clause
    },
    {
        "sentence": "The old man the boats.",
        "issue": "Garden path: 'man' is a verb here, not a noun",
        "expected_vp": "man",
        "regex_problem": "POS tagger likely tags 'man' as NN",
    },
    {
        "sentence": "Flying planes can be dangerous.",
        "issue": "Attachment ambiguity: 'flying' modifies 'planes' or starts a VP",
        "expected": "Ambiguous without context",
    },
]

hard_results = []
for case in hard_cases:
    tokens = word_tokenize(case["sentence"])
    tagged = pos_tag(tokens)
    tree = chunker.parse(tagged)
    found_chunks = tree_to_chunks(tree)
    hard_results.append((case, tagged, found_chunks))