Regular Expressions for NLP: Complete Guide to Pattern Matching in Python

import re

# A simple pattern: find the word "cat"
text = "The cat sat on the mat. The catalog was nearby."
pattern = r"cat"

# Find all matches
matches = re.findall(pattern, text)

# Find match positions
for match in re.finditer(pattern, text):
    start, end = match.span()
    context = text[max(0, start - 5) : end + 5]

import re

text = "Contact us at support@example.com or sales@example.com"

# re.search() - Find first match
first_match = re.search(r"\w+@\w+\.\w+", text)

# re.findall() - Find all matches, return list of strings
all_matches = re.findall(r"\w+@\w+\.\w+", text)

# re.finditer() - Find all matches, return iterator of match objects
match_objects = list(re.finditer(r"\w+@\w+\.\w+", text))

# re.sub() - Replace matches
replaced = re.sub(r"\w+@\w+\.\w+", "[EMAIL]", text)

# re.split() - Split by pattern
parts = re.split(r"\s+", "Hello   world  foo")

# Without raw string: \n becomes a newline character
regular_string = "line1\nline2"

# With raw string: \n stays as backslash-n
raw_string = r"line1\nline2"

# This matters for regex patterns
pattern_wrong = "\bword\b"  # \b becomes backspace character!
pattern_right = r"\bword\b"  # \b stays as word boundary

text = "cat cot cut c@t c9t c\nt"

# . matches any single character
pattern = r"c.t"
matches = re.findall(pattern, text)

text = "The gray grey dog played in the fog"

# [ae] matches either 'a' or 'e'
pattern = r"gr[ae]y"
matches = re.findall(pattern, text)

# Ranges: [a-z] matches any lowercase letter
lowercase = re.findall(r"[a-z]+", "Hello World 123")

# Multiple ranges: [a-zA-Z0-9]
alphanumeric = re.findall(r"[a-zA-Z0-9]+", "user@example.com")

text = "abc123xyz"

# [^0-9] matches any non-digit
non_digits = re.findall(r"[^0-9]+", text)

# [^a-z] matches any non-lowercase letter
non_lower = re.findall(r"[^a-z]+", text)

text = "Call 555-1234 or email bob@mail.com on 2024-01-15"

# \d = digit [0-9]
digits = re.findall(r"\d+", text)

# \w = word character [a-zA-Z0-9_]
words = re.findall(r"\w+", text)

# \s = whitespace [ \t\n\r\f\v]
spaces = re.findall(r"\s+", text)

# Uppercase versions are negations
# \D = non-digit, \W = non-word, \S = non-whitespace
non_digits = re.findall(r"\D+", text)

text = "a aa aaa aaaa b bb bbb"

# * = zero or more
star = re.findall(r"ba*", "b ba baa baaa")

# + = one or more
plus = re.findall(r"ba+", "b ba baa baaa")

# ? = zero or one (optional)
optional = re.findall(r"colou?r", "color colour")

# {n} = exactly n times
exact = re.findall(r"a{3}", text)

# {n,m} = between n and m times
range_q = re.findall(r"a{2,3}", text)

# {n,} = n or more times
at_least = re.findall(r"a{2,}", text)

html = "<div>Hello</div><div>World</div>"

# Greedy: matches as much as possible
greedy = re.findall(r"<div>.*</div>", html)

# Lazy: matches as little as possible
lazy = re.findall(r"<div>.*?</div>", html)

text = "Hello World\nHello Python"

# ^ matches start of string (or line with MULTILINE)
start_matches = re.findall(r"^Hello", text, re.MULTILINE)

# $ matches end of string (or line with MULTILINE)
end_matches = re.findall(r"World$|Python$", text, re.MULTILINE)

# \b matches word boundary
word_boundary = re.findall(r"\bcat\b", "The cat sat on the catalog")

# \B matches non-word boundary
non_boundary = re.findall(r"\Bcat\B", "The cat sat on the catalog")

# Grouping for repetition
text = "abcabcabc"
pattern = r"(abc)+"
match = re.search(pattern, text)

# Grouping for alternation
colors = "The car is red, the bike is blue, the bus is green"
pattern = r"is (red|blue|green)"
matches = re.findall(pattern, colors)

# Extract date components
text = "Meeting on 2024-01-15 and 2024-02-20"
pattern = r"(\d{4})-(\d{2})-(\d{2})"

# findall returns tuples when there are multiple groups
dates = re.findall(pattern, text)

# Using match objects for more control
for match in re.finditer(pattern, text):
    full = match.group(0)
    year = match.group(1)
    month = match.group(2)
    day = match.group(3)

# Named groups with (?P<name>...)
pattern = r"(?P<year>\d{4})-(?P<month>\d{2})-(?P<day>\d{2})"
text = "Event date: 2024-03-15"

match = re.search(pattern, text)

# Capturing group - captures "http" or "https"
capturing = re.findall(
    r"(https?)://(\w+\.\w+)", "Visit https://example.com or http://test.org"
)

# Non-capturing group - only captures the domain
non_capturing = re.findall(
    r"(?:https?)://(\w+\.\w+)", "Visit https://example.com or http://test.org"
)

# Find repeated words
text = "The the quick brown fox jumps over the the lazy dog dog"
pattern = r"\b(\w+)\s+\1\b"
repeated = re.findall(pattern, text, re.IGNORECASE)

# Find matching HTML tags
html = "<div>content</div> <span>text</span> <div>broken</span>"
pattern = r"<(\w+)>.*?</\1>"
valid_tags = re.findall(pattern, html)

text = "100 dollars, 50 euros, 75 pounds"

# Positive lookahead: (?=...)
# Match numbers followed by "dollars"
dollars = re.findall(r"\d+(?= dollars)", text)

# Negative lookahead: (?!...)
# Match numbers NOT followed by "dollars"
not_dollars = re.findall(r"\d+(?! dollars)", text)

text = "$100 €50 £75"

# Positive lookbehind: (?<=...)
# Match numbers preceded by $
usd = re.findall(r"(?<=\$)\d+", text)

# Negative lookbehind: (?<!...)
# Match numbers NOT preceded by $
not_usd = re.findall(r"(?<!\$)\d+", text)

# Practical example: Extract values from key-value pairs
config = "name=Alice, age=30, city=Boston"

# Use lookbehind to find values after specific keys
name = re.search(r"(?<=name=)\w+", config)
age = re.search(r"(?<=age=)\d+", config)

text = """Hello World
hello python
HELLO REGEX"""

# re.IGNORECASE (re.I) - case-insensitive matching
case_insensitive = re.findall(r"hello", text, re.IGNORECASE)

# re.MULTILINE (re.M) - ^ and $ match line boundaries
multiline = re.findall(r"^hello", text, re.IGNORECASE | re.MULTILINE)

# re.DOTALL (re.S) - dot matches newline
dotall = re.findall(r"Hello.*REGEX", text, re.DOTALL)

# re.VERBOSE (re.X) - allows comments and whitespace in patterns
pattern = re.compile(
    r"""
    \d{4}    # Year
    -        # Separator
    \d{2}    # Month
    -        # Separator
    \d{2}    # Day
""",
    re.VERBOSE,
)

import re

sample_text = """
Hey @john_doe! Check out our new product at https://example.com/product?id=123
Contact us at support@company.com or call (555) 123-4567 for help.
Sale ends 2024-12-31! Use code #SAVE20 for 20% off. 
Also follow @tech_news and @deals_daily for updates.
Visit http://blog.example.org or email sales@example.org
#BlackFriday #CyberMonday #Shopping
Meeting scheduled for 01/15/2024. Call +1-800-555-0199.
"""

# Define extraction patterns
patterns = {
    "Emails": r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b",
    "URLs": r"https?://\S+",
    "Mentions": r"@\w+",
    "Hashtags": r"#\w+",
    "Phone Numbers": r"(?:\+?1[-.\s]?)?\(?\d{3}\)?[-.\s]?\d{3}[-.\s]?\d{4}",
    "Dates (ISO)": r"\d{4}-\d{2}-\d{2}",
    "Dates (US)": r"\d{2}/\d{2}/\d{4}",
}

# Extract all entities
for name, pattern in patterns.items():
    matches = re.findall(pattern, sample_text)

# Basic email pattern
email_pattern = r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b"

text = """
Contact us at support@example.com or sales@company.org.
Invalid emails: @missing.com, nodomain@, spaces in@email.com
Edge cases: user.name+tag@sub.domain.co.uk
"""

emails = re.findall(email_pattern, text)

# URL pattern
url_pattern = r"https?://(?:www\.)?[-a-zA-Z0-9@:%._\+~#=]{1,256}\.[a-zA-Z0-9()]{1,6}\b(?:[-a-zA-Z0-9()@:%_\+.~#?&//=]*)"

text = """
Visit https://www.example.com or http://test.org/page?id=123
Also check https://api.service.io/v2/users#section
Not a URL: ftp://other.com or just example.com
"""

urls = re.findall(url_pattern, text)

# US phone number patterns (multiple formats)
phone_pattern = r"""
    (?:
        \+?1[-.\s]?          # Optional country code
    )?
    (?:
        \(?\d{3}\)?          # Area code with optional parens
        [-.\s]?              # Separator
    )
    \d{3}                    # First 3 digits
    [-.\s]?                  # Separator
    \d{4}                    # Last 4 digits
"""

text = """
Call us: (555) 123-4567, 555.123.4567, 555 123 4567
International: +1-555-123-4567, +1 (555) 123-4567
"""

phones = re.findall(phone_pattern, text, re.VERBOSE)

text = """
Dates: 2024-01-15, 01/15/2024, January 15, 2024
Also: 15-Jan-2024, 15 January 2024
"""

# ISO format: YYYY-MM-DD
iso_dates = re.findall(r"\d{4}-\d{2}-\d{2}", text)

# US format: MM/DD/YYYY
us_dates = re.findall(r"\d{2}/\d{2}/\d{4}", text)

# Month name formats
month_names = r"(?:January|February|March|April|May|June|July|August|September|October|November|December|Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)"
named_dates = re.findall(
    rf"(?:\d{{1,2}}[-\s])?{month_names}[-\s]?\d{{1,2}}(?:,?\s*\d{{4}})?", text
)

tweet = "Just learned about #NLP and #MachineLearning! Thanks @professor_ai for the great tutorial. #AI2024"

# Hashtags: # followed by word characters
hashtags = re.findall(r"#\w+", tweet)

# Mentions: @ followed by word characters
mentions = re.findall(r"@\w+", tweet)

text = "Contact john.doe@email.com or jane.smith@company.org"

# Simple replacement
redacted = re.sub(r"\S+@\S+", "[EMAIL REDACTED]", text)

# Using backreferences in replacement
# Swap first and last name in email
swapped = re.sub(r"(\w+)\.(\w+)@", r"\2.\1@", text)


# Using a function for complex replacements
def mask_email(match):
    email = match.group()
    name, domain = email.split("@")
    return f"{name[0]}***@{domain}"


masked = re.sub(r"\S+@\S+", mask_email, text)

import re

# Compile once, use many times
email_regex = re.compile(
    r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b", re.IGNORECASE
)

texts = [
    "Contact support@example.com",
    "No email here",
    "Multiple: a@b.com, c@d.org",
]

results = [email_regex.findall(text) for text in texts]

import time

# Dangerous pattern: nested quantifiers
dangerous_pattern = r"(a+)+"
safe_pattern = r"a+"

# Test string that causes backtracking
test_string = "a" * 25 + "b"

# Time the safe pattern
start = time.perf_counter()
re.search(safe_pattern, test_string)
safe_time = time.perf_counter() - start

# The dangerous pattern would take exponentially longer
# We won't run it with a long string to avoid hanging
short_test = "a" * 10 + "b"
start = time.perf_counter()
re.search(dangerous_pattern, short_test)
dangerous_time = time.perf_counter() - start

# The regex module offers more features and better performance
# Install with: pip install regex
try:
    import regex

    # Possessive quantifiers prevent backtracking
    # a++ means "match one or more 'a', and don't give them back"
    possessive = regex.compile(r"a++b")

    # Atomic groups: (?>...)
    atomic = regex.compile(r"(?>a+)b")

    has_regex = True
except ImportError:
    has_regex = False

import re
from typing import List, Tuple


class TextCleaner:
    """A regex-based text cleaning pipeline for NLP preprocessing."""

    def __init__(self):
        # Compile patterns once
        self.url_pattern = re.compile(r"https?://\S+|www\.\S+")
        self.email_pattern = re.compile(r"\S+@\S+\.\S+")
        self.mention_pattern = re.compile(r"@\w+")
        self.hashtag_pattern = re.compile(r"#\w+")
        self.number_pattern = re.compile(r"\b\d+(?:\.\d+)?\b")
        self.whitespace_pattern = re.compile(r"\s+")
        self.punctuation_pattern = re.compile(r"[^\w\s]")

    def remove_urls(self, text: str) -> str:
        return self.url_pattern.sub(" ", text)

    def remove_emails(self, text: str) -> str:
        return self.email_pattern.sub(" ", text)

    def remove_mentions(self, text: str) -> str:
        return self.mention_pattern.sub(" ", text)

    def extract_hashtags(self, text: str) -> Tuple[str, List[str]]:
        hashtags = self.hashtag_pattern.findall(text)
        cleaned = self.hashtag_pattern.sub(" ", text)
        return cleaned, hashtags

    def normalize_whitespace(self, text: str) -> str:
        return self.whitespace_pattern.sub(" ", text).strip()

    def clean(self, text: str) -> str:
        """Apply full cleaning pipeline."""
        text = self.remove_urls(text)
        text = self.remove_emails(text)
        text = self.remove_mentions(text)
        text, _ = self.extract_hashtags(text)
        text = self.normalize_whitespace(text)
        return text


# Test the pipeline
cleaner = TextCleaner()

sample = """
Check out https://example.com for more info! 
Contact support@company.com or @helpdesk
#MachineLearning is amazing! #NLP #AI
"""

cleaned = cleaner.clean(sample)
_, hashtags = cleaner.extract_hashtags(sample)

# Simple cases: prefer string methods
text = "Hello, World!"

# Bad: regex for simple check
uses_regex = bool(re.search(r"World", text))

# Good: simple 'in' operator
uses_in = "World" in text

# Bad: regex for simple replacement
regex_replace = re.sub(r",", ";", text)

# Good: str.replace()
str_replace = text.replace(",", ";")

# Bad: regex for simple split
regex_split = re.split(r",\s*", "a, b, c")

# Good: str.split() with strip
str_split = [x.strip() for x in "a, b, c".split(",")]