InteractiveSentence Segmentation: From Period Disambiguation to Punkt Algorithm Implementation

# Analyze period usage in sample text
sample_text = """
Dr. Jane Smith, Ph.D., works at the U.S. Department of Energy.
She earned $125.5K last year. Her research on A.I. and M.L. is groundbreaking.
The project, funded by N.I.H., costs approx. $2.5M annually.
Visit her at www.energy.gov for more info. Contact: j.smith@energy.gov
"""

# Count different period types
import re

# Find all periods with context
period_contexts = []
for match in re.finditer(r'.{0,10}\.(?:.{0,10})?', sample_text):
    context = match.group()
    period_contexts.append(context.strip())

# Count total periods
total_periods = sample_text.count('.')

# Count sentence-ending periods (rough heuristic: period followed by space and capital)
sentence_endings = len(re.findall(r'\.\s+[A-Z]', sample_text))

# Count decimal points
decimal_points = len(re.findall(r'\d\.\d', sample_text))

# Count URL/email periods
url_email_periods = len(re.findall(r'www\.|\.gov|\.com|@\w+\.', sample_text))

# Remaining are likely abbreviations
abbreviation_periods = total_periods - sentence_endings - decimal_points - url_email_periods

# Examples of abbreviation ambiguity
ambiguous_cases = [
    ("I work for the U.S. Government.", "U.S. is abbreviation, period ends sentence"),
    ("The U.S. government employs millions.", "U.S. is abbreviation, period does NOT end sentence"),
    ("She has a Ph.D. She teaches at MIT.", "First period ends abbreviation AND sentence"),
    ("Dr. Smith arrived early.", "Dr. is abbreviation, period does NOT end sentence"),
    ("I saw the Dr. He prescribed medicine.", "Unusual: Dr. ends sentence (rare usage)"),
]

# Other sentence terminators and their edge cases
other_terminators = [
    ("What time is it? I need to leave.", "Clear sentence boundary"),
    ("She asked, 'What time is it?' and left.", "Question mark inside quote, sentence continues"),
    ("Yahoo! was founded in 1994.", "Exclamation is part of name, not sentence end"),
    ("Wait! Stop! Don't go!", "Multiple exclamations, each ends a sentence"),
    ("Is this real?! I can't believe it!", "Interrobang usage, each ends sentence"),
]

import re

class SimpleSegmenter:
    """A rule-based sentence segmenter."""
    
    def __init__(self):
        # Common abbreviations that don't end sentences
        self.abbreviations = {
            'mr', 'mrs', 'ms', 'dr', 'prof', 'sr', 'jr',
            'vs', 'etc', 'viz', 'al', 'eg', 'ie', 'cf',
            'inc', 'ltd', 'corp', 'co',
            'jan', 'feb', 'mar', 'apr', 'jun', 'jul', 
            'aug', 'sep', 'oct', 'nov', 'dec',
            'st', 'rd', 'th', 'ave', 'blvd',
            'approx', 'dept', 'est', 'min', 'max',
            'govt', 'natl', 'intl',
        }
        
        # Titles that precede names
        self.titles = {'mr', 'mrs', 'ms', 'dr', 'prof', 'rev', 'gen', 'col', 'lt', 'sgt'}
        
    def is_abbreviation(self, token):
        """Check if token is a known abbreviation."""
        # Remove trailing period for comparison
        word = token.rstrip('.').lower()
        return word in self.abbreviations
    
    def is_likely_sentence_end(self, before, punct, after):
        """Determine if punctuation likely ends a sentence."""
        # Question marks and exclamation points usually end sentences
        if punct in '?!':
            # Unless inside quotes followed by lowercase
            if after and after[0].islower():
                return False
            return True
        
        # For periods, apply heuristics
        if punct == '.':
            # Check if preceding word is abbreviation
            words_before = before.split()
            if words_before:
                last_word = words_before[-1]
                if self.is_abbreviation(last_word + '.'):
                    # Abbreviation followed by lowercase = not sentence end
                    if after and after.strip() and after.strip()[0].islower():
                        return False
                    # Abbreviation followed by capital might still be sentence end
                    # Use additional heuristics...
            
            # Period followed by capital letter suggests sentence boundary
            if after and after.strip():
                first_char = after.strip()[0]
                if first_char.isupper():
                    return True
                    
        return False

def segment(self, text):
    """Split text into sentences."""
    sentences = []
    current = []
    
    # Pattern to find potential sentence boundaries
    # Matches period, question mark, or exclamation followed by space and capital
    boundary_pattern = re.compile(r'([.!?])\s+')
    
    # Split on potential boundaries
    parts = boundary_pattern.split(text)
    
    i = 0
    while i < len(parts):
        current.append(parts[i])
        
        if i + 1 < len(parts) and parts[i + 1] in '.!?':
            punct = parts[i + 1]
            before = ''.join(current)
            after = parts[i + 2] if i + 2 < len(parts) else ''
            
            if self.is_likely_sentence_end(before, punct, after):
                current.append(punct)
                sentences.append(''.join(current).strip())
                current = []
                i += 2
            else:
                current.append(punct)
                i += 2
        else:
            i += 1
    
    # Add remaining text
    if current:
        remaining = ''.join(current).strip()
        if remaining:
            sentences.append(remaining)
    
    return sentences

# Add method to class
SimpleSegmenter.segment = segment

segmenter = SimpleSegmenter()

test_texts = [
    "Hello world. How are you?",
    "Dr. Smith went to Washington. He met the president.",
    "I paid $3.50 for coffee. It was expensive.",
    "She works at U.S. Steel Corp. The company is huge.",
]

results = [(text, segmenter.segment(text)) for text in test_texts]

import math
from collections import defaultdict

class PunktLearner:
    """Simplified Punkt-style abbreviation learner."""
    
    def __init__(self):
        self.word_counts = defaultdict(int)
        self.word_with_period_counts = defaultdict(int)
        self.total_words = 0
        
    def train(self, text):
        """Learn abbreviation patterns from text."""
        # Tokenize simply by whitespace and punctuation
        tokens = re.findall(r'\b\w+\.?|\S', text)
        
        for token in tokens:
            if token.isalpha() or (token.endswith('.') and token[:-1].isalpha()):
                self.total_words += 1
                word = token.rstrip('.').lower()
                self.word_counts[word] += 1
                
                if token.endswith('.'):
                    self.word_with_period_counts[word] += 1
    
    def abbreviation_score(self, word):
        """Calculate likelihood that word is an abbreviation."""
        word = word.lower().rstrip('.')
        
        total = self.word_counts[word]
        with_period = self.word_with_period_counts[word]
        
        if total == 0:
            return 0.0
        
        # Period affinity: fraction of occurrences with period
        period_ratio = with_period / total
        
        # Length penalty: shorter words score higher
        length_factor = 1.0 / (len(word) + 1)
        
        # Frequency weighting: more occurrences = more confidence
        frequency_factor = math.log(total + 1)
        
        # Combine all factors
        score = period_ratio * length_factor * frequency_factor
        
        return score
    
    def get_likely_abbreviations(self, threshold=0.1):
        """Return words likely to be abbreviations."""
        abbrevs = []
        for word in self.word_counts:
            score = self.abbreviation_score(word)
            if score > threshold:
                abbrevs.append((word, score))
        
        return sorted(abbrevs, key=lambda x: -x[1])

# Training corpus with various abbreviations
training_text = """
Dr. Smith and Mrs. Jones met at the U.S. Capitol building.
The meeting was scheduled for 3 p.m. on Jan. 15th.
Mr. Brown, who works at Corp. headquarters, also attended.
Dr. Smith presented findings from the Ph.D. program.
Mrs. Jones discussed the approx. $5M budget for the dept.
The U.S. government approved the proposal. Dr. Smith was pleased.
Mr. Brown noted that Corp. profits exceeded expectations.
The meeting ended at 5 p.m. Everyone agreed it was productive.
"""

learner = PunktLearner()
learner.train(training_text)

# Get learned abbreviations
abbreviations = learner.get_likely_abbreviations(threshold=0.05)

import nltk

# Download the punkt tokenizer data if needed
try:
    nltk.data.find('tokenizers/punkt_tab')
except LookupError:
    nltk.download('punkt_tab', quiet=True)

from nltk.tokenize import sent_tokenize

# Test texts
test_texts = [
    "Dr. Smith went to Washington. He met the president.",
    "I bought 3.5 lbs. of apples. They cost $4.99.",
    "The U.S. economy grew 2.5% in Q3. Experts were surprised.",
    "She asked, 'Are you coming?' He said yes.",
    "Visit us at www.example.com. We're open 24/7!",
]

punkt_results = [(text, sent_tokenize(text)) for text in test_texts]

from nltk.tokenize.punkt import PunktSentenceTokenizer, PunktParameters

# Access the trained parameters
tokenizer = PunktSentenceTokenizer()

# Examine some internal parameters
params = tokenizer._params

# Check if specific words are marked as abbreviations
test_words = ['dr', 'mr', 'inc', 'vs', 'jan', 'approx']
abbrev_status = [(word, word in params.abbrev_types) for word in test_words]

# Edge cases with quotes and parentheses
edge_cases = [
    'He said, "Hello." She waved back.',
    '"Hello," he said. "How are you?"',
    'The book (published in 2020) was popular. It sold millions.',
    'She shouted, "Stop!" He kept running.',
    '(See Appendix A.) The data supports this claim.',
]

edge_results = [(text, sent_tokenize(text)) for text in edge_cases]

# List-style text
list_text = """
The process has three steps:
1. Prepare the data.
2. Train the model.
3. Evaluate results.
Each step is critical for success.
"""

list_sentences = sent_tokenize(list_text.strip())

# Ellipsis examples
ellipsis_texts = [
    "Wait... I think I understand now.",
    "She paused... then continued speaking.",
    "The answer is... well, complicated.",
    "He said he would come... He never did.",
]

ellipsis_results = [(text, sent_tokenize(text)) for text in ellipsis_texts]

# Multiple punctuation
multi_punct = [
    "Really?! I can't believe it!",
    "What...? That makes no sense.",
    'She asked, "Are you sure?!" He nodded.',
    "Wait!! Stop!! Don't do that!!",
]

multi_results = [(text, sent_tokenize(text)) for text in multi_punct]

# Multilingual examples
multilingual_texts = {
    'Spanish': '¿Cómo estás? Estoy bien. ¡Qué bueno!',
    'French': "M. Dupont est arrivé. Il a dit : « Bonjour ! »",
    'German': 'Herr Dr. Müller kam um 15.30 Uhr. Er war pünktlich.',
    'Japanese': '今日は暑いです。明日も暑いでしょう。',
    'Chinese': '今天很热。明天也会很热。',
}

# NLTK has language-specific tokenizers
from nltk.tokenize import sent_tokenize

multi_results = {}
for lang, text in multilingual_texts.items():
    try:
        # Try language-specific tokenization
        if lang == 'German':
            sentences = sent_tokenize(text, language='german')
        elif lang == 'French':
            sentences = sent_tokenize(text, language='french')
        elif lang == 'Spanish':
            sentences = sent_tokenize(text, language='spanish')
        else:
            sentences = sent_tokenize(text)
        multi_results[lang] = sentences
    except Exception as e:
        multi_results[lang] = [f"Error: {e}"]

import spacy

# Load English model
try:
    nlp_en = spacy.load('en_core_web_sm')
except OSError:
    # Model not installed, use blank
    nlp_en = spacy.blank('en')
    nlp_en.add_pipe('sentencizer')

# Test with English
english_text = "Dr. Smith works at U.S. Steel. He's been there for 10 years."
doc = nlp_en(english_text)
spacy_sentences = [sent.text for sent in doc.sents]

def evaluate_segmentation(predicted_sentences, gold_sentences):
    """
    Evaluate sentence segmentation quality.
    
    Compares predicted sentence boundaries against gold standard.
    Returns precision, recall, and F1 score.
    """
    # Convert sentences to boundary positions
    def get_boundaries(sentences):
        boundaries = set()
        pos = 0
        for sent in sentences[:-1]:  # All but last sentence
            pos += len(sent)
            boundaries.add(pos)
        return boundaries
    
    pred_bounds = get_boundaries(predicted_sentences)
    gold_bounds = get_boundaries(gold_sentences)
    
    # Calculate metrics
    true_positives = len(pred_bounds & gold_bounds)
    false_positives = len(pred_bounds - gold_bounds)
    false_negatives = len(gold_bounds - pred_bounds)
    
    precision = true_positives / (true_positives + false_positives) if pred_bounds else 0
    recall = true_positives / (true_positives + false_negatives) if gold_bounds else 0
    f1 = 2 * precision * recall / (precision + recall) if (precision + recall) > 0 else 0
    
    return {
        'precision': precision,
        'recall': recall,
        'f1': f1,
        'true_positives': true_positives,
        'false_positives': false_positives,
        'false_negatives': false_negatives
    }

# Test cases with gold standard segmentation
test_cases = [
    {
        'text': "Dr. Smith arrived. He was early.",
        'gold': ["Dr. Smith arrived.", "He was early."]
    },
    {
        'text': "I paid $3.50 for coffee. It was good.",
        'gold': ["I paid $3.50 for coffee.", "It was good."]
    },
    {
        'text': "The U.S. economy is strong. Growth exceeded 3%.",
        'gold': ["The U.S. economy is strong.", "Growth exceeded 3%."]
    },
]

# Evaluate NLTK Punkt
punkt_scores = []
for case in test_cases:
    predicted = sent_tokenize(case['text'])
    scores = evaluate_segmentation(predicted, case['gold'])
    punkt_scores.append(scores)

# Common error patterns
error_examples = [
    {
        'text': "Prof. Dr. h.c. mult. Hans Schmidt spoke.",
        'issue': "Multiple abbreviated titles",
        'gold': 1,  # One sentence
    },
    {
        'text': "She earned her M.D. She then got a Ph.D.",
        'issue': "Abbreviation at sentence end",
        'gold': 2,  # Two sentences
    },
    {
        'text': "Visit example.com. Click the link.",
        'issue': "Domain name mistaken for abbreviation",
        'gold': 2,  # Two sentences
    },
]

error_analysis = []
for example in error_examples:
    predicted = sent_tokenize(example['text'])
    error_analysis.append({
        'text': example['text'],
        'issue': example['issue'],
        'gold_count': example['gold'],
        'pred_count': len(predicted),
        'correct': len(predicted) == example['gold']
    })

class ProductionSegmenter:
    """
    A production-ready sentence segmenter combining multiple approaches.
    """
    
    def __init__(self, use_spacy=False):
        self.use_spacy = use_spacy
        
        # Precompile regex patterns
        self.url_pattern = re.compile(
            r'https?://\S+|www\.\S+|\S+\.(com|org|net|edu|gov)\b'
        )
        self.email_pattern = re.compile(r'\S+@\S+\.\S+')
        self.number_pattern = re.compile(r'\d+\.\d+')
        
        # Placeholder for protected content
        self.placeholder_map = {}
        
    def _protect_special(self, text):
        """Replace URLs, emails, and numbers with placeholders."""
        self.placeholder_map = {}
        counter = 0
        
        for pattern in [self.url_pattern, self.email_pattern, self.number_pattern]:
            for match in pattern.finditer(text):
                placeholder = f"__PROTECTED_{counter}__"
                self.placeholder_map[placeholder] = match.group()
                text = text.replace(match.group(), placeholder, 1)
                counter += 1
        
        return text
    
    def _restore_special(self, sentences):
        """Restore protected content in sentences."""
        restored = []
        for sent in sentences:
            for placeholder, original in self.placeholder_map.items():
                sent = sent.replace(placeholder, original)
            restored.append(sent)
        return restored
    
    def segment(self, text):
        """Segment text into sentences."""
        # Protect special content
        protected_text = self._protect_special(text)
        
        # Use NLTK Punkt for segmentation
        sentences = sent_tokenize(protected_text)
        
        # Restore protected content
        sentences = self._restore_special(sentences)
        
        # Post-process: merge incorrectly split sentences
        sentences = self._merge_fragments(sentences)
        
        return sentences
    
    def _merge_fragments(self, sentences):
        """Merge sentence fragments that were incorrectly split."""
        if len(sentences) <= 1:
            return sentences
        
        merged = [sentences[0]]
        for sent in sentences[1:]:
            # If sentence starts with lowercase, merge with previous
            if sent and sent[0].islower():
                merged[-1] = merged[-1] + ' ' + sent
            else:
                merged.append(sent)
        
        return merged

prod_segmenter = ProductionSegmenter()

production_tests = [
    "Visit https://example.com/page.html for details. Click the link.",
    "Contact john.doe@company.com for help. Response time is 24hrs.",
    "The price is $19.99 per month. That's a 50% discount.",
    "Dr. Jane Smith, Ph.D., leads the team. She has 20 years of experience.",
]

prod_results = [(text, prod_segmenter.segment(text)) for text in production_tests]