Perplexity › Held-Out Evaluation Methodology

def split_data(sentences, train_ratio=0.7, dev_ratio=0.15, seed=42):
    """Split data into train/dev/test sets."""
    import random

    random.seed(seed)

    shuffled = sentences.copy()
    random.shuffle(shuffled)

    n = len(shuffled)
    train_end = int(n * train_ratio)
    dev_end = int(n * (train_ratio + dev_ratio))

    return {
        "train": shuffled[:train_end],
        "dev": shuffled[train_end:dev_end],
        "test": shuffled[dev_end:],
    }


# Create a larger corpus for demonstration
larger_corpus = """
the quick brown fox jumps over the lazy dog
a quick brown dog runs through the green field
the lazy cat sleeps on the warm mat
a small bird sings in the tall tree
the big dog chases the small cat
a brown fox hides behind the old tree
the green field stretches to the blue horizon
a lazy dog lies in the warm sun
the tall tree provides cool shade
a quick cat catches the small mouse
the warm sun shines on the green grass
a big bird flies over the blue lake
the old tree stands in the open field
a small mouse hides in the tall grass
the blue lake reflects the clear sky
the red fox runs across the dry plain
a gentle breeze blows through the open window
the young girl reads a long book
a fast river flows down the steep hill
the clever rabbit hops over the wooden fence
a hungry wolf howls at the bright moon
the small house sits near the wide river
a white cloud drifts across the pale sky
the bold eagle soars above the rocky cliff
a tired horse rests beside the wooden barn
the cold wind sweeps across the frozen lake
a playful puppy chases the orange butterfly
the old farmer walks along the dusty road
a dark storm gathers above the distant hills
the curious fox peers through the tall bushes
a sleepy owl blinks in the fading light
the noisy crow perches on the broken fence
a swift deer leaps over the mossy log
the happy child runs through the wet grass
a quiet stream winds between the smooth stones
the bright sun warms the empty courtyard
a grey rabbit nibbles the fresh green clover
the spotted fish swims beneath the clear water
a strong wind bends the slender birch trees
the golden harvest fills the wide open barn
""".strip().split("\n")

corpus_sentences = [line.lower().split() for line in larger_corpus]
splits = split_data(corpus_sentences)

def tune_smoothing(train_sentences, dev_sentences, n=2):
    """Find optimal smoothing parameter k using dev set."""
    k_values = [0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1.0]
    results = []

    for k in k_values:
        model = NgramModel(n=n, k=k)
        model.train(train_sentences)
        pp = model.perplexity(dev_sentences)
        results.append((k, pp))

    return results


tuning_results = tune_smoothing(splits["train"], splits["dev"], n=2)
best_k, best_pp = min(tuning_results, key=lambda x: x[1])

# Train final model with best k
final_model = NgramModel(n=2, k=best_k)
final_model.train(splits["train"])

# Evaluate on test set
test_pp = final_model.perplexity(splits["test"])