Naive Bayes › Visualizing Naive Bayes

import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
from scipy.special import logsumexp
from sklearn.calibration import calibration_curve
from sklearn.datasets import make_classification
from sklearn.metrics import accuracy_score, brier_score_loss, log_loss
from sklearn.model_selection import train_test_split
from sklearn.naive_bayes import GaussianNB, MultinomialNB

# Generate a two-feature, two-class problem for the decision-region figure.
X_vis, y_vis = make_classification(
    n_samples=300,
    n_features=2,
    n_informative=2,
    n_redundant=0,
    n_classes=2,
    n_clusters_per_class=1,
    class_sep=1.2,
    flip_y=0.02,
    random_state=42,
)

clf_vis = GaussianNB().fit(X_vis, y_vis)

# Build a grid over the feature space and compute the predicted probability
# of class 1 at each grid point for the decision-region figure.
x_min, x_max = X_vis[:, 0].min() - 0.5, X_vis[:, 0].max() + 0.5
y_min, y_max = X_vis[:, 1].min() - 0.5, X_vis[:, 1].max() + 0.5
xx, yy = np.meshgrid(
    np.linspace(x_min, x_max, 300),
    np.linspace(y_min, y_max, 300),
)
grid = np.column_stack([xx.ravel(), yy.ravel()])
Z = clf_vis.predict_proba(grid)[:, 1].reshape(xx.shape)

# Simulate a latent score, its binary label, and two nearly collinear features.
rng = np.random.default_rng(7)
n_cal = 5000

z = rng.normal(size=n_cal)
p_true = 1.0 / (1.0 + np.exp(-2.0 * z))
y_cal = rng.binomial(1, p_true)

# Feature 2 is almost a copy of feature 1, so the independence assumption fails.

x1 = z + 0.45 * rng.normal(size=n_cal)
x2 = x1 + 0.08 * rng.normal(size=n_cal)
X_cal = np.column_stack([x1, x2])

X_train_cal, X_test_cal, y_train_cal, y_test_cal = train_test_split(
    X_cal, y_cal, test_size=0.5, random_state=7, stratify=y_cal
)

clf_cal = GaussianNB().fit(X_train_cal, y_train_cal)

# Predict on the held-out split and collect calibration diagnostics.
p_hat_cal = clf_cal.predict_proba(X_test_cal)[:, 1]

prob_true, prob_pred = calibration_curve(
    y_test_cal, p_hat_cal, n_bins=10, strategy="quantile"
)
brier_cal = brier_score_loss(y_test_cal, p_hat_cal)