Boosted Trees › Implementation in Scikit-learn

import numpy as np
from sklearn.ensemble import GradientBoostingClassifier
from sklearn.model_selection import cross_val_score, train_test_split

np.random.seed(42)
n_samples = 1000

# Generate features for classification
age = np.random.normal(40, 15, n_samples)
age = np.clip(age, 18, 80)

income = np.random.normal(60, 20, n_samples)
income = np.clip(income, 20, 150)

experience = np.random.normal(10, 8, n_samples)
experience = np.clip(experience, 0, 30)

# Create target based on realistic relationship
log_odds = -5 + 0.02 * age + 0.01 * income + 0.05 * experience
prob_purchase = 1 / (1 + np.exp(-log_odds))
y_class = np.random.binomial(1, prob_purchase, n_samples)

X_class = np.column_stack([age, income, experience])  # Combine features

X_train_class, X_test_class, y_train_class, y_test_class = train_test_split(
    X_class, y_class, test_size=0.2, random_state=42, stratify=y_class
)

gb_classifier = GradientBoostingClassifier(
    n_estimators=100,
    learning_rate=0.1,
    max_depth=3,
    min_samples_split=5,
    min_samples_leaf=2,
    random_state=42,
)

gb_classifier.fit(X_train_class, y_train_class)

y_pred_class = gb_classifier.predict(X_test_class)
y_pred_proba_class = gb_classifier.predict_proba(X_test_class)[:, 1]

# Use a more robust cross-validation approach
import warnings

from sklearn.model_selection import StratifiedKFold

# Suppress the specific warning about failed fits
warnings.filterwarnings("ignore", category=UserWarning, module="sklearn")

# Check class distribution to ensure we have both classes
unique_classes, class_counts = np.unique(y_train_class, return_counts=True)
print(
    f"Class distribution in training set: {dict(zip(unique_classes, class_counts))}"
)

# Use StratifiedKFold with error handling
skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)
cv_scores = cross_val_score(
    gb_classifier, X_train_class, y_train_class, cv=skf, scoring="roc_auc"
)

np.random.seed(42)

# Generate features for regression
house_size = np.random.normal(2000, 500, n_samples)
house_size = np.clip(house_size, 800, 4000)

bedrooms = np.random.poisson(3, n_samples)
bedrooms = np.clip(bedrooms, 1, 6)

bathrooms = np.random.poisson(2, n_samples)
bathrooms = np.clip(bathrooms, 1, 4)

# Create target based on realistic relationship
house_price = (
    50000
    + 100 * house_size
    + 20000 * bedrooms
    + 15000 * bathrooms
    + np.random.normal(0, 10000, n_samples)
)
house_price = np.clip(house_price, 100000, 800000)

X_reg = np.column_stack([house_size, bedrooms, bathrooms])

X_train_reg, X_test_reg, y_train_reg, y_test_reg = train_test_split(
    X_reg, house_price, test_size=0.2, random_state=42
)

gb_regressor = GradientBoostingRegressor(
    n_estimators=100,
    learning_rate=0.1,
    max_depth=3,
    min_samples_split=5,
    min_samples_leaf=2,
    random_state=42,
)

gb_regressor.fit(X_train_reg, y_train_reg)
y_pred_reg = gb_regressor.predict(X_test_reg)

param_grid = {
    "n_estimators": [50, 100, 200],
    "learning_rate": [0.01, 0.1, 0.2],
    "max_depth": [3, 5, 7],
    "min_samples_split": [2, 5, 10],
    "min_samples_leaf": [1, 2, 4],
}

# Use a smaller subset for faster tuning
X_train_small = X_train_class[:500]
y_train_small = y_train_class[:500]

grid_search = GridSearchCV(
    GradientBoostingClassifier(random_state=42),
    param_grid,
    cv=3,  # 3-fold CV for faster tuning
    scoring="roc_auc",
    n_jobs=-1,
)

grid_search.fit(X_train_small, y_train_small)