Model Cards › Code: Building a Model Card

import warnings

import numpy as np

warnings.filterwarnings("ignore")

# Simulate a labeled dataset with demographic metadata
np.random.seed(42)

# Create synthetic reviews with sentiment labels and demographic metadata
# In a real scenario, this would be actual labeled data
n_samples = 2000

# Simulate text features as TF-IDF-like vectors (20 features)
n_features = 20

# Generate base feature distributions for positive and negative classes
pos_mean = np.random.randn(n_features) * 0.5
neg_mean = -pos_mean * 0.8

# Create samples
half = n_samples // 2
X_pos = np.random.randn(half, n_features) * 0.8 + pos_mean
X_neg = np.random.randn(half, n_features) * 0.8 + neg_mean

X = np.vstack([X_pos, X_neg])
y = np.array([1] * half + [0] * half)

# Add demographic metadata for disaggregated evaluation. Each category contains
# both sentiment classes, while apparel reviews receive additional feature noise
# to create a modest, measurable performance difference.
categories = np.tile(["electronics", "apparel"], n_samples // 2)
apparel_mask = categories == "apparel"
X[apparel_mask] += np.random.randn(apparel_mask.sum(), n_features) * 0.45

# Shuffle consistently
shuffle_idx = np.random.permutation(n_samples)
X = X[shuffle_idx]
y = y[shuffle_idx]
categories = categories[shuffle_idx]

from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import train_test_split

# Split into train/test while preserving every class-category combination
strata = np.array(
    [f"{label}-{category}" for label, category in zip(y, categories)]
)
X_train, X_test, y_train, y_test, cat_train, cat_test = train_test_split(
    X, y, categories, test_size=0.2, random_state=42, stratify=strata
)

# Train the model
model = LogisticRegression(max_iter=1000, random_state=42)
model.fit(X_train, y_train)

# Disaggregated evaluation by product category
subgroup_results = {}

for category in np.unique(cat_test):
    mask = cat_test == category
    y_sub = y_test[mask]
    y_pred_sub = model.predict(X_test[mask])
    y_prob_sub = model.predict_proba(X_test[mask])[:, 1]

    subgroup_results[category] = {
        "n": mask.sum(),
        "accuracy": accuracy_score(y_sub, y_pred_sub),
        "f1": f1_score(y_sub, y_pred_sub),
        "auc": roc_auc_score(y_sub, y_prob_sub),
        "pos_rate": y_sub.mean(),
    }

from sklearn.calibration import calibration_curve

# Compute calibration data
fraction_pos, mean_pred_prob = calibration_curve(
    y_test, y_prob, n_bins=6, strategy="quantile"
)

# Also compute calibration error
# Expected Calibration Error (ECE)
bin_edges = np.linspace(0, 1, 11)
ece = 0.0
for i in range(len(bin_edges) - 1):
    mask = (y_prob >= bin_edges[i]) & (y_prob < bin_edges[i + 1])
    if mask.sum() > 0:
        bin_acc = y_test[mask].mean()
        bin_conf = y_prob[mask].mean()
        bin_size = mask.sum() / len(y_test)
        ece += bin_size * abs(bin_acc - bin_conf)

from datetime import datetime


def generate_model_card(
    model_name,
    version,
    model_type,
    training_data_description,
    intended_uses,
    out_of_scope_uses,
    overall_metrics,
    subgroup_metrics,
    known_limitations,
    ethical_considerations,
    recommendations,
):
    """Generate a structured model card as a formatted string."""

    lines = []
    lines.append(f"# Model Card: {model_name}")
    lines.append(f"\n**Version:** {version}")
    lines.append(f"**Type:** {model_type}")
    lines.append(f"**Generated:** {datetime.now().strftime('%Y-%m-%d')}\n")

    lines.append("## Intended Use\n")
    lines.append("**Primary uses:**")
    for use in intended_uses:
        lines.append(f"- {use}")

    lines.append("\n**Out-of-scope uses:**")
    for use in out_of_scope_uses:
        lines.append(f"- {use}")

    lines.append("\n## Training Data\n")
    lines.append(training_data_description)

    lines.append("\n## Evaluation Results\n")
    lines.append("**Overall performance:**")
    for metric, value in overall_metrics.items():
        lines.append(f"- {metric}: {value:.4f}")

    lines.append("\n**Disaggregated performance:**")
    for subgroup, metrics in subgroup_metrics.items():
        lines.append(f"\n*{subgroup}* (n={metrics['n']}):")
        lines.append(f"- Accuracy: {metrics['accuracy']:.4f}")
        lines.append(f"- F1 Score: {metrics['f1']:.4f}")
        lines.append(f"- ROC-AUC:  {metrics['auc']:.4f}")

    lines.append("\n## Known Limitations\n")
    for limitation in known_limitations:
        lines.append(f"- {limitation}")

    lines.append("\n## Ethical Considerations\n")
    for consideration in ethical_considerations:
        lines.append(f"- {consideration}")

    lines.append("\n## Recommendations\n")
    for rec in recommendations:
        lines.append(f"- {rec}")

    return "\n".join(lines)