Classification Applications › Code Implementation

import subprocess

subprocess.run(
    [
        "uv",
        "pip",
        "install",
        "transformers",
        "torch",
        "datasets",
        "scikit-learn",
        "textblob",
    ],
    capture_output=True,
    check=False,
)

import random
from collections import Counter

import numpy as np

# Reproducibility
np.random.seed(42)
random.seed(42)

# Sample sentiment dataset (movie reviews style)
sentiment_data = [
    (
        "This movie was absolutely brilliant. The performances were outstanding.",
        "positive",
    ),
    ("A masterpiece of modern cinema. I was moved to tears.", "positive"),
    ("The story captivated me from the first scene to the last.", "positive"),
    (
        "Wonderful direction and a compelling script. Highly recommended.",
        "positive",
    ),
    ("I loved every minute of this film. Instant classic.", "positive"),
    ("Surprisingly good. Exceeded all my expectations.", "positive"),
    (
        "Beautifully crafted. The cinematography alone is worth the ticket price.",
        "positive",
    ),
    ("One of the best films I've seen this decade.", "positive"),
    ("This movie was a complete waste of time. Terrible acting.", "negative"),
    (
        "Boring, predictable, and painfully slow. Avoid at all costs.",
        "negative",
    ),
    ("The plot made no sense and the dialogue was cringeworthy.", "negative"),
    ("I fell asleep halfway through. Nothing happens.", "negative"),
    ("The worst movie I've seen in years. Deeply disappointing.", "negative"),
    ("A confusing mess. The director clearly had no vision.", "negative"),
    (
        "Poorly written, poorly acted, poorly directed. A complete failure.",
        "negative",
    ),
    ("Two hours of my life I'll never get back.", "negative"),
    ("It was fine. Nothing special, nothing terrible.", "neutral"),
    ("Watchable but forgettable. An average entry in the genre.", "neutral"),
    ("Some good moments but overall not particularly memorable.", "neutral"),
    (
        "Decent enough for a rainy afternoon. Nothing to get excited about.",
        "neutral",
    ),
]

# Sample intent dataset (banking assistant)
intent_data = [
    ("What is my current balance?", "check_balance"),
    ("How much money do I have in my account?", "check_balance"),
    ("Show me my account balance", "check_balance"),
    ("Can you tell me how much is in my savings?", "check_balance"),
    ("Transfer 50 dollars to my sister", "transfer_funds"),
    ("Send money to John Smith", "transfer_funds"),
    ("Move 200 from checking to savings", "transfer_funds"),
    ("I want to wire funds to my friend", "transfer_funds"),
    ("Pay my electric bill", "pay_bill"),
    ("Schedule a payment to Comcast", "pay_bill"),
    ("I need to pay my rent this month", "pay_bill"),
    ("Set up auto-pay for my credit card", "pay_bill"),
    ("Someone used my card without permission", "report_fraud"),
    ("I see a transaction I don't recognize", "report_fraud"),
    ("There's a suspicious charge on my account", "report_fraud"),
    ("I think my card was stolen", "report_fraud"),
    ("Can you recommend a good restaurant nearby?", "out_of_scope"),
    ("What's the weather like today?", "out_of_scope"),
    ("Tell me a joke", "out_of_scope"),
    ("How do I get to downtown?", "out_of_scope"),
]

# Compute label distribution for display
sent_counts = Counter(label for _, label in sentiment_data)

from transformers import pipeline

# Load a pre-trained sentiment classifier
# distilbert-sst2 is a small, fast model trained on movie review sentiment
sentiment_pipeline = pipeline(
    "text-classification",
    model="distilbert-base-uncased-finetuned-sst-2-english",
    device="mps",  # MPS for Apple Silicon; use -1 for CPU-only systems
)

# Test the model on our sample sentences
test_sentences = [
    "This product is absolutely amazing. Best purchase I've made all year.",
    "Terrible quality. Broke after two days of use.",
    "It works as described. Nothing more, nothing less.",
    "I would never say this product is good.",  # Negation challenge
    "Oh great, another software update that breaks everything.",  # Sarcasm
]

results = sentiment_pipeline(test_sentences)

from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import cross_val_score
from sklearn.pipeline import Pipeline

# Prepare data
texts = [text for text, _ in sentiment_data]
labels = [label for _, label in sentiment_data]

# Build TF-IDF + logistic regression pipeline
tfidf_clf = Pipeline(
    [
        (
            "tfidf",
            TfidfVectorizer(ngram_range=(1, 2), max_features=5000, min_df=1),
        ),
        ("clf", LogisticRegression(max_iter=500, random_state=42)),
    ]
)

# Cross-validate (small dataset, so use leave-one-out style)
# With only 20 examples, we use 5-fold CV as a rough estimate
cv_scores = cross_val_score(tfidf_clf, texts, labels, cv=4, scoring="accuracy")

# Fit on all data and inspect top features per class
tfidf_clf.fit(texts, labels)
vectorizer = tfidf_clf.named_steps["tfidf"]
clf = tfidf_clf.named_steps["clf"]
feature_names = vectorizer.get_feature_names_out()

# Top features per class
n_top = 8

from transformers import pipeline

# cross-encoder/nli-deberta-v3-small is a compact NLI model for zero-shot classification
# (facebook/bart-large-mnli is a larger alternative but requires ~1.6GB download)
zero_shot_clf = pipeline(
    "zero-shot-classification",
    model="cross-encoder/nli-deberta-v3-small",
    device="mps",  # MPS for Apple Silicon; use -1 for CPU-only systems
)

# Topic categories for a news classification system
news_categories = [
    "politics",
    "sports",
    "technology",
    "health",
    "business",
    "science",
]

# Test documents
test_articles = [
    "The Senate passed the infrastructure bill with bipartisan support, sending it to the President for signature.",
    "The quarterback threw three touchdown passes in the fourth quarter to complete the comeback victory.",
    "Researchers have developed a new neural network architecture that reduces training time by 40 percent.",
    "A new study links regular exercise to a 30 percent reduction in the risk of cardiovascular disease.",
    "The company reported record earnings this quarter, driven by strong demand in emerging markets.",
]

# Run zero-shot classification on each article
zero_shot_results = []
for article in test_articles:
    result = zero_shot_clf(article, news_categories)
    zero_shot_results.append(result)

# Use the same zero-shot NLI approach for intent detection
# Reuse the zero_shot_clf loaded in the previous section
# In production, you would fine-tune a model on labeled intent data
intent_clf = zero_shot_clf

# Define the in-scope intents
in_scope_intents = [
    "check_balance",
    "transfer_funds",
    "pay_bill",
    "report_fraud",
]

# Out-of-scope threshold: below this confidence, reject
OOS_THRESHOLD = 0.50


def classify_intent(utterance, intents, threshold):
    """Classify an utterance with OOS detection."""
    result = intent_clf(utterance, intents)
    top_intent = result["labels"][0]
    top_score = result["scores"][0]

    if top_score < threshold:
        return "out_of_scope", top_score, top_intent
    return top_intent, top_score, None


# Test utterances including OOS examples
test_utterances = [
    "What's my account balance right now?",
    "Send $200 to my roommate",
    "Pay my internet bill",
    "There's a charge I don't recognize on my Visa",
    "What's a good recipe for pasta?",  # OOS
    "Can you call a taxi for me?",  # OOS
    "Tell me today's weather forecast",  # OOS
]