Term Frequency › Code Implementation

from collections import Counter

import numpy as np

# Sample corpus - three documents of different lengths
corpus = [
    "the cat sat on the mat the cat sat",  # 9 tokens
    "the dog chased the cat across the yard",  # 8 tokens
    "a neural network learns patterns in data patterns patterns patterns",  # 10 tokens
]

# Tokenize and build vocabulary
tokenized = [doc.lower().split() for doc in corpus]
vocab = sorted(set(word for doc in tokenized for word in doc))
word_to_idx = {word: i for i, word in enumerate(vocab)}


# Build raw count matrix
def build_count_matrix(tokenized_docs, vocab, word_to_idx):
    """Build term-document count matrix."""
    n_docs = len(tokenized_docs)
    n_terms = len(vocab)
    matrix = np.zeros((n_docs, n_terms), dtype=float)
    for doc_idx, tokens in enumerate(tokenized_docs):
        counts = Counter(tokens)
        for term, count in counts.items():
            if term in word_to_idx:
                matrix[doc_idx, word_to_idx[term]] = count
    return matrix


count_matrix = build_count_matrix(tokenized, vocab, word_to_idx)

def tf_raw(count_matrix):
    """Raw term frequency: just the counts."""
    return count_matrix.copy()


def tf_log(count_matrix):
    """Log-scaled term frequency: 1 + log(count) for count > 0."""
    tf = np.zeros_like(count_matrix, dtype=float)
    nonzero = count_matrix > 0
    tf[nonzero] = 1 + np.log(count_matrix[nonzero])
    return tf


def tf_boolean(count_matrix):
    """Boolean term frequency: 1 if present, 0 otherwise."""
    return (count_matrix > 0).astype(float)


def tf_augmented(count_matrix, K=0.5):
    """Augmented term frequency: K + (1-K) * count / max_count."""
    tf = np.zeros_like(count_matrix, dtype=float)
    for i in range(count_matrix.shape[0]):
        row = count_matrix[i]
        max_count = row.max()
        if max_count > 0:
            nonzero = row > 0
            tf[i, nonzero] = K + (1 - K) * row[nonzero] / max_count
    return tf


def tf_l2_normalized(count_matrix):
    """L2-normalized term frequency."""
    norms = np.linalg.norm(count_matrix, axis=1, keepdims=True)
    # Avoid division by zero for all-zero documents
    norms[norms == 0] = 1.0
    return count_matrix / norms

l2_matrix = tf_l2_normalized(count_matrix)
norms_after = np.linalg.norm(l2_matrix, axis=1)

from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer

# CountVectorizer gives raw counts
count_vec = CountVectorizer()
sk_counts = count_vec.fit_transform(corpus).toarray().astype(float)
sk_vocab = count_vec.get_feature_names_out()

# TfidfVectorizer with use_idf=False gives log TF with L2 normalization
# sublinear_tf=True applies 1+log(tf), norm='l2' applies L2 normalization
tfidf_log_l2 = TfidfVectorizer(use_idf=False, sublinear_tf=True, norm="l2")
sk_log_l2 = tfidf_log_l2.fit_transform(corpus).toarray()