InteractiveTF-IDF and Bag of Words: Complete Guide to Text Representation & Information Retrieval

1def tokenize(text):
2    """Split text into lowercase words, removing punctuation."""
3    import re
4    # Convert to lowercase and split on non-word characters
5    tokens = re.findall(r'\b\w+\b', text.lower())
6    return tokens
7
8## Example documents
9documents = [
10    "the cat sat on the mat",
11    "the dog sat on the log",
12    "the cat and dog sat"
13]
14
15## Tokenize all documents
16tokenized_docs = [tokenize(doc) for doc in documents]
17print("Tokenized documents:")
18for i, tokens in enumerate(tokenized_docs, 1):
19    print(f"Doc {i}: {tokens}")

1def build_vocabulary(tokenized_documents):
2    """Create a sorted vocabulary from all tokenized documents."""
3    # Collect all unique words
4    all_words = set()
5    for tokens in tokenized_documents:
6        all_words.update(tokens)
7    
8    # Sort for consistent ordering
9    vocabulary = sorted(all_words)
10    
11    # Create word-to-index mapping
12    word_to_idx = {word: idx for idx, word in enumerate(vocabulary)}
13    
14    return vocabulary, word_to_idx
15
16vocabulary, word_to_idx = build_vocabulary(tokenized_docs)
17print(f"Vocabulary ({len(vocabulary)} words): {vocabulary}")
18print(f"\nWord to index mapping:")
19for word, idx in word_to_idx.items():
20    print(f"  {word}: {idx}")

1def bag_of_words_vectorize(tokenized_doc, word_to_idx):
2    """Convert a tokenized document to a Bag of Words vector."""
3    # Initialize vector with zeros
4    vector = [0] * len(word_to_idx)
5    
6    # Count word occurrences
7    for word in tokenized_doc:
8        if word in word_to_idx:
9            idx = word_to_idx[word]
10            vector[idx] += 1
11    
12    return vector
13
14## Vectorize all documents
15bow_vectors = [bag_of_words_vectorize(tokens, word_to_idx) 
16               for tokens in tokenized_docs]
17
18print("Bag of Words vectors:")
19for i, vector in enumerate(bow_vectors, 1):
20    print(f"Doc {i}: {vector}")

1import math
2
3def term_frequency(word, tokenized_doc, use_log=False):
4    """Calculate term frequency for a word in a document."""
5    count = tokenized_doc.count(word)
6    
7    if count == 0:
8        return 0.0
9    
10    if use_log:
11        # Logarithmic scaling
12        return 1 + math.log(count)
13    else:
14        # Normalized by document length
15        return count / len(tokenized_doc)
16
17## Calculate TF for "cat" in each document
18print("Term Frequency for 'cat':")
19for i, tokens in enumerate(tokenized_docs, 1):
20    tf_normalized = term_frequency("cat", tokens, use_log=False)
21    tf_log = term_frequency("cat", tokens, use_log=True)
22    print(f"Doc {i}: normalized={tf_normalized:.3f}, log={tf_log:.3f}")

1def inverse_document_frequency(word, tokenized_documents):
2    """Calculate IDF for a word across a document collection."""
3    # Count documents containing the word
4    doc_count = sum(1 for tokens in tokenized_documents if word in tokens)
5    
6    if doc_count == 0:
7        return 0.0  # Word doesn't appear anywhere
8    
9    total_docs = len(tokenized_documents)
10    # Standard IDF formula
11    idf = math.log(total_docs / doc_count)
12    return idf
13
14## Calculate IDF for each word in vocabulary
15print("Inverse Document Frequency (IDF):")
16idf_scores = {}
17for word in vocabulary:
18    idf = inverse_document_frequency(word, tokenized_docs)
19    idf_scores[word] = idf
20    print(f"  {word:6s}: {idf:.3f} (appears in {sum(1 for tokens in tokenized_docs if word in tokens)}/{len(tokenized_docs)} docs)")

1def tfidf_vectorize(tokenized_doc, word_to_idx, tokenized_documents, 
2                    use_log_tf=False):
3    """Convert a tokenized document to a TF-IDF vector."""
4    vector = [0.0] * len(word_to_idx)
5    
6    for word, idx in word_to_idx.items():
7        # Calculate TF
8        tf = term_frequency(word, tokenized_doc, use_log=use_log_tf)
9        
10        # Calculate IDF
11        idf = inverse_document_frequency(word, tokenized_documents)
12        
13        # TF-IDF is the product
14        vector[idx] = tf * idf
15    
16    return vector
17
18## Compute TF-IDF vectors for all documents
19tfidf_vectors = [tfidf_vectorize(tokens, word_to_idx, tokenized_docs, 
20                                  use_log_tf=False) 
21                 for tokens in tokenized_docs]
22
23print("TF-IDF vectors (using normalized TF):")
24for i, vector in enumerate(tfidf_vectors, 1):
25    print(f"\nDoc {i}:")
26    # Show non-zero values for clarity
27    non_zero = [(vocabulary[j], f"{vector[j]:.3f}") 
28                for j, val in enumerate(vector) if val > 0]
29    for word, score in sorted(non_zero, key=lambda x: float(x[1]), reverse=True):
30        print(f"  {word:6s}: {score}")

1def cosine_similarity(vec1, vec2):
2    """Calculate cosine similarity between two vectors."""
3    import math
4    
5    # Dot product
6    dot_product = sum(a * b for a, b in zip(vec1, vec2))
7    
8    # Magnitudes
9    magnitude1 = math.sqrt(sum(a * a for a in vec1))
10    magnitude2 = math.sqrt(sum(b * b for b in vec2))
11    
12    if magnitude1 == 0 or magnitude2 == 0:
13        return 0.0
14    
15    return dot_product / (magnitude1 * magnitude2)
16
17## Compare all pairs of documents
18print("Cosine similarity between documents (using TF-IDF):")
19for i in range(len(tfidf_vectors)):
20    for j in range(i + 1, len(tfidf_vectors)):
21        similarity = cosine_similarity(tfidf_vectors[i], tfidf_vectors[j])
22        print(f"Doc {i+1} vs Doc {j+1}: {similarity:.3f}")

1from sklearn.feature_extraction.text import TfidfVectorizer
2
3## Initialize vectorizer
4vectorizer = TfidfVectorizer(
5    lowercase=True,
6    token_pattern=r'\b\w+\b',  # Word boundaries
7    max_features=1000,  # Limit vocabulary size
8    min_df=2,  # Ignore words appearing in < 2 documents
9    max_df=0.95  # Ignore words appearing in > 95% of documents
10)
11
12## Fit and transform
13tfidf_matrix = vectorizer.fit_transform(documents)
14
15print(f"TF-IDF matrix shape: {tfidf_matrix.shape}")
16print(f"Vocabulary size: {len(vectorizer.vocabulary_)}")
17print(f"\nSample vocabulary words: {list(vectorizer.vocabulary_.keys())[:10]}")

1import numpy as np
2
3## Convert to dense array for display (not recommended for large datasets)
4dense_matrix = tfidf_matrix.toarray()
5
6print("TF-IDF matrix (documents × words):")
7print(f"\nWords: {vectorizer.get_feature_names_out()}")
8print(f"\nMatrix:\n{dense_matrix}")