Qwen Architecture › Tokenization Strategy

# Simulated tokenization comparison
# (Actual tokenization would require the Qwen tokenizer)


def estimate_tokens_chinese(text: str, vocab_type: str) -> int:
    """Estimate token count for Chinese text"""
    char_count = len(text)

    if vocab_type == "llama":
        # LLaMA often splits Chinese chars into 2-3 tokens
        return int(char_count * 2.5)
    elif vocab_type == "qwen":
        # Qwen has dedicated Chinese character tokens
        return int(char_count * 1.2)
    return char_count


def estimate_tokens_english(text: str, vocab_type: str) -> int:
    """Estimate token count for English text"""
    word_count = len(text.split())

    # Both handle English similarly
    if vocab_type in ["llama", "qwen"]:
        return int(word_count * 1.3)
    return word_count


# Example texts
chinese_text = "人工智能正在改变我们的生活方式和工作模式"
english_text = "Artificial intelligence is transforming how we live and work"

# Estimate token counts
results = {
    "Chinese": {
        "LLaMA": estimate_tokens_chinese(chinese_text, "llama"),
        "Qwen": estimate_tokens_chinese(chinese_text, "qwen"),
    },
    "English": {
        "LLaMA": estimate_tokens_english(english_text, "llama"),
        "Qwen": estimate_tokens_english(english_text, "qwen"),
    },
}