Michael Brenndoerfer | Data & AI, Private Equity, Technology

Michael Brenndoerfer

DeBERTa › Complete DeBERTa Model

Michael BrenndoerferUpdated July 20, 20252 min read

Language AI HandbookAll chapters

Part 24 of 67BERT and Variants

Chapter 176DeBERTa › Complete DeBERTa Model

Reading section 8 of 17 · Complete DeBERTa Model · 2 minSection 8 of 17 · 2 min

Section 8 of 17 · 2 min read

DeBERTa › Complete DeBERTa ModelOne section · Sections in this chapter8 / 17 ↓

Sections in this chapter17

Reading tools Definitions & inline links

Choose your expertise level to adjust how many terms are explained. Beginners see more tooltips, experts see fewer to maintain reading flow. Hover over underlined terms for instant definitions.

Article links

Make inline references clickable

Let's assemble the components into a complete DeBERTa model to see how the pieces fit together end to end:

In[18]:

Code

class DeBERTaEncoderLayer(nn.Module):
    """Single DeBERTa encoder layer."""

    def __init__(
        self,
        hidden_size: int,
        num_heads: int,
        intermediate_size: int,
        max_relative_positions: int,
        dropout: float,
    ):
        super().__init__()
        self.attention = DisentangledAttention(
            hidden_size, num_heads, max_relative_positions, dropout
        )
        self.attention_norm = nn.LayerNorm(hidden_size)
        self.ffn = nn.Sequential(
            nn.Linear(hidden_size, intermediate_size),
            nn.GELU(),
            nn.Dropout(dropout),
            nn.Linear(intermediate_size, hidden_size),
            nn.Dropout(dropout),
        )
        self.ffn_norm = nn.LayerNorm(hidden_size)

    def forward(
        self,
        hidden_states: torch.Tensor,
        attention_mask: torch.Tensor | None = None,
    ) -> torch.Tensor:
        attn_output, _ = self.attention(hidden_states, attention_mask)
        hidden_states = self.attention_norm(hidden_states + attn_output)
        ffn_output = self.ffn(hidden_states)
        hidden_states = self.ffn_norm(hidden_states + ffn_output)
        return hidden_states


class DeBERTaEncoder(nn.Module):
    """DeBERTa encoder with disentangled attention."""

    def __init__(
        self,
        vocab_size: int = 30522,
        hidden_size: int = 768,
        num_layers: int = 12,
        num_heads: int = 12,
        intermediate_size: int = 3072,
        max_relative_positions: int = 512,
        max_position: int = 512,
        dropout: float = 0.1,
    ):
        super().__init__()

        # Token embeddings only (no position embeddings at input)
        self.token_embedding = nn.Embedding(vocab_size, hidden_size)
        self.embedding_norm = nn.LayerNorm(hidden_size)
        self.embedding_dropout = nn.Dropout(dropout)

        self.layers = nn.ModuleList(
            [
                DeBERTaEncoderLayer(
                    hidden_size,
                    num_heads,
                    intermediate_size,
                    max_relative_positions,
                    dropout,
                )
                for _ in range(num_layers)
            ]
        )

        self.emd = EnhancedMaskDecoder(
            hidden_size,
            num_heads,
            intermediate_size,
            max_position,
            num_layers=2,
            dropout=dropout,
        )

    def forward(
        self,
        input_ids: torch.Tensor,
        attention_mask: torch.Tensor | None = None,
    ) -> torch.Tensor:
        hidden_states = self.token_embedding(input_ids)
        hidden_states = self.embedding_norm(hidden_states)
        hidden_states = self.embedding_dropout(hidden_states)

        for layer in self.layers:
            hidden_states = layer(hidden_states, attention_mask)

        hidden_states = self.emd(hidden_states, attention_mask)

        return hidden_states

Out[19]:

Console

DeBERTa Encoder Test:
  Input shape: torch.Size([2, 16])
  Output shape: torch.Size([2, 16, 64])
  Total parameters: 225,024

The complete encoder processes token IDs through disentangled attention layers and the Enhanced Mask Decoder, producing contextualized representations that carry both relative structural information from the encoder and absolute positional information from the EMD. This small test model has around 230K parameters for demonstration. A full DeBERTa-Base would have approximately 140 million parameters. This reflects the additional position projection matrices ( $W_q^p$ and $W_k^p$ ), the separate relative position embedding table, and the EMD layers compared to BERT.