DPO Variants › Implementation Comparison
Michael BrenndoerferJanuary 3, 20265 min read
Language AI HandbookAll chapters Part 37 of 67Alignment and RLHF
DPO Variants › Implementation ComparisonOne section · Sections in this chapter7 / 12 ↓
Sections in this chapter12
Reading tools Definitions & inline links
Choose your expertise level to adjust how many terms are explained. Beginners see more tooltips, experts see fewer to maintain reading flow. Hover over underlined terms for instant definitions.
Article links
Make inline references clickable
With the theory and intuition in place, let's implement all four variants to see the differences in practice. Clear implementations help reveal the structural similarities and differences that the mathematics sometimes obscures.
In[11]:
Codeimport torch
import torch.nn.functional as F
def compute_log_ratios(
policy_chosen_logps: torch.Tensor,
policy_rejected_logps: torch.Tensor,
reference_chosen_logps: torch.Tensor,
reference_rejected_logps: torch.Tensor,
) -> torch.Tensor:
"""Compute the log-ratio difference h_θ used by DPO, IPO, and cDPO."""
chosen_ratio = policy_chosen_logps - reference_chosen_logps
rejected_ratio = policy_rejected_logps - reference_rejected_logps
return chosen_ratio - rejected_ratio
def dpo_loss(
policy_chosen_logps: torch.Tensor,
policy_rejected_logps: torch.Tensor,
reference_chosen_logps: torch.Tensor,
reference_rejected_logps: torch.Tensor,
beta: float = 0.1,
) -> torch.Tensor:
"""Standard DPO loss."""
h_theta = compute_log_ratios(
policy_chosen_logps,
policy_rejected_logps,
reference_chosen_logps,
reference_rejected_logps,
)
return -F.logsigmoid(beta * h_theta).mean()
def ipo_loss(
policy_chosen_logps: torch.Tensor,
policy_rejected_logps: torch.Tensor,
reference_chosen_logps: torch.Tensor,
reference_rejected_logps: torch.Tensor,
beta: float = 0.1,
) -> torch.Tensor:
"""IPO loss with squared error objective."""
h_theta = compute_log_ratios(
policy_chosen_logps,
policy_rejected_logps,
reference_chosen_logps,
reference_rejected_logps,
)
target = 1.0 / (2.0 * beta)
return ((h_theta - target) ** 2).mean()
def cdpo_loss(
policy_chosen_logps: torch.Tensor,
policy_rejected_logps: torch.Tensor,
reference_chosen_logps: torch.Tensor,
reference_rejected_logps: torch.Tensor,
beta: float = 0.1,
epsilon: float = 0.1, # label smoothing
) -> torch.Tensor:
"""Conservative DPO with label smoothing."""
h_theta = compute_log_ratios(
policy_chosen_logps,
policy_rejected_logps,
reference_chosen_logps,
reference_rejected_logps,
)
# Smoothed loss: (1-ε)log σ(βh) + ε log σ(-βh)
loss = -(1 - epsilon) * F.logsigmoid(
beta * h_theta
) - epsilon * F.logsigmoid(-beta * h_theta)
return loss.mean()Now let's implement KTO and ORPO, which have different data requirements:
In[12]:
Codeimport torch
import torch.nn.functional as F
def kto_loss(
policy_logps: torch.Tensor, # log probs for all examples
reference_logps: torch.Tensor, # reference log probs
is_desirable: torch.Tensor, # binary: 1=good, 0=bad
kl_reference: float, # running average KL
beta: float = 0.1,
lambda_d: float = 1.0,
lambda_u: float = 1.0,
) -> torch.Tensor:
"""KTO loss for unpaired binary feedback."""
implicit_reward = beta * (policy_logps - reference_logps)
# Different value functions for desirable vs undesirable
desirable_mask = is_desirable.bool()
values = torch.zeros_like(policy_logps)
values[desirable_mask] = torch.sigmoid(
implicit_reward[desirable_mask] - kl_reference
)
values[~desirable_mask] = torch.sigmoid(
kl_reference - implicit_reward[~desirable_mask]
)
# Weighted loss
weights = torch.where(desirable_mask, lambda_d, lambda_u)
return (weights * (1 - values)).mean()
def orpo_loss(
policy_chosen_logps: torch.Tensor, # per-token log probs, averaged
policy_rejected_logps: torch.Tensor,
chosen_length: torch.Tensor, # sequence lengths
rejected_length: torch.Tensor,
lambda_or: float = 0.1,
) -> torch.Tensor:
"""ORPO loss combining SFT with odds ratio preference."""
# SFT loss on chosen (negative log likelihood)
sft_loss = -policy_chosen_logps.mean()
# Compute log odds (approximation using average log probs)
# log odds = log_prob - log(1 - exp(log_prob))
# For numerical stability with small probabilities:
def log_odds(log_probs):
# Clamp to avoid numerical issues
probs = torch.exp(log_probs.clamp(min=-100))
return log_probs - torch.log1p(-probs.clamp(max=1 - 1e-7))
chosen_log_odds = log_odds(policy_chosen_logps)
rejected_log_odds = log_odds(policy_rejected_logps)
# Odds ratio loss
log_or = chosen_log_odds - rejected_log_odds
or_loss = -F.logsigmoid(log_or).mean()
return sft_loss + lambda_or * or_lossLet's visualize how these losses behave differently:
In[13]:
Codeimport numpy as np
# Create range of h_theta values (log-ratio differences)
h_theta = np.linspace(-2, 12, 200)
beta = 0.1
# DPO loss: -log σ(βh)
dpo = -np.log(1 / (1 + np.exp(-beta * h_theta)))
# IPO loss: (h - 1/2β)²
target = 1 / (2 * beta)
ipo = (h_theta - target) ** 2
# cDPO loss with ε=0.3
epsilon = 0.3
sigmoid_pos = 1 / (1 + np.exp(-beta * h_theta))
sigmoid_neg = 1 / (1 + np.exp(beta * h_theta))
cdpo = -(1 - epsilon) * np.log(sigmoid_pos + 1e-10) - epsilon * np.log(
sigmoid_neg + 1e-10
)
# Normalize for visualization
dpo_norm = (dpo - dpo.min()) / (dpo.max() - dpo.min() + 1e-10)
ipo_norm = (ipo - ipo.min()) / (ipo.max() - ipo.min() + 1e-10)
cdpo_norm = (cdpo - cdpo.min()) / (cdpo.max() - cdpo.min() + 1e-10)Out[14]:
Visualization
Now let's examine the gradients:
In[15]:
Codeimport numpy as np
# Compute gradients analytically
# DPO gradient: -β * σ(-βh)
dpo_grad = -beta * (1 / (1 + np.exp(beta * h_theta)))
# IPO gradient: 2(h - 1/2β)
ipo_grad = 2 * (h_theta - target)
# cDPO gradient: -β[(1-ε)σ(-βh) - ε·σ(βh)]
cdpo_grad = -beta * ((1 - epsilon) * sigmoid_neg - epsilon * sigmoid_pos)Out[16]:
Visualization