% Self-authored reading presentation. Build with local pdfLaTeX. \documentclass[aspectratio=169,11pt]{beamer} \usepackage[T1]{fontenc} \usepackage[utf8]{inputenc} \usepackage{lmodern} \usepackage{amsmath,amssymb,graphicx,booktabs} \definecolor{AcademicBlue}{RGB}{26,57,86} \definecolor{AcademicTeal}{RGB}{0,105,112} \setbeamercolor{structure}{fg=AcademicBlue} \setbeamercolor{frametitle}{fg=AcademicBlue,bg=white} \setbeamercolor{title}{fg=AcademicBlue} \setbeamercolor{alerted text}{fg=AcademicTeal} \setbeamercolor{block title}{fg=white,bg=AcademicBlue} \setbeamercolor{block body}{fg=black,bg=AcademicBlue!5} \setbeamertemplate{navigation symbols}{} \setbeamertemplate{itemize items}[circle] \setbeamertemplate{footline}{% \hfill\usebeamerfont{page number in head/foot}% \insertframenumber\,/\,\inserttotalframenumber\hspace{4mm}\vspace{3mm}} \setbeamersize{text margin left=8mm,text margin right=8mm} \setbeamerfont{frametitle}{size=\Large,series=\bfseries} \setbeamerfont{title}{size=\LARGE,series=\bfseries} \newcommand{\papersource}[1]{\par\smallskip{\scriptsize\color{gray}#1\par}} \newcommand{\vx}{\mathbf{x}} \newcommand{\E}{\mathbb{E}} \newcommand{\mask}{\texttt{[MASK]}} \newcommand{\KL}{\operatorname{KL}} \newcommand{\JSD}{\operatorname{JSD}} \newcommand{\sg}{\operatorname{sg}} \title[CDLM]{Consistent Diffusion Language Models} \subtitle{Exact posterior bridges for few-step discrete generation} \author{\small Paper authors: Hasan Amin, Yuan Gao, Yaser Souri, Subhojit Som,\\ Ming Yin, Rajiv Khanna, and Xia Song\\[2mm] \footnotesize Purdue University and Microsoft} \date{\small ICML 2026 \quad | \quad arXiv:2605.00161v2\\ Reading presentation \quad | \quad 13 October 2026} \begin{document} \begin{frame} \titlepage \vspace{-3mm} \begin{center} \color{AcademicTeal}Train agreement across stochastic bridges,\newline then generate with fewer denoising evaluations. \end{center} \end{frame} \begin{frame}{Parallel updates need a model trained for large jumps} \begin{columns}[T,onlytextwidth] \begin{column}{0.47\textwidth} \begin{block}{The opportunity} Diffusion can update many token positions in one network evaluation. \end{block} \begin{itemize} \item High quality often needs hundreds of refinement steps. \item Fewer steps make each prediction harder. \end{itemize} \end{column} \begin{column}{0.49\textwidth} \begin{block}{The missing construction} Continuous consistency uses a deterministic probability-flow ODE path. \end{block} \begin{itemize} \item Categorical states evolve by stochastic jumps. \item CDLM uses exact posterior bridges to couple noise levels. \end{itemize} \end{column} \end{columns} \medskip \alert{Question: can a single training stage build few-step denoising ability?} \papersource{Paper, Secs. 1--2, PDF pp. 1--3. The ODE obstacle concerns the categorical sample space.} \end{frame} \begin{frame}{A discrete toy task shows structure emerging in fewer steps} \centering \includegraphics[width=0.95\linewidth,height=0.68\textheight,keepaspectratio]{figures/fig-01-moons.pdf} \par\smallskip \small Top: masked diffusion. Bottom: CDLM. Columns: 1, 2, 3, 5, 10, 30 steps. \papersource{Paper, Fig. 1, PDF p. 1. Quantized 2D moons; axes are quantized coordinates. Original figure crop.} \end{frame} \begin{frame}{The masked posterior bridge reveals a known fraction of tokens} Forward masking: a clean token survives at time $t$ with probability $1-t$. \[ q(x_s^i\mid x_t^i,x_0^i)= \begin{cases} \delta_{x_t^i}, & x_t^i\ne\mask,\\[1mm] \dfrac{t-s}{t}\delta_{x_0^i}+\dfrac{s}{t}\delta_{\mask}, & x_t^i=\mask. \end{cases} \] Here $\delta_a$ denotes a point mass at token $a$. \begin{block}{Example: $t=0.8\rightarrow s=0.4$} Each masked position reveals its true token with probability $0.5$.\newline Already visible tokens stay fixed. \end{block} Training knows $\vx_0$, so this intermediate state is sampled analytically. \papersource{Paper, Algorithm 2, PDF p. 14; general bridge in Eq. (2), p. 4. Numerical example is explanatory.} \end{frame} \begin{frame}{MPDC requires agreement after averaging over valid paths} \[ p(\vx_s\mid\vx_t) =\E_{\vx_0\mid\vx_t}[q(\vx_s\mid\vx_t,\vx_0)] \] \begin{block}{Multi-Path Discrete Consistency, per position $i$} \[ f^*(\vx_t,t)_i =\E_{\vx_s\sim p(\cdot\mid\vx_t)}[f^*(\vx_s,s)_i] =[\mathcal C_{s\leftarrow t}f^*](\vx_t,t)_i. \] \end{block} \begin{itemize} \item Direct prediction agrees with the expected prediction after a cleaner hop. \item Bridges compose: marginalizing $t\to s\to u$ recovers $t\to u$. \item Individual sampled paths can have different predictions. \end{itemize} \papersource{Paper, Lemma 3.1, Definitions 3.2--3.3, Eqs. (2)--(4), PDF p. 4.} \end{frame} \begin{frame}{The Bayes fixed point gives token marginals} \[ f^*(\vx_t,t)_i=p(x_0^i\mid\vx_t), \qquad f^*(\vx_0,0)_i=\delta_{x_0^i}. \] \begin{itemize} \item The tower property establishes the fixed point under the true reverse chain. \item A positive data anchor and a mean-eliciting rule support identification. \item Factorized outputs do not represent every joint posterior. \end{itemize} \begin{block}{Why multiple passes help: an illustrative two-token example} Data: $AA$ or $BB$, each with probability $1/2$.\newline From all masks, both token marginals are $A/B=1/2$.\newline Independent one-step sampling assigns $50\%$ probability to $AB$ or $BA$. \end{block} \papersource{Paper, Proposition 3.4, PDF p. 4; Sec. 5, p. 9. Two-token example is our explanation.} \end{frame} \begin{frame}{Exact training pairs remove the need for a pretrained teacher} \[ \vx_0\sim p_{\rm data} \quad\longrightarrow\quad \vx_t\sim q(\cdot\mid\vx_0) \quad\longrightarrow\quad \vx_s\sim q(\cdot\mid\vx_t,\vx_0). \] \[ \mathcal L_{\rm CDLM} =\E\!\left[\frac1\delta\sum_{i\in\mathcal M(\vx_t)} \JSD\!\left(f_\theta(\vx_t,t)_i,\sg[f_{\tilde\theta}(\vx_s,s)_i]\right)\right]. \] \begin{itemize} \item $\delta=t-s$; masked models train on masked positions. \item Uniform models include all positions. \item The target starts with the online network and follows an EMA. \end{itemize} \medskip \alert{At inference: predict clean token probabilities, then use an ancestral sampler.} \papersource{Paper, Eq. (6), PDF p. 4; Sampling, p. 5; Algorithms 1--2, pp. 13--14.} \end{frame} \begin{frame}{A clean-data anchor prevents trivial self-consistency} \[ \mathcal L_{\rm final} =(1-\kappa_{\rm ms})\mathcal L_{\rm CDLM} +\kappa_{\rm ms}\mathcal L_{s=0}. \] \begin{columns}[T,onlytextwidth] \begin{column}{0.47\textwidth} \begin{block}{Cross-time tasks} Match predictions across several step sizes.\newline\smallskip Weight each task by $1/\delta$. \end{block} \end{column} \begin{column}{0.49\textwidth} \begin{block}{Max-step tasks} Set $s=0$ and use clean tokens as the target.\newline\smallskip Ground the model in observed data. \end{block} \end{column} \end{columns} \medskip Stop-gradient, a slow target network, and bounded symmetric JSD stabilize learning. \papersource{Paper, Sec. 3.2 and Eq. (7), PDF p. 5. Exact MDLM recovery additionally requires forward KL/CE.} \end{frame} \begin{frame}{The practical JSD loss differs from the ideal mean target} Let $Q$ be the online prediction and $P$ a random cleaner-state target. \[ \mathbb D(Q,\E[P])\quad\text{and}\quad\E[\mathbb D(Q,P)] \] are generally different objectives. \begin{block}{Forward KL has an exact mean-target decomposition} \[ \E[\KL(P\|Q)] =\underbrace{\E[\KL(P\|\bar P)]}_{\text{target variance}} +\KL(\bar P\|Q),\qquad \bar P=\E[P]. \] \end{block} \begin{itemize} \item JSD was selected for empirical stability; it generally has a different optimum. \item The KL excess-risk proof does not directly certify the JSD implementation. \end{itemize} \papersource{Paper, Eqs. (5)--(6), p. 4; Remark C.2, PDF pp. 14--15. Interpretation of the objective distinction.} \end{frame} \begin{frame}{The main comparison uses 110M models on OpenWebText} \begin{itemize} \item 150K training updates; global batch size 2048. \item Single-stage CDLM: no separate pretrained teacher. \item Distilled models: 100K pretraining + 50K distillation updates. \item AR: 75K updates, to match seen tokens according to the paper. \end{itemize} \medskip \centering \begin{tabular}{lll} \toprule Family & Base models & Distilled comparison\\ \midrule Masked & MDLM, MCDLM & MDLM / MCDLM + SDTT\\ Uniform & UDLM, DUO, UCDLM & DUO + DCD\\ \bottomrule \end{tabular} \par\medskip \raggedright Matching updates does not match training FLOPs: target forwards and rollouts differ. \papersource{Paper, Secs. 4.1--4.2, PDF pp. 6--7; Appendix F.1, p. 20.} \end{frame} \begin{frame}{Quality requires reading PPL together with diversity} \begin{block}{Unconditional generation} 32 samples per model, each 1024 tokens.\newline Generative PPL from \texttt{gpt2-large} $\downarrow$; reported entropy checks repetition. \end{block} \begin{block}{Conditional reconstruction} 32 sequences per dataset; perturb $50\%$ of tokens.\newline PPL $\downarrow$, BLEU2 against the original text $\uparrow$, and MAUVE $\uparrow$. \end{block} \begin{itemize} \item OpenWebText in-domain; Lambada, WikiText-103, PTB out-of-domain. \item Keep FP32 and FP64 sampling results separate. \item NFE counts network evaluations; it does not measure wall-clock latency. \end{itemize} \papersource{Paper, Sec. 4.3, PDF pp. 7--8; Tables 7--8, pp. 20--21.} \end{frame} \begin{frame}{Masked CDLM improves quality at every reported budget} \centering \begin{tabular}{lccc} \toprule Model & 16 steps & 64 steps & 1024 steps\\ \midrule MDLM & 297.1 / 5.6 & 124.4 / 5.6 & 97.7 / 5.6\\ MCDLM & 125.6 / 5.4 & 67.7 / 5.6 & 53.4 / 5.5\\ MCDLM--PPL Opt. & \textbf{71.6 / 5.3} & \textbf{40.1 / 5.3} & \textbf{33.8 / 5.4}\\ \bottomrule \end{tabular} \par\smallskip \small FP64 ancestral sampling; entries: generative PPL $\downarrow$ / entropy. \normalsize\raggedright \medskip \begin{itemize} \item At 64 steps, base MCDLM improves PPL with the same reported entropy. \item At 16 steps, PPL Opt. reduces PPL by $75.9\%$ relative to MDLM. \item PPL optimization introduces a quality--diversity tradeoff. \end{itemize} \papersource{Paper, Table 1, PDF p. 7. All three models: 150K single-stage training updates.} \end{frame} \begin{frame}{CDLM competes with distillation and is a stronger starting point} \centering \small \begin{tabular}{lcccc} \toprule Model & Training & 16 steps & 64 steps & 1024 steps\\ \midrule MCDLM--PPL Opt. & 150K & 71.6 / 5.3 & 40.1 / 5.3 & 33.8 / 5.4\\ MDLM + SDTT & 100K + 50K & 76.0 / 5.4 & 40.1 / 5.3 & 31.2 / 5.4\\ MCDLM + SDTT & 100K + 50K & \textbf{57.5 / 5.1} & \textbf{35.3 / 5.3} & \textbf{27.1 / 5.2}\\ \bottomrule \end{tabular} \par\smallskip FP64 ancestral sampling; entries: generative PPL $\downarrow$ / entropy. \normalsize\raggedright \medskip \begin{itemize} \item Single-stage PPL Opt. is competitive in the few-step regime. \item At 1024 steps, standard SDTT has lower PPL than single-stage PPL Opt. \item Applying SDTT to MCDLM improves on applying it to MDLM. \end{itemize} \papersource{Paper, Table 1, PDF p. 7. Selected columns; entropy varies across configurations.} \end{frame} \begin{frame}{The acceleration claim is a reduction in network evaluations} \begin{block}{A directly traceable FP64 comparison} AR: 1024 evaluations, PPL 40.2, entropy 5.6.\newline MCDLM--PPL Opt.: 64 evaluations, PPL 40.1, entropy 5.3.\newline\smallskip \alert{$16\times$ fewer evaluations at similar generative PPL.} \end{block} \begin{itemize} \item MDLM at 1024 steps: 97.7 / 5.6; PPL Opt. at 16 steps: 71.6 / 5.3. \item This pair shows a $64\times$ NFE reduction with lower PPL and entropy. \item Per-evaluation cost differs between diffusion and cached AR decoding. \end{itemize} \papersource{Paper, Table 1 and Sec. 4.3, PDF p. 7. Authors report 64--128x vs MDLM and 16--32x NFE vs AR; no latency measurements.} \end{frame} \begin{frame}{Uniform CDLM also helps, with sampler-dependent tradeoffs} \centering \small \begin{tabular}{llcc} \toprule Model & Sampler & 4 steps & 64 steps\\ \midrule DUO & Ancestral & 514.4 / 5.6 & 85.1 / 5.4\\ UCDLM & Ancestral & 377.3 / 5.2 & 81.0 / 5.5\\ DUO + DCD & Ancestral & 408.3 / 5.6 & 80.2 / 5.5\\ \midrule UCDLM & Greedy & 110.4 / 4.8 & 64.7 / 5.5\\ DUO + DCD & Greedy & 118.4 / 3.9 & 65.5 / 5.4\\ \bottomrule \end{tabular} \par\smallskip FP64; entries: generative PPL $\downarrow$ / entropy.\par \normalsize\raggedright \medskip UCDLM beats the base uniform models, but the 64-step ancestral result is an exception to the text's claim of strictly beating DCD at all budgets. \papersource{Paper, Table 2, PDF p. 8; accompanying claim in Sec. 4.3, p. 7. Entropy below 5 is flagged by the authors.} \end{frame} \begin{frame}{Conditional reconstruction improves across four datasets} \centering \small \begin{tabular}{lcccc} \toprule Model & OpenWebText & Lambada & WikiText-103 & PTB\\ \midrule MDLM & 40.4 / 61.1 & 62.9 / 57.7 & 39.6 / 62.2 & 220.0 / 50.7\\ SDTT & 30.8 / 62.5 & 46.4 / 59.3 & 30.5 / 63.7 & 125.4 / 52.4\\ PPL Opt. & \textbf{28.6 / 62.9} & \textbf{38.8 / 60.1} & \textbf{27.3 / 64.4} & \textbf{107.5 / 53.3}\\ \bottomrule \end{tabular} \par\smallskip 64 steps, FP64; entries: PPL $\downarrow$ / BLEU2 $\uparrow$. \normalsize\raggedright \medskip \begin{itemize} \item CDLM also leads SDTT at 8 and 512 steps on both metrics. \item PTB at 64 steps: $14.3\%$ lower PPL, $+0.9$ BLEU2 points. \item Greedy DUO-DCD on PTB: PPL 45.0 but BLEU2 only 19.3. \end{itemize} \papersource{Paper, Table 3 and Sec. 4.3.1, PDF p. 8. Protocol: reconstruct sequences after 50\% token corruption.} \end{frame} \begin{frame}{Removing the data anchor creates misleadingly low PPL} \centering \begin{tabular}{lccc} \toprule Max-step weight & 8 steps & 64 steps & 1024 steps\\ \midrule $\kappa_{\rm ms}=0$ & 16.7 / 3.2 & 7.1 / 3.4 & 5.7 / 2.8\\ $\kappa_{\rm ms}=0.4$ & 110.6 / 5.3 & 32.8 / 5.3 & 19.7 / 5.2\\ $\kappa_{\rm ms}=1.0$ & 274.0 / 5.5 & 75.1 / 5.5 & 38.4 / 5.3\\ \bottomrule \end{tabular} \par\smallskip \small Entries: generative PPL $\downarrow$ / entropy.\par \normalsize\raggedright \medskip \begin{block}{A concrete failure mode} Self-consistency without grounding can collapse to repetitive outputs.\newline The smallest PPL in this table is not the desired model. \end{block} \papersource{Paper, Table 5, PDF p. 9. Exact equivalence to MDLM at the max-step boundary requires forward KL/CE.} \end{frame} \begin{frame}{JSD gives the most useful stability--diversity balance here} \centering \begin{tabular}{lcc} \toprule Divergence & 64-step PPL $\downarrow$ & Entropy\\ \midrule JSD & 32.8 & 5.3\\ Forward KL: $\KL(P_{\rm target}\|Q_{\rm online})$ & $8\times10^4$ & 6.9\\ Reverse KL: $\KL(Q_{\rm online}\|P_{\rm target})$ & 64.0 & 4.4\\ \bottomrule \end{tabular} \par\medskip\raggedright \begin{itemize} \item Forward KL: the paper attributes failure to uniform drift. \item Reverse KL: low entropy is consistent with mode seeking. \item A favorable population target can still be hard to learn from scratch. \end{itemize} \papersource{Paper, Table 6 and accompanying discussion, PDF p. 9. This supports the empirical choice of JSD.} \end{frame} \begin{frame}{Step-size schedules favor different inference budgets} \centering \begin{tabular}{lccc} \toprule Training schedule & 8 steps & 64 steps & 1024 steps\\ \midrule Random & 110.6 / 5.3 & 32.8 / 5.3 & 19.7 / 5.2\\ Staged increasing & 160.8 / 5.2 & 45.1 / 5.3 & 25.3 / 5.2\\ Linear increasing & 117.6 / 5.5 & 37.9 / 5.4 & 17.9 / 4.9\\ Linear decreasing & 112.3 / 5.1 & 32.1 / 5.2 & 20.5 / 5.2\\ \bottomrule \end{tabular} \par\smallskip \small Entries: generative PPL $\downarrow$ / entropy.\par \normalsize\raggedright \medskip Decreasing beats increasing at 8 and 64 steps; increasing has lower PPL at 1024 steps with lower entropy. Random remains slightly better at 8 steps by PPL. \papersource{Paper, Table 4, PDF p. 9. Training step size and inference step count are distinct quantities.} \end{frame} \begin{frame}{Additional results qualify the headline metrics} \begin{itemize} \item \textbf{Precision matters.} At 1024 steps, MDLM PPL is 97.7 with FP64 versus 42.4 with FP32. Compare methods within the same precision. \medskip \item \textbf{MAUVE is not uniformly saturated.} At 8 steps on PTB, MDLM / SDTT / CDLM score 0.11 / 0.07 / 0.02.\newline CDLM's PPL and BLEU2 gains do not imply a PTB MAUVE gain. \medskip \item \textbf{Few-step samples remain imperfect.} The 4-step CDLM example is fragmented; longer samples can still repeat themes and lose coherence. \end{itemize} \papersource{Paper, Table 1, p. 7; Table 7, p. 20; Table 8, p. 21; qualitative examples, PDF pp. 22--24.} \end{frame} \begin{frame}{The contribution is a useful stochastic training principle} \begin{enumerate} \item Exact posterior bridges provide cross-time supervision directly from data. \medskip \item A data anchor, target network, and divergence choice make training usable. \medskip \item Few-step gains are strong in this 110M setting and complement distillation. \end{enumerate} \begin{block}{Questions that determine the next step} Can mean-calibrated objectives be as stable as JSD?\newline Do gains persist with larger evaluations and models?\newline What is the latency and total compute at matched quality and diversity? \end{block} \papersource{Synthesis of Secs. 3--5. Evaluation uses 32 generated samples per model; reported efficiency is NFE.} \end{frame} \begin{frame}{Reference and discussion material} Hasan Amin, Yuan Gao, Yaser Souri, Subhojit Som, Ming Yin, Rajiv Khanna, and Xia Song. \medskip \textbf{Consistent Diffusion Language Models.}\newline ICML 2026. arXiv:2605.00161v2, revised 30 May 2026. \medskip \href{https://arxiv.org/abs/2605.00161v2}{https://arxiv.org/abs/2605.00161v2} \medskip \begin{itemize} \item Theory: Sec. 3 and Appendices C--E. \item Main results and ablations: Tables 1--6. \item FP32, MAUVE, and samples: Appendix F. \end{itemize} \papersource{Paper-reading presentation; figure reproduced from the paper, tables selectively retypeset. Backup slides follow.} \end{frame} \appendix \begin{frame}{Backup: the general bridge is Bayes' rule in matrix form} \[ q(x_s^i\mid x_t^i,x_0^i) =\operatorname{Cat}\!\left(x_s^i; \frac{(x_0^iQ_{1:s})\odot(x_t^iQ_{s+1:t}^{\mathsf T})} {\langle x_0^iQ_{1:t},x_t^i\rangle}\right). \] \begin{itemize} \item $x_0^iQ_{1:s}$: prior probability of an intermediate token. \item $x_t^iQ_{s+1:t}^{\mathsf T}$: likelihood of the observed noisy token. \item Denominator: probability of the observation under the forward process. \end{itemize} \[ q(\vx_u\mid\vx_t,\vx_0) =\sum_{\vx_s}q(\vx_u\mid\vx_s,\vx_0)q(\vx_s\mid\vx_t,\vx_0). \] \papersource{Paper, Lemma 3.1, Eq. (2), PDF p. 4; proof in Appendix E.1, pp. 18--19.} \end{frame} \begin{frame}{Backup: local errors accumulate along the true reverse chain} Let $E_{j|k}$ average over the true reverse transition on a time grid. \[ \E\,D_{\rm TV}(f_k,E_{k-1|k}f_{k-1})\le\varepsilon \quad\Longrightarrow\quad \mathcal E(K,m)\le(K-m)\varepsilon. \] \begin{itemize} \item Semigroup composition, the triangle inequality, and convexity yield the bound. \item Under forward KL, tracking error is controlled by excess risk: \end{itemize} \[ \varepsilon\le\sqrt{\tfrac12\E[\mathcal E_{\rm opt}]}, \qquad \mathcal E(K,0)\le K\sqrt{\tfrac12\E[\mathcal E_{\rm opt}]}. \] This controls predictor tracking under the true chain; it is not yet an error bound for the full joint distribution produced by the learned sampler. \papersource{Paper, Lemma C.1 and Remark C.2, PDF pp. 12--15. Uniform control is required across local edges.} \end{frame} \begin{frame}{Backup: the PPL-optimized variant changes three schedules} \begin{block}{Base MCDLM} Random step size in $[1/8,5/8]$; $\kappa_{\rm ms}=0.4$; EMA rate $0.999$. \end{block} \begin{block}{MCDLM--PPLOptimized} Same configuration for the first 100K updates. Then:\newline step-size upper limit shrinks toward $3/8$; anchor weight declines to $0.2$;\newline the target is hard-updated every 10K steps instead of using EMA. \end{block} \begin{block}{Uniform CDLM} Step size increases linearly from $0.125$ to $0.375$. \end{block} \papersource{Paper, Sec. 4.2, PDF pp. 6--7. All valid time pairs must satisfy $0\le s