% Based on the arxiv-paper-reader Beamer template. Compile with pdfLaTeX.
\documentclass[aspectratio=169,11pt]{beamer}
\usepackage[T1]{fontenc}
\usepackage[utf8]{inputenc}
\usepackage{lmodern}
\usepackage{amsmath,amssymb,graphicx,booktabs}
\pdfstringdefDisableCommands{\def\\{ }\def\translate#1{#1}}

\definecolor{AcademicBlue}{RGB}{26,57,86}
\definecolor{AcademicTeal}{RGB}{0,105,112}
\setbeamercolor{structure}{fg=AcademicBlue}
\setbeamercolor{frametitle}{fg=AcademicBlue,bg=white}
\setbeamercolor{title}{fg=AcademicBlue}
\setbeamercolor{alerted text}{fg=AcademicTeal}
\setbeamercolor{block title}{fg=white,bg=AcademicBlue}
\setbeamercolor{block body}{fg=black,bg=AcademicBlue!5}
\setbeamertemplate{navigation symbols}{}
\setbeamertemplate{itemize items}[circle]
\setbeamertemplate{footline}{%
  \hfill\usebeamerfont{page number in head/foot}%
  \insertframenumber\,/\,\inserttotalframenumber\hspace{4mm}\vspace{3mm}}
\setbeamersize{text margin left=8mm,text margin right=8mm}
\setbeamerfont{frametitle}{size=\Large,series=\bfseries}
\setbeamerfont{title}{size=\LARGE,series=\bfseries}
\newcommand{\papersource}[1]{\par\smallskip{\scriptsize\color{gray}#1\par}}
\newcommand{\E}{\mathbb E}
\newcommand{\KL}{D_{\mathrm{KL}}}
\newcommand{\mask}{\mathrm{MASK}}
\newcommand{\bfx}{\boldsymbol{x}}
\newcommand{\bsig}{\bar{\sigma}}
\newcommand{\SE}{\mathrm{SE}}
\newcommand{\DSE}{\mathrm{DSE}}

\title[SEDD]{Discrete Diffusion Modeling by Estimating the Ratios of the Data Distribution}
\subtitle{SEDD: learning positive ratios to reverse token corruption}
\author{Paper by Aaron Lou, Chenlin Meng, and Stefano Ermon\\
{\small Independent research reading presentation}}
\date{ICML 2024 \quad arXiv:2310.16834v3\\
{\small Paper revision: June 6, 2024 \quad Reading: October 14, 2026}}

\begin{document}

\begin{frame}
  \titlepage
\end{frame}

\begin{frame}{The missing ingredient is a trainable discrete score}
  \begin{block}{Research question}
    Can token diffusion compete with autoregressive language modeling
    while updating many positions together?
  \end{block}
  \[
    \underbrace{\frac{p_t(y)}{p_t(x)}}_{\text{reverse-process information}}
    \ \xrightarrow{\ \text{score entropy}\ }\ 
    \underbrace{s_\theta(x,t)_y}_{\text{positive network output}}
  \]
  \begin{itemize}
    \item Discrete ratios replace the continuous score $\nabla_x\log p_t(x)$.
    \item Denoising makes ratio learning tractable with one network pass.
    \item The same objective connects to likelihood and flexible generation.
  \end{itemize}
  \papersource{Paper, Secs. 1--3, PDF pp. 1--4.}
\end{frame}

\begin{frame}{Reverse jump rates require only probability ratios}
  For column probability vectors, $Q_t(y,x)$ is the forward rate $x\to y$:
  \[
    \frac{dp_t}{dt}=Q_tp_t,\qquad
    Q_t(y,x)\ge0\ (y\ne x),\qquad \sum_yQ_t(y,x)=0.
  \]
  \begin{block}{Time reversal}
    \[
      \bar Q_t(y,x)=Q_t(x,y)\frac{p_t(y)}{p_t(x)}
      \ \approx\ Q_t(x,y)s_\theta(x,t)_y,\quad y\ne x.
    \]
  \end{block}
  \begin{itemize}
    \item Reverse probability flow: $p_t(x)\bar Q_t(y,x)=p_t(y)Q_t(x,y)$.
    \item Set each diagonal to minus the sum of its outgoing rates.
    \item Ratios are positive values; they need not sum to one.
  \end{itemize}
  \papersource{Paper, Eqs. (1)--(3), PDF p. 2; Definition 3.5, PDF p. 3.}
\end{frame}

\begin{frame}{Score entropy penalizes missing positive mass}
  Write $r=p(y)/p(x)$ and let $s>0$ be its prediction:
  \[
    \boxed{\ell(s;r)=s-r+r\log(r/s)}
    \qquad
    \mathcal L_{\SE}=\E_{x\sim p}\sum_{y\ne x}w_{xy}\ell(s_\theta(x)_y;r).
  \]
  \[
    \frac{\partial\ell}{\partial s}=1-\frac rs,
    \qquad
    \frac{\partial^2\ell}{\partial s^2}=\frac r{s^2}>0.
  \]
  \begin{itemize}
    \item For $r>0$, the unique minimum is $s=r$ with zero loss.
    \item As $s\to0$ with $r>0$, the logarithmic penalty diverges.
    \item Consistency assumes full support, positive weights, and sufficient capacity.
  \end{itemize}
  \papersource{Paper, Eq. (5), Proposition 3.2, PDF p. 3; proof, PDF p. 12.}
\end{frame}

\begin{frame}{Log outputs reveal why squared error can struggle}
  The network uses positive outputs. For the teaching parameterization $s=e^z$:
  \[
    \frac{\partial\ell_{\SE}}{\partial z}=s-r,
    \qquad
    \frac{\partial\ell_{\rm CSM}}{\partial z}=s(s-r).
  \]
  \begin{block}{Example: true ratio $r=0.2$, prediction $s=0.001$}
    Score entropy gradient: $-0.199$\\[3pt]
    Squared-error gradient: $-0.000199$
  \end{block}
  \begin{itemize}
    \item Squared error adds a small multiplicative factor when $s$ is tiny.
    \item Positive outputs alone do not remove this optimization difference.
  \end{itemize}
  \papersource{Teaching derivation from Eq. (5); exponentiated outputs: Appendix C.2, PDF p. 15.}
\end{frame}

\begin{frame}{Denoising replaces unknown ratios with known kernels}
  Sample $x_0\sim p_0$ and $x_t\sim p_{t|0}(\cdot|x_0)$. Define
  \[
    r_t(y;x_t,x_0)=\frac{p_{t|0}(y|x_0)}{p_{t|0}(x_t|x_0)}.
  \]
  \[
    \mathcal L_{\DSE}
    =\E_{x_0,x_t}\sum_{y\ne x_t}w_{x_ty}
      \bigl[s_\theta(x_t,t)_y-r_t(y;x_t,x_0)\log s_\theta(x_t,t)_y\bigr].
  \]
  \begin{itemize}
    \item Averaging the conditional ratio over $x_0|x_t$ recovers $p_t(y)/p_t(x_t)$.
    \item Equal to score entropy up to a parameter-independent constant.
    \item One network input $x_t$ supplies all required outputs.
  \end{itemize}
  \papersource{Paper, Theorem 3.4 and Eq. (7), PDF p. 3; proof, PDF p. 12.}
\end{frame}

\begin{frame}{Diffusion weighting yields a likelihood upper bound}
  Choose $w_{x_ty}=Q_t(x_t,y)$ and integrate over time:
  \[
    \mathcal L_{\rm DWDSE}(x_0)=
    \int_0^T\E_{x_t|x_0}\sum_{y\ne x_t}
    Q_t(x_t,y)\,\ell\bigl(s_\theta(x_t,t)_y;r_t(y;x_t,x_0)\bigr)\,dt.
  \]
  \[
    \boxed{-\log p_0^\theta(x_0)\le
    \mathcal L_{\rm DWDSE}(x_0)+
    \KL\bigl(p_{T|0}(\cdot|x_0)\,\|\,p_{\rm base}\bigr)}
  \]
  \begin{itemize}
    \item Proof idea: path-space KL bounds endpoint KL by data processing.
    \item Training can drop constants; likelihood evaluation needs the full bound.
    \item Reported diffusion perplexities are Monte Carlo estimates of upper bounds.
  \end{itemize}
  \papersource{Paper, Theorem 3.6, Eqs. (9)--(10), PDF pp. 3--4; Appendix A, pp. 12--13.}
\end{frame}

\begin{frame}{Sparse token corruption avoids an exponential output}
  A sequence has $d$ positions and vocabulary size $n$: $|\mathcal X|=n^d$.
  \[
    s_\theta(\bfx,t)_{i,a}\approx
    \frac{p_t(x^1,\ldots,a,\ldots,x^d)}{p_t(\bfx)}
    \qquad\text{output shape }d\times n.
  \]
  \begin{itemize}
    \item Independent forward token jumps connect only Hamming-distance-one states.
    \item Zero transition weights remove all other ratios from the objective.
    \item A bidirectional Transformer still models dependencies across the sequence.
  \end{itemize}
  \[
    p_{t|0}^{\rm seq}(\bfx_t|\bfx_0)=
    \prod_{i=1}^d p_{t|0}^{\rm tok}(x_t^i|x_0^i).
  \]
  \papersource{Paper, Sec. 3.3, Eqs. (11)--(14), PDF p. 4.}
\end{frame}

\begin{frame}{Absorb and Uniform provide closed-form corruption}
  Let $\alpha(t)=e^{-\bsig(t)}$, with $\bsig(t)=\int_0^t\sigma(u)\,du$.
  \begin{columns}[T,onlytextwidth]
    \begin{column}{0.48\textwidth}
      \begin{block}{Uniform}
        \[
          p_{t|0}^{\rm tok}(\cdot|a)
          =\alpha\delta_a+(1-\alpha)\frac{\mathbf1}{n}.
        \]
        Corrupt into random tokens.\\[5pt]
        Start generation from uniform noise.
      \end{block}
    \end{column}
    \begin{column}{0.48\textwidth}
      \begin{block}{Absorb}
        \[
          p_{t|0}^{\rm tok}(\cdot|a)
          =\alpha\delta_a+(1-\alpha)\delta_{\mask}.
        \]
        Corrupt into an absorbing MASK.\\[5pt]
        Start generation from all MASK.
      \end{block}
    \end{column}
  \end{columns}
  \medskip
  Uniform uses the appendix's $1/n$ generator scaling. Absorb gives the
  strongest combined likelihood and generation results.
  \papersource{Paper, Eqs. (15)--(16), PDF p. 4; Appendix C.1, PDF p. 15; Tables 1--3.}
\end{frame}

\begin{frame}{Training needs one noisy sequence and one forward pass}
  \begin{enumerate}
    \item Draw clean text $\bfx_0$ and a random time $t$.
    \item Corrupt each token with the known transition kernel.
    \item Run a time-conditioned bidirectional Transformer on $\bfx_t$.
    \item Weight local denoising losses by $Q_t^{\rm tok}(x_t^i,a)$ and update.
  \end{enumerate}
  \begin{block}{Architecture and optimization}
    RoPE; adaLN-zero time conditioning; exponentiated score outputs.\\
    Batch 512; learning rate $3\times10^{-4}$; warmup 2000 iterations;\\
    gradient norm clipped to 1; EMA 0.9999.
  \end{block}
  \papersource{Paper, Algorithm 1, PDF p. 14; Appendix C.2--C.4, PDF pp. 15--16.}
\end{frame}

\begin{frame}{Tweedie sampling improves parallel reverse updates}
  Euler $\tau$-leaping samples all positions independently using local reverse rates.
  The alternative uses $\Delta\bsig=\bsig(t)-\bsig(t-\Delta t)$:
  \[
    \widetilde P_i(a|\bfx_t)=
    \left[e^{-\Delta\bsig Q^{\rm tok}}s_\theta(\bfx_t,t)_i\right]_a
    \left[e^{\Delta\bsig Q^{\rm tok}}\right]_{x_t^i,a}.
  \]
  \begin{itemize}
    \item Insert the current token's self-ratio, $s_{i,x_t^i}=1$.
    \item With exact scores, these are the correct reverse token marginals.
    \item Their product minimizes forward KL among independent token updates.
    \item Learned scores may require nonnegative clipping and normalization.
  \end{itemize}
  \papersource{Paper, Theorems 4.1--4.2, Eqs. (18)--(20), PDF p. 5; Algorithm 2, p. 15.}
\end{frame}

\begin{frame}{Conditional ratios reuse the same unconditional network}
  Fix tokens $\boldsymbol y$ outside the generated positions $\Omega$:
  \[
    \frac{p_t(\boldsymbol z'|\boldsymbol y)}
         {p_t(\boldsymbol z|\boldsymbol y)}
    =
    \frac{p_t(\boldsymbol z'\oplus_\Omega\boldsymbol y)}
         {p_t(\boldsymbol z\oplus_\Omega\boldsymbol y)}.
  \]
  \begin{block}{Sampling rule}
    Place the prompt in the sequence. Predict scores for the whole sequence,
    but update only unfilled positions at each step.
  \end{block}
  \begin{itemize}
    \item Prompts can be prefixes, suffixes, or disjoint spans.
    \item In Absorb, an unmasked prompt token also identifies its clean token.
    \item Same-time conditional ratios alone do not justify every corruption scheme.
  \end{itemize}
  \papersource{Paper, Sec. 4.2, Eqs. (21)--(22), PDF p. 5; Algorithm 3, p. 15.
  Last two points: interpretation of the absorbing kernel.}
\end{frame}

\begin{frame}{Three metrics measure different objects}
  \renewcommand{\arraystretch}{1.3}
  \begin{tabular}{p{0.25\linewidth}p{0.31\linewidth}p{0.33\linewidth}}
    \toprule
    Metric & Evaluated object & Interpretation\\
    \midrule
    BPC / model PPL $\downarrow$ &
    Real data under the trained model &
    Density modeling; diffusion reports upper bounds\\
    Generative PPL $\downarrow$ &
    Generated samples under GPT-2 large &
    Typicality under an external evaluator; annealing-sensitive\\
    MAUVE $\uparrow$ &
    Generated and reference text sets &
    Distribution-level quality and coverage\\
    \bottomrule
  \end{tabular}
  \par\medskip
  A low generative PPL is not a measurement of factual accuracy.
  \papersource{Paper, Secs. 5.2--5.3, PDF pp. 5--8; Appendix C.6--C.8, p. 16.}
\end{frame}

\begin{frame}{The three likelihood settings use different protocols}
  \begin{itemize}
    \item \textbf{text8:} character modeling; prior D3PM-style architecture and splits.
    \item \textbf{1BW training:} matched tokenization and size from DiffusionBERT;
    includes a separately trained autoregressive Transformer.
    \item \textbf{GPT-2 zero-shot:} SEDD trained on OpenWebText;
    public GPT-2 trained on WebText.
  \end{itemize}
  \begin{block}{Comparable scale, with parameter differences}
    Non-embedding parameters: SEDD about 90M / 320M versus GPT-2
    about 86M / 304M. SEDD also separates input and output word matrices.
  \end{block}
  \papersource{Paper, Sec. 5.1--5.2, PDF pp. 5--6; Appendix C.2 and C.5, pp. 15--16.}
\end{frame}

\begin{frame}{SEDD improves the non-autoregressive text8 result}
  \begin{columns}[c,onlytextwidth]
    \begin{column}{0.53\textwidth}
      \centering
      \renewcommand{\arraystretch}{1.15}
      \begin{tabular}{lr}
        \toprule
        Model & BPC $\downarrow$\\
        \midrule
        Autoregressive & 1.23\\
        Discrete Flow (AR base) & 1.23\\
        MAC & $\le1.40$\\
        BFN & $\le1.41$\\
        D3PM Absorb & $\le1.45$\\
        SEDD Uniform & $\le1.47$\\
        \textbf{SEDD Absorb} & $\boldsymbol{\le1.39}$\\
        \bottomrule
      \end{tabular}
    \end{column}
    \begin{column}{0.43\textwidth}
      \begin{itemize}
        \item Best reported non-autoregressive value in this comparison.
        \item 0.06 BPC below D3PM Absorb's bound.
        \item Still 0.16 BPC above the autoregressive result.
      \end{itemize}
    \end{column}
  \end{columns}
  \papersource{Paper, Table 2, PDF p. 6; selected rows. Diffusion values are upper bounds.}
\end{frame}

\begin{frame}{On 1BW, Absorb approaches the autoregressive baseline}
  \begin{columns}[c,onlytextwidth]
    \begin{column}{0.49\textwidth}
      \centering
      \renewcommand{\arraystretch}{1.15}
      \begin{tabular}{lr}
        \toprule
        Model & PPL $\downarrow$\\
        \midrule
        Autoregressive & 31.98\\
        D3PM Absorb & $\le77.50$\\
        Diffusion-LM & $\le118.62$\\
        BERT-Mouth & $\le142.89$\\
        DiffusionBERT & $\le63.78$\\
        SEDD Uniform & $\le40.25$\\
        \textbf{SEDD Absorb} & $\boldsymbol{\le32.79}$\\
        \bottomrule
      \end{tabular}
    \end{column}
    \begin{column}{0.47\textwidth}
      \begin{itemize}
        \item Reported bound is 57.7\% below D3PM Absorb.
        \item 48.6\% below DiffusionBERT.
        \item Gap to the AR value: 0.81 PPL.
      \end{itemize}
      \medskip
      This is training on 1BW, distinct from zero-shot 1BW evaluation.
    \end{column}
  \end{columns}
  \papersource{Paper, Table 3, PDF p. 6. Percentages calculated from reported values;
  bound tightness can differ.}
\end{frame}

\begin{frame}{SEDD Absorb leads on 3 of 5 zero-shot columns}
  \centering
  \renewcommand{\arraystretch}{1.18}
  \begin{tabular}{lrrrr}
    \toprule
    & \multicolumn{2}{c}{Small} & \multicolumn{2}{c}{Medium}\\
    Dataset & GPT-2 & SEDD & GPT-2 & SEDD\\
    \midrule
    LAMBADA & \textbf{45.04} & $\le50.92$ & \textbf{35.66} & $\le42.77$\\
    WikiText2 & 42.43 & $\boldsymbol{\le41.84}$ & 31.80 & $\boldsymbol{\le31.04}$\\
    PTB & 138.43 & $\boldsymbol{\le114.24}$ & 123.14 & $\boldsymbol{\le87.12}$\\
    WikiText103 & 41.60 & $\boldsymbol{\le40.62}$ & 31.39 & $\boldsymbol{\le29.98}$\\
    1BW & \textbf{75.20} & $\le79.29$ & \textbf{55.72} & $\le61.19$\\
    \bottomrule
  \end{tabular}
  \par\medskip
  \raggedright
  No sliding window. WikiText2 uses its train split.
  GPT-2 values were recomputed except for 1BW.
  \papersource{Paper, Table 1 and Sec. 5.2.3, PDF p. 6; Appendix C.6, p. 16.
  PPL lower is better; bold compares reported values.}
\end{frame}

\begin{frame}{More reverse steps improve Absorb sample quality}
  \begin{columns}[c,onlytextwidth]
    \begin{column}{0.60\textwidth}
      \centering
      \includegraphics[width=\linewidth,height=0.69\textheight,keepaspectratio]{figures/fig01a-generative-ppl.png}
    \end{column}
    \begin{column}{0.37\textwidth}
      \begin{itemize}
        \item 1000 samples, each 1024 tokens; GPT-2 large evaluator.
        \item 32 steps: comparable or better than unannealed GPT-2.
        \item 2048 steps: reported 6--8$\times$ lower generative PPL.
      \end{itemize}
      \smallskip
      Both axes are logarithmic. No temperature scaling.
    \end{column}
  \end{columns}
  \papersource{Paper, original Fig. 1(a) and Sec. 5.3.1, PDF p. 7; Appendix C.7, p. 16.}
\end{frame}

\begin{frame}{Fewer network calls do not directly specify latency}
  \begin{columns}[T,onlytextwidth]
    \begin{column}{0.48\textwidth}
      \begin{block}{Autoregressive inference}
        About 1024 sequential token steps.\\[5pt]
        KV cache reuses previous states; each new step has a different cost
        from a full-sequence pass.
      \end{block}
    \end{column}
    \begin{column}{0.48\textwidth}
      \begin{block}{SEDD inference}
        32--2048 full-sequence updates.\\[5pt]
        No growing KV cache; each step revisits the sequence.
      \end{block}
    \end{column}
  \end{columns}
  \medskip
  \begin{itemize}
    \item $1024/32=32$: a network-evaluation ratio.
    \item Authors observed similar runtime at about 100 SEDD steps.
    \item Removing KV cache allowed about 4--6$\times$ larger batches in their setup.
  \end{itemize}
  \papersource{Paper, Sec. 6, PDF pp. 8--9. Runtime and batch observations use the authors'
  unoptimized implementation.}
\end{frame}

\begin{frame}{Conditional quality is competitive without annealing}
  \begin{columns}[c,onlytextwidth]
    \begin{column}{0.59\textwidth}
      \centering
      \renewcommand{\arraystretch}{1.16}
      \begin{tabular}{llr}
        \toprule
        Method & Annealing & MAUVE $\uparrow$\\
        \midrule
        GPT-2 & Nucleus .95 & .955\\
        GPT-2 & None & .802\\
        SSD-LM & Threshold .95 & .919\\
        SSD-LM & None & .312\\
        \textbf{SEDD Standard} & None & \textbf{.957}\\
        SEDD Infill & None & .942\\
        \bottomrule
      \end{tabular}
    \end{column}
    \begin{column}{0.37\textwidth}
      All models: medium size.
      \begin{itemize}
        \item Standard: 50 prefix tokens, generate 50.
        \item Infill: 25 prefix + 25 suffix tokens, generate 50.
        \item 5000 generated / 1000 reference samples.
      \end{itemize}
    \end{column}
  \end{columns}
  \medskip
  The .002 lead over nucleus GPT-2 is small; no uncertainty interval is reported.
  \papersource{Paper, Table 5, PDF p. 8; Appendix C.8, PDF p. 16.}
\end{frame}

\begin{frame}{Uniform generation saturates as reverse steps increase}
  \centering
  \includegraphics[width=0.93\linewidth,height=0.80\textheight,keepaspectratio]{figures/fig02-uniform-samplers.png}
  \papersource{Original Fig. 2, PDF p. 17. Blue/orange: small/medium; circles/triangles: Tweedie/Euler.}
\end{frame}

\begin{frame}{Squared error causes a large optimization failure}
  \begin{block}{Loss ablation at GPT-2 scale}
    Replace score entropy with squared error while retaining the $Q_t$ weights.
    The authors report 3--4$\times$ higher likelihood loss, corresponding to
    approximately 10,000$\times$ higher perplexity.
  \end{block}
  \begin{itemize}
    \item The exponential relation between NLL and perplexity explains the
    very different multipliers.
    \item This supports the importance of loss geometry when learning ratios.
    \item Separately, Uniform plateaus; the main generation gains come from Absorb.
  \end{itemize}
  \papersource{Paper, Appendix D.1--D.2, PDF pp. 17--18.
  The loss ablation is reported as an aggregate observation without per-setting values.}
\end{frame}

\begin{frame}{SEDD aligns ratios, loss geometry, and generation}
  \begin{enumerate}
    \item \textbf{A suitable target:} local positive ratios determine the reverse
    continuous-time chain.
    \item \textbf{A scalable objective:} denoising score entropy learns these ratios
    and yields a likelihood bound.
    \item \textbf{Useful generation controls:} parallel reverse steps and prompts
    at arbitrary positions.
  \end{enumerate}
  \begin{block}{Practical tradeoff}
    Absorb is strongest in these GPT-2-scale experiments.
    Full-sequence iteration cost, fixed output length, and scaling remain
    consequential design questions.
  \end{block}
  \papersource{Paper, Secs. 3--7, PDF pp. 3--9.}
\end{frame}

\begin{frame}{Paper and resources}
  \textbf{Aaron Lou, Chenlin Meng, and Stefano Ermon}\\[5pt]
  Discrete Diffusion Modeling by Estimating the Ratios of the Data Distribution.\\
  ICML 2024. arXiv:2310.16834v3, revised June 6, 2024.
  \medskip
  \begin{itemize}
    \item \href{https://arxiv.org/abs/2310.16834v3}{Versioned arXiv paper}
    \item \href{https://github.com/louaaron/Score-Entropy-Discrete-Diffusion}{Author code repository}
    \item Original figures: Fig. 1(a), PDF p. 7; Fig. 2, PDF p. 17.
  \end{itemize}
  \medskip
  Main presentation: approximately 25 minutes.\\
  Backup slides: derivations, protocol details, and formula interpretation.
\end{frame}

\appendix

\begin{frame}{Backup: the denoising identity is an average identity}
  Under the appropriate support conditions,
  \begin{align*}
    &\E_{x_0|x_t=x}\left[\frac{p_{t|0}(y|x_0)}{p_{t|0}(x|x_0)}\right]\\
    &\quad=\sum_{x_0}
      \frac{p_0(x_0)p_{t|0}(x|x_0)}{p_t(x)}
      \frac{p_{t|0}(y|x_0)}{p_{t|0}(x|x_0)}
      =\frac{p_t(y)}{p_t(x)}.
  \end{align*}
  \begin{itemize}
    \item The conditional target varies with the sampled clean sequence.
    \item Its posterior average gives the marginal ratio required for generation.
    \item The objective uses only computable corruption probabilities.
  \end{itemize}
  \papersource{Expanded teaching derivation of Theorem 3.4; Paper, PDF pp. 3 and 12.}
\end{frame}

\begin{frame}{Backup: why the product of marginals is KL-optimal}
  Fix the current sequence $x$ and write the exact reverse joint as $P(z|x)$.
  Restrict the sampler to $q(z|x)=\prod_iq_i(z_i|x)$:
  \[
    \KL\left(P\,\middle\|\,\prod_iq_i\right)
    =C-\sum_i\E_{z_i\sim P_i}\log q_i(z_i|x).
  \]
  Each term is minimized by $q_i=P_i$, the true reverse token marginal.
  \medskip
  \begin{itemize}
    \item Applying $e^{-\Delta\bsig Q}$ only along coordinate $i$
    reverses noise on that coordinate while leaving the others noisy.
    \item Bayes' rule then supplies $P_i(z_i|x)$ from local ratios.
    \item The independent product can still lose within-step correlations.
  \end{itemize}
  \papersource{Teaching explanation of Theorems 4.1--4.2, PDF p. 5; proof, pp. 13--14.}
\end{frame}

\begin{frame}{Backup: absorbing scores contain a masked-token posterior}
  For masked position $i$ and visible context $c$, let $\alpha=e^{-\bsig(t)}$.
  Independent absorbing corruption gives
  \[
    s^*_{i,a}=\frac{\alpha}{1-\alpha}\,p_0(X_0^i=a|c).
  \]
  The sample loss at this position, up to constants, is
  \[
    \sigma(t)\left[\sum_a s_{\theta,i,a}
    -\frac{\alpha}{1-\alpha}\log s_{\theta,i,x_0^i}\right].
  \]
  If $s_{\theta,i,a}=\frac{\alpha}{1-\alpha}\pi_{\theta,i,a}$
  with normalized $\pi_\theta$, this becomes a weighted cross entropy.
  \medskip

  The ratio scale calibrates the reverse jump rates across noise levels.
  \papersource{Teaching derivation from Eqs. (10), (12), (16), PDF p. 4; not an additional paper experiment.}
\end{frame}

\begin{frame}{Backup: small-model diffusion baselines remain behind}
  \centering
  \renewcommand{\arraystretch}{1.2}
  \begin{tabular}{lrrrr}
    \toprule
    Dataset & D3PM & PLAID & Uniform & Absorb\\
    \midrule
    LAMBADA & 93.47 & 57.28 & 65.40 & \textbf{50.92}\\
    WikiText2 & 77.28 & 51.80 & 50.27 & \textbf{41.84}\\
    PTB & 200.82 & 142.60 & 140.12 & \textbf{114.24}\\
    WikiText103 & 75.16 & 50.86 & 49.60 & \textbf{40.62}\\
    1BW & 138.92 & 91.12 & 101.37 & \textbf{79.29}\\
    \bottomrule
  \end{tabular}
  \par\medskip
  \raggedright
  \textbf{All entries are perplexity upper bounds.} D3PM and PLAID were
  retrained using the authors' small-model architecture and training settings.
  \papersource{Paper, Table 1, PDF p. 6; Appendix C.5.3, PDF p. 16.}
\end{frame}

\begin{frame}{Backup: numerical conventions affect implementation}
  \begin{itemize}
    \item \textbf{Uniform scaling:} the appendix scales the generator by $1/n$;
    then the retention factor is $e^{-\bsig}$.
    \item \textbf{Likelihood endpoints:} the absorbing prior has tiny non-MASK
    mass to avoid infinite prior KL at finite noise.
    \item \textbf{Evaluation:} likelihood estimates use 1000 random time points.
    \item \textbf{Loss weights:} retain $Q_t(x_t,y)$ from Eq. (10);
    Absorb trains relevant reverse edges at masked positions.
  \end{itemize}
  \papersource{Paper, Eq. (10), PDF p. 4; Algorithm 1, p. 14; Appendix C.1 and C.6, pp. 15--16.}
\end{frame}

\begin{frame}{Backup: two formula descriptions need care}
  \begin{block}{Score entropy is generally asymmetric}
    $\ell(2;1)=1-\log2$, but $\ell(1;2)=-1+2\log2$.\\
    Use nonnegativity and convexity in the predicted score;
    the p. 3 remark's symmetry claim does not follow from Eq. (5).
  \end{block}
  \begin{block}{The printed log-linear schedule conflicts with the prose}
    Appendix C.1 prints $-\log(1-(1-\epsilon t))=-\log(\epsilon t)$.\\
    A schedule consistent with increasing linear corruption is
    $-\log(1-(1-\epsilon)t)$, derived here from the stated intent.
  \end{block}
  \papersource{Formula interpretation based on Paper, PDF pp. 3, 5, and 15.
  The second expression is an inferred correction, not a verbatim paper formula.}
\end{frame}

\end{document}