% Original reading presentation, based on the arxiv-paper-reader Beamer template.
% Compile with pdfLaTeX using the accompanying build_slides.py.
\documentclass[aspectratio=169,11pt]{beamer}
\usepackage[T1]{fontenc}
\usepackage[utf8]{inputenc}
\usepackage{lmodern}
\usepackage{amsmath,amssymb,graphicx,booktabs}

\definecolor{AcademicBlue}{RGB}{26,57,86}
\definecolor{AcademicTeal}{RGB}{0,105,112}
\setbeamercolor{structure}{fg=AcademicBlue}
\setbeamercolor{frametitle}{fg=AcademicBlue,bg=white}
\setbeamercolor{title}{fg=AcademicBlue}
\setbeamercolor{alerted text}{fg=AcademicTeal}
\setbeamercolor{block title}{fg=white,bg=AcademicBlue}
\setbeamercolor{block body}{fg=black,bg=AcademicBlue!5}
\setbeamertemplate{navigation symbols}{}
\setbeamertemplate{itemize items}[circle]
\setbeamertemplate{footline}{%
  \hspace{8mm}{\color{gray}\scriptsize Smoothie\enspace |\enspace arXiv:2505.18853v2}%
  \hfill\usebeamerfont{page number in head/foot}%
  \insertframenumber\,/\,\inserttotalframenumber\hspace{4mm}\vspace{3mm}}
\setbeamersize{text margin left=8mm,text margin right=8mm}
\setbeamerfont{frametitle}{size=\Large,series=\bfseries}
\setbeamerfont{title}{size=\LARGE,series=\bfseries}
\newcommand{\papersource}[1]{\par\smallskip{\scriptsize\color{gray}#1\par}}
\newcommand{\E}{\mathbf E}
\newcommand{\D}{\mathbf D}
\newcommand{\prob}{\mathbf p}
\newcommand{\noise}{\boldsymbol\varepsilon}
\newcommand{\R}{\mathbb R}
\newcommand{\softmax}{\operatorname{softmax}}
\newcommand{\smoothie}{Smoothie}

\title[Smoothie]{Smoothie: Smoothing Diffusion on\newline Token Embeddings for Text Generation}
\subtitle{Semantic geometry, embedding regression, and sampling trade-offs}
\author{Paper authors: Alexander Shabalin,\newline Viacheslav Meshchaninov, Dmitry Vetrov}
\date{Research group reading\enspace |\enspace 14 October 2026\newline
\href{https://arxiv.org/abs/2505.18853v2}{arXiv:2505.18853v2, revised 15 May 2026}}

\begin{document}

% 1
\begin{frame}
\titlepage
\vspace{-5mm}
\begin{center}
\color{AcademicTeal} Use token distances to define corruption,\newline
and predict embeddings to reverse it.
\end{center}
\end{frame}

% 2
\begin{frame}{Can corruption preserve useful token relationships?}
\begin{block}{The design problem}
Text needs both a meaningful notion of similarity and a clear route back to discrete tokens.
\end{block}
\medskip
\begin{tabular}{@{}p{2.65cm}p{4.8cm}p{5.65cm}@{}}
\toprule
Representation & What carries information? & Main issue or proposed benefit \\
\midrule
Embedding & A continuous vector in $\R^d$ & Semantic geometry; requires token decoding \\[1.0ex]
Simplex & One score per vocabulary item & Clear token coordinates; all wrong tokens start equally far away \\[1.0ex]
\smoothie & Distances to every token & Token scores with semantic similarity \\
\bottomrule
\end{tabular}
\papersource{Paper, Secs. 1--3, PDF pp. 1--3. Earlier discrete methods also explored semantic corruption.}
\end{frame}

% 3
\begin{frame}{Smoothing spreads weight through embedding geometry}
\centering
\includegraphics[width=0.95\linewidth,height=0.65\textheight,keepaspectratio]{figures/fig-01-semantic-smoothing.png}
\begin{itemize}
\item Panel (c): nearby embeddings receive weight before distant ones.
\item Colors are vocabulary weights; the blue circle marks the original token.
\end{itemize}
\papersource{Paper, Fig. 1, PDF p. 2. Original schematic, not an experimental scatter plot.}
\end{frame}

% 4
\begin{frame}{A token becomes a vector of vocabulary distances}
For a target sequence of length $m$ and fixed embeddings $\E\in\R^{V\times d}$:
\[
 [\D_0]_{ij}=-\frac12\|\E_{w_i^y}-\E_j\|^2,
 \qquad \D_0\in\R^{m\times V}.
\]
\begin{itemize}\setlength\itemsep{6pt}
\item The original token has score $0$; nearby tokens have less negative scores.
\item Each coordinate still names a discrete vocabulary item.
\item Clean distance scores decode by nearest embedding:
\[
\arg\max_j[-\tfrac12\|\hat e-\E_j\|^2]
=\arg\min_j\|\hat e-\E_j\|^2.
\]
\end{itemize}
\begin{block}{Where the semantic assumption enters}
Euclidean proximity in the fixed embedding space should reflect useful token similarity.
\end{block}
\papersource{Paper, Sec. 4.1, Eq. (2), PDF p. 3. Ties are possible for identical embeddings.}
\end{frame}

% 5
\begin{frame}{Forward corruption shrinks the signal at fixed noise}
\[
\D_t=\frac{\D_0}{\sigma_t^2}+\delta\noise,
\qquad \noise\sim\mathcal N(0,I),\quad \delta=1.
\]
\[
\prob_t=\softmax(\D_t),\qquad h_t=\prob_t\E\in\R^{m\times d}.
\]
\begin{itemize}\setlength\itemsep{6pt}
\item Larger bandwidth $\sigma_t$ weakens distance differences.
\item Without logit noise, weights are normalized Gaussian kernels:
\[
p_{t,ij}\ \propto\
\exp\!\left(-\frac{\|\E_{w_i^y}-\E_j\|^2}{2\sigma_t^2}\right).
\]
\item The Transformer consumes weighted embeddings, not $V$-dimensional hidden states.
\item Given $\D_0$, noisy states at different times are sampled independently.
\end{itemize}
\papersource{Paper, Sec. 4.1, Eq. (3), PDF pp. 3--4; implementation, p. 5.}
\end{frame}

% 6
\begin{frame}{Embedding MSE recovers the optimal distance weights}
The practical training objective predicts $d$-dimensional embeddings:
\[
L_{\E}=\mathbb E\bigl[\|\E_{\mathbf w^y}-f_\theta(\prob_t,t)\|^2\bigr].
\]
Let $U$ be a clean embedding and $Z$ the observed noisy input and condition.
\[
\begin{aligned}
\mu&=\mathbb E[U\mid Z],\\
\mathbb E[-\tfrac12\|U-\E_j\|^2\mid Z]
&=-\tfrac12\|\mu-\E_j\|^2
-\tfrac12\operatorname{tr}\operatorname{Cov}(U\mid Z).
\end{aligned}
\]
\begin{block}{Why this matters}
The covariance term is the same for every candidate $j$.
Softmax removes this row-wise shift.
\end{block}
\smallskip
The identity concerns optimal regressors with squared Euclidean distances.
\papersource{Paper, Theorem 4.1 and Eq. (8), PDF p. 4; proof in Appendix C, p. 14. Conditional notation made explicit.}
\end{frame}

% 7
\begin{frame}{Sampling rebuilds a noisy state from each prediction}
Start with $\D_T\sim\mathcal N(0,\tilde\delta^2 I)$. At each step:
\[
\prob_t=\softmax(\D_t),\qquad
\hat e_t=f_\theta(\prob_t,t),
\]
\[
\boxed{\D_{t-1}=\frac{\D_0(\hat e_t)}{\sigma_{t-1}^2}
+\tilde\delta\noise.}
\]
\begin{itemize}\setlength\itemsep{7pt}
\item No explicit residual term carries the current noisy latent into the update.
\item The prediction still depends on $\D_t$ through $\prob_t$ and on self-conditioning.
\item $\tilde\delta$ controls generation randomness independently of training noise.
\item Algorithm 2 takes vocabulary argmax of the final scores; padding is removed.
\end{itemize}
\papersource{Paper, Sec. 4.2, Eq. (9), Algorithm 2, PDF p. 4. Eq. (9) includes noise in the final score update.}
\end{frame}

% 8
\begin{frame}{The model processes soft embeddings with a Transformer}
\begin{center}
\fbox{\strut Noisy scores $\D_t$}\quad$\longrightarrow$\quad
\fbox{\strut $\softmax(\D_t)\E$}\quad$\longrightarrow$\quad
\fbox{\strut Predicted embeddings}
\end{center}
\medskip
\begin{columns}[T,onlytextwidth]
\begin{column}{0.48\textwidth}
\begin{block}{Denoiser}
\begin{itemize}\setlength\itemsep{5pt}
\item 12 Transformer layers, width 768, 12 attention heads.
\item U-Net style additive skips between early and late layers.
\item Learned timestep embeddings in each block.
\end{itemize}
\end{block}
\end{column}
\begin{column}{0.48\textwidth}
\begin{block}{Conditioning and tokens}
\begin{itemize}\setlength\itemsep{5pt}
\item Conditional tasks add a 6-layer source encoder and cross-attention.
\item Fixed, normalized BERT embeddings.
\item Predict padding to support different output lengths.
\end{itemize}
\end{block}
\end{column}
\end{columns}
\medskip
Approximately 100M-scale models; denoisers are trained per task, with pretrained embeddings.
\papersource{Paper, Secs. 4.5--5, PDF p. 5; Appendix J, Table 12, p. 17.}
\end{frame}

% 9
\begin{frame}{A bandwidth schedule and self-conditioning shape denoising}
\[
\sigma_t=(\sigma_{\max}-\sigma_{\min})\frac{2}{\pi}
\arctan\!\left(\frac1{d_s}\sqrt{\frac{t}{T-t+\epsilon}}\right)
+\sigma_{\min}.
\]
\begin{itemize}\setlength\itemsep{6pt}
\item $\sigma_{\min}=1.5$, $\sigma_{\max}=200$; schedule shape $d_s\in\{5,7\}$.
\item Smaller $d_s$ means stronger corruption at the same relative time.
\end{itemize}
\medskip
\begin{block}{Self-conditioning}
\textbf{Training:} with probability $1/2$, use zero; otherwise use a detached preliminary prediction at the same noisy input.\par\medskip
\textbf{Generation:} use zero first, then the previous denoising step's prediction.
\end{block}
\papersource{Paper, Secs. 4.3--4.4, Eq. (10), PDF pp. 4--5. Here $d_s$ distinguishes the schedule parameter from embedding width; the paper uses $d$ for both.}
\end{frame}

% 10
\begin{frame}{Evaluation covers five tasks and two views of quality}
\begin{tabular}{@{}lll@{}}
\toprule
Dataset & Task & Main metrics \\
\midrule
ROCStories & Unconditional stories & MAUVE $\uparrow$, PPL $\downarrow$, diversity $\uparrow$ \\[0.7ex]
XSum & Summarization & BERTScore, ROUGE $\uparrow$ \\[0.7ex]
Quasar-T & Question generation & BERTScore, BLEU, R-L $\uparrow$ \\[0.7ex]
QQP & Paraphrase generation & BERTScore, BLEU, R-L $\uparrow$ \\[0.7ex]
ParaDetox & Detoxification & BLEU, J-Score $\uparrow$ \\
\bottomrule
\end{tabular}
\medskip
\begin{itemize}\setlength\itemsep{5pt}
\item ROCStories MAUVE: five generation seeds, 1,000 samples each, then average.
\item Generative PPL evaluates output text; it is not \smoothie's held-out likelihood.
\item Internal variants share architecture; schedules differ, and Simplex uses cross-entropy rather than MSE.
\end{itemize}
\papersource{Paper, Sec. 5, PDF pp. 5--7. External baseline rows combine prior reports and reproductions.}
\end{frame}

% 11
\begin{frame}{Computing over the vocabulary adds sampling cost}
\begin{center}
\begin{tabular}{lrr}
\toprule
Method & Time / s $\downarrow$ & Peak memory / MB $\downarrow$ \\
\midrule
Embedding & 1.642 & 593.3 \\
Simplex & 2.897 & 678.4 \\
\smoothie & 2.897 & 593.3 \\
\bottomrule
\end{tabular}
\end{center}
\medskip
\begin{itemize}\setlength\itemsep{7pt}
\item Measurement: 100 generation steps, batch 32, target length 80.
\item Computing distances to all embeddings costs $O(BmVd)$ per step.
\item The paper gives Embedding $1.75\times$ more steps to approximately match generation cost.
\end{itemize}
\begin{block}{Interpretation}
Low-dimensional network outputs do not eliminate vocabulary-wide work.
\end{block}
\papersource{Paper, Table 2, PDF p. 6; generation-speed protocol, pp. 6--7. The measured time ratio is about 1.76.}
\end{frame}

% 12
\begin{frame}{Sampling noise exposes a quality--diversity trade-off}
\begin{columns}[c,onlytextwidth]
\begin{column}{0.50\textwidth}
\centering
\includegraphics[width=\linewidth,height=0.72\textheight,keepaspectratio]{figures/fig-02-sampling-noise.pdf}
\end{column}
\begin{column}{0.47\textwidth}
\begin{itemize}\setlength\itemsep{8pt}
\item Blue: diversity, left axis.
\item Red: PPL, right axis.
\item Green labels: MAUVE values.
\item Training noise stays at $\delta=1$.
\end{itemize}
\medskip
\alert{ROCStories:} best plotted MAUVE at $\tilde\delta=1.1$.\par\medskip
\alert{Conditional tasks:} use $\tilde\delta=0.1$ after a separate search.
\end{column}
\end{columns}
\papersource{Paper, Fig. 2, Sec. 5.1, PDF pp. 5--6; conditional noise search, Table 8, p. 15.}
\end{frame}

% 13
\begin{frame}{ROCStories shows the largest distribution-quality gain}
\begin{center}
\begin{tabular}{lrrr}
\toprule
Method & MAUVE $\uparrow$ & PPL $\downarrow$ & Div $\uparrow$ \\
\midrule
GPT-2 (AR reference) & 78.9 & 20.5 & 25.2 \\
MDLM & 63.9 & 58.1 & 35.1 \\
TEncDM & 76.2 & 29.1 & 29.5 \\
Embedding & 41.5 & 28.3 & 26.1 \\
Simplex & 15.2 & 25.3 & 12.4 \\
\textbf{\smoothie} & \textbf{80.7} & \textbf{24.9} & 25.1 \\
\bottomrule
\end{tabular}
\end{center}
\medskip
\begin{itemize}\setlength\itemsep{6pt}
\item Versus TEncDM: MAUVE +4.5 points, with lower PPL and lower diversity.
\item Simplex has a competitive PPL but a much worse MAUVE.
\item GPT-2 still has the lowest PPL in this selection.
\end{itemize}
\papersource{Paper, Table 4, PDF p. 8. Selected rows; numbers use the paper's reporting scale.}
\end{frame}

% 14
\begin{frame}{XSum improves ROUGE, while BERTScore favors TEncDM}
\begin{center}
\begin{tabular}{lrrrr}
\toprule
Method & BS $\uparrow$ & R-1 $\uparrow$ & R-2 $\uparrow$ & R-L $\uparrow$ \\
\midrule
FLAN-T5 (AR) & 72.7 & 34.6 & 12.9 & 27.2 \\
TEncDM & \textbf{69.9} & 31.9 & 10.7 & 25.3 \\
Embedding & 68.2 & 32.1 & 10.1 & 24.6 \\
Simplex & 63.8 & 29.6 & 8.5 & 23.0 \\
\textbf{\smoothie} & 68.8 & \textbf{33.7} & \textbf{11.1} & \textbf{26.0} \\
\bottomrule
\end{tabular}
\end{center}
\medskip
\begin{block}{What the result supports}
Better lexical coverage among the compared diffusion models.
\end{block}
\begin{itemize}\setlength\itemsep{5pt}
\item R-L: +1.4 versus internal Embedding, +0.7 versus TEncDM.
\item BERTScore: 1.1 points below TEncDM; FLAN-T5 is stronger on all listed metrics.
\end{itemize}
\papersource{Paper, Table 3 (XSum), PDF p. 7. Bold highlights the best diffusion value in each column.}
\end{frame}

% 15
\begin{frame}{Quasar-T gains depend on the metric}
\begin{center}
\begin{tabular}{lrrr}
\toprule
Method & BS $\uparrow$ & BLEU $\uparrow$ & R-L $\uparrow$ \\
\midrule
BART (AR) & 66.2 & 17.4 & 38.8 \\
SSD-LM & 62.8 & 14.1 & \textbf{38.5} \\
TESS (random initialization) & 60.8 & 19.0 & 36.1 \\
Embedding & 62.0 & 18.9 & 35.2 \\
Simplex & 63.0 & 19.3 & 36.9 \\
\textbf{\smoothie} & \textbf{63.1} & \textbf{19.9} & 36.5 \\
\bottomrule
\end{tabular}
\end{center}
\medskip
\begin{itemize}\setlength\itemsep{7pt}
\item Best diffusion BS and BLEU, but not the best R-L.
\item Internal Simplex already improves on TESS: architecture also matters.
\item Against BART, BLEU is higher while BS and R-L are lower.
\end{itemize}
\papersource{Paper, Table 3 (Quasar-T), PDF p. 7. Selected rows.}
\end{frame}

% 16
\begin{frame}{QQP and detoxification show complementary gains}
\begin{columns}[T,onlytextwidth]
\begin{column}{0.53\textwidth}
\textbf{QQP: reference matching}
\medskip
\begin{tabular}{lrrr}
\toprule
Method & BS & BLEU & R-L \\
\midrule
BART & 85.7 & 30.4 & 61.4 \\
Embedding & 83.4 & 31.3 & 59.4 \\
Simplex & 80.6 & 26.8 & 54.9 \\
\smoothie & 83.9 & 30.8 & 60.9 \\
\bottomrule
\end{tabular}
\par\medskip
Higher BS and R-L than Embedding, but lower BLEU.
\end{column}
\begin{column}{0.43\textwidth}
\textbf{ParaDetox: style conversion}
\medskip
\begin{tabular}{lrr}
\toprule
Method & BLEU & J-Score \\
\midrule
GPT-2 & 67.7 & 60.4 \\
Embedding & 67.6 & 49.1 \\
Simplex & 65.1 & 47.7 \\
\smoothie & 69.2 & 51.7 \\
\bottomrule
\end{tabular}
\par\medskip
Stronger diffusion scores; GPT-2 retains a higher J-Score.
\end{column}
\end{columns}
\medskip
All metrics shown increase with better performance.
\papersource{Paper, Table 4, PDF p. 8. Main-table QQP values differ from some ablation/default-setting entries; see backup.}
\end{frame}

% 17
\begin{frame}{More denoising steps do not consistently improve quality}
\begin{center}
\begin{tabular}{rrrrr}
\toprule
Steps & XSum BS & Quasar-T BS & QQP BS & ParaDetox J \\
\midrule
25 & 67.7 & 63.1 & \textbf{83.9} & 51.1 \\
50 & 68.5 & 63.1 & 83.8 & 51.4 \\
100 & 68.7 & 63.1 & 83.7 & \textbf{51.7} \\
200 & \textbf{68.8} & 63.1 & 83.6 & 51.0 \\
500 & 68.4 & 63.1 & 83.5 & 50.8 \\
\bottomrule
\end{tabular}
\end{center}
\medskip
\begin{itemize}\setlength\itemsep{7pt}
\item XSum benefits up to 200 steps; QQP peaks at 25 in this sweep.
\item Quasar-T BS is unchanged at the reported precision.
\item Authors suggest self-conditioning mismatch as one explanation for late degradation.
\end{itemize}
\papersource{Paper, Sec. 5.3, Table 5, PDF pp. 7--8. Higher is better for every metric shown.}
\end{frame}

% 18
\begin{frame}{Rebuilding from predictions gives modest solver gains}
\begin{center}
\begin{tabular}{lrrrr}
\toprule
Solver & XSum R-L & Quasar-T R-L & QQP R-L & ParaDetox J \\
\midrule
DDPM & 25.6 & 35.5 & 60.7 & 51.5 \\
DDIM & 24.6 & 31.8 & 57.9 & 50.2 \\
Eq. (9) & \textbf{26.0} & \textbf{36.5} & \textbf{60.9} & \textbf{51.7} \\
\bottomrule
\end{tabular}
\end{center}
\medskip
\begin{itemize}\setlength\itemsep{7pt}
\item The proposed update wins on all metrics in this comparison; margins over DDPM are small.
\item The DDPM/DDIM comparison introduces time-dependent forward variance:
\[
\operatorname{Var}(\D_t\mid\D_0)
=\left(1-\sigma_t^{-4}\right)\delta^2 I.
\]
\item Reduced drift is the authors' explanation, not a proved mechanism.
\end{itemize}
\papersource{Paper, Sec. 5.4, Eq. (11), Table 6, PDF p. 8. Selected metric columns, all higher is better.}
\end{frame}

% 19
\begin{frame}{OpenWebText uses a hybrid solver and shows smaller gains}
\begin{center}
\begin{tabular}{lrrr}
\toprule
Method & MAUVE $\uparrow$ & PPL $\downarrow$ & Div $\uparrow$ \\
\midrule
GIDD & 28.6 & 228.3 & 58.8 \\
Embedding & 55.7 & 51.9 & 21.2 \\
\smoothie, $\tilde\delta=0.96$ & 58.9 & \textbf{48.2} & 19.8 \\
\smoothie, $\tilde\delta=1.02$ & \textbf{59.9} & 62.2 & 23.4 \\
\bottomrule
\end{tabular}
\end{center}
\medskip
\begin{itemize}\setlength\itemsep{6pt}
\item Target length 512; \smoothie{} uses 100 steps, Embedding 175.
\item \smoothie: first 40\% DDPM, then Eq. (9); Embedding: DDPM throughout.
\item Best MAUVE gain over Embedding is 4.2 points, with a PPL/diversity trade-off.
\end{itemize}
\papersource{Paper, Appendix D, Table 7, PDF pp. 14--15. This is an explicit exception to the main-text solver description.}
\end{frame}

% 20
\begin{frame}{The embedding geometry is an important design choice}
\begin{center}
\begin{tabular}{lrrr}
\toprule
Embedding source & MAUVE $\uparrow$ & PPL $\downarrow$ & Div $\uparrow$ \\
\midrule
BERT & 80.7 & 24.9 & 25.1 \\
GPT-2 & 64.4 & 23.1 & 25.0 \\
GloVe & 36.8 & 36.4 & 24.6 \\
\bottomrule
\end{tabular}
\end{center}
\medskip
\begin{itemize}\setlength\itemsep{7pt}
\item On ROCStories, BERT embeddings yield much better MAUVE.
\item GPT-2 embeddings lower PPL without matching distribution quality.
\item This also changes token vocabularies: GPT-2 is about 50k; GloVe BPE is 10k.
\end{itemize}
\begin{block}{Design consequence}
The corruption geometry is inherited from a pretrained representation.
\end{block}
\papersource{Paper, Appendix F, Table 9, PDF p. 15. GloVe embeddings were trained by the authors on Wikipedia.}
\end{frame}

% 21
\begin{frame}{Three lessons from Smoothie}
\begin{enumerate}\setlength\itemsep{11pt}
\item \textbf{Design the corruption space.}\par
Distances preserve vocabulary coordinates while giving nearby tokens related scores.
\item \textbf{Use the geometry to simplify learning.}\par
Squared-distance regression reduces to embedding MSE up to a softmax-invariant shift.
\item \textbf{Tune sampling to the task.}\par
Noise, iteration count, and solver choice affect the quality--diversity balance.
\end{enumerate}
\medskip
\alert{Main costs:} fixed embedding geometry and vocabulary-wide computation at each step.
\papersource{Shabalin, Meshchaninov, and Vetrov. Smoothie: Smoothing Diffusion on Token Embeddings for Text Generation.\newline
\href{https://arxiv.org/abs/2505.18853v2}{arXiv:2505.18853v2}, 2026. Main talk ends here; following slides are backup.}
\end{frame}

\appendix

% 22
\begin{frame}{Backup: a three-token example of kernel smoothing}
\textbf{Teaching example:} squared distances to the original token are $0,4,36$.\par
Ignore the stochastic logit term and normalize $\exp[-r^2/(2\sigma^2)]$.
\medskip
\begin{center}
\begin{tabular}{rrrr}
\toprule
Bandwidth $\sigma$ & Original & Nearby & Distant \\
\midrule
1.5 & 0.7085 & 0.2913 & 0.0002 \\
3 & 0.5165 & 0.4136 & 0.0699 \\
6 & 0.3918 & 0.3706 & 0.2376 \\
\bottomrule
\end{tabular}
\end{center}
\begin{itemize}\setlength\itemsep{7pt}
\item Small bandwidth mixes the original token with its neighbor first.
\item With actual noise, the large-bandwidth limit is $\softmax(\delta\noise)$, not a fixed uniform vector.
\end{itemize}
\papersource{Teaching calculation based on Eq. (3), PDF p. 3; not a paper experiment or real-token measurement.}
\end{frame}

% 23
\begin{frame}{Backup: the simplified objective and its indexing}
The variational bound contains Gaussian KL terms for $t\rightarrow t-1$:
\[
\mathrm{KL}\left(q(\D_{t-1}\mid\D_0)\,\|\,
p_\theta(\D_{t-1}\mid\D_t)\right).
\]
With Eq. (3), its model-dependent part for $t>1$ is proportional to
\[
\frac{1}{2\tilde\delta^2}
\left\|\frac{\D_0}{\sigma_{t-1}^2}-\mu_\theta(\prob_t,t)\right\|^2.
\]
\begin{itemize}\setlength\itemsep{6pt}
\item Predict clean distances and drop the time weighting to obtain $L_{\D}$.
\item The paper's Eqs. (6) and (23) use $\sigma_t$ here; Eq. (9) uses $\sigma_{t-1}$.
\item Algorithm 1 also prints $\D_t$ in the clean-input position; Eq. (3) requires $\D_0$.
\end{itemize}
\papersource{Paper, Sec. 4.2 and Algorithm 1, PDF p. 4; Appendix B, pp. 13--14. Index-consistent expression shown above.}
\end{frame}

% 24
\begin{frame}{Backup: the schedule shifts where reconstruction becomes hard}
\centering
\includegraphics[width=0.86\linewidth,height=0.49\textheight,keepaspectratio]{figures/fig-03-noise-schedule.pdf}
\par\medskip
ROCStories MAUVE for schedule $d=4,5,6,7$: \textbf{74.2, 80.7, 76.5, 68.9}.\par
Each setting selects its own best $\tilde\delta$.
\begin{itemize}
\item Left: reconstruction loss; right: non-padding token accuracy.
\item Smaller $d$ removes usable information earlier.
\end{itemize}
\papersource{Paper, Fig. 3, PDF p. 16; Table 10, p. 15. The caption mentions training with $d=5$ while the legend varies $d$; separate training per curve is not clarified.}
\end{frame}

% 25
\begin{frame}{Backup: self-conditioning helps Smoothie unevenly}
\begin{center}
\begin{tabular}{llrr}
\toprule
Task & Metric & With SC & Without SC \\
\midrule
XSum & BERTScore & 68.8 & 67.5 \\
XSum & ROUGE-L & 26.0 & 25.4 \\
Quasar-T & BERTScore & 63.0 & 61.9 \\
Quasar-T & BLEU & 19.0 & 19.0 \\
QQP & BERTScore & 83.9 & 83.2 \\
QQP & BLEU & 30.8 & 29.4 \\
\bottomrule
\end{tabular}
\end{center}
\medskip
\begin{itemize}\setlength\itemsep{6pt}
\item Larger gains on XSum BS and QQP BLEU; Quasar-T BLEU is unchanged.
\item Embedding diffusion on Quasar-T improves when SC is removed in this ablation.
\end{itemize}
\papersource{Paper, Appendix G, Table 11, PDF p. 16. Use within-table pairs; its Quasar-T values differ from the main table.}
\end{frame}

% 26
\begin{frame}{Backup: the MAUVE advantage persists over much of training}
\begin{columns}[c,onlytextwidth]
\begin{column}{0.58\textwidth}
\centering
\includegraphics[width=\linewidth,height=0.67\textheight,keepaspectratio]{figures/fig-04-training-dynamics.pdf}
\end{column}
\begin{column}{0.38\textwidth}
\begin{itemize}\setlength\itemsep{8pt}
\item X-axis: training hours.
\item Y-axis: MAUVE on a 0--1 scale.
\item Evaluation uses 350 steps for Embedding and 200 for the other variants.
\item After the initial phase, \smoothie{} leads through most of the plotted interval.
\end{itemize}
\end{column}
\end{columns}
\papersource{Paper, Appendix I, Fig. 4, PDF p. 16. The main tables report MAUVE on a 0--100 scale.}
\end{frame}

% 27
\begin{frame}{Backup: default configurations and reported variations}
\begin{center}
\begin{tabular}{lrrrrr}
\toprule
 & ROCStories & XSum & Quasar-T & QQP & ParaDetox \\
\midrule
Target length & 80 & 64 & 50 & 50 & 40 \\
Train steps & 1M & 225k & 150k & 50k & 150k \\
Sample steps & 200 & 200 & 100 & 100 & 100 \\
$\tilde\delta$ & 1.1 & 0.1 & 0.1 & 0.1 & 0.1 \\
Schedule $d$ & 5 & 5 & 7 & 5 & 7 \\
\bottomrule
\end{tabular}
\end{center}
\medskip
\begin{itemize}\setlength\itemsep{6pt}
\item AdamW, learning rate $2\times10^{-4}$, 5k warmup, EMA 0.9999.
\item QQP main-table BS is 83.9; the 100-step sweep reports 83.7, despite the listed default of 100 steps.
\item The SC ablation's Quasar-T row also differs from the main table. The paper does not reconcile these entries.
\end{itemize}
\papersource{Paper, Table 12, PDF p. 17; compare Tables 4--5, p. 8, and Table 11, p. 16.}
\end{frame}

% 28
\begin{frame}{Backup: semantic degradation is observed through a decoder}
\begin{tabular}{@{}p{2cm}p{10.8cm}@{}}
\toprule
Noise level & Qualitative change in the example \\
\midrule
Low & Related substitutions: the number of children changes; one US state replaces another. \\[1.4ex]
Intermediate & Semantic relations and sentence structure become unreliable. \\[1.4ex]
High & Frequent tokens and repetitions dominate the reconstructed text. \\
\bottomrule
\end{tabular}
\medskip
\begin{block}{How to interpret the example}
The authors sample noisy forward latents, run the model, and decode its predictions.
These are reconstructions, not direct token samples from the forward weights.
\end{block}
\papersource{Paper, Appendix L, Table 13, PDF p. 18. Paraphrased qualitative description.}
\end{frame}

\end{document}