% Based on the arxiv-paper-reader Beamer template. Compile with pdfLaTeX.
\documentclass[aspectratio=169,11pt]{beamer}
\usepackage[T1]{fontenc}
\usepackage[utf8]{inputenc}
\usepackage{lmodern}
\usepackage{amsmath,amssymb,graphicx,booktabs}
\usepackage{tikz}
\usetikzlibrary{arrows.meta}
\pdfstringdefDisableCommands{\def\translate#1{#1}}
\definecolor{AcademicBlue}{RGB}{26,57,86}
\definecolor{AcademicTeal}{RGB}{0,105,112}
\setbeamercolor{structure}{fg=AcademicBlue}
\setbeamercolor{frametitle}{fg=AcademicBlue,bg=white}
\setbeamercolor{title}{fg=AcademicBlue}
\setbeamercolor{alerted text}{fg=AcademicTeal}
\setbeamercolor{block title}{fg=white,bg=AcademicBlue}
\setbeamercolor{block body}{fg=black,bg=AcademicBlue!5}
\setbeamertemplate{navigation symbols}{}
\setbeamertemplate{itemize items}[circle]
\setbeamertemplate{footline}{\hfill\usebeamerfont{page number in head/foot}\insertframenumber\,/\,\inserttotalframenumber\hspace{4mm}\vspace{3mm}}
\setbeamersize{text margin left=8mm,text margin right=8mm}
\setbeamerfont{frametitle}{size=\Large,series=\bfseries}
\setbeamerfont{title}{size=\LARGE,series=\bfseries}
\newcommand{\papersource}[1]{\par\smallskip{\scriptsize\color{gray}#1\par}}
\newcommand{\RMS}{\operatorname{RMSNorm}}
\newcommand{\softmax}{\operatorname{softmax}}
\newcommand{\src}{\mathcal S_l}
\title[Delta Attention Residuals]{Delta Attention Residuals}
\subtitle{Selective depth readout alongside a preserved residual stream}
\author{Paper by Cheng Luo, Zefan Cai, and Junjie Hu}
\date{Research paper discussion\quad\textbar\quad 4 October 2026\\\small arXiv:2605.18855v1 (13 May 2026)}

\begin{document}

\begin{frame}
  \titlepage
  \vspace{-4mm}
  \centering\small\href{https://arxiv.org/abs/2605.18855v1}{Versioned paper}\quad\textbar\quad Approximately 25 minutes + backup discussion
\end{frame}

\begin{frame}{The useful design is an extra readout from depth}
  \begin{block}{Core computation}
  \[
    \hat{\mathbf h}_l=\tilde{\mathbf h}_l+
    \underbrace{\sum_{i\in\src}\alpha_{i\to l}\mathbf v_i}_{\text{selected historical contributions}}
  \]
  \end{block}
  \begin{itemize}
    \item Preserve the natural residual stream; route into the next sublayer.
    \item At 7.57B: validation PPL \alert{17.43 $\to$ 16.00}, with lower throughput.
    \item Separate measured gains from claims about novelty and initialization.
  \end{itemize}
  \papersource{Paper, Eq. (2), PDF p. 3; Table 3, PDF p. 6.}
\end{frame}

\begin{frame}{Standard residuals expose a sum of earlier contributions}
  \[
    \mathbf h_{i+1}=\mathbf h_i+\mathbf v_i,
    \qquad \mathbf v_i=f_i(\mathbf h_i)
  \]
  \[
    \mathbf h_l=\mathbf h_0+\sum_{i<l}\mathbf v_i
  \]
  \begin{itemize}
    \item A sublayer is attention or MLP; $f_i$ includes its Pre-Norm operation.
    \item Every past output enters the running sum with coefficient one.
    \item The paper asks whether a sublayer can selectively revisit a contribution.
  \end{itemize}
  \begin{block}{Motivation}
    Shared prefixes can make cumulative states difficult to distinguish.
  \end{block}
  \papersource{Paper, Secs. 2.1--2.2, PDF pp. 2--3. The geometric argument is a motivation.}
\end{frame}

\begin{frame}{Original AttnRes already routes over sublayer outputs}
  Original Full AttnRes explicitly defines its sources as
  \[
    \mathbf k_i=\mathbf v_i=
    \begin{cases}
    \mathbf h_1,&i=0,\\
    f_i(\mathbf h_i),&1\leq i<l.
    \end{cases}
  \]
  \begin{itemize}
    \item Original Block AttnRes sums outputs \emph{within} each block.
    \item These are not all full-history cumulative hidden states.
    \item Thus the present paper's baseline description misstates the prior work.
  \end{itemize}
  \begin{block}{Reading implication}
    Focus on the additive residual path and the tested implementations.
  \end{block}
  \papersource{External primary-source comparison: Attention Residuals, arXiv:2603.15031v1, Secs. 3.1--3.2, Eqs. (3), (5), PDF p. 4.}
\end{frame}

\begin{frame}{A learned query scores depth sources for each token}
  \[
    e_{i\to l}=\mathbf w_l^\top\RMS(\mathbf v_i),\qquad
    \alpha_{i\to l}=\frac{\exp e_{i\to l}}{\sum_{j\in\src}\exp e_{j\to l}}
  \]
  \[
    \mathbf r_l=\sum_{i\in\src}\alpha_{i\to l}\mathbf v_i
  \]
  \begin{itemize}
    \item $\mathbf w_l\in\mathbb R^d$: one learned query per receiving sublayer.
    \item Keys are normalized; values retain their original magnitudes.
    \item Queries are parameters, while source content makes weights token-dependent.
    \item Softmax runs over depth sources, independently at each sequence position.
  \end{itemize}
  \papersource{Paper, Eqs. (1)--(2), PDF pp. 2--3; Fig. 3, PDF p. 4.}
\end{frame}

\begin{frame}{Readout changes the sublayer input, not the residual write}
  \begin{align*}
    \hat{\mathbf h}_l&=\tilde{\mathbf h}_l+\mathbf r_l
    &&\text{read from depth}\\[2mm]
    \mathbf v_l&=f_l(\hat{\mathbf h}_l)
    &&\text{compute new contribution}\\[2mm]
    \tilde{\mathbf h}_{l+1}&=\tilde{\mathbf h}_l+\mathbf v_l
    &&\text{update the original stream}
  \end{align*}
  \begin{itemize}
    \item Attention and MLP have separate routing queries and norms.
    \item Delta AttnRes appends each attention/MLP output to the source list.
    \item Adding $\mathbf r_l$ again in the last line would implement a different method.
  \end{itemize}
  \papersource{Paper, Eq. (2), PDF p. 3; forward pseudocode, Fig. 3, PDF p. 4.}
\end{frame}

\begin{frame}{Block grouping reduces the number of selectable sources}
  With sublayer boundary indices $\tau_b$, the intended block definition is
  \[
    \Delta_b=\tilde{\mathbf h}_{\tau_{b+1}}-\tilde{\mathbf h}_{\tau_b}
      =\sum_{i=\tau_b}^{\tau_{b+1}-1}\mathbf v_i.
  \]
  \centering
  \begin{tabular}{lll}
    \toprule
    & Delta AttnRes & Delta Block\\
    \midrule
    Selectable unit & One sublayer output & A grouped contribution\\
    Source count & $O(2L)$ & $O(K)$ blocks\\
    Benefit & Finest selection & Lower readout cost\\
    Tradeoff & More activations & Less granular selection\\
    \bottomrule
  \end{tabular}
  \par\medskip\raggedright
  Residual preservation does not preserve individual access inside a block.
  \papersource{Paper, Appendix C, Eq. (3), PDF pp. 13--14. Here $K$ is block count; the paper overloads $B$ for count and width.}
\end{frame}

\begin{frame}{Block sources require a saved boundary state}
  \begin{columns}[T]
  \begin{column}{0.49\textwidth}
    \begin{block}{Intended computation}
      \texttt{delta = current - block\_start}\par
      \texttt{block\_start = current}
    \end{block}
    \medskip
    Boundary states: $1,3,6$.\\
    Correct deltas: $2,3$.
  \end{column}
  \begin{column}{0.49\textwidth}
    \begin{block}{Published computation}
      \texttt{delta = current - last\_source}
    \end{block}
    \medskip
    After storing $3-1=2$,\\
    the next result is $6-2=4$.
  \end{column}
  \end{columns}
  \medskip
  \begin{itemize}
    \item Fig. 3 and the public \texttt{delta\_block} branch use the latter pattern.
    \item Formula and implementation need reconciliation before reproduction.
  \end{itemize}
  \papersource{Paper, Fig. 3, PDF p. 4; Eq. (3), p. 13. Public code at fc11c3e, modeling\_qwen3\_attnres.py, lines 559--610 and 1104--1108. Scalar example is our analysis.}
\end{frame}

\begin{frame}{A zero query does not generally make readout an identity}
  For $N$ available sources:
  \[
    \mathbf w_l=0\ \Longrightarrow\
    \hat{\mathbf h}_l=\tilde{\mathbf h}_l+\frac{1}{N}\sum_i\mathbf v_i.
  \]
  A zero-valued null source changes $1/N$ to $1/(N+1)$.
  \begin{itemize}
    \item The added mean need not vanish; a null source is not a zero gate.
    \item Complete source coverage can yield a scale change removable by ideal RMSNorm (backup).
    \item A gate $g_l=0$ in $\hat{\mathbf h}_l=\tilde{\mathbf h}_l+g_l\mathbf r_l$ would guarantee identity.
  \end{itemize}
  \papersource{Our substitution into Eq. (2). Paper's identity claim: Sec. 2.3, PDF p. 4, and Sec. 3.5, p. 8. Zero-gate construction is an analytical suggestion, not a main-table result.}
\end{frame}

\begin{frame}{Training is matched by steps and data within each scale}
  \centering
  \begin{tabular}{lrrr}
    \toprule
    Architecture & $d$ / $L$ & Batch / length & Tokens in 10K steps\\
    \midrule
    220M & 768 / 12 & 32 / 1024 & 0.328B\\
    533M & 1024 / 24 & 32 / 1024 & 0.328B\\
    1044M & 1280 / 36 & 32 / 1024 & 0.328B\\
    Qwen3-0.6B (508M) & 1024 / 28 & 64 / 2048 & 1.311B\\
    Qwen3-8B (7.57B) & 4096 / 36 & 64 / 2048 & 1.311B\\
    \bottomrule
  \end{tabular}
  \par\medskip\raggedright
  \begin{itemize}
    \item FineWeb-Edu; AdamW; 500-step warmup + cosine decay.
    \item LR $6\times10^{-4}$; $3\times10^{-4}$ at 7.57B.
    \item 8 H100s, BF16, \texttt{torch.compile}; 7.57B uses FSDP + checkpointing.
    \item Same steps do not mean the same wall-clock or compute budget.
  \end{itemize}
  \papersource{Paper, Secs. 3.1 and 3.3, PDF pp. 4--6. Token totals calculated as steps $\times$ batch $\times$ length.}
\end{frame}

\begin{frame}{Delta variants reduce PPL at all three custom scales}
  \centering
  \begin{tabular}{lrrr}
    \toprule
    Validation PPL $\downarrow$ & 220M & 533M & 1044M\\
    \midrule
    Baseline & 38.71 & 32.00 & 29.70\\
    AttnRes (reimplementation) & 37.39 & 31.75 & 31.76\\
    Full AttnRes (reimplementation) & 37.30 & 31.68 & 33.36\\
    Delta Block & 37.08 & 31.16 & 29.19\\
    Delta AttnRes & \textbf{36.83} & \textbf{31.05} & \textbf{29.13}\\
    \bottomrule
  \end{tabular}
  \par\medskip\raggedright
  \begin{itemize}
    \item Delta AttnRes reductions vs. baseline: 4.86\%, 2.97\%, 1.92\%.
    \item Delta Block stays close to the finest-grained variant.
    \item The tested replacement-routing baselines deteriorate at 1044M.
  \end{itemize}
  \papersource{Paper, Table 1, PDF p. 5. Lower PPL is better; these are fixed-step comparisons.}
\end{frame}

\begin{frame}{At 1044M, grouping recovers much of the lost throughput}
  \centering
  \begin{tabular}{lrrr}
    \toprule
    Method & PPL $\downarrow$ & k token/s $\uparrow$ & Peak GB / GPU\\
    \midrule
    Baseline & 29.70 & 108 & 22.5\\
    AttnRes & 31.76 & 71 & 31.6\\
    Full AttnRes & 33.36 & 49 & 52.0\\
    Delta Block & 29.19 & 86 & 28.4\\
    Delta AttnRes & 29.13 & 34 & 77.7\\
    \bottomrule
  \end{tabular}
  \par\medskip\raggedright
  \begin{block}{Block vs. per-sublayer delta routing}
    0.21\% higher PPL, \alert{2.53$\times$ throughput}, 36.6\% of the memory.
  \end{block}
  Versus baseline: throughput $-20.4\%$, memory $+26.2\%$.
  \papersource{Paper, Table 1, PDF p. 5. 8 H100s, BF16, compiled; peak memory measured at batch 4/GPU and length 1024. Ratios calculated from the table.}
\end{frame}

\begin{frame}{At 7.57B, lower PPL costs about 1.53$\times$ the time}
  \centering
  \begin{tabular}{lrrr}
    \toprule
    Method & PPL $\downarrow$ & k token/s $\uparrow$ & Peak GB / GPU\\
    \midrule
    Baseline & 17.43 & 21.4 & 41.6\\
    AttnRes & 18.58 & 12.5 & 44.0\\
    Delta Block & \textbf{16.00} & 14.0 & 42.7\\
    \bottomrule
  \end{tabular}
  \par\medskip\raggedright
  \begin{itemize}
    \item Versus baseline: PPL $-8.20\%$, throughput $-34.6\%$, memory $+2.6\%$.
    \item At fixed token count, steady-state time ratio is $21.4/14.0=1.53$.
    \item 589.8K extra parameters ($\sim$0.008\%) do not imply negligible runtime.
    \item Trained from scratch for 1.311B input tokens; only three variants tested.
  \end{itemize}
  \papersource{Paper, Table 3 and Sec. 3.3, PDF pp. 6--7. 8 H100s, FSDP, BF16, compilation, gradient checkpointing.}
\end{frame}

\begin{frame}{More blocks do not guarantee better perplexity}
  \begin{columns}[T]
  \begin{column}{0.49\textwidth}
    \centering\textbf{220M, $L=12$}\par\medskip
    \begin{tabular}{rrr}
      \toprule
      Blocks & Sources & PPL $\downarrow$\\
      \midrule
      1 & 2 & 37.44\\
      2 & 3 & 37.08\\
      4 & 5 & 36.98\\
      6 & 7 & 36.92\\
      12 & 13 & 37.34\\
      \bottomrule
    \end{tabular}
    \par\medskip Best block result near 4--6 blocks.
  \end{column}
  \begin{column}{0.49\textwidth}
    \centering\textbf{533M, $L=24$}\par\medskip
    \begin{tabular}{rrr}
      \toprule
      Blocks & Sources & PPL $\downarrow$\\
      \midrule
      2 & 3 & 31.23\\
      4 & 5 & 31.27\\
      6 & 7 & 31.19\\
      12 & 13 & 31.22\\
      24 & 25 & 31.18\\
      \bottomrule
    \end{tabular}
    \par\medskip PPL varies by only 0.09.
  \end{column}
  \end{columns}
  \medskip
  More sources generally increase memory; throughput is not monotonic here.
  \papersource{Paper, Table 4, PDF p. 7. $B$ means block count in this table. Table 1 and Table 4 contain different runs and should not be pooled.}
\end{frame}

\begin{frame}{Fine-tuning compares architecture--optimizer combinations}
  \begin{itemize}
    \item Start from pretrained Qwen3-0.6B; continue on FineWeb-Edu.
    \item 20K steps, batch 32, 4 H100s, 500-step warmup.
    \item Delta Block uses a null source and separate parameter-group LRs.
  \end{itemize}
  \medskip
  \centering
  \begin{tabular}{lrr}
    \toprule
    Method & Backbone LR & Routing LR\\
    \midrule
    Baseline FT & $5\times10^{-5}$ & ---\\
    AttnRes & $5\times10^{-5}$ & $5\times10^{-5}$\\
    Delta Block + null & $5\times10^{-5}$ & $5\times10^{-3}$\\
    \bottomrule
  \end{tabular}
  \par\medskip\raggedright
  The 100$\times$ routing LR and null source are not separately ablated.
  \papersource{Paper, Sec. 3.5, PDF p. 8. Eight 0-shot tasks evaluated with lm-evaluation-harness.}
\end{frame}

\begin{frame}{The +0.6-point mean gain is driven mainly by ARC-Easy}
  \centering
  \begin{tabular}{lrrr}
    \toprule
    0-shot score (\%) & Baseline FT & Delta Block & Change (pp)\\
    \midrule
    HellaSwag & 50.1 & 50.0 & $-0.1$\\
    ARC-Easy & 61.5 & \textbf{66.5} & \textbf{$+5.0$}\\
    ARC-Challenge & 36.7 & 37.3 & $+0.6$\\
    PIQA & 69.2 & 69.4 & $+0.2$\\
    WinoGrande & 57.0 & 56.7 & $-0.3$\\
    BoolQ & 70.3 & 70.3 & $0.0$\\
    MMLU & 45.2 & 44.8 & $-0.4$\\
    LAMBADA & 50.0 & 49.8 & $-0.2$\\
    \midrule
    Mean & 55.0 & \textbf{55.6} & $+0.6$\\
    \bottomrule
  \end{tabular}
  \par\smallskip\raggedright
  3 wins, 1 tie, 4 losses vs. standard FT; AttnRes mean is 54.1\%.
  \papersource{Paper, Table 5, PDF p. 8. Single training runs, no error bars (checklist, PDF pp. 17--18).}
\end{frame}

\begin{frame}{The plotted fine-tuning losses become very close}
  \begin{columns}[c]
  \begin{column}{0.51\textwidth}
    \includegraphics[width=\linewidth,height=5.3cm,keepaspectratio]{figures/finetune-loss.png}
  \end{column}
  \begin{column}{0.46\textwidth}
    \begin{itemize}
      \item Green is lower early; late losses nearly overlap.
      \item The graph contains no standard-FT baseline curve.
      \item Its 2.66--2.80 axis does not show the 3.96 spike claimed in the text.
    \end{itemize}
    \medskip
    This plot cannot establish exact identity initialization.
  \end{column}
  \end{columns}
  \papersource{Paper, Fig. 5(c) and Sec. 3.5, PDF p. 8. Cropped original panel with axes and legend preserved.}
\end{frame}

\begin{frame}{Routing becomes more concentrated in the tested 0.6B model}
  \begin{columns}[c]
  \begin{column}{0.51\textwidth}
    \includegraphics[width=\linewidth,height=5.0cm,keepaspectratio]{figures/scratch-sharpness.png}
  \end{column}
  \begin{column}{0.46\textwidth}
    \begin{itemize}
      \item Deep-layer maximum weights: roughly 0.2 vs. 0.6.
      \item Average maxima: 0.35 vs. 0.62 (1.77$\times$).
      \item Validation PPL: 32.22 baseline, 32.38 AttnRes, 31.45 Delta Block.
    \end{itemize}
    \medskip
    Caption identifies Delta Block; the image legend says Delta-AttnRes.
  \end{column}
  \end{columns}
  \papersource{Paper, Fig. 4(a), Table 2, PDF p. 6. Qwen3-0.6B architecture, $N=L=28$, trained from scratch.}
\end{frame}

\begin{frame}{Heatmaps show localized choices and embedding reuse}
  \centering
  \includegraphics[width=0.89\linewidth,height=5.6cm,keepaspectratio]{figures/fig2_routing_patterns.png}
  \par\smallskip\small
  Rows: receiving sublayers. Columns: sources. Color: attention weight (0--1).
  \papersource{Paper, Fig. 6, PDF p. 9. Right panel shows stronger local peaks; blank upper triangle is unavailable future depth.}
\end{frame}

\begin{frame}{The experiments support a design, not a unique cause}
  \begin{enumerate}
    \item \textbf{Quality:} delta variants improve the reported fixed-step PPLs.
    \item \textbf{Cost:} block grouping offers a useful compromise; 7.57B throughput still falls 34.6\% vs. baseline.
    \item \textbf{Interpretation:} source construction, additive paths, and optimization change together.
  \end{enumerate}
  \begin{block}{Most informative next comparison}
    Fix the source definition; compare additive vs. replacement readout with matched initialization, learning rates, and wall-clock budget.
  \end{block}
  \papersource{Synthesis of Secs. 2--4. Proposed comparison is our analysis, not a reported experiment.}
\end{frame}

\begin{frame}{References and discussion}
  \textbf{Main paper}\par
  Cheng Luo, Zefan Cai, and Junjie Hu.\par
  \emph{Delta Attention Residuals}. arXiv:2605.18855v1, 2026.\par
  \href{https://arxiv.org/abs/2605.18855v1}{arxiv.org/abs/2605.18855v1}\par
  \medskip
  \textbf{Targeted prior-work comparison}\par
  Kimi Team. \emph{Attention Residuals}. arXiv:2603.15031v1, 2026.\par
  \href{https://arxiv.org/abs/2603.15031v1}{arxiv.org/abs/2603.15031v1}\par
  \medskip
  \textbf{Implementation snapshot}\par
  \href{https://github.com/wdlctc/delta-attention-residuals-code/tree/fc11c3e570f44c4d566afac4d7a6eeb372e9aa5c}{wdlctc/delta-attention-residuals-code, commit fc11c3e}
  \papersource{All main results are taken from the paper. Formula checks and proposed controls are labeled as our analysis.}
\end{frame}

\appendix

\begin{frame}{Backup: RMSNorm gives a conditional invariance argument}
  If the sources exactly partition the residual, including the embedding,
  \[
    \sum_{i=1}^{N}\mathbf s_i=\tilde{\mathbf h}_l,
    \qquad \alpha_i=1/N,
  \]
  then
  \[
    \hat{\mathbf h}_l=(1+1/N)\tilde{\mathbf h}_l.
  \]
  \begin{itemize}
    \item For positive $c$, $\RMS(c\mathbf h)=\RMS(\mathbf h)$ when $\epsilon=0$.
    \item The sublayer may therefore be unchanged despite nonzero readout.
    \item This requires complete coverage, correct source bookkeeping, and the normalization assumption.
    \item It is distinct from claiming that \texttt{depth\_route} is itself an identity.
  \end{itemize}
  \papersource{Our conditional derivation from Eq. (2); relevant paper claims on PDF pp. 4 and 8.}
\end{frame}

\begin{frame}{Backup: 0.2 is not uniform over roughly 28 sources}
  \[
    \max_i\alpha_i^{\mathrm{uniform}}=\frac1N;
    \qquad N=28\ \Longrightarrow\ 1/N\simeq0.036.
  \]
  \begin{itemize}
    \item A maximum of 0.2 is diffuse relative to 0.6, but well above uniform.
    \item Candidate count grows with depth; maximum weight alone is incomplete.
    \item A normalized entropy $H(\alpha)/\log N$ would help compare depths.
    \item Larger peaks do not isolate whether source choice caused the PPL gain.
  \end{itemize}
  \papersource{Our interpretation of Fig. 4 and Sec. 4.1, PDF pp. 6 and 8--9. Normalized entropy is a suggested diagnostic.}
\end{frame}

\begin{frame}{Backup: grouped sources trade resolution for memory}
  \centering
  \begin{tikzpicture}[
      box/.style={draw=AcademicBlue,rounded corners=2pt,fill=AcademicBlue!5,
                  minimum height=9mm,align=center,inner sep=3mm},
      flow/.style={-{Latex[length=2mm]},thick,draw=AcademicBlue}]
    \node[box] (v1) at (0,2.5) {$\mathbf v_1$};
    \node[box] (v2) at (0,0.9) {$\mathbf v_2$};
    \node[box] (b1) at (2.9,1.7) {$\Delta_1=\mathbf v_1+\mathbf v_2$};
    \node[box] (b2) at (2.9,-0.1) {$\Delta_2$};
    \node[box] (read) at (6.4,0.9) {$\mathbf r=\sum_b\alpha_b\Delta_b$};
    \node[draw=AcademicTeal,circle,thick,minimum size=8mm] (plus) at (9.1,0.9) {$+$};
    \node (res) at (9.1,2.6) {$\tilde{\mathbf h}$};
    \node[box] (next) at (11.4,0.9) {Next\\sublayer};
    \draw[flow] (v1.east) -- (b1.west);
    \draw[flow] (v2.east) -- (b1.west);
    \draw[flow] (b1.east) -- (read.west);
    \draw[flow] (b2.east) -- (read.west);
    \draw[flow] (read.east) -- (plus.west);
    \draw[flow] (res.south) -- node[right,font=\small] {residual} (plus.north);
    \draw[flow] (plus.east) -- (next.west);
  \end{tikzpicture}
  \par\bigskip\raggedright
  \begin{itemize}
    \item The receiver can select a block sum, not its individual components.
    \item The direct residual input remains available alongside the readout.
  \end{itemize}
  \papersource{Teaching schematic based on Eq. (3) and Fig. 8, PDF pp. 13--14; not an original paper figure. It illustrates the intended block definition, not the inconsistent public bookkeeping.}
\end{frame}

\begin{frame}{Backup: small-model efficiency has its own tradeoffs}
  \centering
  \begin{tabular}{llrrr}
    \toprule
    Scale & Method & PPL & k token/s & Peak GB\\
    \midrule
    220M & Baseline & 38.71 & 289 & 5.6\\
    & Delta Block & 37.08 & 306 & 5.9\\
    & Delta AttnRes & 36.83 & 246 & 7.5\\
    \midrule
    533M & Baseline & 32.00 & 243 & 13.9\\
    & Delta Block & 31.16 & 198 & 16.1\\
    & Delta AttnRes & 31.05 & 80 & 33.6\\
    \bottomrule
  \end{tabular}
  \par\medskip\raggedright
  Delta Block is faster than baseline at 220M in this table, but slower at larger scales. Parameter overhead alone does not predict throughput.
  \papersource{Paper, Table 1, PDF p. 5. All listed memory figures are per GPU.}
\end{frame}

\begin{frame}{Backup: do not merge Table 1 and Table 4 measurements}
  \centering
  \begin{tabular}{llrrr}
    \toprule
    Scale & Delta Block row & PPL & k token/s & Peak GB\\
    \midrule
    220M & Table 1 & 37.08 & 306 & 5.9\\
    & Table 4, $B=4$ & 36.98 & 343 & 9.1\\
    \midrule
    533M & Table 1 & 31.16 & 198 & 16.1\\
    & Table 4, $B=4$ & 31.27 & 160 & 15.4\\
    \bottomrule
  \end{tabular}
  \par\medskip\raggedright
  \begin{itemize}
    \item The paper says $B=4$ is the main-table default.
    \item The repository maps main and ablation entries to separate runs.
    \item Report each table on its own terms; measurement differences remain underexplained.
  \end{itemize}
  \papersource{Paper, Tables 1 and 4, PDF pp. 5 and 7; public WANDB\_RUNS.md at fc11c3e.}
\end{frame}

\end{document}