% Compile with pdflatex. Replace the demonstration content before delivery.
\documentclass[aspectratio=169,11pt]{beamer}
\usepackage[T1]{fontenc}
\usepackage[utf8]{inputenc}
\usepackage{amsmath,amssymb,graphicx,booktabs}

\definecolor{AcademicBlue}{RGB}{26,57,86}
\definecolor{AcademicTeal}{RGB}{0,105,112}
\setbeamercolor{structure}{fg=AcademicBlue}
\setbeamercolor{frametitle}{fg=AcademicBlue,bg=white}
\setbeamercolor{title}{fg=AcademicBlue}
\setbeamercolor{alerted text}{fg=AcademicTeal}
\setbeamercolor{block title}{fg=white,bg=AcademicBlue}
\setbeamercolor{block body}{fg=black,bg=AcademicBlue!5}
\setbeamertemplate{navigation symbols}{}
\setbeamertemplate{itemize items}[circle]
\setbeamertemplate{footline}{%
  \hfill\usebeamerfont{page number in head/foot}%
  \insertframenumber\,/\,\inserttotalframenumber\hspace{4mm}\vspace{3mm}}
\setbeamersize{text margin left=8mm,text margin right=8mm}
\setbeamerfont{frametitle}{size=\Large,series=\bfseries}
\setbeamerfont{title}{size=\LARGE,series=\bfseries}
\newcommand{\papersource}[1]{\par\smallskip{\scriptsize\color{gray}#1\par}}

\begin{document}
\begin{frame}{Retrieve useful depth history inside attention}

\begin{block}{Mixture-of-Depths Attention (MoDA)}
Sequence KV and same-position historical KV compete in one softmax.
\end{block}
\medskip
\textbf{Lianghui Zhu, Yuxin Fang, Bencheng Liao et al.}\\
Huazhong University of Science and Technology / ByteDance Seed
\medskip
\begin{itemize}
\item Paper: \href{https://arxiv.org/abs/2603.15619v1}{arXiv:2603.15619v1}, 16 March 2026.
\item Independent paper-reading presentation; 17 September 2026.
\item 20 main slides + 4 backup slides; approximately 25 minutes.
\end{itemize}
\papersource{Source: author TeX and version-matched PDF, pp. 1--18. No independent model reproduction.}
\end{frame}
\begin{frame}{The question: can deep layers recover shallow features?}

\begin{itemize}
\item Residual addition makes deep optimization easier, but repeatedly mixes history into a fixed-width state.
\item Dense history preserves intermediate states, but can require expensive cross-layer projections.
\item MoDA retains the residual stream and adds content-dependent retrieval of historical KV.
\end{itemize}
\begin{block}{Evidence boundary}
``Information dilution'' motivates the architecture. The paper does not isolate it with a causal information-loss measurement.
\end{block}
\papersource{Paper, Sec. 1, PDF pp. 2--3; Sec. 2.2, pp. 3--5.}
\end{frame}
\begin{frame}{MoDA keeps a residual stream and a depth-KV stream}

\centering
\includegraphics[width=0.96\linewidth,height=0.57\textheight,keepaspectratio]{figures/fig-03.pdf}
\par\medskip
\raggedright Attention supplies a dynamic read operation; MoDA fuses that read with sequence attention.
\papersource{Paper, Fig. 3, PDF p. 4. Original figure.}
\end{frame}
\begin{frame}{Depth retrieval is restricted to the same token position}

\begin{columns}[c]
\column{0.64\textwidth}
\includegraphics[width=\linewidth,height=0.68\textheight,keepaspectratio]{figures/fig-01.pdf}
\column{0.33\textwidth}
\begin{itemize}
\item Horizontal: causal sequence KV.
\item Vertical: same-position depth KV.
\item All Transformer layers still execute.
\end{itemize}
\end{columns}
\papersource{Paper, Fig. 1, PDF p. 1. Original figure; colored regions show two example queries.}
\end{frame}
\begin{frame}{One denominator allocates probability across both memories}

For one query head at layer $l$, position $t$:
\[
a_j=q_{l,t}^{\top}k_{l,j}/\sqrt d,\qquad
b_i=q_{l,t}^{\top}k^{\mathrm{depth}}_{i,t}/\sqrt d
\]
\[
o_{l,t}=\frac{\sum_{j\le t}e^{a_j}v_{l,j}+\sum_{i\in\mathcal H_l}e^{b_i}v^{\mathrm{depth}}_{i,t}}
{\sum_{j\le t}e^{a_j}+\sum_{i\in\mathcal H_l}e^{b_i}}
\]
\begin{itemize}
\item $d$: head dimension; $\mathcal H_l$: previously written depth slots.
\item Values and per-head output have dimension $d$.
\item Separate softmax operations followed by addition give a different operator.
\end{itemize}
\papersource{Equivalent teaching expansion of Sec. 2.2, PDF p. 5 and Algorithm 1, p. 6; not a numbered paper equation.}
\end{frame}
\begin{frame}{Reuse attention KV; add a projection for FFN inputs}

\begin{enumerate}
\item Read the current state and previously stored depth KV.
\item Compute joint sequence/depth attention using the existing query projection.
\item Preserve attention KV for later layers.
\item Project the \textbf{FFN input} into additional depth KV slots.
\end{enumerate}
\medskip
\begin{block}{Default variant}
Reuse attention-side KV + extra FFN KV projection.\newline
An independent attention-side KV projection gives little additional gain.
\end{block}
\papersource{Paper, Sec. 2.2, p. 5; Sec. 4.2.1 and Table 3, PDF pp. 10--11.}
\end{frame}
\begin{frame}{Depth compute still accumulates quadratically in depth}

\centering
\begin{tabular}{lcc}
\toprule
Depth-stream cost & Depth Dense & MoDA\\
\midrule
Parameters & $O(L^2D^2)$ & $O(LD^2/G)$\\
Decode cache & $O(LD)$ & $O(LD/G)$\\
Prefill cache & $O(TLD)$ & $O(TLD/G)$\\
Decode FLOPs & $O(L^2D^2)$ & $O(L^2D)$\\
Prefill FLOPs & $O(TL^2D^2)$ & $O(TL^2D)$\\
\bottomrule
\end{tabular}
\par\medskip\raggedright
$T$: tokens; $L$: depth; $D$: width; $G=H_q/H_k$: GQA group size.\par
These are depth-stream terms, excluding ordinary sequence KV cache and the full attention/FFN cost.
\papersource{Paper, Table 1, PDF p. 5. Attention-only KV reuse adds no projection parameters (Table 3).}
\end{frame}
\begin{frame}{Local packing avoids scanning irrelevant depth entries}

\centering
\includegraphics[width=\linewidth,height=0.43\textheight,keepaspectratio]{figures/fig-04.pdf}
\par\medskip\raggedright
\begin{itemize}
\item Contiguous layout: depth axis of length $TL$; dense utilization $1/T$.
\item Chunk-aware access: local span $CL$; utilization $1/C$.
\item GQA reuse: span $(C/G)L$; utilization $G/C$.
\end{itemize}
\papersource{Paper, Fig. 4 and Sec. 3.2, PDF pp. 7--8. Original figure. $C$: expanded-query block size.}
\end{frame}
\begin{frame}{Shared online-softmax states preserve the joint operator}

Maintain a running maximum $m$, denominator $acc$, and output numerator $o$:
\[
\begin{aligned}
m'&=\max(m,\max S),\\
acc'&=e^{m-m'}acc+\sum e^{S-m'},\\
o'&=e^{m-m'}o+\sum e^{S-m'}V.
\end{aligned}
\]
\begin{itemize}
\item Process sequence blocks, then depth blocks, with the same states.
\item Normalize once: $o/acc$. Do not materialize the full score matrix.
\item Grouped masks preserve causality and same-position retrieval.
\end{itemize}
\papersource{Natural-exponential teaching form of Algorithm 1, PDF p. 6; implementation details, pp. 8--9. See backup caveat.}
\end{frame}
\begin{frame}{Three evaluation settings answer different questions}

\begin{block}{Language-model quality}
700M / 1.5B; 400B OLMo2 tokens; bf16; context 4096; global batch 1024. Ten downstream tasks and ten validation domains.
\end{block}
\begin{block}{Depth and normalization}
24 / 48 layers on FineWeb-Edu; width 384; 6 query heads and 2 KV heads. Held-out validation loss.
\end{block}
\begin{block}{Kernel efficiency}
A100, bf16, forward + backward. Shape sweeps and incremental kernel ablations; no full-model serving benchmark.
\end{block}
\papersource{Paper, Sec. 4.1, p. 10; Sec. 4.3.1, p. 12; Tables 2/7, pp. 9/13.}
\end{frame}
\begin{frame}{At fixed parameter count, depth KV improves quality}

\centering
\begin{tabular}{lrrrr}
\toprule
700M-scale variant & Params M & FLOPs T & C4 PPL & Avg. score\\
\midrule
OLMo2, 36 layers & 669.0 & 8.01 & 18.59 & 56.93\\
+ reused depth KV & 669.0 & 8.02 & 18.48 & 58.10\\
+ FFN KV (default) & 705.7 & 8.33 & 18.21 & 58.87\\
+ separate attn KV & 742.4 & 8.63 & 18.17 & 58.97\\
\bottomrule
\end{tabular}
\par\medskip\raggedright
\begin{itemize}
\item Reused KV: +1.17 percentage points at unchanged parameters.
\item FFN KV: another +0.77 points.
\item Separate attention KV: only +0.10 points for +36.7M parameters.
\end{itemize}
\papersource{Paper, Table 3, PDF p. 10. PPL: lower is better; score: higher is better. FLOPs units as reported.}
\end{frame}
\begin{frame}{Adding depth memory beats adding two baseline layers}

\centering
\begin{tabular}{lrrrr}
\toprule
Model & Layers & Params M & FLOPs T & Avg. score\\
\midrule
OLMo2 & 38 & 700.5 & 8.41 & 57.11\\
MoDA default & 36 & 705.7 & 8.33 & 58.87\\
\bottomrule
\end{tabular}
\par\bigskip\raggedright
\begin{itemize}
\item +1.76 percentage points under approximately matched resources.
\item C4 PPL: 18.31 $\rightarrow$ 18.21 in this ablation table.
\item Resources are close, not exactly equal; no multi-seed uncertainty is reported.
\end{itemize}
\papersource{Paper, Table 3, PDF p. 10. Table 5 reports different 700M C4 values; see backup.}
\end{frame}
\begin{frame}{Average downstream gains persist at both model scales}

\centering
\begin{tabular}{lrrr}
\toprule
Scale & OLMo2 & MoDA & Difference\\
\midrule
700M & 57.11 & 58.87 & +1.76 pp\\
1.5B & 62.28 & 64.39 & +2.11 pp\\
\bottomrule
\end{tabular}
\par\medskip\raggedright
\begin{itemize}
\item 1.5B ARC-C: 42.47 $\rightarrow$ 46.82 (+4.35 pp).
\item 1.5B COPA: 81.00 $\rightarrow$ 85.00 (+4.00 pp).
\item Exceptions: 700M PIQA falls 0.33 pp; 1.5B ARC-E falls 0.17 pp.
\item +2.11 pp is about +3.39\% relative to 62.28.
\end{itemize}
\papersource{Paper, Table 4, PDF p. 11. Averages over ten tasks; pp = percentage points.}
\end{frame}
\begin{frame}{Validation perplexity improves in all ten reported domains}

\centering
\begin{tabular}{lrrr}
\toprule
Scale & OLMo2 mean PPL & MoDA mean PPL & Change\\
\midrule
700M & 15.61 & 15.46 & $-0.15$\\
1.5B & 13.67 & 13.47 & $-0.20$\\
\bottomrule
\end{tabular}
\par\medskip\raggedright
\begin{itemize}
\item 700M m2d2-s2orc: 24.37 $\rightarrow$ 23.64.
\item 1.5B Reddit: 21.21 $\rightarrow$ 20.85.
\item Mean of domain PPLs is not pooled-corpus PPL.
\item These evaluations do not establish long-context retrieval quality.
\end{itemize}
\papersource{Paper, Table 5, PDF p. 11. Domains: C4, ICE, m2d2-s2orc, Pile, Wiki-text, and five Dolma domains.}
\end{frame}
\begin{frame}{Post-norm benefits more in the 48-layer small-model study}

\centering
\begin{tabular}{lrrr}
\toprule
Depth / norm & Baseline & + Depth KV & + FFN KV\\
\midrule
48 / pre-norm & 3.3800 & 3.3759 & 3.3656\\
48 / post-norm & 3.4062 & 3.3653 & 3.3484\\
24 / post-norm & 3.4740 & 3.4537 & 3.4338\\
\bottomrule
\end{tabular}
\par\medskip\raggedright
\begin{itemize}
\item Depth KV gain at 48 layers: 0.0041 vs. 0.0409 loss reduction.
\item FFN KV improves every reported setting further.
\item This is FineWeb-Edu validation loss, not PPL; widths and training regime differ from the main study.
\end{itemize}
\papersource{Paper, Table 6 and Sec. 4.3.1, PDF pp. 12--13. Parameters/FLOPs are not held fixed across all rows.}
\end{frame}
\begin{frame}{The visualized heads use depth memory}

\centering
\includegraphics[width=\linewidth,height=0.42\textheight,keepaspectratio]{figures/fig-05-top-row.png}
\par\medskip\raggedright
\begin{itemize}
\item Query rows attend to sequence KV, then depth KV after the red boundary.
\item Columns: layers 0, 11, 23, 35; one selected head per column shown.
\item Depth attention is visible, but heatmaps alone do not establish that reducing attention sinks causes the quality gains.
\end{itemize}
\papersource{Paper, Fig. 5, PDF p. 14; first-row crop of the original figure. 700M model, 400B training tokens.}
\end{frame}
\begin{frame}{The 97.3\% headline is a 64K kernel runtime ratio}

\centering
\begin{tabular}{rrrr}
\toprule
Tokens & FA2 ms & MoDA ms & Extra time vs. FA2\\
\midrule
4,096 & 7.970 & 10.750 & 34.88\%\\
16,384 & 116.700 & 127.661 & 9.39\%\\
65,536 & 1831.668 & 1883.026 & 2.80\%\\
\bottomrule
\end{tabular}
\par\medskip\raggedright
$1831.668/1883.026=97.27\%$: inverse runtime ratio.\par\medskip
The paper's ``extra time'' uses MoDA time as denominator (2.73\% at 64K); the table above uses FA2 time.\par
This is an attention forward/backward benchmark, not full-model throughput.
\papersource{Recomputed from Table 2, PDF p. 9. A100/bf16; B=1, d=C=L=64, G=8, Hq=64, Hk=8.}
\end{frame}
\begin{frame}{All three layout optimizations contribute to kernel speed}

\centering
\begin{tabular}{lr}
\toprule
Implementation & Forward + backward ms\\
\midrule
Naive PyTorch & 2128.900\\
Flash-compatible & 13.102\\
+ Chunk-aware & 6.286\\
+ Group-aware & 1.460\\
\bottomrule
\end{tabular}
\par\medskip\raggedright
\begin{itemize}
\item Group-aware indexing adds a $4.31\times$ speedup over the prior row.
\item The $1458\times$ total gain is against naive PyTorch, not FA2.
\item Fixed short sequence: $T=1024$; no model-quality comparison here.
\end{itemize}
\papersource{Paper, Table 7, PDF p. 13. A100/bf16; B=1, G=8, Hq=64, Hk=8, d=L=C=64.}
\end{frame}
\begin{frame}{The main open cost is retaining and reading depth history}

\begin{itemize}
\item More depth slots increase cache capacity, bandwidth, and attention work.
\item Large-model evidence stops at the 1.5B class; no serving latency or peak-memory benchmark is reported.
\item No multi-seed confidence intervals or causal test of the proposed information-dilution mechanism.
\end{itemize}
\begin{block}{Future work in the paper}
A bounded $S\ll L$ slot buffer (top-$S$, sliding window, or hybrid) and further CUDA/distributed optimizations are proposals, not evaluated results.
\end{block}
\papersource{Paper, Sec. 6.1--6.2, PDF p. 15; evidence boundaries from reported experiments.}
\end{frame}
\begin{frame}{Take away the retrieval idea and keep the conditions}

\begin{enumerate}
\item \textbf{Architecture:} retrieve same-position depth KV jointly with causal sequence KV.
\item \textbf{Design choice:} attention KV reuse is cheap; FFN KV supplies additional value.
\item \textbf{Evidence:} +2.11 pp average downstream score at the 1.5B scale; 0.20 lower mean domain PPL.
\item \textbf{Systems:} long-context kernel overhead is small under the measured configuration; real training and serving need separate validation.
\end{enumerate}
\medskip
\href{https://arxiv.org/abs/2603.15619v1}{Zhu et al., Mixture-of-Depths Attention, arXiv:2603.15619v1.}
\papersource{Primary source: version-matched author TeX and PDF. All results are paper-reported unless marked recomputed.}
\end{frame}
\begin{frame}{Backup: grouped indexing preserves the correct token map}

\[
T_q=GT_{kv},\qquad t_{base}(i_q)=\lfloor i_q/G\rfloor
\]
\[
\text{Sequence visible: }\lfloor i_q/G\rfloor\ge i_k
\]
\[
\text{Depth visible: }\lfloor i_q/G\rfloor=\lfloor j_d/L\rfloor
\]
\begin{itemize}
\item $i_q$: expanded query row; $i_k$: sequence-key position.
\item $j_d$: flattened depth-key index; $L$: number of depth slots.
\item For $G=4$, rows 8--15 map to positions 2 and 3 only.
\item Query block size must align with $G$; text length is unchanged.
\end{itemize}
\papersource{Paper, Algorithm 1, p. 6 and Sec. 3.3, PDF p. 8.}
\end{frame}
\begin{frame}{Backup: depth and GQA shape sweeps have qualifications}

\begin{block}{Depth sweep: T=16,384; G=8}
$L=64,128,256$: MoDA times 127.661, 138.224, 167.958 ms.\newline
FA2 remains 116.700 ms. More depth is not free.
\end{block}
\begin{block}{Group sweep: T=16,384; L=64; Hk=8}
$G=2\rightarrow32$: utilization $3.12\%\rightarrow50\%$.\newline
But query heads also increase $16\rightarrow256$, changing workload size.
\end{block}
\medskip
All sweeps: A100, bf16, forward + backward; $B=1$, $d=C=64$.
\papersource{Paper, Table 2, PDF p. 9. Utilization here concerns valid depth entries within block computations.}
\end{frame}
\begin{frame}{Backup: retain the reporting discrepancies}

\begin{itemize}
\item \textbf{700M C4 PPL:} Table 3 gives 18.31 / 18.21 for the matched baseline/default; Table 5 gives 18.32 / 18.29. The paper does not explain the difference.
\item \textbf{3.7\% FLOPs:} reported in the abstract alongside 1.5B gains; no corresponding detailed 1.5B cost table. Table 3's 8.01 $\rightarrow$ 8.33 is about +4.00\%.
\item \textbf{Evaluation details:} per-task shots and metric variants, complete 1.5B architecture/cost details, and multi-seed variability are not fully specified.
\end{itemize}
\papersource{Paper, abstract p. 1; Tables 3--5 and Sec. 4.1, PDF pp. 10--11. No silent reconciliation in this reading.}
\end{frame}
\begin{frame}{Backup: pseudocode needs implementation-level checks}

\begin{itemize}
\item Algorithm 1 writes $2^{S-m}$ with $S=QK^\top/\sqrt d$.
\item To reproduce the natural-exponential softmax of Eq. (2), base-2 exponentiation requires logits scaled by $\log_2 e$.
\item Block indices are compared with token ranges in the pseudocode; a kernel needs explicit block offsets and boundary handling.
\item These are notation/implementation checks, not evidence that the released code is incorrect. The code was not executed in this reading.
\end{itemize}
\papersource{Paper, Eq. (2), PDF p. 3; Algorithm 1, p. 6; Sec. 3.3, pp. 8--9. Reader analysis.}
\end{frame}
\end{document}