% Compile with pdflatex. Replace the demonstration content before delivery.
\documentclass[aspectratio=169,11pt]{beamer}
\usepackage[T1]{fontenc}
\usepackage{lmodern}
\usepackage[utf8]{inputenc}
\usepackage{amsmath,amssymb,graphicx,booktabs}

\definecolor{AcademicBlue}{RGB}{26,57,86}
\definecolor{AcademicTeal}{RGB}{0,105,112}
\setbeamercolor{structure}{fg=AcademicBlue}
\setbeamercolor{frametitle}{fg=AcademicBlue,bg=white}
\setbeamercolor{title}{fg=AcademicBlue}
\setbeamercolor{alerted text}{fg=AcademicTeal}
\setbeamercolor{block title}{fg=white,bg=AcademicBlue}
\setbeamercolor{block body}{fg=black,bg=AcademicBlue!5}
\setbeamertemplate{navigation symbols}{}
\setbeamertemplate{itemize items}[circle]
\setbeamertemplate{footline}{%
  \hfill\usebeamerfont{page number in head/foot}%
  \insertframenumber\,/\,\inserttotalframenumber\hspace{4mm}\vspace{3mm}}
\setbeamersize{text margin left=8mm,text margin right=8mm}
\setbeamerfont{frametitle}{size=\Large,series=\bfseries}
\setbeamerfont{title}{size=\LARGE,series=\bfseries}
\newcommand{\papersource}[1]{\par\smallskip{\scriptsize\color{gray}#1\par}}

\title[YOCO]{You Only Cache Once}
\subtitle{Decoder-Decoder Architectures for Language Models}
\author{Paper: Yutao Sun, Li Dong, Yi Zhu, Shaohan Huang,\\Wenhui Wang, Shuming Ma, Quanlu Zhang,\\Jianyong Wang, and Furu Wei}
\date{arXiv:2405.05254v2 (9 May 2024)\\Reading presentation: 20 September 2026}
\begin{document}
\begin{frame}\titlepage\end{frame}
\begin{frame}{One global memory, many layers of queries}
\begin{itemize}
\item A causal self-decoder builds a shared, token-addressable KV memory.
\item A causal cross-decoder repeatedly queries that same memory.
\item History needs no separate full-length KV cache at every layer.
\item Main payoff: lower memory and much faster long-prompt prefill.
\end{itemize}
\papersource{Paper, Sections 1--2, PDF pp. 2--5.}
\end{frame}
\begin{frame}{The bottleneck grows with both depth and length}
\[
\text{Transformer KV memory}\ \propto\ L\times N\times d_{\mathrm{KV}}
\]\begin{itemize}
\item $L$: layers; $N$: cached tokens; $d_{\mathrm{KV}}$: KV width.
\item Grouped-query attention reduces KV width, but retains the depth factor.
\item Pure recurrent compression is cheap, yet exact historical recall is difficult.
\item YOCO keeps global attention while sharing its memory across depth.
\end{itemize}
\papersource{Paper, Sections 1--2, PDF pp. 2--4; interpretation of the design tradeoff.}
\end{frame}
\begin{frame}{The architecture separates memory building and reading}
\begin{columns}
\begin{column}{0.43\textwidth}\centering\includegraphics[width=1\linewidth,height=0.68\textheight,keepaspectratio]{figures/arch.png}\par
\end{column}
\begin{column}{0.55\textwidth}\begin{itemize}
\item First $L/2$ layers: efficient causal self-attention.
\item Last $L/2$ layers: global causal cross-attention.
\item Different queries at each layer; shared keys and values.
\item RMSNorm, SwiGLU, residuals, and GQA remain.
\end{itemize}\end{column}\end{columns}
\papersource{Paper, Figure 2 and Section 2, PDF p. 3; original figure.}
\end{frame}
\begin{frame}{Shared KV does not mean identical attention}
\[
M=X^{L/2},\qquad \hat K=\operatorname{LN}(M)W_K,\quad
\hat V=\operatorname{LN}(M)W_V
\]
\[
\hat Q^l=\operatorname{LN}(X^l)W_Q^l,\qquad
Y^l=\operatorname{Attention}(\hat Q^l,\hat K,\hat V)+X^l
\]
\[
X^{l+1}=\operatorname{SwiGLU}(\operatorname{LN}(Y^l))+Y^l
\]\begin{itemize}
\item $M\in\mathbb R^{N\times d}$; actual KV width depends on GQA.
\item Queries and hidden states evolve; memory remains shared.
\item Causal masking prevents future-token leakage during training.
\end{itemize}
\papersource{Paper, Equations (2)--(3), PDF p. 4. LN denotes RMSNorm.}
\end{frame}
\begin{frame}{Gated retention compresses the self-decoder history}
\[
\gamma_n=\operatorname{sigmoid}(X_nW_\gamma)^{1/\tau},\quad
S_n=\gamma_n S_{n-1}+K_n^\top V_n,\quad O_n=Q_nS_n
\]\begin{itemize}
\item Per-head state $S_n\in\mathbb R^{d_k\times d_v}$ has fixed size in sequence length.
\item Input-dependent, head-wise decay controls forgetting.
\item Position-rotated queries/keys supply relative positional information.
\item The full module also uses head-wise GroupNorm and a swish output gate.
\end{itemize}
\papersource{Paper, Section 3.1, Equations (4), (5), (7), PDF pp. 5--6.}
\end{frame}
\begin{frame}{Three equivalent views serve different workloads}
\[
O=(QK^\top\odot D)V,\qquad
D_{nm}=\mathbf1_{m\le n}\prod_{i=m+1}^{n}\gamma_i
\]\begin{itemize}
\item \textbf{Parallel:} expose all pair interactions; convenient for training.
\item \textbf{Recurrent:} update one state per token; efficient generation.
\item \textbf{Chunkwise:} parallel within a block, recurrent between blocks.
\item A sliding-window self-decoder is an alternative; $C=1{,}024$ in scaling experiments.
\end{itemize}
\papersource{Paper, Sections 3.1--3.2 and 4.2, PDF pp. 5--8. Chunk-index caveat in backup.}
\end{frame}
\begin{frame}{Prefill skips historical cross-decoder outputs}
\centering\includegraphics[width=0.9\linewidth,height=0.43\textheight,keepaspectratio]{figures/inference.pdf}\par
\begin{itemize}
\item Build all prompt KV with the self-decoder; old cross outputs are not future memory.
\item The last prompt position still needs cross-decoder computation for next-token logits.
\item Full training loss and generated tokens still use the cross-decoder.
\end{itemize}
\papersource{Paper, Figure 3 and Section 2.3, PDF pp. 4--5. Last-position detail follows the dependency graph.}
\end{frame}
\begin{frame}{The complexity gain has a precise scope}
\centering
\begin{tabular}{lcc}\toprule
 & Transformer & YOCO\\\midrule
Cache memory & $O(LND)$ & $O((N+L)D)$\\
Prefill attention & $O(LN^2D)$ & $O(LND)$\\\bottomrule
\end{tabular}\par\medskip
\begin{itemize}
\item Paper notation suppresses fixed window/head-state constants.
\item ``Once'' means one global cache; per-layer local states still exist.
\item Weights, FFNs, activations, and communication are separate costs.
\item Generation still reads global KV; full training attention is not made linear.
\end{itemize}
\papersource{Paper, Tables 1--2, PDF p. 4; scope clarified from the architecture.}
\end{frame}
\begin{frame}{Four experiments answer four different questions}
\begin{enumerate}
\item \textbf{Quality at scale:} 3B model trained up to 1.6T tokens.
\item \textbf{Parameter scaling:} 160M--13B, each trained on 10B tokens.
\item \textbf{Long context:} extend 3B progressively to 1M tokens.
\item \textbf{Deployment:} memory, prefill, and total inference throughput.
\end{enumerate}
\medskip
Public-model comparisons and controlled scaling runs offer different evidence.
\papersource{Paper, Section 4, PDF pp. 6--11.}
\end{frame}
\begin{frame}{The 3B model reaches competitive downstream quality}
\centering
\begin{tabular}{lcc}\toprule
Model & Training tokens & 8-task average $\uparrow$\\\midrule
OpenLLaMA-3B-v2 & 1T & 0.619\\
StableLM-base-alpha-3B-v2 & 1T & 0.612\\
YOCO-3B & 1T & \textbf{0.634}\\
YOCO-3B & 1.6T & 0.636\\
YOCO-3B-1M & extended & 0.645\\\bottomrule
\end{tabular}\par\medskip
\begin{itemize}
\item At 1T: +1.5 percentage points over OpenLLaMA on the mean.
\item Not a per-task sweep; training data and recipes are not fully controlled.
\item 5T is the learning-rate schedule, not the actual completed training.
\end{itemize}
\papersource{Paper, Table 3, PDF p. 7; Appendix C, p. 17. Zero-shot LM Eval Harness.}
\end{frame}
\begin{frame}{Fixed-budget scaling favors the hybrid design}
\centering\includegraphics[width=0.9\linewidth,height=0.52\textheight,keepaspectratio]{figures/scaling_curve.pdf}\par
\begin{itemize}
\item Same data/settings; 10B tokens for every size; validation loss is lower-is-better.
\item FFN widths align parameter counts. This is not compute-optimal scaling.
\end{itemize}
\papersource{Paper, Figure 4, PDF p. 7; Section 4.2, p. 8; Appendix D, pp. 17--18.}
\end{frame}
\begin{frame}{One million tokens requires continued training}
\centering
\begin{tabular}{lccc}\toprule
Context length & Tokens & Learning rate & RoPE $\theta$\\\midrule
65,536 & 6B & $8\times10^{-5}$ & 640K\\
262,144 & 4B & $4\times10^{-5}$ & 5M\\
1,048,576 & 1.5B & $2\times10^{-5}$ & 80M\\\bottomrule
\end{tabular}\par\medskip
\begin{itemize}
\item An additional 11.5B tokens, with long-document upsampling.
\item No long-instruction tuning data in this extension.
\item Not zero-training extrapolation from the original 4K model.
\end{itemize}
\papersource{Paper, Section 4.3, PDF p. 8; Appendix E, Table 7, p. 18.}
\end{frame}
\begin{frame}{Single-needle recall stays strong up to 1M}
\centering\includegraphics[width=0.9\linewidth,height=0.56\textheight,keepaspectratio]{figures/1m_retrieval.pdf}\par
\begin{itemize}
\item Axes: context length and insertion depth; color: accuracy, higher is better.
\item 10 trials per depth/length; city + magic-number retrieval. Not general reasoning.
\end{itemize}
\papersource{Paper, Figure 5 and Section 4.3, PDF p. 8; original figure.}
\end{frame}
\begin{frame}{Multi-needle retrieval reveals a harder boundary}
\centering
\begin{tabular}{lrrrr}\toprule
Model & 1 needle & 2 & 4 & 8\\\midrule
LWM-1M-text (7B) & 1.00 & 0.90 & 0.76 & \textbf{0.62}\\
MiniCPM-128K (2.4B) & 1.00 & 1.00 & 0.54 & 0.56\\
ChatGLM3-128K (6B) & 0.94 & 0.72 & 0.52 & 0.44\\
YOCO-3B-1M & 0.98 & 0.98 & \textbf{0.84} & 0.56\\\bottomrule
\end{tabular}\par\medskip
\begin{itemize}
\item All these retrieval tests use \textbf{128K context}, not 1M.
\item YOCO leads at four needles, but trails LWM at eight.
\item Model sizes and context-training histories differ.
\end{itemize}
\papersource{Paper, selected rows from Table 4, PDF p. 8. Accuracy: higher is better.}
\end{frame}
\begin{frame}{Long-document prediction improves with length}
\begin{columns}
\begin{column}{0.49\textwidth}\centering\includegraphics[width=1\linewidth,height=0.45\textheight,keepaspectratio]{figures/book-1m-ppl.png}\par
\end{column}
\begin{column}{0.49\textwidth}\centering\includegraphics[width=1\linewidth,height=0.45\textheight,keepaspectratio]{figures/code-1m-ppl.png}\par
\end{column}\end{columns}\begin{itemize}
\item Books (left), repository-level code (right); examples longer than 1M.
\item Metric is cumulative average NLL, lower is better.
\item Position/content changes also affect this curve; not a fixed-target context ablation.
\end{itemize}
\papersource{Paper, Figure 6, PDF p. 9; original panels. Last point is reading analysis.}
\end{frame}
\begin{frame}{Memory savings grow with context length}
\centering\includegraphics[width=0.9\linewidth,height=0.51\textheight,keepaspectratio]{figures/memory-length.pdf}\par
\begin{itemize}
\item Default 3B configuration: 12.4 GB at 1M; Transformer uses about 9.4 times more.
\item Total inference memory includes weights, activations, and KV.
\end{itemize}
\papersource{Paper, Figure 7(a) and Section 4.4, PDF p. 10; original figure.}
\end{frame}
\begin{frame}{Prefill benefits directly from the dependency change}
\centering\includegraphics[width=0.9\linewidth,height=0.53\textheight,keepaspectratio]{figures/prefilling-length.pdf}\par
\begin{itemize}
\item 512K: about 180 seconds to less than 6 seconds; 32K: 2.87 times faster.
\item 1M: figure reports 71.82 times speedup; hardware/kernel conditions matter.
\end{itemize}
\papersource{Paper, Figure 9, PDF p. 11; Section 4.4, pp. 9--10. H100-80GB profiling.}
\end{frame}
\begin{frame}{Throughput includes more than token decoding}
\centering\includegraphics[width=0.9\linewidth,height=0.49\textheight,keepaspectratio]{figures/throughput-length.pdf}\par
\begin{itemize}
\item 512K: 4.5 to 43.1 tokens/s, about 9.6 times higher.
\item Includes prefill and generation; final 1,024 tokens are generated.
\item Lower memory permits larger batches. Not fixed-batch decode speedup.
\end{itemize}
\papersource{Paper, Figure 10 and Section 4.4, PDF pp. 9--11.}
\end{frame}
\begin{frame}{Small-model diagnostics support global memory}
\centering
\begin{tabular}{lrrr}\toprule
160M model & Validation $\downarrow$ & AR-Hit $\downarrow$ & First-Occur $\downarrow$\\\midrule
Mamba & 3.645 & 1.555 & 4.126\\
gRetNet & 3.600 & 1.354 & 4.116\\
Transformer & 3.564 & 1.219 & 4.104\\
YOCO-gRet & \textbf{3.530} & \textbf{1.199} & \textbf{4.067}\\\bottomrule
\end{tabular}\par\medskip
\begin{itemize}
\item AR-Hit: target bigram already appeared in the context.
\item First-Occur: target cannot be directly recalled from the context.
\item Values and the paper\textquotesingle s ``perplexity'' label are retained as reported.
\end{itemize}
\papersource{Paper, selected rows from Table 8, Appendix G.1, PDF p. 19.}
\end{frame}
\begin{frame}{What is established, and what remains open}
\begin{itemize}
\item \textbf{Established here:} competitive quality and large long-prompt deployment gains.
\item \textbf{Architectural limit:} global KV still grows with length; deep layers cannot rewrite it.
\item \textbf{Evidence limit:} synthetic retrieval is not general long-context reasoning.
\item \textbf{Reproduction limit:} per-length batch/device details and uncertainty are incomplete.
\item BitNet/Groq integration, multimodal fusion, and indexed memory remain proposals.
\end{itemize}
\papersource{Paper, Sections 4--5, PDF pp. 7--12; limitations include reading analysis.}
\end{frame}
\begin{frame}{Reference and discussion}
\textbf{You Only Cache Once: Decoder-Decoder Architectures for Language Models}\par\medskip
Yutao Sun, Li Dong, Yi Zhu, Shaohan Huang, Wenhui Wang, Shuming Ma, Quanlu Zhang, Jianyong Wang, Furu Wei.\par\medskip
\url{https://arxiv.org/abs/2405.05254v2}\par
Author code link: \url{https://aka.ms/YOCO}\par\medskip
\textbf{Discussion:} Which tasks need layer-specific historical KV, and which only need deeper queries over a stable memory?
\papersource{Based on the versioned author TeX and 20-page PDF. Experiments were not reproduced.}
\end{frame}
\appendix
\begin{frame}{Backup: chunkwise recurrence with consistent indices}
Let $a=(i-1)B$ and $\beta_j=\prod_{t=1}^{j}\gamma_{a+t}$.
With query row $j$ and key column $k$:
\[
D^{(i)}_{jk}=\mathbf1_{k\le j}\frac{\beta_j}{\beta_k}
\]
\[
O_{[i]}=(Q_{[i]}K_{[i]}^\top\odot D^{(i)})V_{[i]}
+\operatorname{diag}(\beta)Q_{[i]}R_{i-1}
\]
\[
R_i=\beta_B R_{i-1}+\sum_{k=1}^{B}\frac{\beta_B}{\beta_k}K_{a+k}^\top V_{a+k}
\]
Re-derived from Eqs. (4)--(5). Printed Eq. (6) reverses the mask/ratio under this row convention; Appendix F also has a \texttt{gt}/\texttt{g} typo.
\papersource{Paper, Equations (4)--(6), pp. 5--6; Appendices B/F, pp. 16--19. Re-derived teaching notation.}
\end{frame}
\begin{frame}{Backup: communicate shared KV once}
\centering\includegraphics[width=0.9\linewidth,height=0.52\textheight,keepaspectratio]{figures/chunk_parallelism.pdf}\par
\begin{itemize}
\item Self-decoder exchanges boundary states or local-window information.
\item Cross-decoder gathers shared KV once, rather than at every layer.
\item Dependency advantage is described; no separate communication-speedup ablation.
\end{itemize}
\papersource{Paper, Appendix A, Figure 11, PDF p. 16; original figure.}
\end{frame}
\begin{frame}{Backup: Qasper and GovReport likelihood}
\centering
\includegraphics[width=0.90\linewidth,height=0.48\textheight,keepaspectratio]{figures/scrolls-top.pdf}\par
\includegraphics[width=0.90\linewidth,height=0.07\textheight,keepaspectratio]{figures/scrolls-legend.pdf}\par
\begin{itemize}
\item 160M models; +2B tokens at length 16,384.
\item Answer perplexity, lower is better; not task-level answer accuracy.
\end{itemize}
\papersource{Paper, Appendix G.2, Figure 12 (top panels), PDF p. 20. Original figure split for readability.}
\end{frame}
\begin{frame}{Backup: QMSum and NarrativeQA likelihood}
\centering
\includegraphics[width=0.90\linewidth,height=0.48\textheight,keepaspectratio]{figures/scrolls-bottom.pdf}\par
\includegraphics[width=0.90\linewidth,height=0.07\textheight,keepaspectratio]{figures/scrolls-legend.pdf}\par
\begin{itemize}
\item YOCO and Transformer benefit from longer input context.
\item Sparse baseline: 2,048 window and unchanged RoPE base.
\end{itemize}
\papersource{Paper, Appendix G.2, Figure 12 (bottom panels), PDF p. 20. Original figure split for readability.}
\end{frame}
\end{document}