% Based on the arxiv-paper-reader Beamer template.
% Build: python3 build_slides.py slides.en.tex
\documentclass[aspectratio=169,11pt]{beamer}
\usepackage[T1]{fontenc}
\usepackage[utf8]{inputenc}
\usepackage{lmodern}
\usepackage{amsmath,amssymb,graphicx,booktabs}

\definecolor{AcademicBlue}{RGB}{26,57,86}
\definecolor{AcademicTeal}{RGB}{0,105,112}
\setbeamercolor{structure}{fg=AcademicBlue}
\setbeamercolor{frametitle}{fg=AcademicBlue,bg=white}
\setbeamercolor{title}{fg=AcademicBlue}
\setbeamercolor{alerted text}{fg=AcademicTeal}
\setbeamercolor{block title}{fg=white,bg=AcademicBlue}
\setbeamercolor{block body}{fg=black,bg=AcademicBlue!5}
\setbeamertemplate{navigation symbols}{}
\setbeamertemplate{itemize items}[circle]
\setbeamertemplate{footline}{%
  \hfill\usebeamerfont{page number in head/foot}%
  \insertframenumber\,/\,\inserttotalframenumber\hspace{4mm}\vspace{3mm}}
\setbeamersize{text margin left=8mm,text margin right=8mm}
\setbeamerfont{frametitle}{size=\Large,series=\bfseries}
\setbeamerfont{title}{size=\LARGE,series=\bfseries}
\newcommand{\papersource}[1]{\par\smallskip{\scriptsize\color{gray}#1\par}}
\newcommand{\sm}{\operatorname{softmax}}
\newcommand{\KL}{\operatorname{KL}}
\newcommand{\GOAT}{GOAT}
\newcommand{\K}{\mathcal K}
\newcommand{\R}{\mathbb R}

\title[GOAT: Attention Priors]{You Need Better Attention Priors}
\subtitle{GOAT: learning structural defaults inside standard attention}
\author[Paper discussion]{Paper authors: Elon Litman and Gabe Guo\\[2mm]
\small Research paper discussion}
\date{arXiv:2601.15380v2 \;|\; ICML 2026\\Reading date: October 10, 2026}

\begin{document}

\begin{frame}
  \titlepage
  \vspace{-6mm}
  \begin{center}
    \alert{Content evidence + learned positional log-prior $\longrightarrow$ one SDPA call}
  \end{center}
\end{frame}

\begin{frame}{Weak content still needs a routing decision}
\begin{block}{The design question}
If no key is clearly relevant, should attention be uniform, local, or directed to a default token?
\end{block}
\medskip
\begin{tabular}{@{}p{2.2cm}p{5.2cm}p{5.4cm}@{}}
\toprule
Method & How position enters & Structural choice \\
\midrule
RoPE & Rotate content Q/K & Position interacts with content \\
ALiBi & Add a fixed distance bias & Monotone recency \\
\GOAT & Add a learned log-prior & Relative patterns + key defaults \\
\bottomrule
\end{tabular}
\par\medskip
GOAT reserves independent coordinates for structure in the attention logits.
\papersource{Paper, Secs. 1 and 4; Appendix A and F, PDF pp. 1--5, 12, 15--16.}
\end{frame}

\begin{frame}{Softmax solves a one-sided transport problem}
For one query, distribute unit mass over $L$ keys:
\[
p^*=\arg\min_{p\in\Delta^{L-1}}
\left[-\langle p,s\rangle-\tau H(p)\right],
\qquad H(p)=-\sum_jp_j\log p_j.
\]
\begin{itemize}
\item $s_j$: unscaled query--key affinity; $-s_j$: transport cost.
\item $\tau>0$: temperature; only the query-side simplex is constrained.
\item There is no key marginal constraint and no Sinkhorn iteration.
\end{itemize}
\begin{block}{The implicit prior}
$\KL(p\|\mathcal U)=-H(p)+\log L$: entropy regularization is KL regularization toward a uniform distribution.
\end{block}
\papersource{Paper, Eqs. (2)--(3), Sec. 2 and Appendix B, PDF pp. 2, 12.}
\end{frame}

\begin{frame}{A different prior gives an additive logit term}
\[
\min_{p\in\Delta^{L-1}}-\langle p,s\rangle+\tau\KL(p\|\pi)
\quad\Longrightarrow\quad
p_j^*=\frac{\pi_j e^{s_j/\tau}}{\sum_k\pi_k e^{s_k/\tau}}.
\]
\begin{block}{Closed-form attention}
\[
p^*=\sm(s/\tau+\log\pi).
\]
\end{block}
\begin{itemize}
\item Use positive $\pi$ on allowed keys; apply the causal mask separately.
\item A log-prior $\K$ needs no explicit normalization: row constants cancel.
\item A $9{:}1$ prior is balanced by a $\log9$ content-logit advantage for key 2.
\end{itemize}
\papersource{Paper, Proposition 3.1, Eqs. (4)--(10), PDF p. 3. Last bullet: teaching example.}
\end{frame}

\begin{frame}{GOAT allocates a fixed head budget to structure}
\[
z_{ij}=\underbrace{\frac{q_{c,i}^{\mathsf T}k_{c,j}}{\sqrt{d_c}}}_{\text{content}}
+\underbrace{\K^{\mathrm{rel}}_{ij}}_{\text{relative position}}
+\underbrace{u_h(j)}_{\text{key-only default}},
\qquad o_i=\sum_j\sm_j(z_i)v_j.
\]
\medskip
\begin{columns}[T,onlytextwidth]
\column{0.49\textwidth}
\begin{block}{Dimensions}
$d_p=2R+2$ positional dimensions\\
$d_c=d_h-d_p$ content dimensions\\
Value vectors retain all $d_h$ dimensions.
\end{block}
\column{0.47\textwidth}
\begin{block}{Example: $d_h=64$}
$R=4$: $d_p=10$, $d_c=54$\\
$R=8$: $d_p=18$, $d_c=46$\\
More positional capacity uses content capacity.
\end{block}
\end{columns}
\papersource{Paper, Sec. 4 and Algorithm 1, PDF pp. 4--5. Dimension examples follow the main-text formula.}
\end{frame}

\begin{frame}{Two coordinates realize one directional Fourier mode}
\[
\K^{\mathrm{rel}}_{ij}=\sum_{r=1}^R
\left[\alpha_r\cos\bigl(\omega_r(i-j)\bigr)+\beta_r\sin\bigl(\omega_r(i-j)\bigr)\right].
\]
\[
k^{(r)}_j=\begin{bmatrix}\cos\omega_rj\\\sin\omega_rj\end{bmatrix},\qquad
q^{(r)}_i=\begin{bmatrix}
\alpha_r\cos\omega_ri+\beta_r\sin\omega_ri\\
\alpha_r\sin\omega_ri-\beta_r\cos\omega_ri
\end{bmatrix}.
\]
\begin{itemize}
\item Their dot product is exactly the $r$th relative log-prior term.
\item $\alpha_r$ controls symmetric patterns; $\beta_r$ controls directionality.
\item Signed coefficients are valid: probabilities come from exponentiation.
\end{itemize}
\papersource{Paper, Eqs. (12)--(19); Appendices C--D, PDF pp. 3--4, 12--14.}
\end{frame}

\begin{frame}{One positional coordinate can express a key default}
\[
u_h(j)=\lambda_hj+g_h\bigl(\phi_{\mathrm{abs}}(j)\bigr)
+b_h\mathbf1\{j=0\}.
\]
\begin{itemize}
\item $g_h$: a small SiLU MLP over sinusoidal and normalized position features.
\item $\lambda_hj$: learned key-linear trend; $b_h$: optional first-token bump.
\item Constant query coordinate $1$ times key coordinate $u_h(j)$ adds the bias.
\end{itemize}
\begin{block}{ALiBi fits into the same coordinate}
For causal keys $j\le i$, $-m(i-j)=mj-mi$. The term $-mi$ is a row constant, so a key-linear bias gives the same softmax weights.
\end{block}
\papersource{Paper, Eqs. (21)--(22), Appendix G.3--G.4, PDF pp. 4, 20--22.}
\end{frame}

\begin{frame}{Pre-scaling makes one standard SDPA call exact}
\[
q'_i=\begin{bmatrix}
\sqrt{d_h/d_c}\,q_{c,i}\\
\sqrt{d_h}\,q_{\mathrm{rel},i}\\
\sqrt{d_h}\\0
\end{bmatrix},\qquad
k'_j=\begin{bmatrix}
k_{c,j}\\k_{\mathrm{rel},j}\\u_h(j)\\0
\end{bmatrix}.
\]
\[
\frac{(q'_i)^{\mathsf T}k'_j}{\sqrt{d_h}}
=\frac{q_{c,i}^{\mathsf T}k_{c,j}}{\sqrt{d_c}}+\K^{\mathrm{rel}}_{ij}+u_h(j).
\]
\begin{itemize}
\item No explicit dense positional bias matrix; standard attention kernels apply.
\item Dense attention still has quadratic computation; positional features add work.
\end{itemize}
\papersource{Paper, Eqs. (23)--(27) and Algorithm 1, PDF pp. 4--5. Zero coordinate is alignment padding.}
\end{frame}

\begin{frame}{A copy-mixture task separates global and relative structure}
\begin{columns}[c,onlytextwidth]
\column{0.61\textwidth}
\includegraphics[width=\linewidth,height=0.68\textheight,keepaspectratio]{figures/goat_prior_teaser_2x2.pdf}
\column{0.36\textwidth}
Copy the first token, copy the previous token, or sample noise.
\medskip
\begin{itemize}
\item Key-only term: first-position preference.
\item Relative term: diagonal, repeating patterns.
\item Mask + normalization produce the causal prior.
\end{itemize}
\end{columns}
\papersource{Paper, Fig. 1, PDF p. 1; Sec. 6, p. 6. Columns: key position; rows: query position.}
\end{frame}

\begin{frame}{Low content contrast makes the posterior approach the prior}
Now $s_{ij}=q_{c,i}^{\mathsf T}k_{c,j}/\sqrt{d_c}$ is a scaled logit. Define
\[
\omega_i=\max_k s_{ik}-\min_k s_{ik}.
\]
\begin{block}{Theorem 5.1: collapse to prior}
\[
\pi_{ij}e^{-\omega_i}\le p_{ij}\le\pi_{ij}e^{\omega_i}
\quad\Longrightarrow\quad p_i\to\pi_i\text{ as }\omega_i\to0.
\]
\end{block}
\begin{itemize}
\item Subtract the minimum score: the normalizer lies in $[1,e^{\omega_i}]$.
\item A peaked prior yields a sink; a uniform prior yields uniform weights.
\item The result describes a given prior; it does not force training to learn a sink.
\end{itemize}
\papersource{Paper, Eq. (28), Sec. 5 and Appendix E.1, PDF pp. 5, 14.}
\end{frame}

\begin{frame}{The sink bound retains an explicit dependence on length}
Fix Q/K. Perturb only non-sink values by $\|\Delta v_k\|\le\varepsilon$:
\[
\|\Delta o_i\|\le\varepsilon\Psi,\qquad \Psi=1-p_{ij^*}.
\]
If the prior margin is $\delta=\min_{k\ne j^*}(\K_{ij^*}-\K_{ik})>0$,
\[
\lim_{\omega_i\to0}\Psi\le\frac{L-1}{e^\delta+L-1}.
\]
\begin{block}{What length robustness requires}
To guarantee $\Psi\le\eta$ using this bound:
$\displaystyle \delta\ge\log\frac{(1-\eta)(L-1)}{\eta}$.
Fixed margin does not give a uniform guarantee as $L$ grows.
\end{block}
\papersource{Paper, Eqs. (31)--(34), PDF p. 6; Appendix E.2, p. 15. Rearranged bound: reading analysis.}
\end{frame}

\begin{frame}{Evaluation spans language, retrieval, and spatial structure}
\begin{tabular}{@{}p{2.7cm}p{5.0cm}p{4.9cm}@{}}
\toprule
Experiment & Setup & Main measurement \\
\midrule
C4 & 125M; 4B tokens; train at 2048 & PPL under length extrapolation \\
FineWeb & 1.23B; about 3.13B tokens & PPL from 2K to 16K \\
Passkey / NIAH & Synthetic retrieval; 8-digit key & Span-token accuracy \\
Human genome & 125M decoder LM & NLL, throughput, memory \\
ImageNet-1k & ViT-Small; train at $224^2$ & Top-1 at larger resolutions \\
Spatial VQA & Four-image CIFAR-10 scenes & Held-out scan-order accuracy \\
\bottomrule
\end{tabular}
\par\medskip
Language experiments share a training window, not a dataset or training budget.
\papersource{Paper, Sec. 6; Tables 2, 8, 10, 11, PDF pp. 6--9, 23--25.}
\end{frame}

\begin{frame}{C4 improves the quality--extrapolation trade-off}
\begin{columns}[c,onlytextwidth]
\column{0.57\textwidth}
\includegraphics[width=\linewidth,height=0.64\textheight,keepaspectratio]{figures/colorblind_c4_ppl_extrap.pdf}
\column{0.40\textwidth}
At the 2048 training length:
\par\medskip
\begin{tabular}{@{}lr@{}}
\toprule
Model & PPL $\downarrow$ \\
\midrule
RoPE & 33.24 \\
ALiBi & 32.49 \\
\GOAT & \textbf{30.95} \\
\bottomrule
\end{tabular}
\par\medskip
RoPE degrades rapidly. ALiBi is flatter at very long lengths; GOAT is not best at every length.
\end{columns}
\papersource{Paper, Fig. 2(a), PDF p. 7. Caption reports 1.55 PPL gain; printed labels differ by 1.54.}
\end{frame}

\begin{frame}{At 1.23B, the main gain appears beyond the training window}
\begin{center}
\large
\begin{tabular}{@{}lrrrr@{}}
\toprule
Model & 2K & 4K & 8K & 16K \\
\midrule
RoPE & 20.82 & 28.53 & 89.45 & 417.43 \\
\GOAT & \textbf{20.75} & \textbf{21.04} & \textbf{22.69} & \textbf{24.77} \\
\bottomrule
\end{tabular}
\end{center}
\medskip
\begin{itemize}
\item FineWeb; 22 layers, 32 heads, width 2048; training length 2048.
\item About 3.13B tokens at the selected GOAT checkpoint.
\item In-window PPL is almost unchanged; extrapolation degrades much more slowly.
\end{itemize}
\papersource{Paper, Table 1, PDF p. 8; Table 8, p. 24. All entries are perplexity; lower is better.}
\end{frame}

\begin{frame}{Passkey retrieval stays strong at long distances}
\begin{columns}[c,onlytextwidth]
\column{0.69\textwidth}
\includegraphics[width=\linewidth,height=0.64\textheight,keepaspectratio]{figures/synthetic_passkey_retrieval.pdf}
\column{0.28\textwidth}
At about 16K:
\begin{itemize}
\item GOAT: ${\sim}90\%$
\item ALiBi: ${\sim}48\%$
\item RoPE: ${\sim}11\%$
\end{itemize}
\medskip
Metric: accuracy of tokens within the target span, not full-key exact match.
\end{columns}
\papersource{Paper, Fig. 3(a), PDF p. 7. Values are approximate graph readings; caption uses $L_{\rm train}=1024$ as an example.}
\end{frame}

\begin{frame}{NIAH gains span depths, with some loss at the longest length}
\centering
\includegraphics[width=\linewidth,height=0.54\textheight,keepaspectratio]{figures/niah-selected.pdf}
\par\medskip
\raggedright
GOAT is strongest across the grid. At 16K, shallow needles are visibly less reliable than those in shorter contexts.
\papersource{Paper, Fig. 3(c), PDF p. 7. Selected original GOAT, RoPE, ALiBi panels and shared colorbar; data unchanged.}
\end{frame}

\begin{frame}{DNA modeling improves NLL and memory, with similar throughput}
\begin{center}
\begin{tabular}{@{}lrrl@{}}
\toprule
Metric & RoPE & GOAT & Change \\
\midrule
Validation NLL $\downarrow$ & 1.2054 & \textbf{1.1294} & $-6.30\%$ \\
Bits/base $\downarrow$ & 1.7390 & \textbf{1.6294} & $-0.1096$ \\
Training tokens/s $\uparrow$ & \textbf{139,886} & 138,171 & $-1.23\%$ \\
Peak allocated memory (GB) $\downarrow$ & 2.86 & \textbf{1.83} & $-36.01\%$ \\
\bottomrule
\end{tabular}
\end{center}
\medskip
\begin{itemize}
\item Human reference genome, 125M decoder models, matched training budgets.
\item NLL and bits/base express the same predictive quality in different units.
\item The memory result is for this setup; it is not a fixed saving of the mechanism.
\end{itemize}
\papersource{Paper, Fig. 4(a)--(b), PDF p. 9. Relative changes calculated from displayed values.}
\end{frame}

\begin{frame}{A 2D prior transfers better to higher image resolutions}
\begin{columns}[c,onlytextwidth]
\column{0.56\textwidth}
\includegraphics[width=\linewidth,height=0.60\textheight,keepaspectratio]{figures/imagenet_extrapolation.pdf}
\column{0.40\textwidth}
\begin{itemize}
\item ViT-Small, patch size 16, 150 epochs at $224\times224$.
\item At 512: about 76.6\% vs. 72.8\% Top-1, without fine-tuning.
\item Axis-wise features impose relative 2D structure; learning sets its shape.
\end{itemize}
\end{columns}
\papersource{Paper, Fig. 5(a), PDF p. 8; Table 11, p. 25. Accuracy values are approximate graph readings.}
\end{frame}

\begin{frame}{Canonical coordinates help when image scan order changes}
Four CIFAR-10 images on a canonical $2\times2$ grid; test scan orders were unseen during training.
\medskip
\begin{center}
\large
\begin{tabular}{@{}lrr@{}}
\toprule
Method & Answer NLL $\downarrow$ & Exact answer accuracy $\uparrow$ \\
\midrule
No PE & 0.359 & 17.2\% \\
RoPE & 0.367 & 16.5\% \\
\GOAT & \textbf{0.350} & \textbf{20.3\%} \\
\bottomrule
\end{tabular}
\end{center}
\medskip
\begin{itemize}
\item 50k/10k train/test scenes; answer-only spatial questions.
\item GOAT gains 3.8 percentage points over RoPE; absolute accuracy remains low.
\end{itemize}
\papersource{Paper, Tables 9--10, PDF pp. 24--25. Image tokens use 2D coordinates; text uses 1D positions.}
\end{frame}

\begin{frame}{More Fourier modes mainly improve extrapolation}
\begin{center}
\large
\begin{tabular}{@{}rrrrr@{}}
\toprule
$R$ & $d_p$ & PPL 2K & PPL 4K & PPL 8K \\
\midrule
1 & 4 & 35.49 & 34.81 & 38.28 \\
2 & 6 & 35.52 & 34.95 & 38.64 \\
4 & 10 & 35.49 & 34.57 & 35.97 \\
8 & 18 & \textbf{35.18} & \textbf{34.15} & \textbf{35.15} \\
\bottomrule
\end{tabular}
\end{center}
\medskip
\begin{itemize}
\item $R=2\to8$: 8K PPL improves by 3.49; 2K PPL improves by 0.34.
\item The head budget is fixed: larger $R$ reduces content Q/K dimensions.
\item These ablation PPLs differ from Fig. 2; compare within this table.
\end{itemize}
\papersource{Paper, Table 3, PDF p. 23. Main-figure and ablation configurations are not fully reconciled in the paper.}
\end{frame}

\begin{frame}{Learned structure is directional and stronger in early layers}
\begin{columns}[T,onlytextwidth]
\column{0.50\textwidth}
\begin{block}{1.23B spectral statistics}
\begin{itemize}
\item 287/704 heads have amplitude $>1$.
\item 662/704 have dominant antisymmetric coefficients.
\item Median $|\beta|/|\alpha|=1.84$.
\item Mean amplitude: 2.04 in layer 0, 0.70 in layer 21.
\end{itemize}
\end{block}
\column{0.46\textwidth}
\begin{block}{Training loss checkpoints}
\begin{tabular}{@{}lrr@{}}
 & RoPE & GOAT \\
\midrule
C4, 15k steps & 3.535 & 3.520 \\
FineWeb, 10k & 3.003 & 2.993
\end{tabular}
\par\medskip
Reported runs show no clear early optimization penalty.
\end{block}
\end{columns}
\papersource{Paper, Tables 6--7, PDF p. 24. Spectral statistics describe learned heads; they are not component-removal ablations.}
\end{frame}

\begin{frame}{The useful contribution is an explicit, efficient routing prior}
\begin{enumerate}
\item \textbf{Mechanism:} learn structural logits separately from content, then factor them into ordinary Q/K coordinates.
\medskip
\item \textbf{Evidence:} large extrapolation gains over the tested RoPE baseline, including at 1.23B; gains also appear in retrieval and spatial tasks.
\medskip
\item \textbf{Trade-offs:} positional capacity uses content dimensions; sink bounds need sufficient margin; dense attention cost remains.
\end{enumerate}
\begin{block}{A productive next comparison}
Hold content capacity and training budget fixed, compare adapted long-context baselines, and ablate relative directionality versus key-only defaults.
\end{block}
\papersource{Synthesis of Secs. 4--6 and Appendices G--K. Last block: reading analysis.}
\end{frame}

\begin{frame}{Paper and discussion materials}
\textbf{Elon Litman and Gabe Guo.}\\[2mm]
\emph{You Need Better Attention Priors.}\\[2mm]
ICML 2026; arXiv:2601.15380v2, revised August 23, 2026.
\medskip
\begin{itemize}
\item Versioned paper: \url{https://arxiv.org/abs/2601.15380v2}
\item Author code link: \url{https://github.com/elonlit/goat}
\item Companion Chinese notes: \texttt{reading-notes.zh.md}
\end{itemize}
\medskip
All experiment results and original plots are from this paper.\\
Supplementary derivations and technical qualifications follow.
\end{frame}

\appendix

\begin{frame}{Backup: finite rank and boundedness imply finite frequencies}
Assume a relative log-prior with fixed $d_p$-dimensional positional features:
\[
\K_{ij}=\kappa(i-j)=\langle\varphi(i),\psi(j)\rangle,
\qquad \sup_{d\in\mathbb Z}|\kappa(d)|<\infty.
\]
\begin{enumerate}
\item Translates $\kappa(\cdot-j)$ span a finite-dimensional shift-invariant space.
\item The shift operator yields exponential-polynomial sequences.
\item Boundedness in both directions excludes growth and nonconstant polynomials.
\end{enumerate}
\[
\kappa(d)=\sum_{m=1}^{M}c_m e^{\mathrm{i}\theta_md},\qquad M\le d_p.
\]
This class result does not select the geometric frequency grid, nor constrain all of $u_h(j)$.
\papersource{Paper, Appendix G.1--G.2, Theorem 2, PDF pp. 16--19.}
\end{frame}

\begin{frame}{Backup: a special frequency needs separate dimension counting}
The real-mode statement in Eq. (73) gives $R\le\lfloor d_p/2\rfloor$.
\begin{block}{Counterexample to that bound as stated}
On integer positions, $\kappa(d)=(-1)^d$ is bounded and nonconstant, but
\[
\kappa(i-j)=(-1)^i(-1)^j
\]
uses just one dimension. For $d_p=1$, the printed bound would require $R=0$.
\end{block}
\begin{itemize}
\item Nondegenerate conjugate frequencies use two real dimensions per pair.
\item Constant and Nyquist ($\pi$) modes each use only one dimension.
\item The finite-frequency conclusion and GOAT's ordinary mode construction survive.
\end{itemize}
\papersource{Paper, Appendix G.2, Eq. (73), PDF p. 17. Counterexample and correction: reading analysis.}
\end{frame}

\begin{frame}{Backup: a mean-lag constraint gives a geometric prior}
Let $d=i-j\in\{0,\ldots,L-1\}$ and prescribe mean lag $\mu$:
\[
\max_q H(q)\quad\text{subject to}\quad \sum_dq_d=1,\quad \sum_ddq_d=\mu.
\]
\[
q_d^*=\frac{e^{-\lambda d}}{Z(\lambda)},\qquad
\frac{d}{d\lambda}\mathbb E_{q^*}[d]=-\operatorname{Var}_{q^*}(d)<0.
\]
\begin{itemize}
\item For $L\ge2$ and $0<\mu<L-1$, the required $\lambda$ is unique.
\item Recency corresponds to $\lambda>0$; $\lambda=0$ gives the uniform prior.
\item $\log q_{i-j}^*=\lambda j+c_i$: a key-only trend is sufficient.
\end{itemize}
\papersource{Paper, Appendix G.3, Theorem 3 and Proposition G.6, PDF pp. 20--21.}
\end{frame}

\begin{frame}{Backup: frequency and initialization changes are modest}
\begin{columns}[T,onlytextwidth]
\column{0.50\textwidth}
\textbf{Frequency grid}
\par\medskip
\begin{tabular}{@{}lrr@{}}
\toprule
Setting & 2K PPL & 8K PPL \\
\midrule
Base 1k & 35.48 & 38.42 \\
Base 10k & 35.52 & 38.64 \\
Base 100k & 35.62 & 38.51 \\
Learned & \textbf{35.39} & \textbf{37.99} \\
\bottomrule
\end{tabular}
\column{0.47\textwidth}
\textbf{Initialization}
\par\medskip
\begin{tabular}{@{}lrr@{}}
\toprule
Setting & 2K PPL & 8K PPL \\
\midrule
Zero & \textbf{35.52} & 38.64 \\
Random & 35.54 & 38.88 \\
ALiBi start & 35.56 & \textbf{38.48} \\
\bottomrule
\end{tabular}
\end{columns}
\bigskip
Changes across broad fixed grids are smaller than the mode-count effect. Learning frequencies or using a recency warm start brings a small gain.
\papersource{Paper, Tables 4--5, PDF p. 23.}
\end{frame}

\begin{frame}{Backup: details that matter when implementing the paper}
\begin{itemize}
\item \textbf{2D formula:} Eq. (20) omits the plus signs between cosine and sine terms. Axis-wise concatenation implements the additive form of Eq. (12).
\medskip
\item \textbf{Dimension names:} C4 Table 2 reports $d_{\rm pos}=12$, Pos Rank 2; the mapping to $d_p=2R+2$ is not explained.
\medskip
\item \textbf{C4 protocol:} Fig. 2 and ablation PPLs differ; Table 2's batch and Table 7's tokens/step require an effective-batch explanation.
\medskip
\item \textbf{Sink behavior:} a key-only bias changes routing. It neither removes the sink token nor makes its value vector zero.
\end{itemize}
\papersource{Paper, Eqs. (12), (20), Algorithm 1, Tables 2--7, PDF pp. 3--5, 23--24. Qualifications: reading analysis.}
\end{frame}

\end{document}