% Based on the arxiv-paper-reader Beamer template. Compile with pdfLaTeX.
\documentclass[aspectratio=169,11pt]{beamer}
\usepackage[T1]{fontenc}
\usepackage[utf8]{inputenc}
\usepackage{amsmath,amssymb,graphicx,booktabs}
\definecolor{AcademicBlue}{RGB}{26,57,86}
\definecolor{AcademicTeal}{RGB}{0,105,112}
\setbeamercolor{structure}{fg=AcademicBlue}
\setbeamercolor{frametitle}{fg=AcademicBlue,bg=white}
\setbeamercolor{title}{fg=AcademicBlue}
\setbeamercolor{alerted text}{fg=AcademicTeal}
\setbeamercolor{block title}{fg=white,bg=AcademicBlue}
\setbeamercolor{block body}{fg=black,bg=AcademicBlue!5}
\setbeamertemplate{navigation symbols}{}
\setbeamertemplate{itemize items}[circle]
\setbeamertemplate{footline}{\hfill\usebeamerfont{page number in head/foot}\insertframenumber\,/\,\inserttotalframenumber\hspace{4mm}\vspace{3mm}}
\setbeamersize{text margin left=8mm,text margin right=8mm}
\setbeamerfont{frametitle}{size=\Large,series=\bfseries}
\setbeamerfont{title}{size=\LARGE,series=\bfseries}
\newcommand{\papersource}[1]{\par\smallskip{\scriptsize\color{gray}#1\par}}
\newcommand{\smone}{\mathrm{Softmax}_1}
\newcommand{\softmax}{\mathrm{Softmax}}
\newcommand{\norm}[1]{\left\lVert#1\right\rVert}
\setlength{\tabcolsep}{7pt}
\renewcommand{\arraystretch}{1.2}
\title[OASIS]{Attention Sinks and Outliers\\in Attention Residuals}
\subtitle{OASIS: null routing across tokens and depth}
\author{Paper by Haozheng Luo, Haoran Dai, Ching-Yuen Huang et al.\\\small Research group reading presentation}
\date{arXiv:2605.17887v2 \enspace | \enspace revised 24 September 2026\\Reading: 7 October 2026}

\begin{document}

\begin{frame}
\titlepage
\begin{center}\small Explicit null capacity attenuates contributions; token null evidence guides depth routing.\end{center}
\end{frame}

\begin{frame}{Two routing decisions share the same constraint}
\begin{columns}[T]
\begin{column}{0.48\textwidth}
\begin{block}{Token attention: which context?}
\[
A_t^{(i)}=\sum_j p_{t,j}^{(i)}v_j^{(i)},\qquad\sum_jp_{t,j}^{(i)}=1
\]
All probability goes to real tokens.
\end{block}
\end{column}
\begin{column}{0.48\textwidth}
\begin{block}{Depth routing: which source?}
\[
h_t^{(\ell)}=\sum_{i=0}^{\ell-1}\alpha_{i\to\ell,t}s_{i,t},\qquad\sum_i\alpha_i=1
\]
All probability goes to real branches.
\end{block}
\end{column}
\end{columns}
\medskip
\begin{itemize}
\item $s_{0,t}$ is the initial state; $s_{i,t}=f_i(h_t^{(i)})$ is a transformed source.
\item A small desired contribution can create pressure to find a surrogate no-op.
\item The paper studies this pressure after converting pretrained backbones.
\end{itemize}
\papersource{Paper, Secs. 3--4.1, PDF pp. 3--4. Concentration requires the geometric conditions discussed later.}
\end{frame}

\begin{frame}{The observed activations become much more extreme}
\centering
\includegraphics[width=\linewidth,height=0.53\textheight,keepaspectratio]{figures/fig-03-outliers.pdf}
\medskip
\begin{itemize}
\item Read layer index horizontally; compare the different vertical scales.
\item Original: kurtosis near 3. AttnResidual: about 50--70 after the first layer.
\item Larger ranges make low-bit activation quantization more difficult.
\end{itemize}
\papersource{Original Fig. 3, PDF p. 3. These layer-wise plots differ from the stack-wide statistics in Tables 1--2.}
\end{frame}

\begin{frame}{OASIS connects two null-aware routers}
\centering
\includegraphics[width=0.96\linewidth,height=0.80\textheight,keepaspectratio]{figures/fig-01-overview.pdf}
\papersource{Original Fig. 1, PDF p. 2. Middle: AoS. Right: the token-to-depth coupling added by OASIS.}
\end{frame}

\begin{frame}{A zero-valued null state controls total contribution}
\[
\tilde p_j=\frac{e^{z_j}}{1+\sum_ke^{z_k}},\qquad
\tilde p_\varnothing=\frac{1}{1+\sum_ke^{z_k}}
\]
\begin{block}{Equivalent to one additional softmax category}
Null logit $=0$; null value $=0$. The real-token probabilities sum to $1-\tilde p_\varnothing$.
\end{block}
\[
\tilde p_j=(1-\tilde p_\varnothing)p_j,\qquad
\tilde A=(1-\tilde p_\varnothing)A
\]
\begin{itemize}
\item At fixed logits, relative allocation among real tokens is preserved.
\item A common downward logit shift now reduces the total real contribution.
\end{itemize}
\papersource{Paper, Eq. (4.1), PDF p. 4; Appendix C.5, PDF pp. 20--21.}
\end{frame}

\begin{frame}{AoS applies the same mechanism across depth}
\[
\tilde\alpha_{i\to\ell,t}
=\frac{e^{g_{i\to\ell,t}}}{1+\sum_{r=0}^{\ell-1}e^{g_{r\to\ell,t}}},\qquad
\tilde h_t^{(\ell)}=\sum_{i=0}^{\ell-1}\tilde\alpha_{i\to\ell,t}s_{i,t}
\]
\begin{itemize}
\item \textbf{AoS}: token $\smone$ + depth $\smone$.
\item The real source mixture can now be attenuated.
\item The initial-state branch $i=0$ remains a real candidate.
\end{itemize}
\begin{block}{Depth-null semantics}
The null value is zero. Attenuating the source mixture does not automatically preserve a previous hidden state.
\end{block}
\papersource{Paper, Sec. 4.2 and Eq. (4.2), PDF p. 4; Appendix C.5, PDF pp. 21--22.}
\end{frame}

\begin{frame}{Each branch supplies a token-level null statistic}
\[
\psi_{i,t}=\frac1H\sum_{h=1}^{H}\tilde p_{t,\varnothing}^{(i,h)},\qquad i\geq1
\]
\[
\overline\psi_{\ell,t}=\frac1{\ell-1}\sum_{r=1}^{\ell-1}\psi_{r,t},\qquad
\Delta\psi_{i\to\ell,t}=\begin{cases}
\psi_{i,t}-\overline\psi_{\ell,t},&i\geq1,\\
0,&i=0.
\end{cases}
\]
\begin{itemize}
\item Average over $H$ attention heads at the same token position.
\item Center over transformed source branches available to target layer $\ell$.
\item The initial-state branch has no attention heads; its logit is left unchanged.
\end{itemize}
\papersource{Paper, Sec. 4.3, PDF p. 5. No coupling is applied at $\ell=1$.}
\end{frame}

\begin{frame}{Null evidence changes relative depth preferences}
\[
g^{\mathrm{new}}_{i\to\ell,t}=g^{\mathrm{old}}_{i\to\ell,t}-\beta\Delta\psi_{i\to\ell,t},\qquad\beta\geq0
\]
\[
\tilde\alpha_{i\to\ell,t}=\frac{e^{g^{\mathrm{new}}_{i\to\ell,t}}}{1+\sum_re^{g^{\mathrm{new}}_{r\to\ell,t}}}
\]
\begin{itemize}
\item Above-average null mass receives a negative logit adjustment.
\item Below-average null mass receives a positive adjustment.
\item $\beta=\mathrm{softplus}(\theta)$ is learned; $\theta=-5$ gives $\beta\approx0.0067$ initially.
\end{itemize}
\begin{block}{Design hypothesis}
A branch's tendency to route to null is useful evidence for depth selection.
\end{block}
\papersource{Paper, Sec. 4.3, PDF p. 5; Appendix E, PDF p. 23. The training objective is unchanged.}
\end{frame}

\begin{frame}{A small example separates coupling from attenuation}
\textbf{Teaching example:} two transformed branches, $\psi_1=0.8$, $\psi_2=0.2$; all old logits $=0$; $\beta=1$.
\medskip
\begin{center}
\setlength{\tabcolsep}{11pt}
\begin{tabular}{lrrr}
\toprule
Candidate & New logit & AoS weight & OASIS weight\\
\midrule
Initial state & $0$ & 0.2500 & 0.2445\\
Branch 1: high null & $-0.3$ & 0.2500 & 0.1811\\
Branch 2: low null & $+0.3$ & 0.2500 & 0.3300\\
Null state & $0$ & 0.2500 & 0.2445\\
\bottomrule
\end{tabular}
\end{center}
\begin{itemize}
\item Higher-null branch 1 loses relative weight to branch 2.
\item Null mass need not increase; the initial state's weight can also change.
\end{itemize}
\papersource{Reading analysis derived from Sec. 4.3, PDF p. 5; illustrative numbers, not an experiment.}
\end{frame}

\begin{frame}{Concentration follows from a separation assumption}
\textbf{Token geometry:} $\norm{v_o}\leq c_0<c_1$; every other value has $\langle v_j,u\rangle\geq c_1$ along a shared unit direction $u$.
\[
\norm{A}\leq\delta<c_1
\quad\Longrightarrow\quad
p_o\geq\frac{c_1-\delta}{c_0+c_1}
\]
\textbf{Depth geometry:} for $u_i=s_i-h^{\mathrm{ref}}$, assume $\norm{u_{i^*}}\leq b_0$ and $\langle u_i,w\rangle\geq b_1$ for all $i\neq i^*$, with unit $w$.
\[
\norm{\Delta h}\leq\delta_t<b_1
\quad\Longrightarrow\quad
\alpha_{i^*}\geq\frac{b_1-\delta_t}{b_0+b_1}
\]
\begin{block}{Why the condition matters}
Values $u$ and $-u$ average to zero with equal weights: separation is essential.
\end{block}
\papersource{Lemma 5.1 and Theorem 5.1, PDF pp. 5--6; assumptions/proofs, pp. 17--19, 22. Counterexample: reading analysis.}
\end{frame}

\begin{frame}{Null mass gives a direct bound on contribution size}
Assume $\norm{v_j}\leq V_{\max}$ and $\norm{s_i}\leq B_{\max}$ for all real candidates:
\[
\norm{\tilde A_t^{(\ell)}}\leq V_{\max}(1-\tilde p_{t,\varnothing}^{(\ell)})
\]
\[
\norm{\tilde h_t^{(\ell)}}\leq B_{\max}(1-\tilde\alpha_{\varnothing,t}^{(\ell)})
\]
\begin{itemize}
\item Sufficient null mass allows a small output without a dominant real candidate.
\item The depth bound concerns the mixed state itself, relative to zero.
\item Outlier suppression and quantized accuracy are assessed empirically.
\end{itemize}
\papersource{Theorem 5.2, PDF p. 7; Appendix C.5, PDF pp. 20--22.}
\end{frame}

\begin{frame}{Evaluation combines architecture and quantization controls}
\begin{itemize}
\item \textbf{Backbones:} LLaMA-3.2-1B, Qwen3-0.6B, Phi-4 (the linked checkpoint is Phi-4-reasoning).
\item \textbf{Adaptation:} continued pretraining on BookCorpus + Wiki40B/en; 500 steps in the standard setup.
\item \textbf{Precision:} FP16, W8A8; OmniQuant for W6A6/W4A4.
\item \textbf{Tasks:} WikiText-2/Pile PPL; GSM8K and knowledge/math accuracy; 12K RULER/LongBench.
\item \textbf{Controls:} original backbone, matched CPT, Vanilla AttnResidual, token-only, AoS, clipping/gating and sink baselines.
\end{itemize}
\medskip
The authors report three seeds. Subsequent compact tables show means unless $\pm$ is displayed.
\papersource{Paper, Sec. 6, PDF pp. 7--10; Appendix E, PDF pp. 22--23. The 12K experiment is a separate setup.}
\end{frame}

\begin{frame}{AoS improves quantization beyond token-only null routing}
\begin{center}
\begin{tabular}{llrr}
\toprule
Backbone & Method & W8A8 PPL $\downarrow$ & W4A4 PPL $\downarrow$\\
\midrule
LLaMA-3.2-1B & Vanilla AttnResidual & 30.19 & 74.97\\
 & Token only / OutEffHop & 10.42 & 26.18\\
 & AoS & \alert{9.36} & \alert{22.63}\\
\midrule
Qwen3-0.6B & Vanilla AttnResidual & 36.64 & 50.82\\
 & Token only / OutEffHop & 10.60 & 32.36\\
 & AoS & \alert{9.62} & \alert{17.52}\\
\bottomrule
\end{tabular}
\end{center}
Depth-null routing adds value on top of token-null routing. Clipped and gated combinations also benefit (backup).
\papersource{Paper, Table 1, PDF p. 8. WikiText-2 perplexity; W4A4 uses OmniQuant. Selected mean values.}
\end{frame}

\begin{frame}{Coupling adds 4.67 percentage points over AoS}
\textbf{Qwen3-0.6B, W4A4 GSM8K Pass@1}
\begin{center}
\begin{tabular}{lcccr}
\toprule
Configuration & Token null & Depth null & Coupling & Accuracy\\
\midrule
Vanilla AttnResidual & -- & -- & -- & 45.67\%\\
Token only & Yes & -- & -- & 52.36\%\\
Depth only & -- & Yes & -- & 48.90\%\\
AoS & Yes & Yes & -- & 54.21\%\\
OASIS & Yes & Yes & Yes & \alert{58.88\%}\\
\bottomrule
\end{tabular}
\end{center}
\begin{block}{The key component comparison}
$58.88-54.21=4.67$ points; relative gain over AoS $=8.61\%$.
\end{block}
\papersource{Paper, Sec. 6.2, PDF p. 8; Table 8, PDF p. 24. Higher is better.}
\end{frame}

\begin{frame}{W8A8 perplexity approaches the FP16 result}
\begin{center}
\begin{tabular}{llrr}
\toprule
Backbone & Method & FP16 PPL $\downarrow$ & W8A8 PPL $\downarrow$\\
\midrule
LLaMA-3.2-1B & Vanilla AttnResidual & 8.98 & 30.19\\
 & OASIS & 8.01 & \alert{8.16}\\
Qwen3-0.6B & Vanilla AttnResidual & 10.57 & 36.64\\
 & OASIS & 9.20 & \alert{9.52}\\
Phi-4 & Vanilla AttnResidual & 11.21 & 1176.73\\
 & OASIS & 10.94 & \alert{11.72}\\
\bottomrule
\end{tabular}
\end{center}
Per-model relative W8A8 reductions: \textbf{72.97\%, 74.02\%, 99.00\%}.
Their arithmetic mean is the headline \textbf{82.00\%}.
\papersource{Paper, Table 2, PDF p. 8. WikiText-2; mean values. Relative reductions recalculated from the table.}
\end{frame}

\begin{frame}{GSM8K recovery differs sharply across backbones}
\begin{center}
\begin{tabular}{lrrr}
\toprule
W4A4 Pass@1 (\%) & LLaMA-3.2-1B & Qwen3-0.6B & Phi-4\\
\midrule
Vanilla AttnResidual & $12.97\pm0.24$ & $45.67\pm0.86$ & $43.16\pm0.25$\\
AoS & $13.34\pm0.18$ & $54.21\pm0.66$ & $76.85\pm0.21$\\
OASIS & $13.52\pm0.17$ & $58.88\pm0.59$ & $83.37\pm0.21$\\
\midrule
Gain vs. Vanilla (points) & +0.55 & +13.21 & +40.21\\
Relative gain (\%) & +4.24 & +28.92 & +93.16\\
\bottomrule
\end{tabular}
\end{center}
\begin{itemize}
\item The headline \textbf{42.11\%} averages the three relative gains.
\item FP16 gains are only 0.11, 0.10 and 0.24 points, respectively.
\end{itemize}
\papersource{Paper, Table 2, PDF p. 8. Mean $\pm$ standard deviation over three seeds. Gains: reading calculations.}
\end{frame}

\begin{frame}{Global statistics support model-wide sink suppression}
\begin{center}
\begin{tabular}{llrrr}
\toprule
Backbone & Method & BOS mass $\downarrow$ & Entropy $\uparrow$ & Null mass\\
\midrule
Qwen3-0.6B & Original backbone & 0.1064 & 3.74 & 0\\
 & AttnResidual & 0.4990 & 1.83 & 0\\
 & OASIS & \alert{0.0013} & 4.44 & 0.0004\\
LLaMA-3.2-1B & Original backbone & 0.1091 & 3.34 & 0\\
 & AttnResidual & 0.5630 & 1.86 & 0\\
 & OASIS & \alert{0.0099} & 4.55 & 0.2185\\
\bottomrule
\end{tabular}
\end{center}
\textbf{Interpretation:} Qwen suppresses sinks with very little average null mass. Learned redistribution is relevant beyond directly absorbing BOS mass.
\papersource{Paper, Table 7, PDF p. 24. Aggregated across layers, heads, queries and samples. Interpretation: reading analysis.}
\end{frame}

\begin{frame}{At 12K, OASIS recovers most of the lost performance}
\begin{center}
\begin{tabular}{lrrrr}
\toprule
 & \multicolumn{2}{c}{LLaMA-3.2-1B} & \multicolumn{2}{c}{Qwen3-0.6B}\\
Method & RULER & LongBench & RULER & LongBench\\
\midrule
Original backbone & 0.786 & 0.225 & 0.891 & 0.194\\
Vanilla AttnResidual & 0.500 & 0.115 & 0.592 & 0.133\\
Learnable sink logit & 0.597 & 0.131 & 0.811 & 0.184\\
AoS & 0.567 & 0.121 & 0.781 & 0.174\\
OASIS & \alert{0.721} & \alert{0.213} & \alert{0.832} & \alert{0.189}\\
\bottomrule
\end{tabular}
\end{center}
\begin{itemize}
\item OASIS exceeds the tested stabilization variants; the original backbone remains higher in every column.
\item RULER loss recovered: 77.3\% / 80.3\%; LongBench: 89.1\% / 91.8\%.
\end{itemize}
\papersource{Paper, Table 3, PDF p. 10. Higher scores are better. Recovery = (OASIS--Vanilla)/(Original--Vanilla).}
\end{frame}

\begin{frame}{The low-bit gains extend beyond WikiText-2 and GSM8K}
\begin{center}
\begin{tabular}{llrrrr}
\toprule
 & & Pile & MMLU & TriviaQA & MATH500\\
Model & Method & PPL $\downarrow$ & \% $\uparrow$ & \% $\uparrow$ & \% $\uparrow$\\
\midrule
LLaMA & AttnResidual & 38.24 & 26.42 & 19.13 & 2.81\\
 & AoS & 11.62 & 29.27 & 24.04 & 3.43\\
 & OASIS & 10.17 & 30.19 & 25.56 & 3.77\\
Qwen & AttnResidual & 44.52 & 33.21 & 12.09 & 15.41\\
 & AoS & 11.73 & 39.37 & 15.07 & 19.50\\
 & OASIS & 11.58 & 41.00 & 16.14 & 21.51\\
\bottomrule
\end{tabular}
\end{center}
Pile uses \textbf{W8A8}; accuracy tasks use \textbf{W4A4}. Full-precision accuracy changes are much smaller.
\papersource{Paper, Table 4, PDF p. 10. LLaMA-3.2-1B and Qwen3-0.6B; selected mean values.}
\end{frame}

\begin{frame}{The evidence leaves concrete implementation questions}
\begin{itemize}
\item \textbf{Branch quality proxy:} head-averaged null probability can hide a useful minority of heads or non-attention branch content.
\item \textbf{Attenuation tradeoff:} excessive null allocation may suppress weak useful signals; null usage varies strongly across models.
\item \textbf{Reproducibility:} W8A8 calibration, the 12K training budget and global sink metric definitions need more detail.
\item \textbf{Deployment:} low-bit accuracy is measured; optimized-kernel latency, throughput and memory gains are not reported.
\end{itemize}
\papersource{Paper, Appendix E--F, PDF pp. 22--23; Tables 3 and 7. Proxy and reporting questions: reading analysis.}
\end{frame}

\begin{frame}{The contribution is a feedback path across routing levels}
\begin{enumerate}
\item \textbf{AoS supplies null capacity.} Both token and depth routes can attenuate their real contributions.
\item \textbf{OASIS uses null evidence.} Above-average token null behavior lowers a branch's relative depth preference.
\item \textbf{The ablation isolates coupling.} Qwen W4A4 GSM8K rises from 54.21\% (AoS) to 58.88\% (OASIS).
\end{enumerate}
\begin{block}{Discussion question}
When does a branch's decision to contribute little reliably predict its value to later layers?
\end{block}
\papersource{Paper, Sec. 4.3, PDF p. 5; Table 8, PDF p. 24.}
\end{frame}

\begin{frame}{Paper and supporting material}
\textbf{Haozheng Luo, Haoran Dai, Ching-Yuen Huang et al.}\\
\emph{Attention Sinks and Outliers in Attention Residuals}.\\
arXiv:2605.17887v2, revised 24 September 2026.
\medskip
\begin{itemize}
\item Paper: \href{https://arxiv.org/abs/2605.17887v2}{arxiv.org/abs/2605.17887v2}
\item Code linked by the authors: \href{https://github.com/robinzixuan/OASIS}{github.com/robinzixuan/OASIS}
\item Companion Chinese notes: \texttt{reading-notes.zh.md}
\item Original figures reproduced with paper/page attribution.
\item Numerical examples and explicitly labeled interpretations are reading analysis.
\end{itemize}
\papersource{All experimental numbers in this deck are from the versioned paper. Main talk: slides 1--21; details follow.}
\end{frame}

\appendix

\begin{frame}{Backup: matched CPT supports an architectural effect}
\begin{center}
\begin{tabular}{llrrr}
\toprule
Backbone & Method & Kurtosis $\downarrow$ & Max norm $\downarrow$ & W8A8 PPL $\downarrow$\\
\midrule
LLaMA-3.2-1B & Backbone + CPT & 3.07 & 5.20 & 9.85\\
 & AttnResidual & 91.71 & 84.43 & 30.19\\
 & OASIS & 3.02 & 12.25 & 8.16\\
Qwen3-0.6B & Backbone + CPT & 3.16 & 4.77 & 11.50\\
 & AttnResidual & 93.04 & 74.78 & 36.64\\
 & OASIS & 2.95 & 13.00 & 9.52\\
\bottomrule
\end{tabular}
\end{center}
CPT alone keeps the activation statistics close to the original backbone. OASIS improves low-bit PPL even though maximum magnitude remains higher than the CPT control.
\papersource{Paper, Table 5, PDF p. 23. Matched continued-pretraining setup; selected mean values.}
\end{frame}

\begin{frame}{Backup: clipping and gating remain complementary}
\begin{center}
\begin{tabular}{lrrr}
\toprule
W4A4 PPL $\downarrow$ & LLaMA-3.2-1B & Qwen3-0.6B & Phi-4\\
\midrule
Clipped Softmax & 26.81 & 39.89 & 112.18\\
Clipped AoS & 22.57 & 16.95 & 37.62\\
Gated Attention & 24.27 & 28.78 & 102.45\\
Gated AoS & 21.45 & 15.43 & 31.89\\
\bottomrule
\end{tabular}
\end{center}
\begin{itemize}
\item AoS improves the low-bit results of these existing stabilization designs.
\item FP16 rankings can differ: LLaMA Gated Attention is 8.26 PPL, versus 8.68 for Gated AoS.
\end{itemize}
\papersource{Paper, Table 1, PDF p. 8. WikiText-2; OmniQuant W4A4. Selected mean values.}
\end{frame}

\begin{frame}{Backup: sparsity and null capacity are distinct freedoms}
\begin{center}
\begin{tabular}{lrrr}
\toprule
LLaMA-3.2-1B & Kurtosis $\downarrow$ & Max norm $\downarrow$ & W8A8 PPL $\downarrow$\\
\midrule
Softmax & 91.706 & 84.433 & 30.189\\
Sparsemax & 87.322 & 79.991 & 19.814\\
Entmax15 & 13.134 & 29.381 & 19.600\\
OASIS & 3.024 & 12.250 & 8.162\\
\bottomrule
\end{tabular}
\end{center}
\begin{itemize}
\item Sparse alternatives improve some outlier and quantization metrics.
\item An explicit zero-valued null state also permits the total real mass to shrink.
\end{itemize}
\papersource{Paper, Table 6, PDF p. 23. Selected mean values.}
\end{frame}

\begin{frame}{Backup: centering changes competition with the null state}
For transformed branches $i,k\geq1$:
\[
\frac{\tilde\alpha_i}{\tilde\alpha_k}
=\exp(g_i^{\mathrm{old}}-g_k^{\mathrm{old}})\,
\exp[-\beta(\psi_i-\psi_k)].
\]
\begin{itemize}
\item The mean cancels when comparing two transformed branches.
\item It does not cancel relative to the fixed null logit or the unchanged initial-state logit.
\item With only one transformed branch, its centered statistic is zero.
\end{itemize}
\begin{block}{Stable implementation}
Include the null logit in log-sum-exp. Shifting only real logits changes the null probability.
\end{block}
\papersource{Reading derivation from Sec. 4.3, PDF p. 5.}
\end{frame}

\begin{frame}{Backup: the pathology score has assumed dominance}
\[
\mathcal S_t=L_t+\lambda_1C_t+\lambda_2E_t,\qquad\lambda_1,\lambda_2\geq0
\]
\begin{itemize}
\item $L_t$: irrelevant-token leakage; $C_t$: peak joint routing probability.
\item $E_t=-H(\alpha_t)-\sum_\ell\alpha_t^{(\ell)}H(p_t^{(\ell)})$: negative joint entropy.
\item Proposition 5.1 assumes all three AttnResidual terms are at least their vanilla counterparts, then sums them.
\end{itemize}
\medskip
Sink aggregation is a convex average:
\[
\Sigma_t=\sum_\ell\alpha_t^{(\ell)}\sigma_t^{(\ell)},\qquad
\min_\ell\sigma_t^{(\ell)}\leq\Sigma_t\leq\max_\ell\sigma_t^{(\ell)}.
\]
It approaches a selected branch's sink mass if depth routing concentrates there.
\papersource{Paper, Definitions 5.1--5.2, Proposition 5.1 and Lemma 5.2, PDF pp. 5--6; Assumption D.4, p. 22.}
\end{frame}

\begin{frame}{Backup: standard adaptation and inference settings}
\begin{center}
\begin{tabular}{ll}
\toprule
Setting & Reported value\\
\midrule
Continued pretraining & 500 steps; sequence 2048; block size 512\\
Learning rate & $5\times10^{-5}$; linear schedule; 100 warmup steps\\
Batch / accumulation & 1 / 64\\
Gradient clipping / weight decay & 1.0 / 0.1\\
Coupling parameter & $\beta=\mathrm{softplus}(\theta)$; initialize $\theta=-5$\\
Hardware & $4\times$ H100 80GB\\
Generation & Temperature 0.6; top-p 0.95; max 4096 tokens\\
\bottomrule
\end{tabular}
\end{center}
Long-context results use models trained with maximum length 12K. The paper gives less detail on the budget for that separate experiment.
\papersource{Paper, Appendix E, PDF pp. 22--23; Sec. 6.4, PDF pp. 9--10.}
\end{frame}

\end{document}