\documentclass[aspectratio=169,11pt]{beamer}
\usepackage[T1]{fontenc}
\usepackage[utf8]{inputenc}
\usepackage{lmodern}
\usepackage{amsmath,amssymb,graphicx,booktabs}

\definecolor{AcademicBlue}{RGB}{26,57,86}
\definecolor{AcademicTeal}{RGB}{0,105,112}
\setbeamercolor{structure}{fg=AcademicBlue}
\setbeamercolor{frametitle}{fg=AcademicBlue,bg=white}
\setbeamercolor{title}{fg=AcademicBlue}
\setbeamercolor{alerted text}{fg=AcademicTeal}
\setbeamercolor{block title}{fg=white,bg=AcademicBlue}
\setbeamercolor{block body}{fg=black,bg=AcademicBlue!5}
\setbeamertemplate{navigation symbols}{}
\setbeamertemplate{itemize items}[circle]
\setbeamertemplate{footline}{%
  \hfill\usebeamerfont{page number in head/foot}%
  \insertframenumber\,/\,\inserttotalframenumber\hspace{4mm}\vspace{3mm}}
\setbeamersize{text margin left=8mm,text margin right=8mm}
\setbeamerfont{frametitle}{size=\Large,series=\bfseries}
\setbeamerfont{title}{size=\LARGE,series=\bfseries}
\newcommand{\papersource}[1]{\par\smallskip{\scriptsize\color{gray}#1\par}}

\title[Static Attention Heads]{When Can Attention Heads Be Statically Defined?}
\subtitle{Selective Attention Freezing (SAF): fixed attention weights, live value mixing}
\author{Paper authors: Weixian Waylon Li, Yintao Tai, Marcio Fonseca, Shay B. Cohen}
\date{arXiv:2609.34650v1 \quad 10 November 2026}

\begin{document}

\begin{frame}
  \titlepage
  \vspace{-0.6em}
  \centering
  \scriptsize Publication not found as of 2026-11-10\quad|
  \href{https://github.com/waylonli/Selective-Attention-Freezing}{Official code}\quad|
  \href{https://arxiv.org/abs/2609.34650v1}{Versioned paper}
\end{frame}

\begin{frame}{A subset of heads may repeat the same positional pattern}
  \begin{itemize}
    \item Ordinary attention recomputes input-dependent $QK^\top$ scores and softmax for every head and sequence.
    \item Some heads' attention maps change little across different inputs.
    \item If a map is nearly content-free, repeating its score and softmax work may be unnecessary.
  \end{itemize}
  \begin{block}{Research question}
    Which heads can use a fixed causal pattern, and can a mixed layer turn that change into real training and inference savings?
  \end{block}
  \papersource{Paper, Sec. 1, PDF pp. 1--2.}
\end{frame}

\begin{frame}{SAF freezes the weights, not the token mixing}
  \centering
  \includegraphics[width=0.96\linewidth,height=0.70\textheight,keepaspectratio]{figures/attn-prior-fig.pdf}
  \begin{itemize}
    \item Ordinary heads retain $Q,K,V$ and input-dependent attention.
    \item Replaced heads use a fixed causal $P$ but still mix the current input's $V$ vectors.
    \item Value and output projections keep training.
  \end{itemize}
  \papersource{Paper, Fig. 1, PDF p. 2; Sec. 3, PDF p. 3.}
\end{frame}

\begin{frame}{One calibration event defines the frozen heads}
  \begin{enumerate}
    \item Train with ordinary attention to a chosen checkpoint, usually the midpoint.
    \item Run a small calibration set without updating weights or advancing the training sampler.
    \item Rank heads by attention-map variance; propose the lowest-variance heads.
    \item Fit each mean map to a compact positional pattern and reject fits above 0.2 nats per row.
    \item Install the fixed patterns and resume training with the same model and optimizer state.
  \end{enumerate}
  \papersource{Paper, Sec. 3.1--3.3 and Algorithm 1, PDF pp. 3--5, 16.}
\end{frame}

\begin{frame}{Variance ranks how well one fixed map reconstructs a head}
  For head $(\ell,h)$, calibration estimates
  \[
    \widehat{\mathbf A}_{\ell h}=\frac{1}{N}\sum_{n=1}^N\mathbf A_{\ell h}^{(n)},\qquad
    s_{\ell h}^{\rm var}=\frac{1}{(N-1)T^2}\sum_{n=1}^N
    \left\|\mathbf A_{\ell h}^{(n)}-\widehat{\mathbf A}_{\ell h}\right\|_F^2.
  \]
  \begin{itemize}
    \item Choose the $k=\operatorname{round}(rLH)$ lowest-scoring heads across layers.
    \item This measures attention-map variation, not whether the head is functionally disposable.
    \item At 124M and 4K, 30--31 of 36 quarter-training selections remain selected at the midpoint.
  \end{itemize}
  \papersource{Paper, Eq. (1), Sec. 3.1, PDF p. 3; Appendix B.2, PDF p. 20.}
\end{frame}

\begin{frame}{The post-softmax mean is the best fixed map for squared error}
  \[
    \mathcal R(\mathbf P)=\frac{1}{T^2}\mathbb E\|\mathbf A-\mathbf P\|_F^2
    =\mathcal R(\mathbb E[\mathbf A]) +
    \frac{1}{T^2}\|\mathbf P-\mathbb E[\mathbf A]\|_F^2.
  \]
  \begin{itemize}
    \item The probability mean is causal and row-normalised, so it is a valid pattern.
    \item Attention variance estimates the minimum reconstruction error; this links the choice of pattern and selector.
    \item This argument concerns attention reconstruction, not language-model loss.
    \item Empirically, the mean is best or tied at low replacement rates; sampled Gaussian and Dirichlet patterns do not improve on it.
  \end{itemize}
  \papersource{Paper, Appendix B.1, PDF p. 18; Fig. 3, PDF p. 6.}
\end{frame}

\begin{frame}{Two length-$T$ vectors encode a causal pattern}
  Let $\alpha(j)$ encode absolute key position and $\rho(i-j)$ encode relative distance:
  \[
    \widehat P(i,j)=\begin{cases}
      \exp(\alpha(j)+\rho(i-j))/Z(i), & j\le i,\\
      0, & j>i,
    \end{cases}
    \quad Z(i)=\sum_{k=1}^{i}\exp(\alpha(k)+\rho(i-k)).
  \]
  \begin{itemize}
    \item $Z(i)$ normalises each causal query row.
    \item Fit by row-averaged cross-entropy to the empirical mean pattern.
    \item Storage falls from $O(T^2)$ to $O(T)$ per replaced head.
  \end{itemize}
  \papersource{Paper, Eqs. (3)--(4), PDF p. 4; fit tolerance and settings, Appendix A.2, p. 16.}
\end{frame}

\begin{frame}{The fused kernel reconstructs tiles and keeps head order}
  \begin{columns}[T,onlytextwidth]
    \column{0.49\textwidth}
      \centering
      \includegraphics[height=0.73\textheight,width=\linewidth,keepaspectratio]{figures/compact-kernel.pdf}
    \column{0.51\textwidth}
      \begin{itemize}
        \item One layer launch handles ordinary and fixed-pattern heads.
        \item Ordinary heads use FlashAttention-2.
        \item Fixed heads reconstruct causal tiles in registers, then multiply by live $V$ tiles.
        \item Training skips replaced-head $Q,K$ projections, score computation and softmax.
      \end{itemize}
  \end{columns}
  \papersource{Paper, Fig. 2 and Sec. 3.3, PDF pp. 5, 36--37.}
\end{frame}

\begin{frame}{The evaluation separates model quality from kernel speed}
  \small
  \begin{tabular}{p{0.19\linewidth}p{0.33\linewidth}p{0.37\linewidth}}
    \toprule
    Study & Model and training & Main question \\
    \midrule
    Controlled pretraining & 124M; 12 layers, 12 heads; FineWeb-Edu; 2.4576B tokens & Pattern, selector, rate, schedule, pruning controls \\
    Longer context & 124M at 8K and 16K & PPL and systems behavior across context \\
    Scale-up & 1B; 8K; 19.667B tokens; four GH200 GPUs & PPL and distributed update time \\
    Pretrained transfer & Qwen3-4B; no additional training & Zero-shot tasks and GQA prefill \\
    \bottomrule
  \end{tabular}
  \vspace{0.5em}
  \begin{itemize}
    \item Calibration is separated from final PPL reporting.
    \item Main speedups are paired per-update measurements; 124M quality and runtime use separate runs.
  \end{itemize}
  \papersource{Paper, Sec. 4, PDF pp. 5--6; Appendix C, PDF pp. 22--35.}
\end{frame}

\begin{frame}{The mean pattern edges out alternatives at low replacement rates}
  \begin{columns}[T,onlytextwidth]
    \column{0.53\textwidth}
      \centering
      \includegraphics[width=\linewidth,height=0.67\textheight,keepaspectratio]{figures/prior_quality.pdf}
    \column{0.47\textwidth}
      \begin{itemize}
        \item At 25\% replacement: mean $+0.662\%$, sharp mean $+0.665\%$ PPL.
        \item At 50\%: mean $+2.649\%$, sharp mean $+2.780\%$.
        \item Gaussian, Dirichlet and structured-random controls are not better than the mean.
        \item These figure values are a pattern comparison, distinct from the repeated three-seed main results.
      \end{itemize}
  \end{columns}
  \papersource{Paper, Fig. 3, Sec. 5.1, PDF p. 6.}
\end{frame}

\begin{frame}{Low-variance heads are often early-layer and diffuse}
  \begin{columns}[T,onlytextwidth]
    \column{0.57\textwidth}
      \centering
      \includegraphics[width=\linewidth,height=0.70\textheight,keepaspectratio]{figures/head_selection_profiles.pdf}
    \column{0.43\textwidth}
      \begin{itemize}
        \item At 25\%, selected heads cluster in earlier layers.
        \item Layer-matched normalized entropy: 0.837 selected vs. 0.678 other.
        \item Recent-64-token mass: 19.0\% vs. 40.7\%.
        \item Variance still predicts local replacement cost after controlling for layer and entropy (partial Spearman 0.543).
      \end{itemize}
  \end{columns}
  \papersource{Paper, Appendix B.2, Fig. 7, PDF p. 20.}
\end{frame}

\begin{frame}{At 124M, 25\% replacement is the quality-oriented setting}
  \small
  \begin{tabular}{lrrrr}
    \toprule
    Context & PPL 25\% & PPL 50\% & Speed 25\% & Speed 50\% \\
    \midrule
    4K  & $0.768\pm0.067\%$ & $2.492\pm0.083\%$ & $1.056\times$ & $1.119\times$ \\
    8K  & $0.578\pm0.011\%$ & $2.058\pm0.066\%$ & $1.030\times$ & $1.101\times$ \\
    16K & $0.450\pm0.012\%$ & $1.792\pm0.112\%$ & $1.025\times$ & $1.070\times$ \\
    \bottomrule
  \end{tabular}
  \vspace{0.7em}
  \begin{block}{Interpretation}
    More replacement usually saves more update time while increasing PPL cost. Peak memory falls by 0.82--1.96\% at 25\% and 1.57--3.04\% at 50\%. Each context uses its own trained model and fitted patterns.
  \end{block}
  \papersource{Paper, Table 2a, PDF p. 7. PPL changes are relative percentages; errors are SE over three seeds.}
\end{frame}

\begin{frame}{Measured speedups depend on the workload}
  \centering
  \includegraphics[width=0.92\linewidth,height=0.55\textheight,keepaspectratio]{figures/systems_across_stages.pdf}
  \begin{itemize}
    \item More replaced heads speed pretraining; short-task fine-tuning may slow.
    \item At batch 64, causal prefill gains are $1.09$--$1.12\times$ at 25\% and $1.20$--$1.24\times$ at 50\% across 512--16K tokens.
  \end{itemize}
  \papersource{Paper, Fig. 4 and Table 2b, pp. 7--8; Appendix D.3, p. 38.}
\end{frame}

\begin{frame}{Token and time budgets favor different methods}
  \small
  \begin{tabular}{p{0.42\linewidth}p{0.24\linewidth}p{0.24\linewidth}}
    \toprule
    Operation (25\% heads) & Matched tokens & Matched time \\
    \midrule
    Mean + variance selection & $0.663\pm0.018\%$ & $0.456\pm0.108\%$ \\
    Prune the same heads & $0.806\pm0.018\%$ & $0.113\pm0.169\%$ \\
    Mean + random heads & $1.282\pm0.056\%$ & $1.243\pm0.112\%$ \\
    \bottomrule
  \end{tabular}
  \begin{itemize}
    \item Same-head SAF beats deletion at equal tokens.
    \item At equal time, pruning completes more updates and reaches lower PPL.
    \item Gate-Taylor pruning is stronger than same-head pruning in the later-intervention study.
  \end{itemize}
  \papersource{Paper, Table 1, PDF p. 7; Appendix C.3.4, pp. 28--29. Mean $\pm$ SE over three seeds.}
\end{frame}

\begin{frame}{After MQAR adaptation, SAF handles more pairs at fixed length}
  \begin{columns}[T,onlytextwidth]
    \column{0.59\textwidth}
      \centering
      \includegraphics[width=\linewidth,height=0.66\textheight,keepaspectratio]{figures/mqar_generalisation.pdf}
    \column{0.41\textwidth}
      \begin{itemize}
        \item All models adapt on 8 pairs in 512-token sequences.
        \item Test increases pair count while keeping length at 512 tokens.
        \item At 64 pairs: SAF $54.42\pm8.70\%$; ordinary $26.19\pm2.75\%$; variance-pruned $29.03\pm1.42\%$; Gate-Taylor $26.59\pm4.41\%$.
        \item Length-only tests favor pruning; immediate replacement is less consistent.
      \end{itemize}
  \end{columns}
  \papersource{Paper, Fig. 5, PDF p. 8; Appendix C.3.6, PDF pp. 31--32.}
\end{frame}

\begin{frame}{On Qwen3-4B, variance selection preserves more zero-shot accuracy}
  \begin{columns}[T,onlytextwidth]
    \column{0.57\textwidth}
      \centering
      \includegraphics[width=\linewidth,height=0.65\textheight,keepaspectratio]{figures/qwen_pretrained_tasks.pdf}
    \column{0.43\textwidth}
      \begin{itemize}
        \item No additional training; five zero-shot tasks.
        \item Variance selection beats forward-KL selection in all 15 task-rate comparisons.
        \item Mean accuracy change at 10\% replacement: $-0.37$ pp with variance, $-3.27$ pp with forward KL.
        \item At 30\%, variance selection still averages $-4.79$ pp.
      \end{itemize}
  \end{columns}
  \papersource{Paper, Fig. 6 and Appendix C.6, PDF pp. 9, 35.}
\end{frame}

\begin{frame}{1B: the update trade-off extends to four GPUs}
  \begin{columns}[T,onlytextwidth]
    \column{0.53\textwidth}
      \begin{block}{Quality and update speed}
        Ordinary attention: PPL 12.446.\par
        SAF 25\%: PPL 12.509 (+0.507\%), update speedup $1.068\times$.\par
        SAF 50\%: PPL 12.698 (+2.027\%), update speedup $1.167\times$.\par
        Gate-Taylor pruning 25\%: PPL 12.529.
      \end{block}
      \begin{itemize}
        \item 8K context; 19.667B training tokens; four GH200 GPUs.
        \item Distributed timing includes gradient communication.
      \end{itemize}
    \column{0.47\textwidth}
      \begin{block}{Scale-up caveat}
        One pretraining seed. Three fine-tuning seeds measure task-training variation, not pretraining variation.
      \end{block}
      \begin{itemize}
        \item 25\% calibration and fitting took about 24 minutes.
      \end{itemize}
  \end{columns}
  \papersource{Paper, Table 3, PDF p. 9; Appendix C.3.1/C.5, pp. 27, 34.}
\end{frame}

\begin{frame}{The speed claim has concrete boundaries}
  \begin{itemize}
    \item The fixed path still computes $\widehat P V$ at $O(T^2d_h)$; it removes score/softmax work and lowers pattern storage.
    \item Main speedups are per-update measurements and exclude calibration/fitting. For the reported 124M B=8 control, intervention took about 55 s; the full matched-token path was 184.7 min for SAF vs. 188.7 min for ordinary attention.
    \item Batch-one causal prefill at 4K was near break-even: median $0.969\times$ at 25\% and $0.972\times$ at 50\% due to launch overhead.
    \item Short-task fine-tuning did not consistently speed up; Qwen's 10\% replacement still reduced mean accuracy by 0.37 pp.
  \end{itemize}
  \papersource{Paper, Sec. 3.3 and Appendix C.3.1, D.3, PDF pp. 5, 27, 38; Table 2b, p. 7; Appendix C.6, p. 35.}
\end{frame}

\begin{frame}{Take-home: freeze stable routing, keep useful value mixing}
  \begin{enumerate}
    \item Low attention variance is a practical signal for heads whose routing can be approximated by a fixed causal map.
    \item The post-softmax mean plus a compact position-distance fit gives a simple static pattern.
    \item SAF trades some quality for lower update cost; 25\% is the more quality-oriented tested point.
    \item Retained token mixing can beat deleting the same heads at matched tokens and in fixed-length associative recall, but matched-time pruning can be better.
  \end{enumerate}
  \papersource{Synthesis of Paper, Secs. 3--6, PDF pp. 3--9.}
\end{frame}

\begin{frame}{References and links}
  \begin{itemize}
    \item Li, Tai, Fonseca, and Cohen. \emph{When Can Attention Heads Be Statically Defined?} arXiv:2609.34650v1, 2026. \href{https://arxiv.org/abs/2609.34650v1}{Paper}.
    \item Author-maintained implementation: \href{https://github.com/waylonli/Selective-Attention-Freezing}{Selective-Attention-Freezing on GitHub}.
    \item Related execution baselines discussed by the paper: FlashAttention-2; same-head and Gate-Taylor pruning.
  \end{itemize}
  \vfill
  \begin{block}{Publication status}
    No formal publication record found as of 2026-11-10; the author page labels this work “Preprint 2026”.
  \end{block}
  \papersource{The arXiv version, author page and code repository were checked on 2026-11-10.}
\end{frame}

% Backup slides
\appendix
\begin{frame}{Backup: replacement timing exposes the value of adaptation}
  \begin{itemize}
    \item Replacing 25\% of heads at 25\%, 40\%, 50\%, 60\%, 75\%, and 100\% of training gives PPL increases of 0.724, 0.679, 0.662, 0.711, 0.936, and 8.168\%.
    \item Mid-training replacement leaves room to adapt. Replacing after training leaves no recovery updates.
    \item One-time midpoint replacement matches or beats the tested gradual and adaptive schedules at comparable final rates.
  \end{itemize}
  \papersource{Paper, Appendix C.1, PDF pp. 22--23; Appendix B.3, p. 21.}
\end{frame}

\begin{frame}{Backup: the static matrix saves storage, not all attention arithmetic}
  \begin{itemize}
    \item For the 4K, 36-head, 10-layer representation comparison, compact buffers use 0.133 GB vs. 8.060 GB in the dense implementation (60.6$\times$ less allocated pattern storage).
    \item An ideally packed dense allocation for only those 36 selected matrices is 2.416 GB.
    \item Dense vs. compact mean patterns change PPL by less than 0.014\% in the tested absolute-position and RoPE settings.
    \item Fixed-pattern token mixing still costs $O(T^2d_h)$.
  \end{itemize}
  \papersource{Paper, Appendix D.1--D.2, Tables 24--25, PDF pp. 36--37.}
\end{frame}

\end{document}