% Research discussion deck based on arXiv:2610.13674v1.
% Build: python3 build_slides.py slides.en.tex
\documentclass[aspectratio=169,11pt]{beamer}
\usepackage[T1]{fontenc}
\usepackage[utf8]{inputenc}
\usepackage{lmodern}
\usepackage{amsmath,amssymb,graphicx,booktabs}

\definecolor{AcademicBlue}{RGB}{26,57,86}
\definecolor{AcademicTeal}{RGB}{0,105,112}
\setbeamercolor{structure}{fg=AcademicBlue}
\setbeamercolor{frametitle}{fg=AcademicBlue,bg=white}
\setbeamercolor{title}{fg=AcademicBlue}
\setbeamercolor{alerted text}{fg=AcademicTeal}
\setbeamercolor{block title}{fg=white,bg=AcademicBlue}
\setbeamercolor{block body}{fg=black,bg=AcademicBlue!5}
\setbeamertemplate{navigation symbols}{}
\setbeamertemplate{itemize items}[circle]
\setbeamertemplate{footline}{%
  \hfill\usebeamerfont{page number in head/foot}%
  \insertframenumber\,/\,\inserttotalframenumber\hspace{4mm}\vspace{3mm}}
\setbeamersize{text margin left=8mm,text margin right=8mm}
\setbeamerfont{frametitle}{size=\Large,series=\bfseries}
\setbeamerfont{title}{size=\LARGE,series=\bfseries}
\newcommand{\papersource}[1]{\par\smallskip{\scriptsize\color{gray}#1\par}}
\newcommand{\R}{\mathbb R}
\newcommand{\softmax}{\operatorname{softmax}}
\title[OAL]{Outlier-Aware Attention Linearization}
\subtitle{Not All Dimensions Are Equal:\\Grouped Quadratic Kernels for Pretrained Attention}
\author{Paper authors: Weiqi Sun, Qianyue Wu,\\Wenjia Wang, Tianyang Hu}
\date{arXiv:2610.13674v1 \enspace | \enspace Research discussion, 19 October 2026}

\begin{document}

% 1
\begin{frame}
  \titlepage
  \vspace{-4mm}
  \begin{center}
  \alert{Share coefficients according to query--key contribution scales.}\\[2mm]
  {\small A 25-minute discussion; technical details in backup slides.}
  \end{center}
\end{frame}

% 2
\begin{frame}{Can a small kernel preserve pretrained attention?}
  \[
    y_i=\frac{\sum_{j\le i}e^{q_i^\top k_j/\sqrt{d_h}}v_j}
                 {\sum_{j\le i}e^{q_i^\top k_j/\sqrt{d_h}}}
    \quad\longrightarrow\quad
    \widehat y_i=\frac{\sum_{j\le i}\kappa(q_i,k_j)v_j}
                           {\sum_{j\le i}\kappa(q_i,k_j)}.
  \]
  \begin{itemize}
    \item Separable kernels permit prefix accumulation without an $N\times N$ matrix.
    \item Limited adaptation leaves little room to repair a poor approximation.
    \item A global quadratic $a+bs+cs^2$ shares coefficients across all coordinate contributions.
  \end{itemize}
  \begin{block}{The paper's hypothesis}
    Large differences in coordinate contributions call for nonuniform coefficient sharing.
  \end{block}
  \papersource{Paper, Secs. 1--3, PDF pp. 1--4. $N$: sequence length; $d_h$: Q/K head dimension.}
\end{frame}

% 3
\begin{frame}{A few coordinates dominate query--key contributions}
  \centering
  \includegraphics[width=0.99\linewidth,height=3.65cm,keepaspectratio]{figures/fig-01-qk-outliers.pdf}
  \[
    \chi_{p,r}=\frac1{|\Omega_p|}\sum_{(i,j)\in\Omega_p}
       \left|\frac{q_{p,i,r}k_{p,j,r}}{\sqrt{d_h}}\right|,
    \qquad\chi_r=\frac1P\sum_p\chi_{p,r}.
  \]
  \raggedright
  Post-RoPE Q/K; causal pairs; equal weight per prompt. Coordinate products combine information from both Q and K.
  \papersource{Paper, Fig. 1 and Eq. (2), PDF p. 4. Highlighted coordinates: $r=26,58$.}
\end{frame}

% 4
\begin{frame}{Approximating only 5.5\% of dimensions retains large error}
  \[
  \widetilde\kappa_{ij}=\widehat\kappa_{ij,\mathcal S}
  \kappa^{\rm exact}_{ij,\overline{\mathcal S}},
  \qquad R_{\mathcal S}=\mathcal E_{\mathcal S}/\mathcal E_{\rm full}.
  \]
  \begin{center}
  \begin{tabular}{lrr}
  \toprule
  Method & High-contribution $R_{\mathcal S}$ & Gaussian control\\
  \midrule
  Performer & 37.8\% & 4.7\%\\
  SpectraFormer & 58.1\% & 18.1\%\\
  DiJiang & 64.8\% & 19.4\%\\
  TaylorShift & 63.9\% & 13.6\%\\
  \bottomrule
  \end{tabular}
  \end{center}
  \begin{itemize}
  \item 147 selected dimensions across 42 heads, each of dimension 64.
  \item Separate matched-coordinate controls: $p=1/1001$ for every method.
  \end{itemize}
  \papersource{Paper, Table 1, PDF p. 5; App. A.2, pp. 15--16. Output MSE after normalization; not an additive error decomposition.}
\end{frame}

% 5
\begin{frame}{Representative-head diagnostics support fixed groups}
  \begin{columns}[T]
  \begin{column}{0.44\textwidth}
    \centering
    \includegraphics[width=0.86\linewidth,height=3.65cm,keepaspectratio]{figures/top5_concentration.pdf}
    \par\smallskip
    \raggedright Top-5 concentration is much stronger than in isotropic Gaussian controls.
  \end{column}
  \begin{column}{0.53\textwidth}
    \centering
    \includegraphics[width=0.97\linewidth,height=3.65cm,keepaspectratio]{figures/stability_singlehead_input.pdf}
    \par\smallskip
    \raggedright Input stability: $\rho=0.991$, Top-5 overlap $5/5$. Dominant dimensions persist over 128--4096-token prefixes.
  \end{column}
  \end{columns}
  \papersource{Paper, Figs. 5--6 and Table 4, PDF pp. 15, 17. Single-head diagnostics; GPT-2 evidence in Fig. 4, p. 14.}
\end{frame}

% 6
\begin{frame}{Relax sharing before deciding which parameters to tie}
  Define the coordinate contribution $x_r=q_rk_r/\sqrt{d_h}$, so $s=\sum_r x_r$.
  \begin{block}{Global quadratic}
  \[
  a+bs+cs^2=a+\sum_r bx_r+\sum_{r,t}cx_rx_t.
  \]
  \end{block}
  \begin{block}{Dimension-aware quadratic}
  \[
  \kappa_{\rm dim}(q,k)=a+\sum_r b_rx_r+\sum_{r,t}C_{rt}x_rx_t,
  \qquad C=C^\top.
  \]
  \end{block}
  Linear weights $b_r$; squares $C_{rr}$; cross-coordinate interactions $C_{rt}$.
  \papersource{Paper, Sec. 4.1, Eq. (3), PDF p. 5. Setting every $b_r=b$ and $C_{rt}=c$ recovers the global kernel.}
\end{frame}

% 7
\begin{frame}{Contribution groups concentrate flexibility in the tail}
  \begin{columns}[c]
  \begin{column}{0.53\textwidth}
  \includegraphics[width=\linewidth,height=4.75cm,keepaspectratio]{figures/qwen_qk_contribution_grouped.pdf}
  \end{column}
  \begin{column}{0.44\textwidth}
  \[
    s_g=\sum_{r\in\mathcal G_g}x_r
  \]
  \begin{itemize}
    \item Sort by contribution; inspect gaps in the high-contribution tail.
    \item Keep gaps stable under prompt-level bootstrap.
    \item Share linear coefficients within groups and quadratic coefficients across group pairs.
  \end{itemize}
  \end{column}
  \end{columns}
  \smallskip
  Qwen: 2--5 groups/head. Llama: 2--4. Groups stay fixed during calibration and model adaptation.
  \papersource{Paper, Fig. 2(a), PDF p. 7; App. B, pp. 17--19. Example: Qwen Layer 4, Head 0; group sizes 60, 2, 1, 1.}
\end{frame}

% 8
\begin{frame}{A squared quadratic form guarantees positive weights}
  Let $\bar{\mathbf s}=[1,s_1,\ldots,s_G]^\top$ and $\epsilon>0$; learn a lower-triangular $L$.
  \begin{block}{OAL kernel}
  \[
    \kappa_{\rm OAL}(q,k)
    =\epsilon+\bar{\mathbf s}^\top LL^\top\bar{\mathbf s}
    =\epsilon+\|L^\top\bar{\mathbf s}\|_2^2>0.
  \]
  \end{block}
  With $M=LL^\top$, map back to coordinate coefficients:
  \[
    a=\epsilon+M_{00},\qquad
    b_r=2M_{0,g(r)},\qquad C_{rt}=M_{g(r),g(t)}.
  \]
  \begin{itemize}
    \item Positive denominator for every nonempty causal prefix.
    \item $(G+1)(G+2)/2$ parameters per head: 3 at $G=1$, 21 at $G=5$.
    \item $G=1$ gives the learned global reference.
  \end{itemize}
  \papersource{Paper, Eq. (7), PDF p. 7; App. D, pp. 21--22. Pointwise positivity does not imply Mercer positive semidefiniteness.}
\end{frame}

% 9
\begin{frame}{HD evaluation moves token sums inside a key moment}
  For value channel $\ell$:
  \[
    \sum_j(q_i^\top k_j)^2 V_{j,\ell}
    =q_i^\top\underbrace{\left(\sum_j V_{j,\ell}k_jk_j^\top\right)}_{d_h\times d_h\text{ key moment}}q_i.
  \]
  Equivalently, with $(D_\ell)_{jj}=V_{j,\ell}$:
  \[
  [(QK^\top)\odot(QK^\top)]V_{:,\ell}
  =\operatorname{diag}\!\left(Q[K^\top D_\ell K]Q^\top\right).
  \]
  \begin{itemize}
    \item Accumulate key moments once; read them with each query.
    \item Evaluate only diagonal entries; never build the outer $N\times N$ matrix.
    \item Restrict the inner sum to $j\le i$ for causal attention.
  \end{itemize}
  \papersource{Paper, Eq. (4), PDF p. 5; derivation in App. D, p. 21. HD = Hadamard--diagonal.}
\end{frame}

% 10
\begin{frame}{Three prefix statistics evaluate the chosen kernel exactly}
  Augment values: $u_j=[v_j;1]\in\R^{d_v+1}$.
  \[
  H_{0,i}=\sum_{j\le i}u_j,\quad
  H_{1,i,r}=\sum_{j\le i}k_{j,r}u_j,\quad
  H_{2,i,rt}=\sum_{j\le i}k_{j,r}k_{j,t}u_j.
  \]
  \[
  \begin{aligned}
  \widetilde u_i
    &=aH_{0,i}+\frac1{\sqrt{d_h}}\sum_r b_rq_{i,r}H_{1,i,r}\\
    &\quad+\frac1{d_h}\sum_{r,t}C_{rt}q_{i,r}q_{i,t}H_{2,i,rt},\\[1mm]
  \widehat y_i&=\widetilde u_{i,1:d_v}/\widetilde u_{i,d_v+1}.
  \end{aligned}
  \]
  \alert{The last channel accumulates the normalization denominator.}\\
  Approximation comes from replacing softmax; the recurrence adds none.
  \papersource{Paper, Eqs. (5)--(6), PDF p. 6; Algorithm 1, p. 8; equivalence in Eq. (25), p. 21.}
\end{frame}

% 11
\begin{frame}{Grouping, output calibration, and adaptation are separate}
  \begin{block}{1. Group dimensions from pretrained statistics}
    40 short prompts; per-layer, per-query-head assignments.
  \end{block}
  \begin{block}{2. Fit the normalized attention output}
  \[
    L^\star=\arg\min_L\frac1{|\mathcal C|}\sum_{x\in\mathcal C}\sum_i
    \|\widehat y_i(L;x)-y_i^{\rm softmax}(x)\|_2^2.
  \]
    Frozen Q/K/V and groups; 8 training + 2 validation WikiText-2 blocks.
  \end{block}
  \begin{block}{3. Adapt the converted model}
    Jointly optimize calibrated $L$ and LoRA on WikiText-103 next-token prediction; keep groups fixed.
  \end{block}
  \papersource{Paper, App. B, PDF pp. 19--20, Eq. (20); App. F.2, p. 25. Blocks contain 4096 tokens.}
\end{frame}

% 12
\begin{frame}{A compact kernel can require a large recurrent state}
  \begin{center}
  \begin{tabular}{ll}
  \toprule
  Quantity & HD implementation\\
  \midrule
  Time & $O(Nd_h^2(d_v+1))$\\[1mm]
  Logical state & $O(d_h^2(d_v+1))$\\
  \bottomrule
  \end{tabular}
  \end{center}
  Symmetric packing reduces the state to
  \[
    \left(1+d_h+\frac{d_h(d_h+1)}2\right)(d_v+1)\ \text{entries}.
  \]
  \begin{block}{At $d_h=d_v=64$ and $G=5$}
    139,425 state entries; 21 learned coefficients.\\
    Grouping leaves the coordinate-pair states intact.
  \end{block}
  \papersource{Paper, Sec. 4.4, PDF pp. 7--8; App. D, pp. 21--22. State entries calculated from the paper's formula; not peak GPU memory.}
\end{frame}

% 13
\begin{frame}{Frozen Q/K/V isolate approximation fidelity}
  \begin{itemize}
    \item Qwen2.5-0.5B Base; all query heads in zero-based Layers 4, 12, 20.
    \item 16 held-out WikiText-2 sequences, each 4096 tokens.
    \item Evaluate before and after the frozen output projection $W_o$.
  \end{itemize}
  \[
    \mathrm{MSE}=\frac1B\sum_b\frac{\|\widehat Y_b-Y_b\|_F^2}{n_b},
    \quad
    \mathrm{RelMSE}=\frac1B\sum_b\frac{\|\widehat Y_b-Y_b\|_F^2}{\|Y_b\|_F^2}.
  \]
  Here $n_b$ counts scalar output elements; $B=16$.
  \begin{block}{Aggregation matters}
    RelMSE normalizes by each sequence's exact output energy, then averages. After $W_o$, the denominator also uses the projected output.
  \end{block}
  \papersource{Paper, Sec. 5.1, PDF pp. 8--9; App. B--C, pp. 20--21. Lower is better for both metrics.}
\end{frame}

% 14
\begin{frame}{OAL improves all four frozen-output error metrics}
  \begin{center}
  \begin{tabular}{lrrrr}
  \toprule
  & \multicolumn{2}{c}{Before $W_o$} & \multicolumn{2}{c}{After $W_o$}\\
  \cmidrule(lr){2-3}\cmidrule(lr){4-5}
  Method & MSE & RelMSE & MSE & RelMSE\\
  \midrule
  Performer & 0.103 & 0.569 & 0.038 & 0.710\\
  SpectraFormer & 0.106 & 0.585 & 0.041 & 0.772\\
  DiJiang & 0.086 & 0.474 & 0.033 & 0.631\\
  TaylorShift & 0.088 & 0.490 & 0.034 & 0.649\\
  \midrule
  OAL & \textbf{0.078} & \textbf{0.432} & \textbf{0.031} & \textbf{0.595}\\
  \bottomrule
  \end{tabular}
  \end{center}
  \begin{block}{Relative to DiJiang}
    Pre-projection RelMSE: $-8.86\%$. Post-projection: $-5.71\%$.\\
    The abstract's $8.9\%$ refers to the pre-projection metric.
  \end{block}
  \papersource{Paper, Table 2, PDF p. 8. Percent reductions calculated from reported values. Learned Global is not included in this table.}
\end{frame}

% 15
\begin{frame}{Model conversion uses a matched, limited LoRA budget}
  \begin{center}
  \begin{tabular}{lll}
  \toprule
  Backbone & Replaced layers & Retained softmax\\
  \midrule
  Qwen2.5-0.5B & 3--20 & First/last 3 layers\\
  Llama-3.2-1B & 1--14 & First/last layer\\
  \bottomrule
  \end{tabular}
  \end{center}
  \begin{itemize}
    \item Same ranges for all converted methods; layer indices are zero-based.
    \item LoRA on Q/K/V/O in every layer, including retained softmax layers.
    \item Rank 8, $\alpha=16$, dropout 0.05; 512 updates, batch 1, length 4096.
    \item WikiText-103 next-token training; 2,097,152 input tokens; seed 42.
  \end{itemize}
  \alert{These are hybrid models: retained softmax layers still have quadratic prefill computation.}
  \papersource{Paper, App. F, PDF pp. 22--25. Softmax reference also receives LoRA. Best validation NLL selects the checkpoint.}
\end{frame}

% 16
\begin{frame}{Qwen: grouping improves every reported quality metric}
  \begin{center}
  \begin{tabular}{lrrrr}
  \toprule
  Method & PPL $\downarrow$ & PIQA $\uparrow$ & HellaSwag $\uparrow$ & ARC-E $\uparrow$\\
  \midrule
  Softmax (LoRA) & 10.89 & 70.18 & 51.38 & 62.63\\
  \midrule
  Performer & 37.03 & 57.78 & 30.37 & 38.93\\
  SpectraFormer & 34.78 & 59.19 & 31.77 & 41.84\\
  DiJiang & 36.05 & 59.90 & 31.47 & 41.41\\
  TaylorShift & 39.50 & 59.90 & 31.18 & 40.78\\
  Learned Global & 31.71 & 61.15 & 32.43 & 41.41\\
  OAL & \textbf{27.50} & \textbf{64.69} & \textbf{37.20} & \textbf{50.38}\\
  \bottomrule
  \end{tabular}
  \end{center}
  Against Learned Global: PPL $-13.28\%$; task gains $+3.54$, $+4.77$, $+8.97$ percentage points.
  \papersource{Paper, Table 3, PDF p. 10, Qwen2.5-0.5B. WikiText-103 test PPL; zero-shot normalized accuracy (\%). Bold: best converted model.}
\end{frame}

% 17
\begin{frame}{Llama: gains transfer, but their size depends on the task}
  \begin{center}
  \begin{tabular}{lrrrr}
  \toprule
  Method & PPL $\downarrow$ & PIQA $\uparrow$ & HellaSwag $\uparrow$ & ARC-E $\uparrow$\\
  \midrule
  Softmax (LoRA) & 8.19 & 74.92 & 62.84 & 59.43\\
  \midrule
  Performer & 50.39 & 57.07 & 27.42 & 32.58\\
  SpectraFormer & 45.09 & 59.36 & 30.41 & 36.49\\
  DiJiang & 56.27 & 58.92 & 28.75 & 36.32\\
  TaylorShift & 62.67 & 57.40 & 28.95 & 35.77\\
  Learned Global & 36.03 & 57.40 & 35.02 & 33.46\\
  OAL & \textbf{32.99} & \textbf{63.66} & \textbf{35.42} & \textbf{41.67}\\
  \bottomrule
  \end{tabular}
  \end{center}
  Against Learned Global: PPL $-8.44\%$; task gains $+6.26$, $+0.40$, $+8.21$ percentage points. One adaptation seed.
  \papersource{Paper, Table 3, PDF p. 10, Llama-3.2-1B. WikiText-103 test PPL; zero-shot normalized accuracy (\%). Bold: best converted model.}
\end{frame}

% 18
\begin{frame}{A small group budget captures most of the diagnostic gain}
  \begin{columns}[c]
  \begin{column}{0.57\textwidth}
  \includegraphics[width=\linewidth,height=4.8cm,keepaspectratio]{figures/qwen_group_count_sensitivity.pdf}
  \end{column}
  \begin{column}{0.40\textwidth}
  \begin{itemize}
    \item One head: Qwen Layer 4, Head 0.
    \item Fixed $G$: split at the largest $G-1$ contribution gaps.
    \item Same data and fitting protocol across budgets.
    \item More groups lower error, with diminishing returns.
  \end{itemize}
  \end{column}
  \end{columns}
  \smallskip
  This diagnostic differs from bootstrap-based automatic group selection.
  \papersource{Paper, Fig. 2(b), PDF p. 7; App. B, p. 20. Left axis: held-out output MSE; right axis: learned coefficients/head.}
\end{frame}

% 19
\begin{frame}{OAL tracks Global in cost; FlashAttention remains ahead}
  \centering
  \includegraphics[width=0.94\linewidth,height=4.65cm,keepaspectratio]{figures/oal_runtime_memory.pdf}
  \par\smallskip
  \raggedright
  {\small A100 40GB; forward only; BF16 inputs; $B=1$, $H_q=14$, $H_{kv}=2$, $d_h=d_v=64$. OAL $G=5$ and Global $G=1$ share the HD backend.}
  \papersource{Paper, Fig. 3, PDF p. 9; App. E, p. 22. Precision and optimization differ across other methods. Memory includes inputs and workspace.}
\end{frame}

% 20
\begin{frame}{The results establish fidelity gains under specific tradeoffs}
  \begin{itemize}
    \item \textbf{Quality:} OAL leads the evaluated converted methods, while LoRA-softmax stays substantially stronger.
    \item \textbf{Efficiency:} sequence-linear operator scaling coexists with expensive quadratic head states and slower measured execution than FlashAttention.
    \item \textbf{Comparison scope:} adapted small-parameter kernels; no MLP feature-map baseline such as Hedgehog.
    \item \textbf{Next discriminating test:} compare contribution, random, and uniform partitions at the same group budget.
  \end{itemize}
  \begin{block}{What to carry forward}
  Allocate approximation freedom according to pretrained representation statistics, then account separately for state and execution costs.
  \end{block}
  \papersource{Paper, Sec. 7, PDF p. 11; Apps. D--F, pp. 21--25. Final experiment proposal is this reading's analysis.}
\end{frame}

% 21
\begin{frame}{Three takeaways for attention conversion}
  \begin{enumerate}
    \item Contribution heterogeneity is associated with where linearization error concentrates.
    \item Grouped positive quadratics provide a compact, output-calibrated extension of global quadratic attention.
    \item Exact HD recurrence preserves the selected kernel; practical conversion quality and speed remain open challenges.
  \end{enumerate}
  \bigskip
  {\small Weiqi Sun, Qianyue Wu, Wenjia Wang, Tianyang Hu.\\
  \emph{Not All Dimensions Are Equal: Outlier-Aware Attention Linearization with Grouped Quadratic Kernels}.\\
  \href{https://arxiv.org/abs/2610.13674v1}{arXiv:2610.13674v1}, 9 October 2026.\\[2mm]
  Author-provided code: \href{https://github.com/wuqianyue09/OAL}{github.com/wuqianyue09/OAL}.}
  \papersource{Original paper figures and reported experimental values are attributed at their point of use.}
\end{frame}

\appendix

% 22
\begin{frame}{Backup: bootstrap support determines natural boundaries}
  Fix the contribution ordering $\pi$; search only the top-$\lceil\alpha d_h\rceil$ tail and its preceding boundary.
  \[
  \rho_k=\frac{\chi_{\pi_{k+1}}}{\max(\chi_{\pi_k},\varepsilon)},\quad
  R_k^{(b)}=\frac{\min_{r\in\mathcal U_k}\chi_r^{(b)}}
  {\max(\max_{r\in\mathcal L_k}\chi_r^{(b)},\varepsilon)}.
  \]
  \[
  S_k=\frac1B\sum_b\mathbf1[R_k^{(b)}\ge\tau],\qquad
  \text{keep if }\rho_k\ge\tau\ \text{and }S_k\ge\gamma.
  \]
  \begin{itemize}
    \item 40 prompts, $B=1000$ bootstrap replicates; $\tau=1.5$.
    \item Qwen: $(\alpha,\gamma)=(0.25,0.9)$; Llama: $(0.50,0.5)$.
    \item Require at least 2 groups. Fallback cuts prioritize $(S_k,\rho_k)$, even if natural thresholds fail. No explicit maximum group count.
  \end{itemize}
  \papersource{Paper, App. B, Eqs. (16)--(19), PDF pp. 18--19. Bootstrap resamples prompts, retaining the original coordinate ordering.}
\end{frame}

% 23
\begin{frame}{Backup: coefficient fitting uses disjoint data splits}
  \begin{center}
  \begin{tabular}{lll}
  \toprule
  WikiText-2 blocks & Role & Number of sequences\\
  \midrule
  0--15 & Frozen evaluation & 16\\
  16--23 & Coefficient training & 8\\
  24--25 & Coefficient validation & 2\\
  \bottomrule
  \end{tabular}
  \end{center}
  \begin{itemize}
    \item Adam, learning rate 0.02, FP32, clipping at 10; up to 1000 updates per initialization; 32 causal query positions/update.
    \item Three starts: learned-global embedding, Taylor, perturbed global.
    \item Validate every 100 updates; stop after 400 updates without improvement; choose the best checkpoint per head.
    \item Learned Global uses the same fitting data and optimization budget.
  \end{itemize}
  \papersource{Paper, App. B, PDF pp. 19--20. Each sequence has 4096 tokens; short prompts used for grouping are separate.}
\end{frame}

% 24
\begin{frame}{Backup: the baselines use adapted causal interfaces}
  \begin{itemize}
    \item \textbf{Performer:} 250 fixed positive orthogonal features/head; only LoRA is trained; FP64 features and prefix accumulation.
    \item \textbf{SpectraFormer:} OPRF--FastFoodL, 65 features; three trainable 64D vectors/layer, shared across heads; FP64 features and accumulation.
    \item \textbf{DiJiang:} DCT projection, feature softmax, 64 learned positive weights/head; FP32. Positional decay, GroupNorm, and output gate omitted.
    \item \textbf{TaylorShift:} normalized Q/K, fixed quadratic, learned temperature/head; causal aggregation; no extra output-amplitude scaling.
  \end{itemize}
  \papersource{Paper, App. F.1, PDF pp. 23--25. Comparisons preserve pretrained Q/K/V/O, RoPE, and GQA rather than original full-model recipes.}
\end{frame}

% 25
\begin{frame}{Backup: positive weights do not imply a Mercer kernel}
  \begin{block}{Teaching counterexample, not a paper experiment}
    For one-dimensional Q/K, take $\kappa(q,k)=\epsilon+(1-qk)^2$, $\epsilon>0$.
  \end{block}
  This fits the OAL form with
  \[
  M=\begin{bmatrix}1&-1\\-1&1\end{bmatrix}\succeq0.
  \]
  For the two inputs $q_1=1$ and $q_2=-1$, the Gram matrix is
  \[
  K=\begin{bmatrix}\epsilon&\epsilon+4\\\epsilon+4&\epsilon\end{bmatrix},
  \qquad\lambda(K)=\{2\epsilon+4,-4\}.
  \]
  Every weight is positive, but the Gram matrix is indefinite. HD evaluation requires separable algebraic terms, not Mercer positivity.
  \papersource{Paper, App. D, PDF p. 21 states the distinction. Counterexample derived for this discussion.}
\end{frame}

% 26
\begin{frame}{Backup: layer selection and task scoring affect interpretation}
  \centering
  \includegraphics[width=0.88\linewidth,height=3.8cm,keepaspectratio]{figures/fig-07-layer-selection.pdf}
  \par
  \raggedright
  \begin{itemize}
    \item Layer profiles: per-dimension maximum absolute residual-stream output over 40 prompts; Top-1/2/3 versus median.
    \item Zero-shot candidate scores: mean token log-likelihood for PIQA/HellaSwag; character-normalized for ARC-E.
  \end{itemize}
  \papersource{Paper, Fig. 7, PDF p. 23; App. F.3, p. 25. No chat template or added BOS/EOS tokens.}
\end{frame}

\end{document}