% Research discussion, about 25 minutes plus optional backup slides.
% Based on the arxiv-paper-reader Beamer template. Compile with pdfLaTeX.
\documentclass[aspectratio=169,11pt]{beamer}
\usepackage[T1]{fontenc}
\usepackage[utf8]{inputenc}
\usepackage{lmodern}
\usepackage{amsmath,amssymb,graphicx,booktabs,array}

\definecolor{AcademicBlue}{RGB}{26,57,86}
\definecolor{AcademicTeal}{RGB}{0,105,112}
\definecolor{WarmGray}{RGB}{88,94,102}
\setbeamercolor{structure}{fg=AcademicBlue}
\setbeamercolor{frametitle}{fg=AcademicBlue,bg=white}
\setbeamercolor{title}{fg=AcademicBlue}
\setbeamercolor{alerted text}{fg=AcademicTeal}
\setbeamercolor{block title}{fg=white,bg=AcademicBlue}
\setbeamercolor{block body}{fg=black,bg=AcademicBlue!5}
\setbeamertemplate{navigation symbols}{}
\setbeamertemplate{itemize items}[circle]
\setbeamertemplate{footline}{\hfill\usebeamerfont{page number in head/foot}%
  \insertframenumber\,/\,\inserttotalframenumber\hspace{4mm}\vspace{3mm}}
\setbeamersize{text margin left=8mm,text margin right=8mm}
\setbeamerfont{frametitle}{size=\Large,series=\bfseries}
\setbeamerfont{title}{size=\LARGE,series=\bfseries}
\newcommand{\papersource}[1]{\par\smallskip{\scriptsize\color{WarmGray}#1\par}}
\newcommand{\A}{\mathcal A}
\newcommand{\q}{\mathbf q}
\newcommand{\K}{\mathbf K}
\newcommand{\V}{\mathbf V}
\newcommand{\E}{\mathbb E}
\DeclareMathOperator{\softmax}{softmax}
\DeclareMathOperator{\sigmoid}{sigmoid}
\DeclareMathOperator{\SiLU}{SiLU}
\title[Nectar]{Nectar}
\subtitle{Neural Estimation of Cached-Token Attention via Regression}
\author{Paper by Jo\~ao Monteiro, Michal Klein,\\Pierre Ablin, and Marco Cuturi (Apple)}
\date{Research paper discussion\quad\textbullet\quad 8 October 2026\\
  \small\href{https://arxiv.org/abs/2605.09778v2}{arXiv:2605.09778v2, revised 8 July 2026}}

\begin{document}

\begin{frame}
  \titlepage
  \vspace{-4mm}
  \begin{center}
  \alert{Compile a fixed document's attention behavior into small networks.}
  \end{center}
\end{frame}

\begin{frame}{A fixed document is expensive to read repeatedly}
  \begin{block}{Serving the same book, manual, or knowledge source}
  Every new query reads the document's cached keys and values.
  \end{block}
  \[
    \text{Document KV state}=2nLHd\text{ scalar entries}
  \]
  \begin{itemize}
    \item $n$: document tokens; $L$: layers; $H$: KV heads; $d$: head size.
    \item Great Gatsby, about 61K tokens, on Qwen3-4B:\\
          about \textbf{9 GB of KV state} versus about 8 GB of model weights.
    \item Opportunity: pay once to fit the document's query response,
          then reuse the fitted module across requests.
  \end{itemize}
  \papersource{Paper, Sec. 1, PDF p. 1. Storage example uses bfloat16.}
\end{frame}

\begin{frame}{Nectar stores the function induced by the document}
  \centering
  \includegraphics[width=\linewidth,height=0.60\textheight,keepaspectratio]{figures/fig-01-overview.pdf}
  \begin{block}{One pair per layer and KV head}
    Predict the document's attention output and its log-normalizer.
  \end{block}
  \papersource{Paper, Fig. 1, PDF p. 2. Original figure. Base-model weights remain frozen.}
\end{frame}

\begin{frame}{Two functions separate retrieved content from mass}
  For a fixed document, $\K,\V\in\mathbb R^{n\times d}$ and a rotated
  query $\q\in\mathbb R^d$, let $s_j=\q^\top\mathbf k_j/\sqrt d$.
  \[
     \underbrace{\alpha(\q)=\log\sum_{j=1}^{n}e^{s_j}}_{\text{log attention mass}}
     \qquad
     \underbrace{\A(\q)=\frac{\sum_{j=1}^{n}e^{s_j}\mathbf v_j}
                              {\sum_{j=1}^{n}e^{s_j}}}_{\text{retrieved content}}
  \]
  \begin{itemize}
    \item Score network: $a_\theta(\q)\approx\alpha(\q)$, a scalar.
    \item Target network: $A_\theta(\q)\approx\A(\q)$, a $d$-vector.
    \item GQA: query heads sharing a KV head also share the module;
          each query is still evaluated separately.
  \end{itemize}
  \papersource{Paper, Sec. 3 and Eq. (3.1), PDF p. 4. Layer/head indices suppressed.}
\end{frame}

\begin{frame}{The score makes exact local attention composable}
  For visible local tokens, let $Z_{\rm loc}=\sum_i e^{s_i}$,
  $\mathbf N_{\rm loc}=\sum_i e^{s_i}\mathbf v_i$,
  and $\A_{\rm loc}=\mathbf N_{\rm loc}/Z_{\rm loc}$ when $Z_{\rm loc}>0$.
  \[
   \operatorname{ATT}_{\rm full}
     =\frac{e^{\alpha}\A+\mathbf N_{\rm loc}}{e^{\alpha}+Z_{\rm loc}}
     \quad\longrightarrow\quad
   \widehat{\operatorname{ATT}}
     =\frac{e^{a_\theta}A_\theta+\mathbf N_{\rm loc}}
            {e^{a_\theta}+Z_{\rm loc}}
  \]
  \begin{block}{Equivalent gate, when $Z_{\rm loc}>0$}
  $g=\sigmoid(a_\theta-\log Z_{\rm loc})$,\quad
  $\widehat{\operatorname{ATT}}=gA_\theta+(1-g)\A_{\rm loc}$.
  \end{block}
  \begin{itemize}
    \item Splitting the full softmax sum is exact; learned functions introduce error.
    \item Insert one virtual logit/value pair alongside visible local tokens.
    \item Local instruction and generated-token KV state is retained.
  \end{itemize}
  \papersource{Paper, Eqs. (3.4)--(3.5), PDF p. 5. Gate form is a teaching derivation.}
\end{frame}

\begin{frame}{The representation and the loss are separate choices}
  \begin{columns}[T,onlytextwidth]
  \begin{column}{0.48\textwidth}
    \begin{block}{Quadrature: learned KV pairs}
      $\mathbf W,\mathbf Z\in\mathbb R^{p\times d}$, $p\ll n$
    \end{block}
    \[
      A_\theta(\q)=\softmax(\mathbf W\q/\sqrt d)^\top\mathbf Z
    \]
    \begin{itemize}
      \item $2pd$ parameters; initialize from the first $p$ cached pairs.
      \item Score follows analytically from $\mathbf W$.
      \item With pure KL: Cartridges baseline under this paper's data recipe.
    \end{itemize}
  \end{column}
  \begin{column}{0.48\textwidth}
    \begin{block}{MLP: direct function regression}
      Separate scalar and vector outputs
    \end{block}
    \[
      \mathbf h_{k+1}=\SiLU(\mathbf V_k\q+\mathbf U_k\mathbf h_k+\mathbf b_k)
    \]
    \begin{itemize}
      \item Re-inject the query at intermediate depths.
      \item Optional shared backbone, then score and target heads.
      \item $(D_b,D_s,D_t)$ specifies their depths; width fits the budget.
    \end{itemize}
  \end{column}
  \end{columns}
  \papersource{Paper, Sec. 3.3, Eq. (3.6), PDF pp. 5--6; App. A.3, p. 13.}
\end{frame}

\begin{frame}{Train internal fidelity and final behavior together}
  \[
    \mathcal L_{\A}=\E\sum_{\ell,h,h'}
      \left\|A^{\ell,h}_\theta(\q_i^{\ell,h'})-
                   \A^{\ell,h}(\q_i^{\ell,h'})\right\|_2^2
  \]
  \[
    \mathcal L=\underbrace{\lambda_\alpha\mathcal L_\alpha+
                   \lambda_{\A}\mathcal L_{\A}}_{\mathcal L_{\rm reg}}
              +\lambda_{\rm KL}\mathcal L_{\rm KL}
  \]
  \begin{itemize}
    \item $\mathcal L_\alpha$: analogous scalar score regression.
    \item KL matches the full-cache teacher's next-token distribution
          under teacher forcing.
    \item MLP: $(\lambda_\alpha,\lambda_\A)=(0.1,1)$;
          Quadrature: $(0,1)$. Pure KL sets both to zero.
    \item Only module parameters change; queries come from task-relevant data.
  \end{itemize}
  \papersource{Paper, Sec. 3.1, Eqs. (3.2)--(3.3), PDF pp. 4--5.}
\end{frame}

\begin{frame}{Self-study defines the queries the module learns}
  \begin{enumerate}
    \item Split each document into overlapping chunks of about 2,000 words.
    \item Qwen3-8B generates about 10 pairs per chunk:\\
          5 QA, 1 summary, 2 quoting, 2 paraphrasing.
    \item Qwen2.5-7B-Instruct-1M re-answers using the full document;
          discard low-consistency pairs (confidence below 0.7).
    \item Random 80/20 split; cache the full model's attention targets.
  \end{enumerate}
  \begin{block}{Coverage is part of the approximation problem}
    Train on model-generated queries for one document; the training
    instruction-plus-response length is capped at 196 tokens.
  \end{block}
  \papersource{Paper, Apps. A.1--A.4, PDF pp. 12--14; App. I, pp. 24--25, 34--36.}
\end{frame}

\begin{frame}{The evaluation has three different scopes}
  \centering
  \begin{tabular}{@{}>{\raggedright\arraybackslash}p{0.24\linewidth}>{\raggedright\arraybackslash}p{0.32\linewidth}>{\raggedright\arraybackslash}p{0.35\linewidth}@{}}
  \toprule
  Question & Models & Contexts / measurements \\
  \midrule
  Attention / token fit & Qwen3: 1.7B, 4B, 8B; Qwen2.5-7B-1M & Five documents, 40K--122K tokens \\[2mm]
  Free-form generation & Qwen2.5-7B-1M & Three documents; QA, summary, quoting, paraphrasing \\[2mm]
  Serving cost & Qwen3-1.7B and 4B & TTFT, decode speed, peak GPU memory \\
  \bottomrule
  \end{tabular}
  \vspace{3mm}
  \begin{itemize}
    \item Qwen3 uses YaRN $4\times$; Qwen2.5-7B-1M needs no extension.
    \item $\rho=|\theta|/(2nd)$: module parameters relative to document KV entries.
  \end{itemize}
  \papersource{Paper, Sec. 4, PDF pp. 6--9; App. D, pp. 18--19.}
\end{frame}

\begin{frame}{Internal error and token accuracy can disagree}
  \centering
  \includegraphics[width=\linewidth,height=0.45\textheight,keepaspectratio]{figures/fig-04-fidelity.pdf}
  \begin{itemize}
    \item Qwen2.5-7B-1M, $\rho=2\%$; lower and left are desirable.
    \item Shape: architecture. Color: loss. Vertical axes contain breaks.
    \item Pure regression can have low internal error and large token gaps;
          pure KL can do the reverse. Mixed MLPs balance both.
  \end{itemize}
  \papersource{Paper, Fig. 4, PDF p. 7. Original crop; interpretation distinguishes loss regimes.}
\end{frame}

\begin{frame}{Good surrogate metrics do not ensure usable rollouts}
  \begin{block}{First verify that the teacher can generate}
  All tested YaRN-Qwen3 models failed the long-context generation test,
  including the full-cache teacher. YaRN is the authors' suspected cause.
  \end{block}
  \smallskip
  On Qwen2.5-7B-1M at $\rho=2\%$, MLP generation gaps are:
  \begin{center}
  \begin{tabular}{@{}lrrr@{}}
  \toprule
  Objective & Heart of Darkness & LongHealth & Great Gatsby \\
  \midrule
  Pure regression & 2.22 & 2.22 & 2.36 \\
  Regression + $0.01\,\mathrm{KL}$ & \alert{0.18} & \alert{0.39} & \alert{0.26} \\
  Pure KL & 0.98 & 1.04 & 0.62 \\
  \bottomrule
  \end{tabular}
  \end{center}
  \papersource{Paper, Sec. 4.3, PDF p. 8; Table 14, p. 26. $\Delta_s\downarrow$, 1--5 judge scale.}
\end{frame}

\begin{frame}{Generation is measured by semantic similarity}
  \[
    \Delta_s=\overline{s}_{\rm base}-\overline{s}_{\rm Nectar}
    \qquad (\text{lower is better})
  \]
  \begin{itemize}
    \item Three documents, 500 held-out instructions per document.
    \item GPT-5.2 rates responses against synthetic reference answers
          on a 1--5 scale.
    \item Sampling: temperature 0.7, top-$p=0.8$;
          at most 64 tokens for QA and 196 for other tasks.
    \item Even quoting is judged semantically, with minor wording changes allowed.
  \end{itemize}
  \begin{block}{Selection matters}
  Capacity results select the best configuration per document and capacity,
  then average over three documents; CIs describe these aggregation units.
  \end{block}
  \papersource{Paper, Sec. 4.3, PDF p. 9; Apps. F and H, pp. 21, 23--24, 32--33.}
\end{frame}

\begin{frame}{More capacity brings MLP close to full-cache quality}
  \begin{center}
  \renewcommand{\arraystretch}{1.35}
  \begin{tabular}{@{}lccc@{}}
  \toprule
   & $\rho=2\%$ & $\rho=5\%$ & $\rho=10\%$ \\
  \midrule
  MLP & $0.28\pm0.26$ & $-0.02\pm0.39$ & $\alert{-0.14\pm0.34}$ \\
  Quadrature & $2.25\pm0.22$ & $2.23\pm0.21$ & $2.22\pm0.26$ \\
  \bottomrule
  \end{tabular}
  \end{center}
  \begin{itemize}
    \item Random held-out questions; Qwen2.5-7B-1M; $\Delta_s\downarrow$.
    \item The MLP mean improves across capacities; the same trend is
          not strictly monotonic within every document.
    \item At 10\%, the 95\% CI crosses zero: evidence for near-baseline
          quality, with an uncertain small mean advantage.
    \item Quadrature remains poor under this paper's data and training recipe.
  \end{itemize}
  \papersource{Paper, Table 3, PDF p. 9; per-document values in Table 13, p. 20.}
\end{frame}

\begin{frame}{A small distillation weight helps on average}
  \begin{center}
  \renewcommand{\arraystretch}{1.35}
  \begin{tabular}{@{}lc@{}}
  \toprule
  MLP objective & Random-split $\Delta_s\downarrow$ \\
  \midrule
  Pure KL & $0.36\pm0.33$ \\
  Regression + $0.01\,\mathrm{KL}$ & $\alert{0.25\pm0.13}$ \\
  Regression + $2\,\mathrm{KL}$ & $0.28\pm0.37$ \\
  Regression + $10\,\mathrm{KL}$ & $0.53\pm0.24$ \\
  \bottomrule
  \end{tabular}
  \end{center}
  \begin{itemize}
    \item Fix the loss weight; select within each document--capacity cell;
          average over cells. This differs from capacity-wise selection.
    \item At higher capacity, pure KL wins some cells; mixed training
          wins others. One loss weight does not dominate every setting.
  \end{itemize}
  \papersource{Paper, Table 4, PDF p. 9; detailed cells in Table 14, p. 26.}
\end{frame}

\begin{frame}{Quality varies with the document and task}
  \centering
  \includegraphics[width=\linewidth,height=0.57\textheight,keepaspectratio]{figures/fig-17-task-scores.pdf}
  \begin{itemize}
    \item Absolute judge scores: higher is better. Gray: full cache; gold: MLP.
    \item Best overall run per document: QA gap $-0.46$;
          paraphrasing gap $+0.19$. Task-level CIs are wide.
    \item Semantic match can coexist with unsupported extra details
          in individual responses.
  \end{itemize}
  \papersource{Paper, Fig. 17, PDF p. 25; Table 5, p. 10; example in Table 2, p. 9.}
\end{frame}

\begin{frame}{Dissimilar questions expose a substantial gap}
  Challenge: select the 500 test instructions per document with the
  lowest maximum embedding similarity to any training instruction.
  \begin{center}
  \renewcommand{\arraystretch}{1.3}
  \begin{tabular}{@{}lccc@{}}
  \toprule
  MLP $\Delta_s\downarrow$ & $\rho=2\%$ & $\rho=5\%$ & $\rho=10\%$ \\
  \midrule
  Random & $0.28\pm0.26$ & $-0.02\pm0.39$ & $-0.14\pm0.34$ \\
  Challenge & $1.40\pm0.54$ & $1.28\pm0.49$ & $\alert{1.18\pm0.43}$ \\
  \bottomrule
  \end{tabular}
  \end{center}
  \begin{itemize}
    \item Same documents; a different query distribution.
    \item Challenge quoting gap: 1.98 in the best-run task breakdown.
    \item Quadrature is worse (about 2.78), but MLP no longer matches full cache.
  \end{itemize}
  \papersource{Paper, App. F, PDF pp. 21--22; Tables 19 and 21, p. 27. Encoder: all-MiniLM-L6-v2.}
\end{frame}

\begin{frame}{Spend capacity where it improves the final output}
  Qwen3-1.7B, MLP, $\rho\approx2\%$, $\lambda_{\rm KL}=0.01$:
  \begin{center}
  \renewcommand{\arraystretch}{1.3}
  \begin{tabular}{@{}lccc@{}}
  \toprule
  Three-document mean & Uniform & Weighted & Change \\
  \midrule
  Score MSE & 0.056 & 0.074 & Worse \\
  Target MSE & 0.256 & 0.224 & Better \\
  Token-accuracy gap (pp) & 0.76 & \alert{0.60} & $-0.16$ pp \\
  \bottomrule
  \end{tabular}
  \end{center}
  \begin{itemize}
    \item Layer groups: $[0,9)$, $[9,15)$, $[15,27)$, $[27,28)$.
    \item Multipliers $(1,2,5,2)$, normalized to preserve total capacity.
    \item More middle-to-late capacity helps target fidelity and token accuracy,
          despite a slightly worse score fit.
  \end{itemize}
  \papersource{Paper, App. C.1, Table 10, PDF p. 16. pp = percentage points.}
\end{frame}

\begin{frame}{The large storage reduction applies to document state}
  At about 62K tokens, bfloat16 storage:
  \begin{center}
  \renewcommand{\arraystretch}{1.35}
  \begin{tabular}{@{}lrrrr@{}}
  \toprule
  Model & Weights & Full KV & 2\% module & 10\% module \\
  \midrule
  Qwen3-4B & 8.04 GB & 9.14 GB & 183 MB & 914 MB \\
  Qwen2.5-7B-1M & 14.14 GB & 3.56 GB & 71 MB & 355 MB \\
  \bottomrule
  \end{tabular}
  \end{center}
  \begin{itemize}
    \item 2\% means about $50\times$ less \emph{static document state}.
    \item Model weights, local-token KV, and working memory remain.
    \item A fixed fitted module has no document-length loop at inference.
    \item Across documents at fixed $\rho$, its budget
          $|\theta|=\rho\,2nd$ still grows with $n$.
  \end{itemize}
  \papersource{Paper, Table 9, PDF p. 15; Fig. 1, p. 2. Last distinction is a complexity interpretation.}
\end{frame}

\begin{frame}{Serving speed improves under an asymmetric prefill setup}
  At 122,466 document tokens; each cell is Base $\to$ Nectar:
  \begin{center}
  \renewcommand{\arraystretch}{1.35}
  \begin{tabular}{@{}lrrr@{}}
  \toprule
  Model & TTFT (ms) & Decode (tok/s) & Peak memory (GB) \\
  \midrule
  Qwen3-1.7B & $299{,}900\to29.3$ & $4.4\to37.7$ & $60.4\to8.2$ \\
  Qwen3-4B & $557{,}600\to50.1$ & $1.5\to15.7$ & $84.2\to17.1$ \\
  \bottomrule
  \end{tabular}
  \end{center}
  \begin{itemize}
    \item Decode improves by $8.6\times$ and $10.5\times$ in these rows.
    \item Base includes full-document prefill; Nectar uses a trained module.
          Offline work is excluded; prefix-cache reuse is not the baseline.
    \item These YaRN-Qwen3 models failed the generation-quality test.
          Usable quality and speed are not jointly established in this table.
  \end{itemize}
  \papersource{Paper, App. D, Table 12, PDF pp. 18--19; quality failure in Sec. 4.3, p. 8.}
\end{frame}

\begin{frame}{The trade-off favors stable, frequently queried documents}
  \begin{block}{Offline cost must be amortized}
  Qwen2.5-7B-1M, mixed training on an 8-GPU node:\\
  16.3 h at 2\% / 1M samples; 24.0 h at 10\% / 1.5M samples.\\
  Reported times already subtract one hour of loading and compilation.
  \end{block}
  \begin{itemize}
    \item A module is tied to its document and base model.
    \item Query coverage matters: challenge prompts lose much more quality.
    \item Training and reported generation are short after the document;
          long conversation drift remains an open deployment question.
    \item Needed comparison: quality, speed, and amortization on the
          same usable model against a reused prefix KV-cache.
  \end{itemize}
  \papersource{Paper, Table 8, PDF p. 15; Sec. 5, p. 10; Apps. F and H. Final item: reading analysis.}
\end{frame}

\begin{frame}{Three ideas to take into further research}
  \begin{enumerate}
    \item \textbf{Content plus mass is a useful interface.}\\
    The log-normalizer makes an approximate document response
    composable with exact local attention.
    \medskip
    \item \textbf{Learn the query distribution as carefully as the module.}\\
    The same fixed document supports both near-baseline random-split
    quality and a substantial challenge-split gap.
    \medskip
    \item \textbf{Validate rollouts and costs on one coherent setup.}\\
    Internal regression, token agreement, semantic generation,
    and serving efficiency measure different properties.
  \end{enumerate}
  \papersource{Synthesis of Secs. 3--5 and Apps. C--F. Proposed deployment interpretation, not an additional experiment.}
\end{frame}

\begin{frame}{Reference and discussion route}
  \textbf{Jo\~ao Monteiro, Michal Klein, Pierre Ablin, Marco Cuturi.}
  \medskip

  \emph{Nectar: Neural Estimation of Cached-Token Attention via Regression.}
  arXiv:2605.09778v2, revised 8 July 2026.
  \medskip

  \href{https://arxiv.org/abs/2605.09778v2}{https://arxiv.org/abs/2605.09778v2}
  \bigskip
  \begin{itemize}
    \item Mechanism: Eqs. (3.1)--(3.6), PDF pp. 4--6.
    \item Random generation: Tables 3--6, PDF pp. 9--10.
    \item Pure regression and challenge: Tables 14, 19--23, pp. 26--28.
    \item Costs: Tables 8--9 and 12, pp. 15 and 18.
  \end{itemize}
  \papersource{Original figures reproduced from the paper. All page locators refer to the downloaded v2 PDF.}
\end{frame}

% Backup slides: optional, outside the main 20--30 minute discussion.
\appendix

\begin{frame}{Backup: target capacity matters under mixed supervision}
  Total $\rho=10\%$; score/target splits are percentages of full KV state.
  \begin{center}
  \renewcommand{\arraystretch}{1.35}
  \begin{tabular}{@{}lcc@{}}
  \toprule
  Mean token-accuracy gap (pp) & $1\%\,/\,9\%$ & $2.5\%\,/\,7.5\%$ \\
  \midrule
  Pure KL ($\lambda_{\rm KL}=1$) & 0.24 & 0.20 \\
  Mixed ($\lambda_{\rm KL}=2$) & \alert{0.34} & 0.47 \\
  \bottomrule
  \end{tabular}
  \end{center}
  \begin{itemize}
    \item With direct score supervision, assigning more capacity to the
          target reduces the mean token gap in this ablation.
    \item Pure KL shows a small difference in the opposite direction.
    \item Layer multipliers here are $(1,2,8,12)$, different from the
          Qwen3-1.7B uniform/weighted experiment.
  \end{itemize}
  \papersource{Paper, App. C.2, Table 11, PDF pp. 16--17; model labeled Qwen2-7B by the authors.}
\end{frame}

\begin{frame}{Backup: pure KL can improve output while MSE rises}
  \centering
  \includegraphics[width=\linewidth,height=0.53\textheight,keepaspectratio]{figures/fig-09-training-gatsby.pdf}
  \begin{itemize}
    \item Great Gatsby, $\rho=2\%$; horizontal axes count training pairs.
    \item Solid MLP curves: pure distillation. Dotted: pure regression.
    \item Falling token gap with rising target MSE is visible under pure KL.
          The internal trajectory need not mimic the teacher's.
  \end{itemize}
  \papersource{Paper, Fig. 9, Great Gatsby row, PDF p. 21. Original crop; paper's model label retained.}
\end{frame}

\begin{frame}{Backup: the score connects attention to support functions}
  With $\alpha(\q)=\log\sum_j\exp(\q^\top\mathbf k_j/\sqrt d)$:
  \[
    \nabla_{\q}\alpha(\q)=\frac{1}{\sqrt d}\K^\top
                                     \softmax(\K\q/\sqrt d)
  \]
  \[
    \lim_{\beta\to\infty}\frac1\beta
        \log\sum_j\exp(\beta\q^\top\mathbf k_j)
      =\max_j\q^\top\mathbf k_j
  \]
  \begin{itemize}
    \item Log-sum-exp is convex; the low-temperature limit is a support function.
    \item This motivates the input-skip MLP design from amortized MIPS.
    \item The score gradient averages \emph{keys}; target attention averages
          \emph{values}. They differ in general.
    \item The paper does not impose convexity guarantees on its SiLU MLP.
  \end{itemize}
  \papersource{Paper, App. B, Eqs. (B.1)--(B.2), PDF pp. 15--16; gradient above restores the Sec. 3 scale.}
\end{frame}

\begin{frame}{Backup: RTE needs care when aggregating heads}
  The paper defines the relative transport error as
  \[
    \mathcal E_{\rm rel}
     =\sum_{\ell,h}\E_{\q}\left[
       \log\frac{\|A_\theta^{\ell,h}(\q)-\A^{\ell,h}(\q)\|_2^2}
                     {\|\q-\A^{\ell,h}(\q)\|_2^2}\right].
  \]
  \begin{itemize}
    \item The denominator measures the error of returning the query itself.
    \item For one head, exponentiating the expected log ratio gives its
          geometric-mean error ratio.
    \item With a sum over heads, exponentiation gives a product of those
          means. A global geometric mean needs division by the group count.
  \end{itemize}
  \begin{block}{Interpretation}
  Compare like-for-like architectures and normalization conventions;
  RTE is an internal diagnostic, not a generation score.
  \end{block}
  \papersource{Paper, App. C.4, Eq. (C.1), PDF p. 17. Aggregation clarification is this reading's derivation.}
\end{frame}

\begin{frame}{Backup: document sizes and training budgets}
  \begin{center}
  \renewcommand{\arraystretch}{1.2}
  \begin{tabular}{@{}lrr@{}}
  \toprule
  Document & Tokens & Synthetic pairs \\
  \midrule
  Time Machine & 40,022 & 117,111 \\
  Heart of Darkness & 49,313 & 117,120 \\
  LongHealth & 59,031 & 103,592 \\
  Great Gatsby & 61,651 & 114,896 \\
  IP Intro & 122,466 & 228,193 \\
  \bottomrule
  \end{tabular}
  \end{center}
  \begin{itemize}
    \item Query length: cache up to 512 tokens; train on at most 196.
    \item Table 8 uses 1M training samples at 2\%, but 1.5M for
          Qwen2.5-7B-1M at 5\% and 10\%.
    \item Base peak learning rate $10^{-4}$; warmup/cosine schedule;
          global gradient clipping at 1.0.
  \end{itemize}
  \papersource{Paper, Table 1, PDF p. 7; Apps. A.1--A.6 and Table 8, pp. 12--15.}
\end{frame}

\begin{frame}{Backup: two reporting details affect reproduction}
  \begin{block}{Qwen3-8B layer count and memory}
  Sec. 4.1 reports 64 layers and Table 9 gives 16.25 GB of document KV.
  The official Qwen3-8B config has 36 layers, 8 KV heads, and $d=128$.
  At 62,000 tokens in bfloat16 this gives about 9.14 GB.
  \end{block}
  \begin{block}{Optimizer specification}
  The main text says AdamW; App. A.4 says Adam.
  Weight decay should be resolved against the implementation for reproduction.
  \end{block}
  \papersource{Paper, Sec. 4.1, PDF pp. 6--7; App. A.4, p. 14; Table 9, p. 15.}
  \papersource{External configuration check, 8 Oct. 2026:
    \href{https://huggingface.co/Qwen/Qwen3-8B/blob/main/config.json}{Qwen/Qwen3-8B official config.json}.}
\end{frame}

\end{document}