% Based on the arxiv-paper-reader Beamer template. Compile with pdfLaTeX.
\documentclass[aspectratio=169,11pt]{beamer}
\usepackage[T1]{fontenc}
\usepackage[utf8]{inputenc}
\usepackage{lmodern}
\usepackage{amsmath,amssymb,graphicx,booktabs}
\definecolor{AcademicBlue}{RGB}{26,57,86}
\definecolor{AcademicTeal}{RGB}{0,105,112}
\setbeamercolor{structure}{fg=AcademicBlue}
\setbeamercolor{frametitle}{fg=AcademicBlue,bg=white}
\setbeamercolor{title}{fg=AcademicBlue}
\setbeamercolor{alerted text}{fg=AcademicTeal}
\setbeamercolor{block title}{fg=white,bg=AcademicBlue}
\setbeamercolor{block body}{fg=black,bg=AcademicBlue!5}
\setbeamertemplate{navigation symbols}{}
\setbeamertemplate{itemize items}[circle]
\setbeamertemplate{footline}{\hfill\usebeamerfont{page number in head/foot}\insertframenumber\,/\,\inserttotalframenumber\hspace{4mm}\vspace{3mm}}
\setbeamersize{text margin left=8mm,text margin right=8mm}
\setbeamerfont{frametitle}{size=\Large,series=\bfseries}
\setbeamerfont{title}{size=\LARGE,series=\bfseries}
\newcommand{\papersource}[1]{\par\smallskip{\scriptsize\color{gray}#1\par}}
\newcommand{\E}{\mathbb E}
\newcommand{\R}{\mathbb R}
\newcommand{\x}{\boldsymbol x}
\newcommand{\w}{\boldsymbol w}
\newcommand{\uvec}{\boldsymbol u}
\newcommand{\bvec}{\boldsymbol b}
\newcommand{\hvec}{\boldsymbol h}
\newcommand{\gam}{\boldsymbol\gamma}
\newcommand{\rn}{\rho_{\mathrm{nonlinear}}}
\newcommand{\Ct}{C_{\mathrm{train}}}
\newcommand{\Cs}{C_{\mathrm{test}}}
\newcommand{\tr}{\operatorname{tr}}
\hypersetup{colorlinks=true,urlcolor=AcademicTeal,linkcolor=AcademicBlue}
\title[Linear attention and nonlinear teachers]{What can linear attention learn\\from nonlinear teachers in-context?}
\subtitle{Research group discussion \;|\; approximately 25 minutes}
\author{Paper by Mary Letey, Arman Rysmakhanov, Yue M. Lu,\\Cengiz Pehlevan, and Jacob A. Zavatone-Veth}
\date{\href{https://arxiv.org/abs/2610.10761v1}{arXiv:2610.10761v1} \;|\; 29 October 2026}

\begin{document}

\begin{frame}
  \titlepage
  \vspace{-5mm}
  {\small
  Accepted at NeurIPS 2026 PriGM \textbf{Workshop}, per
  \href{https://jzv.io/publications}{author publication list}.\\
  \href{https://github.com/mletey/single-index-icl}{Code forthcoming}: author-linked repository contains only a README.}
  \papersource{Status checked 2026-10-29; no individual proceedings or decision page verified.}
\end{frame}

\begin{frame}{Only the first Hermite mode becomes signal}
  \begin{block}{Question}
    What can a reduced linear-attention predictor learn from
    $y=f(\x^\top\w)+\varepsilon$?
  \end{block}
  \[
  f(g)=\underbrace{a_1g}_{\text{learnable projection}}
       +\underbrace{\sum_{m\ne1}a_mh_m(g)}_{\text{residual modes}},
  \qquad g\sim\mathcal N(0,1).
  \]
  \begin{itemize}
    \item In the paper's high-dimensional ridgeless limit, residual energy acts as additional noise.
    \item More data can improve estimation of $a_1g$.
    \item The remaining error floor is fixed by the predictor's expressivity.
  \end{itemize}
  \papersource{Paper, Sec. 3, Result 1, Eqs. (8)--(11), PDF pp. 3--4.}
\end{frame}

\begin{frame}{Each prompt hides one task direction}
  \[
  \underbrace{(\x_1,y_1),\ldots,(\x_\ell,y_\ell)}_{\text{context}}
  \quad+\quad \underbrace{\x_{\mathrm{test}}}_{\text{query}}
  \quad\longrightarrow\quad \widehat y_{\mathrm{test}}
  \]
  \[
  y_i=f(\x_i^\top\w)+\varepsilon_i,\qquad
  \x_i,\x_{\mathrm{test}}\overset{\mathrm{i.i.d.}}\sim\mathcal N(0,I_d/d),
  \quad\varepsilon_i\sim\mathcal N(0,\rho).
  \]
  \begin{itemize}
    \item Shared $\w$ within one prompt; fixed link function $f$ across tasks.
    \item Pretraining: $k$ tasks from $\mathcal N(0,\Ct)$, spread uniformly over $n$ prompts.
    \item Testing: a fresh task from $\mathcal N(0,\Cs)$; parameters stay fixed.
  \end{itemize}
  \papersource{Paper, Sec. 2, Eq. (1), PDF p. 2. Independent Gaussian label noise.}
\end{frame}

\begin{frame}{Three ratios describe different data bottlenecks}
  \centering
  \begin{tabular}{lll}
    \toprule
    Ratio & Resource & What it controls \\
    \midrule
    $\alpha=\ell/d$ & Context length & Per-task statistics \\[3mm]
    $\kappa=k/d$ & Task diversity & Coverage of task directions \\[3mm]
    $\tau=n/d^2$ & Pretraining prompts & Parameter estimation \\
    \bottomrule
  \end{tabular}
  \vspace{5mm}
  \begin{block}{Proportional limit}
    $d\to\infty$ with $\alpha,\kappa,\tau$ of order one.\\
    The reduced predictor has $d(d+1)\approx d^2$ parameters.
  \end{block}
  \raggedright
  Repeating the same tasks increases $\tau$ without increasing $\kappa$.
  \papersource{Paper, Sec. 2, Eqs. (3)--(4), PDF pp. 2--3.}
\end{frame}

\begin{frame}{The query enters through a linear map}
  \[
  \uvec=\frac d\ell\sum_i y_i\x_i,\qquad
  s=\frac1\ell\sum_i y_i^2,\qquad
  H=\x_{\mathrm{test}}[\uvec^\top,s].
  \]
  \[
  \Gamma=[A,\bvec]\in\R^{d\times(d+1)}
  \quad\Longrightarrow\quad
  \boxed{\widehat y_{\mathrm{test}}
  =\x_{\mathrm{test}}^\top(A\uvec+s\bvec)}.
  \]
  \begin{itemize}
    \item $\uvec$: empirical input--label correlation; $s$: label second moment.
    \item Context changes the prediction direction.
    \item Squared labels do not create a nonlinear query readout or an intercept.
  \end{itemize}
  \papersource{Paper, Eqs. (4)--(5), PDF p. 3. The $[A,\bvec]$ expansion is a teaching derivation.}
\end{frame}

\begin{frame}{Pretraining is ridge regression across prompts}
  \[
  \gam=\operatorname{vec}(\Gamma),\qquad \hvec_\mu=\operatorname{vec}(H^\mu),
  \]
  \[
  \gam^*=\left(\frac nd\lambda I+\sum_{\mu=1}^n\hvec_\mu\hvec_\mu^\top\right)^{-1}
          \sum_{\mu=1}^n y_{\mathrm{test}}^\mu\hvec_\mu.
  \]
  \begin{columns}[T]
    \begin{column}{0.47\textwidth}
      \begin{block}{Pretraining}
        Fit one shared parameter matrix $\Gamma^*$ from labeled prompts.
      \end{block}
    \end{column}
    \begin{column}{0.47\textwidth}
      \begin{block}{In-context inference}
        Compute $\uvec,s,H$ on a new task; keep $\Gamma^*$ fixed.
      \end{block}
    \end{column}
  \end{columns}
  \smallskip
  Metric: fresh-task mean squared error, including query-label noise.
  \papersource{Paper, Eqs. (6)--(7), PDF p. 3. Main result concerns $\lambda\to0$.}
\end{frame}

\begin{frame}{Hermite projection is different from a Taylor term}
  \[
  f(g)=\sum_{m\ge0}a_mh_m(g),\quad
  a_m=\E[f(g)h_m(g)],\quad \E[h_mh_j]=\delta_{mj}.
  \]
  \[
  h_0=1,\quad h_1=g,\quad h_2=\frac{g^2-1}{\sqrt2},
  \qquad \alert{a_1=\E[g f(g)]}.
  \]
  \centering
  \begin{tabular}{lll}
    \toprule
    Link & First coefficient & Residual energy \\
    \midrule
    $f(g)=g$ & $a_1=1$ & $0$ \\
    $f(g)=g^2-1$ & $a_1=0$ & $2$ \\
    $f(g)=g^3$ & $a_1=3$ & $6$ \\
    \bottomrule
  \end{tabular}
  \par\smallskip\raggedright
  A cubic target contains the learnable projection $3g$.
  \papersource{Definition: paper, Eq. (2), PDF p. 2. Table: teaching examples derived from Gaussian moments.}
\end{frame}

\begin{frame}{Signal energy and residual energy play different roles}
  \[
  \underbrace{\rho_f}_{\text{total label second moment}}
  =\rho+\sum_{m\ge0}a_m^2
  =\underbrace{a_1^2}_{\text{linear signal}}
   +\underbrace{\rn}_{\text{effective noise}},
  \]
  \[
  \boxed{\rn=\rho+\sum_{m\ne1}a_m^2}.
  \]
  \begin{itemize}
    \item The residual includes $a_0^2$: this predictor has no query intercept.
    \item $\rho_f$ is a second moment, not a centered variance when $a_0\ne0$.
    \item ``Effective noise'' describes the asymptotic risk contribution; the residual need not be independent of the input.
  \end{itemize}
  \papersource{Paper, Eq. (8), PDF p. 3; predictor in Eq. (5).}
\end{frame}

\begin{frame}{Task-norm concentration makes Hermite analysis valid}
  \[
  \x^\top\w\mid\w\sim\mathcal N(0,\sigma_{\w}),\quad
  \sigma_{\w}=\|\w\|^2/d,
  \]
  \[
  \E[\sigma_{\w}]=\frac{\tr(C)}d,\qquad
  \operatorname{Var}(\sigma_{\w})=\frac{2\tr(C^2)}{d^2}.
  \]
  \begin{block}{Assumptions on both training and test tasks}
    $\tr(C)=d$ and $\tr(C^2)=o(d^2)$ imply $\sigma_{\w}\to1$.\\
    The effective rank $d^2/\tr(C^2)$ must diverge.
  \end{block}
  The link has at most polynomial growth. A fixed-rank covariance carrying all the signal is outside these assumptions.
  \papersource{Paper, Result 1, PDF p. 3; Appendix A, PDF p. 7.}
\end{frame}

\begin{frame}{The context correlation extracts exactly one mode}
  \[
  \E[y\x\mid\w]\approx\frac{\w}{d}\E[g f(g)]
  =\frac{a_1\w}{d}
  \quad\Longrightarrow\quad
  \E[\uvec\mid\w]\approx a_1\w.
  \]
  \[
  \E[\uvec\uvec^\top\mid\w]
  \approx\underbrace{a_1^2\w\w^\top}_{\text{task structure}}
  +\underbrace{\frac{\rho_f}{\alpha}I_d}_{\text{context fluctuations}}.
  \]
  \begin{itemize}
    \item Orthogonality removes higher Hermite modes from the mean correlation.
    \item All label energy still contributes to finite-context fluctuations.
    \item Matching the leading moments allows reuse of the linear risk theory.
  \end{itemize}
  \papersource{Paper, Appendix B, Lemma 1, PDF pp. 8--9; displayed at $\|\w\|^2/d\to1$.}
\end{frame}

\begin{frame}{The main result rescales signal and replaces noise}
  For $a_1\ne0$, in the high-dimensional ridgeless limit:
  \[
  \boxed{
    e_{\mathrm{ICL}}^{\mathrm{nonlinear}}(\Ct,\Cs,\rho)
    =e_{\mathrm{ICL}}^{\mathrm{linear}}(a_1^2\Ct,a_1^2\Cs,\rn)
  }.
  \]
  \begin{columns}[T]
    \begin{column}{0.46\textwidth}
      \begin{block}{Equivalent linear teacher}
        $\widetilde\w=a_1\w$\\[1mm]
        $\operatorname{Var}(\widetilde\varepsilon)=\rn$
      \end{block}
    \end{column}
    \begin{column}{0.49\textwidth}
      \begin{block}{What is preserved}
        Training--test task geometry; context, diversity, and sample ratios.
      \end{block}
    \end{column}
  \end{columns}
  \smallskip
  Increasing only the noise, while leaving signal strength unchanged, misses part of the transformation.
  \papersource{Paper, Result 1, Eq. (9), PDF p. 3; Appendix C, PDF p. 10.}
\end{frame}

\begin{frame}{Without a linear mode, interpolation amplifies noise}
  If $a_1=0$:
  \[
  e_{\mathrm{ICL}}^{\mathrm{nonlinear}}=\rn
  \begin{cases}
    (1-\tau)^{-1}, & \tau<1,\\[1mm]
    \tau/(\tau-1), & \tau>1.
  \end{cases}
  \]
  \begin{itemize}
    \item No dependence on $\alpha$ or $\kappa$ in this limiting formula.
    \item Positive residual energy produces a singularity at $\tau=1$.
    \item More pretraining eventually returns the error to $\rn$.
  \end{itemize}
  \smallskip
  Formula examples: $\tau=0.5,2\Rightarrow2\rn$;
  $\tau=0.9\Rightarrow10\rn$.
  \papersource{Paper, Eq. (10), PDF p. 3; Appendix C, Eq. (15), PDF p. 10. Examples are derived, not simulation results.}
\end{frame}

\begin{frame}{Figure 1 displays the predicted noise equivalence}
  \centering
  \includegraphics[width=0.91\linewidth,height=0.60\textheight,keepaspectratio]{figures/fig-01.pdf}
  \par\smallskip\raggedright
  $d=64$, $\alpha=\kappa=2$, $\Ct=\Cs=I_d$; sine versus linear teachers.
  \par\smallskip
  Nonlinear and adjusted-noise markers nearly coincide; both show an interpolation peak. \alert{Their scaling needs care (next slide).}
  \papersource{Paper, Fig. 1, PDF p. 4. Original figure. Vertical axis is logarithmic; lower error is better.}
\end{frame}

\begin{frame}{Figure 1 leaves an error-normalization ambiguity}
  For $f(g)=\sin g$ and $\rho=0.1$:
  \[
  a_1^2=e^{-1}\approx0.367879,\qquad
  \rn=0.1+\frac{1-e^{-2}}2-e^{-1}\approx0.164453.
  \]
  \begin{block}{Unit-signal noise variance}
    $\rn/a_1^2\approx0.447029$ matches the linear-noise legend.
  \end{block}
  \begin{itemize}
    \item The prose instead gives $(a_1^2+\rn)/a_1^2\approx1.447029$: the normalized \emph{total} second moment.
    \item Rescaling labels by $1/a_1$ also rescales MSE by $1/a_1^2$; the axis/caption does not specify this.
  \end{itemize}
  \papersource{Reading analysis of paper, Fig. 1 discussion, PDF p. 3, and legend, p. 4. Do not read overlap as equality of unscaled MSE.}
\end{frame}

\begin{frame}{Task diversity removes coverage error, not residual energy}
  With $\alpha,\tau\to\infty$ and $\tau\propto\alpha$, the inherited task-diversity threshold is
  \[
  \kappa_c=\frac{\operatorname{rank}(\Ct)}d.
  \]
  \begin{block}{Isotropic case: $\Ct=\Cs=I_d$}
    If $a_1\ne0$ and $\kappa>1$, then
    \[
      e_{\mathrm{ICL}}\longrightarrow\rn=\rho+\sum_{m\ne1}a_m^2.
    \]
  \end{block}
  Few tasks leave directions poorly covered. Sufficient diversity supports transfer of the linear rule to fresh tasks; the nonlinear residual remains.
  \papersource{Paper, Sec. 3, task-diversity paragraph and Eq. (11), PDF pp. 3--4.}
\end{frame}

\begin{frame}{Figure 2 approaches a raised error floor}
  \begin{columns}[c]
    \begin{column}{0.65\textwidth}
      \centering
      \includegraphics[width=\linewidth,height=0.70\textheight,keepaspectratio]{figures/fig-02.pdf}
    \end{column}
    \begin{column}{0.32\textwidth}
      $f(g)=\tanh g$\\
      $d=32$, $\rho=0.1$\\
      $\tau=\alpha/2$
      \par\medskip
      Error falls around $\kappa\approx1$.
      \par\medskip
      Finite-resource curves remain above the new limiting error.
    \end{column}
  \end{columns}
  \papersource{Paper, Fig. 2 and Eq. (11), PDF p. 4. Original figure; task-ratio axis is logarithmic.}
\end{frame}

\begin{frame}{For tanh, the extra limiting MSE is about 0.0274}
  \[
  a_1\approx0.605706,\quad \E[\tanh^2g]\approx0.394294,
  \]
  \[
  \boxed{\rn=0.1+0.394294-0.605706^2\approx0.127415}.
  \]
  \centering
  \begin{tabular}{rrrr}
    \toprule
    $\alpha$ & Context $\ell$ & $\tau$ & Prompts $n$ \\
    \midrule
    20 & 640 & 10 & 10,240 \\
    40 & 1,280 & 20 & 20,480 \\
    \bottomrule
  \end{tabular}
  \par\medskip\raggedright
  The two curves increase context and pretraining size together. Their difference does not isolate a context-length effect.
  \papersource{Settings: paper, Fig. 2, PDF p. 4. Table derived from ratios; moments computed from Eq. (8), not reported experimental values.}
\end{frame}

\begin{frame}{The conclusion is specific to this predictor and limit}
  \begin{itemize}
    \item \textbf{Architecture:} a reduced predictor with a linear query readout. Nonlinear readouts or multilayer nonlinear transformations change the function class.
    \item \textbf{Distribution:} independent Gaussian inputs and Gaussian tasks with concentrating norms.
    \item \textbf{Asymptotics:} proportional dimensions and ridgeless risk; the derivation keeps leading moments and uses prior resolvent theory.
    \item \textbf{Evidence:} sine at $d=64$ and tanh at $d=32$ illustrate the mechanism; Figure 1 has a normalization ambiguity.
  \end{itemize}
  \papersource{Paper, Secs. 1--3 and Appendices A--C, PDF pp. 1--4, 7--10; normalization issue is reading analysis.}
\end{frame}

\begin{frame}{The useful distinction is estimation versus expressivity}
  \begin{enumerate}
    \item \textbf{Compute the projection.} $a_1=\E[g f(g)]$ identifies the accessible signal.
    \item \textbf{Account for residual energy.} $\rn$ predicts a floor that remains after sufficient task coverage.
    \item \textbf{Match scales.} Signal covariance and error units matter when comparing teachers.
  \end{enumerate}
  \medskip
  \begin{block}{Discussion}
    Which additional Hermite modes would become learnable if we added a nonlinear query readout?
  \end{block}
  \papersource{Paper, Result 1 and Conclusion, PDF pp. 3--4. Discussion question is an extension motivated by the analysis.}
\end{frame}

\begin{frame}{References and paper resources}
  \begin{itemize}
    \item Letey et al. (2026). \emph{What can linear attention learn from nonlinear teachers in-context?}
      \href{https://arxiv.org/abs/2610.10761v1}{arXiv:2610.10761v1}.
    \item Letey et al. (2026). \emph{Pretrain-test task alignment governs generalization in in-context learning.}
      \href{https://openreview.net/forum?id=KZLeg0MQ2r}{ICLR paper cited by the derivation}.
    \item Lu et al. (2025). \emph{Asymptotic theory of in-context learning by linear attention.}
      \href{https://doi.org/10.1073/pnas.2502599122}{PNAS 122(28): e2502599122}.
  \end{itemize}
  \medskip
  \href{https://jzv.io/publications}{Author publication list}:
  PriGM Workshop at NeurIPS 2026.\\
  \href{https://github.com/mletey/single-index-icl}{Author-linked code repository}:
  README only as of 2026-10-29.
  \papersource{Background references are those used by the paper, PDF pp. 5--6. Figures in this deck are the authors' original assets.}
\end{frame}

\appendix

\begin{frame}{Backup: leading moments match a linear teacher}
  At $\|\w\|^2/d\to1$, with the paper's vectorization convention,
  \[
  \E[y^2\mid\w]\approx\rho_f,\qquad
  \E[y_{\mathrm{test}}H\mid\w]
  \approx\frac1d[a_1^2\w\w^\top,\ a_1\rho_f\w],
  \]
  \[
  \E[\hvec\hvec^\top\mid\w]\approx\frac1dI_d\otimes
  \begin{bmatrix}
    \dfrac{\rho_f}{\alpha}I_d+a_1^2\w\w^\top & a_1\rho_f\w\\[2mm]
    a_1\rho_f\w^\top & \rho_f^2
  \end{bmatrix}.
  \]
  \begin{itemize}
    \item Substitute $\widetilde\w=a_1\w$ and noise $\rn$.
    \item Total second moment stays $a_1^2+\rn=\rho_f$.
    \item Appendix B drops finite-$\ell$ corrections to these leading blocks.
  \end{itemize}
  \papersource{Paper, Lemma 1, Eqs. (12)--(14), PDF pp. 8--9; Appendix C, p. 10.}
\end{frame}

\begin{frame}{Backup: the no-signal branch has a scalar equation}
  When $a_1=0$, task randomness disappears from the leading moments.
  With $q=\rn/\alpha$:
  \[
  \chi_0\approx\frac{q}{q\tau/(1+\chi_0)+\tau\lambda}.
  \]
  \[
  \widetilde\lambda=\lambda(1+\chi_0)
  \longrightarrow
  \begin{cases}
    q(1-\tau)/\tau, & \tau<1,\\
    0, & \tau>1.
  \end{cases}
  \]
  Substitution into the inherited linear risk expression yields
  \[
  e_{\mathrm{ICL}}=\rn/(1-\tau)\quad\text{or}\quad
  e_{\mathrm{ICL}}=\rn\tau/(\tau-1).
  \]
  \papersource{Paper, Appendix C, Eq. (15), PDF p. 10.}
\end{frame}

\begin{frame}{Backup: a projection identity explains the error floor}
  \textbf{Teaching derivation.} Fix $\|\w\|^2=d$ and a context-dependent
  vector $v$ independent of the fresh query. Then $g=\x^\top\w\sim\mathcal N(0,1)$.
  \[
  \begin{aligned}
  \E[(f(g)+\varepsilon-\x^\top v)^2]
  &=\rho+\E[f(g)^2]-\frac{2a_1}{d}\w^\top v+\frac{\|v\|^2}{d}\\[2mm]
  &=\boxed{\rn+\frac{\|v-a_1\w\|^2}{d}}.
  \end{aligned}
  \]
  \begin{itemize}
    \item The best linear query prediction uses $v=a_1\w$.
    \item Estimation error is the second term; representational residual is $\rn$.
    \item The paper computes how pretraining and context control the estimation term asymptotically.
  \end{itemize}
  \papersource{Derived explanation based on paper, Eqs. (2), (5), (8). Normalized fixed task used for this identity.}
\end{frame}

\end{document}