\documentclass[aspectratio=169,11pt]{beamer}
\usepackage[T1]{fontenc}
\usepackage[utf8]{inputenc}
\usepackage{amsmath,amssymb,graphicx,booktabs,array}

\definecolor{AcademicBlue}{RGB}{26,57,86}
\definecolor{AcademicTeal}{RGB}{0,105,112}
\setbeamercolor{structure}{fg=AcademicBlue}
\setbeamercolor{frametitle}{fg=AcademicBlue,bg=white}
\setbeamercolor{title}{fg=AcademicBlue}
\setbeamercolor{alerted text}{fg=AcademicTeal}
\setbeamercolor{block title}{fg=white,bg=AcademicBlue}
\setbeamercolor{block body}{fg=black,bg=AcademicBlue!5}
\setbeamertemplate{navigation symbols}{}
\setbeamertemplate{itemize items}[circle]
\setbeamertemplate{footline}{%
  \hfill\usebeamerfont{page number in head/foot}%
  \insertframenumber\,/\,\inserttotalframenumber\hspace{4mm}\vspace{3mm}}
\setbeamersize{text margin left=8mm,text margin right=8mm}
\setbeamerfont{frametitle}{size=\Large,series=\bfseries}
\setbeamerfont{title}{size=\LARGE,series=\bfseries}
\newcommand{\papersource}[1]{\par\smallskip{\scriptsize\color{gray}#1\par}}
\newcommand{\swila}{SwiLA}

\title[Switching Linear Attention]{Switching Linear Attention}
\subtitle{Expressive retrieval with a fixed-size recurrent state}
\author{\small Paper by Hyun Dong Lee, Xavier Gonzalez, Nicolas Zucchet,\\
E. Kelly Buchanan, Emily B. Fox, and Scott W. Linderman}
\date{\small \href{https://arxiv.org/abs/2609.39034v1}{arXiv:2609.39034v1} \quad $\cdot$ \quad Research group discussion, 28 October 2026}

\begin{document}

\begin{frame}
  \titlepage
  \vspace{-2mm}
  \begin{center}
    {\small Published in \href{https://proceedings.colm.cc/paper_files/paper/2026}{COLM 2026 pre-proceedings} (venue DOI pending)
    \quad $\cdot$ \quad \href{https://github.com/lindermanlab/switching-linear-attention}{Official code}}
  \end{center}
\end{frame}

\begin{frame}{SwiLA changes the function stored in recurrent memory}
  \begin{block}{One-sentence takeaway}
    Replace one online linear regressor with a mixture of linear regressors:
    input- and error-dependent updates, query-dependent readout, fixed state size.
  \end{block}
  \begin{columns}[T]
    \begin{column}{0.49\linewidth}
      \textbf{What improves}
      \begin{itemize}
        \item Context-dependent associative recall
        \item Approximation of softmax retrieval at matched recurrent state size
      \end{itemize}
    \end{column}
    \begin{column}{0.49\linewidth}
      \textbf{What costs more}
      \begin{itemize}
        \item Nonlinear, sequential state update
        \item More training memory and lower throughput than GDN
      \end{itemize}
    \end{column}
  \end{columns}
  \papersource{Paper, abstract and Secs. 1, 3, 5--6; PDF pp. 1--10.}
\end{frame}

\begin{frame}{The bottleneck is the query-to-value map}
  \centering
  \includegraphics[width=0.88\linewidth,height=0.53\textheight,keepaspectratio]{figures/fig1-model-clean.png}
  \vspace{1mm}
  \begin{itemize}
    \item Softmax attention retains all past KV pairs; cache grows with sequence length.
    \item Linear attention compresses history into one matrix per head.
    \item SwiLA stores several matrices and mixes their predictions at readout.
  \end{itemize}
  \papersource{Original paper Fig. 1B, PDF p. 3; image cropped from the authors' PDF figure.}
\end{frame}

\begin{frame}{Test-time regression turns attention into online memorization}
  \[
    m_t \approx \arg\min_{m\in\mathcal M}
      \sum_{i=1}^{t}\|v_i-m(k_i)\|_2^2,
    \qquad o_t=m_t(q_t).
  \]
  \begin{itemize}
    \item Keys $k_i$, values $v_i$, and queries $q_t$ are input projections.
    \item The recurrence updates the \emph{fast weights} of $m_t$ inside each sequence.
    \item DeltaNet: one linear map, $m_t(q)=W_tq$.
    \item Softmax: a nonparametric kernel smoother over stored KV pairs.
  \end{itemize}
  \begin{block}{SwiLA design choice}
    Function class: mixture of linear regressors. \quad
    Update: online EM-style stochastic gradient step.
  \end{block}
  \papersource{Paper Sec. 2, Eqs. (2.1)--(2.2), PDF p. 3; Sec. 3, PDF p. 3.}
\end{frame}

\begin{frame}{Each output dimension can choose its own expert}
  \[
    p(z_{td}=j\mid k_t)=\pi_{jd}(k_t),\qquad
    v_{td}\mid z_{td}=j,k_t
      \sim\mathcal N(w_{jd}^{\top}k_t,1).
  \]
  \begin{itemize}
    \item $j=1,\ldots,J$: expert; $d=1,\ldots,D$: value dimension.
    \item $w_{jd}\in\mathbb R^D$; all experts form $W\in\mathbb R^{J\times D\times D}$.
    \item Independent dimension-wise assignments permit $J^D$ combinations,
      while the matrix state has $JD^2$ scalars per head.
  \end{itemize}
  \begin{block}{Interpretation}
    This is a soft, dimension-wise switch among linear retrieval maps.
  \end{block}
  \papersource{Paper Sec. 3.1, Eq. (3.1), PDF p. 3. $J^D$ counts possible assignments, not guaranteed realized modes.}
\end{frame}

\begin{frame}{Write: responsibility weights the delta-rule error}
  \[
    \delta_{tjd}=v_{td}-w_{t-1,jd}^{\top}k_t,\qquad
    r_{tjd}=
      \frac{\pi_{jd}(k_t)e^{-\delta_{tjd}^2/2}}
      {\sum_{j'}\pi_{j'd}(k_t)e^{-\delta_{tj'd}^2/2}}.
  \]
  \[
    \boxed{w_{tjd}=w_{t-1,jd}
      +\beta_{tjd}r_{tjd}\delta_{tjd}k_t.}
  \]
  \begin{itemize}
    \item Prior $\pi$: input-based routing; $r$: posterior after observing $v_{td}$.
    \item $\delta$: prediction error; $\beta\in[0,1]$: learned step size.
    \item $r$ depends on the old state, making the recurrence nonlinear.
  \end{itemize}
  \papersource{Paper Sec. 3.2, Eqs. (3.2)--(3.4), PDF p. 4; gradient derivation App. B.1, p. 20.}
\end{frame}

\begin{frame}{Read: the query prior mixes expert predictions}
  \[
    \boxed{o_{td}=\sum_{j=1}^{J}
       \pi_{jd}(q_t)\,w_{tjd}^{\top}q_t.}
  \]
  \begin{itemize}
    \item The readout uses $\pi(q_t)$, because the value $v_{td}$ is unknown at retrieval.
    \item A weighted sum of linear maps becomes nonlinear in the input when the weights change with the query.
    \item The state is still fixed-size with respect to context length:
      $O(HJD^2)$ matrix scalars for $H$ heads.
  \end{itemize}
  \begin{block}{Key distinction}
    Write routing uses observed prediction error; read routing uses the query-side prior.
  \end{block}
  \papersource{Paper Sec. 3.2, Eq. (3.5), PDF p. 4; App. B.2, p. 20.}
\end{frame}

\begin{frame}{Temporal SwiLA carries context across tokens}
  \[
    \bar r_{tjd}=(1-g^k_{td})r_{t-1,jd}
                  +g^k_{td}\pi^k_{jd}(k_t)
  \]
  \[
    \bar\rho_{tjd}=(1-g^q_{td})\bar\rho_{t-1,jd}
                  +g^q_{td}\pi^q_{jd}(q_t).
  \]
  \begin{itemize}
    \item The filtered key distribution becomes the next write prior.
    \item The filtered query distribution becomes the read prior.
    \item Gate $g$ near zero preserves context; near one accepts current input.
  \end{itemize}
  \begin{block}{Why it matters}
    A context marker may appear once, many positions before the answer.
  \end{block}
  \papersource{Paper Sec. 3.3, PDF p. 4; App. B.3, Eqs. (B.3)--(B.5), PDF pp. 20--21.}
\end{frame}

\begin{frame}{Gating and balanced routing solve different problems}
  \textbf{State decay (the implemented pre-decay form)}
  \[
    w_{tjd}=\alpha_{tjd}w_{t-1,jd}
       +\beta_{tjd}r^-_{tjd}\delta^-_{tjd}k_t,\quad
    \delta^-_{tjd}=v_{td}-\alpha_{tjd}w_{t-1,jd}^{\top}k_t.
  \]
  \textbf{Avoiding expert collapse}
  \[
    \mathcal L_{\rm MI}
      =-\lambda_{\rm marg}H_{\rm marg}
       +\lambda_{\rm cond}H_{\rm cond}.
  \]
  \begin{itemize}
    \item High marginal entropy spreads usage across experts.
    \item Low conditional entropy encourages token-level specialization.
    \item Training-time noisy router logits are another option; the 374M language models disable that noise.
  \end{itemize}
  \papersource{Paper Secs. 3.4--3.5, Eqs. (3.8)--(3.10), PDF p. 5; App. E.3, p. 30.}
\end{frame}

\begin{frame}{SwiLA differs from DeltaNet and input-only memory routing}
  \centering
  \small
  \begin{tabular}{>{\raggedright\arraybackslash}p{2.8cm}>{\raggedright\arraybackslash}p{3.6cm}>{\raggedright\arraybackslash}p{4.7cm}}
    \toprule
    Layer & Stored state & What controls the update?\\
    \midrule
    DeltaNet & One linear matrix per head & Prediction error, single map\\
    MoM & Several memory matrices & Router based on input, independent of old state\\
    SwiLA & Several regressors, each output dimension routed & Posterior uses input prior \emph{and} old-state prediction error\\
    Softmax attention & All earlier KV pairs & Query--key similarities at read time\\
    \bottomrule
  \end{tabular}
  \vspace{3mm}
  \begin{block}{Mechanistic claim}
    SwiLA's state-dependent responsibility creates a nonlinear recurrence;
    its query-side mixture creates a nonlinear retrieval map.
  \end{block}
  \papersource{Paper Sec. 4, PDF pp. 5--6.}
\end{frame}

\begin{frame}{The experiments test expressivity, recall, and systems cost}
  \small
  \begin{tabular}{>{\raggedright\arraybackslash}p{2.8cm}>{\raggedright\arraybackslash}p{5.2cm}>{\raggedright\arraybackslash}p{3.3cm}}
    \toprule
    Setting & Main question & Primary metric\\
    \midrule
    Softmax fit & Better approximation at equal state size? & Test $R^2$ $\uparrow$\\
    CDAR & Store conflicting key--value maps and persist context? & Answer accuracy $\uparrow$\\
    RegBench & Learn a finite automaton from context examples? & Test accuracy $\uparrow$\\
    374M language models & Does retrieval improve after 15B training tokens? & Commonsense / recall $\uparrow$\\
    1.3B profiling & What is the compute cost? & Throughput and memory\\
    115M ablation & When does expert routing collapse? & Test perplexity\\
    \bottomrule
  \end{tabular}
  \papersource{Paper Sec. 5, PDF pp. 6--10; experimental details App. E, pp. 27--33.}
\end{frame}

\begin{frame}{Mixtures better fit a softmax head at matched state size}
  \begin{columns}[c]
    \begin{column}{0.52\linewidth}
      \centering
      \includegraphics[width=\linewidth,height=0.69\textheight,keepaspectratio]{figures/fig1-fit.pdf}
    \end{column}
    \begin{column}{0.45\linewidth}
      \begin{itemize}
        \item Target: causal softmax outputs from clustered inputs, $D=32$.
        \item Budget $B=H J$; each head has a $D\times D$ state.
        \item At $B=2$, $H1J2$ exceeds DeltaNet $H2$ in test $R^2$.
        \item At $B=4$, the best split is $H2J2$.
      \end{itemize}
    \end{column}
  \end{columns}
  \papersource{Original Fig. 1C, PDF p. 3; Sec. 5.1, p. 7. $B=1$ has no multi-expert comparator.}
\end{frame}

\begin{frame}{Temporal SwiLA solves context-dependent recall with less state}
  \begin{columns}[c]
    \begin{column}{0.58\linewidth}
      \centering
      \includegraphics[width=\linewidth,height=0.65\textheight,keepaspectratio]{figures/figure2a.pdf}
    \end{column}
    \begin{column}{0.40\linewidth}
      \begin{itemize}
        \item Four contexts share the same 32 keys; each context assigns different values.
        \item Context marker appears once per query block.
        \item One-layer Temporal SwiLA with $J=4$ is near-perfect at $2^{12}$ state scalars.
        \item One-layer softmax does not solve this setup; two layers do.
      \end{itemize}
    \end{column}
  \end{columns}
  \papersource{Original Fig. 2A, Sec. 5.2, PDF p. 8. Main plot uses best result across learning rates and three seeds; mean results in Fig. S2, p. 28.}
\end{frame}

\begin{frame}{RegBench shows a low-data advantage, then convergence}
  \begin{columns}[c]
    \begin{column}{0.58\linewidth}
      \centering
      \includegraphics[width=\linewidth,height=0.65\textheight,keepaspectratio]{figures/figure2b.pdf}
    \end{column}
    \begin{column}{0.40\linewidth}
      \begin{itemize}
        \item Eight-layer models infer a probabilistic finite automaton in context.
        \item At 1,000 training automata, Temporal SwiLA and DN-MoM exceed Transformer accuracy.
        \item At 20,000, Transformer reaches the highest accuracy; models are close.
      \end{itemize}
    \end{column}
  \end{columns}
  \papersource{Original Fig. 2B, Sec. 5.2, PDF p. 8; protocol in App. E.2, p. 29.}
\end{frame}

\begin{frame}{The language-model comparison matches state and parameter scale}
  \begin{itemize}
    \item 13 models: 24 layers, width 1,024, about 374M parameters.
    \item FineWeb-Edu pretraining: 15B tokens, sequence length 4,096.
    \item SwiLA: $8$ heads $\times$ $4$ experts $\times$ $64^2$
      $=131{,}072$ matrix-state scalars per layer.
    \item DeltaNet, GDN, KDA: $8\times128^2=131{,}072$.
    \item Hybrid models place six full-attention layers among 18 recurrent layers.
  \end{itemize}
  \begin{block}{Evaluation}
    Table 1: eight commonsense accuracies and perplexities.\\
    Table 2: six recall tasks, contains metric, inputs truncated to 2K tokens.
  \end{block}
  \papersource{Paper Sec. 5.3, Tables 1--2, PDF pp. 8--10; App. E.3, p. 30.}
\end{frame}

\begin{frame}{KDA leads commonsense; Gated SwiLA is close}
  \centering
  \small
  \begin{tabular}{lcc}
    \toprule
    Model (374M) & WikiText PPL $\downarrow$ & Commonsense avg.\ \% $\uparrow$\\
    \midrule
    Transformer++ & 28.2 & 45.3\\
    DeltaNet & 29.8 & 43.7\\
    GDN & 27.7 & 45.0\\
    KDA & \textbf{23.4} & \textbf{46.6}\\
    SwiLA & 28.8 & 44.7\\
    Temporal SwiLA & 28.6 & 45.2\\
    Gated SwiLA & 27.8 & 45.4\\
    Gated Temporal SwiLA & 27.8 & 45.4\\
    \bottomrule
  \end{tabular}
  \vspace{2mm}
  \par Gated SwiLA exceeds Transformer++ by 0.1 point; KDA remains strongest.
  \papersource{Selected exact values from original Table 1, PDF p. 9. PPL lower is better; accuracy higher is better.}
\end{frame}

\begin{frame}{SwiLA improves fixed-state recall; full KV caches still lead}
  \centering
  \small
  \begin{tabular}{lcc}
    \toprule
    Model (374M) & Recall avg.\ contains \% $\uparrow$ & SWDE / FDA \%\\
    \midrule
    DeltaNet & 17.3 & 7.9 / 5.4\\
    GDN & 19.2 & 9.1 / 5.5\\
    KDA & 20.3 & 13.9 / 5.5\\
    SwiLA & 19.7 & 12.9 / 8.0\\
    Gated SwiLA & 20.5 & 10.2 / 10.1\\
    Gated Temporal SwiLA & \textbf{20.6} & 12.0 / 7.5\\
    \midrule
    Hybrid GDN & 29.5 & 25.0 / 38.2\\
    Hybrid Temporal SwiLA & 31.8 & 32.3 / 49.1\\
    Transformer++ & 28.3 & 28.4 / 29.5\\
    FoX & \textbf{33.7} & 36.4 / 53.0\\
    \bottomrule
  \end{tabular}
  \papersource{Selected exact values from original Table 2, PDF p. 10. All inputs truncated to 2K tokens; hybrid models include six full-attention layers.}
\end{frame}

\begin{frame}{SwiLA's sequential update halves training throughput vs GDN}
  \centering
  \includegraphics[width=0.88\linewidth,height=0.59\textheight,keepaspectratio]{figures/train-throughput.pdf}
  \par\smallskip
  {\small 1.3B models, one H100, 131,072 tokens/update: SwiLA 19--22K tokens/s vs GDN about 40K tokens/s. SwiLA also allocates more peak memory.}
  \papersource{Original Fig. S3, PDF p. 31; top panel cropped. Timed update includes forward, backward, clipping, and AdamW step.}
\end{frame}

\begin{frame}{Fixed-state inference permits larger batches on one H100}
  \centering
  \includegraphics[width=0.88\linewidth,height=0.59\textheight,keepaspectratio]{figures/infer-throughput.pdf}
  \par\smallskip
  {\small 2,048-token prompt + 128-token decode. Batch groups left to right: 1, 2, 4, 8, 16, 32, 64, 128. Transformer OOMs at large batches; SwiLA reaches 128, but GDN remains faster.}
  \papersource{Original Fig. S4, PDF p. 32; throughput panel cropped. The paper's 4--5$\times$ advantage over Transformer reflects feasible batch sizes, not matched-batch latency.}
\end{frame}

\begin{frame}{Temporal persistence or load balancing prevents collapse}
  \centering
  \small
  \begin{tabular}{lcc}
    \toprule
    115M model, 5B training tokens & Experts & Test PPL $\downarrow$\\
    \midrule
    SwiLA, no load balancing & 4 & 36.769\\
    SwiLA, load balancing & 4 & 16.807\\
    Temporal SwiLA, no load balancing & 4 & 16.718\\
    Temporal SwiLA, load balancing & 4 & \textbf{16.715}\\
    Temporal SwiLA, load balancing & 16 & 16.862\\
    \bottomrule
  \end{tabular}
  \vspace{3mm}
  \begin{itemize}
    \item Without balancing, plain SwiLA's KV-side dead-expert fraction exceeds 40\%.
    \item At fixed state size, more experts shrink each expert's projection; 16 experts slightly worsen PPL.
  \end{itemize}
  \papersource{Exact legend values and router diagnostics from original Fig. S5, PDF p. 33; Sec. 5.5, pp. 9--10.}
\end{frame}

\begin{frame}{What the paper establishes and what remains expensive}
  \begin{enumerate}
    \item A mixture of online regressors yields a nonlinear query readout with a state independent of context length.
    \item Temporal routing is especially useful when a context marker must persist across many tokens.
    \item In 374M models, pure recurrent SwiLA variants improve recall over DeltaNet; FoX's full KV cache still leads on recall.
    \item The nonlinear recurrence currently trains sequentially and costs about 2$\times$ GDN training time in the 1.3B profile.
  \end{enumerate}
  \begin{block}{Open systems question}
    Can chunkwise parallel Newton iterations outperform the current sequential kernel in realistic training?
  \end{block}
  \papersource{Paper Sec. 6, PDF p. 10; App. D, pp. 23--26; App. E.4, pp. 31--32.}
\end{frame}

\begin{frame}{Paper, venue, and code}
  \begin{itemize}
    \item Hyun Dong Lee, Xavier Gonzalez, Nicolas Zucchet, E. Kelly Buchanan, Emily B. Fox, Scott W. Linderman. \emph{Switching Linear Attention}. \href{https://arxiv.org/abs/2609.39034v1}{arXiv:2609.39034v1}.
    \item \href{https://proceedings.colm.cc/paper_files/paper/2026}{COLM 2026 pre-proceedings} and \href{https://colm.eventhosts.cc/Conferences/2026/AcceptedPapers}{official accepted-papers list}. Conference DOI pending as of 28 October 2026.
    \item \href{https://github.com/lindermanlab/switching-linear-attention}{Official implementation}.
  \end{itemize}
  \vspace{4mm}
  {\small All figures shown are from the authors' arXiv v1 source package; selected panels are accurately cropped.}
\end{frame}

\appendix
\begin{frame}{Backup: the update is one marginal-likelihood gradient step}
  \[
  \ell_{td}=\log\sum_j\pi_{jd}(k_t)
       \exp\!\left[-\frac12(v_{td}-w_{t-1,jd}^{\top}k_t)^2\right]+c.
  \]
  \[
    \nabla_{w_{t-1,jd}}\ell_{td}
       =r_{tjd}(v_{td}-w_{t-1,jd}^{\top}k_t)k_t.
  \]
  \begin{itemize}
    \item EM language highlights latent responsibilities.
    \item The actual online parameter update uses one stochastic gradient step;
      no exact batch M-step is performed over stored history.
  \end{itemize}
  \papersource{Paper Eqs. (3.2)--(3.4), PDF p. 4; derivation App. B.1, p. 20.}
\end{frame}

\begin{frame}{Backup: proposed parallelization has not sped up training}
  \[
    \frac{\partial f}{\partial w_{t-1,jd}}
        \approx I-g_{tjd}k_tk_t^\top,\quad
    g_{tjd}=\beta_{tjd}r_{tjd}
       [1-\delta_{tjd}^2(1-r_{tjd})].
  \]
  \begin{itemize}
    \item Each Newton iteration becomes a linear recurrence with DeltaNet's
      identity-minus-rank-one structure.
    \item A chunkwise algorithm could avoid materializing every matrix state.
    \item The Jacobian ignores cross-expert coupling from responsibility normalization.
    \item Around 10 iterations converged on synthetic inputs, but the present Triton implementation was no faster than the sequential kernel.
  \end{itemize}
  \papersource{Paper App. D, Eqs. (D.4)--(D.8), PDF pp. 24--25; limitations in Sec. 6, p. 10.}
\end{frame}

\end{document}