\documentclass[aspectratio=169,11pt]{beamer}
\usepackage[T1]{fontenc}
\usepackage[utf8]{inputenc}
\usepackage{amsmath,amssymb,graphicx,booktabs}

\definecolor{AcademicBlue}{RGB}{26,57,86}
\definecolor{AcademicTeal}{RGB}{0,105,112}
\setbeamercolor{structure}{fg=AcademicBlue}
\setbeamercolor{frametitle}{fg=AcademicBlue,bg=white}
\setbeamercolor{title}{fg=AcademicBlue}
\setbeamercolor{alerted text}{fg=AcademicTeal}
\setbeamercolor{block title}{fg=white,bg=AcademicBlue}
\setbeamercolor{block body}{fg=black,bg=AcademicBlue!5}
\setbeamertemplate{navigation symbols}{}
\setbeamertemplate{itemize items}[circle]
\setbeamertemplate{footline}{%
  \hfill\usebeamerfont{page number in head/foot}%
  \insertframenumber\,/\,\inserttotalframenumber\hspace{4mm}\vspace{3mm}}
\setbeamersize{text margin left=8mm,text margin right=8mm}
\setbeamerfont{frametitle}{size=\Large,series=\bfseries}
\setbeamerfont{title}{size=\LARGE,series=\bfseries}
\newcommand{\papersource}[1]{\par\smallskip{\scriptsize\color{gray}#1\par}}
\newcommand{\Muir}{\textsc{Muir}}
\newcommand{\GDN}{\textsc{GDN-2}}

\title[Muir]{Muir: Linear Attention with Sensitivity-Allocated Gating and Matched-Filter Readout}
\subtitle{A paper discussion on fixed-state memory, readout, and TPU training}
\author{Tushaar Gangavarapu \and Gil Tabak \and Clemens JS Schaefer}
\date{arXiv:2611.08376v1 \quad|\quad 8 November 2026}

\begin{document}

\begin{frame}
  \titlepage
  \vspace{-5mm}
  \begin{center}
    \footnotesize
    Publication: arXiv (2026); no formal venue found.
    \href{https://research.google/pubs/muir-linear-attention-with-sensitivity-allocated-gating-and-matched-filter-readout/}{Google Research record}\\
    Code: JAX reference in Appendix J; no standalone repository identified.
    \href{https://arxiv.org/abs/2611.08376v1}{Paper}
  \end{center}
  \papersource{Paper identity: \href{https://arxiv.org/abs/2611.08376v1}{arXiv v1}; publication status checked 2026-11-08.}
\end{frame}

\begin{frame}{Muir treats three bottlenecks as one design problem}
  \begin{columns}[T,onlytextwidth]
    \column{0.32\textwidth}
      \begin{block}{Gate capacity}
        GDN-2 uses dense erase and write projections, even though state updates respond much more to writes.
      \end{block}
      \centering\alert{Allocate rank by sensitivity.}
    \column{0.32\textwidth}
      \begin{block}{State readout}
        Slow-decaying rows accumulate more energy and can dominate unweighted queries.
      \end{block}
      \centering\alert{Reweight rows when reading.}
    \column{0.32\textwidth}
      \begin{block}{Chunkwise training}
        Large chunks suit TPU matrix units, but decay division becomes unstable and triangular solves become sequential.
      \end{block}
      \centering\alert{Pivot and merge blocks.}
  \end{columns}
  \vspace{4mm}
  \begin{center}
    \large\textbf{Core claim:} better use of a fixed-size state comes from how it is written, read, and computed.
  \end{center}
  \papersource{Paper, Sec. 1, PDF pp. 1--3; Fig. 1, p. 2.}
\end{frame}

\begin{frame}{The gated Delta Rule stores a key-value association in a matrix state}
  For one head, $S_t\in\mathbb{R}^{d_k\times d_v}$ and normalized $k_t,q_t\in\mathbb{R}^{1\times d_k}$:
  \[
    S_t =
    \bigl(I-k_t^\top(b_t\odot k_t)\bigr)
    \operatorname{Diag}(\alpha_t)S_{t-1}
    +k_t^\top(w_t\odot v_t),
    \qquad o_t=q_tS_t.
  \]
  \begin{itemize}
    \item $\alpha_t$: channel-wise decay of the previous state rows.
    \item $b_t$: erase the current key's prediction from the decayed state.
    \item $w_t$: scale the incoming value write; $g_t$ gates the normalized output.
    \item State size is fixed with sequence length; key overlap creates interference.
  \end{itemize}
  \papersource{Paper, Sec. 2.1, Eq. (1), PDF p. 3.}
\end{frame}

\begin{frame}{Erase perturbations are filtered by the current key and stored row energy}
  \[
    D_bS_t[\delta b]
      =-k_t^\top\bigl((\delta b\odot k_t)\hat S_t\bigr),
    \qquad
    D_wS_t[\delta w]
      =k_t^\top(\delta w\odot v_t).
  \]
  Under isotropic unit-variance perturbations:
  \[
    \mathbb{E}\|D_bS_t\|_F^2
      =\sum_j k_{t,j}^2\|\hat S_{t,j,:}\|_2^2,
    \qquad
    \mathbb{E}\|D_wS_t\|_F^2=\|v_t\|_2^2.
  \]
  \begin{columns}[T,onlytextwidth]
    \column{0.49\textwidth}
      A write changes the state whenever a value arrives. Erasure matters when the key activates a row that already holds energy.
    \column{0.49\textwidth}
      Trained keys use about 6 of 128 channels effectively. Measured state sensitivity is 13.3$\times$ larger for writes than erasures.
  \end{columns}
  \papersource{Paper, Proposition 3.1 and Remarks 3.2--3.3, PDF pp. 5--6; trained diagnostics, Fig. 10 / Table 3, pp. 22--23.}
\end{frame}

\begin{frame}{Channel decay creates unequal state-row energies}
  For key energy $c_j=\mathbb{E}[k_{t,j}^2]$, squared decay $\lambda_j=\mathbb{E}[\alpha_{t,j}^2]$, mean erase $\bar b_j$, and write energy $\sigma_v^2$:
  \[
    m_j^\ast \approx
    \frac{c_j\sigma_v^2}
    {1-\lambda_j+2\lambda_j\bar b_jc_j}
    =c_j\sigma_v^2\,\tau_{\mathrm{eff},j}.
  \]
  \begin{itemize}
    \item $m_j^\ast$ is the expected squared norm of state row $j$.
    \item $\tau_{\mathrm{eff},j}$ is its effective memory horizon.
    \item Slow rows collect more writes; fast rows retain mostly recent inputs.
  \end{itemize}
  \begin{alertblock}{Evidence}
    At 166M, predicted and measured row energies have median within-head Spearman $r_s=0.960$ in Muir. At 1.34B, raw row-energy p90/p10 spread is 17.2$\times$.
  \end{alertblock}
  \papersource{Paper, Proposition 3.4, PDF p. 6; mechanistic validation, Fig. 11 / Table 3, p. 23.}
\end{frame}

\begin{frame}{The matched filter balances surviving signal against row interference}
  If a key-value pair is queried after lag $\ell$, the paper models MSE as
  \[
    \mathcal{L}_{\ell}(\gamma)
    =\sigma_v^2\left(1-\sum_j\gamma_jc_j\bar a_{\ell,j}\right)^2
    +\sum_j\gamma_j^2c_jm_j^\ast.
  \]
  The optimal direction is
  \[
    \gamma_j^\ast(\ell)\propto\frac{\bar a_{\ell,j}}{m_j^\ast}
    \approx\frac{\lambda_j^{\ell/2}}{m_j^\ast}.
  \]
  \begin{itemize}
    \item The numerator keeps channels whose target signal survives to the query.
    \item The denominator suppresses channels with high interference energy.
    \item Different lags imply different optimal filters; static and dynamic readout serve different query mixes.
  \end{itemize}
  \papersource{Paper, Proposition 3.6 and Remark 3.7, PDF pp. 7--8.}
\end{frame}

\begin{frame}{Static readout is a horizon average, not full whitening}
  For geometric retrieval lags with mean horizon $H$, the static gain is
  \[
    \gamma_j^{\mathrm{stat}}\propto
    \frac{g_H(\lambda_j)}{m_j^\ast},
    \qquad
    g_H(\lambda_j)=
    \frac{\sqrt{\lambda_j}}
    {1+(H-1)(1-\sqrt{\lambda_j})}.
  \]
  \begin{itemize}
    \item Dividing by row RMS alone over-amplifies fast channels whose target signal has already decayed.
    \item The trained static gain follows an intermediate power law $\gamma_j\propto\rho_j^{-p}$ with median $p\approx0.35$.
    \item Readout suppresses interference at query time, allowing lower write-time erasure and longer retention.
  \end{itemize}
  \papersource{Paper, Corollary 3.8 and Proposition 3.10, PDF pp. 8, 43; trained exponent, Fig. 11, p. 23.}
\end{frame}

\begin{frame}{Controlled memory experiments support the theory}
  \centering
    \includegraphics[width=0.97\linewidth,height=0.54\textheight,keepaspectratio]{figures/fig2_toys.png}
  \begin{itemize}
    \item At mean lag $H=27$, the theory-based filter matches the static least-squares oracle and shifts optimal erase strength from 0.60 to 0.50.
    \item Low-rank erase gates improve long-lag recall; overly compressing the write gate hurts.
  \end{itemize}
  \papersource{Paper, Sec. 3.2 / Fig. 2, PDF pp. 8--9; synthetic summaries over 20 seeds.}
\end{frame}

\begin{frame}{Muir adds a readout gate and shares one projection across five gates}
  \[
    o_t=(\gamma_t\odot q_t)S_t,\qquad
    \gamma_t=\gamma^{\mathrm{stat}}\odot2\sigma(z_t^q).
  \]
  \centering
  \includegraphics[width=0.96\linewidth,height=0.56\textheight,keepaspectratio]{figures/fig3_architecture.png}
  \papersource{Paper, Sec. 4.1 / Fig. 3, PDF p. 10; dynamic gain initializes to one.}
\end{frame}

\begin{frame}{Rank follows sensitivity while the total fills two TPU tiles}
  \begin{center}
    \begin{tabular}{lccccc}
      \toprule
      Gate & Decay $f$ & Erase $b$ & Output $g$ & Write $w$ & Readout $q$ \\
      \midrule
      Rank & 64 & 64 & 128 & 128 & 128 \\
      \bottomrule
    \end{tabular}
  \end{center}
  \begin{itemize}
    \item Shared down-projection has total rank $R_{\mathrm{tot}}=512$; gate-specific up-projections retain channel biases.
    \item At $d=2{,}048$, gate projection shrinks from 9.44M to 2.10M parameters (4.5$\times$ fewer).
    \item Same $12d^2$ layer budget: FFN width grows from 5,888 to 7,680 (+30\%).
    \item Without dynamic readout, its 128 ranks move to the write gate: $r_w=256$.
  \end{itemize}
  \papersource{Paper, Sec. 4.1 / Fig. 3, PDF pp. 10--11; full scales, Appendix G / Table 11, p. 49.}
\end{frame}

\begin{frame}{Chunkwise Delta Rule becomes a triangular solve}
  With relative channel decay $\pi_{t,s}=\exp(\Lambda_t-\Lambda_s)$, one chunk forms
  \[
    T=I+L_{\mathrm{strict}},
    \qquad
    TZ=\begin{bmatrix}\hat V & \hat K_{\odot\pi}\end{bmatrix},
    \qquad
    Z=\begin{bmatrix}U&Y\end{bmatrix}.
  \]
  The chunk output and outgoing state are affine maps of its initial state $S_0$.
  \begin{itemize}
    \item Divide by cumulative decay to build matrix products: fast-channel prefixes underflow.
    \item Form every pairwise decay explicitly: materializes a $C\times C\times d_k$ tensor.
    \item Use forward substitution: serial work on the TPU's slower vector unit.
  \end{itemize}
  \papersource{Paper, Sec. 2.2 and Sec. 4.2, PDF pp. 4, 11--12.}
\end{frame}

\begin{frame}{Boundary pivots keep every decay factor in range}
  Split any cross-block decay at the boundary pivot $p$:
  \[
    \pi_{t,s}
    =\exp(\Lambda_t-\Lambda_p)\odot\exp(\Lambda_p-\Lambda_s),
    \qquad s\le p<t.
  \]
  Both log differences are non-positive, so each exponential lies in $(0,1]$.
  \begin{itemize}
    \item Invert 8-token leaf blocks with a finite Neumann series.
    \item Merge adjacent triangular blocks in a five-level divide-and-conquer tree.
    \item Recompute pivot scalings in the custom backward pass instead of storing tree activations.
  \end{itemize}
  \papersource{Paper, Sec. 4.2, Eqs. (7)--(9), PDF pp. 11--13; $C=256$, leaf size 8.}
\end{frame}

\begin{frame}{The kernel routes most work to TPU matrix units}
  \centering
  \includegraphics[width=0.97\linewidth,height=0.54\textheight,keepaspectratio]{figures/fig4_kernel_design.png}
  \begin{itemize}
    \item At $C=256$, 98\% of chunkwise FLOPs land on MXU; VPU compute falls 24$\times$.
    \item Fused $256\times256$ solves use the full systolic tile; the state scan stays in VMEM.
  \end{itemize}
  \papersource{Paper, Sec. 4.2 / Fig. 4, PDF pp. 13--14; Appendix F / Table 8, p. 46.}
\end{frame}

\begin{frame}{Numerical accuracy depends on the triangular system being well conditioned}
  \centering
  \includegraphics[width=0.97\linewidth,height=0.50\textheight,keepaspectratio]{figures/fig9_kernel_stability.png}
  \begin{itemize}
    \item Naive unpivoted fp32 produces NaNs on 25\% of tested heads; pivoted bf16 output/state errors are about $7\times10^{-3}$.
    \item Sampled training systems have $\kappa_2(T)<40$; synthetic adversarial systems with $\kappa_2(T)\approx12{,}865$ exceed the bf16 solver's stable range.
  \end{itemize}
  \papersource{Paper, Sec. 5.2.4 / Fig. 9, PDF p. 20; Appendix F / Table 10, p. 47.}
\end{frame}

\begin{frame}{Pretraining compares equal-budget models under a shared recipe}
  \begin{itemize}
    \item Four parameter scales: 166M, 406M, 834M, and 1.34B.
    \item 100B FineWeb-Edu tokens, 4K training length, 32K tokenizer, same token order.
    \item 8 TPU7x chips, 524,288 tokens per step, bf16 mixed precision and fp32 state.
    \item Per-layer non-embedding budget: $12d^2$; recurrent state width: $d_k=d_v=128$.
    \item AdamW with 1B-token warmup and cosine decay; pretraining uses fixed seed 0.
  \end{itemize}
  \begin{alertblock}{Comparisons}
    Transformer + Pallas Splash Attention; GDN-2; Muir without, with static, or with dynamic readout gain.
  \end{alertblock}
  \papersource{Paper, Sec. 5.1 and Appendix G, PDF pp. 14--15, 49--50.}
\end{frame}

\begin{frame}{Muir lowers validation NLL across all four scales}
  \centering
  \includegraphics[width=0.98\linewidth,height=0.53\textheight,keepaspectratio]{figures/fig5_nll_curves.png}
  \begin{itemize}
    \item Full Muir is 18.0--20.4 mnat below GDN-2 after 100B tokens.
    \item At 1.34B: 1.9973 vs.\ 2.0177 for GDN-2 and 2.0335 for Transformer.
    \item Muir reaches GDN-2's final validation loss by about 85B tokens.
  \end{itemize}
  \papersource{Paper, Sec. 5.2.1 / Fig. 5 and Table 1, PDF pp. 15--16.}
\end{frame}

\begin{frame}{Readout helps context-sensitive tasks beyond FFN expansion}
  \begin{center}
    \begin{tabular}{lrrr}
      \toprule
      1.34B metric & Transformer & GDN-2 & Muir \\
      \midrule
      8-task zero-shot accuracy & 52.05\% & 52.46\% & 53.84\% \\
      LAMBADA word PPL & 10.30 & 10.38 & 10.00 \\
      5-shot TriviaQA accuracy & 14.56\% & 16.48\% & 19.32\% \\
      Based/JRT 2K-cut average & 45.92\% & 32.08\% & 34.60\% \\
      \bottomrule
    \end{tabular}
  \end{center}
  \begin{itemize}
    \item FFN expansion explains much of the short-prompt commonsense gain.
    \item Dynamic readout contributes more on tasks mixing local context and distant demonstrations.
    \item Muir improves real-document extraction over GDN-2, while Transformer keeps the stronger overall Based/JRT average.
  \end{itemize}
  \papersource{Paper, Table 1, PDF p. 16; Based/JRT protocol and full table, Appendix H / Table 14, p. 53.}
\end{frame}

\begin{frame}{Dynamic readout improves multi-hop and multi-needle recall}
  \centering
  \includegraphics[width=0.98\linewidth,height=0.56\textheight,keepaspectratio]{figures/fig6_recall.png}
  \begin{itemize}
    \item RULER six-task mean at 1K--4K: 73.0\% Muir, 61.8\% GDN-2, 70.9\% Transformer.
    \item Five-hop Variable Tracking: 34.3\% vs.\ 21.5\% GDN-2 and 8.8\% Transformer.
    \item Based/JRT 2K-cut mean: 34.60\% vs.\ 32.08\% GDN-2.
  \end{itemize}
  \papersource{Paper, Sec. 5.2.2 / Fig. 6, PDF p. 17; Appendix H / Tables 13--14, pp. 52--53.}
\end{frame}

\begin{frame}{Dynamic readout keeps language-modeling loss stable on long streams}
  \centering
  \includegraphics[width=0.98\linewidth,height=0.49\textheight,keepaspectratio]{figures/fig7_longctx.png}
  \begin{itemize}
    \item Zero-shot from 4K, full Muir reaches 8.11 PPL at 10M tokens; GDN-2 is 8.31.
    \item The 24-layer fp32 recurrent state is 25 MB, independent of stream length.
    \item Static gain drifts on continuous text; a 64K cooldown recalibrates it.
  \end{itemize}
  \papersource{Paper, Sec. 5.2.3 / Fig. 7, PDF p. 18; Appendix H / Table 16, p. 55.}
\end{frame}

\begin{frame}{Static readout is better for a single passkey query}
  \centering
  \includegraphics[width=0.98\linewidth,height=0.57\textheight,keepaspectratio]{figures/fig8_longctx_niah.png}
  \begin{itemize}
    \item Static-gain Muir reaches 90.0\% zero-shot and 99.4\% after cooldown at 1M tokens.
    \item Dynamic Muir trades write rank for lag-adaptive routing and tapers beyond 32K on this single-needle task.
  \end{itemize}
  \papersource{Paper, Sec. 5.2.3 / Fig. 8, PDF p. 19; RULER S-NIAH-1, Appendix H / Table 17, p. 56.}
\end{frame}

\begin{frame}{Trained models exhibit the predicted gate-sensitivity asymmetry}
  \centering
  \includegraphics[width=0.96\linewidth,height=0.50\textheight,keepaspectratio]{figures/fig10_gate_sensitivity.png}
  \begin{itemize}
    \item SVD truncation: decay and erase saturate by rank 64; write, output, and query readout need rank 128.
    \item Freezing the write gate increases NLL 14.5$\times$ more than freezing erase.
  \end{itemize}
  \papersource{Paper, Sec. 6 / Fig. 10, PDF p. 22; supplementary rank sweep in Appendix I / Table 19, p. 58.}
\end{frame}

\begin{frame}{Readout gain compresses row-energy imbalance}
  \centering
  \includegraphics[width=0.96\linewidth,height=0.52\textheight,keepaspectratio]{figures/fig11_row_energy.png}
  \begin{itemize}
    \item Gain-weighted p90/p10 row-energy spread: 9.6$\times$ in Muir vs.\ 14.7$\times$ in GDN-2.
    \item Learned static gains follow $\gamma_j\propto\rho_j^{-0.35}$, between unit gain and full whitening.
    \item Mean erase rate falls 16.6\%, preserving longer-lived associations.
  \end{itemize}
  \papersource{Paper, Sec. 6 / Fig. 11 and Table 3, PDF p. 23.}
\end{frame}

\begin{frame}{Dynamic gain routes readout across the hops of a binding chain}
  \centering
  \includegraphics[width=0.98\linewidth,height=0.56\textheight,keepaspectratio]{figures/fig13_mv_hop_recall.png}
  \begin{itemize}
    \item Root binding retention at query: 37\% in GDN-2, 45\% in Muir.
    \item Dynamic gain shifts readout toward the target hop across answer steps.
  \end{itemize}
  \papersource{Paper, Sec. 6 / Fig. 13, PDF p. 24; full write-order and hop-resolved results, Appendix I / Table 23, p. 61.}
\end{frame}

\begin{frame}{What the evidence supports, and where the design trades off}
  \begin{columns}[T,onlytextwidth]
    \small
    \column{0.53\textwidth}
      \textbf{Supported by the reported experiments}
        \begin{itemize}\setlength{\itemsep}{1pt}
          \item Sensitivity-aware gate ranks preserve quality and free FFN capacity.
          \item Matched-filter readout reduces row imbalance and write-time erasure.
          \item Boundary pivots enable stable large-chunk TPU training in sampled checkpoints.
        \end{itemize}
    \column{0.43\textwidth}
      \textbf{Specific trade-offs}
        \begin{itemize}\setlength{\itemsep}{1pt}
          \item Static gain: single-lag retrieval to 1M.
          \item Dynamic gain: multi-hop and 10M stream modeling.
          \item Transformer is faster below roughly 11K.
          \item Theory uses stationary and second-order approximations; training uses one seed.
        \end{itemize}
  \end{columns}
  \vspace{-1mm}
  \begin{center}
    \normalsize\textbf{Take-home:} state capacity depends on how the model allocates write, erase, and readout control.
  \end{center}
  \papersource{Paper, Secs. 5--7, PDF pp. 14--25; full identity: \href{https://arxiv.org/abs/2611.08376v1}{arXiv:2611.08376v1}.}
\end{frame}

\end{document}