% Self-authored reading presentation based on the skill's Beamer template.
% Main presentation: frames 1--22, approximately 25 minutes.
% Frames 23--28 are optional technical backup.
% Compile: python3 build_slides.py slides.en.tex
\documentclass[aspectratio=169,11pt]{beamer}
\usepackage[T1]{fontenc}
\usepackage[utf8]{inputenc}
\usepackage{lmodern}
\usepackage{amsmath,amssymb,graphicx,booktabs}

\definecolor{AcademicBlue}{RGB}{26,57,86}
\definecolor{AcademicTeal}{RGB}{0,105,112}
\setbeamercolor{structure}{fg=AcademicBlue}
\setbeamercolor{frametitle}{fg=AcademicBlue,bg=white}
\setbeamercolor{title}{fg=AcademicBlue}
\setbeamercolor{alerted text}{fg=AcademicTeal}
\setbeamercolor{block title}{fg=white,bg=AcademicBlue}
\setbeamercolor{block body}{fg=black,bg=AcademicBlue!5}
\setbeamertemplate{navigation symbols}{}
\setbeamertemplate{itemize items}[circle]
\setbeamertemplate{footline}{%
  \hfill\usebeamerfont{page number in head/foot}%
  \insertframenumber\,/\,\inserttotalframenumber\hspace{4mm}\vspace{3mm}}
\setbeamersize{text margin left=8mm,text margin right=8mm}
\setbeamerfont{frametitle}{size=\Large,series=\bfseries}
\setbeamerfont{title}{size=\LARGE,series=\bfseries}
\newcommand{\papersource}[1]{\par\smallskip{\scriptsize\color{gray}#1\par}}
\newcommand{\mat}[1]{\mathbf{#1}}
\newcommand{\er}{\operatorname{er}}
\newcommand{\rank}{\operatorname{rank}}
\newcommand{\diag}{\operatorname{diag}}
\newcommand{\norm}[1]{\left\|#1\right\|}

\title[State reduction]{On State Reduction\\in Linear Attention}
\subtitle{Research group reading presentation}
\author{Paper by Philipp Nazari and T.\ Konstantin Rusch}
\date{arXiv:2602.04852v3 \quad (29 September 2026)\\
Reading date: 13 October 2026}

\begin{document}

\begin{frame}
\titlepage
\vspace{-6mm}
\begin{center}
\alert{Joint key/query channel selection can halve recurrent state,\\
with task-dependent quality costs and workload-dependent speedups.}
\end{center}
\end{frame}

\begin{frame}{Deleting channels makes the recurrent state smaller}
\centering
\includegraphics[width=0.97\linewidth,height=0.32\textheight,keepaspectratio]{figures/fig-01-structured.pdf}
\[
 \mat S_t\in\mathbb R^{d_v\times d_k}
 \quad\longrightarrow\quad
 \mat S'_t\in\mathbb R^{d_v\times d_k'},\qquad d_k'<d_k
\]
\begin{itemize}
\item Sparse projection weights can leave the state shape unchanged.
\item Select entire key/query channels and their convolution filters.
\item 50\% compression means half the recurrent matrix entries\\
in the pruned layers; total model memory has other components.
\end{itemize}
\papersource{Paper, Fig. 1, PDF p. 2; Secs. 3.2--3.3, pp. 5--6.}
\end{frame}

\begin{frame}{DeltaNet writes a prediction residual into memory}
\[
\mat S_t=\mat S_{t-1}
 +\beta_t(\mat v_t-\mat S_{t-1}\mat k_t)\mat k_t^\top,
\qquad \mat o_t=\mat S_t\mat q_t
\]
\begin{itemize}
\item $\mat k_t,\mat q_t\in\mathbb R^{d_k}$: write and read addresses.
\item $\mat v_t,\mat o_t\in\mathbb R^{d_v}$; $\beta_t$: scalar update strength.
\item The key controls both erasure and writing; the query controls readout.
\end{itemize}
\begin{block}{The state inherits its available directions}
\[
\rank(\mat S_t)\le
\min\{\rank(\mat K_t),\rank(\mat V_t)\}\le t
\]
\end{block}
\papersource{Paper, Eqs. (1)--(3), PDF pp. 2--3; proof in App. F.2, pp. 25--26.
Here $\mat K_t,\mat V_t$ stack historical vectors as columns.}
\end{frame}

\begin{frame}{Stable rank measures concentration of spectral energy}
\begin{columns}[T,onlytextwidth]
\begin{column}{0.47\textwidth}
\[
\er(\mat S)=\frac{\norm{\mat S}_F^2}{\norm{\mat S}_2^2}
=\frac{\sum_i\sigma_i^2}{\sigma_1^2}
\]
\[
u(\mat S)=\frac{\er(\mat S)}{d},
\quad d=\min(d_v,d_k)
\]
\begin{itemize}
\item $u=1$: all $d$ singular values are equal.
\item Low $u$: energy is concentrated in a few directions.
\end{itemize}
\end{column}
\begin{column}{0.49\textwidth}
\includegraphics[width=\linewidth,height=0.48\textheight,keepaspectratio]{figures/fig-02-spectrum.pdf}
\small DeltaNet 370M, one head.\\
75\% reduction, before recovery fine-tuning (RFT).
\end{column}
\end{columns}
\papersource{Paper, Sec. 2.2, Eq. (5), Fig. 2, PDF pp. 3--4.
Both histogram axes are logarithmic; FineWeb, $T=2048$, first 128 tokens skipped.}
\end{frame}

\begin{frame}{Key conditioning enters a lower bound on state rank}
For additive memory, $\mat S=\mat V\mat K^\top$:
\[
\boxed{\er(\mat S)\ge
\frac{\er(\mat V_\parallel)}{\kappa(\mat K)^2}},
\qquad
\kappa(\mat K)=\frac{\sigma_{\max}(\mat K)}{\sigma_{\min}(\mat K)}
\]
\begin{itemize}
\item $\mat K\in\mathbb R^{d_k\times T}$ has full row rank.
\item $\mat V_\parallel\ne0$: value rows projected onto $\operatorname{row}(\mat K)$.
\item DeltaNet uses pseudo-values
$\mat u_t=\beta_t(\mat v_t-\mat S_{t-1}\mat k_t)$.
\end{itemize}
\begin{block}{Interpretation}
Avoid nearly redundant key directions, while preserving useful signals.\\
The projected values also change when channels change.
\end{block}
\papersource{Paper, Eq. (4), PDF p. 3; assumptions and proof in App. F.1, p. 25.
The paper's $\nu(\mat V)=\er(\mat V_\parallel)$ also depends on $\mat K$.}
\end{frame}

\begin{frame}{An uneven spectrum permits large relative readout error}
With $\widetilde{\mat q}=\mat q^*+\varepsilon$, define
\[
R=\frac{\norm{\mat S\varepsilon}_2/\norm{\mat S\mat q^*}_2}
        {\norm{\varepsilon}_2/\norm{\mat q^*}_2}.
\]
For a \alert{full-column-rank} state:
\[
\boxed{\max_{\norm{\mat q^*}_2=1,\;\varepsilon\ne0}R
=\kappa(\mat S)\ge\frac1{\sqrt{u(\mat S)}}}
\]
\begin{itemize}
\item Worst case: query along the weakest direction, noise along the strongest.
\item At exact isotropy, $u=1$ and $R=1$.
\item This describes worst-case sensitivity; actual alignment matters.
\end{itemize}
\papersource{Paper, Proposition 2.1, Eqs. (6)--(7), PDF p. 4; App. F.3, pp. 26--27.}
\end{frame}

\begin{frame}{Joint orthogonal changes preserve the sequence mixer}
For a square orthogonal $\mat T$:
\[
\mat k_t'=\mat T\mat k_t,\qquad
\mat q_t'=\mat T\mat q_t,\qquad
\mat S_t'=\mat S_t\mat T^\top
\]
\[
\mat S_t'\mat q_t'
=\mat S_t\mat T^\top\mat T\mat q_t
=\mat S_t\mat q_t.
\]
\begin{itemize}
\item The Delta recurrence keeps the same form; scalar decay also commutes.
\item Both keys and queries must use the same transformation.
\end{itemize}
\begin{block}{Dimension reduction changes the problem}
For $\mat T\in\mathbb R^{d_k'\times d_k}$, $\mat T^\top\mat T\ne\mat I$.\\
Compression seeks an acceptable approximation to the original dynamics.
\end{block}
\papersource{Paper, Proposition 3.1 and Secs. 3.1--3.2, PDF p. 5.}
\end{frame}

\begin{frame}{Channel selection preserves the efficient layer structure}
\[
\underbrace{\mat T\diag(\mat w^{(j)})\mat T^\top}_{\text{generally dense}}
\quad\text{vs.}\quad
\underbrace{\mat P_{\mathcal I}
=[\mat e_{i_1},\ldots,\mat e_{i_{d_k'}}]^\top}_{\text{select original coordinates}}
\]
\begin{enumerate}
\item Pick one retained set $\mathcal I$ per head.
\item Slice matching columns of $\mat W_K$ and $\mat W_Q$.
\item Slice the matching depthwise convolution filters.
\item Execute the same recurrence with a smaller $d_k'$.
\end{enumerate}
\medskip
Dense PCA rotations conflict with depthwise convolution.\\
Shared filters address convolution, but SiLU still breaks general rotation equivariance.
\papersource{Paper, Secs. 3.2--3.3, PDF pp. 5--6; App. B.3--B.5, pp. 17--18.
The retained channels are normalized in the reduced model.}
\end{frame}

\begin{frame}{Three baselines score individual channels}
\textbf{L1:} projection-weight magnitude
\[
s_j=\norm{\mat W_{Q,:j}}_1+\norm{\mat W_{K,:j}}_1
\]
\textbf{S-Wanda:} magnitude weighted by input activation
\[
s_j=\sum_i(|W_{Q,ij}|+|W_{K,ij}|)\norm{\mat X_{:i}}_2
\]
\textbf{Grad:} first-order task-loss sensitivity
\[
s_j=\sum_i\bigl(
 |W_{Q,ij}\nabla_{W_{Q,ij}}\mathcal L|
 +|W_{K,ij}\nabla_{W_{K,ij}}\mathcal L|\bigr)
\]
Select the top $d_k'$ scores within each head.\\
\alert{Grad alone uses full-model loss gradients.}
\papersource{Paper, Sec. 3.4, PDF pp. 6--7. Rand selects channels randomly.}
\end{frame}

\begin{frame}{DRRQR selects original columns with a stable basis}
\[
\mat M=
\begin{bmatrix}\mat K_{\rm cal}\\\mat Q_{\rm cal}\end{bmatrix}
\in\mathbb R^{2N\times d_k},
\qquad
\mat M\Pi=\mat Q_{\rm QR}
\begin{pmatrix}\mat A_k&\mat B_k\\0&\mat C_k\end{pmatrix}
\]
\begin{enumerate}
\item Start with column-pivoted QR; $k=d_k'$.
\item Evaluate swaps between selected and unselected columns.
\item Accept swaps that improve the selected volume beyond tolerance $f$.
\item Return the first $k$ original channel indices in the permutation.
\end{enumerate}
\begin{block}{What is guaranteed}
Strong RRQR bounds the selected joint activation spectrum.\\
The model uses the selected channels, with no QR rotation at inference.
\end{block}
\papersource{Paper, Sec. 3.4, PDF pp. 6--7; Algorithm 2, p. 15; App. B.2, pp. 15--16.
The full bound and swap rule are in backup slide 23.}
\end{frame}

\begin{frame}{Calibration and recovery answer different questions}
\begin{columns}[T,onlytextwidth]
\begin{column}{0.48\textwidth}
\textbf{Calibration and models}
\begin{itemize}
\item Default: 128 FineWeb samples.
\item DRRQR: subsample 2.5k keys and 2.5k queries.
\item DeltaNet / Gated DeltaNet:\\370M and 1.3B.
\item Mamba-2 370M, Codestral 7B; hybrid Qwen3.5 9B.
\end{itemize}
\end{column}
\begin{column}{0.48\textwidth}
\textbf{Recovery fine-tuning}
\begin{itemize}
\item LoRA rank 32, $\alpha=64$.
\item 32k FineWeb samples; length 2048; batch 16.
\item LR $10^{-4}\to10^{-5}$; 5\% warmup.
\item Train convolutions and norms; distill from original model.
\end{itemize}
\end{column}
\end{columns}
\medskip
\alert{Pre-RFT isolates immediate pruning quality. Post-RFT includes recovery.}
\papersource{Paper, App. A, PDF p. 14. 370M DN/GDN: 10B pretraining tokens.
Mamba results are pre-RFT. Qwen RFT updates recurrent components only.}
\end{frame}

\begin{frame}{Half-sized states often recover language-model quality}
\setlength{\tabcolsep}{8pt}
\textbf{DRRQR at 50\% state reduction}
\medskip
\centering
\begin{tabular}{lrrrrr}
\toprule
& \multicolumn{3}{c}{WikiText-2 PPL $\downarrow$}
& \multicolumn{2}{c}{Zero-shot avg.\ \% $\uparrow$}\\
\cmidrule(lr){2-4}\cmidrule(lr){5-6}
Model & Base & Pre & Post & Base & Post\\
\midrule
DN 370M  &29.8&31.4&29.4&44.2&44.5\\
DN 1.3B  &16.7&20.5&17.5&50.0&49.7\\
GDN 370M &28.8&31.6&28.7&44.4&43.8\\
GDN 1.3B &16.8&17.3&16.3&59.4&58.0\\
\bottomrule
\end{tabular}
\medskip
\begin{itemize}
\item Recovery matters: DN 1.3B LAMBADA PPL is $11.9\to47.2\to11.8$.
\item Small post-RFT gains include added training and distillation.
\end{itemize}
\papersource{Paper, Table 20, PDF p. 29. DN = DeltaNet; GDN = Gated DeltaNet.
Six-task average: ARC-E/C, HellaSwag, WinoGrande, PIQA, LAMBADA.}
\end{frame}

\begin{frame}{More compression exposes a gradual quality trade-off}
\textbf{Gated DeltaNet 1.3B, after RFT}
\medskip
\centering
\begin{tabular}{llrrr}
\toprule
Reduction & Selector & Wiki PPL $\downarrow$ & LMB PPL $\downarrow$ & Avg.\ \% $\uparrow$\\
\midrule
0\%  & Baseline &16.8&9.7&59.4\\
30\% & DRRQR    &15.9&10.7&58.8\\
30\% & Grad     &15.9&9.9&59.0\\
50\% & DRRQR    &16.3&12.0&58.0\\
50\% & Grad     &16.4&10.5&58.3\\
75\% & DRRQR    &17.9&14.7&56.2\\
75\% & Grad     &17.8&12.5&56.8\\
\bottomrule
\end{tabular}
\medskip
\begin{flushleft}
At 50\%, DRRQR has lower WikiText PPL; Grad has better LAMBADA\\
and average task accuracy. The selector ranking depends on the metric.
\end{flushleft}
\papersource{Paper, Table 1, PDF p. 8; full task breakdown in Table 24, p. 33.}
\end{frame}

\begin{frame}{Higher utilization can coexist with lower absolute rank}
\begin{columns}[T,onlytextwidth]
\begin{column}{0.52\textwidth}
\includegraphics[width=\linewidth,height=0.57\textheight,keepaspectratio]{figures/fig-03-utilization.pdf}
\end{column}
\begin{column}{0.44\textwidth}
\textbf{DeltaNet 370M, 75\% reduction}
\medskip
\begin{tabular}{lrr}
\toprule
& $d_k$ & Mean $u$\\
\midrule
Baseline&128&0.053\\
DRRQR&32&0.170\\
L1&32&0.124\\
\bottomrule
\end{tabular}
\medskip
\small
Reading-derived calculation:\\
mean stable rank $\approx6.78\to5.44$.\\[2mm]
Most effective dimensions survive\\
in one quarter of the capacity.
\end{column}
\end{columns}
\papersource{Paper, Fig. 3, PDF p. 7; Table 5, p. 18.
All compressed curves have the same dimension; baseline has a different denominator.
Pre-RFT, FineWeb.}
\end{frame}

\begin{frame}{A low condition number can hide discarded signal}
\textbf{Selected joint key/query columns: DeltaNet 370M, 75\% reduction}
\medskip
\centering
\begin{tabular}{lrrr}
\toprule
Selector & $\sigma_{\min}$ & $\sigma_{\max}$ & $\kappa$\\
\midrule
Baseline (128) &0.10&49.4&529\\
DRRQR (32) &0.96&49.4&65\\
Grad (32) &0.62&48.5&89\\
S-Wanda (32) &0.65&22.5&42\\
L1 (32) &0.64&15.3&32\\
\bottomrule
\end{tabular}
\medskip
\begin{itemize}
\item DRRQR retains the dominant spectral scale and lifts the smallest singular value.
\item L1 has a smaller $\kappa$, but removes much of the dominant signal.
\item DRRQR and Grad overlap on a median 26/32 channels; random overlap is 8.
\end{itemize}
\papersource{Paper, App. C.2, Table 6, PDF p. 18.
Each column reports a per-head median; median ratios need not equal ratios of medians.}
\end{frame}

\begin{frame}{DRRQR transfers more reliably across calibration corpora}
\textbf{DeltaNet 370M, 75\% reduction, before RFT}
\medskip
\centering
\begin{tabular}{lrr}
\toprule
Calibration corpus & DRRQR PPL $\downarrow$ & Grad PPL $\downarrow$\\
\midrule
FineWeb &41.6&44.9\\
PG19 &41.5&53.8\\
Pile &41.7&51.0\\
\bottomrule
\end{tabular}
\medskip
\begin{itemize}
\item Fixed evaluation: WikiText-2; 128 calibration sequences of length 2048.
\item DRRQR: 41.6--41.7 PPL across 8--512 FineWeb samples.
\item Grad is stable in-domain, but its scores depend on calibration loss.
\end{itemize}
\papersource{Paper, Table 2, PDF p. 8; full calibration study, Table 7, p. 19.
These dedicated ablation values differ from the standard 75\% run in Table 21.}
\end{frame}

\begin{frame}{The framework extends to Mamba, with visible quality cost}
\textbf{Mamba-Codestral 7B, 50\% reduction, before RFT}
\medskip
\centering
\begin{tabular}{lrrr}
\toprule
Selector & Wiki PPL $\downarrow$ & LMB PPL $\downarrow$ & Avg.\ \% $\uparrow$\\
\midrule
Baseline &14.5&6.7&64.4\\
Grad &16.7&12.7&59.9\\
DRRQR &17.6&28.7&57.1\\
L1 &25.7&73.7&48.8\\
\bottomrule
\end{tabular}
\medskip
\begin{itemize}
\item Mamba-2's $\mat B,\mat C$ play the roles of keys and queries.
\item Both selectors outperform magnitude pruning; Grad is stronger here.
\item DRRQR loses 7.3 accuracy points; 7B results are reported before recovery.
\end{itemize}
\papersource{Paper, Tables 9--10, PDF pp. 19--20; main-text Table 3, p. 8.}
\end{frame}

\begin{frame}{Retrieval separates pure recurrence from the tested hybrid}
\textbf{50\% recurrent-state reduction, after RFT; scores $\uparrow$}
\medskip
\centering
\begin{tabular}{llrr}
\toprule
Model & Configuration & RULER 4K & NIAH 4K\\
\midrule
GDN 1.3B & Baseline &38.8&81.3\\
& DRRQR &27.5&61.9\\
\midrule
Qwen3.5 9B & Baseline &93.9&100.0\\
& DRRQR &93.6&100.0\\
\bottomrule
\end{tabular}
\medskip
\begin{itemize}
\item GDN loses 19.4 NIAH points despite competitive language-model metrics.
\item Qwen prunes and fine-tunes recurrent components; attention stays frozen.
\item Supports a memory-path hypothesis; models also differ in scale and training.
\end{itemize}
\papersource{Paper, Table 4, PDF p. 9; Tables 11--12, p. 20.
The reported context lengths are 2K and 4K.}
\end{frame}

\begin{frame}{End-to-end speedup grows with decode batch size}
\begin{columns}[T,onlytextwidth]
\begin{column}{0.60\textwidth}
\includegraphics[width=\linewidth,height=0.59\textheight,keepaspectratio]{figures/fig-04-speedup.pdf}
\end{column}
\begin{column}{0.36\textwidth}
\textbf{GDN 1.3B, H100}\\
50\% state reduction
\medskip
\begin{itemize}
\item Decode, $B=1$: $1.03\times$.
\item Decode, $B=64$: $1.47\times$.
\item Prefill: usually $1.14$--$1.16\times$.
\end{itemize}
\small
At $B=64$, $11.61/1.47\approx7.90$ ms/token:\\
47\% more speed, about 32\% less time.
\end{column}
\end{columns}
\papersource{Paper, Fig. 4, PDF p. 9; Tables 17--18, p. 28.
CUDA graphs, unchanged FLA kernels, 10 warmups, 100 measured steps, median of 3 trials.
Decode prompt: 512 tokens. Prefill at 2K, $B=1$--2: $1.00\times$.}
\end{frame}

\begin{frame}{Coupled scoring helps DRRQR, but magnitude has a caveat}
\textbf{DeltaNet 370M, 50\% reduction; WikiText-2 PPL $\downarrow$}
\medskip
\centering
\begin{tabular}{lrrr}
\toprule
Selector & Score from K+Q & Score from K & Score from Q\\
\midrule
L1 &3843.5&425218.3&33.0\\
S-Wanda &915.9&376074.0&33.0\\
DRRQR &31.4&286035.7&32.1\\
Grad &31.7&17709.6&31.9\\
\bottomrule
\end{tabular}
\medskip
\begin{itemize}
\item Every column still applies the same selected indices to both K and Q.
\item Joint scoring works best for DRRQR and Grad.
\item Query-only magnitude is a much stronger baseline than joint magnitude.
\end{itemize}
\papersource{Paper, Table 16 and App. C.7, PDF p. 23. Uncompressed baseline: 29.8.}
\end{frame}

\begin{frame}{Capacity should be chosen for the capability you need}
\begin{enumerate}
\item \textbf{A practical compression primitive.}\\
Coupled channel selection shrinks state while preserving kernel compatibility.
\medskip
\item \textbf{A useful numerical criterion.}\\
DRRQR preserves useful activation directions without task gradients,\\
and transfers well across the tested calibration corpora.
\medskip
\item \textbf{A capability-dependent budget.}\\
Language modeling tolerates more compression than pure recurrent retrieval;\\
batch size determines how much speed is recovered.
\end{enumerate}
\medskip
\alert{Discussion: could per-head budgets and retrieval-aware calibration\\
preserve the rare directions that long-context recall needs?}
\papersource{Synthesis of Secs. 2--6 and Apps. C/G.
The discussion question is a reading-derived research direction.}
\end{frame}

\begin{frame}{Paper and source materials}
\textbf{Philipp Nazari and T.\ Konstantin Rusch}\\
\emph{On State Reduction in Linear Attention}\\
arXiv:2602.04852v3, revised 29 September 2026.

\medskip
\href{https://arxiv.org/abs/2602.04852v3}{arxiv.org/abs/2602.04852v3}\\
\href{https://github.com/camail-official/LinearAttentionPruning}
{github.com/camail-official/LinearAttentionPruning}

\bigskip
\begin{itemize}
\item All experimental numbers are from this paper version.
\item Figures are cropped from the paper; tables select reported entries.
\item Page locators refer to the original 35-page PDF.
\item The following six slides provide optional technical backup.
\end{itemize}
\end{frame}

\appendix
\begin{frame}{Backup: strong RRQR controls the selected spectrum}
Let $k=d_k'$, $n=d_k$, $\rank(\mat M)\ge k$, and $f\ge1$.
\[
\rho_{ij}
=\sqrt{|(\mat A_k^{-1}\mat B_k)_{ij}|^2+
 \left(\frac{\norm{(\mat C_k)_{:,j}}_2}
 {1/\norm{(\mat A_k^{-1})_{i,:}}_2}\right)^2}.
\]
Swap columns while $\max_{ij}\rho_{ij}>f$. At termination:
\[
c_f=\sqrt{1+f^2k(n-k)},\qquad
\sigma_i(\mat A_k)\ge\frac{\sigma_i(\mat M)}{c_f}
\]
\[
\norm{\mat C_k}_2\le c_f\sigma_{k+1}(\mat M),\qquad
\boxed{\kappa(\mat A_k)\le
c_f\frac{\sigma_1(\mat M)}{\sigma_k(\mat M)}}.
\]
Selected original columns share the singular values of $\mat A_k$.\\
The guarantee concerns the calibration matrix, not a bound on task loss.
\papersource{Paper, Algorithm 2 and App. B.2, PDF pp. 15--16; based on Gu \& Eisenstat (1996).}
\end{frame}

\begin{frame}{Backup: the rank bound and local gradient conditioning}
\textbf{Projected-value proof sketch}
\[
\mat S=\mat V_\parallel\mat K^\top,\quad
\norm{\mat S}_F\ge\norm{\mat V_\parallel}_F\sigma_{\min}(\mat K),\quad
\norm{\mat S}_2\le\norm{\mat V_\parallel}_2\sigma_{\max}(\mat K).
\]
Divide the squared inequalities to obtain the stable-rank lower bound.

\medskip
\textbf{Local query-weight gradient}
\[
\mat o=\mat S\mat W_Q^\top\mat x,\qquad
\mat J=\frac{\partial\mat o}{\partial\operatorname{vec}(\mat W_Q)}
=\mat S\otimes\mat x^\top
\]
\[
\sigma_i^{+}(\mat J^\top)=\norm{\mat x}_2\sigma_i(\mat S).
\]
For fixed $\mat S$ of full column rank and $\mat x\ne0$, the backward map\\
inherits $\kappa(\mat S)$ on its nonzero singular subspaces.
\papersource{Paper, App. F.1, PDF p. 25; App. F.4, p. 27.
Projected values must be nonzero and keys full row rank for the first bound.}
\end{frame}

\begin{frame}{Backup: fixing convolution alone does not rescue PCA}
Closest diagonal filters after a square orthogonal rotation:
\[
\mat W'=(\mat T\odot\mat T)\mat W.
\]
Shared filters commute with rotations; elementwise SiLU generally does not.
\medskip
\begin{center}
\begin{tabular}{lrr}
\toprule
DN 370M, 50\% reduction & Standard conv & Shared conv\\
\midrule
Proper PCA, Wiki PPL $\downarrow$ & $3.2\times10^5$ &155.76\\
Grad, Wiki PPL $\downarrow$ &31.75&33.73\\
Uncompressed, Wiki PPL $\downarrow$ &29.83&29.55\\
\bottomrule
\end{tabular}
\end{center}
Keeping high-variance components is insufficient for preserving the full layer.
\papersource{Paper, Proposition B.1, PDF p. 17; SiLU qualification, p. 18;
Table 15, p. 22. Compression results are pre-RFT.}
\end{frame}

\begin{frame}{Backup: hybrid robustness still depends on the selector}
\textbf{Qwen3.5 9B, before RFT}
\par\medskip
\centering
\begin{tabular}{lrrr}
\toprule
Configuration & Wiki PPL $\downarrow$ & LMB PPL $\downarrow$ & Avg.\ \% $\uparrow$\\
\midrule
Baseline &9.0&3.8&73.3\\
DRRQR 25\% &9.7&4.3&73.0\\
DRRQR 50\% &11.6&5.7&71.4\\
L1 25\% &18.8&7.1&61.3\\
L1 50\% &38.0&33.4&50.9\\
\bottomrule
\end{tabular}
\medskip
\begin{flushleft}
After recurrent-only RFT at 50\%, DRRQR also retains completion scores:\\
SQuAD $65.3\to66.0$; SWDE $92.7\to92.3$.\\
Softmax attention layers remain frozen throughout recovery.
\end{flushleft}
\papersource{Paper, Table 13, PDF p. 21; Table 12, p. 20.}
\end{frame}

\begin{frame}{Backup: recovery can further reshape the state spectrum}
\centering
\includegraphics[width=0.93\linewidth,height=0.55\textheight,keepaspectratio]{figures/fig-06-rft-spectrum.pdf}
\medskip
\begin{itemize}
\item One DeltaNet 370M head, 75\% reduction, FineWeb sequence length 2048.
\item Histograms aggregate singular values after the first 128 tokens.
\item Compression shortens the tail; RFT can introduce a spectral gap.
\end{itemize}
\papersource{Paper, Fig. 6 and App. C.5, PDF pp. 20--22.
Both axes are logarithmic; this is a selected-head illustration.}
\end{frame}

\begin{frame}{Backup: state size, mixer memory, and model speed differ}
\textbf{GDN 1.3B, H100, 2K prefill, 50\% state reduction}
\medskip
\centering
\begin{tabular}{crrr}
\toprule
Batch & Mixer speedup & Mixer VRAM ratio & Full-model speedup\\
\midrule
8 &1.26$\times$&0.73$\times$&1.14$\times$\\
16&1.28$\times$&0.73$\times$&1.14$\times$\\
32&1.28$\times$&0.72$\times$&1.14$\times$\\
\bottomrule
\end{tabular}
\medskip
\begin{itemize}
\item Mixer measurements exclude projections, norms, and feed-forward layers.
\item A smaller recurrent matrix is only part of peak inference memory.
\item Larger decode batches expose more benefit from reducing per-sequence state.
\end{itemize}
\papersource{Paper, Tables 18--19 and App. G.1, PDF pp. 27--28.
Mixer setup: 16 heads, hidden size 2048, $d_v=128$.}
\end{frame}

\end{document}