% Based on the arxiv-paper-reader 16:9 Beamer template.
% Main talk: frames 1--21, approximately 25 minutes. Frames 22--26: backup.
% All experimental plots are author-provided figures, unchanged.
\documentclass[aspectratio=169,11pt]{beamer}
\usepackage[T1]{fontenc}
\usepackage[utf8]{inputenc}
\usepackage{amsmath,amssymb,graphicx,booktabs}

\definecolor{AcademicBlue}{RGB}{26,57,86}
\definecolor{AcademicTeal}{RGB}{0,105,112}
\setbeamercolor{structure}{fg=AcademicBlue}
\setbeamercolor{frametitle}{fg=AcademicBlue,bg=white}
\setbeamercolor{title}{fg=AcademicBlue}
\setbeamercolor{alerted text}{fg=AcademicTeal}
\setbeamercolor{block title}{fg=white,bg=AcademicBlue}
\setbeamercolor{block body}{fg=black,bg=AcademicBlue!5}
\setbeamertemplate{navigation symbols}{}
\setbeamertemplate{itemize items}[circle]
\setbeamertemplate{footline}{%
  \hfill\usebeamerfont{page number in head/foot}%
  \insertframenumber\,/\,\inserttotalframenumber\hspace{4mm}\vspace{3mm}}
\setbeamersize{text margin left=8mm,text margin right=8mm}
\setbeamerfont{frametitle}{size=\Large,series=\bfseries}
\setbeamerfont{title}{size=\LARGE,series=\bfseries}
\newcommand{\papersource}[1]{\par\smallskip{\scriptsize\color{gray}#1\par}}
\newcommand{\figfit}[2]{\includegraphics[width=\linewidth,height=#2,keepaspectratio]{figures/#1}}
\newcommand{\E}{\mathbb E}
\newcommand{\R}{\mathbb R}
\newcommand{\D}{\mathcal D}

\title[Length generalization]{Understanding and Improving Length Generalization in Recurrent Models}
\subtitle{Paper discussion: training on attainable states}
\author{Paper authors: Ricardo Buitrago Ruiz and Albert Gu}
\institute{Carnegie Mellon University / Cartesia AI}
\date{arXiv:2507.02782v2 \quad | \quad Reading: 12 October 2026}

\begin{document}

\begin{frame}
  \titlepage
  \vspace{-5mm}
  \begin{center}
  \small A short training chunk can expose a model to a long-history state.\\[1mm]
  \scriptsize\href{https://arxiv.org/abs/2507.02782v2}{arxiv.org/abs/2507.02782v2} \quad Independent reading presentation
  \end{center}
\end{frame}

\begin{frame}{A recurrence can keep running while prediction fails}
  \begin{columns}[c,onlytextwidth]
  \begin{column}{0.55\textwidth}
    \centering\figfit{mamba2-poswise.png}{5.7cm}
  \end{column}
  \begin{column}{0.42\textwidth}
    \begin{itemize}
      \item Mamba-2 was trained with a 2,048-token context.
      \item Some base checkpoints deteriorate sharply after that range.
      \item Dashed curves: 500 steps of State Passing (SP) post-training.
    \end{itemize}
    \medskip
    \alert{Computational reach does not ensure statistical generalization.}
  \end{column}
  \end{columns}
  \papersource{Paper, Fig. 1(b), PDF p. 2. Position-wise perplexity: lower is better.}
\end{frame}

\begin{frame}{Position-wise perplexity separates early and late failure}
  \[
  \operatorname{PPL}(t)=\exp\!\left(\E_{x\sim\D}[-\log q(x_t\mid x_{0:t-1})]\right)
  \]
  \begin{itemize}
    \item Average the negative log-likelihood at a fixed position, then exponentiate.
    \item Let $p^\star$ be the best PPL in the training range, attained at $t^\star$.
  \end{itemize}
  \begin{block}{The paper's formal definition}
    Length generalization to $T>T_{\rm train}$ requires
    $\operatorname{PPL}(t)\le p^\star$ for every $t\in[t^\star,T]$.
  \end{block}
  Empirical plots show approximate trends. Stable PPL alone does not establish use of distant facts.
  \papersource{Paper, Definitions 3.1--3.2, PDF p. 4; motivation for task evaluation, Sec. 5, p. 9.}
\end{frame}

\begin{frame}{Longer training contexts improve extrapolation}
  \begin{columns}[c,onlytextwidth]
  \begin{column}{0.47\textwidth}
    \centering\figfit{pos_ppl_125.png}{5.55cm}
  \end{column}
  \begin{column}{0.49\textwidth}
    \textbf{Mamba-2 125M, trained from scratch}
    \begin{itemize}
      \item Same model size; 25B training tokens.
      \item Training contexts: 1,024, 2,048, 4,096.
      \item The 4,096-token model is substantially more stable at long positions.
    \end{itemize}
    \medskip
    Architecture and state size alone do not determine the failure.
  \end{column}
  \end{columns}
  \papersource{Paper, Sec. 3.2 and Fig. 2(c), PDF p. 4. Other panels cover 45M, 85M, and GLA 70M.}
\end{frame}

\begin{frame}{More short-context training can worsen extrapolation}
  \centering
  \includegraphics[width=0.76\linewidth,height=4.15cm,keepaspectratio]{figures/85m_1024_checkpoints.png}
  \begin{itemize}
    \item Mamba-2 85M, context 1,024; checkpoints span 4.25B--34B tokens.
    \item In-range PPL improves as late-position PPL deteriorates.
    \item This is consistent with overfitting to early-state distributions.
  \end{itemize}
  \papersource{Paper, Sec. 3.3 and Fig. 3, PDF pp. 4--5. Multipliers refer to the paper's Chinchilla budget convention.}
\end{frame}

\begin{frame}{Effective Remembrance measures prefix influence}
  \[
  \operatorname{EffRem}_T(t)=
  d\!\left(q(\cdot\mid x_{0:T}),q(\cdot\mid x_{t:T})\right)
  \]
  \[
  d(p,q)=\operatorname{TV}(p,q)=\frac12\sum_{v\in\mathcal V}|p(v)-q(v)|
  \]
  \begin{itemize}
    \item Fix the prediction position $T$; remove the prefix $[0,t)$.
    \item Recompute the retained suffix from the normal initial state.
    \item High TV means the prefix strongly changes the output distribution.
  \end{itemize}
  \begin{block}{Interpretation}
  Sensitivity can reflect useful information or harmful interference. It is not a memory-capacity score.
  \end{block}
  \papersource{Paper, Definition 3.3 and Eqs. (2)--(3), PDF p. 5. TV is written in its intended standard form.}
\end{frame}

\begin{frame}{Failing models are unusually sensitive to early tokens}
  \begin{columns}[c,onlytextwidth]
  \begin{column}{0.56\textwidth}
    \centering\figfit{mamba2-effrem.png}{5.55cm}
  \end{column}
  \begin{column}{0.41\textwidth}
    \textbf{Mamba-2; fixed $T=8192$}
    \begin{itemize}
      \item Solid: base checkpoints.
      \item Dashed: SP post-training.
      \item Removing a short, distant prefix can strongly alter a failing model's output.
      \item SP reduces this dependence while improving long-position PPL.
    \end{itemize}
  \end{column}
  \end{columns}
  \papersource{Paper, Fig. 1(e), PDF p. 2; Sec. 3.4, p. 5. The vertical scale is logarithmic.}
\end{frame}

\begin{frame}{State drift matters more than a small state scale}
  \begin{columns}[c,onlytextwidth]
  \begin{column}{0.56\textwidth}
    \centering\figfit{statemetrics_full.png}{5.3cm}
  \end{column}
  \begin{column}{0.41\textwidth}
    \textbf{Mamba-2 130M}
    \begin{itemize}
      \item Base statistics keep changing beyond training length 2,048.
      \item SP reaches a more stable range earlier.
      \item SP often has \emph{larger} values, so simply shrinking states is insufficient.
    \end{itemize}
  \end{column}
  \end{columns}
  \papersource{Paper, Sec. 3.5 and Fig. 4, PDF p. 6. The title says ``norm''; the axis says ``Std Dev.'' We interpret the plotted scale trend.}
\end{frame}

\begin{frame}{The hypothesis is a mismatch in attainable states}
  \[
    \D_t=\operatorname{Law}\!\left(h_t(X_{0:t})\right)
  \]
  \begin{columns}[t,onlytextwidth]
    \begin{column}{0.48\textwidth}
      \begin{block}{Zero-initialized short training}
        The model sees early distributions
        $\D_0,\ldots,\D_{T_{\rm train}-1}$ repeatedly.
      \end{block}
    \end{column}
    \begin{column}{0.48\textwidth}
      \begin{block}{Long-sequence evaluation}
        The same recurrence reaches later distributions that may be unfamiliar.
      \end{block}
    \end{column}
  \end{columns}
  \medskip
  \begin{itemize}
    \item With approximately stationary inputs, state shift becomes a useful explanation.
    \item Prediction: expose training to more realistic states, even in short chunks.
  \end{itemize}
  \papersource{Paper, Sec. 3.6, PDF p. 6. This is a supported hypothesis, not a general error-bound theorem.}
\end{frame}

\begin{frame}{A nonzero initial state adds a propagated contribution}
  \[
    h_t=A_t h_{t-1}+B_t x_t,\qquad y_t=C_t^\top h_t
  \]
  \[
    h_t=\underbrace{\sum_{s=0}^t A_{s+1:t}^{\times}B_sx_s}_{\text{zero-initialized result}}
       +\underbrace{A_{0:t}^{\times}h_{-1}}_{\text{initial-state contribution}}
  \]
  \[
    y_t=y_t^{(0)}+C_t^\top A_{0:t}^{\times}h_{-1}
  \]
  \begin{itemize}
    \item $h_t\in\R^N$, $A_t\in\R^{N\times N}$, $B_t,C_t\in\R^{N\times1}$.
    \item For general matrices, $A_{s:t}^{\times}=A_t\cdots A_s$ is ordered.
    \item Initial states can be incorporated into parallel recurrence computation.
  \end{itemize}
  \papersource{Paper, Eqs. (1), (4), PDF pp. 3, 7. Ordered-product clarification for the general matrix case.}
\end{frame}

\begin{frame}{Four interventions preserve different state structure}
  \renewcommand{\arraystretch}{1.35}
  \begin{tabular}{@{}p{0.21\linewidth}p{0.42\linewidth}p{0.31\linewidth}@{}}
    \toprule
    Method & Initial state & What it preserves \\
    \midrule
    Random Noise & IID $\mathcal N(0,\sigma^2)$ & A global noise scale \\
    Fitted Noise & Gaussian samples fitted per layer and head & Per-head mean and variance \\
    State Passing & Final state of a shuffled sequence & Actual joint state structure \\
    TBTT & Final state from the preceding chunk of the same sequence & State + sequence order \\
    \bottomrule
  \end{tabular}\par\medskip
  \alert{Test whether realistic state coverage is more useful than arbitrary noise.}
  \papersource{Paper, Secs. 4.2--4.5, PDF pp. 7--9. TBTT: truncated backpropagation through time.}
\end{frame}

\begin{frame}{Fitted Noise matches moments within each head}
  \begin{align*}
    \mu^{(li)}&\leftarrow(1-\beta)\operatorname{Mean}(h_T^{(li)})+\beta\mu^{(li)},\\
    {\sigma^2}^{(li)}&\leftarrow(1-\beta)\operatorname{Var}(h_T^{(li)})+\beta{\sigma^2}^{(li)},\\
    h_{-1,j}^{(li)}&\ \overset{\rm independent}{\sim}\ \mathcal N\!\left(\mu^{(li)},{\sigma^2}^{(li)}\right).
  \end{align*}
  \begin{itemize}
    \item $l$: layer; $i$: head; $j$: an element of that head's state.
    \item Statistics aggregate head and state-expansion dimensions.
    \item $\beta=0.1$: \textbf{90\% current statistics, 10\% previous estimate.}
    \item Independent sampling discards dependencies between state elements.
  \end{itemize}
  \papersource{Paper, Sec. 4.3 and Eqs. (5)--(6), PDF p. 8. Noise is used during training, with zero initialization at validation.}
\end{frame}

\begin{frame}{State Passing changes training starts, not test-time inputs}
  \begin{block}{Training data flow}
  Previous shuffled batch $\longrightarrow$ final states $\longrightarrow$
  stop gradient $\longrightarrow$ next batch's initial states.
  \end{block}
  \begin{itemize}
    \item Reset the initial state to zero with probability $p=0.1$.
    \item Carry forward state history; optimize using the current short chunk.
    \item TBTT instead preserves the order of chunks from the same long sequence.
    \item At inference, start a new stream normally and continue the recurrence.
  \end{itemize}
  \medskip
  \alert{SP does not require semantically adjacent training chunks.}
  \papersource{Paper, Secs. 4.4--4.5, PDF pp. 8--9; chunk-order distinction in footnote 3, p. 10. Flow is a teaching schematic.}
\end{frame}

\begin{frame}{The 32k and 128k results use different budgets}
  \renewcommand{\arraystretch}{1.4}
  \begin{tabular}{@{}p{0.34\linewidth}p{0.23\linewidth}p{0.36\linewidth}@{}}
    \toprule
    Experiment & Adaptation budget & Context and evaluation \\
    \midrule
    Four-way comparison (Fig. 5) & 100 steps & Mamba-2: 2k training; 32k test \\
    Broad evaluation (Fig. 1) & 500 steps & Mamba-2 / GLA; up to 128k \\
    Mamba-1 (Fig. 1) & 1000 steps & 2k training; up to 128k \\
    Passkey (Fig. 7) & 1000 task steps & 2k training; up to 256k \\
    \bottomrule
  \end{tabular}\par\medskip
  Pile post-training uses $1/10$ of the base peak learning rate.\\
  The reported 100 / 500-step budgets are approximately 0.02\% / 0.1\% of pre-training.
  \papersource{Paper, Figs. 1, 5, PDF pp. 2, 7; Appendix B, pp. 16--17. GLA's training context is 512, not 2k.}
\end{frame}

\begin{frame}{Real states succeed where independent noise fails}
  \begin{columns}[c,onlytextwidth]
  \begin{column}{0.63\textwidth}
    \centering\figfit{1.3b_interventions.png}{5.35cm}
  \end{column}
  \begin{column}{0.34\textwidth}
    \textbf{100-step intervention}
    \begin{itemize}
      \item Fitted Noise helps 370M and 780M, but fails at 1.3B.
      \item SP and TBTT remain stable for all three sizes through 32k.
      \item SP and TBTT are very close despite different sequence continuity.
    \end{itemize}
  \end{column}
  \end{columns}
  \papersource{Paper, Fig. 5(c), PDF p. 7; interpretation in Secs. 4.2--4.5, pp. 7--9.}
\end{frame}

\begin{frame}{State Passing also transfers to Mamba-1 and GLA}
  \begin{columns}[c,onlytextwidth]
    \begin{column}{0.48\textwidth}
      \centering\figfit{mamba1-poswise.png}{4.65cm}\\[-1mm]
      \small Mamba-1: 2,048-token context; 1,000 SP steps
    \end{column}
    \begin{column}{0.48\textwidth}
      \centering\figfit{glas-poswise.png}{4.65cm}\\[-1mm]
      \small GLA: 512-token context; 500 SP steps
    \end{column}
  \end{columns}
  \medskip
  A shared training intervention improves long-position behavior across distinct recurrent mechanisms.
  \papersource{Paper, Fig. 1(a,c), PDF p. 2. Dashed lines denote SP; solid lines denote base models. RWKV-v6 is in backup.}
\end{frame}

\begin{frame}{BABILong improves, with substantial task error remaining}
  \begin{columns}[c,onlytextwidth]
    \begin{column}{0.49\textwidth}\centering\figfit{babilong_fs.png}{3.2cm}\end{column}
    \begin{column}{0.49\textwidth}\centering\figfit{babilong_finetuned.png}{3.2cm}\end{column}
  \end{columns}
  \begin{itemize}
    \item Mamba-2 780M; tasks qa1--qa10; training / finetuning context 2,048.
    \item Task finetuning uses one epoch \emph{without} SP or TBTT for every model.
    \item Earlier Pile post-training benefits survive later ordinary task finetuning.
    \item Few-shot accuracy at the longest inputs remains very low.
  \end{itemize}
  \papersource{Paper, Sec. 5.1 and Fig. 6, PDF p. 9; Appendix B.3, p. 17. Accuracy is higher-is-better.}
\end{frame}

\begin{frame}{Fitted Noise enables 256k passkey retrieval at 780M}
  \begin{columns}[c,onlytextwidth]
    \begin{column}{0.47\textwidth}
      \centering\figfit{passkey_official_780m.png}{3.45cm}
    \end{column}
    \begin{column}{0.51\textwidth}
      \centering\figfit{passkey_fitted_780m.png}{3.45cm}
    \end{column}
  \end{columns}
  \begin{itemize}
    \item Same task: 1,000 finetuning steps, sequences of length 2,048.
    \item Fitted Noise reaches 100\% on the reported grid, including 256k.
    \item A terminal passkey state is a poor start for another sample: task phases are nonstationary.
  \end{itemize}
  \papersource{Paper, Sec. 5.2 and Fig. 7(e,f), PDF p. 10; Appendix B.4, p. 17. Horizontal: length; vertical: depth; green: high accuracy.}
\end{frame}

\begin{frame}{Copying improves by 20 percentage points, yet stays hard}
  \begin{center}
    \renewcommand{\arraystretch}{1.6}
    \begin{tabular}{lc}
      \toprule
      Training method & Character accuracy $\uparrow$ \\
      \midrule
      Mamba-2 & $0.27\pm0.03$ \\
      Mamba-2 + State Passing & $0.47\pm0.11$ \\
      \bottomrule
    \end{tabular}
  \end{center}
  \begin{itemize}
    \item 45M parameters; trained from scratch on one million sequences.
    \item Training lengths 50--100; evaluation length 300.
    \item Length robustness improves; arbitrary-sequence storage remains difficult.
  \end{itemize}
  \papersource{Paper, Sec. 5.3 and Table 1, PDF p. 10. The paper does not define the statistical meaning of the reported $\pm$ values.}
\end{frame}

\begin{frame}{State coverage is a useful explanation with concrete limits}
  \begin{itemize}
    \item \textbf{Mechanism:} realistic initial states improve both predictions and state dynamics; the experiments do not isolate these two effects.
    \medskip
    \item \textbf{Task dependence:} nonstationary task phases can make direct state reuse inappropriate.
    \medskip
    \item \textbf{Capacity:} success on one passkey does not imply reliable copying of arbitrary long sequences.
    \medskip
    \item \textbf{Scope:} evidence covers Mamba-1/2, GLA, and RWKV-v6; hybrid architectures and other domains remain open.
  \end{itemize}
  \papersource{Synthesis of Paper, Secs. 3--5 and 7, PDF pp. 4--11; Appendix B.4, p. 17. Mechanism separation is our analysis.}
\end{frame}

\begin{frame}{Train on the states that long sequences will produce}
  \begin{enumerate}
    \item Diagnose prediction stability, prefix influence, and distant-information use separately.
    \medskip
    \item Treat the initial-state distribution as part of the training recipe.
    \medskip
    \item Test inexpensive state interventions before attributing failure solely to architecture or capacity.
  \end{enumerate}
  \medskip
  \small
  \textbf{Novelty in context:} SP/TBTT existed in prior work. This paper contributes the state-distribution account, Effective Remembrance, intervention comparisons, and efficient post-training evidence.
  \papersource{Ricardo Buitrago Ruiz and Albert Gu. Understanding and Improving Length Generalization in Recurrent Models. arXiv:2507.02782v2 (2025). \href{https://arxiv.org/abs/2507.02782v2}{Versioned paper}. Related-work attribution: Sec. 6, PDF p. 11.}
\end{frame}

\appendix

\begin{frame}{Backup: RWKV-v6 and warm-up support the broader picture}
  \begin{columns}[t,onlytextwidth]
    \begin{column}{0.49\textwidth}
      \centering\figfit{fig_rwkv-v6.png}{4.15cm}\par
      \small\raggedright
      RWKV-v6 70M, context 256.\\
      35B pretraining tokens; 250M SP tokens.
    \end{column}
    \begin{column}{0.49\textwidth}
      \centering\figfit{warmup.png}{4.15cm}\par
      \small\raggedright
      Mamba-2 45M, context 1,024, 9B tokens.\\
      25\% warm-up hurts extrapolation relative to 10\%.
    \end{column}
  \end{columns}
  \papersource{Paper, Appendices D--E, Figs. 10--11, PDF pp. 17--19. These are separate experiments with separate budgets.}
\end{frame}

\begin{frame}{Backup: arbitrary noise requires a suitable scale}
  \begin{columns}[c,onlytextwidth]
  \begin{column}{0.59\textwidth}
    \centering\figfit{125m_mamba2_initial_stds.png}{5.45cm}
  \end{column}
  \begin{column}{0.38\textwidth}
    \textbf{Mamba-2 125M; 100 steps}
    \begin{itemize}
      \item Initial state elements are IID $\mathcal N(0,\sigma^2)$.
      \item Too little noise may fail to fix generalization.
      \item Too much noise worsens PPL.
      \item Validation still starts from zero.
    \end{itemize}
  \end{column}
  \end{columns}
  \papersource{Paper, Appendix F and Fig. 12, PDF pp. 19--20; training/validation distinction in Sec. 4.2, p. 8.}
\end{frame}

\begin{frame}{Backup: output-distance choices preserve the main pattern}
  \begin{columns}[c,onlytextwidth]
    \begin{column}{0.32\textwidth}\centering\figfit{effrem_tot_var.png}{3.9cm}\end{column}
    \begin{column}{0.32\textwidth}\centering\figfit{effrem_jenshen_shannon.png}{3.9cm}\end{column}
    \begin{column}{0.32\textwidth}\centering\figfit{effrem_cos_sim.png}{3.9cm}\end{column}
  \end{columns}
  \begin{itemize}
    \item Mamba-2 base and SP models, fixed prediction position $T=8192$.
    \item Absolute values differ, but the characteristic curves are similar.
    \item TV is bounded by 1; reaching 1 is not a mathematical necessity.
  \end{itemize}
  \papersource{Paper, Appendix G and Fig. 13, PDF p. 20. The paper notes that $1-$cosine similarity is not a proper metric.}
\end{frame}

\begin{frame}{Backup: selected heads reveal where the state drift occurs}
  \centering\includegraphics[width=0.90\linewidth,height=4.35cm,keepaspectratio]{figures/statemetrics_heads.png}
  \begin{itemize}
    \item Heads were selected for pronounced variation; most heads are less abnormal.
    \item SP can have larger standard deviation and still be more stable over time.
  \end{itemize}
  \papersource{Paper, Appendix C and Fig. 9, PDF p. 18. These moments do not fully characterize the joint state distribution.}
\end{frame}

\begin{frame}{Backup: preserve matrix order and parameter conventions}
  \[
  \Phi(t,s)=A_tA_{t-1}\cdots A_s,\qquad
  h_t=\Phi(t,0)h_{-1}+\sum_{s=0}^t\Phi(t,s+1)B_sx_s
  \]
  \begin{itemize}
    \item General matrices need ordered products; Appendix H swaps factors in an intermediate line.
    \medskip
    \item Appendix I assumes diagonal transitions. Its \texttt{exp(cumsum(A))} requires log-transition values, inferred from the operation.
    \medskip
    \item The function's output \texttt{y} is only the initial-state contribution. Add it to the zero-state output.
  \end{itemize}
  \papersource{Paper, Appendices H--I and Fig. 14, PDF pp. 20--21. Parameter-convention and ordering clarifications are our analysis.}
\end{frame}

\end{document}