% Based on the arxiv-paper-reader Beamer template.
% Compile: python3 build_slides.py slides.en.tex
\documentclass[aspectratio=169,11pt]{beamer}
\usepackage[T1]{fontenc}
\usepackage[utf8]{inputenc}
\usepackage{amsmath,amssymb,graphicx,booktabs}
\definecolor{AcademicBlue}{RGB}{26,57,86}
\definecolor{AcademicTeal}{RGB}{0,105,112}
\setbeamercolor{structure}{fg=AcademicBlue}
\setbeamercolor{frametitle}{fg=AcademicBlue,bg=white}
\setbeamercolor{title}{fg=AcademicBlue}
\setbeamercolor{alerted text}{fg=AcademicTeal}
\setbeamercolor{block title}{fg=white,bg=AcademicBlue}
\setbeamercolor{block body}{fg=black,bg=AcademicBlue!5}
\setbeamertemplate{navigation symbols}{}
\setbeamertemplate{itemize items}[circle]
\setbeamertemplate{footline}{%
  \hfill\usebeamerfont{page number in head/foot}%
  \insertframenumber\,/\,\inserttotalframenumber\hspace{4mm}\vspace{3mm}}
\setbeamersize{text margin left=8mm,text margin right=8mm}
\setbeamerfont{frametitle}{size=\Large,series=\bfseries}
\setbeamerfont{title}{size=\LARGE,series=\bfseries}
\newcommand{\papersource}[1]{\par\smallskip{\scriptsize\color{gray}#1\par}}
\newcommand{\panel}[2]{\includegraphics[width=#1\linewidth,height=4.25cm,keepaspectratio]{figures/#2}}
\newcommand{\E}{\mathbb E}
\newcommand{\PPL}{\operatorname{PPL}}
\newcommand{\TV}{\operatorname{TV}}
\newcommand{\EffRem}{\operatorname{EffRem}}
\newcommand{\Var}{\operatorname{Var}}
\newcommand{\Mean}{\operatorname{Mean}}
\title[Length Generalization in Recurrent Models]{Understanding and Improving\\Length Generalization in Recurrent Models}
\subtitle{Research discussion -- 20--30 minute main presentation}
\author{Paper authors: Ricardo Buitrago Ruiz and Albert Gu}
\date{arXiv:2507.02782v2 -- Revised October 12, 2025\\Reading prepared October 13, 2026}
\begin{document}

\begin{frame}
  \titlepage
  \vspace{-5mm}
  \begin{center}
  \alert{Broaden the states seen during training to improve long-sequence behavior.}
  \end{center}
\end{frame}

\begin{frame}{A recurrent update can run longer than it works well}
  \begin{columns}[c,onlytextwidth]
    \begin{column}{0.53\textwidth}
      \panel{1}{mamba2-poswise.png}
    \end{column}
    \begin{column}{0.44\textwidth}
      \[h_t=A_t h_{t-1}+B_t x_t\]
      \begin{itemize}
        \item Fixed-size state; linear sequence cost.
        \item Mamba-2 training context: 2048.
        \item Solid: baseline; dashed: State Passing (SP).
        \item The same architecture can behave very differently beyond training.
      \end{itemize}
    \end{column}
  \end{columns}
  \papersource{Paper, Fig. 1(b), PDF p. 2; Eq. (1), p. 3. Lower perplexity is better.}
\end{frame}

\begin{frame}{Length generalization concerns prediction quality}
  \[
    \PPL(t)=\exp\!\left(\E_{x\sim\mathcal D}
        [-\log q(x_t\mid x_{0:t-1})]\right).
  \]
  Average negative log-likelihood at a fixed position, then exponentiate.
  \begin{block}{The paper's definition}
    Let \(p^\star\) be the minimum training-range PPL, reached at \(t^\star\).
    Generalization to \(T>T_{\rm train}\) requires
    \[\PPL(t)\le p^\star\quad\text{for all }t\in[t^\star,T].\]
  \end{block}
  Stable PPL still leaves open whether the model uses distant information.
  \papersource{Paper, Definitions 3.1--3.2, PDF p. 4; motivation for long-context tasks, p. 9.}
\end{frame}

\begin{frame}{Longer training contexts reduce extrapolation failure}
  \begin{columns}[c,onlytextwidth]
    \begin{column}{0.48\textwidth}
      \centering\panel{0.92}{pos_ppl_125.png}
    \end{column}
    \begin{column}{0.48\textwidth}
      \centering\panel{0.92}{pos_ppl_gla_70m.png}
    \end{column}
  \end{columns}
  \smallskip
  Same model within each panel; colors vary training context length.
  Longer contexts expose the model to more of its state trajectories.
  \papersource{Paper, Fig. 2(c--d), PDF p. 4. Mamba-2 125M: 25B tokens; GLA 70M: 34B tokens.}
\end{frame}

\begin{frame}{More training can improve local fit and harm extrapolation}
  \begin{columns}[c,onlytextwidth]
    \begin{column}{0.60\textwidth}
      \panel{1}{85m_1024_checkpoints.png}
    \end{column}
    \begin{column}{0.37\textwidth}
      Mamba-2 85M, context 1024.
      \medskip
      \begin{itemize}
        \item Checkpoints: 4.25B--34B tokens.
        \item Legend: multiples of the Chinchilla allocation.
        \item Later checkpoints fit short contexts better but degrade farther out.
      \end{itemize}
    \end{column}
  \end{columns}
  \papersource{Paper, Sec. 3.3 and Fig. 3, PDF pp. 4--5. Warm-up also affects extrapolation (backup).}
\end{frame}

\begin{frame}{Effective Remembrance probes sensitivity to an old prefix}
  \[
    \EffRem_T(t)=
    \TV\!\left(q(\cdot\mid x_{0:T}),q(\cdot\mid x_{t:T})\right),
  \]
  \[\TV(q,q')=\frac12\sum_{v\in\mathcal V}|q(v)-q'(v)|.\]
  \begin{itemize}
    \item Fix prediction position \(T\); restart on suffix \(x_{t:T}\).
    \item Large distance at small \(t\): an old prefix strongly affects the output.
    \item Task accuracy tells us whether the influence is useful.
  \end{itemize}
  \papersource{Paper, Definition 3.3 and Eqs. (2)--(3), PDF p. 5. TV expression corrects a prime-placement typo.}
\end{frame}

\begin{frame}{State Passing reduces distant-prefix sensitivity on text}
  \begin{columns}[c,onlytextwidth]
    \begin{column}{0.49\textwidth}
      \centering\panel{1}{mamba2-effrem.png}
    \end{column}
    \begin{column}{0.49\textwidth}
      \centering\panel{1}{gla-effrem.png}
    \end{column}
  \end{columns}
  Solid: baseline. Dashed: SP. Colors identify model sizes.
  Mamba-2 uses \(T=8192\); GLA uses \(T=2048\).
  \papersource{Paper, Fig. 1(e--f), PDF p. 2; Sec. 3.4, p. 5. Vertical axes are logarithmic.}
\end{frame}

\begin{frame}{The hypothesis concerns which states training covers}
  \begin{columns}[c,onlytextwidth]
    \begin{column}{0.47\textwidth}
      \panel{1}{statemetrics_full.png}
    \end{column}
    \begin{column}{0.50\textwidth}
      \[\mathcal D_t=\operatorname{Law}(h_t(X_{0:t}))\]
      Short training exposes only early \(\mathcal D_t\).
      Long evaluation can reach unfamiliar distributions.
      \par\medskip
      SP produces more stable state statistics, even when their magnitude is larger.
    \end{column}
  \end{columns}
  \papersource{Paper, Secs. 3.5--3.6 and Fig. 4, PDF p. 6. Plot mixes a Norm title with a Std Dev axis.}
\end{frame}

\begin{frame}{A nonzero initial state adds a propagated contribution}
  For a given coefficient sequence, unroll the linear recurrence:
  \[
  \begin{aligned}
    h_t&=\underbrace{\sum_{s=0}^{t} A_{s+1:t}^{\times}B_sx_s}_{\text{current input}}
       +\underbrace{A_{0:t}^{\times}h_{-1}}_{\text{initial-state contribution}},\\[2mm]
    y_t&=C_t^\top\sum_{s=0}^{t} A_{s+1:t}^{\times}B_sx_s
       +C_t^\top A_{0:t}^{\times}h_{-1}.
  \end{aligned}
  \]
  \[
    A_{s:t}^{\times}=A_tA_{t-1}\cdots A_s,\qquad
    h_t,B_t,C_t\in\mathbb R^N,\quad A_t\in\mathbb R^{N\times N}.
  \]
  Scan or chunk computations can incorporate the extra initial-state term.
  \papersource{Paper, Eq. (4), PDF p. 7; Appendix H, p. 20. Ordered products are required for general matrices.}
\end{frame}

\begin{frame}{Four interventions vary the initial-state distribution}
  \renewcommand{\arraystretch}{1.45}
  \begin{tabular}{@{}p{2.6cm}p{5.1cm}p{5.4cm}@{}}
    \toprule
    Intervention & Initial state & Structure retained\\
    \midrule
    Random Noise & IID \(\mathcal N(0,\sigma^2)\) & One global scale\\
    Fitted Noise & Gaussian fitted per layer/head & Means and variances\\
    State Passing & Final state of another sequence & Full generated state\\
    TBTT & Final state of preceding chunk & State and input continuity\\
    \bottomrule
  \end{tabular}\par\medskip
  \alert{The comparison tests state coverage as well as context continuity.}
  \papersource{Paper, Secs. 4.2--4.5, PDF pp. 7--9.}
\end{frame}

\begin{frame}{State Passing carries states between shuffled batches}
  \begin{block}{Training flow (teaching schematic)}
    Shuffled batch \(k\) \(\longrightarrow\) final states
    \(\longrightarrow\) detach
    \(\longrightarrow\) initial states of batch \(k+1\).
  \end{block}
  \begin{itemize}
    \item Keep the original short sequence length for each training example.
    \item Reset the initial state to zero with probability \(p=0.1\).
    \item Maintain state tensors for the model's layers and heads.
    \item TBTT instead preserves consecutive chunks from one long sequence.
  \end{itemize}
  \medskip
  After training, use the learned weights with ordinary recurrent inference.
  \papersource{Paper, Secs. 4.4--4.5, PDF pp. 8--9. Schematic reorganizes the described procedure.}
\end{frame}

\begin{frame}{Fitted Noise matches statistics by layer and head}
  For layer \(l\), head \(i\), update final-state statistics:
  \[
  \begin{aligned}
    \mu^{(li)}&\leftarrow
      (1-\beta)\Mean(h_T^{(li)})+\beta\mu^{(li)},\\
    (\sigma^2)^{(li)}&\leftarrow
      (1-\beta)\Var(h_T^{(li)})+\beta(\sigma^2)^{(li)}.
  \end{aligned}
  \]
  \begin{itemize}
    \item \(\beta=0.1\): new statistics receive weight \(0.9\).
    \item Sample entries independently from
      \(\mathcal N(\mu^{(li)},(\sigma^2)^{(li)})\).
    \item Statistics cover head channels and state expansion dimensions.
    \item Noise is used in training; validation starts from zero.
  \end{itemize}
  \papersource{Paper, Secs. 4.2--4.3 and Eqs. (5)--(6), PDF pp. 7--8.}
\end{frame}

\begin{frame}{At 100 steps, SP and TBTT are the most consistent}
  \begin{columns}[c,onlytextwidth]
    \begin{column}{0.41\textwidth}
      \centering\panel{1}{780m_interventions.png}
    \end{column}
    \begin{column}{0.57\textwidth}
      \centering\panel{1}{1.3b_interventions.png}
    \end{column}
  \end{columns}
  Fitted Noise helps 370M and 780M, but fails at long positions for 1.3B.
  SP and TBTT remain effective through 32K in all three tested sizes.
  \papersource{Paper, Fig. 5 (780M and 1.3B shown), PDF p. 7. All interventions: 100 post-training steps.}
\end{frame}

\begin{frame}{State Passing also improves Mamba-1 and GLA}
  \begin{columns}[c,onlytextwidth]
    \begin{column}{0.49\textwidth}
      \centering\panel{1}{mamba1-poswise.png}
    \end{column}
    \begin{column}{0.49\textwidth}
      \centering\panel{1}{glas-poswise.png}
    \end{column}
  \end{columns}
  Mamba-1: \(T_{\rm train}=2048\), 1000 SP steps.
  GLA: \(T_{\rm train}=512\), 500 SP steps.
  \papersource{Paper, Fig. 1(a,c), PDF p. 2. Tests extend to roughly 128K; curves need not be perfectly flat.}
\end{frame}

\begin{frame}{The headline results use several distinct budgets}
  \renewcommand{\arraystretch}{1.35}
  \begin{tabular}{@{}p{4.1cm}p{2.3cm}p{3.1cm}p{3.6cm}@{}}
    \toprule
    Experiment & Train context & Added training & Evaluation\\
    \midrule
    Mamba-2, four methods & 2K & 100 steps & 32K PPL\\
    Mamba-2, SP & 2K & 500 steps & \(\sim128\)K PPL\\
    Mamba-1, SP & 2K & 1000 steps & \(\sim128\)K PPL\\
    GLA, SP & 512 & 500 steps & \(\sim128\)K PPL\\
    Passkey fine-tuning & 2K & 1000 steps & Up to 256K\\
    \bottomrule
  \end{tabular}\par\medskip
  Pile interventions use a peak learning rate one tenth of the tabulated base recipe.
  \papersource{Paper, Figs. 1 and 5; Appendix B.2/B.4, PDF p. 17. Percent-budget headlines are setting-specific.}
\end{frame}

\begin{frame}{BABILong improves, while accuracy still falls with length}
  \begin{columns}[c,onlytextwidth]
    \begin{column}{0.49\textwidth}
      \centering\panel{1}{babilong_fs.png}
    \end{column}
    \begin{column}{0.49\textwidth}
      \centering\panel{1}{babilong_finetuned.png}
    \end{column}
  \end{columns}
  \medskip
  Mamba-2 780M; qa1--qa10; 2K training/fine-tuning context.
  Task fine-tuning is one epoch \emph{without} SP or TBTT in every group.
  \papersource{Paper, Fig. 6, PDF p. 9; Appendix B.3, p. 17. At 256K, fine-tuned SP is about 0.24 (visual estimate).}
\end{frame}

\begin{frame}{Fitted Noise enables 256K passkey retrieval for 780M}
  \begin{columns}[c,onlytextwidth]
    \begin{column}{0.46\textwidth}
      \centering\panel{1}{passkey_official_780m.png}
    \end{column}
    \begin{column}{0.52\textwidth}
      \centering\panel{1}{passkey_fitted_780m.png}
    \end{column}
  \end{columns}
  \medskip
  \begin{itemize}
    \item Both are fine-tuned for 1000 steps on 2K sequences.
    \item Green denotes accuracy 1; rows vary passkey depth.
    \item Fitted Noise avoids carrying an answer-bearing terminal state into a new task.
  \end{itemize}
  \papersource{Paper, Fig. 7(e--f) and Sec. 5.2, PDF p. 10; Appendix B.4, p. 17. Task-specific fine-tuning.}
\end{frame}

\begin{frame}{Copying improves by 20 percentage points, but remains hard}
  \begin{center}
  \renewcommand{\arraystretch}{1.7}
  \begin{tabular}{lc}
    \toprule
    Training method & Character accuracy at length 300\\
    \midrule
    Mamba-2 & \(0.27\pm0.03\)\\
    Mamba-2 + State Passing & \alert{\(0.47\pm0.11\)}\\
    \bottomrule
  \end{tabular}
  \end{center}
  \begin{itemize}
    \item 45M parameters, trained from random initialization.
    \item One million sequences; training lengths 50--100.
    \item Character accuracy differs from exact whole-sequence copying.
  \end{itemize}
  \papersource{Paper, Table 1 and Sec. 5.3, PDF p. 10. The paper does not define the reported \(\pm\).}
\end{frame}

\begin{frame}{The contribution is a state-distribution explanation}
  \begin{block}{Prior work acknowledged by the authors}
    LongSSM already examined zero initialization and proposed
    State Passing / TBTT pre-training.
  \end{block}
  This paper adds:
  \begin{itemize}
    \item The unexplored-states hypothesis and an output-level diagnostic.
    \item A controlled comparison of initial-state distributions.
    \item Low-budget post-training across several recurrent architectures.
    \item Evidence on reasoning, retrieval, and copying.
  \end{itemize}
  \papersource{Paper, Sec. 6, PDF p. 11; Appendix A, p. 16. Prior-work discussion follows the paper's attribution.}
\end{frame}

\begin{frame}{Practical choices depend on the state and input together}
  \begin{itemize}
    \item \textbf{State Passing:} retains a full generated state; works well with shuffled text.
    \item \textbf{TBTT:} retains real temporal continuity; needs ordered chunks.
    \item \textbf{Fitted Noise:} controls the initial distribution; omits correlations.
  \end{itemize}
  \begin{block}{What changes the interpretation}
    Stable low-order statistics do not identify a full state distribution.
    Passkey inputs are nonstationary, so an actual terminal state may be a poor initial state.
  \end{block}
  Fig. 5 uses the original checkpoint as Base; no matched 100-step zero-state continuation control is shown.
  \papersource{Paper, Secs. 3.6 and 4, pp. 6--9; Appendix B.4, p. 17. Joint-distribution framing is reading analysis.}
\end{frame}

\begin{frame}{Train the model on states it will encounter later}
  \begin{enumerate}
    \item Short, zero-initialized training can leave long-run states unexplored.
    \medskip
    \item A small amount of training with realistic initial states can repair severe extrapolation failure.
    \medskip
    \item Check distant-information tasks as well as PPL: retrieval can succeed while copying and reasoning remain difficult.
  \end{enumerate}
  \medskip
  \alert{Initial-state distributions are a useful training design variable.}
  \papersource{Synthesis of the results. Other distributions, S4, and hybrid models are future directions (Sec. 7, p. 11).}
\end{frame}

\begin{frame}{Reference and backup material}
  \textbf{Ricardo Buitrago Ruiz and Albert Gu.}\\[2mm]
  \emph{Understanding and Improving Length Generalization in Recurrent Models.}\\[2mm]
  arXiv:2507.02782v2, October 12, 2025.\\[2mm]
  \href{https://arxiv.org/abs/2507.02782v2}{https://arxiv.org/abs/2507.02782v2}
  \medskip
  \begin{itemize}
    \item Experimental plots are author figures from this version.
    \item Tables reproduce selected results or reorganize reported settings.
    \item Teaching examples and interpretation are identified where used.
  \end{itemize}
  \papersource{Prepared as an independent research discussion. All page references refer to the paper PDF.}
\end{frame}

\renewcommand{\appendixname}{Appendix}
\appendix

\begin{frame}{Backup: RWKV-v6 provides another architecture test}
  \begin{columns}[c,onlytextwidth]
    \begin{column}{0.50\textwidth}
      \centering\panel{0.95}{fig_rwkv-v6.png}
    \end{column}
    \begin{column}{0.47\textwidth}
      \begin{itemize}
        \item 70M parameters, context 256.
        \item Pre-training: 35B tokens.
        \item SP post-training: 250M tokens.
        \item Baseline diverges; SP stays much more stable.
      \end{itemize}
      \medskip
      Added token budget: about \(0.71\%\) in this experiment.
    \end{column}
  \end{columns}
  \papersource{Paper, Appendix D and Fig. 10, PDF pp. 17--18. Percentage computed from the token counts.}
\end{frame}

\begin{frame}{Backup: warm-up and noise scale both matter}
  \begin{columns}[c,onlytextwidth]
    \begin{column}{0.47\textwidth}
      \centering\panel{1}{warmup.png}
    \end{column}
    \begin{column}{0.51\textwidth}
      \centering\panel{1}{125m_mamba2_initial_stds.png}
    \end{column}
  \end{columns}
  Longer warm-up worsens extrapolation in the 45M example.
  Excessive initial-state noise hurts PPL in the 125M example.
  \papersource{Paper, Figs. 11--12, PDF p. 19. Left: 9B tokens, context 1024. Right: 100 post-training steps.}
\end{frame}

\begin{frame}{Backup: a few heads explain much of the state shift}
  \begin{center}
    \includegraphics[width=0.96\linewidth,height=4.55cm,keepaspectratio]{figures/statemetrics_heads.png}
  \end{center}
  Heads were selected for strong variation. SP sometimes increases standard deviation while stabilizing its long-position behavior.
  \papersource{Paper, Fig. 9 and Appendix C, PDF pp. 17--18. Mamba-2 130M; most other heads lack this abnormal growth.}
\end{frame}

\begin{frame}{Backup: the sensitivity pattern survives other distances}
  \begin{columns}[c,onlytextwidth]
    \begin{column}{0.49\textwidth}
      \centering\panel{1}{effrem_jenshen_shannon.png}
    \end{column}
    \begin{column}{0.49\textwidth}
      \centering\panel{1}{effrem_cos_sim.png}
    \end{column}
  \end{columns}
  Shapes are similar to TV, but absolute values differ.
  The right panel uses \(1-\)cosine similarity despite its abbreviated title.
  \papersource{Paper, Fig. 13(b--c) and Appendix G, PDF p. 20. Jensen--Shannon distance/divergence naming varies.}
\end{frame}

\begin{frame}{Backup: implement the transition product with care}
  For diagonal positive transitions, let \(a_t\) be their entries
  and \(\ell_t=\log a_t\).
  \[
     A_{0:t}^{\times}h_{-1}
        =\exp\!\left(\sum_{u=0}^{t}\ell_u\right)\odot h_{-1}.
  \]
  \begin{itemize}
    \item A prefix sum in log space implements the elementwise product.
    \item Add the initial-state contribution to the zero-state output.
    \item The paper uses \(\exp(\mathrm{cumsum}(A))\):
          interpret its code variable in log-transition space.
    \item General matrices require ordered products \(A_t\cdots A_s\).
  \end{itemize}
  \papersource{Paper, Appendices H--I and Fig. 14, PDF pp. 20--21. Clarification from the recurrence and code.}
\end{frame}

\begin{frame}{Backup: a stable recurrence can have an unseen state scale}
  \textbf{Teaching example, not a paper experiment:}
  \[
    h_t=a h_{t-1}+x_t,\quad h_{-1}=0,\quad |a|<1,
  \]
  with IID, zero-mean, unit-variance inputs.
  \[
    \Var(h_t)=\frac{1-a^{2(t+1)}}{1-a^2},
    \qquad \lim_{t\to\infty}\Var(h_t)=\frac{1}{1-a^2}.
  \]
  For \(a=0.99\):
  \begin{itemize}
    \item After 32 tokens: \(47.4\%\) of the limiting variance.
    \item After 512 tokens: approximately the limiting variance.
  \end{itemize}
  The state variance stays finite, yet changes beyond a short training horizon.
  \papersource{Illustrative derivation for the state-distribution idea in Sec. 3.6, PDF p. 6.}
\end{frame}

\end{document}