% Adapted from the arxiv-paper-reader Beamer template.
% Frames 1--23: approximately 28 minutes. Remaining frames: backup.
% Compile with: python3 build_slides.py slides.en.tex
\documentclass[aspectratio=169,11pt]{beamer}
\usepackage[T1]{fontenc}
\usepackage[utf8]{inputenc}
\usepackage{lmodern}
\usepackage{amsmath,amssymb,graphicx,booktabs,array}
\definecolor{AcademicBlue}{RGB}{26,57,86}
\definecolor{AcademicTeal}{RGB}{0,105,112}
\definecolor{BPTTRed}{RGB}{193,55,39}
\definecolor{SMTGreen}{RGB}{31,169,97}
\definecolor{TeacherBlue}{RGB}{42,128,182}
\setbeamercolor{structure}{fg=AcademicBlue}
\setbeamercolor{frametitle}{fg=AcademicBlue,bg=white}
\setbeamercolor{title}{fg=AcademicBlue}
\setbeamercolor{alerted text}{fg=AcademicTeal}
\setbeamercolor{block title}{fg=white,bg=AcademicBlue}
\setbeamercolor{block body}{fg=black,bg=AcademicBlue!5}
\setbeamertemplate{navigation symbols}{}
\setbeamertemplate{itemize items}[circle]
\setbeamertemplate{footline}{%
  \hspace{4mm}{\scriptsize\color{gray}SMT \;|\; Kumar \& Isola, 2026}
  \hfill\usebeamerfont{page number in head/foot}%
  \insertframenumber\,/\,\inserttotalframenumber\hspace{4mm}\vspace{3mm}}
\setbeamersize{text margin left=8mm,text margin right=8mm}
\setbeamerfont{frametitle}{size=\Large,series=\bfseries}
\setbeamerfont{title}{size=\LARGE,series=\bfseries}
\newcommand{\papersource}[1]{\par\smallskip{\scriptsize\color{gray}#1\par}}
\newcommand{\E}{\mathcal E_\phi}
\newcommand{\D}{\mathcal D_\psi}
\newcommand{\Ls}{\mathcal L}
\newcommand{\fig}[2][0.68\textheight]{%
  \includegraphics[width=\linewidth,height=#1,keepaspectratio]{figures/#2}}
\newcommand{\methodlegend}{%
  {\small\textcolor{BPTTRed}{BPTT RNN}\quad
  \textcolor{SMTGreen}{SMT$\rightarrow$DMT RNN}\quad
  \textcolor{TeacherBlue}{SMT Encoder (teacher)}}}
\hypersetup{colorlinks=true,urlcolor=AcademicTeal}
\pdfstringdefDisableCommands{\def\translate#1{#1}}
\title[SMT]{Pretraining Recurrent Networks\\without Recurrence}
\subtitle{Supervised Memory Training}
\author{Paper authors: Akarsh Kumar and Phillip Isola\texorpdfstring{\\}{, }MIT}
\date{arXiv:2606.06479v2 \;|\; 24 July 2026}
\begin{document}

\begin{frame}[plain]
  \vspace{3mm}
  {\usebeamerfont{title}\usebeamercolor[fg]{title}\inserttitle\par}
  \vspace{5mm}
  {\large Supervised Memory Training (SMT)\par}
  \vspace{4mm}
  Paper authors: Akarsh Kumar and Phillip Isola, MIT\\
  \href{https://arxiv.org/abs/2606.06479v2}{arXiv:2606.06479v2}, revised 24 July 2026
  \begin{block}{Central idea}
    Learn useful memory states in parallel, then supervise an RNN's
    one-step updates between those states.
  \end{block}
  {\small Research discussion \;|\; 29 October 2026\\[2pt]
  Publication not found as of 2026-10-29;
  \href{https://akarshkumar.com/smt/}{author page: Preprint 2026}.\\
  \href{https://github.com/akarshkumar0101/smt}{Official code: SMT and DMT in PyTorch}.}
\end{frame}

\begin{frame}{Memory labels shorten the credit assignment path}
  \centering
  \fig[0.54\textheight]{teaser_methods.pdf}
  \begin{columns}[T]
    \begin{column}{0.48\textwidth}
      \textbf{BPTT}\\
      Learn representation and dynamics through a recurrent chain.
    \end{column}
    \begin{column}{0.48\textwidth}
      \textbf{SMT}\\
      Predict the future from an encoded memory; fit one-step transitions.
    \end{column}
  \end{columns}
  \papersource{Paper, Fig. 1, PDF p. 1; Secs. 1--2, pp. 2--4. Path length is relative to sequence length.}
\end{frame}

\begin{frame}{The deployed model has a fixed memory}
  \centering
  \fig[0.58\textheight]{architecture.pdf}
  \begin{itemize}
    \item The teacher encodes a past window into memory tokens.
    \item The RNN attends only to old memory tokens and one new input.
    \item Default state: 16 tokens $\times$ 256 dimensions = 4096 scalars.
  \end{itemize}
  \papersource{Paper, Fig. 15 and Appendix B.1, PDF pp. 20--21; defaults, pp. 22--23.}
\end{frame}

\begin{frame}{Future prediction gives memory its meaning}
  \[
    m_t=\E(\mathbf x_t^{\mathrm{ctx}}),\qquad
    \mathbf x_t^{\mathrm{ctx}}=[x_0,\ldots,x_t]
  \]
  \[
    p(\mathbf y_t^{\mathrm{fut}}\mid
      \mathbf x_t^{\mathrm{ctx}},\mathbf x_t^{\mathrm{fut}})
    =\prod_{\tau=t}^{T}p_\psi(y_\tau\mid m_t,\mathbf x_{t+1:\tau})
  \]
  \begin{block}{Predictive-state objective}
    $\Ls_t^{\mathrm{dec}}
    =\mathrm{CE}\bigl(\mathbf y_t^{\mathrm{fut}},
      \D(m_t,\mathbf x_t^{\mathrm{fut}})\bigr)$
  \end{block}
  \begin{itemize}
    \item Bidirectional encoding stays within the past context.
    \item Causal decoding uses teacher-forced future inputs.
    \item To predict $y_\tau$, the decoder sees inputs only through $\tau$.
  \end{itemize}
  \papersource{Paper, Sec. 2.2, PDF pp. 3--4, Eq. (2); Appendix B.1, p. 20.}
\end{frame}

\begin{frame}{Three losses shape useful, updatable memories}
  \[
    \hat m_{t+1}=f_\theta(m_t,x_{t+1}),\qquad
    \Ls_t^{\mathrm{dyn}}=\mathrm{MSE}(\hat m_{t+1},m_{t+1})
  \]
  \[
    \Ls^{\mathrm{unif}}
    =\log\mathbb E_{a,b}\exp\!\left(-2\|m_a-m_b\|_2^2\right)
  \]
  \[
    \boxed{\Ls^{\mathrm{smt}}
      =\lambda_{\mathrm{dec}}\mathbb E_t\Ls_t^{\mathrm{dec}}
      +\lambda_{\mathrm{dyn}}\mathbb E_t\Ls_t^{\mathrm{dyn}}
      +\lambda_{\mathrm{unif}}\Ls^{\mathrm{unif}}}
  \]
  \begin{itemize}
    \item Prediction: retain information useful for future outputs.
    \item Dynamics: train the updater \emph{and} encourage Markovian states.
    \item Uniformity: discourage collapse of distinct memories.
  \end{itemize}
  \alert{Default SMT jointly trains the teacher and the RNN.}
  \papersource{Paper, Sec. 2.2, PDF p. 4, Eqs. (3)--(5).}
\end{frame}

\begin{frame}{DMT corrects drift on the RNN's own states}
  \begin{columns}[c]
    \begin{column}{0.40\textwidth}
      \centering\fig[0.46\textheight]{smt_dmt_path.pdf}
      \papersource{Paper, Fig. 2, PDF p. 4.}
    \end{column}
    \begin{column}{0.57\textwidth}
      \[
      \begin{gathered}
        \underbrace{(m_t,x_{t+1})}_{\text{SMT}}
        \;\longrightarrow\;
        \underbrace{(\hat m_t,x_{t+1})}_{\text{DMT}}\\[2pt]
        \Ls^{\mathrm{dmt}}=\mathbb E_t
          [\mathrm{MSE}(\hat m_t,m_t)]
      \end{gathered}
      \]
      \begin{itemize}
        \item Freeze the teacher; roll out the RNN.
        \item Inputs remain teacher-forced.
        \item Rollout is sequential; temporal gradients are optional.
      \end{itemize}
    \end{column}
  \end{columns}
  \begin{block}{Readout adaptation}
    Copy decoder weights to the RNN readout, then fine-tune the readout
    with next-token loss. This loss does not update RNN dynamics.
  \end{block}
  \papersource{Paper, Sec. 2.3, PDF pp. 4--5, Eq. (6); Appendix B.3, p. 22.}
\end{frame}

\begin{frame}{Parallel pretraining leads to recurrent deployment}
  \centering
  \begin{tabular}{lccc}
    \toprule
    & BPTT & SMT & DMT\\
    \midrule
    RNN rollout in training & Yes & No & Yes\\
    Serial operations vs.\ $T$ & $O(T)$ & $O(1)$ & $O(T)$\\
    Supervisory signal & Task targets & Teacher states & Teacher states\\
    \bottomrule
  \end{tabular}
  \vspace{4mm}
  \begin{itemize}
    \item SMT's $O(1)$ path is relative to time horizon, at fixed depth.
    \item Deployment uses fixed memory and constant per-token work
          \emph{with respect to history length}.
    \item The time-parallel teacher still has expressivity limits.
  \end{itemize}
  \alert{The full evaluated method is SMT$\rightarrow$DMT.}
  \papersource{Paper, Table 1, PDF p. 5; Discussion, p. 11; Appendices A and E, pp. 20, 25.}
\end{frame}

\begin{frame}{The experiments isolate training and memory effects}
  \begin{columns}[T]
    \begin{column}{0.48\textwidth}
      \textbf{Comparison}\\[2mm]
      \begin{itemize}
        \item BPTT RNN.
        \item SMT$\rightarrow$DMT RNN.
        \item SMT Encoder*: teacher with the same memory bottleneck.
      \end{itemize}
      \medskip
      \textbf{Updater backbones}\\
      Transformer, MLP, GRU.
    \end{column}
    \begin{column}{0.48\textwidth}
      \textbf{Tasks}\\[2mm]
      \begin{itemize}
        \item Five controlled synthetic tasks.
        \item TinyStories: character-level language modeling.
        \item MNIST: 784 raw-pixel tokens per image.
        \item Sketchy: 1024 binary-patch tokens per image.
      \end{itemize}
    \end{column}
  \end{columns}
  \vspace{3mm}
  Test loss $\downarrow$; compare serial compute and processed tokens.
  \papersource{Paper, Sec. 3, PDF pp. 5--7; Appendix B, pp. 20--23.}
\end{frame}

\begin{frame}{SMT improves long-range retrieval and state use}
  \begin{columns}[c]
    \begin{column}{0.69\textwidth}
      \centering\fig[0.69\textheight]{synthetic-first-three.pdf}
    \end{column}
    \begin{column}{0.29\textwidth}
      \begin{itemize}
        \item Retrieval resists longer delays.
        \item Copying uses larger memory.
        \item Stack tracking remains more accurate.
      \end{itemize}
    \end{column}
  \end{columns}
  {\small Top: BPTT. Bottom: SMT$\rightarrow$DMT.\\
  Horizontal axes: sequence length, 4--64. Test loss: blue $0$, red $3$.}
  \papersource{Paper, Fig. 3, PDF p. 6. First three task columns and two RNN rows cropped; teacher row omitted.}
\end{frame}

\begin{frame}{Associative recall remains difficult}
  \begin{columns}[c]
    \begin{column}{0.56\textwidth}
      \centering\fig[0.67\textheight]{synthetic-last-two.pdf}
    \end{column}
    \begin{column}{0.41\textwidth}
      \begin{itemize}
        \item More pairs and more complex strings still hurt recall.
        \item In-context arithmetic improves, with failures at high difficulty.
        \item The full figure also shows limitations in the teacher.
      \end{itemize}
    \end{column}
  \end{columns}
  {\small Top: BPTT. Bottom: SMT$\rightarrow$DMT. Test loss: blue $0$, red $3$.\\
  Horizontal axes: number of key--value pairs / in-context examples.}
  \papersource{Paper, Fig. 3, PDF p. 6. Last two task columns, two RNN rows; task definitions, pp. 21--22.}
\end{frame}

\begin{frame}{Pixel streams reveal the value of long-range memory}
  \begin{columns}[c]
    \begin{column}{0.49\textwidth}
      \centering\fig[0.58\textheight]{mnist_generations.pdf}
      \papersource{MNIST: data, BPTT Transformer, BPTT GRU, SMT$\rightarrow$DMT Transformer.}
    \end{column}
    \begin{column}{0.49\textwidth}
      \centering\fig[0.58\textheight]{sketchy_generations.pdf}
      \papersource{Sketchy: data above, SMT$\rightarrow$DMT generations below.}
    \end{column}
  \end{columns}
  \vspace{2mm}
  SMT$\rightarrow$DMT captures coherent strokes; some generated sketches
  remain hard to identify.
  \papersource{Paper, Figs. 4--5, PDF p. 6; additional samples and discussion, pp. 23--24, 27.}
\end{frame}

\begin{frame}{TinyStories gains are clearest in serial compute}
  \centering
  \fig[0.63\textheight]{seqflops-tinystories-serial.pdf}
  \par\methodlegend
  \begin{itemize}
    \item Transformer/MLP RNNs reach lower loss more efficiently in SeqFLOPs.
    \item Token efficiency is roughly similar to BPTT in the paper's sweep.
  \end{itemize}
  \papersource{Paper, Fig. 6, PDF p. 7 (TinyStories). Stars: best RNN loss. SeqFLOPs measure serial work, not elapsed time.}
\end{frame}

\begin{frame}{MNIST gains coexist with a GRU failure}
  \centering
  \fig[0.58\textheight]{seqflops-mnist-serial.pdf}
  \par\methodlegend
  \vspace{2mm}
  \begin{itemize}
    \item Transformer/MLP improve in both serial compute and data efficiency.
    \item GRU training suffers memory collapse under this SMT recipe.
  \end{itemize}
  \papersource{Paper, Fig. 6, PDF p. 7 (MNIST). Stars: best RNN loss. Each run in this sweep is capped at one H200-day.}
\end{frame}

\begin{frame}{Context and state capacity provide complementary gains}
  \begin{columns}[c]
    \begin{column}{0.64\textwidth}
      \centering\fig[0.66\textheight]{memory_Tc_scaling.pdf}
    \end{column}
    \begin{column}{0.33\textwidth}
      \begin{itemize}
        \item TinyStories, SMT$\rightarrow$DMT.
        \item Longer context helps at fixed state size.
        \item Larger states retain more predictive information.
        \item Context gains diminish near $T_c=256$--512.
      \end{itemize}
    \end{column}
  \end{columns}
  \papersource{Paper, Fig. 7, PDF p. 7. Memory-size labels count scalars, not memory tokens.}
\end{frame}

\begin{frame}{Larger RNNs approach their teacher's performance}
  \centering
  \fig[0.65\textheight]{model_scaling.pdf}
  \begin{itemize}
    \item Width and depth are varied jointly across a model-size sweep.
    \item Both improve with scale; the student gap narrows at larger sizes.
  \end{itemize}
  \papersource{Paper, Fig. 8, PDF p. 7. TinyStories; teacher left, RNN right. No universal fitted exponent is reported.}
\end{frame}

\begin{frame}{More training compute can buy a smaller memory}
  \begin{columns}[c]
    \begin{column}{0.57\textwidth}
      \centering\fig[0.72\textheight]{compression_scaling.pdf}
    \end{column}
    \begin{column}{0.40\textwidth}
      \begin{itemize}
        \item Black curves are equal-loss contours.
        \item Moving right permits moving down: more compute, less state.
        \item These measurements are for the \alert{SMT encoder}.
        \item RNN rollout quality still depends on fitting its dynamics.
      \end{itemize}
    \end{column}
  \end{columns}
  \papersource{Paper, Fig. 9, PDF p. 8. Horizontal axis: total training FLOPs; vertical axis: memory scalars.}
\end{frame}

\begin{frame}{Joint training compensates for short future windows}
  \begin{columns}[c]
    \begin{column}{0.59\textwidth}
      \centering\fig[0.61\textheight]{needle_Tf_sweep.pdf}
    \end{column}
    \begin{column}{0.38\textwidth}
      \begin{itemize}
        \item Retrieval horizon: $T=64$.
        \item Detached dynamics need $T_f=64$ to capture the distant target.
        \item Joint training succeeds even at $T_f=1$.
      \end{itemize}
    \end{column}
  \end{columns}
  \begin{block}{Authors' mechanism interpretation}
    Credit can propagate across optimization steps through jointly trained
    adjacent memories, even without an unrolled RNN graph.
  \end{block}
  \papersource{Paper, Fig. 10 and Sec. 3.6, PDF p. 8. This figure does not establish a general iteration lower bound.}
\end{frame}

\begin{frame}{DMT improves long rollouts beyond one-step fitting}
  \centering
  \fig[0.48\textheight]{drift_analysis.pdf}
  \begin{itemize}
    \item Left: post-DMT drift often plateaus at a lower level.
    \item Middle: prediction loss improves across many configurations.
    \item Right: one-step and rollout drift are only partially correlated
          (reported correlations: 0.50 before, 0.45 after).
  \end{itemize}
  \papersource{Paper, Fig. 12, PDF p. 9. Plotted drift uses $1-R^2$; the training objective uses MSE.}
\end{frame}

\begin{frame}{Regularization trades loss against state dispersion}
  \centering
  {\small
  \begin{tabular}{llrrrr}
    \toprule
    Dataset & Setting & $\lambda_{\rm dyn}$ & $\lambda_{\rm unif}$
      & Test loss $\downarrow$ & $\Ls^{\rm unif}$\\
    \midrule
    TinyStories & Default & 0.1 & 0.001 & 0.567 & $-3.294$\\
    TinyStories & Grid minimum & 0.01 & 0.003 & \textbf{0.565} & $-3.682$\\
    MNIST & Default & 0.1 & 0.001 & 0.516 & $-3.618$\\
    MNIST & Grid minimum & 0.1 & 0 & \textbf{0.507} & $-1.273$\\
    MNIST & High dynamics & 10 & 0 & 0.521 & $-0.058$\\
    \bottomrule
  \end{tabular}}
  \vspace{4mm}
  \begin{itemize}
    \item The default balances task loss and memory geometry.
    \item The text's \emph{best} setting differs from numerical grid minima.
    \item A nearly collapsed state can still have a competitive task loss.
  \end{itemize}
  \papersource{Exact cells from paper, Fig. 16, PDF p. 23; compare the recommendation in Sec. 3.6, p. 8.}
\end{frame}

\begin{frame}{The RNN can extrapolate beyond its teacher}
  \begin{columns}[c]
    \begin{column}{0.63\textwidth}
      \centering\fig[0.68\textheight]{seqlen_generalization.pdf}
    \end{column}
    \begin{column}{0.34\textwidth}
      \begin{itemize}
        \item Synthetic stack tracking.
        \item Training length: 64.
        \item At length 512: teacher $\approx0.39$, RNN $\approx0.16$ loss.
        \item Recurrent updates provide a useful inductive bias.
      \end{itemize}
    \end{column}
  \end{columns}
  \papersource{Paper, Fig. 13, PDF p. 9. Approximate graph readings; a synthetic length-generalization result.}
\end{frame}

\begin{frame}{Memory geometry reflects what the task must retain}
  \centering
  \fig[0.64\textheight]{memory_space.pdf}
  \begin{itemize}
    \item Retrieval merges irrelevant histories into a few useful states.
    \item String copy preserves more distinctions in a tree-like geometry.
  \end{itemize}
  \papersource{Paper, Fig. 14, PDF p. 10. True 2D memories; arrows show transitions and the learned update field.}
\end{frame}

\begin{frame}{SMT offers a practical pretraining decomposition}
  \begin{block}{What the evidence supports}
    Parallel predictive-state learning plus supervised dynamics makes
    long-range memory easier to learn in the tested nonlinear RNNs.
  \end{block}
  \begin{itemize}
    \item \textbf{Full recipe:} SMT pretraining and sequential DMT correction.
    \item \textbf{Main costs:} teacher training, rollout drift, architecture sensitivity.
    \item \textbf{Expressivity:} the parallel teacher may miss computations
          that require deep recurrence.
    \item \textbf{Next experiment:} evaluate total cost and rollout quality jointly
          as state size and sequence horizon grow.
  \end{itemize}
  \papersource{Paper, Discussion, PDF p. 11; empirical limits, pp. 7--9. Final experimental suggestion is reader analysis.}
\end{frame}

\begin{frame}{References and discussion prompts}
  \textbf{Primary paper}\\
  Akarsh Kumar and Phillip Isola.\\
  \emph{Pretraining Recurrent Networks without Recurrence}. 2026.\\
  \href{https://arxiv.org/abs/2606.06479v2}{arXiv:2606.06479v2}
  \quad
  \href{https://akarshkumar.com/smt/}{Author project page}
  \quad
  \href{https://github.com/akarshkumar0101/smt}{Official implementation}
  \vspace{4mm}
  \begin{itemize}
    \item Can one-step pretraining reduce the need for DMT?
    \item How should future horizon, state capacity and compute be allocated?
    \item What post-training can teach a student beyond teacher expressivity?
  \end{itemize}
  \papersource{Experimental figures reproduced from the paper's v2 author source and checked against the PDF.
  Publication lookup: 2026-10-29; no formal publication found.}
\end{frame}

\appendix

\begin{frame}{Backup: timestamps preserve sequence information}
  \[
    [x_0,x_1,\ldots,x_t]
    \quad\longleftrightarrow\quad
    \{(x_0,0),(x_1,1),\ldots,(x_t,t)\}
  \]
  \begin{enumerate}
    \item Sort the set by its unique timestamps.
    \item Recover the original sequence.
    \item Apply any recurrent memory function $\mathcal Q$.
  \end{enumerate}
  \begin{block}{What the construction establishes}
    A permutation-invariant set function can represent the same mapping.
    The construction itself still uses the original sequential computation.
  \end{block}
  A bounded-depth Transformer may need depth that grows with sequence length.
  \papersource{Paper, Appendix E, PDF p. 25.}
\end{frame}

\begin{frame}{Backup: Markovian memory requires ideal sufficiency}
  Assume $m_t$ is an \alert{optimal minimal sufficient statistic}
  for future prediction at every $t$.
  \[
    H(\mathbf y_t^{\rm fut}\mid m_t,\mathbf x_t^{\rm fut})
    =
    H(\mathbf y_t^{\rm fut}\mid\mathbf x_t^{\rm ctx},
                                  \mathbf x_t^{\rm fut})
  \]
  \begin{enumerate}
    \item $(m_t,x_{t+1})$ remains sufficient for the remaining future.
    \item Minimality makes $m_{t+1}$ a function of that statistic.
  \end{enumerate}
  \[
    m_{t+1}=f(m_t,x_{t+1}),\qquad
    H(m_{t+1}\mid m_t,x_{t+1})=0
  \]
  \begin{block}{Practical gap}
    Finite horizons and approximate optimization do not ensure minimal
    sufficiency. Joint dynamics explicitly encourages updatable states.
  \end{block}
  \papersource{Paper, Appendix F, PDF p. 26. Cross-entropy alone does not explicitly impose minimality.}
\end{frame}

\begin{frame}{Backup: the two training protocols differ}
  \centering
  {\small
  \begin{tabular}{lll}
    \toprule
    & Synthetic tasks & Natural data defaults\\
    \midrule
    Context / future & $T_c=T_f=T$ & $T_c=256,\ T_f=64$\\
    Time splits & All timesteps & One sampled timestep\\
    Batch & 32 sequences & 128 sequences\\
    Encoder contexts & $32T$ & One per sampled split\\
    Prediction loss & Defined target positions & Future target window\\
    \bottomrule
  \end{tabular}}
  \vspace{4mm}
  \begin{itemize}
    \item AdamW; weight decay 0.01; gradient clipping at norm 1.
    \item Learning rate tuned separately for each method.
    \item Natural-data SMT: 150,000 steps; loss weights $(1,0.1,0.001)$.
    \item Encoder/updater: 8 layers; decoder/readout: 4 layers.
  \end{itemize}
  \papersource{Paper, Appendix B.3, PDF pp. 22--23. Width 256, 16 memory tokens unless otherwise specified.}
\end{frame}

\begin{frame}{Backup: gradient probes support the short-path mechanism}
  \centering
  \fig[0.62\textheight]{vanishing_gradients.pdf}
  \begin{itemize}
    \item Needle retrieval; loss applied at the end of the sequence.
    \item Across initializations, BPTT varies with temporal distance;
          SMT gradients are much flatter.
  \end{itemize}
  \papersource{Paper, Fig. 11, PDF p. 8; Sec. 3.7, p. 9. Probe at initialization, not a general convergence theorem.}
\end{frame}

\begin{frame}{Backup: more image context improves continuations}
  \centering
  \fig[0.73\textheight]{cat_write.pdf}
  \par
  {\small Each row supplies a larger known prefix of Attneave's cat image.}
  \papersource{Paper, Fig. 20, PDF p. 28. Sketchy-trained RNN; samples retain errors, especially with short context.}
\end{frame}

\begin{frame}{Backup: local fitting and rollout stability differ}
  Define the deviation $e_t=\hat m_t-m_t$.
  If the updater is locally Lipschitz in memory,
  \[
    \|e_{t+1}\|\le L_t\|e_t\|+\epsilon_t,
    \qquad
    \epsilon_t=\|f_\theta(m_t,x_{t+1})-m_{t+1}\|.
  \]
  \begin{itemize}
    \item Small teacher-state error $\epsilon_t$ can coexist with amplification
          of off-trajectory errors.
    \item DMT trains on those off-trajectory states.
    \item Lower equilibrium drift may matter more than first-step error.
  \end{itemize}
  \begin{block}{Interpretation, not a measured contraction bound}
    The paper does not establish $L_t<1$ for DMT.
  \end{block}
  \papersource{Reader's explanatory bound; empirical motivation: paper, Fig. 12, PDF p. 9.}
\end{frame}

\begin{frame}{Backup: resource estimates assume a sampled split}
  \centering
  \begin{tabular}{lccc}
    \toprule
    & BPTT & SMT & DMT\\
    \midrule
    Training memory & $O(MT)$ & $O(M+T_c)$ & $O(MT)$\\
    Training compute & $O(T)$ & $O(T_c^2)$ & $O(T_c^2T)$\\
    Serial operations & $O(T)$ & $O(1)$ & $O(T)$\\
    \bottomrule
  \end{tabular}
  \vspace{3mm}
  \begin{itemize}
    \item Simplified orders: model size is fixed and logarithmic factors omitted.
    \item SMT samples one split; the synthetic all-split setup changes work
          and activation memory.
    \item Future decoding and attention implementation affect actual costs.
    \item DMT has direct state supervision; optional temporal backpropagation
          still creates long paths.
  \end{itemize}
  \papersource{Paper, Table 1, PDF p. 5; Secs. 2.2--2.3, pp. 4--5; Appendix B.3, p. 22.}
\end{frame}

\end{document}