% Self-authored paper discussion. Compile with the adjacent build_slides.py.
\documentclass[aspectratio=169,11pt]{beamer}
\usepackage[T1]{fontenc}
\usepackage[utf8]{inputenc}
\usepackage{amsmath,amssymb,graphicx,booktabs,array}
\usepackage{tikz}
\usetikzlibrary{positioning,arrows.meta}

\definecolor{AcademicBlue}{RGB}{26,57,86}
\definecolor{AcademicTeal}{RGB}{0,105,112}
\setbeamercolor{structure}{fg=AcademicBlue}
\setbeamercolor{frametitle}{fg=AcademicBlue,bg=white}
\setbeamercolor{title}{fg=AcademicBlue}
\setbeamercolor{alerted text}{fg=AcademicTeal}
\setbeamercolor{block title}{fg=white,bg=AcademicBlue}
\setbeamercolor{block body}{fg=black,bg=AcademicBlue!5}
\setbeamertemplate{navigation symbols}{}
\setbeamertemplate{itemize items}[circle]
\setbeamertemplate{footline}{%
  \hfill\usebeamerfont{page number in head/foot}%
  \insertframenumber\,/\,\inserttotalframenumber\hspace{4mm}\vspace{3mm}}
\setbeamersize{text margin left=8mm,text margin right=8mm}
\setbeamerfont{frametitle}{size=\Large,series=\bfseries}
\setbeamerfont{title}{size=\LARGE,series=\bfseries}
\setbeamerfont{author}{size=\small}
\newcommand{\papersource}[1]{\par\smallskip{\scriptsize\color{gray}#1\par}}
\newcommand{\gap}{\par\vspace{2mm}}

\title[Transformer topology]{The Topological Trouble\\With Transformers}
\subtitle{State tracking, feedback paths, and serial computation}
\author{Paper by Michael C. Mozer, Shoaib Ahmed Siddiqui, and Rosanne Liu\\Google DeepMind\\[2mm]Independent research group discussion}
\date{\href{https://arxiv.org/abs/2604.17121v5}{arXiv:2604.17121v5} (September 1, 2026)\\Reading: October 19, 2026}

\begin{document}

% Main talk: frames 1--22, approximately 25 minutes.
\begin{frame}
  \titlepage
  \vspace{-3mm}
  \centering\small\color{AcademicTeal}
  A computed state must become available to the next computation.
\end{frame}

\begin{frame}{The question is how a computed state is reused}
  \begin{block}{State tracking}
    Update a representation as new events arrive:
    \(s_t=f(s_{t-1},x_t)\).
  \end{block}
  \begin{itemize}
    \item Long context provides access to historical information.
    \item Sequential updates also require access to prior \emph{computed} states.
    \item The paper links this requirement to the model's computation graph.
  \end{itemize}
  \gap
  \textbf{Contribution:} a structural argument, a recurrence taxonomy, and a research agenda.
  \papersource{Position and synthesis paper; no new controlled benchmark study. Paper, Secs. 1--6, PDF pp. 1--10.}
\end{frame}

\begin{frame}{A short dialogue can require persistent constraints}
  \centering
  \begin{tabular}{lll}
    \toprule
    Guess & Reported response & Consistent integer range\\
    \midrule
    60 & lower & $[1,59]$\\
    41 & lower & $[1,40]$\\
    70 & higher & $\varnothing$\\
    \bottomrule
  \end{tabular}
  \gap
  \begin{block}{The task can be solved without choosing a hidden number}
    Maintaining a nonempty interval is enough to remain consistent.
  \end{block}
  \begin{itemize}
    \item A second trace explicitly selects 42, then misses the guess 42.
    \item A state written in context must still be used correctly later.
  \end{itemize}
  \papersource{Paper, Sec. 2, PDF p. 2. Gemini 3 Fast / Thinking examples; illustrative traces, not failure-rate estimates.}
\end{frame}

\begin{frame}{Direct state-update chains climb the layer stack}
  \[
    h_t^{(\ell)}=F_\ell\bigl(h_1^{(\ell-1)},\ldots,h_t^{(\ell-1)}\bigr)
  \]
  \centering
  \includegraphics[width=0.92\linewidth,height=0.45\textheight,keepaspectratio]{figures/fig-01.pdf}
  \begin{itemize}
    \item Horizontal: input positions. Vertical: complete Transformer blocks.
    \item A result formed at layer $\ell$ reaches later positions through deeper blocks.
  \end{itemize}
  \papersource{Paper, Fig. 1, PDF p. 3. Equation is explanatory notation; shortcuts can change the computation path.}
\end{frame}

\begin{frame}{A correct deep state can arrive too late}
  \begin{columns}[c,onlytextwidth]
    \begin{column}{0.61\textwidth}
      \includegraphics[width=\linewidth,height=0.71\textheight,keepaspectratio]{figures/fig-03.pdf}
    \end{column}
    \begin{column}{0.36\textwidth}
      Fishing context resolves \emph{bank} as a river bank.
      \gap
      Later shallow computation can still use the bank--ATM association.
      \gap
      \alert{State availability has a timing requirement.}
    \end{column}
  \end{columns}
  \papersource{Paper, Sec. 2 and Fig. 3, PDF pp. 4--5; adapted from Lepori et al. (2025). Layer six is schematic.}
\end{frame}

\begin{frame}{Intermediate outputs create a feedback path}
  \begin{center}
    \begin{tikzpicture}[node distance=8mm,>=Stealth,
      box/.style={draw=AcademicBlue,fill=AcademicBlue!5,rounded corners,align=center,minimum height=12mm,text width=33mm}]
      \node[box] (deep) {Deep-layer\\computation};
      \node[box,right=of deep] (token) {Intermediate\\output};
      \node[box,right=of token] (input) {Next input\\representation};
      \draw[->,thick,AcademicTeal] (deep)--(token);
      \draw[->,thick,AcademicTeal] (token)--(input);
    \end{tikzpicture}
  \end{center}
  \begin{itemize}
    \item CoT exposes intermediate results as tokens.
    \item Latent thought can feed back continuous representations.
    \item Both may add serial passes and memory positions.
  \end{itemize}
  \begin{block}{The authors' efficiency hypothesis}
    Routine state updates could be handled by implicit activation dynamics.
  \end{block}
  \papersource{Teaching schematic based on Paper, Sec. 2, p. 4 and Sec. 4, pp. 8--9. No matched-cost study in this paper.}
\end{frame}

\begin{frame}{Layer, input, and execution steps are distinct}
  \begin{tabular}{p{0.19\linewidth}p{0.73\linewidth}}
    \toprule
    Layer $\ell$ & Depth within the network; a shared block can be repeated.\\[2mm]
    Input $t$ & Position in the input sequence; several positions may run together.\\[2mm]
    Execution $r$ & A feedback update that may precede the next input or block.\\
    \bottomrule
  \end{tabular}
  \gap
  \begin{block}{Teacher forcing supplies tokens, not computed hidden states}
    If block $b$ consumes the memory produced by block $b-1$, that dependency remains during training.
  \end{block}
  \papersource{Paper, Sec. 3, PDF p. 6. Ordinary token-by-token decoding alone is a different notion of sequentiality.}
\end{frame}

\begin{frame}{A feedback arrow needs an execution schedule}
  \begin{columns}[c,onlytextwidth]
    \begin{column}{0.72\textwidth}
      \centering
      \includegraphics[width=\linewidth,height=0.76\textheight,keepaspectratio]{figures/fig-05.pdf}
    \end{column}
    \begin{column}{0.25\textwidth}
      \textbf{(b)} Depth loops
      \gap
      \textbf{(c)} Block updates
      \gap
      \textbf{(d)} Continuing historical updates
      \gap
      {\small Gray fill: frozen computation that can be cached.}
    \end{column}
  \end{columns}
  \papersource{Paper, Fig. 5, PDF p. 6. Colored inputs are distinct tokens; pale empty boxes are inactive positions.}
\end{frame}

\begin{frame}{Fixed depth loops increase a finite compute budget}
  \[
    L_{\mathrm{effective}}=K L
  \]
  \centering
  $K$ repetitions of a group containing $L$ blocks
  \gap
  \begin{itemize}
    \item Shared weights reduce parameter duplication.
    \item More repetitions create a longer computation path.
    \item A fixed $K$ still limits direct dependent updates.
  \end{itemize}
  \begin{block}{The budget distinction matters}
    If $K$ grows with input length, depth scaling can solve additional state-tracking tasks.
  \end{block}
  \papersource{Paper, Sec. 3, PDF pp. 6--7. Dynamic-depth qualification: Merrill and Sabharwal (2025), Theorem 1.}
\end{frame}

\begin{frame}{Block recurrence carries a computed memory forward}
  \[
    (Y_b,m_b)=U(X_b,m_{b-1})
  \]
  \begin{center}
    \begin{tikzpicture}[node distance=16mm,>=Stealth,
      box/.style={draw=AcademicBlue,fill=AcademicBlue!5,align=center,minimum height=13mm,text width=27mm}]
      \node[box] (b1) {$X_1$\\block 1};
      \node[box,right=of b1] (b2) {$X_2$\\block 2};
      \node[box,right=of b2] (b3) {$X_3$\\block 3};
      \draw[->,thick,AcademicTeal] (b1)--node[above] {$m_1$}(b2);
      \draw[->,thick,AcademicTeal] (b2)--node[above] {$m_2$}(b3);
    \end{tikzpicture}
  \end{center}
  \begin{itemize}
    \item Parallel work inside each block; state dependency between blocks.
    \item For block size $B$, approximately $\lceil n/B\rceil$ memory updates.
    \item Larger blocks reduce update frequency but require more local integration.
  \end{itemize}
  \papersource{Teaching notation and schematic. Paper, Fig. 5c, p. 6; Sec. 5.3, pp. 9--10. Update count is not a speedup claim.}
\end{frame}

\begin{frame}{Latent thought can spend multiple updates per input}
  \centering
  \includegraphics[width=0.98\linewidth,height=0.44\textheight,keepaspectratio]{figures/fig-06.pdf}
  \[
    \rho=\frac{\text{new input tokens}}{\text{recurrence updates}}<1
  \]
  \begin{itemize}
    \item Intermediate latent outputs feed back before the next external input.
    \item Persistent tracking depends on which state survives that transition.
  \end{itemize}
  \papersource{Paper, Fig. 6 and Sec. 3, PDF p. 7. The paper notes that not every latent-thought model tracks state.}
\end{frame}

\begin{frame}{Continuing feedback may create attractor dynamics}
  \begin{block}{The schedule in Figure 5d}
    Earlier deep activations feed shallower computation while past positions may continue to update.
  \end{block}
  \begin{itemize}
    \item One new input is introduced per execution step in the illustration.
    \item Historical activations need not all freeze after their first pass.
    \item Convergence depends on the update dynamics and previous states.
  \end{itemize}
  \gap
  \alert{The architecture permits iterative refinement; the paper does not prove convergence.}
  \papersource{Paper, Fig. 5d, PDF p. 6; discussion in Sec. 3, p. 7.}
\end{frame}

\begin{frame}{Within-layer recurrence changes the state path}
  \centering
  \includegraphics[width=0.94\linewidth,height=0.42\textheight,keepaspectratio]{figures/fig-07.pdf}
  \[
    S_t=A_tS_{t-1}+B_t
  \]
  \begin{itemize}
    \item State moves directly across input steps at the same layer.
    \item The transition matrix determines what this recurrent state can do.
  \end{itemize}
  \papersource{Paper, Fig. 7, p. 7; Secs. 4--5.1, pp. 8--9. Equation illustrates an affine state update.}
\end{frame}

\begin{frame}{The taxonomy separates path and update frequency}
  \begingroup\small
  \setlength{\tabcolsep}{4pt}
  \begin{tabular}{>{\raggedright\arraybackslash}p{0.15\linewidth}>{\raggedright\arraybackslash}p{0.27\linewidth}>{\raggedright\arraybackslash}p{0.25\linewidth}>{\raggedright\arraybackslash}p{0.23\linewidth}}
    \toprule
    Axis & $\rho>1$ & $\rho=1$ & $\rho<1$\\
    \midrule
    Depth & Looped / Universal Transformer, RINS & Recirculation & Empty in Table 1\\[3mm]
    Step & Block-Recurrent Transformers & Linear Attention, DeltaNet, Mamba & DeltaProduct, FSRM\\[3mm]
    Depth + Step & Recurrent Memory Transformer & Feedback Transformer & COCONUT, CYB\\
    \bottomrule
  \end{tabular}
  \par\endgroup
  \gap
  \begin{itemize}
    \item $\rho>1$: several input tokens per update.
    \item $\rho<1$: several updates before advancing the input.
  \end{itemize}
  \papersource{Selected entries from Paper, Table 1, PDF p. 8. These are the table's labels, not an expressivity ranking.}
\end{frame}

\begin{frame}{Transformers often find a cheaper computation}
  \begin{block}{Retrieve an old fact}
    A latch task can become a lookup into the original input history.
  \end{block}
  \begin{block}{Compose updates in parallel}
    When the operation permits it, merge partial results in a tree.
  \end{block}
  \begin{block}{Factor the state}
    Independent entity variables can be updated separately and asynchronously.
  \end{block}
  \papersource{Paper, Sec. 4, PDF p. 9. These mechanisms explain why a monolithic state-chain picture is incomplete.}
\end{frame}

\begin{frame}{Parity illustrates a parallel composition shortcut}
  \[
  \begin{array}{c}
    1\quad0\quad1\quad1\quad0\quad0\quad1\quad0\\[2mm]
    \downarrow\quad\text{pairwise XOR}\quad\downarrow\\[2mm]
    1\qquad0\qquad0\qquad1\\[2mm]
    \downarrow\qquad\qquad\downarrow\\[2mm]
    1\qquad1\\[2mm]
    \downarrow\\[1mm]
    0
  \end{array}
  \]
  \centering
  Eight inputs, three combination levels: $\log_2 8=3$.
  \papersource{Teaching example of the construction in Paper, Sec. 4, PDF p. 9. This is a construction, not a universal lower bound.}
\end{frame}

\begin{frame}{Depth bounds depend on the computational model}
  \begin{block}{Dynamic depth}
    Formal constructions with $\Theta(\log n)$ depth recognize regular languages and solve graph connectivity.
  \end{block}
  \begin{itemize}
    \item Fixed-depth separations use standard complexity assumptions.
    \item Allowing depth to grow differs from holding the loop budget fixed.
  \end{itemize}
  \gap
  \textbf{Entity tracking, with a local dependency assumption:}
  \[
    L_{\min}(G)=\left\lceil\log_2\bigl(\operatorname{depth}(G)+1\bigr)\right\rceil
  \]
  \papersource{Merrill and Sabharwal (2025), Secs. 2, 4--5; Fagnou et al. (2024), Thm. 1, PDF p. 4. The main paper's $\log n+1$ paraphrase is inaccurate.}
\end{frame}

\begin{frame}{Negative eigenvalues enable qualitatively new updates}
  \[
    S_t=\underbrace{(I-\beta_t k_tk_t^\top)}_{A_t}S_{t-1}
      +\beta_t k_tv_t^\top
  \]
  \centering
  $S_t\in\mathbb R^{d_k\times d_v}$, $k_t\in\mathbb R^{d_k}$, $v_t\in\mathbb R^{d_v}$
  \gap
  \[
    \|k_t\|_2=1\quad\Longrightarrow\quad A_tk_t=(1-\beta_t)k_t
  \]
  \begin{itemize}
    \item $\beta_t\in[0,1]$: preserve, attenuate, or erase along the key.
    \item $1<\beta_t\leq2$: the key direction can change sign.
    \item Expressivity depends on transition structure as well as recurrence.
  \end{itemize}
  \papersource{Paper, Sec. 5.1, PDF p. 9. Formula: Grazzi et al. (2025), Table 1, PDF p. 3. Eigenvalue calculation is explanatory.}
\end{frame}

\begin{frame}{State dependence can coexist with parallel training}
  \[
    S_t=A_tS_{t-1}+B_t
  \]
  \[
    (A_2,B_2)\star(A_1,B_1)
       =(A_2A_1,\ A_2B_1+B_2)
  \]
  \begin{itemize}
    \item If the coefficients can be computed first, composition is associative.
    \item A prefix scan uses a tree of combinations across the sequence.
    \item General nonlinear updates need not have a compact, cheap composition.
  \end{itemize}
  \begin{block}{Three separate questions}
    Is there a state? What can it express? How is its update executed?
  \end{block}
  \papersource{Teaching derivation clarifying Paper, Sec. 3, p. 7 and Sec. 5.1, p. 9. Scan stages and matrix-operation cost are distinct.}
\end{frame}

\begin{frame}{Three directions change the state computation}
  \begin{enumerate}
    \item \textbf{Enhance transition structure.}\\
      Broaden SSM updates and combine recurrence with attention.
      \gap
    \item \textbf{Improve the representation and objective.}\\
      Encourage useful state estimates and compositional variables.
      \gap
    \item \textbf{Choose a coarser update unit.}\\
      Pass state at chunk or sentence boundaries.
  \end{enumerate}
  \gap
  These are research directions supported by cited work, not a newly evaluated system.
  \papersource{Paper, Secs. 5.1--5.3, PDF pp. 9--10.}
\end{frame}

\begin{frame}{Training must make the feedback path affordable}
  \begin{block}{Exploit representational alignment}
    Residual streams may make cross-layer and cross-step communication easier to introduce.
  \end{block}
  \begin{block}{Introduce recurrence after parallel pretraining}
    The authors propose staged training, truncated gradients, and recurrent backpropagation for attractor dynamics.
  \end{block}
  \gap
  \textbf{Practical tradeoff:} persistent forward state, gradient horizon, and hardware efficiency must be designed together.
  \papersource{Paper, Secs. 5.4--5.5, PDF p. 10. Representational alignment and the staged recipe are proposals here.}
\end{frame}

\begin{frame}{Evaluate usable state under a concrete compute budget}
  \begin{enumerate}
    \item \textbf{Trace the dependency.}\\
      Identify where a computed state reaches the next update.
      \gap
    \item \textbf{Specify the update class and budget.}\\
      Fixed loops, growing depth, affine scans, and arbitrary recurrence differ.
      \gap
    \item \textbf{Measure the tradeoff.}\\
      Compare long-sequence consistency with latency, memory, and training cost.
  \end{enumerate}
  \papersource{Synthesis of Paper, Secs. 2--6. The evaluation recommendation is this discussion's interpretation.}
\end{frame}

\appendix
\begin{frame}{References for the central claims}
  \begingroup\small
  \begin{itemize}
    \item Mozer, Siddiqui, Liu. \emph{The Topological Trouble With Transformers}.\\
      \href{https://arxiv.org/abs/2604.17121v5}{arXiv:2604.17121v5}, 2026. Main paper; all original figures.
      \gap
    \item Merrill, Sabharwal. \emph{A Little Depth Goes a Long Way}.\\
      \href{https://arxiv.org/abs/2503.03961v3}{arXiv:2503.03961v3}, 2025. Dynamic-depth constructions.
      \gap
    \item Fagnou et al. \emph{Chain and Causal Attention for Efficient Entity Tracking}.\\
      \href{https://aclanthology.org/2024.emnlp-main.731/}{EMNLP 2024}. Theorem 1 and its dependency-graph assumption.
      \gap
    \item Merrill, Petty, Sabharwal. \emph{The Illusion of State in State-Space Models}.\\
      \href{https://arxiv.org/abs/2404.08819v3}{arXiv:2404.08819v3}. Scope of SSM expressivity results.
      \gap
    \item Grazzi et al. \emph{Unlocking State-Tracking in Linear RNNs Through Negative Eigenvalues}. \href{https://arxiv.org/abs/2411.12537v5}{arXiv:2411.12537v5}, 2025.
  \end{itemize}
  \endgroup
\end{frame}

\begin{frame}{Backup: the feedback operator remains a design choice}
  \begin{tabular}{p{0.29\linewidth}p{0.64\linewidth}}
    \toprule
    Connection & What it changes\\
    \midrule
    Ordinary KV caching & Reuses past computation at its original layer.\\[3mm]
    Deep-to-shallow attention & Lets shallow queries read deeper keys and values.\\[3mm]
    Linear / MLP / adapter & Transforms and injects a feedback representation.\\[3mm]
    Mutable recurrent state & Updates a persistent representation across events.\\
    \bottomrule
  \end{tabular}
  \gap
  \alert{An arrow does not specify memory capacity, update dynamics, or a training algorithm.}
  \papersource{Paper, Sec. 3, PDF p. 8. Explanatory comparison of the implementation choices listed by the authors.}
\end{frame}

\begin{frame}{Backup: read Table 1 with its definition of the ratio}
  \begin{itemize}
    \item \textbf{Multiple updates per input means $\rho<1$.}\\
      The prose on p. 8 uses $\rho>1$ at one point; the table caption gives the opposite and consistent definition.
      \gap
    \item \textbf{Figure 5c has inconsistent axis labels.}\\
      The p. 7 discussion groups it with depth recurrence; Table 1 places it under Depth + Step.
      \gap
    \item \textbf{A logical recurrence need not ban all parallel algorithms.}\\
      The strict footnote on p. 7 needs qualification for the parallelizable updates discussed in Sec. 5.1.
  \end{itemize}
  \papersource{Paper, Sec. 3, PDF pp. 7--8 and Sec. 5.1, p. 9. Classifications in this deck follow Table 1.}
\end{frame}

\begin{frame}{Backup: the entity-tracking bound has a local assumption}
  \[
    L_{\min}(G)
      =\left\lceil\log_2\bigl(\operatorname{depth}(G)+1\bigr)\right\rceil
  \]
  \begin{itemize}
    \item $G$ is the task's computational dependency graph.
    \item $\operatorname{depth}(G)$ is the length of its longest path.
    \item Each attention layer is assumed to have receptive field one in that graph.
  \end{itemize}
  \begin{block}{The assumption controls the claim}
    This is not an unconditional lower bound on every algorithm that a Transformer could learn for the task.
  \end{block}
  \papersource{Fagnou et al. (2024), Theorem 1, PDF p. 4 (printed p. 13177); assumption discussed in Appendix B.}
\end{frame}

\end{document}