\documentclass[aspectratio=169,11pt]{beamer}
\usepackage[T1]{fontenc}
\usepackage[utf8]{inputenc}
\usepackage{lmodern}
\usepackage{amsmath,amssymb,graphicx,booktabs,array}

\definecolor{AcademicBlue}{RGB}{26,57,86}
\definecolor{AcademicTeal}{RGB}{0,105,112}
\definecolor{AcademicOrange}{RGB}{171,85,0}
\setbeamercolor{structure}{fg=AcademicBlue}
\setbeamercolor{frametitle}{fg=AcademicBlue,bg=white}
\setbeamercolor{title}{fg=AcademicBlue}
\setbeamercolor{alerted text}{fg=AcademicOrange}
\setbeamercolor{block title}{fg=white,bg=AcademicBlue}
\setbeamercolor{block body}{fg=black,bg=AcademicBlue!5}
\setbeamertemplate{navigation symbols}{}
\setbeamertemplate{itemize items}[circle]
\setbeamertemplate{footline}{%
  \hfill\usebeamerfont{page number in head/foot}%
  \insertframenumber\,/\,\inserttotalframenumber\hspace{4mm}\vspace{3mm}}
\setbeamersize{text margin left=8mm,text margin right=8mm}
\setbeamerfont{frametitle}{size=\Large,series=\bfseries}
\setbeamerfont{title}{size=\LARGE,series=\bfseries}
\newcommand{\papersource}[1]{\par\smallskip{\scriptsize\color{gray}#1\par}}

\title[StateLoop]{StateLoop: Persistent State}
\subtitle{for Looped Transformers}
\author[Feichen Song et al.]{Paper by Feichen Song et al.\\Research group paper discussion}
\date{\href{https://arxiv.org/abs/2610.35544v1}{arXiv:2610.35544v1} \quad 28 October 2026}

\begin{document}

\begin{frame}
  \titlepage
  \vspace{-4mm}
  \begin{center}
    \small \href{https://arxiv.org/abs/2610.35544}{Publication not found as of 2026-10-28}\\
    Public code not found as of 2026-10-28
  \end{center}
\end{frame}

\begin{frame}{The problem: one hidden stream must compute and remember}
  \begin{itemize}
    \item A looped Transformer reuses one backbone \(T\) across \(K\) iterations.
    \item Vanilla feedback sends each backbone output directly into the next iteration.
    \item Useful early information may need to survive several later transformations.
  \end{itemize}
  \begin{block}{StateLoop's design}
    Keep the direct backbone stream and add one gated persistent state per token,
    carried across the \emph{loop-depth} axis.
  \end{block}
  \papersource{Paper, Secs.\ 1--2, PDF pp.\ 1--2. Persistence here is across internal loop iterations.}
\end{frame}

\begin{frame}{Each token has a direct path and a state path}
  \begin{columns}[T]
    \begin{column}{0.47\linewidth}
      \begin{block}{Direct backbone stream}
        \centering
        \vspace{2mm}
        \(H^k\ \longrightarrow\ Z^{k+1}\ \xrightarrow{\ T\ }\ H^{k+1}\)
        \vspace{2mm}
      \end{block}
      The current backbone output remains available to the next iteration.
    \end{column}
    \begin{column}{0.47\linewidth}
      \begin{block}{Persistent state stream}
        \centering
        \vspace{2mm}
        \(S^k\ \xrightarrow[\ H^k\ ]{\ \text{gated update}\ }\ S^{k+1}\)
        \vspace{2mm}
      \end{block}
      Each token selectively carries state across loop iterations.
    \end{column}
  \end{columns}
  \vspace{5mm}
  \begin{alertblock}{The streams meet before the next loop}
    \centering
    \(Z^{k+1}=H^k+\operatorname{Read}(S^{k+1})\)
  \end{alertblock}
  \papersource{Paper, Fig.\ 1 and Sec.\ 3.1, PDF p.\ 3. Backbone and state-update parameters are shared across iterations.}
\end{frame}

\begin{frame}{The gate chooses how much old state to retain}
  \[
    U^k=\operatorname{RMSNorm}_h(H^k),\qquad
    P^k=U^kW_u^\top
  \]
  \[
    \Delta^k=\operatorname{softplus}(U^kW_\Delta^\top+b_\Delta),\qquad
    g^k=\exp[-\exp(A_{\log})\odot\Delta^k]
  \]
  \[
    \boxed{\widetilde S^{k+1}
    =g^k\odot S^k+(1-g^k)\odot P^k}
  \]
  \begin{itemize}
    \item \(g^k\approx1\): retain; \(g^k\approx0\): write from this backbone output.
    \item Each token and state channel can have a different gate.
  \end{itemize}
  \papersource{Paper, Sec.\ 3.2.1, PDF p.\ 4, Eqs.\ (5)--(6).}
\end{frame}

\begin{frame}{The state is read without replacing the backbone stream}
  \[
    S^{k+1}=\mathcal X_\psi(\widetilde S^{k+1}),\qquad
    N^{k+1}=\operatorname{RMSNorm}_s(S^{k+1})
  \]
  \[
    \boxed{Z^{k+1}=H^k+N^{k+1}W_{s\to z}^{\top}},\qquad
    H^{k+1}=T(Z^{k+1})
  \]
  \[
    Y=H^{K-1}+N^KW_{s\to out}^{\top}
  \]
  \begin{itemize}
    \item One projection feeds the next loop; a separate projection feeds final prediction.
    \item Standard next-token cross-entropy trains the full model end-to-end.
  \end{itemize}
  \papersource{Paper, Secs.\ 3.1 and 3.3, PDF pp.\ 3--5, Eqs.\ (3)--(4), (10)--(12).}
\end{frame}

\begin{frame}{State update and token mixing act on different axes}
  \begin{block}{Across loop depth}
    Every variant uses the same selective gate to update each token's state.
  \end{block}
  \vspace{2mm}
  \centering
  \begin{tabular}{lp{5.5cm}}
    \toprule
    Variant & Mixing across tokens\\
    \midrule
    StateLoop & Identity\\
    + Attn & Causal attention\\
    + Mamba-2 & Residual Mamba-2 block\\
    \bottomrule
  \end{tabular}
  \vspace{4mm}
  \begin{block}{Autoregressive decoding}
    The identity variant keeps a transient state for the current token across its
    \(K\) iterations. The token-mixing variants also need loop-specific caches.
  \end{block}
  \papersource{Paper, Secs.\ 3.2.2 and 4.4, PDF pp.\ 4 and 7--8.}
\end{frame}

\begin{frame}{The comparison fixes data and backbone at 1.4B}
  \begin{itemize}
    \item Pile pretraining from scratch: 30B tokens, 2,048-token sequences, batch 256.
    \item Shared 24-layer LLaMA-style backbone; width \(d=2048\), \(K=4\).
    \item StateLoop state width \(d_s=2048\); three mixer variants.
    \item Controls: Vanilla Looped, wider FFN, and alternative loop connections.
  \end{itemize}
  \begin{block}{Metrics}
    Pile and LAMBADA token PPL; WikiText-2 word PPL; eight-task zero-shot
    and five-shot accuracy; decoding latency in ms/token.
  \end{block}
  \papersource{Paper, Sec.\ 4.1, PDF p.\ 5. MeSH and Parcae match 96 layer applications but retain their own structures.}
\end{frame}

\begin{frame}{Persistent state lowers language-modeling perplexity}
  \centering
  \begin{tabular}{lrrrr}
    \toprule
    1.4B model & Params (M) & Pile & LAMBADA & WikiText-2\\
    \midrule
    Vanilla Looped & 1420.92 & 7.79 & 6.94 & 16.50\\
    Vanilla + FFN & 1467.22 & 7.75 & 7.00 & 16.34\\
    MeSH & 1421.51 & 7.74 & 6.77 & 16.43\\
    \midrule
    StateLoop & 1437.71 & 7.61 & 6.71 & 15.96\\
    StateLoop + Attn & 1454.49 & \textbf{7.60} & \textbf{6.64} & \textbf{15.93}\\
    StateLoop + Mamba-2 & 1467.23 & \textbf{7.60} & 6.88 & 15.95\\
    \bottomrule
  \end{tabular}
  \vspace{4mm}
  \begin{itemize}
    \item Identity StateLoop adds 1.18\% parameters and reduces Pile PPL by 0.18.
    \item A parameter-matched wider FFN does not reproduce the gain.
  \end{itemize}
  \papersource{Paper, Table 1, PDF p.\ 6. Lower PPL is better; WikiText-2 is word PPL.}
\end{frame}

\begin{frame}{Eight-task average accuracy rises under both prompt settings}
  \centering
  \begin{tabular}{lrr}
    \toprule
    1.4B model & Zero-shot Avg-8 (\%) & Five-shot Avg-8 (\%)\\
    \midrule
    Vanilla Looped & 53.31 & 55.12\\
    Vanilla + FFN & 52.85 & 55.54\\
    StateLoop & 54.28 & 56.60\\
    StateLoop + Attn & 54.74 & \textbf{57.15}\\
    StateLoop + Mamba-2 & \textbf{54.75} & 57.13\\
    \bottomrule
  \end{tabular}
  \vspace{5mm}
  \begin{itemize}
    \item Identity StateLoop: \(+0.97\) zero-shot and \(+1.48\) five-shot percentage points.
    \item Average gains do not mean every task improves; zero-shot ARC-Easy dips slightly.
  \end{itemize}
  \papersource{Paper, Table 2, PDF p.\ 7. Avg-8 is the unweighted mean of eight task accuracies.}
\end{frame}

\begin{frame}{The StateLoop advantage grows with trained loop depth}
  \centering
  \includegraphics[width=0.85\linewidth,height=0.58\textheight,keepaspectratio]{figures/recurrent_depth_ppl.pdf}
  \begin{itemize}
    \item Pile PPL gap: 0.14 at \(K=3\), 0.30 at \(K=7\).
    \item Every plotted \(K\) is a separately trained 410M, 25B-token pair.
  \end{itemize}
  \papersource{Paper, Sec.\ 4.3 and Fig.\ 2, PDF pp.\ 6--7. Both methods improve with more loops.}
\end{frame}

\begin{frame}{The state branch costs less than one more backbone loop}
  \centering
  \begin{tabular}{lrrr}
    \toprule
    1.4B decoding & \(K\) & ms/token & Change vs.\ Vanilla \(K=4\)\\
    \midrule
    Vanilla Looped & 4 & 121.224 & baseline\\
    StateLoop & 4 & 122.943 & \(+1.418\%\)\\
    StateLoop + Attn & 4 & 123.778 & \(+2.106\%\)\\
    StateLoop + Mamba-2 & 4 & 123.397 & \(+1.792\%\)\\
    Vanilla Looped & 5 & 151.527 & \(+24.997\%\)\\
    \bottomrule
  \end{tabular}
  \vspace{4mm}
  \begin{itemize}
    \item The comparison measures state-branch cost against a fifth full iteration.
    \item BF16, batch 1, 1,024-token prefill + 1,024-token continuation.
  \end{itemize}
  \papersource{Paper, Table 3, PDF p.\ 8. Means over five runs on one GPU.}
\end{frame}

\begin{frame}{Even a narrow state preserves much of the gain}
  \centering
  \includegraphics[width=0.93\linewidth,height=0.60\textheight,keepaspectratio]{figures/state_width_ppl.pdf}
  \begin{itemize}
    \item Vanilla: PPL 9.34; \(d_s=128\): 9.16; \(d_s=1024\): 9.04.
    \item Halving state width from 1,024 to 512 yields PPL 9.08.
  \end{itemize}
  \papersource{Paper, Sec.\ 5.1 and Fig.\ 3, PDF p.\ 8. 410M, \(K=7\), 25B tokens, identity mixer.}
\end{frame}

\begin{frame}{Ablations separate the branch, persistence, and update rule}
  \centering
  \begin{tabular}{llrr}
    \toprule
    Intervention & Variant & Params (M) & Pile PPL\\
    \midrule
    Feedback & State-only next input & 409.00 & 9.44\\
     & StateLoop direct path & 411.10 & \textbf{9.04}\\
    \midrule
    Persistence & Reset state each loop & 411.10 & 9.25\\
     & Persist across loops & 411.10 & \textbf{9.04}\\
    \midrule
    Update & Vanilla Looped & 406.90 & 9.34\\
     & StateLoop with GRU & 416.34 & 9.09\\
     & StateLoop selective SSM & 411.10 & \textbf{9.04}\\
    \bottomrule
  \end{tabular}
  \vspace{3mm}
  \begin{itemize}
    \item The branch helps without persistence; persistence adds a further 0.21 PPL gain.
    \item GRU retains most of the gain: the result is not unique to one gate formula.
  \end{itemize}
  \papersource{Paper, Sec.\ 5.2 and Table 4, PDF p.\ 9. 410M, \(K=7\), 25B tokens.}
\end{frame}

\begin{frame}{Late state reads receive most attribution}
  \begin{columns}[T]
    \column{0.58\linewidth}
    \centering
    \includegraphics[width=\linewidth,height=0.65\textheight,keepaspectratio]{figures/state_reads.pdf}
    \column{0.40\linewidth}
    \begin{itemize}
      \item Mask all \(2^7\) combinations of state reads.
      \item Feedback reads: 63.70\% of attributed NLL increase.
      \item Terminal read: 36.30\%.
      \item \(R_5\), \(R_6\), \(R_7\) dominate.
    \end{itemize}
  \end{columns}
  \papersource{Paper, Sec.\ 5.3 and Fig.\ 4(a), PDF p.\ 10. Shapley shares refer to the read-masking intervention.}
\end{frame}

\begin{frame}{Earlier states can be reused once; freezing hurts}
  \begin{columns}[T]
    \column{0.58\linewidth}
    \centering
    \includegraphics[width=\linewidth,height=0.65\textheight,keepaspectratio]{figures/state_reuse.pdf}
    \column{0.40\linewidth}
    \begin{itemize}
      \item Calibrate an earlier state's channel mean and scale.
      \item Replace \(S^4\) once using \(S^3\): PPL 9.08.
      \item Freeze reuse from \(S^3\): PPL 18.41.
      \item Unmodified model: 9.04.
    \end{itemize}
  \end{columns}
  \papersource{Paper, Sec.\ 5.3 and Fig.\ 4(b), PDF p.\ 10. Backbone computation continues in both interventions.}
\end{frame}

\begin{frame}{The evidence supports a specific design claim}
  \begin{enumerate}
    \item A direct backbone path plus a gated extra state beats state-only feedback.
    \item Resetting the state loses part of the gain at equal parameter count.
    \item Later reads and continued updates contribute materially to prediction.
  \end{enumerate}
  \begin{block}{Scope of the reported evidence}
    Main 1.4B comparisons use \(K=4\); systematic depth and state interventions
    use 410M models. Decoding cost is measured under a specified batch-1 protocol.
  \end{block}
  \papersource{Paper, Tables 1--4 and Figs.\ 2--4, PDF pp.\ 6--10; Sec.\ 6, p.\ 11.}
\end{frame}

\begin{frame}{Take-home: explicit state helps recurrent depth}
  \begin{itemize}
    \item StateLoop retains a small, gated state per token across loop iterations.
    \item It improves Pile PPL and eight-task average accuracy with modest added latency.
    \item Its PPL advantage grows with trained depth in the 410M study.
    \item The next test is the quality--latency tradeoff at larger depths and scales.
  \end{itemize}
  \vspace{3mm}
  {\small\textbf{Paper:} Feichen Song, Boyi Zeng, He Li, Shixiang Song, Yiqin Hao,
  Zitong Wang, Yizhou Yang, Yifan Liu, Ziwei He, Zhouhan Lin.
  \emph{StateLoop: Persistent State for Looped Transformers}.
  \href{https://arxiv.org/abs/2610.35544v1}{arXiv:2610.35544v1}.\\
  \href{https://arxiv.org/abs/2610.35544}{Publication not found as of 2026-10-28};
  public code not found as of 2026-10-28.}
\end{frame}

\appendix

\begin{frame}{Backup: initialization and next-token training}
  \[
    E=\sqrt d\,\mathrm{Embed}(x),\qquad S^0=0,\qquad H^0=T(E)
  \]
  \[
    p_\theta(\cdot\mid x_{\le t})
    =\mathrm{softmax}(\mathrm{LMHead}(Y_t))
  \]
  \[
    \mathcal L_{\rm LM}
    =-\frac1{n-1}\sum_{t=1}^{n-1}
    \log p_\theta(x_{t+1}\mid x_{\le t})
  \]
  \begin{itemize}
    \item \(H^k\in\mathbb R^{n\times d}\); \(S^k\in\mathbb R^{n\times d_s}\).
    \item The whole network is trained end-to-end with ordinary causal LM loss.
  \end{itemize}
  \papersource{Paper, Secs.\ 3.1 and 3.3, PDF pp.\ 3--5, Eqs.\ (1), (11)--(12).}
\end{frame}

\begin{frame}{Backup: benchmark interpretation}
  \begin{itemize}
    \item Eight-task Avg-8 is an unweighted mean of accuracy percentages.
    \item Pile and LAMBADA PPL are token-level; WikiText-2 PPL is word-level.
    \item MeSH and Parcae match the 96 backbone layer applications but keep their original partial-loop designs.
    \item StateLoop + Attn and + Mamba-2 add token-axis mixing and loop-specific caches.
  \end{itemize}
  \papersource{Paper, Secs.\ 4.1 and 4.4, PDF pp.\ 5 and 7--8.}
\end{frame}

\end{document}