% Research discussion slides, based on the arxiv-paper-reader Beamer template.
% Build locally with: python3 build_slides.py slides.en.tex
\documentclass[aspectratio=169,11pt]{beamer}
\usepackage[T1]{fontenc}
\usepackage[utf8]{inputenc}
\usepackage{amsmath,amssymb,graphicx,booktabs}
\DeclareFontShape{T1}{cmss}{b}{n}{<->ssub*cmss/bx/n}{}
\pdfstringdefDisableCommands{\def\\{ }\def\translate#1{#1}}

\definecolor{AcademicBlue}{RGB}{26,57,86}
\definecolor{AcademicTeal}{RGB}{0,105,112}
\setbeamercolor{structure}{fg=AcademicBlue}
\setbeamercolor{frametitle}{fg=AcademicBlue,bg=white}
\setbeamercolor{title}{fg=AcademicBlue}
\setbeamercolor{alerted text}{fg=AcademicTeal}
\setbeamercolor{block title}{fg=white,bg=AcademicBlue}
\setbeamercolor{block body}{fg=black,bg=AcademicBlue!5}
\setbeamertemplate{navigation symbols}{}
\setbeamertemplate{itemize items}[circle]
\setbeamertemplate{footline}{%
  \hfill\usebeamerfont{page number in head/foot}%
  \insertframenumber\,/\,\inserttotalframenumber\hspace{4mm}\vspace{3mm}}
\setbeamersize{text margin left=8mm,text margin right=8mm}
\setbeamerfont{frametitle}{size=\Large,series=\bfseries}
\setbeamerfont{title}{size=\LARGE,series=\bfseries}
\newcommand{\papersource}[1]{\par\smallskip{\scriptsize\color{gray}#1\par}}
\newcommand{\W}{\mathbf W}
\newcommand{\HH}{\mathbf H}
\newcommand{\Ht}{\widetilde{\mathbf H}}
\newcommand{\RMS}{\operatorname{RMSNorm}}
\newcommand{\softmax}{\operatorname{softmax}}

\title[DiscoLoop]{DiscoLoop}
\subtitle{Looping Discrete Embeddings and Continuous Hidden States\\for Multi-hop Reasoning}
\author{Paper by Hengyu Fu, Tianyu Guo, Zixuan Wang, Hanlin Zhu,\\
Jason D. Lee, Jiantao Jiao, Stuart Russell, and Song Mei}
\date{arXiv:2607.00341v2 (27 July 2026)\\Research discussion \;|\; 10 October 2026}

\begin{document}

\begin{frame}
  \titlepage
  \vspace{-4mm}
  \begin{center}\small
    Shared memory needs an intermediate representation it can reuse.
  \end{center}
\end{frame}

\begin{frame}{Can stored facts compose without an explicit trace?}
  \begin{block}{The controlled task}
    Alice's son is Bob.\qquad Bob's wife is Carol.\\[2mm]
    Query: Who is the wife of Alice's son?\quad Answer: Carol.
  \end{block}
  \[
    (a,r_1,b),\ (b,r_2,c)\quad\Longrightarrow\quad (a,r_1,r_2,c)
  \]
  \begin{itemize}
    \item Facts are learned in weights; the query does not supply them.
    \item Predict $c$ directly; do not generate the bridge entity $b$.
    \item One overall forward pass contains $K$ backbone applications.
  \end{itemize}
  \papersource{Paper, Secs. 2--3.1, PDF pp. 3--4, Eqs. (1)--(2).}
\end{frame}

\begin{frame}{OOD isolates transfer of the composition rule}
  \begin{center}
  \begin{tabular}{lcc}
    \toprule
    & Graph $\mathcal G_A$ & Graph $\mathcal G_B$ \\
    \midrule
    Entities & 500 & 500, disjoint from A \\
    Atomic facts in training & All 5,000 & All 5,000 \\
    Two-hop training queries & 10,000 & None \\
    Two-hop test queries & 2,000 ID & 2,000 OOD \\
    \bottomrule
  \end{tabular}
  \end{center}
  \begin{itemize}
    \item Shared vocabulary of 50 relations; out-degree 10 per entity.
    \item OOD entities and facts have been seen as atomic examples.
    \item Their use in a composition is held out entirely.
  \end{itemize}
  \papersource{Paper, Secs. 2--3.1, PDF pp. 3--4. Each graph contains 50,000 two-hop chains.}
\end{frame}

\begin{frame}{Memory reuse leaves a representation bottleneck}
  \centering
  \includegraphics[width=0.98\linewidth,height=0.68\textheight,keepaspectratio]{figures/fig-01.pdf}
  \par\smallskip
  \raggedright
  Earlier facts must be both accessible and usable by the next loop.
  \papersource{Paper, Fig. 1, PDF p. 3; Sec. 3, PDF pp. 4--6. Original author figure.}
\end{frame}

\begin{frame}{Two loops recall facts but generalize imperfectly}
  \[
    \HH^{(0)}=\W[x],\qquad \HH^{(k+1)}=f_\theta(\HH^{(k)}),\qquad K=2
  \]
  \begin{center}
  \begin{tabular}{lrrrr}
    \toprule
    Accuracy (\%) & Atomic train & 2-hop train & ID test & OOD test \\
    \midrule
    Stage 1 & 100.0 & 8.8 & 0.9 & 0.0 \\
    Stage 2 & 100.0 & 100.0 & 71.1 & 8.3 \\
    \bottomrule
  \end{tabular}
  \end{center}
  \begin{itemize}
    \item Stage 1 reads the LM head after one backbone application.
    \item Stage 2 is the actual prediction after two applications.
    \item Composition uses the second loop, yet OOD remains poor.
  \end{itemize}
  \papersource{Paper, Eq. (2), PDF p. 4; Table 1a, PDF p. 6. After 3,000 training epochs.}
\end{frame}

\begin{frame}{The bridge is decodable but geometrically different}
  \begin{center}
  \begin{tabular}{lcc}
    \toprule
    At position 1, after loop 1 & ID & OOD \\
    \midrule
    Correct bridge probability $P(b\mid\HH^{(1)}_1)$ & 1.000 & 1.000 \\
    Cosine with input embedding $\W[b]$ & 0.327 & 0.266 \\
    \bottomrule
  \end{tabular}
  \end{center}
  \begin{block}{Different questions for the same representation}
    Can the LM head identify $b$?\quad Can $f_\theta$ consume it as $b$?
  \end{block}
  \begin{itemize}
    \item A dominant logit does not require embedding alignment.
    \item Bridge readout is at $r_1$; final-answer readout targets $c$.
  \end{itemize}
  \papersource{Paper, Sec. 3.2, PDF pp. 5--6, Table 1b. Probabilities shown at reported precision.}
\end{frame}

\begin{frame}{A local intervention tests the loop interface}
  \[
    b_{\max}=\arg\max_v(\W\HH^{(1)}_1)_v
  \]
  \[
    \HH^{(1)}_1\leftarrow
    (1-\alpha)\HH^{(1)}_1+\alpha\,\operatorname{Norm}(\W[b_{\max}])
  \]
  \begin{itemize}
    \item Modify position 1 only, between loops 1 and 2.
    \item Use the model's decoded token; no oracle bridge is supplied.
    \item No weight update is needed.
  \end{itemize}
  \begin{block}{Interpretation}
    Better composition after changing the interface supports a\newline
    representation bottleneck beyond memory availability.
  \end{block}
  \papersource{Paper, Sec. 3.2, PDF p. 5, Eq. (3).}
\end{frame}

\begin{frame}{Embedding injection nearly closes the OOD gap}
  \centering
  \includegraphics[width=0.98\linewidth,height=0.66\textheight,keepaspectratio]{figures/fig-02.pdf}
  \par\smallskip\raggedright
  OOD rises from 8.3\% to about 99\% near $\alpha=0.7$--$0.8$.
  \papersource{Paper, Fig. 2, PDF p. 5; approximate reading of the plotted curve. The prose reports different intervention strengths; see backup.}
\end{frame}

\begin{frame}{Decode softly, then return to embedding space}
  \[
    p_v(h)=\frac{\exp((\W h)_v/\tau)}{\sum_{v'}\exp((\W h)_{v'}/\tau)},
    \qquad \Phi(h)=\sum_{v=1}^{V}p_v(h)\W[v]
  \]
  \begin{center}
    Hidden state $h$ $\longrightarrow$ vocabulary distribution $p$
    $\longrightarrow$ soft embedding $\Phi(h)$
  \end{center}
  \begin{itemize}
    \item $\W\in\mathbb R^{V\times d}$ is tied across input and output.
    \item If $p_b\approx1$, the returned vector approaches $\W[b]$.
    \item The ``discrete'' channel uses token embeddings but stays soft.
    \item Temperature $\tau>0$ controls distribution sharpness.
  \end{itemize}
  \papersource{Paper, Sec. 4, PDF p. 6, Eq. (5).}
\end{frame}

\begin{frame}{DiscoLoop adds the soft embedding to the residual}
  \[
    \Ht^{(0)}=\W[x],\qquad \HH^{(k+1)}=f_\theta(\Ht^{(k)})
  \]
  \[
    \boxed{\Ht^{(k+1)}=\HH^{(k+1)}+
    \alpha^{(k)}\odot\RMS\!\left(\Phi(\HH^{(k+1)})\right)}
  \]
  \begin{itemize}
    \item Preserve the continuous state and add an embedding channel.
    \item Both channels occupy the same $T\times d$ residual space.
    \item Apply at every position; no bridge location is hard-coded.
    \item Set $\alpha^{(K-1)}=0$; read final logits from $\HH^{(K)}\W^\top$.
  \end{itemize}
  \papersource{Paper, Sec. 4, PDF pp. 6--7, Eq. (4). There are $K-1$ loop-to-loop injections.}
\end{frame}

\begin{frame}{A shared gate controls the injection strength}
  \[
    \alpha_t^{(k)}=\sigma\!\left(
      \left\langle\mathbf w_\alpha,\Phi(\HH_t^{(k+1)})\right\rangle+b_\alpha
    \right)
  \]
  \begin{itemize}
    \item Gate input: the decoded embedding before RMSNorm.
    \item One vector and bias, shared over tokens and loops: $d+1$ parameters.
    \item Symbolic two-hop: fixed $\alpha^\star=1$, temperature $\tau=1$.
    \item Synthetic language: a learned token-wise gate.
  \end{itemize}
  \begin{block}{Two distinct mixing rules}
    Diagnostic intervention: $(1-\alpha)h+\alpha e$.\\
    Learned architecture: $h+\alpha e$.
  \end{block}
  \papersource{Paper, Eqs. (3)--(6), PDF pp. 5--7; Sec. 5.2, PDF p. 8.}
\end{frame}

\begin{frame}{Symbolic generalization improves as shortcuts fade}
  \centering
  \includegraphics[width=0.98\linewidth,height=0.65\textheight,keepaspectratio]{figures/fig-03.pdf}
  \par\smallskip\raggedright
  Around epoch 500: ID/OOD exceed 80\%; Stage-1 answers collapse.
  \papersource{Paper, Sec. 5.1, PDF pp. 7--8, Fig. 3. $K=2$, 3,000 epochs. Right: Stage-1 training accuracy; non-looped curve uses its final training readout.}
\end{frame}

\begin{frame}{The effect survives two English question formats}
  \centering
  {\small Direct: Alice's son's wife.\quad Reverse: The wife of Alice's son.\par}
  \smallskip
  \includegraphics[width=0.98\linewidth,height=0.57\textheight,keepaspectratio]{figures/fig-04.pdf}
  \par\smallskip\raggedright
  Direct and reverse formats: nearly 100\% ID, about 95\% OOD.
  \papersource{Paper, Sec. 5.2, PDF pp. 8--9, Fig. 4. $K=2$, learned gate, 5,000 epochs. Solid = ID, dashed = OOD. Phrasing above is a teaching example.}
\end{frame}

\begin{frame}{Three-hop OOD improves to about 65\%}
  \centering
  \includegraphics[width=0.98\linewidth,height=0.60\textheight,keepaspectratio]{figures/fig-05.pdf}
  \par\smallskip\raggedright
  Both baselines stay near zero on three-hop OOD.
  \papersource{Paper, App. A.2, PDF pp. 13--14, Fig. 5. $K=3$; training contains both two-hop and three-hop queries. This evaluates transfer across graphs at trained depths.}
\end{frame}

\begin{frame}{Pretraining compares three recurrent signals}
  \begin{center}
  \begin{tabular}{ll}
    \toprule
    Model & Information passed across loops \\
    \midrule
    Vanilla loop & Continuous hidden states \\
    PonderLM & Decoded embedding-space states \\
    DiscoLoop & Continuous states + decoded embeddings \\
    \bottomrule
  \end{tabular}
  \end{center}
  \begin{itemize}
    \item 440M parameters; 24-layer backbone, $d=1024$, 16 heads.
    \item Four backbone applications for every recurrent model.
    \item 20B tokens: 60\% FineWeb-Edu, 40\% FineMath-4plus.
    \item Context 8,192; tied embeddings. DiscoLoop uses top-128 return.
  \end{itemize}
  \papersource{Paper, Sec. 5.3 and App. B, PDF pp. 9, 14--15. PonderLM: three ponder steps plus the final backbone application.}
\end{frame}

\begin{frame}{Zero-shot average improves by 1.2 points}
  \centering
  \setlength{\tabcolsep}{9pt}
  \begin{tabular}{lrrr}
    \toprule
    Score (\%) & Vanilla loop & PonderLM & DiscoLoop \\
    \midrule
    ARC-C & 30.5 & 31.7 & \textbf{32.6} \\
    ARC-E & 57.2 & 55.6 & \textbf{57.6} \\
    HellaSwag & \textbf{44.2} & 43.9 & \textbf{44.2} \\
    LAMBADA & 35.1 & \textbf{38.0} & 37.6 \\
    PIQA & 67.4 & 67.7 & \textbf{68.4} \\
    RACE & 31.0 & 31.2 & \textbf{31.3} \\
    SciQ & 79.5 & 80.5 & \textbf{81.5} \\
    \midrule
    Average & 49.3 & 49.8 & \textbf{50.5} \\
    \bottomrule
  \end{tabular}
  \papersource{Paper, Table 2, PDF p. 9. Zero-shot acc\_norm for ARC-C/E, HellaSwag, PIQA, SciQ; acc for LAMBADA and RACE. DiscoLoop is best or tied on six of seven tasks.}
\end{frame}

\begin{frame}{The loss advantage emerges late in pretraining}
  \centering
  \includegraphics[width=0.98\linewidth,height=0.65\textheight,keepaspectratio]{figures/fig-07-zoom.pdf}
  \par\smallskip\raggedright
  Global average loss: DiscoLoop overtakes Vanilla near 13B tokens.
  \papersource{Paper, Fig. 7 lower panel, PDF p. 16; cropped from the author figure. Loss is EMA-smoothed; lower is better. Peak late-stage separation is approximately $10^{-2}$.}
\end{frame}

\begin{frame}{Few new parameters still add vocabulary computation}
  \[
    \mathbf P=\softmax(\HH\W^\top/\tau),\qquad
    \Phi(\HH)=\mathbf P\W
  \]
  \begin{itemize}
    \item Full decode and return each involve $O(TVd)$ arithmetic.
    \item Top-128 reduces selected embedding aggregation to $O(Tkd)$.
    \item Finding the top tokens still requires vocabulary scoring.
    \item Faster learning in epochs does not establish faster wall time.
  \end{itemize}
  \begin{block}{Comparison to request next}
    Report end-to-end throughput, memory, and equal-total-compute quality.
  \end{block}
  \papersource{Complexity interpretation of Eq. (5), PDF p. 6, and App. B, PDF pp. 14--15. The paper does not report end-to-end overhead for the added channel.}
\end{frame}

\begin{frame}{The next tests are depth and scale}
  \begin{itemize}
    \item \textbf{Depth:} train up to $k$ hops, test beyond $k$.\\
      The existing three-hop study includes three-hop training data.
    \item \textbf{Scale:} move beyond 440M parameters and 20B tokens;\\
      test adaptation of existing pretrained models.
    \item \textbf{Mechanism:} probe the loop interface in pretrained models;\\
      broad benchmark gains alone do not localize their cause.
    \item \textbf{Reproducibility:} clarify synthetic configurations and\\
      pretraining gate, temperature, and top-k normalization.
  \end{itemize}
  \papersource{Paper, Sec. 6, PDF pp. 9--10; Apps. A--B, PDF pp. 13--15. Mechanistic follow-up and configuration observations are reading analysis.}
\end{frame}

\begin{frame}{A useful loop must pass a useful representation}
  \begin{enumerate}
    \item \textbf{Diagnosis:} shared memory alone is insufficient;\\
      the bridge is decodable but poorly aligned with its embedding.
    \item \textbf{Design:} retain hidden states and inject soft decoded\\
      embeddings between backbone applications.
    \item \textbf{Evidence:} large controlled-task gains; smaller positive\\
      pretraining gains under the reported 440M / 20B-token setting.
  \end{enumerate}
  \begin{block}{Discussion question}
    How much of the gain requires semantic embedding alignment,\\
    compared with an equally costly learned residual transformation?
  \end{block}
  \papersource{Synthesis of Secs. 3--5 and App. A.2. Discussion question is proposed analysis.}
\end{frame}

\begin{frame}{Primary reference}
  Hengyu Fu, Tianyu Guo, Zixuan Wang, Hanlin Zhu,\\
  Jason D. Lee, Jiantao Jiao, Stuart Russell, and Song Mei.
  \bigskip

  \textbf{DiscoLoop: Looping Discrete Embeddings and Continuous\\
  Hidden States for Multi-hop Reasoning.}
  \medskip

  arXiv:2607.00341v2, revised 27 July 2026.\\[2mm]
  \href{https://arxiv.org/abs/2607.00341v2}{https://arxiv.org/abs/2607.00341v2}
  \bigskip

  All empirical figures and benchmark values in this deck come from\\
  this version. Figures retain the original data and legends.
  \papersource{Research discussion slides; not an author presentation. Backup slides follow.}
\end{frame}

\appendix

\begin{frame}{Backup: intervention strengths differ in text and figure}
  \begin{center}
  \begin{tabular}{lcc}
    \toprule
    OOD accuracy & Sec. 3.2 prose & Fig. 2, approximate \\
    \midrule
    $\alpha=0.1$ & 25.9\% & 15\% \\
    $\alpha=0.2$ & --- & 26\% \\
    $\alpha=0.5$ & Nearly 100\% & 87\% \\
    $\alpha\approx0.7$--$0.8$ & --- & 99\% \\
    \bottomrule
  \end{tabular}
  \end{center}
  \begin{itemize}
    \item Both support the large benefit of embedding injection.
    \item The plotted high-accuracy region is used in this presentation.
    \item Exact strength selection needs clarification of the discrepancy.
  \end{itemize}
  \papersource{Paper, Sec. 3.2 and Fig. 2, PDF p. 5. Figure values are visual estimates, not raw measurements.}
\end{frame}

\begin{frame}{Backup: language data show the same Stage-1 transition}
  \centering
  \includegraphics[width=0.98\linewidth,height=0.63\textheight,keepaspectratio]{figures/fig-06.pdf}
  \par\smallskip\raggedright
  Single-loop answer accuracy falls earlier with DiscoLoop.
  \papersource{Paper, App. A.3, PDF p. 14, Fig. 6. Roughly 400 epochs for DiscoLoop; much later for Vanilla. Non-looped model uses its final training readout.}
\end{frame}

\begin{frame}{Backup: the pretraining recipe}
  \begin{center}
  \begin{tabular}{ll}
    \toprule
    Component & Reported setting \\
    \midrule
    Tokenizer / vocabulary & DeepSeek-V3-0324 / 129,280 \\
    Positional encoding & RoPE base $10^6$; max positions 16,384 \\
    Activations / QKV bias & SwiGLU / no QKV bias \\
    Optimizers & AdamW and Muon enabled \\
    Learning rate & $3\times10^{-3}$ to $10^{-5}$, linear decay \\
    Warmup / clipping & 1,000 steps / gradient norm 1.0 \\
    AdamW $\epsilon$ / Muon momentum & $10^{-15}$ / 0.95 \\
    Global batch / sequence length & 128 / 8,192 \\
    Loop launcher budget & 19,074 steps; 20,000,538,624 tokens \\
    \bottomrule
  \end{tabular}
  \end{center}
  \papersource{Paper, App. B, PDF p. 15. These are language-model settings, not the missing synthetic-task hyperparameters.}
\end{frame}

\begin{frame}{Backup: top-k normalization needs clarification}
  \[
    \Phi_k(h)=\sum_{v\in\mathcal T_k(h)}p_v(h)\W[v],\qquad k=128
  \]
  \begin{itemize}
    \item The formula retains probabilities from the full distribution.
    \item The prose describes masking logits outside the top-k set.
    \item Renormalizing on that support changes the vector's scale.
    \item RMSNorm largely removes scale; a gate reading raw $\Phi$\\
      can still respond differently.
  \end{itemize}
  \papersource{Paper, App. B, PDF pp. 14--15; gate in Eq. (6), PDF p. 6. The scale/gate implication is an analytical observation.}
\end{frame}

\end{document}