% Self-authored reading companion. Compile with pdflatex; shell escape disabled.
% Suggested timing: slides 2--8, 9 minutes; 9--14, 10 minutes; 15--17, 4 minutes.
\documentclass[aspectratio=169,11pt]{beamer}
\usepackage[T1]{fontenc}
\usepackage[utf8]{inputenc}
\usepackage{lmodern}
\usepackage{amsmath,amssymb,graphicx,booktabs}

\definecolor{AcademicBlue}{RGB}{26,57,86}
\definecolor{AcademicTeal}{RGB}{0,105,112}
\setbeamercolor{structure}{fg=AcademicBlue}
\setbeamercolor{frametitle}{fg=AcademicBlue,bg=white}
\setbeamercolor{title}{fg=AcademicBlue}
\setbeamercolor{alerted text}{fg=AcademicTeal}
\setbeamercolor{block title}{fg=white,bg=AcademicBlue}
\setbeamercolor{block body}{fg=black,bg=AcademicBlue!5}
\setbeamertemplate{navigation symbols}{}
\setbeamertemplate{itemize items}[circle]
\setbeamertemplate{footline}{%
  \hfill\usebeamerfont{page number in head/foot}%
  \insertframenumber\,/\,\inserttotalframenumber\hspace{4mm}\vspace{3mm}}
\setbeamersize{text margin left=8mm,text margin right=8mm}
\setbeamerfont{frametitle}{size=\Large,series=\bfseries}
\setbeamerfont{title}{size=\LARGE,series=\bfseries}
\newcommand{\papersource}[1]{\par\smallskip{\scriptsize\color{gray}#1\par}}
\newcommand{\Attn}{\mathrm{Attn}}
\newcommand{\LSTM}{\mathrm{LSTM}}
\newcommand{\DAT}{\textsc{DAT}}
\newcommand{\gain}[1]{{\color{AcademicTeal}\textbf{#1}}}
\pdfstringdefDisableCommands{\def\translate#1{#1}}

\title[Delayed Attention Training]{Delayed Attention Training Improves Length Generalization in Transformer--RNN Hybrids}
\subtitle{A research paper discussion}
\author{\texorpdfstring{\small Paper authors: Buu Phan, Reza Ebrahimi,\\ Sanjay Haresh, and Roland Memisevic}{Paper authors: Buu Phan, Reza Ebrahimi, Sanjay Haresh, Roland Memisevic}}
\date{\small arXiv:2510.00258v1 \quad | \quad Reading: 13 October 2026}

\begin{document}

\begin{frame}
  \titlepage
  \vspace{-5mm}
  \begin{center}
    \color{AcademicTeal} Train the recurrent path first; then enable attention.
  \end{center}
\end{frame}

\begin{frame}{Can one model preserve two complementary skills?}
  \begin{columns}[T,onlytextwidth]
    \begin{column}{0.47\textwidth}
      \begin{block}{State tracking}
        Update a running state after every new value.
      \end{block}
      \medskip
      Modulo addition has a compact recursive solution.
      \medskip

      \textbf{LSTM bias:} process the sequence step by step.
    \end{column}
    \begin{column}{0.47\textwidth}
      \begin{block}{Associative recall}
        Retrieve a past value when its key is queried.
      \end{block}
      \medskip
      Longer sequences contain more associations to distinguish.
      \medskip

      \textbf{Attention bias:} access past positions by content.
    \end{column}
  \end{columns}
  \bigskip
  \alert{The question: does joint training produce this division of labor?}
  \papersource{Paper, Sec. 1, PDF p. 1.}
\end{frame}

\begin{frame}{Each example requires both answers to be correct}
  \[
    \langle\mathrm{bos}\rangle\; k_1v_1\cdots k_nv_n\;
    \langle\mathrm{modulo}\rangle\;m\;
    \langle\mathrm{recall}\rangle\; k_jv_j
  \]
  \begin{block}{Teaching example}
    Prefix: \texttt{A 7\quad B 8\quad C 4}\qquad Query key: \texttt{B}
    \[
      m=(7+8+4)\bmod 10=\gain{9},\qquad v_j=\gain{8}.
    \]
  \end{block}
  \[
    \mathrm{Acc}_{\mathrm{joint}}=
      \frac{1}{N_{\mathrm{eval}}}\sum_{r=1}^{N_{\mathrm{eval}}}
      \mathbf{1}[\hat m_r=m_r\ \land\ \hat v_{j,r}=v_{j,r}]
  \]
  \textbf{One failure makes the whole example incorrect.}
  \papersource{Paper, Sec. 2, PDF pp. 1--2. Example and metric notation are explanatory.}
\end{frame}

\begin{frame}{A reusable recurrence explains the desired extrapolation}
  \[
    s_0=0,\qquad s_i=(s_{i-1}+v_i)\bmod 10,\qquad m=s_n
  \]
  \begin{center}
    \Large $0\ \xrightarrow{+7}\ 7\ \xrightarrow{+8}\ 5\ \xrightarrow{+4}\ 9$
  \end{center}
  \medskip
  \begin{itemize}
    \item The state has only ten possible values.
    \item The same update can be repeated on a longer sequence.
    \item Recall additionally requires the association \texttt{B $\mapsto$ 8}.
  \end{itemize}
  \begin{block}{Why a hybrid is attractive}
    A compact running state and access to past associations can coexist.
  \end{block}
  \papersource{Teaching explanation of the task in Sec. 2; not a measured hidden-state trajectory.}
\end{frame}

\begin{frame}{Five hybrids combine the same component types}
  \centering
  \renewcommand{\arraystretch}{1.45}
  \begin{tabular}{ll}
    \toprule
    Architecture & Computation / flow \\
    \midrule
    $\Attn\circ\LSTM$ & $H=\LSTM(X),\ Z=\Attn(H)$ \\
    $\LSTM\circ\Attn$ & $H=\Attn(X),\ Z=\LSTM(H)$ \\
    $\LSTM+\Attn$ & $Z=H_{\mathrm{lstm}}+H_{\mathrm{attn}}$ \\
    Hybrid Block & LSTM, MHA, MLP within a residual block \\
    $\LSTM\circ\Attn\circ\LSTM$ & LSTM $\to$ attention $\to$ LSTM \\
    \bottomrule
  \end{tabular}
  \medskip

  Function composition is read \alert{right to left}.
  \papersource{Paper, Appendix A, PDF p. 4. Attn includes MHA + MLP. Sec. 2 says four variants; Fig. 1 and Appendix A list five.}
\end{frame}

\begin{frame}{DAT removes the attention module's early contribution}
  \begin{columns}[c,onlytextwidth]
    \begin{column}{0.28\textwidth}
      \centering
      \includegraphics[width=\linewidth,height=0.76\textheight,keepaspectratio]{figures/fig-01-hybrid-block.pdf}
    \end{column}
    \begin{column}{0.68\textwidth}
      \begin{itemize}
        \item Read the original diagram from bottom to top.
        \item The LSTM path remains available during the first phase.
        \item The red region includes MHA, MLP, and their LayerNorms.
        \item Later, attention and recurrence train jointly.
      \end{itemize}
      \medskip
      \alert{Forward contribution is disabled during the early phase.}
      \papersource{Original crop: Fig. 1 left, PDF p. 2. The caption repeats the red attention module $\times 4$.}
    \end{column}
  \end{columns}
\end{frame}

\begin{frame}{The recurrent path receives a 1500-epoch head start}
  \begin{columns}[T,onlytextwidth]
    \begin{column}{0.31\textwidth}
      \begin{block}{Phase 1}
        Attention contribution off
        \medskip

        Train LSTM component
        \medskip

        \textbf{1500 epochs}\\
        Learning rate $10^{-4}$
      \end{block}
    \end{column}
    \begin{column}{0.31\textwidth}
      \begin{block}{Phase 2a}
        Enable attention
        \medskip

        Joint training
        \medskip

        \textbf{500 epochs}\\
        Learning rate $10^{-4}$
      \end{block}
    \end{column}
    \begin{column}{0.31\textwidth}
      \begin{block}{Phase 2b}
        Full hybrid active
        \medskip

        Continue joint training
        \medskip

        \textbf{Duration unspecified}\\
        Learning rate $10^{-5}$
      \end{block}
    \end{column}
  \end{columns}
  \medskip
  Adam; $2^{14}=16{,}384$ randomly generated samples per epoch.
  \medskip

  Ordinary training: $10^{-4}$ for 500 epochs, then $10^{-5}$.
  \papersource{Paper, Sec. 2, PDF p. 2; Appendix A, PDF p. 4.}
\end{frame}

\begin{frame}{The evaluation separates joint success from single skills}
  \begin{itemize}
    \item \textbf{Shared model settings:} embedding dimension $d=256$;
      default RoPE; Transformer components use four blocks by default.
    \item \textbf{Joint task:} two pure baselines and five hybrids;
      hybrids compared with and without DAT.
    \item \textbf{Single tasks:} recall-only and modulo-only models are
      trained and evaluated separately; no DAT rows in these tables.
    \item \textbf{Length:} reported ID range $L\le25$; OOD points extend to 100.
  \end{itemize}
  \begin{block}{Interpret the length scale as reported}
    The paper does not explicitly map table length $L$ to total token count.
    The threefold comparison is $75/25$.
  \end{block}
  \papersource{Paper, Sec. 2 / Fig. 1, PDF p. 2; Appendices A--B, PDF p. 4.}
\end{frame}

\begin{frame}{DAT preserves high joint accuracy beyond the ID boundary}
  \centering
  \includegraphics[width=0.94\linewidth,height=0.65\textheight,keepaspectratio]{figures/joint-generalization.pdf}
  \smallskip

  Hybrid Block at $L=75$: $0.09\ \to\ \gain{0.93}$ \quad (\gain{+84 percentage points}).
  \papersource{Redrawn from Fig. 1 right, PDF p. 2. Higher is better; shaded region is ID. Lines connect reported points.}
\end{frame}

\begin{frame}{Four of five DAT hybrids exceed 90\% at threefold length}
  \centering
  \renewcommand{\arraystretch}{1.4}
  \begin{tabular}{lrrrr}
    \toprule
     & \multicolumn{2}{c}{$L=75$} & \multicolumn{2}{c}{$L=100$} \\
    Architecture & Ordinary & DAT & Ordinary & DAT \\
    \midrule
    $\LSTM\circ\Attn$ & 0.05 & \gain{0.93} & 0.04 & \gain{0.88} \\
    $\Attn\circ\LSTM$ & 0.07 & 0.80 & 0.05 & 0.56 \\
    Hybrid Block & 0.09 & \gain{0.93} & 0.08 & 0.85 \\
    $\LSTM\circ\Attn\circ\LSTM$ & 0.06 & \gain{0.91} & 0.06 & 0.80 \\
    $\LSTM+\Attn$ & 0.12 & \gain{0.95} & 0.08 & 0.85 \\
    \bottomrule
  \end{tabular}
  \medskip

  Joint accuracy, 0--1. \quad All five DAT hybrids reach 0.99 at $L=50$.
  \papersource{Paper, Fig. 1 right, PDF p. 2. Architecture still matters at longer lengths.}
\end{frame}

\begin{frame}{Recall-only results reveal the LSTM retrieval bottleneck}
  \centering
  \setlength{\tabcolsep}{10pt}
  \renewcommand{\arraystretch}{1.5}
  \begin{tabular}{lrrrr}
    \toprule
    Model & $L=25$ & $L=50$ & $L=75$ & $L=100$ \\
    \midrule
    LSTM & 0.83 & 0.53 & 0.37 & 0.28 \\
    Transformer & 1.00 & 0.99 & 0.90 & 0.80 \\
    Hybrid Block & 1.00 & 0.99 & 0.98 & \gain{0.92} \\
    \bottomrule
  \end{tabular}
  \bigskip
  \begin{itemize}
    \item Attention-based models retrieve well beyond length 25.
    \item Their recall also degrades at the longest tested lengths.
    \item These are separately trained recall-only models.
  \end{itemize}
  \papersource{Selected rows from Table 1, Appendix B, PDF p. 4. Single-task accuracy; no DAT rows.}
\end{frame}

\begin{frame}{Adding attention loses the LSTM's modulo extrapolation}
  \centering
  \renewcommand{\arraystretch}{1.45}
  \begin{tabular}{lrrrr}
    \toprule
    Model & $L=25$ & $L=30$ & $L=50$ & $L=100$ \\
    \midrule
    LSTM & 1.00 & 1.00 & \gain{1.00} & \gain{1.00} \\
    Transformer & 0.98 & 0.51 & 0.10 & 0.10 \\
    Hybrid Block & 1.00 & 0.45 & 0.10 & 0.10 \\
    $\Attn\circ\LSTM$ & 1.00 & 0.63 & 0.10 & 0.10 \\
    \bottomrule
  \end{tabular}
  \bigskip
  \begin{block}{The informative failure}
    The recurrent component remains in the hybrid, but ordinary training
    no longer yields the observed LSTM extrapolation.
  \end{block}
  \papersource{Selected rows from Table 2, Appendix B, PDF p. 4. Modulo-only accuracy; no DAT rows.}
\end{frame}

\begin{frame}{The evidence supports a training-path explanation}
  \begin{enumerate}
    \item \textbf{Capability:} LSTM alone solves the tested modulo lengths.
      \medskip
    \item \textbf{Interference:} ordinary hybrids fail on longer modulo sequences.
      \medskip
    \item \textbf{Intervention:} delaying attention strongly improves joint-task OOD accuracy.
  \end{enumerate}
  \medskip
  \begin{block}{Authors' interpretation}
    Early attention enables shortcut solutions; DAT first establishes a
    more robust recurrent computation.
  \end{block}
  \papersource{Paper, Sec. 2, Fig. 1 and Table 2, PDF pp. 2 and 4. The exact shortcut and internal state representation are not measured.}
\end{frame}

\begin{frame}{More candidates dilute attention at a fixed score margin}
  $N\ge2$ positions: correct score $a$, distractor scores $b$; $\Delta=a-b$.
  \[
    p_\star=\frac{e^a}{e^a+(N-1)e^b}
           =\frac{1}{1+(N-1)e^{-\Delta}}
  \]
  \begin{itemize}
    \item At fixed $\Delta$, the correct position receives less weight as $N$ grows.
    \item For target weight $0<\rho<1$, maintaining $p_\star\ge\rho$ requires
      $\Delta\ge\log(N-1)+\log\!\frac{\rho}{1-\rho}$.
  \end{itemize}
  \begin{block}{How far the paper's evidence goes}
    The authors attribute the long-length drop to softmax sharpness.
    Table 1 shows recall degradation; it does not measure sharpness or
    isolate this cause in the DAT model.
  \end{block}
  \papersource{Paper's attribution: Sec. 2, PDF p. 2. Equations are a teaching derivation assuming equal distractor scores.}
\end{frame}

\begin{frame}{Training budget and implementation detail limit attribution}
  \begin{itemize}
    \item \textbf{Budget:} DAT's first phase processes
      $1500\times2^{14}=24{,}576{,}000$ generated samples.
      Final epoch counts and matched total-budget controls are unspecified.
      \medskip
    \item \textbf{Reproduction:} the precise loss mask, decoding protocol,
      and architecture-specific bypass implementations are not provided.
      \medskip
    \item \textbf{Scope:} synthetic LSTM--Transformer tasks at $d=256$;
      no language-modeling result or reported seed variability.
  \end{itemize}
  \papersource{Paper, Sec. 2, PDF p. 2; Appendix A, PDF p. 4. Sample count is arithmetic from the reported schedule.}
\end{frame}

\begin{frame}{Three experiments would make the mechanism clearer}
  \begin{enumerate}
    \item \textbf{Match total training budget and vary the delay.}\\
      Separate scheduling effects from total samples or compute.
      \medskip
    \item \textbf{Probe the same joint-task model.}\\
      Measure both subtask accuracies and decode the running remainder.
      \medskip
    \item \textbf{Intervene on retrieval sharpness.}\\
      Track score margins and target weights; test a temperature intervention.
  \end{enumerate}
  \papersource{Reading discussion: proposed follow-ups, not experiments reported in the paper.}
\end{frame}

\begin{frame}{Training order is part of hybrid-model design}
  \begin{block}{Main result}
    Delaying attention strongly improves length generalization on a task
    requiring both state tracking and associative recall.
  \end{block}
  \medskip
  \begin{itemize}
    \item Final architecture alone does not determine which solution is learned.
    \item At $L=75$, four DAT variants score 0.91--0.95; one scores 0.80.
    \item Farther extrapolation and transfer to language modeling remain open.
  \end{itemize}
  \papersource{Paper, Fig. 1 and Sec. 3, PDF p. 2.}
\end{frame}

\begin{frame}{Primary source and reading map}
  \textbf{Buu Phan, Reza Ebrahimi, Sanjay Haresh, Roland Memisevic.}
  \medskip

  \emph{Delayed Attention Training Improves Length Generalization in
  Transformer--RNN Hybrids.}
  \medskip

  arXiv:2510.00258v1, submitted 30 September 2025.\\
  \href{https://arxiv.org/abs/2510.00258v1}{https://arxiv.org/abs/2510.00258v1}
  \bigskip
  \begin{itemize}
    \item PDF p. 2: task, DAT, architecture diagram, joint results.
    \item PDF p. 4: training details, architecture definitions, single tasks.
    \item Teaching examples and follow-up ideas are identified on their slides.
  \end{itemize}
\end{frame}

\appendix

\begin{frame}{Backup: DAT across the reported length transition}
  \centering
  \renewcommand{\arraystretch}{1.5}
  \begin{tabular}{lrrrrrr}
    \toprule
    DAT architecture & 25 & 30 & 35 & 50 & 75 & 100 \\
    \midrule
    $\LSTM\circ\Attn$ & 1.00 & 0.99 & 0.99 & 0.99 & 0.93 & 0.88 \\
    $\Attn\circ\LSTM$ & 1.00 & 0.99 & 0.99 & 0.99 & 0.80 & 0.56 \\
    Hybrid Block & 1.00 & 1.00 & 1.00 & 0.99 & 0.93 & 0.85 \\
    Sandwich & 1.00 & 1.00 & 0.99 & 0.99 & 0.91 & 0.80 \\
    Parallel & 1.00 & 1.00 & 1.00 & 0.99 & 0.95 & 0.85 \\
    \bottomrule
  \end{tabular}
  \bigskip

  Joint accuracy. Sandwich: $\LSTM\circ\Attn\circ\LSTM$.
  \smallskip

  Parallel: $\LSTM+\Attn$. Column headers are reported lengths.
  \papersource{Selected columns from Fig. 1 right, PDF p. 2.}
\end{frame}

\begin{frame}{Backup: disabling contribution requires a suitable bypass}
  \begin{columns}[T,onlytextwidth]
    \begin{column}{0.47\textwidth}
      \begin{block}{Serial composition}
        Preserve the information path when the attention module is absent.
        \[
          A_{\mathrm{early}}(H)=H
        \]
      \end{block}
    \end{column}
    \begin{column}{0.47\textwidth}
      \begin{block}{Parallel addition}
        Remove the branch contribution before summing.
        \[
          Z_{\mathrm{early}}=H_{\mathrm{lstm}}+0
        \]
      \end{block}
    \end{column}
  \end{columns}
  \bigskip
  \begin{itemize}
    \item These illustrate bypass semantics; the paper does not supply this code.
    \item Keeping a frozen attention output active changes the forward function.
    \item A switch to modulo-only supervision is not specified for DAT.
  \end{itemize}
  \papersource{Implementation interpretation of Sec. 2 and Fig. 1, PDF p. 2; conceptual formulas, not author pseudocode.}
\end{frame}

\end{document}