% Self-authored reading presentation based on arXiv:2609.33093v1.
% Build: python3 build_slides.py slides.en.tex
\documentclass[aspectratio=169,11pt]{beamer}
\usepackage[T1]{fontenc}
\usepackage[utf8]{inputenc}
\usepackage{amsmath,amssymb,graphicx,booktabs}

\definecolor{AcademicBlue}{RGB}{26,57,86}
\definecolor{AcademicTeal}{RGB}{0,105,112}
\setbeamercolor{structure}{fg=AcademicBlue}
\setbeamercolor{frametitle}{fg=AcademicBlue,bg=white}
\setbeamercolor{title}{fg=AcademicBlue}
\setbeamercolor{alerted text}{fg=AcademicTeal}
\setbeamercolor{block title}{fg=white,bg=AcademicBlue}
\setbeamercolor{block body}{fg=black,bg=AcademicBlue!5}
\setbeamertemplate{navigation symbols}{}
\setbeamertemplate{itemize items}[circle]
\setbeamertemplate{footline}{%
  \hspace{8mm}{\scriptsize\color{gray}How Linear Attention Remembers\;|\;v1}
  \hfill\usebeamerfont{page number in head/foot}%
  \insertframenumber\,/\,\inserttotalframenumber\hspace{4mm}\vspace{3mm}}
\setbeamersize{text margin left=8mm,text margin right=8mm}
\setbeamerfont{frametitle}{size=\Large,series=\bfseries}
\setbeamerfont{title}{size=\LARGE,series=\bfseries}
\newcommand{\papersource}[1]{\par\smallskip{\scriptsize\color{gray}#1\par}}
\newcommand{\figwide}[1]{\begin{center}\includegraphics[width=.97\linewidth,height=.52\textheight,keepaspectratio]{#1}\end{center}}
\newcommand{\R}{\mathbb{R}}
\title[How Linear Attention Remembers]{How Linear Attention Remembers}
\subtitle{Causal evidence for writing, retention, and readout}
\author{Paper authors: Kichang Lee, JaeYeon Park,\\Songkuk Kim, and JeongGil Ko}
\date{arXiv:2609.33093v1\quad |\quad 27 September 2026\\[2mm]
\small Reading presentation\quad |\quad 8 October 2026}

\begin{document}

\begin{frame}
  \titlepage
  \vspace{-4mm}
  \begin{center}
  \alert{Selective recall can emerge from shared, causally coupled state.}
  \end{center}
  \papersource{20--30 minute main discussion; slides 21--28 provide additional evidence and details.}
\end{frame}

\begin{frame}{Fixed memory changes the organization of history}
  \begin{columns}[T,onlytextwidth]
    \column{.47\textwidth}
    \begin{block}{Full attention}
      Past tokens retain explicit key--value entries.\\[3mm]
      The cache grows with context length.
    \end{block}
    \column{.49\textwidth}
    \begin{block}{Recurrent linear attention}
      Many past tokens update one matrix per head.\\[3mm]
      State size stays fixed as context grows.
    \end{block}
  \end{columns}
  \vspace{3mm}
  \begin{itemize}
    \item Which writes carry the remembered content?
    \item How does a query access one fact among many?
    \item What fails as writes compete, or when KV memory is available?
  \end{itemize}
  \papersource{Paper, Sec. 1, PDF pp. 1--2. This is a mechanism study of pretrained models.}
\end{frame}

\begin{frame}{A head transforms old state and adds a new write}
  \[
    S_t=\mathcal{T}_t(S_{t-1})+W_t,
    \qquad o_t=q_t^\top S_t
  \]
  \begin{itemize}
    \item $S_t,W_t\in\R^{d_k\times d_v}$; $q_t\in\R^{d_k}$; $o_t\in\R^{d_v}$.
    \item A common write is $W_t=u_tv_t^\top$: direction $u_t$, content $v_t$.
    \item Conditioned on token representations, $\mathcal T_t$ is linear in state.
  \end{itemize}
  \begin{block}{The update rules differ}
    GLA uses data-dependent multiplicative gating. GDN also corrects the content associated with the current key direction.
  \end{block}
  \papersource{Paper, Sec. 2.1, Eqs. (1)--(2), PDF p. 2. Core read omits output gating and normalization.}
\end{frame}

\begin{frame}{Retention and query access are separate operations}
  \[
    C_i^{(t)}=(\mathcal T_t\circ\cdots\circ\mathcal T_{i+1})(W_i)
  \]
  \[
    \underbrace{S_t=\sum_{i=1}^{t} C_i^{(t)}}_{\text{contributions retained in state}}
    \qquad
    \underbrace{o_t=\sum_{i=1}^{t}q_t^\top C_i^{(t)}}_{\text{contributions accessed by the query}}
  \]
  \begin{itemize}
    \item Exact for $S_0=0$ along a \alert{fixed forward trajectory}.
    \item A retained contribution need not influence the current output.
    \item The aggregate state does not uniquely identify its source terms.
  \end{itemize}
  \papersource{Paper, Sec. 2.2, Eqs. (3)--(5), PDF pp. 2--3. End-to-end interventions can change the later trajectory.}
\end{frame}

\begin{frame}{Shared storage can support selective access}
  \begin{block}{Teaching example: two writes, no gating}
  \[
    S=u_Av_A^\top+u_Bv_B^\top,\qquad q=u_A,\qquad\|u_A\|=1
  \]
  \[
    q^\top S=v_A^\top+\underbrace{(u_A^\top u_B)}_{\rho}\,v_B^\top
  \]
  \end{block}
  \begin{itemize}
    \item Orthogonal directions give a selective read of $A$.
    \item Overlap introduces a contribution from $B$.
    \item Learned gates and downstream computation determine the real effect.
  \end{itemize}
  \papersource{Presenter's illustration of the paper's framework, not a reported experiment.}
\end{frame}

\begin{frame}{Paired counterfactuals connect writing to recall}
  \begin{center}
  Recipient: Alice lives in \textbf{Paris}.\quad Donor: Alice lives in \alert{London}.
  \end{center}
  \vspace{1mm}
  \begin{tabular}{@{}p{.24\linewidth}p{.69\linewidth}@{}}
    \toprule
    Intervention & Causal question \\
    \midrule
    Block / replace write & Is the fact-time update necessary and content-specific? \\[2mm]
    Swap state & Is donor information already present before the query? \\[2mm]
    Patch query output & Can this read pathway transmit the donor effect? \\[2mm]
    Restore clean output & Does it block the effect of a donor-state swap? \\
    \bottomrule
  \end{tabular}
  \par\vspace{3mm}
  Only a development-selected head is changed in the main single-fact tests.
  \papersource{Paper, Fig. 1, PDF p. 4; Secs. 3.1--3.2. Example follows the intervention schematic.}
\end{frame}

\begin{frame}{Frozen models answer controlled recall questions}
  \begin{center}
  \begin{tabular}{lccc}
    \toprule
    Model label & Recurrent layers & Heads/layer & State/head \\
    \midrule
    GLA-340M & 24 & 4 & $128\times256$ \\
    GDN-340M & 24 & 4 & $256\times256$ \\
    GDN-1.3B & 24 & 8 & $256\times256$ \\
    \bottomrule
  \end{tabular}
  \end{center}
  \begin{itemize}
    \item WikiText-103 context, controlled associations, four balanced answers.
    \item Donor/recipient pairs differ in the target value; base weights stay frozen.
    \item Development and confirmation use separate source passages.
    \item Additional hybrids: GDN-340M and Qwen3.5-4B/9B.
  \end{itemize}
  \papersource{Paper, Sec. 2.3, PDF p. 3; Appendix A, Table 1, PDF pp. 12--14. Model labels follow released checkpoints.}
\end{frame}

\begin{frame}{A small set of heads has strong causal effects}
  \begin{columns}[c,onlytextwidth]
    \column{.45\textwidth}
      \includegraphics[width=\linewidth,height=.70\textheight,keepaspectratio]{figures/fig-02.pdf}
    \column{.51\textwidth}
      \begin{itemize}
        \item Screen all 96 / 96 / 192 heads on development data.
        \item $\Delta(z_d-z_r)>0$ means a shift toward the donor answer.
        \item Freeze GLA \textbf{L21H3}, GDN-340M \textbf{L20H2}, and GDN-1.3B \textbf{L20H3}.
        \item Keep those heads fixed for confirmation.
      \end{itemize}
  \end{columns}
  \papersource{Paper, Fig. 2, PDF p. 4; Appendix B, pp. 14--15. Indices are zero-based; $z_d,z_r$ are donor/recipient logits. See slide 21 for the v1 score notation issue.}
\end{frame}

\begin{frame}{Fact-time writes carry persistent, specific content}
  \figwide{figures/fig-03.pdf}
  \begin{itemize}
    \item Blocking the selected write lowers recall by \textbf{36.0--78.9 pp}.
    \item Donor writes transfer the donor answer in \textbf{68--90\%} of cases.
    \item GLA joint effects are non-additive: the selected head interacts with its circuit.
  \end{itemize}
  \papersource{Paper, Fig. 3, Sec. 3.1, PDF p. 5. Error bars: source-bootstrap 95\% CIs; pp = percentage points.}
\end{frame}

\begin{frame}{Query-time output mediates the state-swap effect}
  \begin{columns}[c,onlytextwidth]
    \column{.47\textwidth}
    \includegraphics[width=\linewidth,height=.70\textheight,keepaspectratio]{figures/fig-04.pdf}
    \column{.49\textwidth}
    \begin{enumerate}
      \item Donor state before the query shifts the answer.
      \item Donor output alone reproduces a similar shift.
      \item Restoring recipient output after the state swap removes the shift.
    \end{enumerate}
    \vspace{3mm}
    \alert{Stored information affects behavior through the selected read pathway.}
  \end{columns}
  \papersource{Paper, Fig. 4, Sec. 3.2, PDF p. 5. Horizontal axis: donor-versus-recipient margin change.}
\end{frame}

\begin{frame}{Four facts remain behaviorally distinguishable}
  \figwide{figures/fig-05.pdf}
  \begin{itemize}
    \item Donor-answer rates: \textbf{89.1 / 71.9 / 61.3\%} across the three models.
    \item Non-target break rates: \textbf{1.42 / 0.41 / 0.56\%}.
    \item Break rate: previously correct non-target answers turned wrong.
  \end{itemize}
  \papersource{Paper, Fig. 5, Sec. 3.3, PDF p. 6. Model order: GLA-340M, GDN-340M, GDN-1.3B.}
\end{frame}

\begin{frame}{A selective edit contains opposing cross-fact effects}
  Let $\Delta_A,\Delta_B$ be donor--recipient state differences for facts $A,B$.
  \[
    \Delta_{A\parallel B}
      =\frac{\langle\Delta_A,\Delta_B\rangle_F}{\|\Delta_B\|_F^2}\Delta_B,
    \qquad
    \Delta_{A\perp B}=\Delta_A-\Delta_{A\parallel B}
  \]
  \begin{itemize}
    \item Add each component separately, then query fact $B$.
    \item Parallel and orthogonal components have \alert{opposing} effects on $B$.
    \item The full edit has a weaker signed effect, consistent with cancellation.
    \item Norm-matched random directions have much smaller mean effects.
  \end{itemize}
  \papersource{Paper, Sec. 3.3, Fig. 5(c), PDF p. 6. Projection written out from the text; $\Delta_B\ne0$. Geometric orthogonality does not guarantee behavioral independence.}
\end{frame}

\begin{frame}{Hybrid recall is carried mainly by the KV cache}
  \begin{columns}[c,onlytextwidth]
    \column{.44\textwidth}
    \includegraphics[width=\linewidth,height=.70\textheight,keepaspectratio]{figures/fig-06.pdf}
    \column{.53\textwidth}
    \begin{itemize}
      \item Near-matched hybrid GDN: recurrent/local \textbf{$<1\%$}, KV \textbf{$>99\%$}.
      \item Qwen3.5-4B: recurrent \textbf{0.2--3.6\%}, KV \textbf{96--100\%}.
      \item Denominator: effect of replacing \emph{both} cache substrates.
    \end{itemize}
    \vspace{2mm}
    This identifies the direct pre-query carrier. Recurrent layers may still form KV contents and queries.
  \end{columns}
  \papersource{Paper, Sec. 4.1, Fig. 6, PDF p. 7. Separately trained checkpoints; two chance-level clean conditions excluded.}
\end{frame}

\begin{frame}{Competing and overlapping writes weaken recall}
  \figwide{figures/fig-07.pdf}
  \begin{itemize}
    \item Fixed length, $K=1\to16$: recall falls by about \textbf{30--40 pp}.
    \item Fixed $K=4$, distance $128\to1024$: a much smaller decline.
    \item Aligning a later write key lowers target margin by \textbf{0.27--0.44 logits}.
  \end{itemize}
  \papersource{Paper, Sec. 4.2, Fig. 7, PDF pp. 7--8. Key norm is preserved. $K=32$ uses a separate high-load layout.}
\end{frame}

\begin{frame}{Retention, edit strength, and collateral fail differently}
  \figwide{figures/fig-08.pdf}
  \begin{itemize}
    \item At $K=32$, recall is about \textbf{47\%, 60\%, 69\%}; edit strength also falls.
    \item GDN collateral stays below \textbf{2\%}; GLA is higher, near or above \textbf{5\%}.
    \item Low collateral alone is insufficient if the intended edit is already weak.
  \end{itemize}
  \papersource{Paper, Fig. 8, PDF p. 8. GLA points at $K=8,32$ are slightly below 5\%, qualifying the prose. Missing GLA edit points were not evaluated. Fig. 5 uses a separate comparison.}
\end{frame}

\begin{frame}{Single-component dominance depends on the checkpoint}
  \figwide{figures/fig-14.pdf}
  \begin{itemize}
    \item GLA-340M top-1: \textbf{94.7\%}; GLA-7B top-1: \textbf{12.3\%}, top-16: \textbf{92.3\%}.
    \item RWKV7-2.9B / 13.3B top-32: \textbf{97.8\% / 91.6\%}.
    \item Joint top-$k$ effects are measured directly; there is no monotonic scale trend.
  \end{itemize}
  \papersource{Paper, Appendix E.2, Fig. 14, PDF pp. 19--20. 32 development / 64 test sources; formats differ across model families.}
\end{frame}

\begin{frame}{Native errors can retain useful state information}
  \begin{center}
  \begin{tabular}{lcc}
    \toprule
    Frozen backbone / condition & Native accuracy & With correction \\
    \midrule
    GLA, colors & 24.2\% & \textbf{52.3\%} \\
    GLA, shapes + changed template & 30.5\% & \textbf{49.2\%} \\
    GDN, colors & 46.9\% & 48.4\% \\
    Qwen3.5-4B recurrent feature & 78.9\% & 46.9\% \\
    \bottomrule
  \end{tabular}
  \end{center}
  \begin{itemize}
    \item A separate correction model combines native logits and state features.
    \item Trained on source-disjoint data, then frozen for evaluation.
    \item GLA supports a storage/access distinction; benefits do not transfer uniformly.
  \end{itemize}
  \papersource{Paper, Appendix F.1, Fig. 15, PDF pp. 20--21. This is a four-choice diagnostic, not a general decoding method.}
\end{frame}

\begin{frame}{The causal account has concrete experimental boundaries}
  \begin{itemize}
    \item \textbf{Task competence:} the prose pilot misses an 80\% top-1 gate, so its lifecycle interventions are not run.
    \vspace{2mm}
    \item \textbf{Architecture attribution:} pure and hybrid checkpoints were trained separately; GLA and GDN also differ in state size.
    \vspace{2mm}
    \item \textbf{Localization:} a high-impact head is an interacting pathway; dominance varies with checkpoint and input format.
    \vspace{2mm}
    \item \textbf{Practical editing:} donor and neutral counterfactuals require information unavailable to a general memory manager.
  \end{itemize}
  \papersource{Paper, Sec. 6; Appendices A, D--F, PDF pp. 9, 13, 17--21.}
\end{frame}

\begin{frame}{Measure how memories compete, survive, and are read}
  \begin{block}{Mechanistic picture}
    Content-specific writes persist in a shared state and influence answers through learned read pathways. Selectivity coexists with causal coupling.
  \end{block}
  \begin{enumerate}
    \item Separate memory load from elapsed context in evaluation.
    \item Report retention, intended edit strength, and collateral together.
    \item In hybrids, distinguish representation formation from the direct runtime carrier of recalled identity.
  \end{enumerate}
  \papersource{Synthesis of Secs. 3--6, PDF pp. 4--9. Write partitioning, refresh, and dynamic recurrent/KV allocation remain design directions.}
\end{frame}

\begin{frame}{Paper reference and discussion prompts}
  \textbf{Kichang Lee, JaeYeon Park, Songkuk Kim, JeongGil Ko.}\\[2mm]
  \emph{How Linear Attention Remembers.}\\
  arXiv:2609.33093v1, 27 September 2026.\\[2mm]
  \href{https://arxiv.org/abs/2609.33093v1}{\texttt{https://arxiv.org/abs/2609.33093v1}}
  \vspace{5mm}
  \begin{itemize}
    \item Would the GLA/GDN gap persist with matched state capacity and training?
    \item Can useful edits be found without an oracle donor execution?
    \item Which facts should a hybrid preserve in growing KV memory?
  \end{itemize}
  \papersource{All experimental figures in this presentation are reproduced from the version above. The two-write example is a teaching illustration.}
\end{frame}

\appendix

\begin{frame}{Backup: head scoring needs a v1 notation clarification}
  The prose defines a mean \emph{change} in the donor--recipient margin:
  \[
    m_i=z_{i,d_i}-z_{i,r_i},\qquad
    J_{\ell,h}^{\text{prose}}=\frac1{16}\sum_i
    \bigl(m_i^{(\ell,h)}-m_i^{\mathrm{clean}}\bigr).
  \]
  \begin{itemize}
    \item Printed Eq. (6) has adjacent margin factors, with no minus sign.
    \item The text and Fig. 9 describe a signed margin \emph{shift}; implementation needs clarification.
    \item The 340M screen replicates earlier selection; the 1.3B screen is prospective.
    \item Source-wise winners: 16/16, 16/16, 15/16; leave-one-source-out winners are stable.
  \end{itemize}
  \papersource{Paper, Appendix B, Eq. (6), Fig. 9, PDF pp. 14--15. Equation above follows the prose, not the printed formula.}
\end{frame}

\begin{frame}{Backup: model names do not imply matched budgets}
  \begin{center}
  \begin{tabular}{lccc}
    \toprule
    Model & Actual parameters & Recurrent & Full attention \\
    \midrule
    GLA-340M & 341.7M & 24 & 0 \\
    GDN-340M & 399.5M & 24 & 0 \\
    Hybrid GDN-340M & 382.6M & 16 & 8 \\
    GDN-1.3B & 1.466B & 24 & 0 \\
    Qwen3.5-4B / 9B & 4.206B / 8.954B text & 24 & 8 \\
    \bottomrule
  \end{tabular}
  \end{center}
  \begin{itemize}
    \item Hybrid GDN attention indices: $2,5,8,11,14,17,20,23$.
    \item These counts imply 2:1 recurrent/attention; Appendix A calls it 3:1.
    \item Qwen's 24/8 pattern is 3:1. GLA's state/head has half as many elements as GDN's.
  \end{itemize}
  \papersource{Paper, Appendix A, Table 1, PDF p. 13. Counts and explicit layer list are used here to resolve the ratio inconsistency.}
\end{frame}

\begin{frame}{Backup: held-out hubs agree, global rankings do not}
  \begin{center}
  \includegraphics[width=.84\linewidth,height=.50\textheight,keepaspectratio]{figures/fig-10.pdf}
  \end{center}
  \begin{itemize}
    \item Selected heads rank first at both write and pre-query stages on 32 sources.
    \item Bootstrap rank intervals: $[1,1]$ in all four model/stage comparisons.
    \item Across all 96 heads, write/read rank correlations are only 0.173 and 0.149.
  \end{itemize}
  \papersource{Paper, Appendix C.1, Fig. 10, PDF pp. 15--16. Correlation 95\% CIs cross zero for both models.}
\end{frame}

\begin{frame}{Backup: answer readout rises sharply at the hub layer}
  \begin{columns}[c,onlytextwidth]
    \column{.60\textwidth}
    \includegraphics[width=\linewidth,height=.67\textheight,keepaspectratio]{figures/fig-11.pdf}
    \column{.36\textwidth}
    Independent confirmation:
    \begin{itemize}
      \item GLA retention: 28.9\% $\to$ 95.3\%.
      \item GDN retention: 25.8\% $\to$ 100\%.
      \item Three updates: +28.9 / +43.0 pp.
    \end{itemize}
    \vspace{2mm}
    Logit lens localizes accessible readout, not the first existence of information.
  \end{columns}
  \papersource{Paper, Appendix C.2, Fig. 11, PDF pp. 16--17. Curves: 96-source localization; annotations: disjoint 128-source confirmation.}
\end{frame}

\begin{frame}{Backup: a neutral state selectively suppresses a fact}
  \figwide{figures/fig-12.pdf}
  \begin{itemize}
    \item Replace the chosen head with a state from a target-absent execution.
    \item Target accuracy: GLA \textbf{91.8\% $\to$ 7.6\%}; GDN \textbf{96.1\% $\to$ 37.1\%}.
    \item Other-fact mean accuracy changes by +2.15 / +1.11 pp, despite some individual disruptions.
  \end{itemize}
  \papersource{Paper, Appendix D.1, Fig. 12, PDF p. 17. 128 sources; oracle counterfactual, not a guaranteed deletion procedure.}
\end{frame}

\begin{frame}{Backup: the same head carries task-dependent directions}
  \figwide{figures/fig-13.pdf}
  \begin{itemize}
    \item Frozen-head effects persist across bindings, templates, values, and updates.
    \item A rank-4 color basis transfers colors but yields no shape donor answers.
    \item Most shape transfer lies in the displayed rank-8 orthogonal component.
  \end{itemize}
  \papersource{Paper, Appendix D.2, Fig. 13, PDF p. 18. Panel (a): 128 sources, qualified rates. Panel (b): exploratory 32-source analysis.}
\end{frame}

\begin{frame}{Backup: KV dominance survives more natural tasks}
  \begin{center}
  \small
  \begin{tabular}{lrrrrr}
    \toprule
    Qwen3.5-4B task & $n$ & Clean & Rec. donor & KV donor & KV/full \\
    \midrule
    Entity update, WikiText & 128 & 93.0 & 0.0 & 85.2 & 97.2 \\
    Entity update, PG19 & 128 & 89.1 & 0.0 & 89.8 & 96.2 \\
    RULER single-NIAH, $\sim$4K & 64 & 100.0 & 0.0 & 100.0 & 97.2 \\
    \bottomrule
  \end{tabular}
  \end{center}
  \begin{itemize}
    \item All performance/effect entries are percentages.
    \item Donor columns measure answer rates; KV/full measures margin-effect recovery.
    \item Recurrence can still influence how KV contents were formed.
  \end{itemize}
  \papersource{Paper, Appendix E.1, Table 2, PDF p. 19. Source-bootstrap / paired intervention framework follows Sec. 4.1.}
\end{frame}

\begin{frame}{Backup: the prose pilot fails a prerequisite for attribution}
  \begin{center}
  \begin{tabular}{rccc}
    \toprule
    Filler tokens & Recipient & Donor & Evidence removed \\
    \midrule
    512 & 56.3\% & 53.1\% & 0.0\% \\
    128 & 71.9\% & 68.8\% & 3.1\% \\
    \bottomrule
  \end{tabular}
  \end{center}
  \begin{itemize}
    \item Localized evidence sentence; 31 city identities; two fixed query paraphrases.
    \item 32 source families; joint full-vocabulary top-1 accuracy across the paraphrases.
    \item Both recipient and donor must reach 80\%; neither setting qualifies.
    \item Lifecycle interventions are therefore not applied in this pilot.
  \end{itemize}
  \papersource{Paper, Appendix F.2, Table 3, PDF p. 21. An unmet competence gate leaves this extension open.}
\end{frame}

\end{document}