% Research discussion; compile with pdfLaTeX and shell escape disabled. % Based on the arxiv-paper-reader Beamer template. \documentclass[aspectratio=169,11pt]{beamer} \usepackage[T1]{fontenc} \usepackage[utf8]{inputenc} \usepackage{lmodern} \usepackage{amsmath,amssymb,graphicx,booktabs} \definecolor{AcademicBlue}{RGB}{26,57,86} \definecolor{AcademicTeal}{RGB}{0,105,112} \setbeamercolor{structure}{fg=AcademicBlue} \setbeamercolor{frametitle}{fg=AcademicBlue,bg=white} \setbeamercolor{title}{fg=AcademicBlue} \setbeamercolor{alerted text}{fg=AcademicTeal} \setbeamercolor{block title}{fg=white,bg=AcademicBlue} \setbeamercolor{block body}{fg=black,bg=AcademicBlue!5} \setbeamertemplate{navigation symbols}{} \setbeamertemplate{itemize items}[circle] \setbeamertemplate{footline}{\hfill\usebeamerfont{page number in head/foot}\insertframenumber\,/\,\inserttotalframenumber\hspace{4mm}\vspace{3mm}} \setbeamersize{text margin left=8mm,text margin right=8mm} \setbeamerfont{frametitle}{size=\Large,series=\bfseries} \setbeamerfont{title}{size=\LARGE,series=\bfseries} \newcommand{\papersource}[1]{\par\smallskip{\scriptsize\color{gray}#1\par}} \newcommand{\R}{\mathbb R} \newcommand{\diag}{\operatorname{diag}} \newcommand{\tril}{\operatorname{tril}} \newcommand{\fig}[2]{\includegraphics[width=\linewidth,height=#2,keepaspectratio]{figures/#1}} \title[Triadic Linear Attention]{Triadic Linear Attention} \subtitle{Three-Dimensional Recurrent States\\for Long-Context Sequence Modeling} \author{\small Paper by Oliver Sieberling, Bharat Runwal, David Jin,\\Ryan Chin, Rameswar Panda, and Yoon Kim} \institute{MIT \quad / \quad MIT-IBM Computing Research Lab\\[3mm]Independent research paper discussion} \date{arXiv:2609.36529v1 \quad $\cdot$ \quad 9 October 2026} \begin{document} \begin{frame} \titlepage \vspace{-4mm} \begin{center}\color{AcademicTeal} Two short addresses create a larger memory with few new parameters. \end{center} \end{frame} \begin{frame}{Memory capacity is a separate scaling axis} \[ S_t=S_{t-1}+k_tv_t^\top,\qquad o_t=\sum_{s\leq t}(q_t^\top k_s)v_s \] \begin{itemize} \item A $d\times d$ matrix stores superposed associations. \item Overlapping keys produce retrieval interference; $d$ dimensions allow at most $d$ mutually orthogonal keys. \item More heads or wider values also enlarge projections, competing with the MLP for a fixed parameter budget. \end{itemize} \begin{block}{Research question} How can recurrent memory grow much faster than model parameters? \end{block} \papersource{Paper, Secs. 1--2.1, PDF pp. 1--2; budget comparison in Sec. 3.2, p. 7.} \end{frame} \begin{frame}{A second key expands the joint address space} \[ \underbrace{S_t}_{d\times E\times d} =S_{t-1}+\underbrace{k_t}_{d}\otimes \underbrace{k'_t}_{E}\otimes\underbrace{v_t}_{d} \] \[ o_t=S_t\times_1q_t\times_2q'_t =\sum_{s\leq t}(q_t^\top k_s)(q'_t{}^\top k'_s)v_s \] \begin{itemize} \item The effective key is $\kappa_t=k_t\otimes k'_t\in\R^{dE}$. \item State size grows by $E$; only two short address projections are added, plus small gates and convolutions. \item $E=1$ with $k'_t=q'_t=1$ recovers ordinary linear attention. \end{itemize} \papersource{Paper, Sec. 2.2, Eq. (2), PDF p. 3. $\times_n$ contracts tensor axis $n$.} \end{frame} \begin{frame}{Two records can share one key without colliding} \begin{columns}[T,onlytextwidth] \column{0.46\textwidth} \begin{block}{One address} Both records use $k=e_1$: \[ S=e_1a^\top+e_1b^\top \] Querying with $e_1$ returns \[ S^\top e_1=a+b. \] \end{block} \column{0.50\textwidth} \begin{block}{A joint address} Use orthogonal $f_1,f_2$: \[ S=e_1\otimes f_1\otimes a +e_1\otimes f_2\otimes b. \] Querying with $(e_1,f_1)$ returns $a$, since $f_1^\top f_2=0$. \end{block} \end{columns} \medskip There are $dE$ orthogonal address pairs in an ideal product basis. Learned addresses need not be orthogonal. \papersource{Teaching example derived from Eq. (2), PDF p. 3; not a paper experiment.} \end{frame} \begin{frame}{The delta rule updates the two-key association jointly} \[ \widetilde S_t[:,e,:]=\alpha_{t,e}S_{t-1}[:,e,:],\qquad \widehat v_t=\widetilde S_t\times_1 k_t\times_2 k'_t \] \[ S_t=\widetilde S_t+\beta_t k_t\otimes k'_t\otimes(v_t-\widehat v_t) \] \begin{itemize} \item Forget each slice, then correct the residual at the joint address. \item With unit-norm keys, the same-address read becomes $(1-\beta_t)\widehat v_t+\beta_tv_t$. \item All slices contribute to one residual: this couples their updates. \item sGLA keeps the write strength but drops the erase term: $S_t=\widetilde S_t+\beta_t k_t\otimes k'_t\otimes v_t$. \end{itemize} \papersource{Paper, Eqs. (3)--(5), PDF p. 4; Algorithm 2, p. 20. Interpolation is a derived identity.} \end{frame} \begin{frame}{The construction fits inside a modern GDN block} \begin{enumerate} \item RMSNorm the input; project and causally convolve. \item Apply SiLU to $q,k,v$ and softplus to $q',k'$. \item Normalize $q,k,q',k'$ individually to unit length. \item Apply slice-wise forgetting and the joint delta update. \item Read out, apply RMSNorm and an output gate, then project. \item Add the residual and the standard SwiGLU MLP block. \end{enumerate} \medskip \[ \alpha_{t,e}=\exp[-\exp(A_e)\,\mathrm{softplus}(W_{\alpha,e}u_t+b_e)], \quad \beta_t=\sigma(W_\beta u_t) \] At $E=1$, positivity plus normalization makes $q'=k'=1$ exactly. \papersource{Paper, Appendix D / Algorithm 2, PDF pp. 19--20. Projections are fused; Appendix A, p. 18.} \end{frame} \begin{frame}{Factorization avoids an $E$-fold block-attention cost} For a chunk of $C$ tokens, without forgetting or the delta rule: \[ A=\tril\!\left[(QK^\top)\odot(Q'K'{}^\top)\right] \] \[ O=\sum_e\diag(q'_e)QS_{[i],e}+AV \] \[ S_{[i+1],e}=S_{[i],e}+K^\top\diag(k'_e)V \] \begin{block}{Where the saving occurs} Forming $A$: $O(C^2dE)\ \longrightarrow\ O(C^2(d+E))$.\\ State reads and writes still grow with $E$. \end{block} \papersource{Paper, Sec. 2.5, Eqs. (6)--(8), PDF pp. 4--5. $q'_e,k'_e$ are columns over a chunk.} \end{frame} \begin{frame}{Tiling the value axis makes the large state practical} \begin{columns}[T,onlytextwidth] \column{0.49\textwidth} \begin{block}{One head, $d=128$, $E=8$} \[ 128\times8\times128\times4 \] \centering $=512$ KiB in FP32 \end{block} Too large for one Hopper SM's register file. \column{0.47\textwidth} \begin{block}{One tile, 32 value columns} \[ 128\times8\times32\times4 \] \centering $=128$ KiB in FP32 \end{block} Each tile keeps all $E$ slices for its value columns. \end{columns} \medskip \begin{itemize} \item CuTe kernels; 64-token chunks; independent value columns. \item Training saves BF16 chunk-boundary states for backward. \item Fixed inference state does not imply fixed training activation memory. \end{itemize} \papersource{Paper, Sec. 2.5, PDF p. 5; Appendix A, pp. 17--18.} \end{frame} \begin{frame}{All language models receive long-context extension} \centering \begin{tabular}{lrr} \toprule Setting & 400M scale & 1.3B scale\\ \midrule Layers / width / heads & 24 / 1024 / 8 & 24 / 2048 / 16\\ Key and value dimension & 128 & 128\\ Pretraining tokens & 20B & 65B\\ Pretraining context & 4K & 4K\\ Extension context & 64K & 64K\\ \bottomrule \end{tabular} \medskip \begin{itemize} \item FineWeb-Edu pretraining; extension adds 5 tokens per parameter using web, books, and scientific PDFs. \item Transformer: GQA-8, QK-norm, RoPE base $10\mathrm K\to2\mathrm M$. \item Report PPL, six-task recall, ten-task zero-shot, and RULER NIAH. \end{itemize} \papersource{Paper, Sec. 3.1, PDF pp. 5--6; Appendix D, pp. 19--20. All evaluations follow extension.} \end{frame} \begin{frame}{MQAR isolates a substantial increase in capacity} Multi-query associative recall: read $N$ key--value pairs, then answer $N$ queries. \par\smallskip \begin{columns}[c,onlytextwidth] \column{0.58\textwidth} \fig{fig01-mqar.png}{4.7cm} \column{0.39\textwidth} Two layers, four heads, $d=16$; no forgetting, convolution, or delta rule in the mixer. \medskip Increasing $E$ moves the accuracy curve to the right. \medskip $E=16$: roughly $16\times$ more associations with only $1.08\times$ non-embedding parameters. \end{columns} \papersource{Paper, Fig. 1 / Sec. 2.3, PDF p. 3; Appendix C, p. 19. 25 seeds per $(N,E)$ cell; read-only queries.} \end{frame} \begin{frame}{Larger states extend the long-book prediction advantage} \begin{columns}[c,onlytextwidth] \column{0.49\textwidth} \centering 400M parameters\\[1mm] \fig{fig02-pg19-400m.png}{4.45cm} \column{0.49\textwidth} \centering 1.3B parameters\\[1mm] \fig{fig02-pg19-1p3b.png}{4.45cm} \end{columns} \smallskip Negative loss difference favors the recurrent model. $E=8$ remains below the Transformer at 64K in both panels. \papersource{Paper, Fig. 2 left, PDF p. 5. Vertical dashed lines mark memory-size crossovers, not loss crossovers.} \end{frame} \begin{frame}{Recall improves strongly, especially on copying tasks} \begin{columns}[c,onlytextwidth] \column{0.65\textwidth} \centering 1.3B task breakdown\\[1mm] \fig{fig02-recall-1p3b.png}{4.8cm} \column{0.32\textwidth} \textbf{GDN $\to$ Triadic, $E=8$} \medskip 400M recall:\\ $26.2\to33.1$\\ \alert{$+6.9$ percentage points} \medskip 1.3B recall:\\ $36.1\to44.4$\\ \alert{$+8.3$ percentage points} \end{columns} \medskip FDA and SWDE gain most. Individual tasks need not improve monotonically with $E$; the Transformer remains stronger on average. \papersource{Paper, Fig. 2 right, PDF p. 5; exact averages from Table 2, p. 7. 400M: three seeds; 1.3B: one seed.} \end{frame} \begin{frame}{At equal state size, how it is expanded matters} 400M GDN family; $4\times$ state (25.2 MB), approximately matched parameters. \medskip \centering \begin{tabular}{lrrr} \toprule Expansion & Wiki PPL $\downarrow$ & PG19 PPL $\downarrow$ & Recall $\uparrow$\\ \midrule Larger heads, $d=512$ & 11.25 & 14.24 & 28.5\\ Four value heads / key & 11.35 & 14.36 & 27.3\\ Wider values, $d_v=512$ & 11.41 & 14.44 & 27.5\\ \textbf{Triadic, $E=4$} & \textbf{10.93} & \textbf{13.79} & \textbf{31.1}\\ \bottomrule \end{tabular} \medskip \begin{itemize} \item Wider/grouped values shrink the MLP from 2816 to 640 to fit the budget; Triadic preserves it. \item Larger heads preserve projection width and still underperform. \item sGLA shows similar PPL gains; its $2\times$ recall has a small exception (28.4 vs. 28.5). \end{itemize} \papersource{Paper, Table 1 / Sec. 3.2, PDF pp. 6--7. PG19 column: positions 16K--64K.} \end{frame} \begin{frame}{Larger states delay retrieval failure, but do not remove it} \centering \fig{fig03-niah.png}{4.1cm} \begin{itemize} \item Increasing $E$ improves retrieval at intermediate context lengths. \item At 64K, all four pure Triadic curves remain low; the Transformer retains a clear advantage. \item Book PPL and exact retrieval expose different memory demands. \end{itemize} \papersource{Paper, Fig. 3 / Sec. 3.2, PDF p. 7. Four representative RULER tasks at 1.3B.} \end{frame} \begin{frame}{Continued pretraining can recover much of the benefit} Copy forget gates from $E=1$ to 8 slices; initialize new addresses and convolutions; run long-context extension.\par \medskip \centering \begin{tabular}{llrr} \toprule Scale & GDN variant & PG19 PPL $\downarrow$ & Recall $\uparrow$\\ \midrule 400M & Base & 14.15 & 26.2\\ & Upcycled to $E=8$ & 13.86 & 29.7\\ & $E=8$ from scratch & 13.74 & 33.1\\ \midrule 1.3B & Base & 10.20 & 36.1\\ & Upcycled to $E=8$ & 10.00 & 42.0\\ & $E=8$ from scratch & 9.94 & 44.4\\ \bottomrule \end{tabular}\par \medskip 400M: about 71\% of the long-PPL improvement and 51\% of the recall improvement are recovered. sGLA also benefits. \papersource{Paper, Table 2, PDF p. 7. PG19: 16K--64K. Recovery ratios calculated from table values.} \end{frame} \begin{frame}{Hybrid models benefit more from expanding GDN state} 400M, 3:1 recurrent/softmax blocks, NoPE. Compare a larger KV cache with $E=4$ recurrent state.\par \medskip \centering \begin{tabular}{lrrrr} \toprule Hybrid & MB @64K & PG19 $\downarrow$ & Recall $\uparrow$ & NIAH $\uparrow$\\ \midrule GDN / GQA-8 & 206 & 13.50 & 43.7 & 51.5\\ GDN / GQA-4 & 407 & 13.47 & \textbf{45.3} & 53.0\\ Triadic / GQA-8 & 220 & \textbf{13.39} & 44.5 & \textbf{56.3}\\ \bottomrule \end{tabular} \medskip \begin{itemize} \item At 64K: 46\% less state memory than the larger-KV alternative, with better PPL and NIAH. \item At 4K: 31 MB vs. 30 MB; memory crossover is about 4.6K. \item Recall favors larger KV by 0.8 points (within seed spread, per authors). \end{itemize} \papersource{Paper, Table 3 / Sec. 3.4, PDF p. 8. PG19: 4K--64K. Three-seed averages. Memory excludes weights.} \end{frame} \begin{frame}{Eight times the state adds about 30\% block training time} \begin{columns}[c,onlytextwidth] \column{0.57\textwidth} \fig{fig04-training.png}{5.25cm} \column{0.40\textwidth} \textbf{Measured setting}\\ 1.3B block; H100; batch 4; forward + backward. \medskip Overhead over GDN:\\ $E=2$: 9--11\%\\ $E=4$: 14--15\%\\ $E=8$: 28--30\% \medskip $E=8$ vs. Transformer:\\ 3\% slower at 4K;\\ \alert{$5.1\times$ faster at 64K.} \end{columns} \papersource{Paper, Fig. 4 / Sec. 3.5, PDF p. 8. CuTe vs. FlashQLA/TileLang vs. FlexAttention; not decoding throughput.} \end{frame} \begin{frame}{Both address factorization and activation affect quality} \begin{columns}[T,onlytextwidth] \column{0.48\textwidth} \textbf{Fixed $d_kE=1024$}\\ One scalar forget gate per head. \medskip \begin{tabular}{lrr} \toprule $d_k\times E$ & Wiki & Long PPL\\ \midrule $128\times8$ & 10.80 & 13.70\\ $64\times16$ & 10.84 & 13.73\\ $32\times32$ & 10.90 & 13.78\\ \bottomrule \end{tabular} \medskip Balanced factors save projection parameters but slightly worsen PPL; MLP width is adjusted. \column{0.48\textwidth} \textbf{Second-key/query activation}\\ $E=8$, separate activation study. \medskip \begin{tabular}{lrr} \toprule Activation & Wiki & Long PPL\\ \midrule Softplus & 10.86 & 13.77\\ Sigmoid & 10.86 & 13.77\\ SiLU & 10.98 & 13.84\\ Linear & 11.06 & 13.98\\ \bottomrule \end{tabular} \medskip Nonnegative activations win on PPL. Signed cancellation is the authors' explanation. \end{columns} \papersource{Paper, Table 4 / Sec. 3.6, PDF p. 9. Long PPL: PG19 16K--64K; lower is better.} \end{frame} \begin{frame}{The added slices learn a wider range of timescales} \centering \fig{fig05-analysis.png}{4.0cm} \begin{itemize} \item Fixed final 4K targets: Triadic uses more preceding context. \item Mean write strength $\beta$: $0.31\to0.50$. \item Ranked-slice median half-lives: $0.20$--$1080$ tokens; GDN: $5.1$. \end{itemize} \papersource{Paper, Fig. 5 / Appendix B, PDF pp. 18--19. Half-life measures forget-gate decay, not complete recall lifetime.} \end{frame} \begin{frame}{The gain is a better capacity--parameter trade-off} \begin{block}{What the evidence supports} Structured product addresses enlarge recurrent memory cheaply in parameters and improve long-context PPL and recall. \end{block} \begin{itemize} \item $E=8$: about 1.2--1.3\% more parameters, $8\times$ recurrent state, and about 30\% more measured block training time. \item Upcycling and hybrid results make the idea relevant beyond training a pure model from scratch. \item Fixed memory still limits exact long-context retrieval; hardware efficiency depends on the supplied kernels. \end{itemize} \medskip \alert{For memory research: compare state size and compute alongside parameters.} \papersource{Paper, Tables 1--3, Figs. 3--4, PDF pp. 6--8; Discussion and Conclusion, pp. 9--10.} \end{frame} \begin{frame}{Paper and discussion directions} \textbf{Oliver Sieberling, Bharat Runwal, David Jin, Ryan Chin,\\Rameswar Panda, and Yoon Kim.} \medskip \emph{Triadic Linear Attention: Three-Dimensional Recurrent States for Long-Context Sequence Modeling.} Preprint, 2026. \medskip \href{https://arxiv.org/abs/2609.36529v1}{arXiv:2609.36529v1}\quad \href{https://github.com/OliverSieberling/TriadicLinearAttention}{Author-provided code link} \medskip \begin{itemize} \item Higher-order memories predate this work; the contribution combines modern state management, scale, and efficient kernels. \item Promising extension: grow state during training as contexts grow. \item Open design issue: allocate a fixed budget between recurrent state and exact KV storage. \end{itemize} \papersource{Figures reproduced from the paper's author source under CC BY 4.0. Discussion based on Secs. 3--5.} \end{frame} \appendix \begin{frame}{Backup: the delta correction shares one triangular solve} For a $C=64$ chunk, define $\gamma_e^r=\prod_{j=1}^r\alpha_e^j$ and \[ (\Gamma_e)_{rs}=\begin{cases} \gamma_e^r/\gamma_e^s,&r\ge s,\\0,&r