% Paper discussion based on arXiv:2610.26834v1. Compile with pdfLaTeX.
\documentclass[aspectratio=169,11pt]{beamer}
\usepackage[T1]{fontenc}
\usepackage[utf8]{inputenc}
\usepackage{amsmath,amssymb,graphicx,booktabs}

\definecolor{AcademicBlue}{RGB}{26,57,86}
\definecolor{AcademicTeal}{RGB}{0,105,112}
\setbeamercolor{structure}{fg=AcademicBlue}
\setbeamercolor{frametitle}{fg=AcademicBlue,bg=white}
\setbeamercolor{title}{fg=AcademicBlue}
\setbeamercolor{alerted text}{fg=AcademicTeal}
\setbeamercolor{block title}{fg=white,bg=AcademicBlue}
\setbeamercolor{block body}{fg=black,bg=AcademicBlue!5}
\setbeamertemplate{navigation symbols}{}
\setbeamertemplate{itemize items}[circle]
\setbeamertemplate{footline}{%
  \hfill\usebeamerfont{page number in head/foot}%
  \insertframenumber\,/\,\inserttotalframenumber\hspace{4mm}\vspace{3mm}}
\setbeamersize{text margin left=8mm,text margin right=8mm}
\setbeamerfont{frametitle}{size=\Large,series=\bfseries}
\setbeamerfont{title}{size=\LARGE,series=\bfseries}
\newcommand{\papersource}[1]{\par\smallskip{\scriptsize\color{gray}#1\par}}
\newcommand{\good}[1]{{\color{AcademicTeal}\bfseries #1}}

\title[Kronecker Delta Rule]{Kronecker Delta Rule for Long Context Linear Attention}
\subtitle{KronLSTM: targeted associative memory revision}
\author{Paper by A.-R. Hartl, L. Z\'olyomi, M. List, G. Klambauer, S. B\"ock, and S. Hochreiter}
\date{Research group discussion \quad 22 October 2026\\\href{https://arxiv.org/abs/2610.26834v1}{arXiv:2610.26834v1}}

\begin{document}

\begin{frame}
  \titlepage
\end{frame}

\begin{frame}{The question: can a bounded matrix remember more selectively?}
  \begin{block}{Context}
    Recurrent linear attention stores history in a fixed-size matrix $S$, so its recurrent state does not grow with context length.
  \end{block}
  \begin{itemize}
    \item Additive writes can accumulate without control.
    \item Forget gates suppress old information, including useful long-range facts.
    \item Delta-rule corrections bound memory but can disturb earlier associations sharing a key direction.
  \end{itemize}
  \papersource{Paper, Introduction and Sec.~2.1, PDF pp.~2--3.}
\end{frame}

\begin{frame}{Three updates differ in what they revise}
  \centering
  \includegraphics[width=.88\linewidth,height=.70\textheight,keepaspectratio]{figures/fig-01-revision.pdf}
  \papersource{Paper, Fig.~1, PDF p.~1. Conceptual illustration, not measured interference.}
\end{frame}

\begin{frame}{DeltaNet corrects a full vector; KronLSTM corrects a score}
  \begin{align*}
    \text{DeltaNet:}\quad&S^+=S-\beta(Sk-v)k^\top, \\
    \text{Kron rule:}\quad&S^+=S-\beta(v^\top Sk-c)vk^\top.
  \end{align*}
  \begin{itemize}
    \item $S\in\mathbb R^{d_v\times d_k}$; $k$ is a key, $v$ a value, $c$ a scalar target.
    \item DeltaNet imposes $Sk\approx v$; the Kron rule imposes $v^\top Sk\approx c$.
    \item The Kron correction points along $vk^\top$, a direction specified by the input pair.
  \end{itemize}
  \papersource{Paper, Eqs.~(2)--(3), Sec.~2.1, PDF pp.~3--4.}
\end{frame}

\begin{frame}{The rule is gradient descent on one scalar association}
  \[
    \ell(S)=\tfrac12(v^\top Sk-c)^2,\qquad
    \nabla_S\ell=(v^\top Sk-c)vk^\top.
  \]
  \begin{block}{Correction of the addressed pair}
    $e^+=e\bigl(1-\beta\|k\|^2\|v\|^2\bigr)$, where $e=v^\top Sk-c$.
  \end{block}
  \begin{itemize}
    \item The error contracts for $0<\beta\|k\|^2\|v\|^2<2$.
    \item $\beta^*=1/(\|k\|^2\|v\|^2)$ hits the scalar target in one step.
  \end{itemize}
  \papersource{Paper, Sec.~2.1 and Appendix A3.3, PDF pp.~3--4, 22--24.}
\end{frame}

\begin{frame}{Why ``Kronecker'': a delta rule in matrix space}
  \[
    a=k\otimes v=\operatorname{vec}(vk^\top),\quad
    s=\operatorname{vec}(S),\quad v^\top Sk=a^\top s.
  \]
  \[
    s^+=(I-\beta aa^\top)s+\beta ca.
  \]
  \begin{itemize}
    \item $a$ has $d_kd_v$ components, but no implementation needs to form it explicitly.
    \item $a^\top s=c$ is a hyperplane; at $\beta^*$ the update is its orthogonal projection.
    \item That step is the smallest exact correction in Frobenius norm.
  \end{itemize}
  \papersource{Paper, Eq.~(4), Sec.~2.1, PDF p.~4; Appendix A3.4, PDF pp.~26--27.}
\end{frame}

\begin{frame}{Interference requires overlap in both key and value}
  \[
    \Delta(v_i^\top Sk_i)=-\beta e_t\,
    \underbrace{(v_i^\top v_t)}_{\text{value overlap}}
    \underbrace{(k_t^\top k_i)}_{\text{key overlap}}.
  \]
  \begin{itemize}
    \item Orthogonality in either factor protects an old scalar association.
    \item At the exact-correction step, the paper proves minimum expected interference for independent isotropic probe keys and values.
    \item The formula is exact; the statistical optimality statement needs its probe assumptions.
  \end{itemize}
  \papersource{Paper, Eq.~(5), PDF p.~4; Appendix A3.4, PDF p.~27.}
\end{frame}

\begin{frame}{One key can support two orthogonal value directions}
  \begin{columns}[T]
    \column{.52\linewidth}
    \begin{block}{Teaching example}
      $k=e_1$, initially $S=e_1e_1^\top$.\\[1mm]
      Write $(k,e_2)$ with $c=\beta=1$.
    \end{block}
    \[
      S_{\rm Kron}^+=(e_1+e_2)e_1^\top,
      \quad S_{\rm Delta}^+=e_2e_1^\top.
    \]
    \column{.45\linewidth}
    \begin{itemize}
      \item Kron preserves both scalar probes: $e_1^\top S^+e_1=e_2^\top S^+e_1=1$.
      \item Its vector read-out is a \emph{sum}; downstream queries must disambiguate it.
      \item Capacity counts independent scalar constraints, not arbitrary full vectors.
    \end{itemize}
  \end{columns}
  \papersource{Teaching example from Paper Sec.~2.1 and Appendix A3.3, PDF pp.~4, 24--25.}
\end{frame}

\begin{frame}{KronLSTM separates forgetting, erasing, and writing}
  \[
    S_{t+1}=\underbrace{S_tF_t}_{\text{forget}}
      -\underbrace{\beta_t(v_t^\top S_tF_tk_t)v_tk_t^\top}_{\text{erase}}
      +\underbrace{c_tv_tk_t^\top}_{\text{write}}.
  \]
  \begin{itemize}
    \item $F_t=\operatorname{diag}(f_t)$ decays each key channel separately.
    \item Sigmoid $\beta_t,c_t\in(0,1)$ are per-head scalar gates.
    \item $q,k,v$ are $\ell_2$-normalized; read-out still uses an output gate and RMSNorm.
  \end{itemize}
  \papersource{Paper, Eqs.~(10)--(12), Sec.~2.2, PDF pp.~5--6.}
\end{frame}

\begin{frame}{The block keeps only matrix memories as recurrent state}
  \centering
  \includegraphics[width=.79\linewidth,height=.72\textheight,keepaspectratio]{figures/fig-03-block.pdf}
  \papersource{Paper, Fig.~3, PDF p.~6. Projections, short causal convolutions, gates, and per-head memory.}
\end{frame}

\begin{frame}{The theorem gives a conditional second-moment bound}
  \begin{itemize}
    \item Analyze the \emph{ungated} rule with i.i.d. pairs, fixed $\beta$, and $\bar A=\mathbb E[aa^\top]\succ0$.
    \item Mean memory approaches least squares; the state law approaches a unique stationary distribution.
    \item If $\kappa=\beta\max_i\|a_i\|^2\le1$ and $S_0=0$:
  \end{itemize}
  \[
    \mathbb E\|S_t\|_F^2\le
      (1+q_\beta^{t/2})^2\frac{\mathbb E[c^2]}{\lambda_{\min}(\bar A)}
      \le4\frac{\mathbb E[c^2]}{\lambda_{\min}(\bar A)}.
  \]
  \centering $q_\beta=1-\beta(2-\kappa)\lambda_{\min}(\bar A)\in[0,1).$
  \papersource{Paper, Theorems 1--2, PDF pp.~4--5; Appendix A3.5--A3.6, PDF pp.~29--40.}
\end{frame}

\begin{frame}{Interpret the bound at its actual scope}
  \begin{itemize}
    \item At fixed $\beta$, inconsistent associations leave stationary fluctuations of order $O(\beta)$ in covariance.
    \item The bound is on $\mathbb E\|S_t\|_F^2$, with size controlled by feature coverage $\lambda_{\min}(\bar A)$.
    \item The trained network has data-dependent $F_t,\beta_t,c_t$; the theorem does not directly cover every such trajectory.
    \item The high-learning-rate stress test supplies separate empirical evidence of smoother gradients than mLSTM.
  \end{itemize}
  \papersource{Paper, Fig.~2 and Theorems 1--2, PDF pp.~2, 4--5; Appendix A3.5.}
\end{frame}

\begin{frame}{Evaluation matches pretraining budgets across mixers}
  \begin{itemize}
    \item 1.5B parameters, 24 layers, 100B FineWeb-Edu tokens, 8k training context.
    \item Same corpus, tokenizer, optimizer, batch size, and seed in each mixer comparison.
    \item Pure recurrent model; or $5{:}1$ recurrent / sliding-window attention layers (window 2048).
  \end{itemize}
  \begin{block}{Questions asked}
    Six language accuracies + LAMBADA perplexity; six real-world recall tasks at 2k; RULER at 4k, 8k, and 16k.
  \end{block}
  \papersource{Paper, Sec.~3, PDF p.~6; Appendix A7, PDF p.~48.}
\end{frame}

\begin{frame}{Language modeling: highest average in both settings}
  \centering
  \begin{tabular}{lrrrr}
    \toprule
    & \multicolumn{2}{c}{Linear} & \multicolumn{2}{c}{SWA hybrid} \\
    Model & Avg-6 $\uparrow$ & LMB ppl $\downarrow$ & Avg-6 $\uparrow$ & LMB ppl $\downarrow$ \\
    \midrule
    KDA & 58.12 & 12.67 & 57.85 & 12.60 \\
    GDN-2 & 57.94 & 12.54 & 58.79 & 11.85 \\
    \good{KronLSTM} & \good{59.72} & \good{11.72} & \good{59.56} & \good{11.18} \\
    \bottomrule
  \end{tabular}
  \vspace{3mm}
  \begin{itemize}
    \item Linear Avg-6 lead over KDA: \good{+1.60 accuracy points}.
    \item KronLSTM's own hybrid Avg-6 is slightly lower than its linear Avg-6.
  \end{itemize}
  \papersource{Paper, Table 1, PDF p.~7. Avg-6 averages five reasoning accuracies and LAMBADA accuracy.}
\end{frame}

\begin{frame}{Real-world recall: small average lead, mixed task results}
  \centering
  \begin{tabular}{lrr}
    \toprule
    2k recall Avg-6 $\uparrow$ & Linear & SWA hybrid \\
    \midrule
    GDN & 29.81 & 35.27 \\
    Mamba-3 & 28.41 & 36.96 \\
    \good{KronLSTM} & \good{30.31} & \good{37.40} \\
    \bottomrule
  \end{tabular}
  \vspace{3mm}
  \begin{itemize}
    \item KronLSTM wins three of six tasks in each setting.
    \item Linear SWDE/SQuAD/FDA improve; linear DROP is 19.31 vs GDN's 21.18.
    \item Hybrid gains are substantial for all families, so setting matters.
  \end{itemize}
  \papersource{Paper, Table 2, PDF p.~8. Average of SWDE, SQuAD, FDA, TriviaQA, NQ, DROP.}
\end{frame}

\begin{frame}{RULER reveals the benefit and a sharp S1 failure}
  \centering
  \begin{tabular}{lrrrr}
    \toprule
    Linear, 16k evaluation & S1 & S3 & MQ & MV \\
    \midrule
    GDN & 99.2 & 5.4 & 8.6 & 13.8 \\
    KronLSTM & 11.6 & \good{11.8} & \good{12.7} & \good{17.6} \\
    \bottomrule
  \end{tabular}
  \vspace{3mm}
  \begin{itemize}
    \item Models trained at 8k; 16k is length extrapolation.
    \item S1 repeats artificial noise sentences; the authors suggest recurring-key residual accumulation.
    \item Natural-text needles favor KronLSTM more consistently.
  \end{itemize}
  \papersource{Paper, Table 3 and Sec.~3.2, PDF p.~9. All cells are accuracy percentages.}
\end{frame}

\begin{frame}{32k continuation improves natural-text retrieval}
  \begin{itemize}
    \item Branch at 75B tokens; continue 25B at 32,768 context on a long-document mix.
    \item Token-matched comparison with GDN-2; architecture and tokenizer unchanged.
  \end{itemize}
  \centering
  \begin{tabular}{lrrrr}
    \toprule
    Linear RULER & \multicolumn{2}{c}{16k evaluation} & \multicolumn{2}{c}{32k evaluation} \\
    Model & S3 & MV & S3 & MV \\
    \midrule
    GDN-2 & 7.4 & 8.9 & 0.8 & 7.2 \\
    \good{KronLSTM} & \good{41.8} & \good{30.1} & \good{8.6} & \good{11.3} \\
    \bottomrule
  \end{tabular}
  \vspace{3mm}
  \papersource{Paper, Sec.~3.3, PDF p.~9; Appendix Table A4, PDF p.~47. Accuracy \%.}
\end{frame}

\begin{frame}{Length extension still leaves difficult 32k cases}
  \begin{itemize}
    \item At 32k, KronLSTM's five natural-text NIAH accuracies are only 3.0--11.3\%.
    \item It leads GDN-2 on S2/S3/MK/MQ/MV, while GDN-2 wins artificial S1 (97.8 vs 81.4\%).
    \item The long-document mixture changes along with sequence length, so the gain cannot be assigned to length alone.
  \end{itemize}
  \begin{block}{Reading the claim}
    Better relative retention at long contexts; retrieval remains hard in absolute terms.
  \end{block}
  \papersource{Paper, Fig.~4, PDF p.~10; Appendix Table A4 and pretraining details, PDF pp.~47--48.}
\end{frame}

\begin{frame}{State tracking is competitive on two of three tasks}
  \centering
  \includegraphics[width=.94\linewidth,height=.67\textheight,keepaspectratio]{figures/fig-05-state-tracking.pdf}
  \papersource{Paper, Fig.~5, PDF p.~10. Trained up to length 128, evaluated to 1024; mean $\pm$ SD over 3 seeds. State-tracking KronLSTM uses $\beta\in(0,2)$.}
\end{frame}

\begin{frame}{Write-gate ablation isolates the retrieval gain}
  \centering
  \begin{tabular}{lrrrr}
    \toprule
    1.5B linear & Avg-6 & NIAH @4k & @8k & @16k \\
    \midrule
    Base KronLSTM & 59.72 & \good{59.5} & 30.5 & \good{14.3} \\
    No $\ell_2$ norm on $v$ & 59.41 & 39.6 & \good{32.7} & 12.6 \\
    Coupled erase/write & \good{60.00} & 46.9 & 29.4 & 11.8 \\
    Fixed unit target & 58.17 & 41.9 & 26.2 & 10.1 \\
    \bottomrule
  \end{tabular}
  \vspace{2mm}
  \begin{itemize}
    \item A learned, independent write target matters especially for retrieval.
    \item The coupled variant is slightly better on language Avg-6.
  \end{itemize}
  \papersource{Paper, Appendix Table A3, PDF p.~47. NIAH values average six tasks at each length.}
\end{frame}

\begin{frame}{Throughput is above GDN-2, below GDN}
  \centering
  \includegraphics[width=.70\linewidth,height=.67\textheight,keepaspectratio]{figures/fig-06-throughput.pdf}
  \papersource{Paper, Fig.~6 and Sec.~3.5, PDF p.~10. Single H100-80GB; 16,384 tokens/step; KronLSTM 36.8--44.5k tokens/s.}
\end{frame}

\begin{frame}{Take-home messages}
  \begin{enumerate}
    \setlength{\itemsep}{3mm}
    \item A scalar key--value target makes each memory correction local in \emph{both} key and value directions.
    \item Conditional theory bounds the ungated rule's expected squared memory norm; the full network is supported by design constraints and experiments.
    \item At matched pretraining budgets, KronLSTM improves average language and recall scores and often long natural-text retrieval.
    \item Artificial repeated-noise S1 and low absolute 32k retrieval remain the main practical cautions.
  \end{enumerate}
  \papersource{Paper, Secs.~2--4; Tables 1--4 and Appendix Table A4.}
\end{frame}

\appendix

\begin{frame}{Backup: chunkwise parallelization without explicit $k\otimes v$}
  \begin{itemize}
    \item For chunk token $r$, write coefficient $u_r=c_r-\beta_r v_r^\top S_{r-1}F_rk_r$.
    \item Inside a chunk, earlier writes affect $u_r$ through key and value Gram products.
  \end{itemize}
  \[
    \bigl[I+\operatorname{diag}(\beta)(G^{kk}\odot G^{vv})\bigr]u
      =c-\beta\odot p.
  \]
  \begin{itemize}
    \item $G^{kk}$ is strictly lower triangular and decay-weighted; $G^{vv}=VV^\top$.
    \item The system is unit lower triangular; only matrix states pass between chunks.
  \end{itemize}
  \papersource{Paper, Appendix A8, PDF pp.~49--51.}
\end{frame}

\begin{frame}{Backup: retention depends on the feature spectrum}
  \[
    \mathbb E[s_t]=(I-\beta\bar A)^ts_0+
      \bigl[I-(I-\beta\bar A)^t\bigr]s_{\rm LS}.
  \]
  \begin{itemize}
    \item Initial-memory modes decay at rates $(1-\beta\lambda_j(\bar A))^t$.
    \item If keys and values are independent, $\bar A=\mathbb E[kk^\top]\otimes\mathbb E[vv^\top]$.
    \item Value-direction diversity can slow forgetting; the result needs the stated independence and non-expansive step conditions.
  \end{itemize}
  \papersource{Paper, Appendix A5, PDF pp.~44--46.}
\end{frame}

\end{document}