% Based on the arxiv-paper-reader Beamer template. Compile with pdfLaTeX.
\documentclass[aspectratio=169,11pt]{beamer}
\usepackage[T1]{fontenc}
\usepackage[utf8]{inputenc}
\usepackage{amsmath,amssymb,graphicx,booktabs}
\definecolor{AcademicBlue}{RGB}{26,57,86}
\definecolor{AcademicTeal}{RGB}{0,105,112}
\setbeamercolor{structure}{fg=AcademicBlue}
\setbeamercolor{frametitle}{fg=AcademicBlue,bg=white}
\setbeamercolor{title}{fg=AcademicBlue}
\setbeamercolor{alerted text}{fg=AcademicTeal}
\setbeamercolor{block title}{fg=white,bg=AcademicBlue}
\setbeamercolor{block body}{fg=black,bg=AcademicBlue!5}
\setbeamertemplate{navigation symbols}{}
\setbeamertemplate{itemize items}[circle]
\setbeamertemplate{footline}{\hfill\usebeamerfont{page number in head/foot}\insertframenumber\,/\,\inserttotalframenumber\hspace{4mm}\vspace{3mm}}
\setbeamersize{text margin left=8mm,text margin right=8mm}
\setbeamerfont{frametitle}{size=\Large,series=\bfseries}
\setbeamerfont{title}{size=\LARGE,series=\bfseries}
\newcommand{\papersource}[1]{\par\smallskip{\scriptsize\color{gray}#1\par}}
\newcommand{\diag}{\operatorname{diag}}
\newcommand{\KL}{\operatorname{KL}}
\newcommand{\tr}{\operatorname{tr}}
\newcommand{\R}{\mathbb R}
\title[KDN paper discussion]{Kalman Delta Networks}
\subtitle{Uncertainty-aware Associative Memory}
\author{Paper: Ngoc Bui, Tinglin Huang, Rex Ying\\Yale University}
\date{arXiv:2609.07816v1 \;\textbullet\; 7 September 2026\\Independent reading presentation \;\textbullet\; 22 September 2026}

\begin{document}
\begin{frame}
\titlepage
\vspace{-5mm}
\begin{center}
\alert{Track confidence in a recurrent memory to decide how strongly to edit it.}\\[2mm]
{\small 22 main slides for a 20--30 minute discussion; 4 backup slides.}\\
{\scriptsize\url{https://arxiv.org/abs/2609.07816v1}}
\end{center}
\end{frame}

\begin{frame}{A fixed memory must decide what to overwrite}
\begin{block}{The online decision}
Each token edits a compressed prefix before future queries are known.
\end{block}
\[
S_t=D_tS_{t-1}+\beta_tk_t\bigl(v_t-(D_tS_{t-1})^\top k_t\bigr)^\top
\]
\begin{itemize}
\item $S_t\in\R^{d_k\times d_v}$ stores a key--value map; read $o_t=S_t^\top q_t$.
\item A large residual can mean stale memory or an unreliable observation.
\item A token-predicted gate has no explicit covariance state recording prior support.
\end{itemize}
\papersource{Paper, Secs. 1--2, PDF pp. 1--4. ``Fixed gain'' does not mean a constant gate across tokens.}
\end{frame}

\begin{frame}{Separate the memory transition from the write gain}
\centering
\begin{tabular}{lll}
\toprule
Model & Transition $D_t$ & Gain $\kappa_t$\\
\midrule
DeltaNet & $I$ & $\beta_t k_t$\\
Gated DeltaNet & $\alpha_t I$ & $\beta_t k_t$\\
KDA & $\diag(\alpha_t)$ & $\beta_t k_t$\\
Isotropic KDN & $\diag(\alpha_t)$ & covariance-derived $\beta_t k_t$\\
Diagonal KDN & $\diag(\alpha_t)$ & covariance-derived vector\\
\bottomrule
\end{tabular}
\medskip
\begin{block}{What changes in KDN?}
Propagate uncertainty with the transition; derive the residual gain from uncertainty and observation noise.
\end{block}
\papersource{Paper, Sec. 3.3, Eqs. (18)--(20), PDF pp. 7--8; Sec. 4, pp. 8--11.}
\end{frame}

\begin{frame}{Treat each key--value pair as a noisy measurement}
\[
\widetilde S_t=D_t\widetilde S_{t-1}+W_t,
\qquad W_t\sim\mathcal N_{\mathrm{col}}(0,\Omega_t)
\]
\[
v_t=\widetilde S_t^\top k_t+e_t,
\qquad e_t\sim\mathcal N(0,r_tI_{d_v})
\]
\begin{itemize}
\item $\widetilde S_t$: latent map; $S_t$: its estimated mean.
\item $P_t\in\R^{d_k\times d_k}$: key-space covariance shared by independent value columns.
\item $\Omega_t$: uncertainty from drift; $r_t$: reliability of the current value.
\end{itemize}
\begin{block}{Scope of the model}
Linear--Gaussian dynamics with conditionally independent noises. These are modeling assumptions, not measured laws of language.
\end{block}
\papersource{Paper, Sec. 3.1, Eqs. (7), (9)--(11), PDF pp. 4--6.}
\end{frame}

\begin{frame}{The Kalman gain balances uncertainty and reliability}
\[
\widehat S_t=D_tS_{t-1},\qquad
\widehat P_t=D_tP_{t-1}D_t^\top+\Omega_t
\]
\[
\kappa_t=\frac{\widehat P_tk_t}{r_t+k_t^\top\widehat P_tk_t},\qquad
S_t=\widehat S_t+\kappa_t\underbrace{(v_t-\widehat S_t^\top k_t)^\top}_{\text{innovation}}
\]
\[
P_t=(I-\kappa_tk_t^\top)\widehat P_t,\qquad o_t=S_t^\top q_t
\]
\begin{itemize}
\item Effective same-key write: $\beta_t^{\mathrm{eff}}=z_t/(r_t+z_t)$, where $z_t=k_t^\top\widehat P_tk_t$.
\item Larger directional uncertainty permits a stronger correction.
\end{itemize}
\papersource{Paper, Proposition 3.1, PDF p. 7; Appendix B, pp. 21--22. MMSE optimality is conditional on the assumed model.}
\end{frame}

\begin{frame}{A measurement creates correlations between key channels}
\centering
\includegraphics[width=.94\linewidth,height=.57\textheight,keepaspectratio]{figures/approximate.pdf}
\begin{itemize}
\item Prediction can stay diagonal; the exact posterior is generally dense.
\item Reverse-KL projection restores a diagonal state and preserves the one-step mean.
\end{itemize}
\papersource{Paper, original Fig. 1, PDF p. 5. Exactness refers to the retained predictive prior.}
\end{frame}

\begin{frame}{Diagonal KDN projects precision, not covariance entries}
\[
\widehat p_t=\alpha_t^2\odot p_{t-1}+\omega_t,\qquad
\kappa_t=\frac{\widehat p_t\odot k_t}{r_t+\sum_i\widehat p_{t,i}k_{t,i}^2}
\]
\[
q_t=\arg\min_{q\in\mathcal Q_{\mathrm{col}}}
\KL\bigl(q\,\|\,\mathcal N_{\mathrm{col}}(S_t^\star,P_t^\star)\bigr)
\]
\[
S_t=S_t^\star,\qquad p_{t,i}=\frac{1}{(P_t^{\star-1})_{ii}}
=\left(\widehat p_{t,i}^{-1}+\frac{k_{t,i}^2}{r_t}\right)^{-1}
\]
\begin{itemize}
\item The approximation retains $d_k$ uncertainty scalars per head.
\item One-step mean preservation does not recover the full exact filtering trajectory.
\end{itemize}
\papersource{Paper, Sec. 4.1, Eqs. (23)--(30), PDF pp. 8--9; proof in Appendix A.2, p. 21. Here $\mu=1$.}
\end{frame}

\begin{frame}{Information scaling protects subsequent writes}
\[
u_t=\frac{\mu}{r_t}k_t\odot k_t,\qquad
p_t=\frac{\widehat p_t}{1+u_t\odot\widehat p_t}
\]
\begin{itemize}
\item Compute the current gain from $\widehat p_t$ \emph{before} this update.
\item Larger $\mu$ lowers retained variance and weakens later overlapping writes.
\item $\mu=1$ gives the original variational solution; main runs use $\mu=d_k$.
\end{itemize}
\begin{block}{A balanced-key example}
For $m$ active coordinates with $k_i^2=1/m$ and variance $p$:
\[
V_{\mathrm{exact}}=\frac{rp}{r+p},\qquad
V_{\mathrm{proj}}(\mu)=\frac{mrp}{mr+\mu p}.
\]
$\mu=m$ matches this direction, not the full covariance.
\end{block}
\papersource{Paper, Eq. (34), PDF p. 9; Appendix B, Eqs. (60)--(65), p. 22.}
\end{frame}

\begin{frame}{A nonlinear variance recurrence admits an associative scan}
\[
p_{t,i}=\frac{\alpha_{t,i}^2p_{t-1,i}+\omega_{t,i}}
{u_{t,i}\alpha_{t,i}^2p_{t-1,i}+1+u_{t,i}\omega_{t,i}}
\]
\[
M_{t,i}=\begin{bmatrix}
\alpha_{t,i}^2&\omega_{t,i}\\
u_{t,i}\alpha_{t,i}^2&1+u_{t,i}\omega_{t,i}
\end{bmatrix},\quad
\begin{bmatrix}n_{t,i}\\d_{t,i}\end{bmatrix}
=M_{t,i}\begin{bmatrix}n_{t-1,i}\\d_{t-1,i}\end{bmatrix}
\]
\begin{block}{M\"obius representation}
$p_{t,i}=n_{t,i}/d_{t,i}$. Composition becomes $2\times2$ matrix multiplication: associative, but order-sensitive.
\end{block}
\begin{itemize}
\item First scan uncertainty to obtain gains; then scan the affine memory recurrence.
\item The associative formulation permits $O(\log T)$ parallel depth.
\end{itemize}
\papersource{Paper, Eqs. (32)--(39), PDF pp. 9--11.}
\end{frame}

\begin{frame}{Isotropic KDN trades directionality for a scalar state}
\[
a_t=\frac{1}{d_k}\sum_i\alpha_{t,i}^2,\qquad
\widehat b_t=a_tb_{t-1}+\omega_t,\qquad P_t=b_tI
\]
\[
\beta_t=\frac{\widehat b_t}{r_t+\widehat b_t\|k_t\|^2},\qquad
b_t=\frac{\widehat b_t}{1+\mu\widehat b_t\|k_t\|^2/(d_kr_t)}
\]
\begin{itemize}
\item Prediction: trace projection, preserving the average eigenvalue.
\item Posterior: reverse-KL projection, yielding a harmonic mean when $\mu=1$.
\item The gain stays parallel to $k_t$, but depends on accumulated uncertainty.
\end{itemize}
\begin{block}{State cost per head}
Extra uncertainty: $O(1)$ isotropic or $O(d_k)$ diagonal. Both still store the $O(d_kd_v)$ associative memory.
\end{block}
\papersource{Paper, Sec. 4.2, Eq. (40), PDF p. 11; Appendix C, pp. 23--26.}
\end{frame}

\begin{frame}{The practical kernel uses two chunkwise stages}
\begin{enumerate}
\item \textbf{Gain computation:} reduce token-local M\"obius matrices, obtain exclusive chunk carries, replay chunks to emit $\kappa_t$.
\item \textbf{Memory computation:} use compact WY for the asymmetric delta rule:
\[
S_t=(I-\kappa_tk_t^\top)D_tS_{t-1}+\kappa_tv_t^\top.
\]
\end{enumerate}
\begin{itemize}
\item Gain chunks: $C_g=256$; memory chunks: $C_m\in\{32,64\}$.
\item The implemented short gain carry pass is serial across chunks.
\item FP32 homogeneous rescaling and log-decay differences improve stability.
\end{itemize}
\papersource{Paper, Appendix D, PDF pp. 26--27. Scan compatibility is not a claim of zero serial work in the implementation.}
\end{frame}

\begin{frame}{Evaluation controls data and capacity at two scales}
\centering
\begin{tabular}{lll}
\toprule
 & Smaller run & Larger run\\
\midrule
Parameters / tokens & 750M / 50B & 1.3B / 100B\\
Layers / hidden width & 16 / 2048 & 18 / 2304\\
KDN heads / key width & 16 / 128 & 18 / 128\\
Training data & \multicolumn{2}{c}{FineWeb-Edu, 4K sequences}\\
Hybrid setting & --- & 9 recurrent + 9 SWA layers\\
\bottomrule
\end{tabular}
\medskip
\begin{itemize}
\item Recurrent non-embedding parameters matched within 0.03\%; Mamba-3 states matched too.
\item Evaluate perplexity, six-task zero-shot accuracy, synthetic and real retrieval.
\item Main KDN runs: $\mu=d_k=128$, $r_{\min}=0.01$, learned initial variance per head.
\end{itemize}
\papersource{Paper, Sec. 5.1, PDF pp. 11--13. Hybrid and attention-only quality baselines use 2K sliding windows.}
\end{frame}

\begin{frame}{At 750M, both variants improve recurrent-model quality}
\centering
\begin{tabular}{lrrr}
\toprule
Model (750M / 50B) & Wiki. PPL $\downarrow$ & LMB. PPL $\downarrow$ & Avg. \% $\uparrow$\\
\midrule
KDA & 18.85 & 15.06 & 53.87\\
Mamba-3 MIMO & 18.99 & 15.67 & 54.39\\
Isotropic KDN & \textbf{18.42} & 14.68 & 54.41\\
Diagonal KDN & 18.64 & \textbf{14.15} & \textbf{54.97}\\
\bottomrule
\end{tabular}
\medskip
\begin{itemize}
\item Diagonal mean accuracy: +1.10 percentage points over KDA.
\item Against the strongest non-KDN mean (Mamba-3 MIMO): +0.58 points.
\item Individual tasks differ: Mamba-3 MIMO still leads HellaSwag and ARC-c.
\end{itemize}
\papersource{Paper, selected rows of Table 1, PDF p. 12. Avg.: unweighted mean over six zero-shot tasks.}
\end{frame}

\begin{frame}{At 1.3B, average gains are small and hybrid results vary}
\centering
\begin{tabular}{lrrr}
\toprule
Model (1.3B / 100B) & Wiki. PPL $\downarrow$ & LMB. PPL $\downarrow$ & Avg. \% $\uparrow$\\
\midrule
KDA & 15.40 & 10.09 & 60.28\\
Isotropic KDN & 15.30 & 9.98 & 60.35\\
Diagonal KDN & \textbf{15.04} & \textbf{9.75} & \textbf{60.45}\\
\midrule
KDA + SWA & 15.08 & 10.81 & 59.94\\
Isotropic KDN + SWA & \textbf{14.98} & 10.90 & 59.38\\
Diagonal KDN + SWA & 15.10 & \textbf{10.41} & \textbf{60.11}\\
\bottomrule
\end{tabular}
\par
\medskip
\alert{Diagonal's recurrent mean gain over KDA is only +0.17 points.}
\begin{itemize}
\item No reported multi-seed uncertainty establishes statistical significance.
\item In hybrids, neither KDN variant dominates every aggregate metric.
\end{itemize}
\papersource{Paper, selected rows of Table 1, PDF p. 12. Bold values compare only within each displayed block.}
\end{frame}

\begin{frame}{Diagonal KDN shows stronger synthetic retrieval gains}
\centering
\begin{tabular}{lrr}
\toprule
Recurrent model & 750M / 50B & 1.3B / 100B\\
\midrule
KDA & 62.11 & 64.31\\
GDN-2 & 52.54 & 71.00\\
Isotropic KDN & 62.36 & 68.66\\
Diagonal KDN & \textbf{70.33} & \textbf{76.51}\\
\bottomrule
\end{tabular}
\medskip
\begin{itemize}
\item Values: \textbf{our equal-weight recomputation} of Table 2's 14 cells (\%).
\item At 1.3B: +12.20 points over KDA; +5.51 over the strongest baseline, GDN-2.
\item Not every cell wins: S-NIAH-2 / 8K is 25.8 vs GDN-2's 28.0.
\item Modified RULER sampling, 500 matched examples/cell; not a full-suite score.
\end{itemize}
\papersource{Paper, Table 2 and caption, PDF p. 13. Differences use unrounded means. Contexts span 1K--8K; training is 4K.}
\end{frame}

\begin{frame}{Local attention changes the real-retrieval ranking}
\centering
\begin{tabular}{lrr}
\toprule
Model (1.3B / 100B) & Recurrent & Hybrid + SWA\\
\midrule
KDA & 33.76 & 45.09\\
Mamba-3 MIMO & 32.23 & 44.76\\
GDN-2 & 32.09 & 44.24\\
Isotropic KDN & 33.40 & \textbf{45.94}\\
Diagonal KDN & \textbf{34.86} & 45.55\\
\bottomrule
\end{tabular}
\medskip
\begin{itemize}
\item Six-task mean accuracy (\%); all inputs limited to 2K tokens.
\item Diagonal wins recurrent average; Isotropic wins hybrid average.
\item Hybridization improves Diagonal by 10.69 points, larger than the mixer-only gap.
\end{itemize}
\papersource{Paper, Table 3, PDF p. 14. Tasks: SWDE, SQuAD, FDA, TriviaQA, NQ, DROP. These are short-input retrieval results.}
\end{frame}

\begin{frame}{The best information scale depends on the metric}
\centering
\begin{tabular}{lrrr}
\toprule
$\mu$ & Wiki. PPL $\downarrow$ & LMB. PPL $\downarrow$ & Avg. \% $\uparrow$\\
\midrule
$1$ & 18.69 & 15.51 & 54.12\\
$\sqrt{d_k}$ & 18.93 & 15.10 & 54.20\\
$d_k$ & 18.64 & 14.15 & \textbf{54.97}\\
$4d_k$ & 18.97 & \textbf{13.91} & 54.92\\
Learned & \textbf{18.51} & 14.68 & 54.62\\
\bottomrule
\end{tabular}
\medskip
\begin{itemize}
\item Diagonal KDN, 750M / 50B; learned scale initialized at $d_k$.
\item Larger fixed scales improve LAMBADA PPL monotonically in this sweep.
\item $d_k$ is a practical compromise, not a universal optimum.
\end{itemize}
\papersource{Paper, Table 4, PDF p. 14; $d_k=128$.}
\end{frame}

\begin{frame}{Larger scales reduce the observed effective-write strength}
\centering
\includegraphics[width=.91\linewidth,height=.57\textheight,keepaspectratio]{figures/kdn_overwrite_distributions.pdf}
\begin{itemize}
\item Top: initialization. Bottom: trained. Dashed lines: pooled means.
\item Trained Diagonal means: $0.943\to0.891\to0.826$ as $\mu$ increases.
\item This supports reduced overwrite, not calibrated factual confidence.
\end{itemize}
\papersource{Paper, original Fig. 2, PDF p. 10. Panels have different density-axis ranges. Fixed-weight probe: Appendix B, Table 6, p. 23.}
\end{frame}

\begin{frame}{Learning both noise terms does not win every metric}
\centering
\begin{tabular}{lrrr}
\toprule
Setting ($\mu=1$) & Wiki. PPL $\downarrow$ & LMB. PPL $\downarrow$ & Avg. \% $\uparrow$\\
\midrule
Learn $r_t,\omega_t$ & 18.69 & \textbf{15.51} & \textbf{54.12}\\
Fix $r_t=1$ & \textbf{18.46} & 15.72 & 53.77\\
Fix both at 1 & 18.64 & 15.84 & 53.87\\
\bottomrule
\end{tabular}
\medskip
\begin{itemize}
\item Fixing observation noise improves WikiText, but lowers average accuracy.
\item The evidence supports mixed, metric-dependent effects.
\item This ablation uses $\mu=1$, not the main setting $\mu=d_k$.
\end{itemize}
\begin{block}{Missing comparison}
The paper does not report the full noise-factorial ablation at the main information scale.
\end{block}
\papersource{Paper, Table 5, PDF p. 15.}
\end{frame}

\begin{frame}{Isotropic stays close to KDA; Diagonal costs more}
\begin{columns}[T,onlytextwidth]
\begin{column}{.56\textwidth}
\includegraphics[width=\linewidth,height=.68\textheight,keepaspectratio]{figures/mixer_throughput_scaling.pdf}
\end{column}
\begin{column}{.41\textwidth}
\begin{itemize}
\item One H200; mixer layers only.
\item Batch 4, width 2048; median of 30 runs.
\item BF16 setup; Diagonal uses FP32/TF32x3.
\item Mamba-3 SISO leads at long lengths.
\end{itemize}
\end{column}
\end{columns}
\medskip
\alert{Full attention here; 2K SWA in the quality tables.}
\papersource{Paper, original Fig. 4 and throughput paragraph, PDF p. 15. Beyond 4K: runtime scaling probes, not quality validation. No end-to-end speedup is established.}
\end{frame}

\begin{frame}{The evidence does not imply universally better memory}
\begin{itemize}
\item \textbf{Approximation:} diagonal/scalar states discard correlations; $\mu\ne1$ adds a correction beyond the original variational posterior.
\item \textbf{Statistics:} modest 1.3B accuracy gains without multi-seed uncertainty.
\item \textbf{Architecture dependence:} at S-NIAH-1 / 8K, 1.3B Diagonal scores 99.8 recurrent vs 28.0 hybrid.
\item \textbf{Efficiency:} extra gain scan and precision cost; no measured end-to-end decode benefit.
\item \textbf{Interpretation:} learned covariance is not demonstrated to be calibrated factual confidence.
\end{itemize}
\papersource{Paper, Secs. 4--5, Tables 1--2, PDF pp. 8--15; Appendix D, pp. 26--27. Calibration and generalization cautions are this reading's analysis.}
\end{frame}

\begin{frame}{Carry uncertainty forward, then let evidence control edits}
\begin{enumerate}
\item \textbf{Concept:} interpret delta writes as Kalman innovations in a drifting associative map.
\item \textbf{Algorithm:} compress covariance to recover a M\"obius gain scan and an affine memory scan.
\item \textbf{Evidence:} improved recurrent-model aggregates; the strongest gains appear in selected synthetic retrieval settings.
\end{enumerate}
\begin{block}{Open question}
Can richer covariance and rotating transitions preserve both useful correlations and efficient scans?
\end{block}
\papersource{Paper, Sec. 7, PDF p. 16. Rotation is proposed future work.}
\medskip
{\small Ngoc Bui, Tinglin Huang, Rex Ying. \emph{Kalman Delta Networks: Uncertainty-aware Associative Memory}. arXiv:2609.07816v1, 2026.}
\par{\scriptsize\url{https://arxiv.org/abs/2609.07816v1}}
\end{frame}

\appendix
\begin{frame}{Backup: reverse KL explains the diagonal update}
Let $\Lambda^\star=(P^\star)^{-1}$ and $q=\mathcal N_{\mathrm{col}}(S,\diag(p))$.
\[
\KL(q\|p^\star)=\frac12\tr\bigl((S-S^\star)^\top\Lambda^\star(S-S^\star)\bigr)
\]
\[
\hspace{22mm}+\frac{d_v}{2}\sum_i\bigl(\Lambda_{ii}^\star p_i-\log p_i\bigr)+C.
\]
\begin{itemize}
\item Positive definite $\Lambda^\star$ gives $S=S^\star$.
\item $\partial\KL/\partial p_i=0$ gives $p_i=1/\Lambda_{ii}^\star$.
\item $\Lambda^\star=\widehat P^{-1}+kk^\top/r$ yields the elementwise recurrence.
\end{itemize}
\begin{block}{Which posterior is exact?}
The one formed from the already approximated predictive prior. The mean statement is local to this step.
\end{block}
\papersource{Paper, Appendix A.2, Eqs. (54)--(57), PDF p. 21.}
\end{frame}

\begin{frame}{Backup: protection has a causal ordering}
For a token-local intervention $\mu_t$, hold the predictive prior fixed:
\[
\frac{\partial\kappa_t}{\partial\mu_t}=0,\qquad
\frac{\partial S_t}{\partial\mu_t}=0,\qquad
\frac{\partial p_{t,i}}{\partial\mu_t}=-\frac{k_{t,i}^2}{r_t}p_{t,i}^2\leq0.
\]
With next-token parameters fixed, this implies
\[
\frac{\partial\beta_{t+1}^{\mathrm{eff}}}{\partial\mu_t}\leq0.
\]
\centering
\begin{tabular}{lrrrr}
\toprule
$\mu$ & $1$ & $\sqrt{d_k}$ & $d_k$ & $4d_k$\\
\midrule
Trained-at-scale mean & .943 & .891 & .826 & .805\\
Fixed-weight probe & .886 & .873 & .862 & .857\\
\bottomrule
\end{tabular}
\papersource{Paper, Appendix B, Eqs. (64)--(65), Table 6, PDF pp. 22--23. Probe protocols differ; compare within each row. Fixed weights do not fix downstream hidden states.}
\end{frame}

\begin{frame}{Backup: compact WY handles asymmetric write directions}
Within a chunk, let $\gamma_r=\prod_{j\leq r}\alpha_j$ and
\[
\overline k_r=\gamma_r\odot k_r,\qquad
\overline\kappa_r=\kappa_r\oslash\gamma_r.
\]
Stack rows into $\overline K$, $\overline K^{(\kappa)}$, and $V$. Then
\[
T=\operatorname{tril}\bigl(\overline K(\overline K^{(\kappa)})^\top,-1\bigr),
\quad A=(I+T)^{-1},\quad R=AV-A\overline K S^{\mathrm{in}}.
\]
\begin{itemize}
\item A unit-lower-triangular solve computes chunk-local factors in parallel.
\item Write direction $\kappa_t$ and erase direction $k_t$ need not be parallel.
\item Include the query--write diagonal because output follows the current write.
\item Chunk memory carries remain; decay ratios use log differences for stability.
\end{itemize}
\papersource{Paper, Appendix D.2, PDF p. 27. $R$ here stacks chunk residuals, not the observation-noise covariance.}
\end{frame}

\begin{frame}{Backup: training and evaluation details that affect scope}
\begin{itemize}
\item AdamW: peak LR $4\times10^{-4}$, $\beta=(0.9,0.95)$, weight decay 0.1; gradient clipping 1.0.
\item Cosine schedule; 1\% token-budget warm-up; global batch 0.5M tokens.
\item Learned positive $c_{0,h}$ per head replaces the theoretical unit initial variance.
\item RULER: 25-word increments and deterministic random essay windows; matched keys, values, and depths.
\item Real retrieval: cloze formatting following JRT, inputs limited to 2K tokens.
\item No training reproduction or independent author-code audit in this reading.
\end{itemize}
\papersource{Paper, Sec. 5.1 and Table 2 caption, PDF pp. 11--13; Table 3, p. 14. Sources read: author TeX and all substantive PDF sections/appendices, v1.}
\end{frame}
\end{document}