% Compile with pdflatex. Replace the demonstration content before delivery.
\documentclass[aspectratio=169,11pt]{beamer}
\usepackage[T1]{fontenc}
\usepackage[utf8]{inputenc}
\usepackage{amsmath,amssymb,graphicx,booktabs}

\definecolor{AcademicBlue}{RGB}{26,57,86}
\definecolor{AcademicTeal}{RGB}{0,105,112}
\setbeamercolor{structure}{fg=AcademicBlue}
\setbeamercolor{frametitle}{fg=AcademicBlue,bg=white}
\setbeamercolor{title}{fg=AcademicBlue}
\setbeamercolor{alerted text}{fg=AcademicTeal}
\setbeamercolor{block title}{fg=white,bg=AcademicBlue}
\setbeamercolor{block body}{fg=black,bg=AcademicBlue!5}
\setbeamertemplate{navigation symbols}{}
\setbeamertemplate{itemize items}[circle]
\setbeamertemplate{footline}{%
  \hfill\usebeamerfont{page number in head/foot}%
  \insertframenumber\,/\,\inserttotalframenumber\hspace{4mm}\vspace{3mm}}
\setbeamersize{text margin left=8mm,text margin right=8mm}
\setbeamerfont{frametitle}{size=\Large,series=\bfseries}
\setbeamerfont{title}{size=\LARGE,series=\bfseries}
\newcommand{\papersource}[1]{\par\smallskip{\scriptsize\color{gray}#1\par}}

\title[Memory by Design]{Memory by Design:\\Probabilistic Sequence Layers}
\subtitle{Paper discussion: covariance as a memory-writing rule}
\author{Paper: Matthew Dowling, Hyungju Jeon,\\Cristina Savin, Il Memming Park}
\date{arXiv:2605.31163v1 (29 May 2026)\\Reading: 22 September 2026}
\begin{document}
\begin{frame}\titlepage\end{frame}
\begin{frame}{The question is where a new write should go}
\begin{itemize}
\item Fixed-size recurrent memory can confuse nearby addresses.
\item A smaller scalar gain slows both overwriting and new learning.
\item Propagated covariance can redirect the write away from resolved directions.
\end{itemize}\begin{block}{Central claim}Store the association and the uncertainty geometry that should govern future writes.\end{block}
\papersource{Sec. 1--2, PDF pp. 1--5.}
\end{frame}
\begin{frame}{A design model separates writing from reading}
\begin{center}\includegraphics[width=.97\linewidth,height=.59\textheight,keepaspectratio]{figures/figure_0_compact.pdf}\end{center}\begin{itemize}
\item Inputs parameterize an auxiliary model; its filter updates memory.
\item A query-dependent readout produces the deterministic layer output.
\end{itemize}
\papersource{Fig. 1, PDF p. 2. Original author figure.}
\end{frame}
\begin{frame}{Bayesian here describes the layer recurrence}
\begin{itemize}
\item Inputs $x_t$ are controls, not samples generated by the design model.
\item Memory belief: $\operatorname{vec}(S_t)\sim\mathcal N(\operatorname{vec}(M_t),I_m\otimes P_t)$.
\item $M_t\in\mathbb R^{D\times m}$ stores content; $P_t\in\mathbb R^{D\times D}$ stores directional uncertainty.
\item Task training learns the maps that produce keys, values and dynamics.
\end{itemize}
\papersource{Sec. 2--2.1, PDF pp. 2--4. This is not a Bayesian posterior over all network weights.}
\end{frame}
\begin{frame}{Linear--Gaussian assumptions determine the write}
\[
S_t\mid S_{t-1}\sim\mathcal N(A_tS_{t-1},\ell_t^2I),\qquad
v_t\mid S_t\sim\mathcal N(S_t^\top k_t,r_t^2I).
\]\begin{itemize}
\item $A_t$: memory dynamics; $k_t\in\mathbb R^D$: write key.
\item $v_t\in\mathbb R^m$: pseudo-observed value; $r_t^2$: distrust of the write.
\item $\ell_t^2$: process noise replenishes uncertainty.
\item Matrix-Gaussian shorthand: columns share a key-space covariance.
\end{itemize}
\papersource{Definition 1, Sec. 2.1, PDF pp. 3--4.}
\end{frame}
\begin{frame}{Prediction creates an uncertainty-weighted gain}
\begin{align*}
\bar M_t&=A_tM_{t-1}, &\bar P_t&=A_tP_{t-1}A_t^\top+\ell_t^2I,\\
u_t&=\bar P_tk_t, &\beta_t&=(r_t^2+k_t^\top u_t)^{-1},\\
K_t&=\beta_tu_t.
\end{align*}\begin{itemize}
\item $u_t$ changes the direction, not just the magnitude, of a write.
\item $\beta_t$ is innovation precision; $K_t$ is the vector Kalman gain.
\item Initialize $M_0=0$ and $P_0=p_0I$.
\end{itemize}
\papersource{Proposition 1, Eqs. (9)--(11), PDF p. 4; Appendix B, pp. 13--14.}
\end{frame}
\begin{frame}{The mean corrects errors; covariance records evidence}
\begin{align*}
e_t&=v_t-\bar M_t^\top k_t,\\
M_t&=\bar M_t+K_te_t^\top,\\
P_t&=\bar P_t-\beta_tu_tu_t^\top,\qquad y_t=M_t^\top q_t.
\end{align*}\begin{itemize}
\item Conditioning reduces uncertainty along the observed direction.
\item The query reads the mean and does not enter the write update.
\item Output uncertainty is not used by the evaluated downstream losses.
\end{itemize}
\papersource{Eqs. (8), (12)--(13), PDF p. 4; Appendix B; limitations, p. 10.}
\end{frame}
\begin{frame}{Covariance rotates writes away from confident memories}
\begin{center}\includegraphics[width=.97\linewidth,height=.59\textheight,keepaspectratio]{figures/figure_2_geometry.pdf}\end{center}\[k_A=\rho k_B+\sqrt{1-\rho^2}k_\perp,\quad u_A\approx\rho\bar\sigma_B^2k_B+\sqrt{1-\rho^2}\bar\sigma_\perp^2k_\perp.\]
\papersource{Fig. 2 and Eq. (15), PDF p. 5. Original author figure.}
\end{frame}
\begin{frame}{Two existing families require two different explanations}
\begin{tabular}{p{.23\linewidth}p{.65\linewidth}}
\toprule Family & Mechanism \\\midrule
Bayesian Layer & Propagate covariance; correct along $\bar P_tk_t$.\\[5pt]
Delta-rule & Reset $\bar P_t=\lambda_t I$; correct along raw $k_t$.\\[5pt]
Additive & Observe a transient write coefficient in a different design model.\\\bottomrule
\end{tabular}\begin{itemize}
\item DeltaNet / GDN / KDA differ in the transition structure.
\item Linear Attention / RetNet / diagonal GLA / SSD arise from additive means.
\end{itemize}
\papersource{Table 1, PDF p. 6. These are core-recurrence correspondences, not full-network equivalences.}
\end{frame}
\begin{frame}{Covariance reset recovers the Delta-rule form}
\[
\eta_t=\frac{\lambda_t}{r_t^2+\lambda_t\|k_t\|^2},\qquad
M_t=\bar M_t+\eta_tk_t(v_t-\bar M_t^\top k_t)^\top.
\]\begin{itemize}
\item $A_t=I$: DeltaNet; $A_t=\alpha_tI$: Gated DeltaNet.
\item $A_t=\operatorname{diag}(\alpha_t)$: KDA core.
\item The reset preserves error correction but removes covariance history.
\end{itemize}
\papersource{Proposition 2 and Eq. (17), PDF p. 6.}
\end{frame}
\begin{frame}{Three evaluations form an evidence ladder}
\begin{enumerate}
\item Controlled collisions: test the predicted mechanism with fixed addresses, then train and extrapolate.
\item MQAR: learn associations from discrete tokens; stress updates and delayed values.
\item A 340M pretrained model: replace four layers, distill, and measure retrieval plus language-modeling cost.
\end{enumerate}
\papersource{Sec. 4, PDF pp. 7--9; full protocols in Appendices E--G.}
\end{frame}
\begin{frame}{Boost B, flood A, then query B}
\begin{itemize}
\item Two unit keys overlap by $\rho=0.92$; values are one-hot.
\item $D=16$, $m=6$, $P_0=3I$, $r^2=\ell^2=0.05$, $A=I$.
\item 12 seed writes, 40 B-boost writes, 60 A-flood writes.
\item Compare full covariance with reset, raw-key updates and a diagonal ablation.
\end{itemize}\[p_B^{AB}=\frac{\exp(y_B)}{\exp(y_A)+\exp(y_B)}\quad\text{(pairwise probability, not classification accuracy).}\]
\papersource{Sec. 4.1, PDF p. 7; Appendix E.3--E.4, pp. 18--20.}
\end{frame}
\begin{frame}{The controlled curves support directional protection}
\begin{center}\includegraphics[width=.97\linewidth,height=.58\textheight,keepaspectratio]{figures/experiment_1_2_compact.pdf}\end{center}\begin{itemize}
\item Left: retrieval and directional uncertainty. Right: learned extrapolation.
\item Panel (c) is labeled accuracy; panel (d) is labeled margin.
\end{itemize}
\papersource{Fig. 3, PDF p. 7. Original figure; v1 metric definitions are not fully consistent.}
\end{frame}
\begin{frame}{Exact limits show protection with a small residual loss}
\begin{center}\begin{tabular}{lrr}\toprule
Pairwise B probability & Bayesian & Reset / Delta\\\midrule
Infinite B-boost limit & 0.73106 & 0.73106\\
A-flood limit from actual pre-flood state & 0.72309 & 0.35138\\\bottomrule
\end{tabular}\end{center}\begin{itemize}
\item $0.73106=\operatorname{sigmoid}(1)$ comes from the one-hot logit gap.
\item Flood gain approaches $0.618$, not zero; the innovation vanishes.
\item At $\rho=.92$, cross-direction variance grows by $0.00768$ per step asymptotically.
\end{itemize}
\papersource{Appendix E.1--E.3, PDF pp. 16--19. Conflicting tables on p. 20 are not used here.}
\end{frame}
\begin{frame}{Learned tests stress length and address overlap}
\begin{itemize}
\item Train: $n_f\le8$, $\rho\in[.60,.80]$; 8 pairs in 16 dimensions.
\item Test in Fig. 3: $n_f=256$ (32$\times$) and overlap up to $.95$.
\item Addresses bypass learned projections; values are re-assigned each sequence.
\item 2 layers, width 64; 2,500 steps; 3 seeds with one-standard-deviation bars.
\item BL retains its advantage; reset and Delta-style fail at high overlap.
\end{itemize}
\papersource{Fig. 3, PDF pp. 7--8; Appendix E.5, pp. 21--24. Appendix lists only 16/32/64 for flood test settings.}
\end{frame}
\begin{frame}{Random-key crowding still degrades every finite memory}
\begin{center}\includegraphics[width=.97\linewidth,height=.55\textheight,keepaspectratio]{figures/appendix_experiment_2_random_key_crowding.pdf}\end{center}\begin{itemize}
\item Fixed projected addresses, $D=16$; test occupancy $n/D=6,8,12,16$.
\item Lower query CE favors BL; this does not remove the linear interpolation limit.
\end{itemize}
\papersource{Fig. 7, PDF p. 24; Appendix E.5.4, p. 23. Original author figure.}
\end{frame}
\begin{frame}{MQAR tests learned binding, updating and delayed values}
\begin{center}\includegraphics[width=.97\linewidth,height=.57\textheight,keepaspectratio]{figures/figure_4_mqar.pdf}\end{center}\begin{itemize}
\item Base: competitive near-perfect recall at larger widths.
\item Update: approximately 88\% BL vs. 60\% GDN/Mamba-2 in the caption.
\item Block: both convolution-free BL and GDN bridge delayed values.
\end{itemize}
\papersource{Fig. 4, PDF p. 8. Approximate percentages, not an average across all configurations.}
\end{frame}
\begin{frame}{Four Bayesian layers are distilled into a 340M GDN}
\begin{itemize}
\item Replace layers $\{5,11,17,23\}$; training context is 2,048.
\item Phase A: about 100M tokens per layer, parallel hidden-state matching.
\item Phase B: about 300M tokens, teacher-logit KL plus boundary MSE.
\item GDN-FT control: same reported 400M-token budget and trainable mask, but next-token CE.
\item One training seed; token matching is not a reported end-to-end FLOP benchmark.
\end{itemize}
\papersource{Sec. 4.3, PDF pp. 8--9; Appendix G, pp. 25--27.}
\end{frame}
\begin{frame}{Mean retrieval improves, but 16k performance remains low}
\begin{center}\begin{tabular}{rrrrr}\toprule
Context & GDN-zero & GDN-FT & BL & BL$-$zero\\\midrule
2k &49.52&48.46&50.81&+1.29\\
4k &31.08&31.58&34.02&+2.94\\
8k &16.81&16.06&20.00&+3.19\\
16k &7.85&8.36&8.81&+0.96\\\bottomrule
\end{tabular}\end{center}\begin{itemize}
\item Mean across six RULER NIAH tasks; CWE excluded.
\item Scores are percentages; differences are percentage points.
\item At 2k, single-2 drops from 96.50 to 89.00 (7.50 points).
\end{itemize}
\papersource{Table 4 (Appendix Table A1), PDF p. 26.}
\end{frame}
\begin{frame}{Position-dependent gains are based on sparse cells}
\begin{center}\includegraphics[width=.97\linewidth,height=.56\textheight,keepaspectratio]{figures/fig5_tradeoff_position_task_composite.pdf}\end{center}\begin{itemize}
\item Green favors BL; depth increases from early to late context.
\item 208 of 240 task/length/depth cells contain fewer than 30 samples.
\end{itemize}
\papersource{Fig. 5, PDF p. 9; sample counts, Appendix G, p. 27. Avoid treating peaks as precise population effects.}
\end{frame}
\begin{frame}{Retrieval gains have a language-modeling cost}
\begin{center}\begin{tabular}{lrrr}\toprule
8k context & GDN-zero PPL & BL PPL & Relative increase\\\midrule
WikiText-103 &14.57&14.97&2.75\%\\
SlimPajama-heldout &12.15&12.43&2.30\%\\
PG-19 validation &19.29&19.81&2.70\%\\\bottomrule
\end{tabular}\end{center}\begin{itemize}
\item PPL is lower-is-better; the gap is roughly flat across context lengths.
\item Table-derived range is about 2.3--2.75\%, broader than the prose summary.
\item This is a retrieval/quality trade-off, not a universal improvement.
\end{itemize}
\papersource{Table 5 (Appendix Table A2), PDF p. 27. Percentages recalculated from printed PPL values.}
\end{frame}
\begin{frame}{The strongest result is the memory-writing mechanism}
\begin{itemize}
\item Covariance history changes where errors are written.
\item Resetting that history recovers familiar Delta-rule dynamics.
\item Controlled tests are the clearest evidence; pretrained-model gains are modest and task-specific.
\item Open questions: state cost, throughput, larger scales, multi-seed replication and uncertainty-aware readout.
\end{itemize}
\papersource{Sec. 5, PDF pp. 9--10; interpretation of the evidence in this reading.}
\end{frame}
\begin{frame}{Reference and scope of this discussion}
\textbf{Memory by Design: Probabilistic Sequence Layers}\\[5pt]
Matthew Dowling, Hyungju Jeon, Cristina Savin, Il Memming Park.\\[5pt]
\href{https://arxiv.org/abs/2605.31163v1}{arXiv:2605.31163v1}, submitted 29 May 2026.\\[9pt]
Based on the version-matched 27-page PDF, author TeX and original figures.\\[6pt]
Main text and Appendices A--G were read. No model retraining was performed.\\[6pt]
Companion Chinese notes contain full protocols, derivations and v1 consistency checks.

\papersource{Prepared for paper discussion on 22 September 2026. Later arXiv versions are outside scope.}
\end{frame}
\begin{frame}{Backup: a scalar Riccati map gives the gain floor}
\[
x_{t+1}=\frac{r^2x_t}{r^2+x_t}+\ell^2,\qquad
x_\infty=\frac{\ell^2+\sqrt{\ell^4+4r^2\ell^2}}2,
\]
\[
g_\infty=\frac{x_\infty}{r^2+x_\infty},\qquad
\Delta_B\longrightarrow(1-\rho^2)\ell^2.
\]\begin{itemize}
\item Assumptions: $A=I$, a repeated unit key, constant positive noise.
\item Cross-covariance with the orthogonal subspace contracts to zero.
\item A positive gain floor is compatible with a vanishing innovation.
\end{itemize}
\papersource{Appendix E.1--E.2, PDF pp. 15--17; Eq. (42).}
\end{frame}
\begin{frame}{Backup: efficient filtering still carries a covariance cost}
\begin{itemize}
\item Base state and per-token work: $O(Dm+D^2)$ with diagonal dynamics.
\item Grouped noise: $G$ covariance blocks, state $O(Dm+GD^2)$.
\item Compute gain trajectories, then apply affine mean-update machinery.
\item Chunks inherit boundary states sequentially; covariance arithmetic uses fp32.
\item The implementation also describes damped rotations and real-row noise injection.
\end{itemize}
\papersource{Appendix D, PDF pp. 14--15. Reported implementation details are not a measured throughput comparison.}
\end{frame}
\begin{frame}{Backup: important v1 inconsistencies remain unresolved}
\begin{itemize}
\item Gain table says about 2.0, but $g=x/(r^2+x)<1$ (p. 20 vs. p. 17).
\item Flood margins +0.20 / -0.44 conflict with the exact-limit section and overlap table.
\item Pairwise vs. full-vocabulary softmax margins differ; Fig. 3(c) says accuracy.
\item The additive mean closes without covariance, but latent write noise still implies uncertain memory.
\end{itemize}
\papersource{PDF pp. 6, 8, 17--20, 24. Reader checks; these are not claimed author corrections.}
\end{frame}
\end{document}