% Based on the arxiv-paper-reader Beamer template; self-authored discussion.
% Main talk: frames 1--24, approximately 25--30 minutes. Backups: 25--27.
\documentclass[aspectratio=169,11pt]{beamer}
\usepackage[T1]{fontenc}
\usepackage[utf8]{inputenc}
\usepackage{amsmath,amssymb,graphicx,booktabs,lmodern,array}
\definecolor{AcademicBlue}{RGB}{26,57,86}
\definecolor{AcademicTeal}{RGB}{0,105,112}
\setbeamercolor{structure}{fg=AcademicBlue}
\setbeamercolor{frametitle}{fg=AcademicBlue,bg=white}
\setbeamercolor{title}{fg=AcademicBlue}
\setbeamercolor{alerted text}{fg=AcademicTeal}
\setbeamercolor{block title}{fg=white,bg=AcademicBlue}
\setbeamercolor{block body}{fg=black,bg=AcademicBlue!5}
\setbeamertemplate{navigation symbols}{}
\setbeamertemplate{itemize items}[circle]
\setbeamertemplate{footline}{\hspace{4mm}{\color{gray}\scriptsize Palimpsa / arXiv v4}\hfill\usebeamerfont{page number in head/foot}\insertframenumber\,/\,\inserttotalframenumber\hspace{4mm}\vspace{3mm}}
\setbeamersize{text margin left=8mm,text margin right=8mm}
\setbeamerfont{frametitle}{size=\Large,series=\bfseries}
\setbeamerfont{title}{size=\LARGE,series=\bfseries}
\newcommand{\papersource}[1]{\par\smallskip{\scriptsize\color{gray}#1\par}}
\newcommand{\m}{\boldsymbol\mu}
\newcommand{\bt}{\boldsymbol\beta}
\newcommand{\Ip}{\mathbf I_{\mathrm{prior}}}
\newcommand{\pp}{\,\mathrm{pp}}
\title[Palimpsa]{Learning to Remember, Learn, and Forget in Attention-Based Models}
\subtitle{Palimpsa: Bayesian metaplasticity for recurrent attention}
\author{Paper authors: Djohan Bonnet, Jamie Lohoff, Jan Finkbeiner,\texorpdfstring{\\}{ }Elidona Shiqerukaj, and Emre Neftci}
\date{Research group discussion \quad $\vert$ \quad 30 October 2026}
\begin{document}

% 1
\begin{frame}
\titlepage
\vspace{-5mm}
\begin{center}
\alert{Protect important memory states; gradually release old evidence.}
\end{center}
\end{frame}

% 2
\begin{frame}{A dual-state memory that can upgrade Mamba2}
\begin{block}{One-sentence takeaway}
Track both memory content and its precision to adapt each state's plasticity within the context.
\end{block}
\begin{itemize}
\item A discounted Bayesian regression view leads to an efficient approximation.
\item Pretrain a fast non-metaplastic model, then add metaplasticity by fine-tuning.
\item Better recall and selected long-context results come with extra state and kernel cost.
\end{itemize}
\medskip
{\small
Read: \href{https://arxiv.org/abs/2602.09075v4}{arXiv:2602.09075v4}, revised 3 June 2026.\\
Published at \href{https://proceedings.mlr.press/v306/bonnet26a.html}{ICML 2026, PMLR 306:9129--9151}.\\
\href{https://github.com/djo1996/Palimpsa}{Official public implementation} available; status checked 30 October 2026.}
\papersource{Paper, Secs. 1--3, PDF pp. 1--9. All source page numbers refer to arXiv v4.}
\end{frame}

% 3
\begin{frame}{Fixed memory creates a stability--plasticity dilemma}
\begin{columns}[T]
\begin{column}{0.52\textwidth}
\[
\mathbf S_t=\mathbf S_{t-1}+\mathbf v_t\mathbf k_t^\top
\]
For an old key $\mathbf k_a$, the new write adds
\[
\Delta\mathbf y_a=\mathbf v_t(\mathbf k_t^\top\mathbf k_a).
\]
Non-orthogonal keys interfere.
\begin{itemize}
\item Fast writes can erase useful associations.
\item Excessive consolidation can stop new learning.
\end{itemize}
\end{column}
\begin{column}{0.45\textwidth}
\begin{block}{Two controls are needed}
\textbf{Protection:} adapt plasticity to each state's accumulated evidence.\par\medskip
\textbf{Release:} discount stale evidence so the state remains adaptable.
\end{block}
An input gate alone does not store this history.
\end{column}
\end{columns}
\papersource{Paper, Secs. 1 and 2.1, PDF pp. 1--4. Interference expression is an explanatory expansion.}
\end{frame}

% 4
\begin{frame}{The attention state is an online regression model}
\[
\mathbf k_t,\mathbf q_t\in\mathbb R^{d_k},\quad
\mathbf v_t,\bt_t\in\mathbb R^{d_v},\quad
\mathbf S\in\mathbb R^{d_v\times d_k}
\]
\[
-\log p(\mathbf v_t\mid\mathbf k_t,\bt_t,\mathbf S)
=\frac12\sum_i\beta_{t,i}(\mathbf S_i^\top\mathbf k_t-v_{t,i})^2+\mathrm{const.}
\]
\begin{itemize}
\item $\bt_t\geq0$ weights the \emph{current} observation by value channel.
\item A posterior over $\mathbf S$ stores both content and uncertainty.
\item Read with the posterior mean: $\mathbf y_t=\m_t\mathbf q_t$.
\end{itemize}
\begin{block}{Inner vs. outer learning}
The forward pass updates memory states. Backpropagation trains the projections and gates that generate those updates.
\end{block}
\papersource{Paper, Sec. 2.2, Eqs. (1)--(2), PDF p. 4.}
\end{frame}

% 5
\begin{frame}{Forgetting discounts evidence and restores the prior}
\[
p_{w,t}(\mathbf S)\propto
p(\mathbf d_t\mid\mathbf S)\,
\underbrace{p_{w,t-1}(\mathbf S)^{\alpha_t}}_{\text{discounted history}}\,
\underbrace{p_0(\mathbf S)^{1-\alpha_t}}_{\text{prior restoration}}
\]
\[
\alpha_t=e^{-Ad_t}=1-\frac1{N_t},\qquad
N_t=\frac1{1-\alpha_t}
\]
\begin{itemize}
\item Small $\alpha_t$: release historical evidence more quickly.
\item Large $\alpha_t$: retain evidence and build stronger consolidation.
\item For constant $\alpha$, total observation weight is
$\sum_{j=0}^{t-1}\alpha^j\leq N$.
\end{itemize}
\alert{$N_t$ is an effective evidence horizon, not a hard sliding window.}
\papersource{Paper, Sec. 2.2, PDF p. 4; Appendix A.2, Eq. (9), pp. 14--15. Geometric sum: explanatory derivation.}
\end{frame}

% 6
\begin{frame}{Three terms balance learning, protection, and release}
\[
\begin{aligned}
\mathcal F_{t,i}(\m_i)={}&
\underbrace{\frac{\beta_{t,i}}2(\m_i^\top\mathbf k_t-v_{t,i})^2}_{\text{fit the new association}}\\[2mm]
&+\underbrace{\frac{1-\alpha_t}{2\sigma_{\mathrm{prior}}^2}\|\m_i\|^2}_{\text{restore the base prior}}\\[2mm]
&+\underbrace{\frac{\alpha_t}2(\m_i-\m_{t-1,i})^\top
\boldsymbol\Sigma_{t-1,i}^{-1}(\m_i-\m_{t-1,i})}_{\text{protect confident directions}}
+C_{\boldsymbol\Sigma}.
\end{aligned}
\]
\begin{itemize}
\item High historical precision makes a change more expensive.
\item Discounting limits evidence accumulation; the prior supplies a precision floor.
\end{itemize}
\papersource{Paper, Eq. (3), PDF p. 5; Appendix A.3, pp. 15--17. $C_{\boldsymbol\Sigma}$ contains covariance-dependent terms.}
\end{frame}

% 7
\begin{frame}{Palimpsa updates content and importance together}
\[
\boxed{\mathbf I_t=\alpha_t\mathbf I_{t-1}
+(1-\alpha_t)\Ip+\bt_t\otimes\mathbf k_t^2}
\]
\[
\boxed{\m_t=\alpha_t\frac{\mathbf I_{t-1}}{\mathbf I_t}\odot\m_{t-1}
+\frac1{\mathbf I_t}\odot[(\bt_t\odot\mathbf v_t)\otimes\mathbf k_t]}
\]
\begin{itemize}
\item $\mathbf I_t\in\mathbb R^{d_v\times d_k}$ stores per-state precision.
\item Its inverse scales new writes; its ratio also modulates retained content.
\item Squares and divisions are elementwise; $\otimes$ is an outer product.
\end{itemize}
\begin{block}{Approximation that matters}
The implementation discards off-diagonal precision terms from the coupled Gaussian solution. This is stronger than ordinary mean-field VI.
\end{block}
\papersource{Paper, Eq. (5), PDF p. 5; Appendix A.5, Eqs. (28)--(31), pp. 17--18.}
\end{frame}

% 8
\begin{frame}{A confident state resists the same conflicting write}
\textbf{Illustrative scalar example:}
$\mu_{t-1}=1$, $v_t=0$, $k_t=\beta_t=I_{\mathrm{prior}}=1$, $\alpha_t=0.9$.
\medskip
\begin{center}
\begin{tabular}{lrr}
\toprule
Old importance & New importance & New mean \\
\midrule
$I_{t-1}=1$ & $2.0$ & $0.4500$ \\
$I_{t-1}=9$ & $9.2$ & $0.8804$ \\
\bottomrule
\end{tabular}
\end{center}
\medskip
\begin{itemize}
\item Higher importance preserves more of the old value.
\item With no new evidence, excess importance decays:
\end{itemize}
\[
I_t-I_{\mathrm{prior}}=\alpha_t(I_{t-1}-I_{\mathrm{prior}}).
\]
\alert{Consolidation is reversible, so future inputs can still change the state.}
\papersource{Teaching example calculated from paper Eq. (5); not an experiment from the paper.}
\end{frame}

% 9
\begin{frame}{Natural parameters expose two affine scans}
\[
\mathbf M_t=\mathbf I_t\odot\m_t,\qquad
\bar{\mathbf I}_t=\mathbf I_t-\Ip
\]
\[
\begin{aligned}
\mathbf M_t&=\alpha_t\mathbf M_{t-1}+(\bt_t\odot\mathbf v_t)\otimes\mathbf k_t,\\
\bar{\mathbf I}_t&=\alpha_t\bar{\mathbf I}_{t-1}+\bt_t\otimes\mathbf k_t^2,\\
\mathbf y_t&=\left(\frac{\mathbf M_t}{\Ip+\bar{\mathbf I}_t}\right)\mathbf q_t.
\end{aligned}
\]
\begin{itemize}
\item Parallel associative scan within each chunk; sequential carry between chunks.
\item Two recurrent arrays: $O(d_vd_k)$ storage, independent of sequence length.
\item Per-token states needed for readout create register pressure in training.
\end{itemize}
\papersource{Paper, Appendix A.12, PDF pp. 21--22. Recurrences follow by rearranging Eq. (5).}
\end{frame}

% 10
\begin{frame}{Two backbones separate memory rules from architecture}
\begin{center}
\begin{tabular}{>{\raggedright\arraybackslash}p{3.2cm}>{\raggedright\arraybackslash}p{4.4cm}>{\raggedright\arraybackslash}p{4.4cm}}
\toprule
 & \textbf{Palimpsa-M} & \textbf{Palimpsa-D} \\
\midrule
Backbone & Mamba2 style & DeltaNet style \\
\addlinespace
Separate MLP & No & Yes \\
\addlinespace
Input/forgetting gates & Coupled through $d_t$ & Separate $b_t$ \\
\addlinespace
Query/key L2 norm & No & Yes \\
\addlinespace
Without Meta & Mamba2 & D-backbone ablation \\
\bottomrule
\end{tabular}
\end{center}
\medskip
\alert{Palimpsa-D without metaplasticity is a separate ablation, not the Gated Deltanet baseline.}
\papersource{Paper, Sec. 2.2, PDF p. 5; Appendix A.1 and Fig. 5, p. 14.}
\end{frame}

% 11
\begin{frame}{The Mamba2 limit enables a gradual model upgrade}
For shared scalar prior precision $I_{\mathrm{prior}}>0$ and $|\mathbf I_t-\Ip|/\Ip\ll1$,
\[
\m_t\approx\alpha_t\m_{t-1}
+\left(\frac{\bt_t\odot\mathbf v_t}{I_{\mathrm{prior}}}\right)\otimes\mathbf k_t.
\]
\begin{block}{Preserve writes while introducing importance}
Use $\mathbf v_t^*=\bt_t\odot\mathbf v_t=\operatorname{SiLU}(\boldsymbol\theta_v\mathbf x_t)$ as the kernel input. Small $\bt_t$ need not make the content write vanish.
\end{block}
\begin{itemize}
\item Initialize per-head $b_{\mathrm{scale}}\in[0.1,1.0]$ for fine-tuning.
\item Learn stronger importance accumulation as training progresses.
\item The required limit is small \emph{excess precision}; a strong forgetting gate alone is insufficient if new evidence is large.
\end{itemize}
\papersource{Paper, Secs. 2.3--2.4, PDF pp. 5--6. Last point follows directly from the precision recurrence.}
\end{frame}

% 12
\begin{frame}{Evaluation separates capacity, quality, and adaptation}
\begin{center}
\begin{tabular}{>{\raggedright\arraybackslash}p{2.7cm}>{\raggedright\arraybackslash}p{4.7cm}>{\raggedright\arraybackslash}p{4.7cm}}
\toprule
Experiment & Setup & Main question \\
\midrule
MQAR & 2 layers, width 128; 8 seeds; dense KV pairs & Does importance help under interference? \\
\addlinespace
170M / 760M & FineWeb-Edu; 15B / 30B tokens & Does it improve language tasks at matched token budgets? \\
\addlinespace
2.7B surgery & Replace 8 of 64 layers; matched adaptation recipe & Can an existing model gain long-context benefits? \\
\bottomrule
\end{tabular}
\end{center}
\medskip
\textbf{Metrics:} recall accuracy $\uparrow$, perplexity $\downarrow$, seven-task average accuracy $\uparrow$, long-context scores $\uparrow$.
\papersource{Paper, Sec. 3 and Appendices A.8--A.11, PDF pp. 6--9, 19--21. Equal token budgets do not imply equal runtime.}
\end{frame}

% 13
\begin{frame}{MQAR gains grow as fixed memory becomes crowded}
\begin{columns}[T]
\begin{column}{0.62\textwidth}
\includegraphics[width=\linewidth,height=5.3cm,keepaspectratio]{figures/fig-02-long.pdf}
\end{column}
\begin{column}{0.35\textwidth}
\textbf{Curriculum:} $L=128\to256\to512\to1024$, with $L/4$ KV pairs.
\medskip
\begin{itemize}
\item Palimpsa-D has the highest mean at each plotted length.
\item At 1024 tokens: roughly 0.69 vs. 0.58 without metaplasticity.
\item Black dots: 8 runs. Bars: $\pm1$ standard deviation.
\end{itemize}
\end{column}
\end{columns}
\papersource{Paper, Fig. 2, PDF p. 6; largest-context panel cropped from author figure. Values quoted here are approximate readings.}
\end{frame}

% 14
\begin{frame}{Weaker forgetting allows importance to differentiate}
\begin{columns}[T]
\begin{column}{0.62\textwidth}
\includegraphics[width=\linewidth,height=4.7cm,keepaspectratio]{figures/fig-03.pdf}
\end{column}
\begin{column}{0.35\textwidth}
\begin{itemize}
\item Blue: mean $\log N_t$ over context positions.
\item Orange: importance spread
\end{itemize}
\[
r=\frac{I_{\max}-I_{\min}}{I_{\min}}.
\]
\begin{itemize}
\item Pink: loss trend.
\item Late $r\approx40$ implies $I_{\max}/I_{\min}\approx41$.
\end{itemize}
\end{column}
\end{columns}
\medskip
\alert{The observed sequence is longer retention, stronger differentiation, then lower loss.}
\papersource{Paper, Sec. 3.2 and Fig. 3, PDF p. 7. Shading: standard deviation over 8 seeds; loss has no separate numerical axis.}
\end{frame}

% 15
\begin{frame}{Fine-tuning helps, but gains depend on the backbone}
\begin{center}
\begin{tabular}{llrrr}
\toprule
Scale & Backbone & Without Meta & Fine-tuned & Change \\
\midrule
170M & D & 43.91 & 43.96 & $+0.05\pp$ \\
170M & M & 42.90 & 43.48 & $+0.58\pp$ \\
\addlinespace
760M & D & 51.82 & 52.27 & $+0.45\pp$ \\
760M & M & 51.53 & 52.16 & $+0.63\pp$ \\
\bottomrule
\end{tabular}
\end{center}
\smallskip
Seven-task mean accuracy (\%); final 1B / 2B tokens use metaplasticity.
\begin{itemize}
\item 170M D trained from scratch scores 43.45, below its non-metaplastic ablation.
\item 170M M from scratch scores 43.57, slightly above its fine-tuned version.
\item Fine-tuning is a practical route; it is not uniformly superior in every case.
\end{itemize}
\papersource{Paper, Table 2, PDF p. 8. Changes calculated from printed scores. Total budgets: 15B / 30B tokens.}
\end{frame}

% 16
\begin{frame}{At 760M, Palimpsa-D leads the task average}
\begin{center}
\begin{tabular}{lrrr}
\toprule
Model & WikiText ppl $\downarrow$ & LAMBADA ppl $\downarrow$ & Avg. acc. $\uparrow$ \\
\midrule
Transformer++ & \textbf{18.91} & 18.84 & 51.28 \\
Gated Deltanet & 19.06 & 17.06 & 51.50 \\
Palimpsa-D, without Meta & 19.33 & 16.54 & 51.82 \\
Palimpsa-D, fine-tuned & 19.02 & \textbf{14.86} & \textbf{52.27} \\
Palimpsa-M, without Meta & 19.96 & 16.34 & 51.53 \\
Palimpsa-M, fine-tuned & 19.60 & 16.35 & 52.16 \\
\bottomrule
\end{tabular}
\end{center}
\medskip
\begin{itemize}
\item D vs. Gated Deltanet: $+0.77$ percentage points in average accuracy.
\item D LAMBADA accuracy: $41.98\%\to43.55\%$ ($+1.57\pp$).
\item M's LAMBADA perplexity is effectively unchanged; WikiText still favors the Transformer.
\end{itemize}
\papersource{Paper, Table 2, PDF p. 8. All models trained for 30B tokens; architectures/state sizes are not identical.}
\end{frame}

% 17
\begin{frame}{Position-wise loss supports more persistent context use}
\begin{columns}[T]
\begin{column}{0.62\textwidth}
\includegraphics[width=\linewidth,height=4.9cm,keepaspectratio]{figures/fig-04.pdf}
\end{column}
\begin{column}{0.35\textwidth}
\begin{itemize}
\item X: token position, log scale.
\item Y: NLL difference relative to Transformer.
\item Below zero is better.
\item Metaplastic models cross zero later.
\end{itemize}
\medskip
The $\sim5\times$ annotation refers to a crossover-position interval in this plot.
\end{column}
\end{columns}
\papersource{Paper, Fig. 4, PDF p. 9; 760M models on WikiText. This is not a universal 5x capacity or speed claim.}
\end{frame}

% 18
\begin{frame}{At 2.7B, replace only the eight weakest-forgetting layers}
\begin{columns}[T]
\begin{column}{0.57\textwidth}
\includegraphics[width=\linewidth,height=4.4cm,keepaspectratio]{figures/fig-06.png}
\end{column}
\begin{column}{0.40\textwidth}
\textbf{Base:} Mamba2, 64 layers, pretrained on 300B tokens.
\medskip
\textbf{Matched adaptation for both models:}
\begin{enumerate}
\item 10B tokens continued pretraining.
\item 5B tokens context extension to 32k.
\item LongAlpaca instruction tuning.
\end{enumerate}
\end{column}
\end{columns}
\medskip
Red dots select high-$N_t$ layers; the remaining 56 layers retain Mamba2 blocks.
\papersource{Paper, Sec. 3.4, PDF pp. 8--9; Appendix A.10 and Fig. 6, pp. 20--21.}
\end{frame}

% 19
\begin{frame}{The 2.7B upgrade improves averages, with uneven gains}
\begin{center}
\begin{tabular}{lrrr}
\toprule
Metric & Mamba2 control & Palimpsa-M & Change \\
\midrule
Seven-task average & 59.22 & 60.06 & $+0.84$ \\
LongBench overall & 19.3 & 20.4 & $+1.1$ \\
\addlinespace
LongBench short & 18.9 & 16.1 & $-2.8$ \\
LongBench medium & 19.5 & 23.3 & $+3.8$ \\
LongBench long & 19.4 & 20.4 & $+1.0$ \\
\bottomrule
\end{tabular}
\end{center}
\medskip
\begin{itemize}
\item Both models receive the same post-pretraining adaptation recipe.
\item WikiText ppl slightly worsens: $14.46\to14.51$; LAMBADA ppl stays at 4.79.
\item LongBench refers to v2 in the bibliography. Length groups describe benchmark documents, not a demonstrated usable memory window.
\end{itemize}
\papersource{Paper, Tables 2--3, PDF pp. 8--9. Accuracy changes are percentage points; LongBench changes are score points.}
\end{frame}

% 20
\begin{frame}{Variable tracing exposes a strong length-dependent trade-off}
\begin{center}
\begin{tabular}{rrrr}
\toprule
Context & Mamba2 control & Palimpsa-M & Difference \\
\midrule
4k & 59.6 & 33.5 & $-26.1$ \\
8k & 49.7 & 24.9 & $-24.8$ \\
16k & 26.0 & 22.5 & $-3.5$ \\
32k & 14.9 & 24.4 & $+9.5$ \\
\bottomrule
\end{tabular}
\end{center}
\medskip
\begin{block}{What the full curve says}
Palimpsa retains more tracing ability at 32k, but sacrifices substantial performance at shorter contexts.
\end{block}
\papersource{Paper, RULER Variable Tracing, Table 3 p. 9 and Table 6 p. 21; 2.7B models, higher is better.}
\end{frame}

% 21
\begin{frame}{RULER benefits concentrate in multi-item retrieval}
\begin{center}
\begin{tabular}{llrr}
\toprule
Task & Length & Mamba2 control & Palimpsa-M \\
\midrule
Multi-query retrieval (N-MQ) & 4k & 53.1 & 57.5 \\
Multi-value retrieval (N-MV) & 4k & 51.5 & 59.9 \\
Multi-key retrieval (N-MK) & 8k & 24.6 & 21.8 \\
Single needle (N-1) & 32k & 94.4 & 93.0 \\
Frequent words (FWE) & 16k & 59.6 & 43.8 \\
\bottomrule
\end{tabular}
\end{center}
\medskip
\begin{itemize}
\item N-MQ and N-MV improve at all four lengths, but 32k scores remain low.
\item N-MK is mixed; the 8k row contradicts a blanket improvement claim.
\item Aggregation favors the baseline; QA gains vary by context length.
\end{itemize}
\papersource{Paper, Appendix A.11 and Table 6, PDF p. 21. Selected task rows; higher is better.}
\end{frame}

% 22
\begin{frame}{Constant recurrent state still has a substantial kernel cost}
\begin{columns}[T]
\begin{column}{0.54\textwidth}
\includegraphics[width=\linewidth,height=4.5cm,keepaspectratio]{figures/fig-07.pdf}
\end{column}
\begin{column}{0.43\textwidth}
\textbf{Training vs. Simple GLA:}
\begin{itemize}
\item 170M: about $1.5\times$ time.
\item 760M: about $3\times$ time.
\end{itemize}
\medskip
\textbf{Recurrent attention kernel:}
\begin{itemize}
\item RTX 3090; $B=16$, $L=2048$, $H=1$.
\item $D=2048$: about 50k vs. 188k tokens/s.
\item No measured full-model inference speedup is reported.
\end{itemize}
\end{column}
\end{columns}
\papersource{Paper, Appendices A.12--A.13, PDF pp. 21--23; Fig. 7. Main-text Mamba2 speed comparison uses a different baseline.}
\end{frame}

% 23
\begin{frame}{The useful contribution is adaptive memory protection}
\begin{enumerate}
\item \textbf{Mechanism:} importance controls per-state plasticity, while prior restoration prevents permanent consolidation.
\medskip
\item \textbf{Practical route:} fine-tune a fast pretrained backbone; selective layer replacement limits expensive updates.
\medskip
\item \textbf{Trade-off:} approximate precision, extra state, slower kernels, and task-dependent adaptation costs.
\end{enumerate}
\medskip
\begin{block}{Discussion}
Can metaplasticity be activated only where it helps, and can limited cross-key correlations improve the diagonal approximation?
\end{block}
\papersource{Synthesis of paper Secs. 2--4 and Appendices A.5, A.11--A.13. Discussion questions are proposed follow-up work.}
\end{frame}

% 24
\begin{frame}{Paper and implementation}
\textbf{Djohan Bonnet, Jamie Lohoff, Jan Finkbeiner, Elidona Shiqerukaj, Emre Neftci.}
\medskip

\emph{Learning to Remember, Learn, and Forget in Attention-Based Models.}
\medskip
\begin{itemize}
\item Read version: \href{https://arxiv.org/abs/2602.09075v4}{arXiv:2602.09075v4}, 3 June 2026.
\item Publication: \href{https://proceedings.mlr.press/v306/bonnet26a.html}{ICML 2026, PMLR 306:9129--9151}.
\item Official code: \href{https://github.com/djo1996/Palimpsa}{github.com/djo1996/Palimpsa}.
\end{itemize}
\medskip
\textbf{Source use:} figures are from the authors' v4 source package; tables select and reorganize v4 values. Numerical teaching examples are explicitly labeled.
\papersource{Publication/code checked 30 October 2026. Later repository optimizations are not treated as v4 experimental results.}
\end{frame}

\appendix
% 25
\begin{frame}{Backup: dropping precision correlations changes the mean}
Let $\mathbf P=\boldsymbol\Sigma^{-1}$. The full Gaussian update is
\[
\begin{aligned}
\mathbf P_{t,i}&=\alpha_t\mathbf P_{t-1,i}
+(1-\alpha_t)\mathbf P_{\mathrm{prior}}
+\beta_{t,i}\mathbf k_t\mathbf k_t^\top,\\
\m_{t,i}&=\mathbf P_{t,i}^{-1}
\left[\alpha_t\mathbf P_{t-1,i}\m_{t-1,i}
+\beta_{t,i}v_{t,i}\mathbf k_t\right].
\end{aligned}
\]
\textbf{Teaching example:} zero prior mean, identity prior precision,
$\mathbf k=(1,1)^\top$, $v=\beta=1$.
\[
\mathbf P=\begin{pmatrix}2&1\\1&2\end{pmatrix},\quad
\mathbf h=\begin{pmatrix}1\\1\end{pmatrix},\quad
\m_{\mathrm{full}}=\begin{pmatrix}1/3\\1/3\end{pmatrix},\quad
\m_{\mathrm{diag}}=\begin{pmatrix}1/2\\1/2\end{pmatrix}.
\]
Restricting posterior covariance alone would not yield the latter mean.
\papersource{Paper, Appendix A.3--A.5, PDF pp. 15--18. Numerical example is an explanatory calculation.}
\end{frame}

% 26
\begin{frame}{Backup: token parity does not mean identical architecture}
\begin{center}
\begin{tabular}{llrrrr}
\toprule
Scale & Model & Layers & Width & Heads & $E_v$ \\
\midrule
170M & Transformer++ & 20 & 768 & 16 & -- \\
 & Gated Deltanet & 19 & 768 & 16 & 2 \\
 & Palimpsa-D & 19 & 768 & 16 & 2 \\
 & Palimpsa-M & 30 & 768 & 16 & 2 \\
\addlinespace
760M & Transformer++ & 25 & 1536 & 16 & -- \\
 & Gated Deltanet & 19 & 1536 & 16 & 1 \\
 & Palimpsa-D & 19 & 1536 & 16 & 2 \\
 & Palimpsa-M & 38 & 1536 & 16 & 2 \\
\bottomrule
\end{tabular}
\end{center}
Fine-tuning uses a 200-step warmup and one-tenth of the original peak learning rate. Initial training uses a 2000-step warmup.
\papersource{Paper, Appendix A.9 and Table 5, PDF pp. 19--20. $E_v$: value expansion.}
\end{frame}

% 27
\begin{frame}{Backup: the forgetting gate alone does not define the limit}
For a scalar state with constant $\alpha<1$, $\beta$, and $k$,
\[
I_t=\alpha I_{t-1}+(1-\alpha)I_{\mathrm{prior}}+\beta k^2.
\]
The stationary precision is
\[
I_\infty=I_{\mathrm{prior}}+\frac{\beta k^2}{1-\alpha}
=I_{\mathrm{prior}}+N\beta k^2.
\]
\begin{itemize}
\item Longer retention increases the precision that can accumulate.
\item The Mamba-like regime requires
$N\beta k^2/I_{\mathrm{prior}}\ll1$ in this simple setting.
\item Reparameterizing the write as $v^*$ permits small $\beta$ while preserving finite content updates.
\end{itemize}
\papersource{Explanatory stationary-state derivation from paper Eq. (5); compare Sec. 2.4, PDF p. 6.}
\end{frame}
\end{document}