\documentclass[aspectratio=169,11pt]{beamer}
\usepackage[T1]{fontenc}
\usepackage[utf8]{inputenc}
\usepackage{amsmath,amssymb,graphicx,booktabs}

\definecolor{AcademicBlue}{RGB}{26,57,86}
\definecolor{AcademicTeal}{RGB}{0,105,112}
\setbeamercolor{structure}{fg=AcademicBlue}
\setbeamercolor{frametitle}{fg=AcademicBlue,bg=white}
\setbeamercolor{title}{fg=AcademicBlue}
\setbeamercolor{alerted text}{fg=AcademicTeal}
\setbeamercolor{block title}{fg=white,bg=AcademicBlue}
\setbeamercolor{block body}{fg=black,bg=AcademicBlue!5}
\setbeamertemplate{navigation symbols}{}
\setbeamertemplate{itemize items}[circle]
\setbeamertemplate{footline}{\hfill\usebeamerfont{page number in head/foot}\insertframenumber\,/\,\inserttotalframenumber\hspace{4mm}\vspace{3mm}}
\setbeamersize{text margin left=8mm,text margin right=8mm}
\setbeamerfont{frametitle}{size=\Large,series=\bfseries}
\setbeamerfont{title}{size=\LARGE,series=\bfseries}
\newcommand{\papersource}[1]{\par\smallskip{\scriptsize\color{gray}#1\par}}
\newcommand{\mm}{Memory Mosaics v2}
\newcommand{\norm}[1]{\operatorname{Norm}\!\left(#1\right)}
\newcommand{\widefig}[1]{\centering\includegraphics[width=\linewidth,height=0.56\textheight,keepaspectratio]{figures/#1.pdf}\par}

\title[Memory Mosaics]{Memory Mosaics at scale}
\subtitle{Associative memory for learning from new context}
\author{Paper authors: Jianyu Zhang and L\'eon Bottou}
\date{Research group reading\quad\textbullet\quad 19 October 2026\\[2mm]
\small arXiv:2507.03285v3 \; (14 January 2026)\quad NeurIPS 2025}

\begin{document}

\begin{frame}
\titlepage
\vspace{-5mm}
\begin{center}\color{AcademicTeal}
Similar standard benchmark scores, stronger learning from new context.
\end{center}
\end{frame}

\begin{frame}{Three questions separate knowledge from learning}
\begin{enumerate}
\item \textbf{Persistent knowledge:} What did pretraining teach the model?
\medskip
\item \textbf{New knowledge:} Can it retain documents before seeing a question?
\medskip
\item \textbf{In-context learning:} Can it infer a new input--label mapping from demonstrations?
\end{enumerate}
\begin{block}{Research question}
Can associative memories improve the last two abilities at roughly 10B parameters, while preserving the first?
\end{block}
\papersource{Paper, Secs. 1 and 5, PDF pp. 1, 5--7.}
\end{frame}

\begin{frame}{A block combines contextual and persistent memory}
\begin{columns}[c,onlytextwidth]
\column{0.61\textwidth}
\includegraphics[width=\linewidth,height=0.72\textheight,keepaspectratio]{figures/architecture.pdf}
\column{0.36\textwidth}
\begin{itemize}
\item Separate short- and long-term KV retrieval.
\medskip
\item Concatenate heads, then project with $W_o$.
\medskip
\item Persistent knowledge uses a SwiGLU feedforward network.
\end{itemize}
\end{columns}
\papersource{Paper, Fig. 2 (right) and Sec. 3.3, PDF p. 4. The experiments use dense SwiGLU persistent memory.}
\end{frame}

\begin{frame}{Retrieval estimates a conditional expectation}
For stored pairs $\mathcal M=\{(k_i,v_i)\}_{i=1}^{n}$:
\[
f(k;\mathcal M)\approx\mathbb E[V\mid K=k]
\]
\[
f(k;\mathcal M)=\sum_{i=1}^{n}
\underbrace{\frac{e^{\beta k^\top k_i}}{\sum_j e^{\beta k^\top k_j}}}_{\text{similarity weight}}v_i.
\]
\begin{itemize}
\item Keys are $L_2$ normalized; $\beta$ controls retrieval sharpness.
\item The current key also serves as the query.
\item Key construction and causal masks still encode sequence order.
\end{itemize}
\papersource{Paper, Eqs. (2)--(4), PDF p. 2. Gaussian distance gives a factor 2, absorbed into the dot-product $\beta$.}
\end{frame}

\begin{frame}{A value stores what followed a past situation}
\[
k_t=\varphi_\theta(x_t,x_{t-1},\ldots),\qquad
v_t=\alpha_\psi\norm{\gamma W_\psi x_t+(1-\gamma)W_\psi x_{t+1}}.
\]
\begin{block}{Teaching example: $[A,B,\ldots,A]$}
The earlier $A$ provides a matching key; its value contains information about the following $B$.
\end{block}
\begin{itemize}
\item At position $T$, read only values indexed by $i<T$.
\item Each retrieved $v_i$ uses at most $x_{i+1}$, already observed by $T$.
\item Complete $v_T$ only after the next input arrives.
\end{itemize}
\papersource{Paper, Eqs. (5), (9)--(10), PDF pp. 3--4. The causal timeline is an explanatory derivation.}
\end{frame}

\begin{frame}{Content gates decide what contributes to a key}
\[
\begin{aligned}
\tilde k_t&=W_\varphi x_t,&g_t&=e^{W_gx_t},&\lambda_t&=e^{-|W_\lambda x_t|},\\
\bar k_t&=g_t\tilde k_t+\lambda_t\bar k_{t-1},&&k_t=\norm{\bar k_t}.
\end{aligned}
\]
\begin{itemize}
\item $g_t>0$ controls the current write; $0<\lambda_t\leq1$ controls retention.
\item Both gates are scalars driven by the current content.
\item Semantically irrelevant insertions can receive less weight.
\end{itemize}
\begin{block}{Where the recurrence stops}
The recurrence creates keys. The associative memory still stores historical KV pairs.
\end{block}
\papersource{Paper, Sec. 3.2, Eq. (8) and footnote 6, PDF p. 3.}
\end{frame}

\begin{frame}{Larger memories use narrower retrieval kernels}
\[
\boxed{\beta(n)=\beta_1n^\alpha+\beta_0},\qquad
\beta_0\geq0,\ \beta_1>0,\ 0<\alpha<1.
\]
\begin{itemize}
\item $n$: number of available key--value pairs.
\item More examples $\Rightarrow$ larger $\beta$ $\Rightarrow$ smaller bandwidth $1/\sqrt{\beta}$.
\end{itemize}
\begin{block}{Teaching calculation: one relevant key, $n-1$ distractors}
If the similarity gap is $\Delta>0$, the relevant weight is
\[
a_\star=\frac{1}{1+(n-1)e^{-\beta\Delta}}.
\]
Increasing sharpness can counter dilution as memory grows.
\end{block}
\papersource{Paper, Sec. 3.1, Eq. (6), PDF p. 3. Calculation illustrates the direction; it does not prove optimality.}
\end{frame}

\begin{frame}{Overlapping time scales support different roles}
\[
\mathcal M_t^S=\{(k_i^S,v_i^S):t-h+1\leq i\leq t-1\},
\]
\[
\mathcal M_t^L=\{(k_i^L,v_i^L):1\leq i\leq t-m\}.
\]
\begin{itemize}
\item \textbf{Short term:} local relationships within window parameter $h=256$.
\item \textbf{Long term:} distant context, excluding the nearest positions.
\item Independent parameters; $m<h$ creates overlap.
\item Train with random $m\in[64,256]$; infer with $m=64$.
\end{itemize}
\medskip
\alert{Persistent memory stores learned parameters; long-term memory stores context KV pairs.}
\papersource{Paper, Secs. 3.3--4, Eq. (10), PDF pp. 4--5; Appendix C, PDF p. 14.}
\end{frame}

\begin{frame}{The main comparison matches training recipes}
\centering
\begin{tabular}{lcc}
\toprule
Configuration & small & large\\
\midrule
Layers / hidden width / heads & 24 / 2048 / 16 & 32 / 4096 / 32\\
Pretraining tokens & 200B & 1T\\
Pretraining $\rightarrow$ adapted context & 4K $\rightarrow$ 32K & 4K $\rightarrow$ 32K\\
\bottomrule
\end{tabular}
\medskip
\begin{itemize}
\item Baseline: Llama-style Transformer with multi-head attention.
\item Same data mix and hyperparameters; document-wise training masks.
\item Large parameter counts: \textbf{8.8B Transformer vs. 9.9B MM v2}.
\item At length 256: 16.7B vs. 18.9B FLOPs/token.
\end{itemize}
\papersource{Paper, Sec. 4 and Table 2, PDF pp. 4--5; Appendix C, PDF p. 14. FLOPs are short-context estimates.}
\end{frame}

\begin{frame}{Standard benchmark averages are nearly unchanged}
\centering
\begin{tabular}{lrr}
\toprule
19-benchmark average & Transformer & MM v2\\
\midrule
small & 37.8 & 38.0\\
large & 52.2 & 52.2\\
\bottomrule
\end{tabular}
\medskip
\begin{block}{Ablating long-term memory separates two groups}
\begin{tabular}{lrr}
& Full MM v2 & Without long-term memory\\
13 benchmark average & 56.8 & 56.6\\
Other 6 benchmark average & 42.1 & 34.9
\end{tabular}
\end{block}
The first group is a proxy for persistent-knowledge ability. The second group needs more contextual support.
\papersource{Paper, Tables 1--2, PDF p. 5; Table 9, PDF p. 16. Scores in percent; large model for the ablation.}
\end{frame}

\begin{frame}{At 32K, multi-document QA improves by 12--15 points}
Read multiple unrelated documents, then answer a question about one of them.
\medskip
\begin{center}
\begin{tabular}{lrrr}
\toprule
32K task length & Transformer & MM v2 & Difference\\
\midrule
small & 22.1 & 36.9 & \alert{+14.8 pp}\\
large & 41.1 & 53.4 & \alert{+12.3 pp}\\
\bottomrule
\end{tabular}
\end{center}
\begin{itemize}
\item Both architectures were adapted from 4K to 32K context.
\item The question arrives after the information must be stored.
\item Simple single-needle retrieval nearly saturates: 99.6 vs. 100.0 for large models at 32K.
\end{itemize}
\papersource{Paper, Sec. 5.2 and Table 4, PDF p. 6; Table 13, PDF p. 17. QA scores in percent; pp = percentage points.}
\end{frame}

\begin{frame}{Length extrapolation works, with declining accuracy}
\centering
\begin{tabular}{lrrrr}
\toprule
4K pretrained model & 4K test & 8K & 16K & 32K\\
\midrule
Transformer large & 57.7 & $\times$ & $\times$ & $\times$\\
MM v2 large & 59.3 & 48.8 & 46.4 & 26.5\\
MM v2 small & 45.0 & 35.0 & 34.1 & 31.7\\
\bottomrule
\end{tabular}
\medskip
\begin{itemize}
\item No length adaptation for the rows above.
\item After 32K adaptation, MM v2 large reaches \textbf{46.4 at 64K}.
\item $\times$ preserves the paper's failure marker; it is not a numerical zero.
\end{itemize}
\begin{block}{Interpretation}
Extrapolation retains useful retrieval, but longer contexts remain harder.
\end{block}
\papersource{Paper, Tables 3--4, PDF p. 6. QA scores in percent.}
\end{frame}

\begin{frame}{Ablations connect long-term memory to retrieval}
\begin{columns}[t,onlytextwidth]
\column{0.48\textwidth}
\begin{block}{Remove the long-term branch}
Large model, adapted to 32K:
\[
53.4\ \longrightarrow\ 20.2
\]
\centering 32K QA: \alert{$-33.2$ pp}
\end{block}
\column{0.48\textwidth}
\begin{block}{Randomize the training boundary}
Small model, trained at 4K:
\[
15.9\ \longrightarrow\ 31.7
\]
\centering 32K QA: \alert{$+15.8$ pp}
\end{block}
\end{columns}
\medskip
The second comparison uses no length fine-tuning. Random overlap helps avoid dependence on a fixed distance boundary.
\papersource{Paper, Tables 10 and 12, PDF p. 17. The two ablations use different scales and adaptation settings.}
\end{frame}

\begin{frame}{One shot means one example from every class}
\centering
\begin{tabular}{lr@{\hspace{8mm}}r}
\toprule
Task & Classes & Mean tokens/example\\
\midrule
Banking77: intent & 77 & 24\\
Tacred: entity relation & 41 & 77\\
Goemotion: emotion & 28 & 26\\
\bottomrule
\end{tabular}
\medskip
\begin{itemize}
\item Banking77, 16 shots: \textbf{1,232 demonstrations}.
\item Semantic labels: use meaningful names such as \texttt{joy}.
\item Anonymous labels: learn mappings to names such as \texttt{class\_00}.
\item Choose the best delimiter/order strategy for each $n$-shot setting.
\end{itemize}
\papersource{Paper, Sec. 5.3 and footnote 13, PDF p. 7; prompt examples in Appendix I, PDF pp. 19--21.}
\end{frame}

\begin{frame}{Semantic-label gains depend on the task and shot count}
\widefig{icl-semantic-large}
\begin{itemize}
\item Tacred benefits strongly; the marked gap is about \textbf{10.2 pp}.
\item Banking77 improves for both models; low-shot Goemotion favors the Transformer.
\end{itemize}
\papersource{Paper, Fig. 3, PDF p. 8. Large models; blue = MM v2, red = Transformer. Accuracy uses different vertical scales.}
\end{frame}

\begin{frame}{Anonymous labels reveal stronger mapping from examples}
\widefig{icl-anonymous-large}
\begin{itemize}
\item MM v2 leads across the displayed curves; Tacred's marked gap is \textbf{15.5 pp}.
\item Goemotion remains difficult: accuracy is roughly 0.07--0.11 for MM v2.
\end{itemize}
\papersource{Paper, Fig. 4, PDF p. 8. Large models. The curves are not all monotonic in shot count.}
\end{frame}

\begin{frame}{Splitting Transformer attention alone is insufficient}
\begin{columns}[c,onlytextwidth]
\column{0.55\textwidth}
\includegraphics[width=\linewidth,height=0.66\textheight,keepaspectratio]{figures/transformer-split-ablation.pdf}
\column{0.42\textwidth}
\begin{itemize}
\item Long/short attention alone does not close the anonymous Tacred gap.
\medskip
\item This motivates studying the full combination of key, value, and retrieval design.
\medskip
\item A full ablation of all three v2 modifications is still missing.
\end{itemize}
\end{columns}
\papersource{Paper, Fig. 5, PDF p. 8. Small models. The last point is a reading assessment.}
\end{frame}

\begin{frame}{Eight times more data does not close the QA gap}
\centering
\begin{tabular}{lrr}
\toprule
Large model & Training tokens & 32K QA\\
\midrule
Transformer & 200B & 33.8\\
Transformer & 1T & 41.1\\
Transformer* & 8T & 46.9\\
MM v2 & 1T & \textbf{53.4}\\
\bottomrule
\end{tabular}
\medskip
\begin{itemize}
\item 8T Transformer still trails by \alert{6.5 percentage points}.
\item Transformer* also uses GQA and 8K pretraining context.
\item This comparison changes more than the training-token count.
\end{itemize}
\papersource{Paper, Sec. 6.1 and Table 5, PDF p. 9. All rows use the 32K-adapted models.}
\end{frame}

\begin{frame}{More pretraining can match semantic-label performance}
\widefig{data-scaling-semantic}
\begin{itemize}
\item Dashed red: 8T Transformer. Blue: 1T MM v2.
\item The 8T model reaches comparable results and leads on Goemotion.
\end{itemize}
\papersource{Paper, Fig. 6, PDF p. 9. Meaningful class names allow use of pretrained semantic knowledge.}
\end{frame}

\begin{frame}{Anonymous-label performance remains harder to recover}
\widefig{data-scaling-anonymous}
\begin{itemize}
\item The 8T Transformer remains below MM v2 on all three curves.
\item It also falls below the 1T Transformer in these displayed tasks.
\end{itemize}
\papersource{Paper, Fig. 7, PDF p. 10. The 8T model also differs in attention type and pretraining context length.}
\end{frame}

\begin{frame}{One optimization step greatly improves length adaptation}
\begin{columns}[c,onlytextwidth]
\column{0.62\textwidth}
\includegraphics[width=\linewidth,height=0.67\textheight,keepaspectratio]{figures/finetuning.pdf}
\column{0.35\textwidth}
\begin{itemize}
\item 4K $\rightarrow$ 32K adaptation.
\medskip
\item MM v2: about \textbf{+22 pp} after one mini-batch.
\medskip
\item Two steps approach the plateau.
\medskip
\item This is an optimization step, not one example or weight-free ICL.
\end{itemize}
\end{columns}
\papersource{Paper, Sec. 7 and Fig. 8, PDF p. 10. Transformer at 800 steps remains below MM v2 at one step.}
\end{frame}

\begin{frame}{The capability gain has a concrete storage cost}
\begin{itemize}
\item \textbf{Capacity:} full contextual KV storage grows with context length.
\medskip
\item \textbf{Computation:} exact long-term retrieval still scans history; full-sequence interactions remain quadratic in length.
\medskip
\item \textbf{Budget evidence:} Appendix E compares parameter/FLOP curves, but gives no serving throughput or latency results.
\medskip
\item \textbf{Mechanism evidence:} long-term memory and random overlap have ablations; separate contributions of all new modules remain open.
\end{itemize}
\papersource{Paper, footnote 6, PDF p. 3; Sec. 8, PDF p. 10; Appendix E, PDF p. 16. Complexity statement follows from exact KV retrieval.}
\end{frame}

\begin{frame}{The main contribution is better use of new context}
\begin{enumerate}
\item Associative-memory design scales to \textbf{9.9B parameters and 1T tokens}.
\medskip
\item Comparable standard benchmarks can hide large differences in \textbf{new-knowledge retrieval and ICL}.
\medskip
\item More pretraining helps semantic tasks, but does not recover all \textbf{anonymous-label learning} in these comparisons.
\end{enumerate}
\begin{block}{A useful next experiment}
Separate the effects of successor values, gated keys, and adaptive bandwidth under matched budgets; then reduce long-term retrieval cost.
\end{block}
\papersource{Jianyu Zhang and L\'eon Bottou. Memory Mosaics at scale. NeurIPS 2025.\\
\href{https://arxiv.org/abs/2507.03285v3}{arXiv:2507.03285v3}, revised 14 January 2026. The proposed experiment is a reading perspective.}
\end{frame}

\appendix

\begin{frame}{Backup: six benchmarks rely more on long context}
\centering
\begin{tabular}{lrrr}
\toprule
Benchmark & Full MM v2 & No long-term memory & Difference\\
\midrule
SQuAD & 78.2 & 69.4 & $-8.8$\\
BBH & 47.8 & 24.6 & $-23.2$\\
MATH & 8.8 & 5.4 & $-3.4$\\
MBPP & 9.6 & 6.8 & $-2.8$\\
RACE middle & 61.6 & 59.5 & $-2.1$\\
RACE high & 46.5 & 43.6 & $-2.9$\\
\midrule
Average & 42.1 & 34.9 & $-7.2$\\
\bottomrule
\end{tabular}
\papersource{Paper, Table 9, PDF p. 16. Large model; scores in percent and differences in percentage points.}
\end{frame}

\begin{frame}{Backup: the small model also benefits on semantic ICL}
\widefig{icl-semantic-small}
\begin{itemize}
\item Tacred's marked gap is approximately 16.7 percentage points.
\item Small-model Goemotion is relatively flat for MM v2 as shots increase.
\end{itemize}
\papersource{Paper, Fig. 13, PDF p. 18. Small models trained on 200B tokens.}
\end{frame}

\begin{frame}{Backup: anonymous ICL gains persist at small scale}
\widefig{icl-anonymous-small}
\begin{itemize}
\item Tacred's marked gap is approximately 12.7 percentage points.
\item Anonymous Goemotion stays low in absolute accuracy for both models.
\end{itemize}
\papersource{Paper, Fig. 14, PDF p. 18.}
\end{frame}

\begin{frame}{Backup: performance--budget curves favor MM v2}
\centering
\includegraphics[width=0.88\linewidth,height=0.62\textheight,keepaspectratio]{figures/efficiency.pdf}
\par\smallskip
Sparse scale points connected by curves; intermediate-budget paired runs and the FLOPs counting convention are not detailed.
\papersource{Paper, Fig. 12, PDF p. 16. Top: 32K RULER QA. Bottom: 10-shot Tacred. This is not a latency benchmark.}
\end{frame}

\begin{frame}{Backup: initialization makes the parameter constraints explicit}
\centering
\begin{tabular}{lll}
\toprule
Parameter & Reparameterization & Initial $\theta$\\
\midrule
$\beta_0,\beta_1$ & $e^{\min(\theta,10)}$ & $1.5$\\
$\alpha$ & $\min(|\theta|,1)$ & $1/3$\\
$\alpha_\psi$ & $e^{\min(|\theta|,15)}$ & $0$\\
$\gamma$ & No reparameterization & $U(0,1)$ directly\\
\bottomrule
\end{tabular}
\medskip
\begin{itemize}
\item The implemented exponent can reach 0 or 1.
\item $\gamma$ starts in $(0,1)$, but no ongoing interval constraint is specified.
\item Pretraining: AdamW, batch 1024, 2000 warmup steps, cosine decay.
\end{itemize}
\papersource{Paper, Appendix C and Table 6, PDF p. 14. Initialization details refer to pretraining.}
\end{frame}

\begin{frame}{Backup: external baselines provide supporting context}
\begin{itemize}
\item Appendix D reuses published RWKV and Mamba results with poor long-context retrieval and Tacred ICL.
\medskip
\item These are particular models and protocols, not a proof that all compressed-memory architectures must fail.
\medskip
\item At 32K QA, public-model scores include Mistral-base 44.7 and Mixtral-base 41.3, versus MM v2 53.4.
\medskip
\item Public comparisons differ in training and generation length: 128 output tokens for the other models, 32 for MM v2.
\end{itemize}
\papersource{Paper, Tables 7--8, PDF p. 15; Table 11, PDF p. 17. Controlled in-paper Transformer comparisons remain the main evidence.}
\end{frame}

\end{document}