% Self-authored paper discussion, based on the arxiv-paper-reader template.
% Build: python3 build_slides.py slides.en.tex
\documentclass[aspectratio=169,11pt]{beamer}
\usepackage[T1]{fontenc}
\usepackage[utf8]{inputenc}
\usepackage{lmodern}
\usepackage{amsmath,amssymb,graphicx,booktabs,array}
\definecolor{AcademicBlue}{RGB}{26,57,86}
\definecolor{AcademicTeal}{RGB}{0,105,112}
\setbeamercolor{structure}{fg=AcademicBlue}
\setbeamercolor{frametitle}{fg=AcademicBlue,bg=white}
\setbeamercolor{title}{fg=AcademicBlue}
\setbeamercolor{alerted text}{fg=AcademicTeal}
\setbeamercolor{block title}{fg=white,bg=AcademicBlue}
\setbeamercolor{block body}{fg=black,bg=AcademicBlue!5}
\setbeamertemplate{navigation symbols}{}
\setbeamertemplate{itemize items}[circle]
\setbeamertemplate{footline}{\hfill\usebeamerfont{page number in head/foot}\insertframenumber\,/\,\inserttotalframenumber\hspace{4mm}\vspace{3mm}}
\setbeamersize{text margin left=8mm,text margin right=8mm}
\setbeamerfont{frametitle}{size=\Large,series=\bfseries}
\setbeamerfont{title}{size=\LARGE,series=\bfseries}
\newcommand{\papersource}[1]{\par\smallskip{\scriptsize\color{gray}#1\par}}
\newcommand{\M}{\mathcal M}
\newcommand{\vct}[1]{\boldsymbol{#1}}
\newcommand{\fig}[2]{\includegraphics[width=#1\linewidth,height=4.4cm,keepaspectratio]{figures/#2}}
\title[Nested Learning]{Nested Learning}
\subtitle{The Illusion of Deep Learning Architectures}
\author{\texorpdfstring{Paper discussion\\[2mm]\small Paper authors: Ali Behrouz, Meisam Razaviyayn,\\Peilin Zhong, and Vahab Mirrokni}{Paper discussion}}
\date{arXiv:2512.24695v1 \quad | \quad 10 October 2026}
\hypersetup{pdftitle={Nested Learning: paper discussion (arXiv 2512.24695v1)},pdfauthor={Paper reading notes}}

\begin{document}

\begin{frame}
\titlepage
\vspace{-6mm}
\begin{center}\color{AcademicTeal}
Design what learns, how often it learns, and how knowledge persists.
\end{center}
\end{frame}

\begin{frame}{Deployment leaves a gap between two memories}
\begin{columns}[T]
\column{0.48\textwidth}
\begin{block}{Persistent parameters}
FFNs and projections retain pre-training knowledge.\par\medskip
Usually frozen during inference.
\end{block}
\column{0.48\textwidth}
\begin{block}{Immediate context}
Attention reads the current KV history.\par\medskip
Knowledge disappears when this context is discarded.
\end{block}
\end{columns}
\medskip
\begin{itemize}
\item More network layers do not specify how new knowledge is consolidated.
\item NL adds a design axis: \alert{learning processes at different time scales}.
\end{itemize}
\papersource{Paper, Secs. 1--1.1, PDF pp. 1--4. Brain analogies motivate the design.}
\end{frame}

\begin{frame}{A learning level differs from a network layer}
\centering
\includegraphics[width=0.94\linewidth,height=4.1cm,keepaspectratio]{figures/fig03-mlp-memory.png}
\begin{itemize}
\item A learned initial memory supplies persistent knowledge.
\item An additional update process lets that memory adapt within a sequence.
\end{itemize}
\papersource{Paper, Fig. 3, PDF p. 10; Secs. 3.1--3.2, pp. 7--13.}
\end{frame}

\begin{frame}{Associative memory unifies several update rules}
\[
\M^*=\arg\min_{\M}\widetilde{\mathcal L}\bigl(\M(\mathcal K);\mathcal V\bigr)
\]
\begin{columns}[T]
\column{0.49\textwidth}
\begin{block}{Linear-layer training}
\[
\nabla_W\mathcal L=\vct\delta\vct x^\top
\]
\[
W^+=W-\eta\vct\delta\vct x^\top
\]
Input $\vct x$ is associated with a local error signal $\vct\delta$.
\end{block}
\column{0.49\textwidth}
\begin{block}{Linear attention}
\[
\widetilde{\mathcal L}=-\langle\M\vct k,\vct v\rangle
\]
\[
\M^+=\M+\vct v\vct k^\top
\]
Key $\vct k$ is associated with value $\vct v$ (step size 1).
\end{block}
\end{columns}
\medskip
Backpropagation is self-referential: $\vct\delta$ depends on current weights.
\papersource{Paper, Sec. 3.1, PDF pp. 7--9; Sec. 4.1, Eqs. (29)--(31), p. 15.}
\end{frame}

\begin{frame}{Optimizers learn from a stream of gradients}
\[
\vct m_{t+1}=\alpha_t\vct m_t-\eta_t\nabla_W\mathcal L_t,
\qquad W_{t+1}=W_t+\vct m_{t+1}
\]
\begin{itemize}
\item Momentum compresses gradient history; weights use the resulting update.
\item The network generates the optimizer's context, so their designs interact.
\item NL suggests richer objectives, nonlinear memory, feature maps, and multiple time scales for momentum.
\end{itemize}
\begin{block}{A useful distinction}
An associative-memory interpretation is a design tool. Optimality for an internal regression objective does not imply optimality for the original task.
\end{block}
\papersource{Paper, Secs. 4.2--4.4, PDF pp. 15--20; Appendix B, pp. 50--51.}
\end{frame}

\begin{frame}{DGD adds explicit shrinkage along the input direction}
Let $\vct u_t=-\nabla_{\vct y_t}\mathcal L(W_t;\vct x_t)$ be fixed for this update.
\[
W^+=\arg\min_W\left\{\tfrac12\|W\vct x_t-\vct u_t\|^2
+\tfrac1{2\eta_t}\|W-W_t\|_F^2\right\}
\]
\[
\boxed{W^+=W_t(I-\gamma_t\vct x_t\vct x_t^\top)
-\gamma_t\nabla_{\vct y_t}\mathcal L\,\vct x_t^\top}
\qquad
\gamma_t=\frac{\eta_t}{1+\eta_t\|\vct x_t\|^2}
\]
\begin{itemize}
\item Reduce the old component along $\vct x_t$, then write a new error signal.
\item Unit-norm inputs give $\gamma_t=\eta_t/(1+\eta_t)$, as in the main text.
\item This changes the update objective; ordinary GD already depends on state.
\end{itemize}
\papersource{Paper, Eqs. (56)--(57), PDF p. 20. General-norm solution re-derived here; Appendix C has inconsistent intermediate steps.}
\end{frame}

\begin{frame}{CMS provides several rates of adaptation and retention}
\[
\vct y_t=\operatorname{MLP}^{(f_k)}\circ\cdots\circ
\operatorname{MLP}^{(f_1)}(\vct x_t)
\]
\begin{center}
\begin{tabular}{lll}
\toprule
Memory & Update schedule & Intended role\\
\midrule
Fast & Short interval & Adapt to recent evidence\\
Intermediate & Longer interval & Retain across local contexts\\
Slow & Rare updates & Preserve more persistent knowledge\\
\bottomrule
\end{tabular}
\end{center}
\begin{itemize}
\item Update each block only at its scheduled boundary.
\item Connect levels through reads, gradients, or learned initial states.
\item Slower updates help only if useful knowledge reaches those states.
\end{itemize}
\papersource{Paper, Sec. 7.1, Eqs. (70)--(74), PDF pp. 27--29. Schedule table is an explanatory summary.}
\end{frame}

\begin{frame}{Hope's fast memory generates its own update targets}
\begin{align*}
\vct k_t&=\M_{k,t-1}(\vct x_t),
&\vct v_t&=\M_{v,t-1}(\vct x_t),\\
\eta_t&=\M_{\eta,t-1}(\vct x_t),
&\alpha_t&=\M_{\alpha,t-1}(\vct x_t).
\end{align*}
For each memory module $\M_\square$:
\[
\widehat{\vct v}_{\square,t}=\M_{\square,t-1}(\vct v_t),
\qquad
\widetilde{\mathcal L}_\square=
\|\M_\square(\vct k_t)-\widehat{\vct v}_{\square,t}\|_2^2.
\]
\begin{itemize}
\item Each module learns a key-to-target mapping using DGD and retention.
\item The state influences both the representation and its learning signal.
\item Two-layer residual MLP memories; initial states learned across contexts.
\end{itemize}
\papersource{Paper, Sec. 8.1, Eqs. (83)--(89), PDF pp. 31--32. Targets are generated from the pre-update state.}
\end{frame}

\begin{frame}{Hope combines an adaptive memory with CMS}
\centering
\includegraphics[width=0.95\linewidth,height=4.2cm,keepaspectratio]{figures/fig05-hope.png}
\begin{itemize}
\item \textbf{Hope:} self-modifying Titans $\rightarrow$ CMS.
\item \textbf{Hope-Attention:} global softmax attention $\rightarrow$ CMS.
\end{itemize}
\papersource{Paper, Fig. 5, PDF p. 30; Sec. 8.3, Eqs. (94)--(97), p. 33. Diagram values are illustrative, not shared experiment settings.}
\end{frame}

\begin{frame}{Chunking trades immediate adaptation for parallel work}
\begin{enumerate}
\item Freeze the state available at the start of a chunk.
\item Generate projections, targets, and gradients for that chunk in parallel.
\item Apply scheduled memory updates and advance to the next chunk.
\end{enumerate}
\medskip
\begin{columns}[T]
\column{0.48\textwidth}
\begin{block}{Benefit}
Fewer sequential nonlinear gradient evaluations; matrix operations inside chunks.
\end{block}
\column{0.48\textwidth}
\begin{block}{Cost and limitation}
Extra adaptive states and update work. Chunk-start gradients differ from fresh token-wise gradients.
\end{block}
\end{columns}
\papersource{Paper, Sec. 8.2, PDF pp. 32--33; text describes the previous chunk's state. Eq. (90)'s printed index is inconsistent with that description.}
\end{frame}

\begin{frame}{The experiments evaluate distinct systems and budgets}
\begin{center}\small
\begin{tabular}{>{\raggedright\arraybackslash}p{2.8cm}>{\raggedright\arraybackslash}p{4.9cm}>{\raggedright\arraybackslash}p{5.3cm}}
\toprule
Question & System & Training / evaluation protocol\\
\midrule
Continual learning & Llama-based Hope-Attention & Class-incremental: 3B/8B backbones, 15B extra tokens; matched ICL token budget\\[1mm]
Long context & Hope from scratch & About 50B tokens; BABILong small models additionally task-fine-tuned\\[1mm]
Language modeling & Hope and architecture baselines & 760M / 30B tokens; 1.3B / 100B tokens; AdamW\\[1mm]
Optimizer quality & M3 on ViT; timing on Transformer & ViT 24M/86M; timing at 140M/1.3B\\
\bottomrule
\end{tabular}\end{center}
\medskip
\alert{M3 is a separate experiment: the Hope language models use AdamW.}
\papersource{Paper, Secs. 9.1--9.3 and 9.7, PDF pp. 34--40.}
\end{frame}

\begin{frame}{CMS improves the reported class-incremental results}
\begin{columns}[T]
\column{0.49\textwidth}
\centering CLINC\par\smallskip
\includegraphics[width=\linewidth,height=3.5cm,keepaspectratio]{figures/fig06-clinc.png}
\column{0.49\textwidth}
\centering Banking\par\smallskip
\includegraphics[width=\linewidth,height=3.5cm,keepaspectratio]{figures/fig06-banking.png}
\end{columns}
\begin{itemize}
\item Hope-Attention leads ICL, EWC, and the external-learner baseline InCA.
\item Both 3B and 8B groups improve; DBpedia follows the same pattern.
\item This experiment directly tests the CMS adaptation of pre-trained FFNs.
\end{itemize}
\papersource{Paper, Fig. 6 (selected original panels), PDF p. 33; setup in Sec. 9.1, p. 34. No exact values inferred from bar heights.}
\end{frame}

\begin{frame}{More memory levels improve these context tasks}
\begin{columns}[T]
\column{0.49\textwidth}
\centering Multi-key retrieval: accuracy $\uparrow$\par
\includegraphics[width=\linewidth,height=3.5cm,keepaspectratio]{figures/fig07-niah-levels.png}
\column{0.49\textwidth}
\centering QASPER: perplexity $\downarrow$\par
\includegraphics[width=\linewidth,height=3.5cm,keepaspectratio]{figures/fig07-qasper-levels.png}
\end{columns}
\begin{itemize}
\item Higher level counts improve both metrics in the shown configurations.
\item The 512 group performs best; the paper proposes 2K as an efficiency compromise.
\end{itemize}
\papersource{Paper, Fig. 7, PDF p. 34; discussion pp. 35--36. ``Lowest Freq'' labels are retained as printed; their units are not consistently specified.}
\end{frame}

\begin{frame}{Additional levels reduce interference across languages}
\begin{columns}[c]
\column{0.57\textwidth}
\includegraphics[width=\linewidth,height=5.2cm,keepaspectratio]{figures/fig08-translation.png}
\column{0.40\textwidth}
\begin{itemize}
\item Learn Manchu and Kalamang, then translate into English.
\item \textcolor{red!70!black}{Red}: learn and test separately.
\item \textcolor{blue!70!black}{Blue}: learn sequentially, then test.
\item More CMS levels move the sequential result toward the upper right.
\end{itemize}
\end{columns}
\papersource{Paper, Fig. 8, PDF p. 35; CTNL protocol, Sec. 9.1, p. 36. Both axes are ChrF scores (higher is better).}
\end{frame}

\begin{frame}{At 16K, Hope improves compressed-memory retrieval}
\begin{center}
\begin{tabular}{lrrrr}
\toprule
Model & Number & UUID & Multi-key & Multi-query\\
\midrule
Transformer & 94.2 & 40.8 & 61.4 & 29.8\\
Hope-Attention & 94.4 & 42.4 & 60.8 & 30.6\\
RWKV-7 & 12.6 & 5.8 & 9.6 & 8.6\\
Comba & 13.4 & 7.4 & 8.2 & 6.4\\
Titans & 75.4 & 21.2 & 8.2 & 9.4\\
\textbf{Hope} & \textbf{78.2} & \textbf{24.8} & \textbf{14.8} & \textbf{14.2}\\
\bottomrule
\end{tabular}
\end{center}
\begin{itemize}
\item Better than the shown attention-free baselines at this length.
\item A large precision gap remains: multi-key 14.8 vs. Transformer 61.4.
\item At 4K, Titans slightly beats Hope on number and UUID retrieval.
\end{itemize}
\papersource{Paper, Table 1, PDF p. 35. Selected 16K accuracies (\%). Setup: about 50B training tokens, Sec. 9.2.}
\end{frame}

\begin{frame}{The 10M BABILong result uses task fine-tuning}
\begin{columns}[c]
\column{0.58\textwidth}
\includegraphics[width=\linewidth,height=5.2cm,keepaspectratio]{figures/fig09-babilong.png}
\column{0.39\textwidth}
\begin{itemize}
\item Hope retains strong performance at the longest plotted context.
\item Red curves: task-fine-tuned small models.
\item Blue curves: zero-shot large models / RAG baseline.
\item The paper reports substantial degradation without fine-tuning.
\end{itemize}
\end{columns}
\papersource{Paper, Fig. 9, PDF p. 35; protocol and limitations, Sec. 9.2, pp. 36--37. This is not a matched-compute comparison to general-purpose large models.}
\end{frame}

\begin{frame}{Hope improves language modeling at both reported scales}
\begin{center}\small
\begin{tabular}{llrrr}
\toprule
Parameters / tokens & Model & Wiki PPL$\downarrow$ & LMB PPL$\downarrow$ & Avg.$\uparrow$\\
\midrule
760M / 30B & Transformer++ & 24.18 & 24.27 & 50.11\\
& Titans & 20.08 & 21.52 & 51.68\\
& \textbf{Hope} & \textbf{18.68} & \textbf{20.07} & \textbf{52.28}\\
\midrule
1.3B / 100B & Transformer++ & 17.92 & 17.73 & 53.38\\
& Titans & 15.60 & 11.41 & 56.82\\
& \textbf{Hope} & \textbf{14.39} & \textbf{10.08} & \textbf{58.04}\\
\bottomrule
\end{tabular}\end{center}
\begin{itemize}
\item At 1.3B vs. Titans: Wiki PPL $-7.76\%$; average accuracy $+1.22$ pp.
\item Eight-task average; individual tasks still have exceptions (e.g., BoolQ).
\item Model size and token budget change together across the two scales.
\end{itemize}
\papersource{Paper, Table 2, PDF p. 37; training details pp. 37--38. PPL = perplexity; LMB = LAMBADA; pp = percentage points.}
\end{frame}

\begin{frame}{Natural recall still favors full attention}
\begin{columns}[T]
\column{0.49\textwidth}
\begin{block}{Short-context recall: scores $\uparrow$}
\centering
\begin{tabular}{lrrr}
\toprule
Model & SWDE & NQ & FDA\\
\midrule
Transformer & 71.4 & 22.0 & 67.3\\
Titans & 60.8 & 20.3 & 37.6\\
Hope & 65.9 & 21.2 & 41.9\\
\bottomrule
\end{tabular}
\end{block}
\column{0.48\textwidth}
\begin{block}{Synthetic MAD: scores $\uparrow$}
Hope vs. Transformer:
\begin{itemize}
\item Fuzzy ICR: 52.1 vs. 47.9
\item Selective copy: 99.7 vs. 96.2
\item Memory: 85.2 vs. 83.7
\item Standard ICR: all models 100
\end{itemize}
\end{block}
\end{columns}
\medskip
Different strengths: precise access to stored context versus learned state updates.
\papersource{Paper, Tables 3--4, PDF p. 38. Selected columns; Table 3 does not specify a common EM/F1 scoring convention.}
\end{frame}

\begin{frame}{State tracking benefits from nonlinear recurrence}
\begin{center}
\begin{tabular}{lccc}
\toprule
Model & Parity (Bin0/Bin1) & $(aa)^*$ & $(abab)^*$\\
\midrule
Transformer & 46.4 / 0.0 & 0.0 / 0.0 & 0.0 / 0.0\\
DeltaNet & 98.2 / 10.1 & 0.0 / 0.0 & 0.0 / 0.0\\
LSTM & 100 / 100 & 100 / 100 & 100 / 100\\
SRWM & 100 / 100 & 100 / 100 & 100 / 100\\
\textbf{Hope} & \textbf{100 / 100} & \textbf{100 / 100} & \textbf{100 / 100}\\
\bottomrule
\end{tabular}
\end{center}
\begin{itemize}
\item The shown non-star-free languages distinguish the models.
\item All models score 100 on the table's three counter-language tasks.
\item Hope also supports chunked training; the table provides no speed comparison.
\end{itemize}
\papersource{Paper, Table 5 and Sec. 9.5, PDF pp. 38--39. Accuracies (\%); exact Bin0/Bin1 length ranges are not specified in v1.}
\end{frame}

\begin{frame}{Ablations support several components, with a q tradeoff}
\begin{center}
\begin{tabular}{lrr}
\toprule
Variant & LM PPL$\downarrow$ & Reasoning accuracy$\uparrow$\\
\midrule
Hope & 12.24 & 58.1\\
Without DGD & 13.41 & 56.5\\
Without momentum & 13.58 & 56.9\\
Without CMS & 13.04 & 57.3\\
Fixed v projection & 13.90 & 55.1\\
Fixed q projection & \textbf{12.19} & 57.4\\
\bottomrule
\end{tabular}
\end{center}
\begin{itemize}
\item Value-projection adaptation has the largest accuracy effect among these rows.
\item Removing q adaptation slightly improves PPL but reduces accuracy.
\end{itemize}
\papersource{Paper, Table 6, PDF p. 39. These averaged PPL values differ from individual datasets in Table 2. The final q configuration is inconsistent across the text and ablation.}
\end{frame}

\begin{frame}{M3 extends gradient memory, at a runtime cost}
\[
\Delta\Theta_t\propto-\frac{\operatorname{NS}(M_t^{(1)})+
\alpha\operatorname{NS}(M_t^{(2)})}{\sqrt{V_t+\epsilon}}
\]
\begin{columns}[T]
\column{0.49\textwidth}
\centering\small 24M ViT: lower left is better\par
\includegraphics[width=0.88\linewidth,height=3.05cm,keepaspectratio]{figures/fig11-vit24m.png}
\column{0.49\textwidth}
\centering\small Transformer training time: lower is better\par
\includegraphics[width=0.88\linewidth,height=3.05cm,keepaspectratio]{figures/fig12-optimizer-time.png}
\end{columns}
\smallskip
Fast/slow momentum $+$ orthogonalization $+$ second-moment scaling.\par
M3 improves the displayed loss tradeoff but takes longer than Muon.
\papersource{Paper, Algorithm 1, PDF p. 29; Figs. 11--12 and Sec. 9.7, pp. 39--40. Formula summarizes the mechanism. Timing compares optimizers, not Hope vs. Transformer.}
\end{frame}

\begin{frame}{The design opportunity is persistent, multi-rate learning}
\begin{enumerate}
\item \textbf{Represent the whole learning system.}\par
Weights, fast memories, and optimizer states all retain different histories.
\medskip
\item \textbf{Design update rates and transfer paths together.}\par
CMS supplies multiple time scales; self-generated targets make fast memory adaptive.
\medskip
\item \textbf{Judge the complete tradeoff.}\par
Reported gains coexist with recall gaps, adaptation costs, and unresolved implementation details in v1.
\end{enumerate}
\medskip
\alert{Finite-capacity compression still forgets. The paper explicitly says forgetting is not solved.}
\papersource{Paper, Sec. 10, PDF p. 40; assessment based on Tables 1--6 and the method analysis.}
\end{frame}

\begin{frame}{Paper and discussion material}
\textbf{Ali Behrouz, Meisam Razaviyayn, Peilin Zhong, Vahab Mirrokni.}\par
\medskip
\emph{Nested Learning: The Illusion of Deep Learning Architectures}.\par
arXiv:2512.24695v1, submitted 31 December 2025.
\medskip
\url{https://arxiv.org/abs/2512.24695v1}
\begin{itemize}
\item All experimental numbers and plots are from this version.
\item Figures retain the authors' original axes and legends.
\item Derivations and interpretive qualifications are identified where used.
\end{itemize}
\medskip
The following slides provide backup detail for discussion.
\papersource{Companion: reading-notes.zh.md. The PDF title page uses singular ``Architecture''; the arXiv record uses plural ``Architectures''.}
\end{frame}

\appendix

\begin{frame}{Backup: levels need an explicit knowledge-transfer path}
\begin{center}\small
\begin{tabular}{>{\raggedright\arraybackslash}p{3.0cm}>{\raggedright\arraybackslash}p{5.0cm}>{\raggedright\arraybackslash}p{4.8cm}}
\toprule
Connection & Operation & Example\\
\midrule
Conditioning & Read another state or its context & Fast-weight retrieval; attention\\[2mm]
Backpropagation & Propagate a shared task signal through modules & Sequential CMS\\[2mm]
Initialization & Learn the start state of a faster process & MAML; memory initialization\\[2mm]
Generation & Produce weights or training signals for another process & Hypernetworks; network-generated gradients\\
\bottomrule
\end{tabular}
\end{center}
\medskip
For each level, specify the context, objective, parameter space, update schedule, and reset policy.
\papersource{Paper, Sec. 3.3, PDF pp. 13--15; CMS variants, Sec. 7.1, p. 28.}
\end{frame}

\begin{frame}{Backup: a regression interpretation of Adam-like updates}
A scalar version of the internal objective:
\[
\min_m\sum_{i=1}^t(mg_i-p_t)^2+\lambda m^2
\quad\Longrightarrow\quad
m^*=\frac{p_t\sum_i g_i}{\sum_i g_i^2+\lambda}.
\]
Choose $p_t=\sqrt{\sum_i g_i^2}$ and let $\lambda\rightarrow0$:
\[
m^*=\frac{\sum_i g_i}{\sqrt{\sum_i g_i^2}}.
\]
\begin{itemize}
\item This produces the first-/second-moment normalization structure.
\item The printed cumulative updates omit standard Adam's EMA decay and bias correction.
\item The optimum is for this constructed regression, not for the model's task loss.
\end{itemize}
\papersource{Paper, Appendix B, Eqs. (101)--(105), PDF pp. 50--51. Scalar simplification and qualifications are this reading's analysis.}
\end{frame}

\begin{frame}{Backup: derive DGD from the stated proximal objective}
With $\vct u$ fixed and $\eta>0$, stationarity gives
\[
(W^+\vct x-\vct u)\vct x^\top+\eta^{-1}(W^+-W_t)=0.
\]
Thus
\[
W^+(I+\eta\vct x\vct x^\top)=W_t+\eta\vct u\vct x^\top.
\]
Using $(I+\eta\vct x\vct x^\top)^{-1}
=I-\frac{\eta\vct x\vct x^\top}{1+\eta\|\vct x\|^2}$,
\[
W^+=W_t-\frac{\eta}{1+\eta\|\vct x\|^2}
(W_t\vct x-\vct u)\vct x^\top.
\]
\begin{itemize}
\item Set $\vct u=-\nabla_{\vct y}\mathcal L$ to obtain the main-slide equation.
\item If the task gradient is zero, DGD can still shrink $W_t\vct x$.
\end{itemize}
\papersource{Re-derived from paper Eq. (56), PDF p. 20 / Eq. (113), p. 52. Appendix C changes signs and proximal coefficients in intermediate steps.}
\end{frame}

\begin{frame}{Backup: v1 leaves several implementation choices unclear}
\begin{itemize}
\item \textbf{q projection:} p. 31 says it is non-adaptive, but the update set and Table 6 include q adaptation.
\medskip
\item \textbf{Chunk indexing:} Eq. (90) uses $C\lceil t/C\rceil$; the surrounding text requires the previous chunk's end state. A causal chunk-start index is $C\lfloor(t-1)/C\rfloor$.
\medskip
\item \textbf{M3:} Algorithm 1 prints additive accumulators, not ordinary EMA updates; the slow-state recurrence has inconsistent indices.
\medskip
\item \textbf{Muon interpretation:} the gradient in Eq. (44) contains a closeness term absent from Eq. (43)'s objective.
\end{itemize}
\papersource{Paper, PDF pp. 17, 29, 31--33, 39. These are specification issues; no inference about unseen implementation or data leakage is made.}
\end{frame}

\begin{frame}{Backup: two additional empirical patterns}
\begin{columns}[T]
\column{0.49\textwidth}
\centering More context reduces PPL\par\smallskip
\includegraphics[width=\linewidth,height=3.55cm,keepaspectratio]{figures/fig10-context.png}
\par\smallskip
\small Hope benefits from context up to 32K in the plotted experiment.
\column{0.49\textwidth}
\centering 86M ViT shows a loss tradeoff\par\smallskip
\includegraphics[width=\linewidth,height=3.55cm,keepaspectratio]{figures/fig11-vit86m.png}
\par\smallskip
\small M3 has low test-loss points; Muon has points with lower train loss.
\end{columns}
\papersource{Paper, Figs. 10 and 11 (right panel), PDF p. 39. Curves and scatter points are unmodified author images.}
\end{frame}

\begin{frame}{Backup: preserve the distinction between three claims}
\begin{block}{Mathematical representation}
Certain updates can be expressed as associative-memory optimization. Exact equalities require consistent objectives and conventions.
\end{block}
\begin{block}{Design hypothesis}
Multiple update rates and self-generated targets may improve adaptation and retention.
\end{block}
\begin{block}{Empirical finding}
Hope improves several reported tasks; exact recall, compute matching, and long-term retention remain consequential evaluation dimensions.
\end{block}
\papersource{Discussion synthesis based on paper Secs. 3--10 and Appendices A--C, arXiv:2512.24695v1.}
\end{frame}

\end{document}