% Compile with pdflatex. Replace the demonstration content before delivery.
\documentclass[aspectratio=169,11pt]{beamer}
\usepackage[T1]{fontenc}
\usepackage[utf8]{inputenc}
\usepackage{amsmath,amssymb,graphicx,booktabs}
\usepackage{lmodern}

\definecolor{AcademicBlue}{RGB}{26,57,86}
\definecolor{AcademicTeal}{RGB}{0,105,112}
\setbeamercolor{structure}{fg=AcademicBlue}
\setbeamercolor{frametitle}{fg=AcademicBlue,bg=white}
\setbeamercolor{title}{fg=AcademicBlue}
\setbeamercolor{alerted text}{fg=AcademicTeal}
\setbeamercolor{block title}{fg=white,bg=AcademicBlue}
\setbeamercolor{block body}{fg=black,bg=AcademicBlue!5}
\setbeamertemplate{navigation symbols}{}
\setbeamertemplate{itemize items}[circle]
\setbeamertemplate{footline}{%
  \hfill\usebeamerfont{page number in head/foot}%
  \insertframenumber\,/\,\inserttotalframenumber\hspace{4mm}\vspace{3mm}}
\setbeamersize{text margin left=8mm,text margin right=8mm}
\setbeamerfont{frametitle}{size=\Large,series=\bfseries}
\setbeamerfont{title}{size=\LARGE,series=\bfseries}
\newcommand{\papersource}[1]{\par\smallskip{\scriptsize\color{gray}#1\par}}

\begin{document}
\begin{frame}{Proteus: when should memory become available?}
\begin{center}
{\Large Incremental Memory Activation\\for Long-Context Sequence Modeling}\par\bigskip
Reza Bayat, Ali Behrouz, Vahab Mirrokni, Aaron Courville\par\medskip
\href{https://arxiv.org/abs/2608.16844v1}{arXiv:2608.16844v1 --- 17 August 2026}\par\smallskip
Paper discussion prepared on 16 September 2026
\end{center}
\begin{block}{One-sentence takeaway}
A fixed memory budget can work better when its read/write capacity is unlocked progressively.
\end{block}
\end{frame}
\begin{frame}{A fixed memory budget still leaves a scheduling choice}
\begin{itemize}
\item Attention retains historical keys and values; full attention has quadratic sequence-processing cost.
\item Recurrent memory compresses history into a fixed-size state, but long-context retrieval remains difficult.
\item Common design axes: memory architecture, initialization, objective, and update rule.
\item Proteus asks a different question: how much capacity should be accessible at each position?
\end{itemize}
\papersource{Secs. 1--2, PDF pp. 2--4.}
\end{frame}
\begin{frame}{The proposed mechanism targets two failure modes}
\begin{columns}[T]
\begin{column}{0.47\textwidth}\begin{block}{Early context}
Full capacity may fit early inputs too freely.\\\medskip
\textbf{Intervention:} impose a small active subspace.
\end{block}\end{column}
\begin{column}{0.47\textwidth}\begin{block}{Later context}
New writes may interfere with an already-written state.\\\medskip
\textbf{Intervention:} unlock unused blocks over time.
\end{block}\end{column}
\end{columns}\medskip
These are the authors' explanations. The paper does not directly measure memory pollution or information compression.
\papersource{Secs. 2.4--3, PDF pp. 4--5; evidence boundary: reader analysis.}
\end{frame}
\begin{frame}{New blocks open while old blocks remain writable}
\centering
\includegraphics[width=\linewidth,height=0.56\textheight,keepaspectratio]{figures/proteus.png}
\par\medskip\raggedright
Total state size is constant. Previously activated blocks remain part of the read/write set; old content is not frozen.
\papersource{Original Fig. 1, PDF p. 1.}
\end{frame}
\begin{frame}{Online memory learning differs from model training}
\[
\mathcal M_t=\mathcal M_{t-1}-\theta_t\nabla\widetilde{\mathcal L}(\mathcal M_{t-1};k_t,v_t),
\qquad y_t=\mathrm{Read}(\mathcal M_{t-1},q_t).
\]
\begin{itemize}
\item $k_t,v_t,q_t$: key, value, query; $\theta_t$: online memory step size.
\item $\widetilde{\mathcal L}$ fits key--value associations; the outer loss $L$ trains next-token prediction.
\item Hebbian, delta-rule, and Titans updates instantiate different memory learners.
\item Proteus schedules accessible coordinates, rather than replacing the inner objective.
\end{itemize}
\papersource{Sec. 2, Eqs. (1)--(5), PDF pp. 3--4.}
\end{frame}
\begin{frame}{Gate the state used for both reading and learning}
\[
\mathcal M_{t-1}^{(g)}=g_t\odot\mathcal M_{t-1},\qquad
 y_t=\mathrm{Read}(\mathcal M_{t-1}^{(g)},q_t)
\]
\[
\mathcal M_t=\mathcal M_{t-1}-\theta_t\left[g_t\odot
\nabla\widetilde{\mathcal L}(\mathcal M_{t-1}^{(g)};k_t,v_t)\right].
\]
\begin{itemize}
\item $g_t$ is a binary mask over memory coordinates.
\item Compute with the active state, then apply only the active update to the full state.
\item Locked coordinates must satisfy $(\mathbf 1-g_t)\odot\mathcal M_t=(\mathbf 1-g_t)\odot\mathcal M_{t-1}$.
\end{itemize}
\papersource{Definition 2, Eq. (7), p. 5; Eqs. (8)--(11), p. 6.}
\end{frame}
\begin{frame}{A deterministic prefix mask controls effective capacity}
\[
\Delta=\max\left(1,\left\lfloor\frac{N}{E}\right\rfloor\right),\qquad
k(t)=\min\left(E,1+\left\lfloor\frac{t-1}{\Delta}\right\rfloor\right).
\]
\[
g_t[j]=\begin{cases}1,&1\le j\le k(t)d,\\0,&\text{otherwise},\end{cases}
\qquad d=\dim(\mathcal M)/E.
\]
\begin{itemize}
\item $N$: training context length; $E$: block count; $d$: coordinates per equal block.
\item $k(t)$ is a block count, distinct from the key vector $k_t$.
\item Default $E=16$; $E=1$ recovers the base model.
\end{itemize}
\papersource{Sec. 4.1, Eqs. (12)--(13) and intervening definition, PDF p. 6.}
\end{frame}
\begin{frame}{The schedule finishes inside the training window}
\begin{center}
\begin{tabular}{lcc}\toprule
Token positions & Active blocks & Active fraction\\\midrule
1--512 & 1 & 1/16\\
513--1024 & 2 & 2/16\\
3585--4096 & 8 & 8/16\\
7681 onward & 16 & 1\\\bottomrule
\end{tabular}\end{center}
\medskip
Example computed from $N=8192$, $E=16$, $\Delta=512$.\par\medskip
\alert{At 16K or 32K, no new blocks are being unlocked.} Evaluation keeps the original schedule.
\papersource{Teaching example derived from Sec. 4.1, PDF p. 6; not an extra experiment.}
\end{frame}
\begin{frame}{Hope extends the idea to a different time axis}
\[
\boldsymbol\theta_{i+1}=\boldsymbol\theta_i-
\eta_{i+1}(g_{i+1}\odot\boldsymbol e_i).
\]
\begin{itemize}
\item Here $i$ indexes training samples/updates, not positions within one context.
\item Hope-Attention passes attention output through a chain of MLP blocks updated at different frequencies.
\item The extension gates parameter updates over training; it is demonstrated on one architecture.
\item Forward masking and the detailed mapping to training steps are not fully specified.
\end{itemize}
\papersource{Sec. 4.2, Eqs. (14)--(18), PDF pp. 6--8; implementation boundary: reader analysis.}
\end{frame}
\begin{frame}{Evaluation pairs each backbone with its Proteus variant}
\begin{center}\begin{tabular}{ll}\toprule
Training data & FineWeb\\
Scale / training tokens & 760M / 50B; 1.3B / 100B\\
Context / partition count & 8K / $E=16$\\
Optimizer & AdamW, $4\times10^{-4}$, cosine schedule\\
Batch / weight decay & 0.5M tokens / 0.1\\\bottomrule\end{tabular}\end{center}
\medskip
Paired families: Hope-Attention, SWLA, Comba, Titans.\par\smallskip
Reference models include Transformer++, RetNet, and DeltaNet.\par\smallskip
Long-context tables do not explicitly identify the model scale per result.
\papersource{Sec. 5 setup, PDF p. 8; Tables 1--3, pp. 7, 9--10.}
\end{frame}
\begin{frame}{Mean downstream accuracy improves in all eight pairs}
\centering
\begin{tabular}{lrr}\toprule
Backbone & 760M: base $\to$ Proteus & 1.3B: base $\to$ Proteus\\\midrule
Hope-Attention & $53.15\to53.99$ & $56.93\to57.30$\\
SWLA & $50.12\to50.90$ & $53.67\to54.20$\\
Comba & $51.43\to52.15$ & $55.42\to56.00$\\
Titans & $52.65\to53.36$ & $56.95\to58.00$\\\bottomrule
\end{tabular}\par\medskip\raggedright
Avg. is the mean of eight accuracy columns: LAMBADA plus seven commonsense benchmarks. HellaSwag and ARC-c use normalized accuracy.\par\medskip
Improvements span 0.37--1.05 percentage points. No variance estimates are reported.
\papersource{Selected values from Table 1, PDF p. 7; differences computed from reported means.}
\end{frame}
\begin{frame}{Perplexity improves often, but not universally}
\begin{center}\begin{tabular}{llr}\toprule
Scale / backbone & Metric & Base $\to$ Proteus\\\midrule
760M Hope & Wiki. ppl & $20.67\to19.87$\\
760M Hope & LAMBADA ppl & $21.36\to19.72$\\
1.3B Titans & Wiki. ppl & $15.36\to14.94$\\
1.3B Hope & Wiki. ppl & $15.66\to15.70$\\
1.3B SWLA & LAMBADA ppl & $16.95\to17.18$\\\bottomrule\end{tabular}\end{center}
\medskip
Lower is better. At 1.3B, the best LAMBADA ppl is \textbf{13.01 (Hope+Proteus)}, not 13.03 (Titans+Proteus), despite the prose claim.
\papersource{Table 1, PDF p. 7; conflicting best-ppl statement in Sec. 5.1, p. 9.}
\end{frame}
\begin{frame}{Hard 16K retrieval shows larger absolute gains}
\centering
\begin{tabular}{llrr}\toprule
Task & Backbone & Base $\to$ Proteus & Gain (pp)\\\midrule
S-NIAH-2 & Comba & $13.4\to21.2$ & +7.8\\
S-NIAH-3 & Titans & $21.4\to29.8$ & +8.4\\
MK-NIAH-1 & Titans & $11.8\to16.8$ & +5.0\\
MQ-NIAH & Hope & $30.6\to34.2$ & +3.6\\
MV-NIAH & Hope & $23.0\to25.8$ & +2.8\\\bottomrule
\end{tabular}\par\medskip\raggedright
S-NIAH-2/3: numeric/UUID needles; MK/MQ/MV: multiple keys/queries/values.\par\smallskip
The +8.4 pp UUID gain still leaves Titans at 29.8: difficult retrieval remains unsolved.
\papersource{Selected 16K results from Table 2, PDF p. 9.}
\end{frame}
\begin{frame}{Short-context and saturated tasks limit the claim}
\begin{itemize}
\item S-NIAH-1 is already near 100 for most paired models: little room to improve.
\item Comba, S-NIAH-3 at 4K: $62.4\to60.8$.
\item Hope, MK-NIAH-1 at 4K: $81.4\to80.8$.
\item At 16K, Comba/Titans remain well below Transformer/Hope on multi-needle tasks.
\item Long-context benefits are compatible with better early compression; they do not isolate that causal mechanism.
\end{itemize}
\papersource{Table 2, PDF p. 9; interpretation boundary: reader analysis.}
\end{frame}
\begin{frame}{Retrieval benefits depend on the dataset and backbone}
\centering
\includegraphics[width=\linewidth,height=0.57\textheight,keepaspectratio]{figures/longbench.png}
\par\medskip\raggedright
Read within each panel and backbone. SWDE/FDA show stronger length degradation; SQuAD does not show the same downward trend. No error bars are given.
\papersource{Original Fig. 2, PDF p. 8; discussion in Sec. 5.3, p. 10.}
\end{frame}
\begin{frame}{Six-task LongBench means improve modestly}
\begin{center}\begin{tabular}{lrrr}\toprule
Backbone & Base & Proteus & Difference\\\midrule
Hope-Attention & 15.72 & 16.65 & +0.93\\
Comba & 13.05 & 13.23 & +0.18\\
Titans & 13.80 & 14.15 & +0.35\\\bottomrule\end{tabular}\end{center}
\begin{itemize}
\item Six tasks: Narrative, Qasper, MultiField, Hotpot, 2WikiMulti, Musique.
\item Exceptions: Comba Hotpot $14.6\to14.5$; Titans Musique $6.7\to6.7$.
\item These are mean task scores for the reported subset, not a full LongBench aggregate.
\end{itemize}
\papersource{Table 3, PDF p. 10.}
\end{frame}
\begin{frame}{More partitions help only up to a point}
\begin{columns}
\begin{column}{0.57\textwidth}\includegraphics[width=\linewidth,height=0.67\textheight,keepaspectratio]{figures/blocks.pdf}\end{column}
\begin{column}{0.4\textwidth}
\begin{itemize}
\item $E=1$: no scheduling.
\item $E=16$: best tested setting.
\item $E=32$: perplexity rises again.
\item Block count changes both initial capacity and unlock frequency.
\end{itemize}
\end{column}\end{columns}
\papersource{Original Fig. 3 and Appendix A, PDF p. 18. No error bars; exact values not tabulated.}
\end{frame}
\begin{frame}{An advantage persists after all blocks are open}
\begin{columns}
\begin{column}{0.62\textwidth}\includegraphics[width=\linewidth,height=0.67\textheight,keepaspectratio]{figures/ppl.pdf}\end{column}
\begin{column}{0.35\textwidth}\begin{itemize}
\item Blue: Titans+Proteus; red: Titans.
\item Lower ppl at all plotted points.
\item Large gap near 8K; positive gap at 32K.
\item No new blocks beyond the training window.
\end{itemize}\end{column}\end{columns}
\papersource{Original Fig. 4 and Appendix A, PDF p. 18. Discrete plotted points, not every token.}
\end{frame}
\begin{frame}{The evidence supports scheduling, with clear limits}
\begin{itemize}
\item Cross-backbone mean gains make capacity scheduling a useful design axis.
\item No new learned parameters or larger main state; runtime overhead is not benchmarked.
\item No random-seed variation or significance estimates are reported.
\item No causal separation of early bottleneck versus fresh later capacity.
\item Fixed hand-designed schedule; one-architecture MLP proof of concept.
\end{itemize}
\papersource{Secs. 5 and 7; Appendix A. Cost/statistical boundaries: reader analysis.}
\end{frame}
\begin{frame}{Three takeaways for a research discussion}
\begin{enumerate}
\item \textbf{Capacity has a time dimension.} A fixed budget need not be accessible uniformly.
\item \textbf{Longer-context gains do not require continued expansion.} Here the schedule has already saturated by 8K.
\item \textbf{Mechanism and implementation deserve further tests.} Separate early regularization, later free capacity, and optimizer effects.
\end{enumerate}\medskip
Learned schedules, genuinely growing memory, and stretched inference schedules are future directions, not evaluated solutions.
\papersource{Sec. 7, PDF pp. 11--12; discussion synthesis.}
\end{frame}
\begin{frame}{Reference and reading materials}
\textbf{Primary source}\par\medskip
Reza Bayat, Ali Behrouz, Vahab Mirrokni, and Aaron Courville.\par
\emph{Proteus: Incremental Memory Activation for Long-Context Sequence Modeling.}\par
\href{https://arxiv.org/abs/2608.16844v1}{arXiv:2608.16844v1}, 17 August 2026.\par\bigskip
This presentation uses the version-matched author TeX, three source tables, four original figures, and the final 18-page PDF.\par\medskip
Figures are reproduced from the paper. Selected tables are re-typeset from reported values. Teaching calculations and reader critiques are labeled.
\end{frame}
\begin{frame}{Backup: dimension conventions need care}
For column vectors, $\mathcal M k_t\approx v_t$ implies
\[
\mathcal M\in\mathbb R^{d_v\times d_k},\qquad
\nabla_{\mathcal M}\tfrac12\|\mathcal M k_t-v_t\|^2
=(\mathcal M k_t-v_t)k_t^\top.
\]
Thus a consistent delta update is
\[
\mathcal M_t=\mathcal M_{t-1}(I-\theta_t k_tk_t^\top)
+\theta_t v_tk_t^\top.
\]
The paper's Eq. (4) instead left-multiplies the erase factor, while p. 5 states the transposed matrix shape. This is a notation concern, not evidence that the experiments used incorrect code.
\papersource{Reader derivation; compare Eqs. (3)--(4), PDF p. 3, and matrix shape on p. 5.}
\end{frame}
\begin{frame}{Backup: masked gradients do not mask weight decay}
The paper's AdamW expression includes
\[
\boldsymbol\theta_{i+1}=\alpha_{i+1}\boldsymbol\theta_i
-\eta_{i+1}\frac{\hat{\boldsymbol m}_{i+1}}
{\sqrt{\hat{\boldsymbol v}_{i+1}}+\varepsilon},
\]
while moments use masked gradients.\par\medskip
\begin{itemize}
\item Even with zero moments, an inactive nonzero parameter changes if decay is unmasked.
\item A monotone mask avoids previous nonzero momentum in never-active coordinates, but does not remove this decay issue.
\item Exact freezing, forward masking, and the schedule-to-training-step mapping require clarification.
\end{itemize}
\papersource{Eq. (15), pp. 7--8; discrepancy with the freezing description: reader analysis.}
\end{frame}
\end{document}