% Adapted from arxiv-paper-reader/assets/beamer-template.tex.
% Main talk: frames 1--22, about 25 minutes. Frames 23--26 are backups.
\documentclass[aspectratio=169,11pt]{beamer}
\usepackage[T1]{fontenc}
\usepackage[utf8]{inputenc}
\usepackage{lmodern}
\usepackage{amsmath,amssymb,graphicx,booktabs}
\definecolor{AcademicBlue}{RGB}{26,57,86}
\definecolor{AcademicTeal}{RGB}{0,105,112}
\setbeamercolor{structure}{fg=AcademicBlue}
\setbeamercolor{frametitle}{fg=AcademicBlue,bg=white}
\setbeamercolor{title}{fg=AcademicBlue}
\setbeamercolor{alerted text}{fg=AcademicTeal}
\setbeamercolor{block title}{fg=white,bg=AcademicBlue}
\setbeamercolor{block body}{fg=black,bg=AcademicBlue!5}
\setbeamertemplate{navigation symbols}{}
\setbeamertemplate{itemize items}[circle]
\setbeamertemplate{footline}{\hfill\usebeamerfont{page number in head/foot}\insertframenumber\,/\,\inserttotalframenumber\hspace{4mm}\vspace{3mm}}
\setbeamersize{text margin left=8mm,text margin right=8mm}
\setbeamerfont{frametitle}{size=\Large,series=\bfseries}
\setbeamerfont{title}{size=\LARGE,series=\bfseries}
\newcommand{\papersource}[1]{\par\smallskip{\scriptsize\color{gray}#1\par}}
\newcommand{\M}{\mathcal M}
\newcommand{\W}{\mathbf W}
\newcommand{\vect}[1]{\mathbf #1}
\title[MoNe]{MoNe: Modular Neural Memory for Efficient Long Context Inference}
\author{Research paper discussion}
\date{arXiv:2608.17616v1, 18 August 2026}
\hypersetup{pdftitle={MoNe -- Research Paper Discussion},pdfauthor={Paper reading presentation},colorlinks=true,urlcolor=AcademicTeal}

\begin{document}

\begin{frame}
  \raggedright
  {\usebeamerfont{title}\color{AcademicBlue}\inserttitle\par}
  \par\vspace{4mm}
  {\small Paper authors: Wonguk Cho, Kyubyung Chae, Tribhuvanesh Orekondy,\par
  Sunghyun Park, Hyoungwoo Park, Jeongho Kim, Arash Behboodi,\par
  Kyuwoong Hwang, and Sungrack Yun\par}
  \par\smallskip
  Qualcomm AI Research
  \par\vspace{3mm}
  \begin{block}{Core idea}
    Write context into small neural memories; let a frozen Transformer\par
    read them through its existing attention interface.
  \end{block}
  \small Research discussion, 9 October 2026\par
  \href{https://arxiv.org/abs/2608.17616v1}{arXiv:2608.17616v1}\quad
  \textcolor{gray}{AdaptFM Workshop at ICML 2026}
\end{frame}

\begin{frame}{Can a frozen model use a fixed-size context state?}
  \begin{columns}[T]
  \begin{column}{.47\textwidth}
    \begin{block}{Full-context attention}
      Store token-level keys and values.
    \end{block}
    \begin{itemize}
      \item Attention prefill cost: $O(N^2)$.
      \item Context KV storage: $O(N)$.
      \item Small models may struggle even within their native window.
    \end{itemize}
  \end{column}
  \begin{column}{.47\textwidth}
    \begin{block}{MoNe's proposal}
      Store context in adaptable weights.
    \end{block}
    \begin{itemize}
      \item Keep pretrained backbone weights frozen.
      \item Learn a task-relevant write rule offline.
      \item Pay for context ingestion once, then reuse the state.
    \end{itemize}
  \end{column}
  \end{columns}
  \papersource{Paper, Secs. 1--2, PDF pp. 1--3. $N$: context tokens.}
\end{frame}

\begin{frame}{Separate context writing from question answering}
  \begin{columns}[c]
    \begin{column}{.66\textwidth}
      \includegraphics[width=\linewidth,height=.75\textheight,keepaspectratio]{figures/fig-01-overview.png}
    \end{column}
    \begin{column}{.30\textwidth}
      \textbf{Write}\par
      Consume 512-token segments and update each layer's memory.
      \par\vspace{5mm}
      \textbf{Read}\par
      Generate memory KV from the question and final state.
      \par\vspace{5mm}
      Original context tokens are absent at answering time.
    \end{column}
  \end{columns}
  \papersource{Paper, Fig. 1, PDF p. 1; Sec. 3, PDF pp. 3--4.}
\end{frame}

\begin{frame}{The backbone is frozen; the memory system is trained}
  \renewcommand{\arraystretch}{1.35}
  \begin{tabular}{p{.19\textwidth}p{.35\textwidth}p{.35\textwidth}}
    \toprule
    Stage & Objective / input & Parameters that change \\
    \midrule
    Offline learning & Answer generation after context writing & LoRA and update/read meta-parameters \\
    Context writing & Local key--value association & Per-layer fast weights and update state \\
    Answering & Question and generated tokens & Memory weights remain fixed \\
    \bottomrule
  \end{tabular}\par
  \par\vspace{4mm}
  \alert{No backbone retraining} still involves offline adaptation\par
  and gradient-based context ingestion at deployment.
  \papersource{Paper, Secs. 3.3--3.4, PDF p. 4; Appendix A, PDF p. 10.}
\end{frame}

\begin{frame}{A nonlinear fast-weight network carries the history}
  \[
    \M(\vect x;\W^{(l)})=
    \W_{\rm out}^{\top}
    \bigl[\operatorname{SiLU}(\W_{\rm in}\vect x)
    \odot(\W_{\rm gate}\vect x)\bigr]
  \]
  \begin{itemize}
    \item Three $d_h\times d_h$ matrices per memory head.
    \item Experimental configuration: $H=4$, $d_h=224$, all 24 layers.
    \item Fixed-size state; authors report 6.4\% total parameter overhead.
    \item The memory output becomes an additional attention input.
  \end{itemize}
  \begin{block}{Capacity implication}
    A fixed state can learn useful associations, but information from\par
    different context segments must share that state.
  \end{block}
  \papersource{Paper, Eq. (3), PDF p. 3; Appendix A.2, PDF p. 10. Head dimensions refer to memory.}
\end{frame}

\begin{frame}{Writing maximizes agreement with layer-local values}
  \[
    \ell_{s,j}^{(l)}(\W)=-\bigl(\vect v_{s,j}^{(l)}\bigr)^\top
    \M(\vect k_{s,j}^{(l)};\W),\qquad
    \mathcal L_s^{(l)}=\frac1T\sum_{j=1}^{T}\ell_{s,j}^{(l)}
  \]
  \[
    \W_s^{(l)}=\W_{s-1}^{(l)}-\boldsymbol\mu_s^{(l)}
  \]
  \begin{itemize}
    \item $\vect k,\vect v$ come from the current layer's segment activations.
    \item $\boldsymbol\mu$: gradients with learned per-token rates and momentum.
    \item Key/query: affine rescaling, SiLU, token-wise L2 normalization.
    \item Value: SiLU; weights: channel-wise L2 renormalization after updates.
  \end{itemize}
  \papersource{Paper, Eqs. (6)--(8), PDF p. 4; Appendix A.2, PDF p. 10. The loss uses a negative inner product.}
\end{frame}

\begin{frame}{Local write losses avoid a full backward pass}
  \begin{columns}[c]
    \begin{column}{.57\textwidth}
      \includegraphics[width=\linewidth,height=.68\textheight,keepaspectratio]{figures/fig-03-local-update.png}
    \end{column}
    \begin{column}{.39\textwidth}
      \begin{itemize}
        \item Each write loss updates only its layer's memory.
        \item No answer-logit gradient must traverse subsequent layers.
        \item Forward activations still flow through the backbone.
      \end{itemize}
      \par\smallskip
      This boundary describes test-time writing; offline training uses answer supervision.
    \end{column}
  \end{columns}
  \papersource{Paper, Fig. 3, PDF p. 3; Eqs. (10)--(11), PDF p. 4.}
\end{frame}

\begin{frame}{The question reads memory through extra attention KV}
  \[
    \vect h_j=\M(\W_q\vect x_j;\W_S^{(l)})
  \]
  \[
    \mathrm{out}=\operatorname{Attention}\bigl(
    \vect Q,[\vect K\,\|\,\W_k\vect h],
    [\vect V\,\|\,\W_v\vect h]\bigr)
  \]
  \begin{itemize}
    \item Read the state after all $S$ context segments.
    \item Apply RMSNorm and a learned gate to the raw memory output.
    \item Reuse backbone projections, with offline-trained LoRA adapters.
    \item Attention combines ordinary query entries with memory entries.
  \end{itemize}
  \papersource{Paper, Eqs. (4)--(5), PDF p. 3; Sec. 3.2, PDF pp. 3--4. Equations omit normalization details.}
\end{frame}

\begin{frame}{Local positions stay in range as context grows}
  \[
    \underbrace{\vect s_1}_{[0,T)}\ \longrightarrow\
    \underbrace{\vect s_2}_{[0,T)}\ \longrightarrow\ \cdots\ \longrightarrow\
    \underbrace{\vect s_S}_{[0,T)}
  \]
  \begin{itemize}
    \item Segment-local RoPE reuses positions in $[0,T)$; $T=512$.
    \item Query positions lie in $[0,Q)$, assuming $Q\ll T$.
    \item Segment order enters through the sequence of memory updates.
  \end{itemize}
  \begin{block}{Two different extrapolation factors}
    Offline training: up to 4K $\rightarrow$ evaluation: 128K \quad ($32\times$).\par
    Backbone native window: 32K $\rightarrow$ evaluation: 128K \quad ($4\times$).
  \end{block}
  Bounded positions remove a positional obstacle; memory capacity remains finite.
  \papersource{Paper, Sec. 3.3, PDF p. 4; Table 1, PDF p. 5; Appendix A.1, PDF p. 10.}
\end{frame}

\begin{frame}{The constant query cost is relative to context length}
  \[
    F_{\rm write}(N)=\left\lceil\frac NT\right\rceil c(T)=O(N)
  \]
  \[
    F_{\rm total}=F_{\rm write}(N)+
    \sum_{i=1}^{m}F_{\rm query}(Q_i,A_i)
  \]
  \begin{itemize}
    \item Fix segment size, backbone, memory size, and answer length.
    \item Query cost does not grow with the original context length $N$.
    \item $Q_i$ and $A_i$ still affect compute and generation storage.
    \item Multi-query reuse amortizes writing; ICL can also reuse prefix KV.
  \end{itemize}
  \papersource{Paper, Sec. 1, PDF p. 2; Secs. 3--4.3, PDF pp. 3--6. Cost decomposition is an explanatory derivation.}
\end{frame}

\begin{frame}{The tasks test retrieval and frequency aggregation}
  \renewcommand{\arraystretch}{1.3}
  \begin{tabular}{p{.15\textwidth}p{.50\textwidth}p{.23\textwidth}}
    \toprule
    Task & What the model must recover & Metric $\uparrow$ \\
    \midrule
    S-NIAH & One planted key--value pair & Substring EM \\
    MK-NIAH & One queried value among four planted pairs; others are distractors & Substring EM \\
    FWE & Top three non-noise words from a synthetic Zipfian stream & Target-word recall \\
    \bottomrule
  \end{tabular}\par
  \par\vspace{4mm}
  MK-NIAH does not require chaining the four facts.\par
  FWE measures recall, despite the Sub-EM wording in Sec. 4.2.
  \papersource{Paper, Sec. 4.1, PDF p. 4; task definitions in Appendix A.1, PDF p. 10.}
\end{frame}

\begin{frame}{Shared backbone, different task-adaptation budgets}
  \begin{itemize}
    \item \textbf{Backbone:} Qwen2.5-0.5B-Instruct; native window 32K.
    \item \textbf{MoNe training:} 1K--4K contexts; 30K samples per segment count\par
      $S\in\{2,\ldots,8\}$, with 512-token segments.
    \item \textbf{Evaluation:} 100 samples per length setting; 4K--128K.
    \item \textbf{ICL:} use the full context as a prompt.
    \item \textbf{RAG:} 128-token chunks, BGE-Large, cosine retrieval;\par
      report the best $K\in\{1,4,8\}$ at each length.
  \end{itemize}
  \begin{block}{Comparison to keep in mind}
    MoNe receives offline task supervision for its adapters and update rule.\par
    The baselines are described as direct ICL and retrieval prompting.
  \end{block}
  \papersource{Paper, Sec. 4.1, PDF pp. 4--5; Sec. 3.3, PDF p. 4; Appendix A.1, PDF p. 10.}
\end{frame}

\begin{frame}{Gains already appear within the native window}
  \centering
  \renewcommand{\arraystretch}{1.16}
  \begin{tabular}{llrrr}
    \toprule
    Task & Length & ICL & RAG & MoNe \\
    \midrule
    S-NIAH & 4K & 0.95 & 0.94 & \textbf{1.00} \\
           & 32K & 0.94 & 0.93 & \textbf{1.00} \\
    MK-NIAH & 4K & 0.89 & 0.92 & \textbf{1.00} \\
            & 32K & 0.93 & 0.79 & \textbf{0.99} \\
    FWE & 4K & 0.59 & 0.58 & \textbf{1.00} \\
        & 32K & 0.41 & 0.61 & \textbf{1.00} \\
    \bottomrule
  \end{tabular}\par
  \par\vspace{3mm}
  \begin{block}{Largest gap: global frequency information}
    At 32K, FWE recall improves by 59 percentage points over ICL\par
    and 39 points over RAG.
  \end{block}
  \papersource{Paper, Table 1, PDF p. 5. NIAH: Sub-EM; FWE: variable recall. Differences calculated from table.}
\end{frame}

\begin{frame}{At 128K, MoNe retains high scores on all three tasks}
  \centering
  \renewcommand{\arraystretch}{1.5}
  \begin{tabular}{lrrrl}
    \toprule
    Task at 128K & ICL & RAG & MoNe & Gain vs. RAG \\
    \midrule
    S-NIAH & 0.28 & 0.89 & \textbf{0.96} & +7 pp \\
    MK-NIAH & 0.00 & 0.71 & \textbf{0.94} & +23 pp \\
    FWE & 0.23 & 0.60 & \textbf{0.96} & +36 pp \\
    \bottomrule
  \end{tabular}\par
  \par\vspace{5mm}
  \raggedright
  \begin{itemize}
    \item Training contexts stop at 4K.
    \item ICL is evaluated beyond the backbone's native 32K window.
    \item Evidence covers controlled retrieval and aggregation workloads.
  \end{itemize}
  \papersource{Paper, Table 1, PDF p. 5. NIAH: Sub-EM; FWE: variable recall. pp = percentage points.}
\end{frame}

\begin{frame}{Length scaling separates the methods clearly}
  \centering
  \includegraphics[width=\linewidth,height=.60\textheight,keepaspectratio]{figures/table-01-results-redrawn.pdf}
  \par\vspace{2mm}
  \raggedright
  The dashed boundary is 32K; shading marks the region beyond it.\par
  MoNe degrades gradually, while ICL retrieval falls sharply.
  \papersource{Redrawn from the exact values in Table 1, PDF p. 5. The right panel uses variable recall.}
\end{frame}

\begin{frame}{At 128K, total FLOPs and peak memory fall by about 80\%}
  \begin{columns}[c]
    \begin{column}{.64\textwidth}
      \includegraphics[width=\linewidth,height=.65\textheight,keepaspectratio]{figures/fig-02-cost.png}
    \end{column}
    \begin{column}{.32\textwidth}
      \textbf{Peak GPU memory}\par
      7.07 $\rightarrow$ 1.41 GB\par
      \alert{80.06\% lower}
      \par\vspace{5mm}
      \textbf{Total compute}\par
      786.33 $\rightarrow$ 149.61 T\par
      \alert{80.97\% lower}
      \par\vspace{5mm}
      Includes context writing\par
      and answering.
    \end{column}
  \end{columns}
  \par\smallskip
  Logarithmic axes. FLOPs are operation counts, not measured latency.
  \papersource{Paper, Fig. 2, PDF p. 1; Table 2, PDF p. 5. T FLOPs = $10^{12}$ operations. Offline training excluded.}
\end{frame}

\begin{frame}{Writing accounts for almost all single-query compute}
  \centering
  \renewcommand{\arraystretch}{1.3}
  \begin{tabular}{lrr}
    \toprule
    FLOPs (T) & 32K & 128K \\
    \midrule
    ICL & 58.06 & 786.33 \\
    MoNe: context writing & 37.24 & 148.97 \\
    MoNe: answering & 0.86 & 0.64 \\
    \textbf{MoNe total} & \textbf{38.10} & \textbf{149.61} \\
    \bottomrule
  \end{tabular}\par
  \par\vspace{4mm}
  \raggedright
  \begin{itemize}
    \item Writing grows about $4\times$ as context grows from 32K to 128K.
    \item Peak memory stays at 1.41 GB for writing and 1.29 GB for reading.
    \item At 128K, writing is 99.6\% of the reported total compute.
    \item Multi-query amortization is proposed, without a dedicated timing study.
  \end{itemize}
  \papersource{Paper, Table 2, PDF p. 5; multi-query reuse in Sec. 1, PDF p. 2. Peak memory uses max, not sum.}
\end{frame}

\begin{frame}{Removing memory from lower layers is costly in accuracy}
  \centering
  \includegraphics[width=.86\linewidth,height=.46\textheight,keepaspectratio]{figures/fig-04ab-layers.png}
  \par\smallskip
  \begin{tabular}{lrrr}
    \toprule
    At 128K & All 24 layers & Last 16 layers & Last 8 layers \\
    \midrule
    MK-NIAH Sub-EM & 0.94 & 0.70 & 0.02 \\
    FLOPs (T) & 149.1 & 132.6 & 116.1 \\
    \bottomrule
  \end{tabular}\par
  \papersource{Paper, Fig. 4(a--b) and Sec. 4.4, PDF p. 6. The ablation reports 149.1 T; Table 2 reports 149.61 T.}
\end{frame}

\begin{frame}{Larger segments give a better tested trade-off}
  \centering
  \includegraphics[width=.86\linewidth,height=.46\textheight,keepaspectratio]{figures/fig-04cd-segments.png}
  \par\smallskip
  \begin{tabular}{lrrr}
    \toprule
    Segment size & 512 & 256 & 128 \\
    \midrule
    64K MK-NIAH & 0.98 & 0.91 & 0.85 \\
    128K MK-NIAH & 0.94 & 0.75 & 0.53 \\
    \bottomrule
  \end{tabular}\par
  \papersource{Paper, Fig. 4(c--d) and Sec. 4.4, PDF p. 6. Smaller segments save little compute in these experiments.}
\end{frame}

\begin{frame}{The evidence is strongest for trained retrieval memory}
  \begin{itemize}
    \item \textbf{Demonstrated:} one small backbone, three controlled tasks,\par
      length extrapolation, and lower deployment FLOPs / peak memory.
    \item \textbf{Comparison scope:} MoNe has offline task supervision;\par
      ICL beyond 32K also faces native-window extrapolation.
    \item \textbf{Task scope:} MK-NIAH queries one of four keys;\par
      natural document reasoning remains future work.
    \item \textbf{System scope:} no measured latency or energy result;\par
      RAG is absent from the computational cost table.
  \end{itemize}
  \papersource{Paper, Secs. 4--5, PDF pp. 4--6; Appendix A.1, PDF p. 10. Interpretation based on the reported protocol.}
\end{frame}

\begin{frame}{Three experiments would clarify practical usefulness}
  \begin{enumerate}
    \item \textbf{Measure actual memory capacity.}\par
      Increase independent facts, temporal dependencies, and continued writes;\par
      track old-fact retention as well as new-fact accuracy.
    \medskip
    \item \textbf{Match task-adaptation budgets.}\par
      Compare trained baselines and isolate memory, position, and update choices.
    \medskip
    \item \textbf{Measure a complete serving workload.}\par
      Fix query/output lengths; include ingestion, multiple queries, and RAG costs;\par
      allow ICL prefix-cache reuse.
  \end{enumerate}
  \papersource{Discussion proposals. Paper's stated future directions: larger backbones and natural workloads, Sec. 5, PDF p. 6.}
\end{frame}

\begin{frame}{A useful interface between learned memory and attention}
  \begin{block}{Method}
    Local associative updates compress context into nonlinear fast weights.\par
    Query-conditioned reads inject that information as extra attention KV.
  \end{block}
  \begin{block}{Result}
    4K-trained memory reaches 128K on controlled tasks, with\par
    0.96 / 0.94 / 0.96 scores and about 80\% lower FLOPs / peak memory.
  \end{block}
  \begin{block}{Practical interpretation}
    Offline-trained memory can reduce dependence on retained context tokens;\par
    natural workloads and measured serving speed are the next tests.
  \end{block}
  \papersource{Cho et al., MoNe: Modular Neural Memory for Efficient Long Context Inference.\par
  \href{https://arxiv.org/abs/2608.17616v1}{arXiv:2608.17616v1}; Tables 1--2, PDF p. 5; Sec. 5, PDF p. 6. FWE score is recall.}
\end{frame}

\appendix

\begin{frame}{Backup: a linear example explains associative writing}
  \[
    \M(\vect k;A)=A\vect k,\qquad
    \ell=-\vect v^\top A\vect k
  \]
  \[
    \nabla_A\ell=-\vect v\vect k^\top,\qquad
    A'=A+\eta\vect v\vect k^\top
  \]
  \[
    A'\vect u=A\vect u+\eta\vect v(\vect k^\top\vect u)
  \]
  \begin{itemize}
    \item Matching the key increases the contribution of its associated value.
    \item Repeated related updates can accumulate signal or interference.
    \item Actual MoNe uses nonlinear memory, momentum, and renormalization.
  \end{itemize}
  \papersource{Teaching derivation from the form of Eq. (7), PDF p. 4. This linear example is not the implemented MoNe update.}
\end{frame}

\begin{frame}{Backup: the SwiGLU write gradient stays inside the MLP}
  Define $a=\W_{\rm in}k$, $b=\W_{\rm gate}k$,\par
  $u=\operatorname{SiLU}(a)\odot b$, and $z=\W_{\rm out}v$.
  \[
    \ell=-v^\top\W_{\rm out}^\top u=-z^\top u
  \]
  \begin{align*}
    \nabla_{\W_{\rm out}}\ell&=-uv^\top,\\
    \nabla_{\W_{\rm in}}\ell&=-[z\odot b\odot\operatorname{SiLU}'(a)]k^\top,\\
    \nabla_{\W_{\rm gate}}\ell&=-[z\odot\operatorname{SiLU}(a)]k^\top.
  \end{align*}
  The segment's keys and values are treated as fixed local inputs.\par
  Learned rates, momentum, and weight normalization act around this gradient.
  \papersource{Derivation for one head/token from Eqs. (3) and (7), PDF pp. 3--4; update modifiers in Appendix A.2, PDF p. 10.}
\end{frame}

\begin{frame}{Backup: offline training configuration}
  \centering
  \renewcommand{\arraystretch}{1.22}
  \begin{tabular}{ll}
    \toprule
    Component & Reported value \\
    \midrule
    Backbone & Qwen2.5-0.5B-Instruct, frozen \\
    Memory & SwiGLU; 4 heads; 224 dimensions/head \\
    LoRA & Rank 128; $\alpha=128$ \\
    Optimizer & AdamW; $\beta_1=0.9$, $\beta_2=0.95$; decay 0.1 \\
    Learning rate & $10^{-3}$ non-LoRA; $5\times10^{-5}$ LoRA \\
    Training & 1 epoch; batch 16; bf16 \\
    Schedule & 200 warmup steps; cosine to $10^{-5}$ \\
    Gradient clipping & 1.0 \\
    \bottomrule
  \end{tabular}\par
  \papersource{Paper, Appendix A.2, PDF p. 10. These are offline optimizer settings, not the test-time memory rule.}
\end{frame}

\begin{frame}{Backup: details that require clarification for reproduction}
  \begin{itemize}
    \item \textbf{Memory entry count.}\par
      Eq. (4) generates a read per query token; Sec. 3.2 also states\par
      a fixed $T$-entry memory KV. The slot-construction step is unspecified.
    \medskip
    \item \textbf{Update recurrence and tensor layout.}\par
      Appendix A.2 describes rates and momentum activations, but\par
      does not fully spell out their recurrence or all head/projection reshapes.
    \medskip
    \item \textbf{Cost protocol.}\par
      Answering costs differ (0.86 T vs. 0.64 T); output-length and\par
      hardware details are insufficient to infer a latency speedup.
  \end{itemize}
  \papersource{Paper, Sec. 3.2, PDF pp. 3--4; Table 2, PDF p. 5; Appendix A.2, PDF p. 10.}
\end{frame}

\end{document}