\documentclass[aspectratio=169,11pt]{beamer}
\usepackage[T1]{fontenc}
\usepackage[utf8]{inputenc}
\usepackage{amsmath,amssymb,graphicx,booktabs,array}

\definecolor{AcademicBlue}{RGB}{26,57,86}
\definecolor{AcademicTeal}{RGB}{0,105,112}
\setbeamercolor{structure}{fg=AcademicBlue}
\setbeamercolor{frametitle}{fg=AcademicBlue,bg=white}
\setbeamercolor{title}{fg=AcademicBlue}
\setbeamercolor{alerted text}{fg=AcademicTeal}
\setbeamercolor{block title}{fg=white,bg=AcademicBlue}
\setbeamercolor{block body}{fg=black,bg=AcademicBlue!5}
\setbeamertemplate{navigation symbols}{}
\setbeamertemplate{itemize items}[circle]
\setbeamertemplate{footline}{\hfill\usebeamerfont{page number in head/foot}\insertframenumber\,/\,\inserttotalframenumber\hspace{4mm}\vspace{3mm}}
\setbeamersize{text margin left=8mm,text margin right=8mm}
\setbeamerfont{frametitle}{size=\Large,series=\bfseries}
\setbeamerfont{title}{size=\LARGE,series=\bfseries}
\newcommand{\papersource}[1]{\par\smallskip{\scriptsize\color{gray}#1\par}}
\newcommand{\takeaway}[1]{\begin{block}{Takeaway}#1\end{block}}

\title[Cortex]{Cortex: Consolidating Context into Parametric Memory}
\subtitle{Reading and discussion of arXiv:2610.20399v1}
\author{Paper: Jintian Zhang, Feng He, Xinnian Liang, Ziwen Xu, Haoming Xu, Haojing Huang, Shuangzhi Wu, Ningyu Zhang}
\date{Research group presentation | 25 October 2026}

\begin{document}

\begin{frame}
  \titlepage
  \vspace{-0.5em}
  {\scriptsize\centering\url{https://arxiv.org/abs/2610.20399v1}\par}
\end{frame}

\begin{frame}{The problem: context helps, then the gain disappears}
  \begin{columns}[T,onlytextwidth]
    \column{0.54\textwidth}
      \begin{enumerate}
        \item Give model $M$ useful context $C$.
        \item Observe better responses to target queries $Q_C^+$.
        \item Remove $C$: the benefit often vanishes.
      \end{enumerate}
      \vspace{0.3em}
      \takeaway{Can a weight update preserve the \emph{context-induced gain} while retaining behavior outside the intended change?}
    \column{0.42\textwidth}
      \begin{block}{Crucial scope}
        A large context gap identifies a \emph{memory failure}: the model can already exploit $C$. A small gap with poor performance may instead be a capability failure.
      \end{block}
  \end{columns}
  \papersource{Paper, Secs. 1--2, PDF pp. 1--4.}
\end{frame}

\begin{frame}{Define the recoverable gain and what must be preserved}
  \[
    \Delta_{\rm ctx}=\mathrm{Perf}(M,Q_C^+,C)-\mathrm{Perf}(M,Q_C^+)
  \]
  \vspace{-0.4em}
  \[
    \mathrm{CR}=\frac{\mathrm{Perf}(M',Q_C^+)-\mathrm{Perf}(M,Q_C^+)}{\Delta_{\rm ctx}},
    \qquad
    \mathrm{RR}=\frac{\mathrm{Perf}(M',Q_C^-)}{\mathrm{Perf}(M,Q_C^-)}.
  \]
  \begin{itemize}
    \item $M'=\mathcal A(M,C)$ answers \emph{without} the source context.
    \item CR = 100\% recovers the measured context gain; RR = 100\% retains the measured baseline.
    \item Target and retain sets may overlap when scoring different response attributes.
  \end{itemize}
  \papersource{Paper, Sec. 2, PDF pp. 3--4. Ratios use unrounded aggregate scores and may exceed 100\%.}
\end{frame}

\begin{frame}{The testbed spans five different kinds of memory}
  \small
  \begin{tabular}{@{}>{\raggedright\arraybackslash}p{0.22\linewidth}>{\raggedright\arraybackslash}p{0.37\linewidth}>{\raggedright\arraybackslash}p{0.33\linewidth}@{}}
    \toprule
    Task & Target change & Retention probe \\
    \midrule
    Agent experience & ALFWorld / $\tau^2$ success & MMLU, SimpleQA, math \\
    New APIs & Tests for 15 Python packages & MMLU, SimpleQA, LiveCodeBench \\
    Persona steering & Power-seeking / Psychopathy tendency & Fluency for Power-seeking only \\
    Knowledge suppression & Less overlap with RWKU target answers & Neighbor facts and BBH \\
    Factual editing & Efficacy, paraphrase, portability & Locality and general benchmarks \\
    \bottomrule
  \end{tabular}
  \vspace{0.5em}
  \begin{block}{Three protocols}
    Single context; compatible batch of contexts; sequential arrivals with no access to earlier \emph{source contexts}.
  \end{block}
  \papersource{Paper, Sec. 4, Fig. 4 / Table 1, PDF pp. 8--10. Coverage differs across task--model--protocol pairs.}
\end{frame}

\begin{frame}{Cortex: rehearse uses, then distill into FFN memory}
  \centering
  \includegraphics[width=0.98\linewidth,height=0.75\textheight,keepaspectratio]{figures/fig-02-workflow.pdf}
  \par\smallskip\small Prospective uses $\rightarrow$ FFN writes $\rightarrow$ context-conditioned teacher / context-free student.
  \papersource{Paper, Fig. 2 and Sec. 3, PDF pp. 5--7. Original paper figure.}
\end{frame}

\begin{frame}{What to write: prospective rehearsal}
  \[
    q_i\sim M(\cdot\mid I_q,C),\qquad
    y_i\sim M(\cdot\mid C,q_i).
  \]
  \begin{itemize}
    \item $I_q$ asks the pre-update model to anticipate future uses of $C$.
    \item The same model answers with $C$; an optional filter removes unsuitable examples.
    \item Agent tasks instead use training-environment queries and successful offline trajectories.
    \item Batch contexts can be concatenated or rehearsed separately before merging examples.
  \end{itemize}
  \takeaway{Rehearsal coverage determines which future uses are exposed to the write procedure.}
  \papersource{Paper, Sec. 3.1, PDF p. 5; Appendix B.1, PDF pp. 34--36. Held-out evaluation items are excluded.}
\end{frame}

\begin{frame}{Where to write: two mergeable FFN updates}
  \[
    F(x)=W_D\left[(W_Ux)\odot\mathrm{SiLU}(W_Gx)\right]
  \]
  \vspace{-0.3em}
  \begin{columns}[T,onlytextwidth]
    \column{0.48\textwidth}
      \begin{block}{Synaptic: low-rank changes}
        $W'=W+B_WA_W$ for $W_G,W_U,W_D$.\par
        Original matrices frozen; rank $\leq r$.
      \end{block}
    \column{0.48\textwidth}
      \begin{block}{Structural: new nonlinear channels}
        $F'(x)=F(x)+D_s[(U_sx)\odot\mathrm{SiLU}(G_sx)]$.\par
        Adds width $k$; old channels frozen.
      \end{block}
  \end{columns}
  \vspace{0.3em}
  \small At $k=r$ in a dense layer: Structural trains $3kd_h$ matrix weights; Synaptic trains $3r(d_h+d_{\rm ff})$. Structural adds deployed FFN width.
  \papersource{Paper, Sec. 3.2 / Fig. 3, PDF p. 6; Appendix A, PDF pp. 30--32.}
\end{frame}

\begin{frame}{For MoEs, a shared branch sees every token}
  \[
    F_{\rm MoE}'(x)=\sum_e g_e(x)E_e(x)+E_s(x)
  \]
  \begin{itemize}
    \item Independent expert expansions receive direct gradient only on selected routes.
    \item Shared Structural adds one branch $E_s$ across all routed tokens.
    \item If $\sum_e g_e(x)=1$, then $F_{\rm MoE}'=\sum_e g_e(x)(E_e(x)+E_s(x))$.
  \end{itemize}
  \begin{block}{Deployment detail}
    Folding the shared branch into each expert is algebraically valid under normalized routing, but copies its storage and executes the new channels on active routes.
  \end{block}
  \papersource{Paper, Sec. 3.2, PDF p. 6; Appendix A, PDF p. 33.}
\end{frame}

\begin{frame}{How to write: context-to-memory distillation}
  \begin{center}
    \begin{tabular}{ll}
      Teacher $M_{t-1}$: & $[\text{prefix},C,\text{suffix},q,y]$ \\
      Student $M_t$: & $[\text{prefix},\text{suffix},q,y]$
    \end{tabular}
  \end{center}
  \vspace{-0.3em}
  \[
    \mathcal L_t=\mathbb E_{i\in\mathcal J}\!\left[
    \beta T^2\mathrm{KL}_{K_v}(p_i^T\Vert p_i^S)
    +\frac{\alpha}{Ld_h}\sum_{\ell=1}^{L}\|h_{\ell,i}^S-h_{\ell,i}^T\|_2^2\right].
  \]
  \begin{itemize}
    \item KL matches next-token behavior; MSE aligns post-FFN residual states.
    \item Only the current FFN write $\psi_t$ receives gradients.
    \item Teacher targets store top-$K_v$ logits; student normalizes over the full vocabulary.
  \end{itemize}
  \papersource{Paper, Sec. 3.3, PDF pp. 6--7; Appendix B.3, PDF pp. 48--50.}
\end{frame}

\begin{frame}{Sequential writes also need behavioral replay}
  \begin{enumerate}
    \item Freeze $M_{t-1}$, including earlier write modules; generate current-edit rehearsal with $C_t$.
    \item Store earlier \emph{queries}; regenerate their answers from $M_{t-1}$ without old contexts.
    \item Include generic questions with targets cached once from $M_0$.
    \item Train only the new FFN adapter and evaluate all still-valid edits without source contexts.
  \end{enumerate}
  \begin{block}{Why replay matters}
    Frozen old parameters do not freeze the function: later writes can change the hidden states reaching them.
  \end{block}
  \papersource{Paper, Sec. 3.3, PDF p. 7; Appendix B.2, PDF p. 39. Up to 100 historical queries are stored.}
\end{frame}

\begin{frame}{Evaluation compares three conditions on paired outcomes}
  \small
  \begin{tabular}{@{}p{0.22\linewidth}p{0.28\linewidth}p{0.37\linewidth}@{}}
    \toprule
    Condition & Target input & Role \\
    \midrule
    Vanilla & Query only & Context-free baseline \\
    Context & Query + source $C$ & Measured recoverable gain \\
    Updated model & Query only & CR and RR after consolidation \\
    \bottomrule
  \end{tabular}
  \vspace{0.7em}
  \begin{itemize}
    \item Baselines include specialized editing, distillation, hypernetworks, and test-time training.
    \item Trainable baseline hyperparameters/duration are selected by held-out CR, then retrained and tested.
    \item Repeated generations are aggregated within each item before task scores and rates.
  \end{itemize}
  \papersource{Paper, Secs. 4--5, PDF pp. 8--11; Appendix B.1, PDF pp. 34--38.}
\end{frame}

\begin{frame}{Single and batch: strong recovery is often possible}
  \small
  \begin{tabular}{@{}p{0.38\linewidth}rrp{0.29\linewidth}@{}}
    \toprule
    Setting & CR & RR & Comparison \\
    \midrule
    Qwen2.5 batch API, Synaptic & 66\% & 99\% & OPCD: 41\% / 100\% \\
    Qwen2.5 batch API, Structural & 74\% & 98\% & Same 15 packages \\
    Qwen2.5 batch editing, Synaptic & 91\% & 98\% & SDCD: 63\% / 94\% \\
    Qwen2.5 batch editing, Structural & 89\% & 94\% & Same 50 edits \\
    \bottomrule
  \end{tabular}
  \vspace{0.7em}
  \takeaway{At least one Cortex variant is on the empirical CR--RR frontier among context-free methods with both rates in each of Fig. 5's 15 panels.}
  \papersource{Paper, Sec. 5.1 / Fig. 5, PDF pp. 10--12; Tables 11, 13, PDF pp. 76, 78. Rounded display values.}
\end{frame}

\begin{frame}{Global behavioral steering also persists}
  \small
  \begin{tabular}{@{}p{0.48\linewidth}rr@{}}
    \toprule
    Power-seeking, Structural & CR & Fluency RR \\
    \midrule
    Qwen2.5-14B-Instruct & 96.9\% & 99.6\% \\
    Gemma3-12B-it & 100.3\% & 99.3\% \\
    \bottomrule
  \end{tabular}\par\medskip
  \begin{itemize}
    \item The target is an LLM-judged tendency in open-ended answers.
    \item RR measures \emph{fluency on those same answers}, not broad benchmark retention.
    \item Psychopathy uses behavior-matching accuracy and has no reported RR; Qwen3-4B Structural reaches 98.0\% CR.
  \end{itemize}
  \papersource{Paper, Secs. 4.2 and 5.1, PDF pp. 9--11; Tables 17--18, PDF pp. 82--83.}
\end{frame}

\begin{frame}{A high CR can hide serious retention loss}
  \small
  \begin{tabular}{@{}p{0.46\linewidth}rr@{}}
    \toprule
    Setting and write & CR & RR \\
    \midrule
    GPT-OSS-20B ALFWorld, Synaptic & 100.6\% & 23.0\% \\
    GPT-OSS-20B ALFWorld, Structural & 96.6\% & 95.5\% \\
    GPT-OSS-20B suppression, Synaptic & 99.5\% & 75.9\% \\
    GPT-OSS-20B suppression, Structural & 97.4\% & 83.3\% \\
    GPT-OSS-20B $\tau^2$, Synaptic & $-92.3$\% & 98.5\% \\
    GPT-OSS-20B $\tau^2$, Structural & 38.5\% & 101.0\% \\
    \bottomrule
  \end{tabular}\par\medskip
  \small In $\tau^2$, target success is 60.3\% Vanilla, 64.7\% Context, 56.3\% Synaptic, 62.0\% Structural. Suppression's retention loss concentrates on nearby knowledge.
  \papersource{Paper, Sec. 5.1, PDF p. 12; Tables 10, 19, 20, PDF pp. 75, 83--84.}
\end{frame}

\begin{frame}{Continual editing: a controlled 50-update sequence}
  \begin{columns}[T,onlytextwidth]
    \column{0.49\textwidth}
      \begin{block}{Training per edit}
        Qwen2.5-14B-Instruct; fixed sequence of 50 zsRE facts; 2 current QA pairs; up to 50 replay queries; 16 generic retention examples; 30 epochs; KL + MSE.
      \end{block}
    \column{0.49\textwidth}
      \begin{block}{Evaluation}
        All seen facts after every edit; 7 generations per fact/facet. Fixed MMLU, SimpleQA, GSM8K subsets every five edits. Three training seeds, same edit order.
      \end{block}
  \end{columns}
  \vspace{0.6em}
  Target score: equal mean of efficacy, paraphrase, portability.\par
  Retention score: equal mean of locality, MMLU, SimpleQA, GSM8K.
  \papersource{Paper, Sec. 5.2, PDF pp. 12--14; Appendix B.2, PDF pp. 39--41.}
\end{frame}

\begin{frame}{After 50 edits, Structural recovers 88.9\% of the gap}
  \small
  \begin{tabular}{@{}lrrrr@{}}
    \toprule
    Method & Final CR & Final RR & Portability & Added/edited per step \\
    \midrule
    ROME & 28.5\% & 94.4\% & 39.0\% & 70.78M \\
    Cortex-Synaptic ($r=1$) & 80.6\% & 97.2\% & 30.7\% & 2.73M \\
    \alert{Cortex-Structural ($k=1$)} & \alert{88.9\%} & \alert{99.8\%} & \alert{49.4\%} & \alert{0.74M} \\
    \bottomrule
  \end{tabular}
  \vspace{0.8em}
  \begin{itemize}
    \item Context-free target average: 8.76\%; with context: 88.10\%; Structural after 50 edits: 79.3\%.
    \item After 50 edits, cumulative added/edited weights: Structural 36.86M; Synaptic 136.40M.
  \end{itemize}
  \papersource{Paper, Fig. 6, PDF p. 13; Table 14, PDF p. 79. Means over three seeds; Structural CR SD 3.5 points.}
\end{frame}

\begin{frame}{Read the raw scores behind a 99.8\% retention rate}
  \small
  \begin{tabular}{@{}lrrrr@{}}
    \toprule
    & Locality & MMLU & SimpleQA & GSM8K \\
    \midrule
    Vanilla & 40.3\% & 80.9\% & 4.6\% & 94.2\% \\
    Structural, edit 50 & 38.3\% & 81.7\% & 4.8\% & 94.7\% \\
    \bottomrule
  \end{tabular}
  \vspace{0.8em}
  \begin{itemize}
    \item RR compares equal-weight means: 54.9\% / 55.0\% $\approx$ 99.8\%.
    \item Locality still falls by 2.0 percentage points.
    \item Direct fact efficacy reaches 95.3\%, but compositional portability reaches 49.4\%.
  \end{itemize}
  \takeaway{The aggregate rate summarizes the measured suite; it does not imply every neighboring fact is unchanged.}
  \papersource{Paper, Fig. 6a, PDF p. 13; Table 14, PDF p. 79.}
\end{frame}

\begin{frame}{Final performance combines acquisition and preservation}
  \small
  \begin{tabular}{@{}lrrrr@{}}
    \toprule
    & CR at acquisition & CR at edit 50 & Portability then & Portability at 50 \\
    \midrule
    Structural & 91.3\% & 88.9\% & 52.4\% & 49.4\% \\
    Synaptic & 85.8\% & 80.6\% & 36.1\% & 30.7\% \\
    WISE & 52.0\% & 28.0\% & -- & -- \\
    \bottomrule
  \end{tabular}
  \vspace{0.8em}
  \begin{itemize}
    \item Compare each of the same 50 facts immediately after its own edit and at checkpoint 50.
    \item Structural loses only 2.4 CR points; its portability lead is already visible at acquisition.
    \item Separately, the first ten facts retain 97.2\% mean CR after 40 later edits.
  \end{itemize}
  \papersource{Paper, Fig. 6c, PDF p. 13; Appendix B.2, PDF pp. 40--41.}
\end{frame}

\begin{frame}{Historical-query replay protects old edits}
  \begin{block}{Age-aligned ablation on the first 25 facts}
    After each fact has undergone 25 later edits, original-question + paraphrase accuracy stays above 96\% \emph{with} replay.
  \end{block}
  \vspace{0.3em}
  \begin{center}
    \begin{tabular}{@{}lrr@{}}
      \toprule
      At age 25 & With replay & Without replay \\
      \midrule
      Structural & $>96$\% & 40.5\% \\
      Synaptic & $>96$\% & 8.0\% \\
      \bottomrule
    \end{tabular}
  \end{center}
  \vspace{0.3em}
  The no-replay control still keeps current-edit examples and 16 generic $M_0$ examples per update. A fixed historical QA control ends within one CR point of query replay.
  \papersource{Paper, Sec. 5.3 / Fig. 7e, PDF pp. 14--15; Appendix B.3, PDF pp. 58--61.}
\end{frame}

\begin{frame}{The data and the FFN location both matter}
  \begin{itemize}
    \item \textbf{Same data:} on Qwen2.5 batch API, Structural exceeds LoRA-KL by 23.0 CR points.
    \item \textbf{Attention updates:} in continual editing at rank 4 with KL, FFN-only beats FFN + attention by 3.8 CR and 4.0 RR points.
    \item \textbf{Hidden-state MSE:} at Synaptic rank 4, adding MSE raises RR by 2.7 points while lowering CR by 4.0; Structural width 16 improves both.
  \end{itemize}
  \takeaway{The rehearsal data, supervision signal, and update parameterization jointly shape the trade-off.}
  \papersource{Paper, Sec. 5.3 / Fig. 7a--g, PDF pp. 14--15; Appendix B.3, PDF pp. 50--63.}
\end{frame}

\begin{frame}{Shared MoE capacity and compact writes are effective here}
  \begin{itemize}
    \item \textbf{Equal total expansion weights:} shared MoE branch gains 4.6 CR points on ALFWorld and 10.3 on API code versus independent branches, at about one point lower RR.
    \item \textbf{Equal active expansion weights:} shared remains within two CR points while training one eighth as many expansion weights.
    \item \textbf{Exposure:} every token trains the shared branch; 32 of 768 layer--expert branches receive under 1\% of mean exposure in every ALFWorld seed.
    \item \textbf{Capacity sweep:} the smallest tested $k=1$ and $r=1$ have the highest mean CR under the common training recipe.
  \end{itemize}
  \papersource{Paper, Sec. 5.3 / Fig. 7h--n, PDF pp. 15--16; Appendix B.3, PDF pp. 64--69.}
\end{frame}

\begin{frame}{The appendix explains capacity on finite states}
  \begin{block}{FFN-local correction}
    At a fixed layer, align student state with the teacher by learning $R(u_i)=\delta_i=h_i^T-\bar h_i$ at the observed states.
  \end{block}
  \[
    R(u)=D_s\phi_s(u),\qquad
    \min_{D_s}\|D_s\Phi-\Delta\|_F^2
    =\|\Delta(I-\Phi^\dagger\Phi)\|_F^2.
  \]
  \begin{itemize}
    \item Exact fit with fixed features iff $\operatorname{row}(\Delta)\subseteq\operatorname{row}(\Phi)$.
    \item With trainable Structural features and $k\ge n$, the appendix gives sufficient conditions to interpolate any $n$ finite states.
    \item The experiment's $k=1$ relies on learned approximation and rehearsal coverage beyond this guarantee.
  \end{itemize}
  \papersource{Paper, Appendix A, PDF pp. 30--32. Finite-set result does not establish unseen-query generalization.}
\end{frame}

\begin{frame}{Interpret the evidence within its measured scope}
  \begin{itemize}
    \item The 50-edit sequence uses one fact order and three training seeds; cross-task continual consolidation remains future work.
    \item Sequential consolidation stores historical \emph{queries} and uses $M_0$ generic targets; it does not revisit old source contexts.
    \item Near-perfect aggregate RR coexists with a locality decrease; suppression has a larger neighbor-knowledge cost.
    \item Structural writes add capacity at every update, and distillation still computes through a large backbone for many optimization steps.
  \end{itemize}
  \papersource{Paper, Secs. 5--8, PDF pp. 12--20; Appendix B.2, PDF p. 39.}
\end{frame}

\begin{frame}{Take-home messages}
  \begin{enumerate}
    \item \textbf{A precise target:} recover the measurable context gap and retain specified non-target behavior.
    \item \textbf{A practical write path:} prospective rehearsal + context-conditioned self-teacher + mergeable FFN updates.
    \item \textbf{Strongest evidence:} 50 sequential factual edits, 88.9\% CR and 99.8\% RR for Structural with 0.74M parameters per edit.
    \item \textbf{The remaining challenge:} protect local knowledge and improve compositional portability as memories accumulate.
  \end{enumerate}
  \vspace{0.5em}
  {\small Paper: Zhang et al., \emph{Cortex: Consolidating Context into Parametric Memory}, arXiv:2610.20399v1.}
\end{frame}

\end{document}