% Research-group paper discussion. Build with the local build_slides.py helper.
\documentclass[aspectratio=169,11pt]{beamer}
\usepackage[T1]{fontenc}
\usepackage[utf8]{inputenc}
\usepackage{amsmath,amssymb,graphicx,booktabs}

\definecolor{AcademicBlue}{RGB}{26,57,86}
\definecolor{AcademicTeal}{RGB}{0,105,112}
\setbeamercolor{structure}{fg=AcademicBlue}
\setbeamercolor{frametitle}{fg=AcademicBlue,bg=white}
\setbeamercolor{title}{fg=AcademicBlue}
\setbeamercolor{alerted text}{fg=AcademicTeal}
\setbeamercolor{block title}{fg=white,bg=AcademicBlue}
\setbeamercolor{block body}{fg=black,bg=AcademicBlue!5}
\setbeamertemplate{navigation symbols}{}
\setbeamertemplate{itemize items}[circle]
\setbeamertemplate{footline}{\hfill\usebeamerfont{page number in head/foot}\insertframenumber\,/\,\inserttotalframenumber\hspace{4mm}\vspace{3mm}}
\setbeamersize{text margin left=8mm,text margin right=8mm}
\setbeamerfont{frametitle}{size=\Large,series=\bfseries}
\newcommand{\papersource}[1]{\par\smallskip{\scriptsize\color{gray}#1\par}}
\newcommand{\emphblue}[1]{{\color{AcademicTeal}\bfseries #1}}
\hypersetup{pdfauthor={Research-group paper discussion},pdftitle={Deep Fusion Memory: Paper Discussion}}

\begin{document}

\begin{frame}[plain]
  \noindent{\LARGE\bfseries\color{AcademicBlue}Deep Fusion Memory\par}
  \vspace{2mm}
  {\large A Chunk-Level Parametric Memory for Large Language Models\par}
  \vspace{7mm}
  {\normalsize Paper by Rubin Wei, Siyuan Sun, Hao Sun, Yuliang Liu,\par
  Xiang Shi, Jiarui Wang, Junming Zhang, and Zhouhan Lin\par}
  \vspace{6mm}
  {\normalsize\href{https://arxiv.org/abs/2610.25100v1}{arXiv:2610.25100v1} \quad Paper discussion, 26 October 2026\par}
  \vfill
  {\small Publication not found as of 26 October 2026.\par
  Code forthcoming, according to the \href{https://arxiv.org/pdf/2610.25100v1}{paper}.\par}
\end{frame}

\begin{frame}{The paper asks how to learn external knowledge without online search}
  \begin{block}{Research question}
    Can retrieval supervise a compact, reusable memory that enters the decoder's hidden states, while the backbone stays frozen?
  \end{block}
  \begin{itemize}
    \item Token-level parametric memory needs extensive \emphblue{offline retrieval}.
    \item Output-probability interpolation gives memory no further decoder layers to work through.
    \item DFM predicts \emphblue{one memory set per chunk} and uses gated cross-attention inside selected layers.
  \end{itemize}
  \papersource{Paper, Secs. 1--3, PDF pp. 1--5.}
\end{frame}

\begin{frame}{DFM changes both supervision granularity and fusion location}
  \begin{center}
  \begin{tabular}{@{}lll@{}}
    \toprule
    Approach & Knowledge access & Decoder interface \\
    \midrule
    CPT / LoRA & Train on corpus & Adapt model weights \\
    RAG / RETRO & Retrieve at inference & Prompt or cross-attention \\
    MLP Memory & Learn offline retrieval & Output distribution mix \\
    \textbf{DFM} & \textbf{Predict chunk memory} & \textbf{Hidden-state fusion} \\
    \bottomrule
  \end{tabular}
  \end{center}
  \vspace{3mm}
  \begin{block}{Key architectural distinction}
    A frozen backbone can process the injected memory through subsequent decoder layers.
  \end{block}
  \papersource{Paper, Secs. 1--2, PDF pp. 1--3.}
\end{frame}

\begin{frame}{Three components turn offline neighbors into internal memory}
  \begin{columns}[T,onlytextwidth]
    \begin{column}{0.31\textwidth}
      \begin{block}{1. Offline teacher}
        Retrieve ranked chunk embeddings from the training corpus. These become target memory $V_j$.
      \end{block}
    \end{column}
    \begin{column}{0.31\textwidth}
      \begin{block}{2. Causal predictor}
        Predict $\widehat V_j$ from preceding chunks; no retrieval is needed at inference.
      \end{block}
    \end{column}
    \begin{column}{0.31\textwidth}
      \begin{block}{3. Hidden-state fusion}
        Project predicted slots to $M_j$ and read them through gated cross-attention.
      \end{block}
    \end{column}
  \end{columns}
  \vspace{7mm}
  \[
     X_{j-1}\xrightarrow{\mathrm{retrieve}}V_j
     \quad\big|\quad
     X_{<j}\xrightarrow{\mathrm{predict}}\widehat V_j
     \xrightarrow{\mathrm{project}}M_j
  \]
  \papersource{Paper, Sec. 3, Fig. 1, PDF pp. 3--5. See backup slide for the original schematic.}
\end{frame}

\begin{frame}{One retrieved target supervises a full 32-token chunk}
  \begin{columns}[T,onlytextwidth]
    \begin{column}{0.55\textwidth}
      \begin{enumerate}
        \item Partition training text into chunks $X_0,\ldots,X_{J-1}$ of $C=32$ tokens.
        \item For $j\geq1$, retrieve neighbors using the \emphblue{previous} chunk $X_{j-1}$.
        \item Store ranked Top-16 chunk embeddings as $V_j\in\mathbb{R}^{16\times1024}$.
      \end{enumerate}
      \vspace{2mm}
      $X_0$ has no previous-chunk query and receives no memory supervision.
    \end{column}
    \begin{column}{0.42\textwidth}
      \begin{block}{Offline teacher}
        Frozen Qwen3-Embedding-0.6B; normalized vectors; training-only FAISS datastore.
      \end{block}
      \begin{block}{Why this saves queries}
        About $N/32$ queries for $N$ training tokens, rather than one per token.
      \end{block}
    \end{column}
  \end{columns}
  \papersource{Paper, Sec. 3.1 and Appendix A.1, PDF pp. 3--4, 14.}
\end{frame}

\begin{frame}{A causal predictor replaces retrieval at inference}
  \[
    \widehat V_j=\operatorname{Predictor}_{\phi}(C_{\leq j})
    \in\mathbb{R}^{K\times d_m}
  \]
  \begin{itemize}
    \item $C_j$ mean-pools frozen token embeddings from $X_{j-1}$.
    \item Causal attention lets $\widehat V_j$ use only $X_0,\ldots,X_{j-1}$.
    \item $K$ output heads predict the Top-$K$ ranked memory slots.
    \item Default predictor: 28 layers, width 1,024, $K=16$.
  \end{itemize}
  \begin{block}{The learned object}
    Continuous memory vectors, not retrieved passages or next-token distributions.
  \end{block}
  \papersource{Paper, Sec. 3.2, Eq. (1), PDF p. 4; Appendix A.1, p. 14.}
\end{frame}

\begin{frame}{Gated cross-attention injects memory into selected layers}
  \begin{columns}[T,onlytextwidth]
    \begin{column}{0.55\textwidth}
      \[
        M_j=P_\psi(\widehat V_j)\in\mathbb{R}^{K\times d}
      \]
      \[
        g_t=\sigma(W_g z_t+b_g)
      \]
      \[
        u_t=W_O\operatorname{Concat}_{a=1}^{H}(g_{t,a}o_{t,a})
      \]
      \[
        \widetilde h_t=h_t+\mathrm{SelfAttn}(z_{\leq t})+u_t
      \]
      \begin{itemize}
        \item Token state queries $K$ shared slots; gate scales each head.
        \item Fusion layers: $[9,16,24,31]$ (zero-based).
      \end{itemize}
    \end{column}
    \begin{column}{0.41\textwidth}
      \centering
      \includegraphics[width=0.95\linewidth,height=0.69\textheight,keepaspectratio]{figures/fig-01-fusion.png}
    \end{column}
  \end{columns}
  \papersource{Original Fig. 1(c); Paper, Sec. 3.2, Eqs. (2)--(3), PDF pp. 4--5.}
\end{frame}

\begin{frame}{The LM sees predicted memory throughout joint training}
  \[
    \mathcal L=\mathcal L_{\mathrm{LM}}+\beta\mathcal L_{\mathrm{MSE}},\qquad
    \mathcal L_{\mathrm{MSE}}=
    \frac{\sum_{j=1}^{J-1}\|\widehat V_j-V_j\|_F^2}{(J-1)Kd_m},\quad\beta=1.
  \]
  \begin{itemize}
    \item Retrieval vectors $V_j$ are \emphblue{targets only}; they do not enter the LM forward pass.
    \item LM gradients flow through fusion back into the predictor.
    \item Train predictor, projector, cross-attention and gates; freeze the backbone and encoder.
    \item Gate weights start at zero; default bias $-2$ gives $\sigma(-2)\approx0.119$.
  \end{itemize}
  \papersource{Paper, Sec. 3.2, Eq. (4), PDF p. 5; Appendix A.1, p. 14.}
\end{frame}

\begin{frame}{Reported QA inference uses fixed prompt memory}
  \begin{block}{General inference path}
    Input embeddings $\rightarrow$ memory predictor $\rightarrow$ selected decoder layers. No datastore lookup or external embedding encoder.
  \end{block}
  \begin{itemize}
    \item In QA evaluation, the \emphblue{visible prompt chunks} form memory queries.
    \item Predicted memory stays fixed while the answer is generated.
    \item Generated answer chunks do not refresh queries or receive direct memory updates.
  \end{itemize}
  \vspace{2mm}
  The QA results therefore test prompt-conditioned memory during answer generation.
  \papersource{Paper, Sec. 3.3, PDF p. 5; Appendix A.1, PDF pp. 14--15.}
\end{frame}

\begin{frame}{The six-task QA mean combines different task metrics}
  \begin{columns}[T,onlytextwidth]
    \begin{column}{0.49\textwidth}
      \begin{block}{Training and evaluation}
        2018 English Wikipedia; roughly 4B tokens; Mistral-7B-v0.3 and Qwen3-8B-Base.
      \end{block}
      Zero-shot greedy decoding; no online retrieval, gold documents or reasoning demonstrations.
    \end{column}
    \begin{column}{0.49\textwidth}
      \begin{block}{Six benchmarks}
        NQ, WebQA; HotpotQA, 2Wiki, MuSiQue, Bamboogle.
      \end{block}
      F1 on five tasks; \emphblue{substring match} on WebQA. The six scores receive equal weight.
    \end{column}
  \end{columns}
  \vspace{5mm}
  Baselines: RAG, xRAG, RETRO, CPT, LoRA, MLP Memory.
  \papersource{Paper, Sec. 4.1 and Appendix A.1, PDF pp. 6, 14--15; Table 4.}
\end{frame}

\begin{frame}{DFM raises the six-task QA average on both main backbones}
  \begin{center}
  \begin{tabular}{@{}lrrrl@{}}
    \toprule
    Backbone & Base & Best baseline & DFM & Gain over baseline \\
    \midrule
    Mistral-7B-v0.3 & 13.63 & xRAG 24.14 & \textbf{27.21} & +3.07 points; +12.71\% relative \\
    Qwen3-8B-Base & 22.71 & MLP 23.80 & \textbf{26.46} & +2.66 points; +11.19\% relative \\
    \bottomrule
  \end{tabular}
  \end{center}
  \vspace{5mm}
  \begin{block}{Read the percentages correctly}
    Relative percentage gain divides the \emph{point difference} by the baseline mean; it is not a percentage-point change.
  \end{block}
  \papersource{Paper, Sec. 4.2, Table 1, PDF pp. 6--7. Relative values as reported by the authors.}
\end{frame}

\begin{frame}{Task-level wins and failures explain the mean}
  \begin{center}
  \begin{tabular}{@{}llrrr@{}}
    \toprule
    Backbone & Task & Base & DFM & Relevant comparator \\
    \midrule
    Mistral & Bamboogle & 3.91 & \textbf{22.65} & RAG 18.56 \\
    Mistral & 2Wiki & 5.87 & 25.00 & \textbf{RAG 31.02} \\
    Qwen3 & 2Wiki & 24.83 & \textbf{32.38} & MLP 27.47 \\
    Qwen3 & WebQA & 41.78 & 39.47 & \textbf{MLP 43.26} \\
    \bottomrule
  \end{tabular}
  \end{center}
  \vspace{3mm}
  DFM leads in the \emphblue{average}, not every task. Qwen3 WebQA falls below Base.
  \papersource{Paper, Sec. 4.2, Table 1, PDF p. 7. Bamboogle has 125 test examples (Table 4, p. 15).}
\end{frame}

\begin{frame}{A third QA backbone shows the same average trend}
  \begin{center}
  \begin{tabular}{@{}lrrr@{}}
    \toprule
    OLMo-3-1025-7B & Base & Best baseline & DFM \\
    \midrule
    Six-task mean & 17.99 & RETRO 23.63 & \textbf{25.49} \\
    \bottomrule
  \end{tabular}
  \end{center}
  \vspace{5mm}
  \begin{itemize}
    \item DFM improves the mean by 7.50 points over Base and 1.86 over RETRO.
    \item It improves four of six tasks over Base.
    \item WebQA and HotpotQA decrease by 1.28 and 0.63 points.
  \end{itemize}
  \papersource{Paper, Appendix B.1, Table 5, PDF p. 16.}
\end{frame}

\begin{frame}{Domain adaptation tests code, math and law separately}
  \begin{itemize}
    \item Separate DFM checkpoint for each domain on Qwen3-4B-Base and Qwen3-8B-Base.
    \item Code and Math: 100M training tokens each; Law: 123.5M.
    \item HumanEval and HumanEval+ use execution correctness (estimated pass@1).
    \item GSM8K tests math; LawBench averages 20 legal task metrics.
  \end{itemize}
  \begin{block}{Reported domain mean}
    Equal-weight average of HumanEval, HumanEval+, GSM8K and LawBench scores.
  \end{block}
  \papersource{Paper, Sec. 4.1 and Appendix A.2, PDF pp. 6, 15.}
\end{frame}

\begin{frame}{DFM also has the highest domain mean at both scales}
  \begin{center}
  \begin{tabular}{@{}lrrrrr@{}}
    \toprule
    Backbone & Base & CPT & LoRA & DFM & DFM $-$ Base \\
    \midrule
    Qwen3-4B-Base & 58.18 & 62.92 & 60.70 & \textbf{64.69} & +6.51 \\
    Qwen3-8B-Base & 63.14 & 65.06 & 66.96 & \textbf{67.52} & +4.38 \\
    \bottomrule
  \end{tabular}
  \end{center}
  \vspace{4mm}
  \begin{itemize}
    \item 4B: CPT leads LawBench (41.66 vs DFM 40.41).
    \item 8B: LoRA leads HumanEval+ (65.85 vs 64.02) and GSM8K (87.19 vs 86.43).
  \end{itemize}
  \papersource{Paper, Sec. 4.3, Table 2, PDF p. 8. All differences are points on the reported 0--100 scale.}
\end{frame}

\begin{frame}{Construction counts improve far more than measured training time}
  \begin{center}
  \begin{tabular}{@{}lrr@{}}
    \toprule
    Compared with MLP Memory & MLP Memory & DFM \\
    \midrule
    Added parameters & 1.50B & 0.75B \\
    Training on 16 A800 GPUs & 64 h & 24 h \\
    Queries for 4B tokens & 4B & 125M \\
    Returned neighbor slots & 4.096T & 2B \\
    \bottomrule
  \end{tabular}
  \end{center}
  \vspace{3mm}
  $32\times$ fewer queries; $2048\times$ fewer returned slots because Top-1024 becomes Top-16 too.
  \papersource{Paper, Sec. 4.4 and Appendix D, Fig. 2, Table 13, PDF pp. 9, 23. Training hours exclude target construction and evaluation; slot counts are not timing measurements.}
\end{frame}

\begin{frame}{Fusion placement matters more than simply adding modules}
  \centering
  \includegraphics[width=0.92\linewidth,height=0.60\textheight,keepaspectratio]{figures/fig-03-ablations.pdf}
  \vspace{1mm}
  \begin{itemize}
    \item Four distributed layers: 27.21; final four layers: 22.07.
    \item Two layers at $[9,31]$: 27.54; last layer alone: 21.11.
  \end{itemize}
  \papersource{Original Fig. 3, PDF p. 10; Appendix C.1, pp. 20--21. Six-task means; single runs.}
\end{frame}

\begin{frame}{The default gate is conservative, but not the best sweep result}
  \begin{center}
  \begin{tabular}{@{}lrr@{}}
    \toprule
    Gate configuration & Six-task mean & Difference vs default \\
    \midrule
    No gate & 26.83 & $-0.38$ \\
    Bias $b_g=-2$ (default) & 27.21 & --- \\
    Bias $b_g=0$ & \textbf{27.90} & $+0.69$ \\
    \bottomrule
  \end{tabular}
  \end{center}
  \vspace{4mm}
  \begin{itemize}
    \item Zero-initialized gate weights and $b_g=-2$ start at $\sigma(-2)\approx0.119$.
    \item Other tested variations are close: Top-2 memory 26.88; 14-layer predictor 27.31.
  \end{itemize}
  \papersource{Paper, Sec. 5.2, Table 3, PDF pp. 10--11; Appendix C, pp. 20--22. Single runs.}
\end{frame}

\begin{frame}{The evidence supports average gains under a specific protocol}
  \begin{itemize}
    \item QA means mix F1 with WebQA substring match; domain means mix four benchmarks.
    \item Offline retrieval \emphblue{workload counts} do not establish end-to-end construction time or inference latency.
    \item QA answer generation uses fixed prompt memory, not refreshed answer-chunk memory.
    \item Ablations are single runs; Bamboogle has only 125 test questions.
  \end{itemize}
  \vspace{2mm}
  The current corpus is learned into extra parameters; updating its facts would require further adaptation or another update mechanism.
  \papersource{Paper, Tables 1--4 and Appendices A.1, C, D, PDF pp. 7--15, 20--23. Final sentence is an architectural inference.}
\end{frame}

\begin{frame}{Takeaways}
  \begin{enumerate}
    \item \emphblue{One memory target per chunk} sharply reduces offline retrieval queries.
    \item \emphblue{Predicted continuous memory} can be fused into frozen decoder hidden states without inference-time search.
    \item DFM improves reported \emphblue{mean QA and domain scores}, with clear task-specific exceptions.
    \item Fusion placement and gate initialization deserve attention; the paper does not time full inference or target construction.
  \end{enumerate}
  \papersource{Paper, Secs. 3--6, PDF pp. 3--11; Appendix D, p. 23.}
\end{frame}

\begin{frame}{Paper and source links}
  \begin{itemize}
    \item Rubin Wei, Siyuan Sun, Hao Sun, Yuliang Liu, Xiang Shi, Jiarui Wang, Junming Zhang, Zhouhan Lin. \emph{Deep Fusion Memory: A Chunk-Level Parametric Memory for Large Language Models}. \href{https://arxiv.org/abs/2610.25100v1}{arXiv:2610.25100v1}, 2026.
    \item \href{https://arxiv.org/pdf/2610.25100v1}{Versioned paper PDF}; author TeX source retrieved from the same version.
    \item Publication not found as of 26 October 2026; the paper says code will be released soon.
  \end{itemize}
  \vspace{4mm}
  All figures in this talk come from the paper. Performance tables reproduce selected original values.
\end{frame}

\appendix
\begin{frame}{Backup: original architecture figure}
  \centering
  \includegraphics[width=0.97\linewidth,height=0.76\textheight,keepaspectratio]{figures/fig-01-architecture.pdf}
  \papersource{Original paper figure, Fig. 1, PDF p. 4.}
\end{frame}

\end{document}