% Paper discussion deck. Compile with the adjacent build_slides.py (pdfLaTeX).
\documentclass[aspectratio=169,11pt]{beamer}
\usepackage[T1]{fontenc}
\usepackage[utf8]{inputenc}
\usepackage{lmodern}
\usepackage{amsmath,amssymb,graphicx,booktabs}

\definecolor{AcademicBlue}{RGB}{26,57,86}
\definecolor{AcademicTeal}{RGB}{0,105,112}
\setbeamercolor{structure}{fg=AcademicBlue}
\setbeamercolor{frametitle}{fg=AcademicBlue,bg=white}
\setbeamercolor{title}{fg=AcademicBlue}
\setbeamercolor{alerted text}{fg=AcademicTeal}
\setbeamercolor{block title}{fg=white,bg=AcademicBlue}
\setbeamercolor{block body}{fg=black,bg=AcademicBlue!5}
\setbeamertemplate{navigation symbols}{}
\setbeamertemplate{itemize items}[circle]
\setbeamertemplate{footline}{\hfill\usebeamerfont{page number in head/foot}\insertframenumber\,/\,\inserttotalframenumber\hspace{4mm}\vspace{3mm}}
\setbeamersize{text margin left=8mm,text margin right=8mm}
\setbeamerfont{frametitle}{size=\Large,series=\bfseries}
\setbeamerfont{title}{size=\LARGE,series=\bfseries}
\newcommand{\papersource}[1]{\par\smallskip{\scriptsize\color{gray}#1\par}}
\newcommand{\context}[1]{\textcolor{AcademicTeal}{#1}}

\title{Equivalence of Context and Parameter Updates\\in Modern Transformer Blocks}
\subtitle{A constructive view of token-specific implicit weight patches}
\author{Paper by Adrian Goldwaser, Michael Munn, Javier Gonzalvo, Benoit Dherin}
\date{\href{https://arxiv.org/abs/2511.17864v3}{arXiv:2511.17864v3}\quad\textbar\quad Paper discussion, 26 October 2026}

\begin{document}

\begin{frame}
  \titlepage
\end{frame}

\begin{frame}{Can a prompt be represented as a weight patch?}
  \begin{block}{The paper's question}
    For a fixed query token, can a model with reduced context and updated parameters produce the same output as the original model with full context?
  \end{block}
  \vspace{2mm}
  \[
    T_{\Theta'}(C\setminus Y,\mathbf{x})
    \stackrel{?}{=} T_{\Theta}(C,\mathbf{x})
  \]
  \begin{itemize}
    \item $Y$: the context to absorb; $\Theta'$: patches to existing parameters.
    \item Equality is constructed for the current query and generation history.
  \end{itemize}
  \papersource{Paper, Secs. 1 and 3, PDF pp. 1--3.}
\end{frame}

\begin{frame}{Modern blocks remove the bias used by the earlier proof}
  \begin{columns}[T]
    \begin{column}{0.47\textwidth}
      \begin{block}{Earlier vanilla block}
        \begin{itemize}
          \item Rank-one input matrix patch aligns the MLP activation.
          \item Output bias absorbs the attention residual shift.
        \end{itemize}
      \end{block}
    \end{column}
    \begin{column}{0.47\textwidth}
      \begin{block}{Modern LLM blocks}
        \begin{itemize}
          \item Gated MLPs, pre/post normalization, often no MLP bias.
          \item Need another output parameter to carry the residual shift.
        \end{itemize}
      \end{block}
    \end{column}
  \end{columns}
  \vspace{5mm}
  \alert{The paper derives the replacement for Gemma, then isolates the general condition.}
  \papersource{Paper, Secs. 1--2, PDF pp. 1--2.}
\end{frame}

\begin{frame}{Gemma has two places where a patch can act}
  \begin{columns}[c]
    \begin{column}{0.44\textwidth}
      \centering
      \includegraphics[width=0.98\linewidth,height=0.75\textheight,keepaspectratio]{figures/fig-01-gemma-block.pdf}
    \end{column}
    \begin{column}{0.52\textwidth}
      \begin{enumerate}
        \item $W_{\rm gate},W_{\rm up}$ control the inputs to the gated MLP.
        \item The trainable post-RMSNorm scale $\mathbf m$ controls its final output, coordinate by coordinate.
      \end{enumerate}
      \vspace{3mm}
      \begin{block}{Division of labor}
        Input patches reproduce the internal MLP state; output-scale patch restores the missing residual contribution.
      \end{block}
    \end{column}
  \end{columns}
  \papersource{Original paper Fig. 1, PDF p. 3; cropped from the paper PDF.}
\end{frame}

\begin{frame}{Notation: isolate the change caused by context}
  \[
    \mathbf v_C=A(C,\mathbf x),\qquad
    \mathbf v=A(C\setminus Y,\mathbf x),\qquad
    \Delta\mathbf v=\mathbf v_C-\mathbf v
  \]
  \[
    \mathbf z_C=N_{\rm RMS}(\mathbf v_C),\qquad
    \mathbf z=N_{\rm RMS}(\mathbf v)
  \]
  \begin{block}{Gemma-style residual MLP output}
    \[
      T(C,\mathbf x)=\mathbf v_C+\mathbf m\odot
      f(W_{\rm gate}\mathbf z_C,W_{\rm up}\mathbf z_C)
    \]
  \end{block}
  $f$ includes gating, $W_{\rm down}$, and post-RMSNorm \emph{without} its scale $\mathbf m$.
  \papersource{Paper, Sec. 3.1, Eq. (1), PDF pp. 2--3.}
\end{frame}

\begin{frame}{A rank-one patch aligns each input projection}
  \begin{block}{Construction for $i\in\{\rm gate,up\}$}
    \[
      \Delta W_i =
      \frac{W_i(\mathbf z_C-\mathbf z)\mathbf z^{\top}}
           {\|\mathbf z\|_2^2},\qquad \mathbf z\ne\mathbf 0
    \]
  \end{block}
  \[
    (W_i+\Delta W_i)\mathbf z
    =W_i\mathbf z+W_i(\mathbf z_C-\mathbf z)
    =W_i\mathbf z_C.
  \]
  \begin{itemize}
    \item The gate and up activations now match their full-context targets.
    \item $\Delta W_i$ is an outer product, hence has rank at most one.
  \end{itemize}
  \papersource{Paper, Theorem 1 and Eqs. (2)--(3), PDF p. 3.}
\end{frame}

\begin{frame}{The output scale restores the missing residual}
  Let $\mathbf h=f(W_{\rm gate}\mathbf z_C,W_{\rm up}\mathbf z_C)$ after the input patches.
  \begin{block}{Output correction}
    \[
      \Delta\mathbf m=\Delta\mathbf v\oslash\mathbf h,
      \qquad h_j\ne0\ \text{for every coordinate }j.
    \]
  \end{block}
  \[
    \underbrace{\mathbf v+(\mathbf m+\Delta\mathbf m)\odot\mathbf h}_{\text{reduced context + patches}}
    =\underbrace{\mathbf v_C+\mathbf m\odot\mathbf h}_{\text{full context}}.
  \]
  \alert{The equality is algebraically exact when its nonzero conditions hold.}
  \papersource{Paper, Theorem 1 and Eq. (4), PDF p. 3.}
\end{frame}

\begin{frame}{The unified theorem needs two controllable functions}
  \[
    T(C,\mathbf x)=A(C,\mathbf x)
       +g\!\left(f(A(C,\mathbf x);\theta_f);\theta_g\right)
  \]
  \begin{enumerate}
    \item \textbf{Input controllability:} update $\theta_f$ so the reduced-context input gives the full-context inner activation.
    \item \textbf{Output controllability:} update $\theta_g$ so the outer branch adds any required $\Delta\mathbf v$.
  \end{enumerate}
  \begin{block}{Sufficient condition}
    When both properties hold, with the required activations nonzero, a context-equivalent patch exists for the residual block.
  \end{block}
  \papersource{Paper, Definitions 1--2 and Theorem 3, PDF pp. 7--8.}
\end{frame}

\begin{frame}{Different architectures offer different output controls}
  \renewcommand{\arraystretch}{1.25}
  \begin{tabular}{@{}p{0.27\linewidth}p{0.44\linewidth}p{0.22\linewidth}@{}}
    \toprule
    Outer control & Patch that adds $\Delta\mathbf v$ & Condition \\
    \midrule
    Bias $\mathbf b$ & $\Delta\mathbf b=\Delta\mathbf v$ & trainable bias \\
    Output matrix $W'$ & $\Delta W'=\Delta\mathbf v\,\mathbf h^\top/\|\mathbf h\|^2$ & $\mathbf h\ne0$ \\
    RMSNorm scale $\mathbf m$ & $\Delta\mathbf m=\Delta\mathbf v\oslash\mathbf h$ & every $h_j\ne0$ \\
    MoE experts & controllable experts share correction & $S\ne0$ \\
    \bottomrule
  \end{tabular}
  \vspace{4mm}
  \alert{Gated and pre-normalized inputs remain controllable through their input matrices.}
  \papersource{Paper, Table 1 and Appendix A, PDF pp. 8, 12--13.}
\end{frame}

\begin{frame}{A fixed post-RMSNorm scale gives a counterexample}
  \begin{block}{Geometric bound}
    With fixed $\mathbf m$, the post-RMSNorm output lies in a bounded set:
    \[
      \|g(\mathbf z')-g(\mathbf z)\|_2
      \le 2\sqrt d\,\|\mathbf m\|_\infty.
    \]
  \end{block}
  \begin{itemize}
    \item The attention-induced $\Delta\mathbf v$ can exceed this bound.
    \item Then no change upstream of the fixed normalization can compensate it.
  \end{itemize}
  \alert{Output controllability has a real architectural boundary.}
  \papersource{Paper, Sec. 5.1 and Appendix D, PDF pp. 9, 19--20.}
\end{frame}

\begin{frame}{The single-block result composes layer by layer}
  \begin{columns}[c]
    \begin{column}{0.43\textwidth}
      \centering
      \includegraphics[width=0.92\linewidth,height=0.69\textheight,keepaspectratio]{figures/fig-02-multilayer.pdf}
    \end{column}
    \begin{column}{0.53\textwidth}
      \begin{enumerate}
        \item Start from the same query-token representation.
        \item If layer $l$ inputs match, patch layer $l$ to match its outputs.
        \item The common output becomes the input to layer $l+1$.
      \end{enumerate}
      \vspace{2mm}
      \alert{Induction gives equality at the final layer.}
    \end{column}
  \end{columns}
  \papersource{Paper, Fig. 2 and Theorem 2, PDF p. 4; cropped from the paper PDF.}
\end{frame}

\begin{frame}{The algorithm records targets, then patches sequentially}
  \begin{enumerate}
    \item \textbf{Full-context pass:} save the target activation after each layer.
    \item \textbf{Reduced-context pass:} at layer $l$, compute its patch against the saved target.
    \item Forward the layer's \emph{actual patched output} to layer $l+1$.
  \end{enumerate}
  \vspace{4mm}
  \begin{block}{During autoregressive generation}
    Recompute patches for each next-token query because the generation history changes.
  \end{block}
  \papersource{Paper, Sec. 3.3, Algorithm 1 and Sec. 4.1, PDF pp. 4--6.}
\end{frame}

\begin{frame}{Small output coordinates destabilize the direct Gemma patch}
  \[
    \Delta m_j=\frac{\Delta v_j}{h_j}
  \]
  \begin{itemize}
    \item A small $h_j$ can magnify the scale patch and rounding error.
    \item The ideal equality from Theorem 1 can drift in \texttt{bfloat16}.
    \item Figure 3 exhibits token mismatches for the direct low-precision update.
  \end{itemize}
  \begin{block}{Design response}
    Move most of the correction into $W_{\rm down}$; use $\mathbf m$ for the remaining error.
  \end{block}
  \papersource{Paper, Sec. 4.2 and Appendix B, PDF pp. 6, 13--15.}
\end{frame}

\begin{frame}{Stable Gemma update: matrix first, scale second}
  \begin{enumerate}
    \item Set target output $\mathbf g=\Delta\mathbf v+\mathbf m\odot N(\mathbf h_{{\rm down},C})$.
    \item Find $\mathbf h_{\rm target}$ with a fixed RMS value that best fits $\mathbf g$ after scaled RMSNorm.
    \item Patch $W_{\rm down}$ to reach $\mathbf h_{\rm target}$; patch $\mathbf m$ for the residual.
  \end{enumerate}
  \[
    \Delta W_{\rm down}=
    \frac{(\mathbf h_{\rm target}-\mathbf h_{{\rm down},C})
          \mathbf h_{{\rm gated},C}^{\top}}
         {\|\mathbf h_{{\rm gated},C}\|^2},
    \qquad
    \Delta\mathbf m=\mathbf r\oslash N(\mathbf h_{\rm target}).
  \]
  \papersource{Paper, Appendix B.1--B.2, PDF pp. 13--15.}
\end{frame}

\begin{frame}{Evaluation compares identical next-token histories}
  \begin{block}{Two generation paths}
    Original model with full context \quad vs.\quad patched model with no explicit context.
  \end{block}
  \begin{itemize}
    \item Recompute the patch at each token, including the prompt and prior generated tokens.
    \item If argmax tokens differ, continue both comparisons on the baseline history.
    \item Metrics: token match $\uparrow$, $L_\infty$ logit difference $\downarrow$, TVD $\downarrow$.
  \end{itemize}
  \[
    \operatorname{TVD}(p,q)=\tfrac12\|p-q\|_1.
  \]
  \papersource{Paper, Secs. 4.1--4.2, PDF pp. 5--6.}
\end{frame}

\begin{frame}{Gemma text matching depends strongly on precision}
  \renewcommand{\arraystretch}{1.35}
  \begin{tabular}{@{}p{0.34\linewidth}p{0.24\linewidth}p{0.34\linewidth}@{}}
    \toprule
    Gemma 3 1B setting & Token match & Distribution comparison \\
    \midrule
    \texttt{float32}, direct & 100\% & nearly identical \\
    \texttt{bfloat16}, direct & 87.5\% & clear logit / TVD drift \\
    \texttt{bfloat16}, stable & 100\% & drift reduced \\
    \bottomrule
  \end{tabular}
  \vspace{4mm}
  \small The 87.5\% figure is the main text's low-precision case; Figure 4 pools five text generations.
  \papersource{Paper, Sec. 4.2, Figs. 3--4, PDF pp. 5--7.}
\end{frame}

\begin{frame}{Across five prompts, stable updates recover token match}
  \centering
  \includegraphics[width=0.96\linewidth,height=0.67\textheight,keepaspectratio]{figures/fig-04-generation-summary.pdf}
  \papersource{Original paper Fig. 4, PDF p. 7. Left: TVD (log scale; lower is better). Right: token match (higher is better).}
\end{frame}

\begin{frame}{The image-context example follows the same pattern}
  \centering
  \includegraphics[width=0.96\linewidth,height=0.62\textheight,keepaspectratio]{figures/fig-05a-image-logits.pdf}
  \par\smallskip
  \small Gemma 3 4B: direct \texttt{bfloat16} (blue) has token mismatches; the stable update (orange) reduces logit drift. Both \texttt{float32} variants (green/red) stay close.
  \papersource{Original paper Fig. 5, top panel cropped, and Sec. 4.2, PDF p. 7.}
\end{frame}

\begin{frame}{Falcon needs an outer matrix patch, not a scale patch}
  \begin{columns}[T]
    \begin{column}{0.48\textwidth}
      \begin{block}{Architecture}
        Falcon's tested block lacks Gemma's post-normalization scale, so its outer output matrix absorbs $\Delta\mathbf v$.
      \end{block}
    \end{column}
    \begin{column}{0.48\textwidth}
      \begin{block}{Observed fidelity}
        Across five prompts, both \texttt{bfloat16} and \texttt{float32} achieve 100\% token match; \texttt{float32} has smaller TVD.
      \end{block}
    \end{column}
  \end{columns}
  \vspace{5mm}
  \[
    \Delta W_{\rm out}=\frac{\Delta\mathbf v\,\mathbf h^\top}{\|\mathbf h\|^2}
  \]
  \papersource{Paper, Table 1 and Appendix C.3, Fig. 9, PDF pp. 8, 17--18.}
\end{frame}

\begin{frame}{Ablation: distributing patches across layers helps}
  \begin{itemize}
    \item Updating only the last layer causes numerical problems in the reported sweep.
    \item With a few adjustable layers, token accuracy is usually restored.
    \item Starting earlier lowers the stable scheme's scale-vector update norm.
  \end{itemize}
  \vspace{3mm}
  \begin{block}{Interpretation}
    Existence of a patch leaves many valid constructions; placement and parameter choice determine finite-precision behavior.
  \end{block}
  \papersource{Paper, Appendix C.4--C.5 and Figs. 11--13, PDF pp. 17--18, 23--25.}
\end{frame}

\begin{frame}{The result is local to a query and its history}
  \begin{enumerate}
    \item \textbf{Token-specific:} changing the next-token history changes the target activations and required patches.
    \item \textbf{Conditional:} input and output controls need the stated nonzero activations.
    \item \textbf{Numerical:} small divisors make direct low-precision evaluation unstable.
  \end{enumerate}
  \vspace{4mm}
  \begin{block}{Reading of the claim}
    The construction expresses the context effect as parameter changes at one query; the experiments measure how accurately those changes reproduce logits.
  \end{block}
  \papersource{Paper, Sec. 6, Appendix B, PDF pp. 9, 13--15.}
\end{frame}

\begin{frame}{Take-home messages}
  \begin{enumerate}
    \item Input projections can absorb a context-induced activation shift using rank-one updates.
    \item An output-controllable parameter must also absorb the attention residual shift.
    \item Layerwise construction extends the equality to deep models; stable parameter choices matter in low precision.
  \end{enumerate}
  \vspace{5mm}
  \alert{Controllability is the useful architectural test.}
  \papersource{Paper, Theorems 1--3, Secs. 3--6, PDF pp. 3--9.}
\end{frame}

\begin{frame}{Paper and discussion pointers}
  \begin{itemize}
    \item Goldwaser, Munn, Gonzalvo, Dherin. \emph{Equivalence of Context and Parameter Updates in Modern Transformer Blocks.}
    \item \href{https://arxiv.org/abs/2511.17864v3}{arXiv:2511.17864v3}, revised 6 July 2026.
    \item Main proof: Theorems 1--3; numerics: Appendix B; experiments: Sec. 4 and Appendix C.
  \end{itemize}
  \vspace{6mm}
  \centering\Large Questions and discussion
\end{frame}

\appendix

\begin{frame}{Backup: RMSNorm inversion is constrained fitting}
  For target $\mathbf g$ and fixed RMS $C$, find $\mathbf x$ minimizing
  \[
    \left\|\mathbf m\odot\frac{\mathbf x}{\operatorname{RMS}(\mathbf x)}-\mathbf g\right\|_2^2
    \quad\text{subject to}\quad \operatorname{RMS}(\mathbf x)=C.
  \]
  With $\mathbf y=\mathbf x/C$, the stationarity condition gives
  \[
    y_k=\frac{g_km_k}{m_k^2-\mu},\qquad
    \frac1n\sum_k\frac{(g_km_k)^2}{(m_k^2-\mu)^2}=1.
  \]
  The paper solves for the scalar $\mu$ and uses the result as $\mathbf h_{\rm target}$.
  \papersource{Paper, Appendix B.2, PDF pp. 14--15.}
\end{frame}

\begin{frame}{Backup: the gradient-step form is constructed from the patch}
  \[
    \mathcal L(W)=-\operatorname{Tr}(\Delta^\top W)
    \quad\Longrightarrow\quad -\nabla_W\mathcal L=\Delta.
  \]
  \begin{itemize}
    \item Choose $\Delta$ from the context-equivalent patch and a norm-based step size.
    \item Summing context increments telescopes to the full patch.
    \item This supplies an algebraic gradient representation for the desired update.
  \end{itemize}
  \papersource{Paper, Appendix E, PDF pp. 20--22.}
\end{frame}

\begin{frame}{Backup: per-token Mars forecast example}
  \centering
  \includegraphics[width=0.90\linewidth,height=0.71\textheight,keepaspectratio]{figures/fig-03-generation-metrics.pdf}
  \papersource{Original paper Fig. 3, PDF p. 5. Red crosses mark token mismatches.}
\end{frame}

\end{document}