\documentclass[aspectratio=169,11pt]{beamer}
\usepackage[T1]{fontenc}
\usepackage[utf8]{inputenc}
\usepackage{amsmath,amssymb,graphicx,booktabs}

\definecolor{AcademicBlue}{RGB}{26,57,86}
\definecolor{AcademicTeal}{RGB}{0,105,112}
\definecolor{AcademicOrange}{RGB}{190,87,24}
\setbeamercolor{structure}{fg=AcademicBlue}
\setbeamercolor{frametitle}{fg=AcademicBlue,bg=white}
\setbeamercolor{title}{fg=AcademicBlue}
\setbeamercolor{alerted text}{fg=AcademicTeal}
\setbeamercolor{block title}{fg=white,bg=AcademicBlue}
\setbeamercolor{block body}{fg=black,bg=AcademicBlue!5}
\setbeamertemplate{navigation symbols}{}
\setbeamertemplate{itemize items}[circle]
\setbeamertemplate{footline}{\hfill\usebeamerfont{page number in head/foot}\insertframenumber\,/\,\inserttotalframenumber\hspace{4mm}\vspace{3mm}}
\setbeamersize{text margin left=8mm,text margin right=8mm}
\setbeamerfont{frametitle}{size=\Large,series=\bfseries}
\setbeamerfont{title}{size=\LARGE,series=\bfseries}
\newcommand{\papersource}[1]{\par\smallskip{\scriptsize\color{gray}#1\par}}
\newcommand{\cp}{\mathrm{CP}}
\newcommand{\lm}{\mathrm{LM}}

\title[Copy-Paste Language Models]{Copy-Paste Language Models}
\subtitle{A direct output and gradient path for in-context copying}
\author{Paper by Nathan Godey and Yoav Artzi}
\institute{Research group discussion}
\date{\href{https://arxiv.org/abs/2610.19247v1}{arXiv:2610.19247v1} \quad 23 October 2026}

\begin{document}
\begin{frame}
  \titlepage
  \vspace{-2mm}
  \begin{center}\small\color{AcademicTeal}
  A lightweight pointer head improves pretraining and long-context access across tested backbones.
  \end{center}
\end{frame}

\begin{frame}{Copying is common, but the output layer makes it expensive to learn}
\begin{itemize}
  \item About \textbf{60\%} of FineWeb-Edu tokens have appeared earlier in their 4k-token context.
  \item Self-attention can retrieve a past token; the final prediction still passes through a fixed vocabulary projection $W\in\mathbb{R}^{V\times D}$.
  \item With $V\gg D$, this head limits output expressivity and compresses the backward signal before it reaches the backbone.
\end{itemize}
\begin{block}{Research question}
Can a direct pointer to previous positions improve modern decoder-only LM pretraining, even when the backbone already has attention?
\end{block}
\papersource{Paper, Secs. 1--2.1, PDF pp. 1--3.}
\end{frame}

\begin{frame}{The contribution is a second prediction \emph{and} gradient path}
\begin{columns}[T,onlytextwidth]
\begin{column}{0.48\textwidth}
\begin{block}{Standard LM head}
Predicts over the fixed vocabulary $\mathcal V$ using $w_v^\top h_t$. Its backward projection maps vocabulary-logit gradients to $D$ hidden dimensions.
\end{block}
\end{column}
\begin{column}{0.48\textwidth}
\begin{block}{Copy-Paste head}
Predicts over context positions $1{:}t$. Its target set changes with the context and provides a shorter gradient projection when $t\ll V$.
\end{block}
\end{column}
\end{columns}
\vspace{5mm}
\begin{center}\Large\color{AcademicTeal}Point to a position $\rightarrow$ recover its token $\rightarrow$ mix with the LM distribution.\end{center}
\papersource{Paper, Secs. 1--2.2, PDF pp. 2--3. Pointer mechanisms themselves predate this work.}
\end{frame}

\begin{frame}{One extra head points at previous positions}
\centering
\includegraphics[width=0.94\linewidth,height=0.67\textheight,keepaspectratio]{figures/copysink_schema.pdf}
\vspace{1mm}
\begin{itemize}
  \item \texttt{<copy>} gates the pointer; \texttt{<sink>} lets the pointer defer to the LM head.
\end{itemize}
\papersource{Paper, Fig. 1 (left), PDF p. 1. Original author figure.}
\end{frame}

\begin{frame}{The pointer distribution becomes a token distribution}
\begin{align*}
q_t&=W_Qh_t,\qquad k_\tau=W_Kh_\tau,\\[1mm]
\alpha_t(\tau)&=\operatorname{softmax}_{\tau'\le t}
\left(\frac{k_{\tau'}^\top q_t}{\sqrt D}\right)_\tau,\\[1mm]
p_{\cp}(v\mid x_{\le t})&=\sum_{\tau\le t:\,x_\tau=v}\alpha_t(\tau).
\end{align*}
\begin{itemize}
\item A token appearing twice receives the sum of both position probabilities.
\item An unseen token receives zero CP probability and remains available through $p_{\lm}$.
\end{itemize}
\papersource{Paper, Sec. 2.2, Eqs. (2)--(3), PDF p. 3. Default query/key dimension is $D$.}
\end{frame}

\begin{frame}{The gate and sink form a valid mixture over the vocabulary}
\[
\boxed{p(v\mid x_{\le t})=(1-\lambda_t)p_{\lm}(v\mid x_{\le t})
 +\lambda_t p_{\cp}(v\mid x_{\le t}),\qquad
\lambda_t=c_t(1-s_t)}
\]
\begin{itemize}
\item $c_t$ is the LM head probability of a special \texttt{<copy>} token.
\item $s_t$ is CP attention on a learned \texttt{<sink>} key.
\item $p_{\lm}$ is normalized over the \emph{real} vocabulary, without \texttt{<copy>}.
\end{itemize}
\begin{block}{Training signal}
Only standard next-token cross-entropy on $p$; no auxiliary loss or labeled copy decisions.
\end{block}
\papersource{Paper, Sec. 2.2, Eqs. (4)--(7), PDF p. 3.}
\end{frame}

\begin{frame}{The gate also makes exact conditional sampling possible}
\begin{enumerate}
\item Sample from the LM head over $\mathcal V\cup\{\texttt{<copy>}\}$.
\item If \texttt{<copy>} is sampled, compute copy attention over cached keys.
\item If the sink is sampled, fall back to the normalized LM distribution.
\end{enumerate}
\vspace{3mm}
\begin{block}{Implementation consequence}
Copy attention can be skipped on sampling steps that do not choose \texttt{<copy>}. At inference, one additional copy key is cached per token.
\end{block}
\begin{itemize}
\item Greedy decoding still requires the argmax of the \emph{mixture}; independent argmax decisions are not equivalent.
\end{itemize}
\papersource{Paper, Appendix B and C.4, PDF pp. 19, 23.}
\end{frame}

\begin{frame}{Matched comparisons span scale and backbone context access}
\centering
\begin{tabular}{lrrr}
\toprule
Scale & Training tokens & Train length & Hidden size \\
\midrule
135M & 20B & 2,048 & 576 \\
360M & 20B & 2,048 & 960 \\
1.7B & 200B & 4,096 & 2,048 \\
\bottomrule
\end{tabular}
\vspace{4mm}
\begin{itemize}
\item FineWeb-Edu pretraining; each vanilla/CPLM pair uses the same recipe.
\item Full attention, sliding-window (SW), and 3:1 Hybrid backbones; SW/Hybrid use a 256-token window in the main 1.7B comparison.
\item Hybrid provides global backbone access as a comparison to CP-mediated global access.
\end{itemize}
\papersource{Paper, Sec. 3.1 and Appendix A, PDF pp. 4, 18.}
\end{frame}

\begin{frame}{CPLMs reach the vanilla final loss with fewer tokens}
\begin{columns}[T,onlytextwidth]
\begin{column}{0.61\textwidth}
\centering
\includegraphics[width=\linewidth,height=0.67\textheight,keepaspectratio]{figures/val_nll_curves_1p7B.pdf}
\end{column}
\begin{column}{0.35\textwidth}
\begin{itemize}
\item 1.7B, 200B-token runs.
\item \textbf{Full / Hybrid:} about $1.25\times$ fewer tokens to the vanilla final loss.
\item \textbf{SW:} about $2\times$ fewer tokens.
\item Solid: CPLM; dashed: vanilla. Blue: Full; green: Hybrid; orange: SW.
\end{itemize}
\end{column}
\end{columns}
\papersource{Paper, Fig. 2, PDF p. 4. Curves show validation NLL before LR decay.}
\end{frame}

\begin{frame}{Base-model accuracy improves modestly; LAMBADA gains most}
\centering
\begin{tabular}{lrrr}
\toprule
1.7B zero-shot average (\%) & Vanilla & CPLM & Gain (pt) \\
\midrule
Full & 62.45 & \textbf{63.06} & +0.61 \\
SW-256 & 61.63 & \textbf{62.48} & +0.85 \\
Hybrid 3:1 & 62.22 & \textbf{62.59} & +0.37 \\
\bottomrule
\end{tabular}
\vspace{5mm}
\begin{block}{Task with a clear copy opportunity}
LAMBADA: Full $44.46\rightarrow46.92$; SW $42.46\rightarrow46.42$. The target word often appeared earlier in the passage.
\end{block}
\papersource{Paper, Sec. 3.1, Table 1, PDF p. 4. Average of seven benchmark columns.}
\end{frame}

\begin{frame}{The copy cache shifts the perplexity--memory frontier}
\begin{columns}[T,onlytextwidth]
\begin{column}{0.59\textwidth}
\centering
\includegraphics[width=\linewidth,height=0.72\textheight,keepaspectratio]{figures/mem_vs_ppl_norm.pdf}
\end{column}
\begin{column}{0.37\textwidth}
\begin{itemize}
\item On FinePDFs, 1.7B CPLMs lower PPL by about 0.5--1.5 points against paired vanilla backbones.
\item SW CPLM matches the Full vanilla model at about \textbf{12$\times$ less cache per token}, including its copy cache.
\item Lower and further left is better.
\end{itemize}
\end{column}
\end{columns}
\papersource{Paper, Fig. 1 (right), PDF p. 1. Original author plot.}
\end{frame}

\begin{frame}{CPLMs keep exploiting document context beyond train length}
\begin{columns}[T,onlytextwidth]
\begin{column}{0.59\textwidth}
\centering
\includegraphics[width=\linewidth,height=0.72\textheight,keepaspectratio]{figures/rolling16k_5000_ppl.pdf}
\end{column}
\begin{column}{0.37\textwidth}
\begin{itemize}
\item 5,000 FinePDFs documents, 16k tokens each; train length was 4k.
\item CPLM PPL falls to about 6.5--7 at 16k; vanilla SW/Hybrid plateau around 10--13.
\item SW CPLM copies from over 2,000 tokens back on average at position 16k, beyond its 256-token window.
\end{itemize}
\end{column}
\end{columns}
\papersource{Paper, Fig. 4, PDF p. 5; Appendix G.2, Fig. 15, PDF p. 28. Full/Hybrid use YaRN $\times4$.}
\end{frame}

\begin{frame}{Supervised fine-tuning preserves the gains}
\centering
\begin{tabular}{lrrr}
\toprule
1.7B SFT task average & Vanilla & CPLM & Gain (pt) \\
\midrule
Full & 24.2 & \textbf{26.4} & +2.2 \\
Hybrid 3:1 & 24.3 & \textbf{25.5} & +1.2 \\
SW-256 & 18.9 & \textbf{24.0} & +5.1 \\
\bottomrule
\end{tabular}
\vspace{4mm}
\begin{itemize}
\item Tulu-3 SFT, up to 16k context; math, code and instruction evaluations.
\item SW: MATH $7.2\rightarrow16.4$, GSM8K $18.6\rightarrow25.2$.
\item Hybrid MBPP is an exception: $19.0\rightarrow17.4$.
\end{itemize}
\papersource{Paper, Sec. 3.2, Table 2, PDF p. 6. MATH/GSM8K use pass@4; code uses pass@10; see Appendix D.}
\end{frame}

\begin{frame}{Long-context gains are strongest for the local backbone}
\centering
\begin{tabular}{lccc}
\toprule
1.7B base-model score & Full V $\to$ CP & Hybrid V $\to$ CP & SW V $\to$ CP \\
\midrule
BABILong & 26.4 $\to$ 32.0 & 27.7 $\to$ 32.9 & 13.8 $\to$ 16.1 \\
RULER & 33.6 $\to$ 39.6 & 33.2 $\to$ 37.8 & 7.9 $\to$ 29.2 \\
LongBench & 10.4 $\to$ 12.2 & 9.5 $\to$ 14.3 & 7.4 $\to$ 13.4 \\
\bottomrule
\end{tabular}
\vspace{5mm}
\begin{itemize}
\item RULER: mean of 13 tasks across 4k--16k; BABILong: qa1--qa5 across 0k--16k.
\item LongBench: mean over 16 English tasks at 16k.
\item The small SW BABILong gain shows that long-range copying alone does not solve multi-fact reasoning.
\end{itemize}
\papersource{Paper, Sec. 3.3, Table 3, PDF p. 6. Full/Hybrid use YaRN $\times4$; SW uses base RoPE.}
\end{frame}

\begin{frame}{A 256-token-window model retrieves needles through 64k}
\begin{columns}[T,onlytextwidth]
\begin{column}{0.39\textwidth}
\centering
\includegraphics[width=\linewidth,height=0.64\textheight,keepaspectratio]{figures/niah_sw_vanilla.png}\\[-1mm]
{\small Vanilla}
\end{column}
\begin{column}{0.39\textwidth}
\centering
\includegraphics[width=\linewidth,height=0.64\textheight,keepaspectratio]{figures/niah_sw_cplm.png}\\[-1mm]
{\small CPLM}
\end{column}
\begin{column}{0.20\textwidth}
\small
\textbf{Grid mean}\par
14.3 $\to$ 99.4\%\par\medskip
\textbf{Protocol}\par
7-digit needle; 200 examples per cell; exact number in output.
\end{column}
\end{columns}
\papersource{Paper, Appendix E, Fig. 13, PDF p. 25. Cropped original SW panels; columns are 1k--64k context, rows are needle depths.}
\end{frame}

\begin{frame}{Copy keys can serve as compact document memory}
\begin{columns}[T,onlytextwidth]
\begin{column}{0.57\textwidth}
\centering
\includegraphics[width=\linewidth,height=0.62\textheight,keepaspectratio]{figures/hotpot_yarn4_distractor_dropdocs.pdf}
\end{column}
\begin{column}{0.39\textwidth}
\begin{itemize}
\item HotpotQA with documents hidden from answer-side backbone attention.
\item CP keys alone give \textbf{34--38\% EM} at 500 distractors.
\item Reported cache is about \textbf{50$\times$ smaller} than Full document KV.
\item Last-layer KV control stays near 14\% EM.
\end{itemize}
\end{column}
\end{columns}
\centering\includegraphics[width=0.65\linewidth,height=0.045\textheight,keepaspectratio]{figures/hotpot_legend.pdf}
\papersource{Paper, Sec. 3.4, Fig. 5 (middle), PDF pp. 7--8. 200 validation questions; each regime is fine-tuned separately.}
\end{frame}

\begin{frame}{The CP path has a more favorable gradient shape}
\begin{columns}[T,onlytextwidth]
\begin{column}{0.55\textwidth}
\centering
\includegraphics[width=\linewidth,height=0.54\textheight,keepaspectratio]{figures/lmhead_nullspace_seqlen.pdf}\\[1mm]
\includegraphics[width=\linewidth,height=0.04\textheight,keepaspectratio]{figures/gradient_legend.pdf}
\end{column}
\begin{column}{0.41\textwidth}
\begin{itemize}
\item LM head: over 99\% of logit-gradient norm is lost in the measured setup.
\item CP projections use $t$ context positions rather than $V$ vocabulary items; at early positions, up to 85\% of gradient norm is retained.
\item SpamLang: at $V=512$k, vanilla reaches 34\% accuracy; CPLM solves the copying task across tested $V$.
\end{itemize}
\end{column}
\end{columns}
\papersource{Paper, Sec. 4.1, Fig. 6, PDF pp. 8--9. Plot: lost gradient fraction vs. sequence position.}
\end{frame}

\begin{frame}{Two interventions test whether the gradient path matters}
\centering
\begin{tabular}{lrr}
\toprule
135M token model & Val. NLL $\downarrow$ & PPL $\downarrow$ \\
\midrule
Vanilla & 2.794 & 16.34 \\
CPLM & \textbf{2.763} & \textbf{15.85} \\
CPLM, stop CP $\to$ backbone gradient & 2.786 & 16.22 \\
\bottomrule
\end{tabular}
\vspace{3mm}
\begin{itemize}
\item Stop-gradient leaves $0.008$ of the $0.031$ nat/token gain: about one quarter.
\item Byte-level $V=257 < D=576$: vanilla NLL 0.6908, CPLM 0.6918; no benefit when this LM-head rank bottleneck is removed.
\item These targeted tests support the authors' gradient-path explanation.
\end{itemize}
\papersource{Paper, Appendix H, Tables 15--16, PDF pp. 30--31. Byte NLL is per byte and is not comparable to token NLL.}
\end{frame}

\begin{frame}{The backbone delegates some copying to the CP head}
\begin{itemize}
\item On a repeated random-sequence probe, CPLMs predict the repeated half more accurately.
\item At sequence length $T=128$, Full/Hybrid CPLM backbones contain \textbf{4} detected induction heads versus \textbf{13--15} in vanilla backbones.
\item Low-copy-gate tokens also receive lower NLL in CPLMs, consistent with a better-trained shared backbone.
\end{itemize}
\vspace{3mm}
\begin{block}{Interpretation}
The CP head can carry copying work that vanilla models spread over backbone attention heads; the effect reaches the LM branch as well.
\end{block}
\papersource{Paper, Sec. 4.2, Fig. 7, PDF p. 9; Appendix G.1, Fig. 14, PDF p. 27.}
\end{frame}

\begin{frame}{Measured overhead is small, with clear accounting}
\centering
\begin{tabular}{lp{0.63\textwidth}}
\toprule
Resource & Reported measurement \\
\midrule
Parameters & Two $D\!\times\!D$ projections + sink; about +0.5\% at 1.7B. \\
Training & 1.7B: 55,862 $\to$ 54,113 tokens/s on one B300 at 4k (0.969$\times$). \\
Inference & 1.7B Full: copy cache = 2.1\% of KV cache; decoding step about 1.7--1.8\% slower in reported settings. \\
\bottomrule
\end{tabular}
\vspace{4mm}
\begin{itemize}
\item 9B and 30B results in the appendix measure overhead on \emph{untrained} architectures, not task quality.
\item Fused and chunked training kernels control temporary memory.
\end{itemize}
\papersource{Paper, Appendix B, Tables 5--6, PDF p. 19.}
\end{frame}

\begin{frame}{What the evidence establishes---and where it stops}
\begin{enumerate}
\item A trained pointer head improves LM pretraining across the tested 135M--1.7B sizes and Full, SW, Hybrid and GDN backbones.
\item Gradient interventions explain much of the gain; the pointer design itself is related to prior copy mechanisms.
\item SW gains are striking for retrieval, but complex multi-fact tasks remain hard; Full models still fail the 16k+ needle test.
\end{enumerate}
\vspace{3mm}
\begin{block}{Take-home message}
Giving the model an explicit way to copy changes both what it can access and how the backbone is trained.
\end{block}
\papersource{Paper, Secs. 3--6 and Appendices E--H, PDF pp. 4--10, 25--32.}
\end{frame}

\begin{frame}{Paper and pointers for discussion}
\begin{itemize}
\item Nathan Godey and Yoav Artzi. \emph{Copy-Paste Language Models}. arXiv:2610.19247v1, 2026.\\\href{https://arxiv.org/abs/2610.19247v1}{https://arxiv.org/abs/2610.19247v1}
\item Main mechanism: Sec. 2, Eqs. (2)--(7), PDF p. 3.
\item Evaluation: Sec. 3, Tables 1--3, PDF pp. 4--8.
\item Mechanistic interventions: Appendix H, Tables 15--16, PDF pp. 30--32.
\end{itemize}
\vspace{4mm}
\begin{center}\color{AcademicTeal}\Large Discussion: when is a learned pointer a better use of memory than longer backbone attention?\end{center}
\end{frame}

\appendix
\begin{frame}{Backup: pointer designs are close on small-scale perplexity}
\centering
\begin{tabular}{lrrrr}
\toprule
 & \multicolumn{2}{c}{Validation PPL} & \multicolumn{2}{c}{FinePDF PPL} \\
Method & Full & SW & Full & SW \\
\midrule
Vanilla & 16.34 & 16.88 & 14.25 & 16.52 \\
Sigmoid gate & 15.94 & 16.20 & 13.39 & 14.25 \\
Shared softmax & 15.88 & 16.09 & 13.32 & \textbf{13.82} \\
CPLM + sink & \textbf{15.85} & \textbf{16.02} & \textbf{13.31} & \textbf{13.82} \\
\bottomrule
\end{tabular}
\vspace{4mm}
\begin{itemize}
\item Margins among pointer variants are small; the paper emphasizes training with a pointer, not a novel pointer operator.
\end{itemize}
\papersource{Paper, Appendix C.4, Table 8, PDF p. 23. 135M, 20B tokens, seed 42.}
\end{frame}

\begin{frame}{Backup: retrieval and cache comparisons}
\begin{itemize}
\item A tuned inference-time neural cache barely changes 1.7B FineWeb-Edu PPL: Full $9.17\to9.17$, Hybrid $9.20\to9.20$, SW $9.65\to9.63$.
\item Trained CPLMs score 9.06, 9.06 and 9.24, respectively, on the same 100-document evaluation.
\item At 135M, reducing CP key/query dimension from 576 to 64 changes validation PPL by less than 0.02 while using $9\times$ fewer copy-key values per token.
\end{itemize}
\papersource{Paper, Appendix C.2 and C.5, Fig. 8 and Table 9, PDF pp. 20, 24.}
\end{frame}

\end{document}