% Self-authored reading deck, based on the arxiv-paper-reader Beamer template.
% Build with: python3 build_slides.py slides.en.tex
\documentclass[aspectratio=169,11pt]{beamer}
\usepackage[T1]{fontenc}
\usepackage[utf8]{inputenc}
\usepackage{amsmath,amssymb,graphicx,booktabs}
\definecolor{AcademicBlue}{RGB}{26,57,86}
\definecolor{AcademicTeal}{RGB}{0,105,112}
\setbeamercolor{structure}{fg=AcademicBlue}
\setbeamercolor{frametitle}{fg=AcademicBlue,bg=white}
\setbeamercolor{title}{fg=AcademicBlue}
\setbeamercolor{alerted text}{fg=AcademicTeal}
\setbeamercolor{block title}{fg=white,bg=AcademicBlue}
\setbeamercolor{block body}{fg=black,bg=AcademicBlue!5}
\setbeamertemplate{navigation symbols}{}
\setbeamertemplate{itemize items}[circle]
\setbeamertemplate{footline}{\hfill\usebeamerfont{page number in head/foot}\insertframenumber\,/\,\inserttotalframenumber\hspace{4mm}\vspace{3mm}}
\setbeamersize{text margin left=8mm,text margin right=8mm}
\setbeamerfont{frametitle}{size=\Large,series=\bfseries}
\setbeamerfont{title}{size=\LARGE,series=\bfseries}
\newcommand{\papersource}[1]{\par\smallskip{\scriptsize\color{gray}#1\par}}
\newcommand{\sg}{\operatorname{sg}}
\newcommand{\ntp}{\mathcal L_{\mathrm{NTP}}}
\newcommand{\ncp}{\mathcal L_{\mathrm{NCP}}}
\newcommand{\vq}{\mathcal L_{\mathrm{VQ}}}
\newcommand{\avgppl}{\text{Avg PPL}}
\hypersetup{colorlinks=true,urlcolor=AcademicTeal}
\title[ConceptLM]{Next Concept Prediction in Discrete Latent Space Leads to Stronger Language Models}
\subtitle{ConceptLM: paper reading and evidence review}
\author{Yuliang Liu, Yunchong Song, Yixuan Wang, Kewen Ge, Alex Lamb,\\
Qipeng Guo, Kai Chen, Bowen Zhou, Zhouhan Lin}
\date{arXiv:2602.08984v1 \quad | \quad Reading: 28 September 2026}

\begin{document}

\begin{frame}
\titlepage
\vspace{-5mm}
\centering\small Reading presentation; the presenter is not a paper author.\\[1mm]
\href{https://arxiv.org/abs/2602.08984v1}{Versioned paper}\quad $\cdot$ \quad 20 main slides + 8 backup slides
\end{frame}

\begin{frame}{Predict a future chunk, then generate its tokens}
\begin{block}{Research question}
Can supervision on future multi-token representations improve a language model trained with next-token prediction?
\end{block}
\begin{itemize}
\item Pool adjacent token states into a concept; learn a segmented codebook.
\item Predict the next concept and inject it into token generation.
\item Evidence is strongest for small-model pretraining; the 8B gain is modest.
\end{itemize}
\medskip
\alert{A concept is a learned chunk representation, not a labeled semantic unit.}
\papersource{Paper, Sec. 1, PDF pp. 1--2; Tables 2--3, pp. 8--9.}
\end{frame}

\begin{frame}{Two resolutions share one language model}
\centering
\includegraphics[width=0.99\linewidth,height=0.81\textheight,keepaspectratio]{figures/fig-02-architecture.pdf}
\papersource{Paper, Fig. 2, PDF p. 3. Original diagram uses 3-token groups; main experiments use $k=4$.}
\end{frame}

\begin{frame}{Product quantization builds a compact vocabulary}
\[
c_j=\frac{1}{k}\sum_{r=0}^{k-1}h_{jk+r}\in\mathbb R^d,
\qquad M=T/k
\]
\begin{itemize}
\item Split each $c_j$ into $S$ feature segments of width $d/S$.
\item Each segment has $N$ code vectors $e_n^s\in\mathbb R^{d/S}$.
\item $N^S$ possible index combinations; only $Nd$ raw code scalars.
\item Main setting: $k=4$, $N=64$, $S=$ attention head count.
\end{itemize}
\begin{block}{Avoiding codebook collapse}
Two MLP layers with ReLU transform the codebook embeddings, inspired by SimVQ.
\end{block}
\papersource{Paper, Secs. 2.2 and 4.1, PDF pp. 4, 6; App. E.3, p. 21. Pooling equation expanded from the text.}
\end{frame}

\begin{frame}{The predicted concept is a weighted code mixture}
\[
\hat c_j^s=\sum_{n=1}^{N} w_{j,n}^s e_n^s,
\qquad
\hat c_j=\operatorname{concat}(\hat c_j^1,\ldots,\hat c_j^S)
\]
\begin{itemize}
\item Concept Transformers read past \emph{continuous} states $c_{<j}$.
\item $S$ heads output $N$ coefficients each; no $N^S$-way head.
\item The decoder receives a continuous mixture, not a sampled code ID.
\item NCP fits future continuous states with MSE, not code-ID cross-entropy.
\end{itemize}
\medskip
\alert{Normalization is ambiguous in the paper; do not assume a softmax.}
\papersource{Paper, Secs. 2.2--3, PDF pp. 4--5; App. A, p. 18. Public Pythia code uses raw head outputs; see backup.}
\end{frame}

\begin{frame}{Causal alignment is the essential constraint}
For $k=4$, the full first chunk is available only after reading $x_3$.
\medskip
\centering
\begin{tabular}{lll}
\toprule
Decoder position & Next-token targets & Added prediction \\
\midrule
$t=0,1,2$ & $x_1,x_2,x_3$ & zero \\
$t=3,4,5,6$ & $x_4,x_5,x_6,x_7$ & $\hat c_1$ from $c_0$ \\
$t=7,8,9,10$ & $x_8,x_9,x_{10},x_{11}$ & $\hat c_2$ from $c_0,c_1$ \\
\bottomrule
\end{tabular}
\medskip
\raggedright
\begin{itemize}
\item Repeat each concept $k$ times; delay by $k-1$ token positions.
\item The future chunk is a loss target, never a decoder input.
\item Tokens are still generated autoregressively, one at a time.
\end{itemize}
\papersource{Teaching example from Sec. 2.3, PDF pp. 4--5. The printed $\lfloor t/(k-1)\rfloor$ index conflicts with this rule.}
\end{frame}

\begin{frame}{Three losses support different learning roles}
\begin{columns}[c,onlytextwidth]
\column{0.57\textwidth}
\[\mathcal L=\ntp+\ncp+\vq\]
\textbf{Token prediction}\par
Cross-entropy trains token generation.
\par\medskip
\textbf{Concept prediction}
\[\ncp=\frac{1}{M-1}\sum_{j=1}^{M-1}\|\hat c_j-c_j\|_2^2\]
Fit the next chunk's continuous representation.
\column{0.39\textwidth}
\centering
\includegraphics[width=\linewidth,height=0.64\textheight,keepaspectratio]{figures/fig-03-gradients.pdf}
\end{columns}
\papersource{Paper, Fig. 3 and Sec. 3, PDF pp. 3, 5--6. Public Pythia code stops gradients through the NCP target.}
\end{frame}

\begin{frame}{Codebook fitting needs an explicit gradient boundary}
The paper writes the VQ objective as
\[
\vq=\|\sg(c)-d_q\|_2^2+\beta\|c-\sg(d_q)\|_2^2.
\]
\begin{itemize}
\item $d_q$ denotes the quantized vector (written $d$ in the paper).
\item The text says VQ loss updates only the quantizer.
\item The second term would update $c$ unless it is detached upstream.
\item Public Pythia code detaches the pooled states before VQ, consistent with the stated intent.
\end{itemize}
\medskip
\alert{A formula alone does not specify the full training graph.}
\papersource{Paper, Sec. 3, PDF p. 5; Fig. 3, p. 3. Code check is supplementary, not an experimental reproduction.}
\end{frame}

\begin{frame}{Three training regimes test different claims}
\centering
\begin{tabular}{llll}
\toprule
Backbone labels & Training data & Tokens & Context \\
\midrule
GPT-2 124M--1.5B & OpenWebText & 8B & 1024 \\
Pythia 70M--410M & The Pile & 300B & 2048 \\
Llama-3.1-8B & LongCtxEng & 9.6B & 8192 \\
\bottomrule
\end{tabular}
\medskip\raggedright
\begin{itemize}
\item PM adds two token layers; ConceptLM adds two concept layers.
\item ContextLM tests a related hierarchy without the explicit NCP vocabulary.
\item Main pretraining: 9 accuracy tasks + 4 PPL evaluations.
\item Default ablations: Pythia-160M on \alert{30B}, not 300B tokens.
\end{itemize}
\papersource{Paper, Sec. 4 and App. B, PDF pp. 6--9, 18--20. Sizes label the base backbone; added-layer totals differ.}
\end{frame}

\begin{frame}{GPT-2 improves over token baselines at every size}
\centering
\begin{tabular}{lrrrr}
\toprule
Base size & Base & PM & ContextLM & ConceptLM \\
\midrule
124M & 101.94 & 99.14 & 87.19 & \textbf{84.75} \\
355M & 52.57 & 52.55 & 43.06 & \textbf{40.51} \\
774M & 36.35 & 32.89 & 31.80 & \textbf{29.61} \\
1.5B & 29.68 & 30.01 & \textbf{25.93} & 26.38 \\
\bottomrule
\end{tabular}
\medskip\raggedright
\begin{itemize}
\item Reported Avg PPL $\downarrow$: arithmetic mean across four datasets.
\item At 355M, in-domain PPL also improves: 17.97 $\to$ 16.81.
\item At 1.5B, ContextLM wins Avg PPL and Lambada Standard PPL.
\end{itemize}
\papersource{Paper, Table 1, PDF p. 7. OpenWebText, 8B tokens; main ConceptLM uses insertion before layer 1.}
\end{frame}

\begin{frame}{Pythia gains depend on scale and task}
\centering
\begin{tabular}{lrrr}
\toprule
Base size & Base & ContextLM & ConceptLM \\
\midrule
70M & 34.6 & 38.2 & 38.2 \\
160M & 40.3 & 42.0 & \textbf{42.9} \\
410M & 47.5 & 48.2 & \textbf{48.3} \\
\bottomrule
\end{tabular}
\medskip\raggedright
\begin{itemize}
\item Nine-task Avg Acc (\%) $\uparrow$, after 300B Pile tokens.
\item At 160M: +2.6 percentage points vs Base; +0.9 vs ContextLM.
\item Individual regressions remain: ARC-C 19.5 $\to$ 18.3 vs ContextLM.
\item Pythia Base/ContextLM results are taken from prior work, with settings aligned by the authors.
\end{itemize}
\papersource{Paper, Table 2 and Sec. 4.2, PDF pp. 7--8. No reported run-to-run uncertainty.}
\end{frame}

\begin{frame}{Scaling curves suggest separate resource savings}
\centering
\includegraphics[width=\linewidth,height=0.57\textheight,keepaspectratio]{figures/fig-01-scaling.pdf}
\medskip\raggedright
Near the largest GPT-2 baseline's PPL, the authors report:
\begin{itemize}
\item 37\% fewer parameters, 24\% fewer tokens, or 19\% fewer FLOPs.
\item These are \alert{separate comparisons}, not simultaneous savings.
\end{itemize}
\papersource{Paper, Fig. 1, PDF p. 1; Sec. 4.2, p. 7. Original panels; no fitted law or uncertainty intervals reported.}
\end{frame}

\begin{frame}{The 8B improvement is small and mixed}
\centering
\begin{tabular}{lrrrrr}
\toprule
Model & MMLU & ARC-C & AGIEval & SQuAD 2.0 & Avg \\
\midrule
Trained & 64.3 & 34.8 & 54.2 & \textbf{36.7} & 47.5 \\
PM & 64.5 & 35.0 & 53.3 & 36.1 & 47.2 \\
ConceptLM & \textbf{64.6} & \textbf{35.1} & \textbf{54.2} & 36.5 & \textbf{47.6} \\
\bottomrule
\end{tabular}
\medskip\raggedright
\begin{itemize}
\item All receive 9.6B tokens of continual pretraining.
\item +0.4 average points vs PM; only +0.1 vs Trained.
\item No confidence interval or multi-seed variance is reported.
\end{itemize}
\papersource{Paper, Table 3 and Sec. 4.3, PDF p. 9. SQuAD metric variant is not specified; retain the reported score.}
\end{frame}

\begin{frame}{Joint objectives beat either auxiliary loss alone}
\centering
\begin{tabular}{lrr}
\toprule
Training objective & Avg Acc $\uparrow$ & Avg PPL $\downarrow$ \\
\midrule
Token-model MTP & 38.4 & 81.16 \\
ConceptLM: NTP only & 39.6 & 69.28 \\
NTP + VQ & 39.8 & 75.64 \\
NTP + NCP & 39.5 & 76.30 \\
NTP + NCP + VQ & \textbf{40.3} & \textbf{68.13} \\
\bottomrule
\end{tabular}
\medskip\raggedright
\begin{itemize}
\item Same ConceptLM architecture for the four objective variants.
\item Full objective vs NTP-only: +0.7 accuracy points, $-1.15$ Avg PPL.
\item VQ alone worsens PPL but slightly improves accuracy.
\end{itemize}
\papersource{Paper, Table 4, PDF p. 10; Pythia-160M, 30B tokens. MTP is a small-model comparison only.}
\end{frame}

\begin{frame}{Longer training context increases the loss advantage}
\begin{columns}[c,onlytextwidth]
\column{0.57\textwidth}
\includegraphics[width=\linewidth,height=0.72\textheight,keepaspectratio]{figures/fig-04-length.pdf}
\column{0.39\textwidth}
\begin{itemize}
\item Training lengths: 512, 1024, 2048.
\item Top: language modeling loss.
\item Bottom: PM minus ConceptLM loss.
\item Approximate gap: 0.006 $\to$ 0.011.
\end{itemize}
\medskip
\alert{No retrieval or context-extrapolation test here.}
\end{columns}
\papersource{Paper, Fig. 4 and Sec. 5.3, PDF pp. 10--11. Gap values are approximate readings of the plot.}
\end{frame}

\begin{frame}{Depth results require checking the reported averages}
\centering
\begin{tabular}{lrrr}
\toprule
Added layers & Token Avg PPL & Concept Avg PPL & Acc: token / concept \\
\midrule
2 & 71.3500 & 68.1325 & 40.1 / 40.3 \\
6 & 65.2525 & 54.4550 & 40.1 / 40.9 \\
10 & 75.1350 & 58.3550 & 40.2 / 40.4 \\
\bottomrule
\end{tabular}
\medskip\raggedright
\begin{itemize}
\item PPL means above are \alert{recomputed from the four printed entries}.
\item Paper reports 59.80 for +2 token layers; its entries average 71.35.
\item The concept advantage remains, but quality is not monotonic in depth.
\end{itemize}
\papersource{Paper, Table 5 and Sec. 5.4, PDF p. 11. Recalculation, not a new model evaluation. Fixed-width Pythia experiment.}
\end{frame}

\begin{frame}{Cheaper concept layers reduce time relative to PM}
\[
\text{Token layer: }O(T^2d+Td^2)
\quad\longrightarrow\quad
\text{Concept layer: }O(T^2d/k^2+Td^2/k)
\]
\centering
\begin{tabular}{lrr}
\toprule
Llama configuration & Training & SQuAD inference \\
\midrule
34 token layers (PM) & 80.0 h & 99.0 min \\
32 token + 2 concept layers & 71.0 h & 86.0 min \\
Time reduction & 11.25\% & 13.13\% \\
\bottomrule
\end{tabular}
\medskip\raggedright
\begin{itemize}
\item Training uses 8 H200 GPUs; theoretical FLOPs saving is about 4.69\%.
\item IO savings are the authors' explanation, without detailed profiling.
\item The 32 token layers remain; this is not a $k$-fold whole-model speedup.
\end{itemize}
\papersource{Paper, Table 6, PDF p. 12; App. C, pp. 19--20. Savings are against PM, not the unexpanded 32-layer model.}
\end{frame}

\begin{frame}{What the evidence does not establish}
\begin{itemize}
\item \textbf{Semantic interpretability:} fixed token chunks need not be concepts meaningful to people.
\item \textbf{A unique causal mechanism:} architecture, auxiliary supervision, and codebooks change together.
\item \textbf{Reliable large-model gains:} +0.1 average points vs continued training has no uncertainty estimate.
\item \textbf{Universal scaling:} tested depth, context length, and training budgets are limited.
\end{itemize}
\begin{block}{Reproduction details matter}
Shift indices, gradient boundaries, metric definitions, and several table averages need clarification.
\end{block}
\papersource{Reader analysis based on Secs. 2--5 and Apps. B--E. Numerical issues do not by themselves invalidate all results.}
\end{frame}

\begin{frame}{A useful direction, with a precise claim}
\begin{enumerate}
\item \textbf{Method:} learn a segmented concept vocabulary and predict future chunk states alongside tokens.
\medskip
\item \textbf{Evidence:} generally better small-model pretraining results, with lower time than added token layers in the reported 8B setup.
\medskip
\item \textbf{Next step:} isolate NCP under matched compute and repeated runs; test real long-context tasks and larger-scale training.
\end{enumerate}
\medskip
\alert{Discrete codebooks support continuous predictions; causal token generation remains intact.}
\papersource{Synthesis of the paper and this reading. Proposed next steps are reader suggestions, not reported experiments.}
\end{frame}

\begin{frame}{Sources and scope of this reading}
\textbf{Primary paper}\par
Yuliang Liu et al. (2026). \emph{Next Concept Prediction in Discrete Latent Space Leads to Stronger Language Models}. arXiv:2602.08984v1.\par
\href{https://arxiv.org/abs/2602.08984v1}{https://arxiv.org/abs/2602.08984v1}
\medskip
\begin{itemize}
\item Read the same-version author TeX, full main text, and Appendices A--E; checked figures and tables against the 25-page PDF.
\item Original figures are reused with page locators. Recomputed values are explicitly labeled.
\item Supplement: static Pythia code inspection at commit \texttt{a0ab281}; no training or benchmark reproduction.
\end{itemize}
\href{https://github.com/LUMIA-Group/ConceptLM}{Official repository: github.com/LUMIA-Group/ConceptLM}
\papersource{Accessed 28 September 2026. The following eight slides are backup material.}
\end{frame}

% Backup slides follow.
\appendix

\begin{frame}{Backup: the main configurations are not identical}
\centering
\begin{tabular}{llll}
\toprule
Backbone & Insert point & Batch & Learning rates \\
\midrule
GPT-2 & Before layer 1 & 512 & $10^{-3}$ to $4\!\times\!10^{-4}$ \\
Pythia & After layer 1 & 1024 & $10^{-3}$ to $4\!\times\!10^{-4}$ \\
Llama & After layer 1 & 96 & $10^{-5}$ \\
\bottomrule
\end{tabular}
\medskip\raggedright
\begin{itemize}
\item Main models use two concept layers, $k=4$, $N=64$, $S=$ head count.
\item GPT-2 also tests insertion after layer 1 in Appendix E.1.
\item Paper's GPT overview says 160M; detailed tables use a 124M base.
\item New Llama layers initialize from the first two base-model layers.
\end{itemize}
\papersource{Paper, Secs. 4.1 and 4.3; Tables 8--12, PDF pp. 18--20. Optimizer schedules and seeds are not fully specified.}
\end{frame}

\begin{frame}{Backup: GPT accuracy improves, with a labeling caveat}
\centering
\begin{tabular}{lrrrr}
\toprule
Base size & Base & PM & ConceptLM L0 & ConceptLM L1 \\
\midrule
124M & 38.0 & 38.4 & 38.9 & 38.7 \\
355M & 40.6 & 41.4 & 42.3 & 41.5 \\
774M & 42.8 & 42.8 & 44.0 & 43.5 \\
1.5B & 43.9 & 43.5 & 45.1 & 44.7 \\
\bottomrule
\end{tabular}
\medskip\raggedright
\begin{itemize}
\item Nine-task Avg Acc (\%); L0 is the main setting.
\item L0 beats L1 here, so insertion depth affects the comparison.
\item The 124M group has another row labeled ``ConceptLM'' at 39.1; its identity is ambiguous in Tables 2 and 14.
\end{itemize}
\papersource{Paper, Table 2, PDF p. 8; Table 14, p. 23. The ambiguous row is not silently renamed as ContextLM.}
\end{frame}

\begin{frame}{Backup: the best insertion depth depends on the metric}
\centering
\begin{tabular}{lrrrrrrr}
\toprule
Depth & L0 & L1 & L3 & L5 & L7 & L9 & L11 \\
\midrule
Avg Acc & \textbf{40.5} & 40.3 & 39.6 & 40.3 & 40.3 & 40.4 & 40.3 \\
Avg PPL & 63.25* & 68.13 & 62.09 & 60.64 & 64.34 & \textbf{55.04} & 59.79 \\
\bottomrule
\end{tabular}
\medskip\raggedright
\begin{itemize}
\item Pythia-160M, 30B tokens; reported values shown.
\item L9 is best for reported Avg PPL; L0 wins Avg Acc.
\item *L0's four PPL entries average 62.5025, not 63.25.
\item ``Three-quarters depth is best'' needs a metric and setup qualifier.
\end{itemize}
\papersource{Paper, Table 15, PDF p. 24; App. B.2, pp. 18--19 and E.2, p. 21.}
\end{frame}

\begin{frame}{Backup: broad code usage is not semantic validation}
\centering
\includegraphics[width=0.96\linewidth,height=0.57\textheight,keepaspectratio]{figures/fig-05-codebook-selected.pdf}
\par\medskip\raggedright
Components 0 and 1 of 12; 1024 entries each.\par
Axes: code index and use count. Full figure is in the paper and notes.\par\medskip
\alert{High utilization does not imply uniform usage or interpretable concepts.}
\papersource{Paper, Fig. 5 and App. E.3, PDF p. 21. Two original panels cropped; $N=1024$, not the main $N=64$.}
\end{frame}

\begin{frame}{Backup: vocabulary size has a nonmonotonic effect}
\begin{columns}[T,onlytextwidth]
\column{0.48\textwidth}
\textbf{Codebook size $N$}\par\medskip
\begin{tabular}{rrr}
\toprule
$N$ & Avg Acc & Avg PPL \\
\midrule
8 & 40.0 & 66.88 \\
16 & 40.3 & \textbf{63.03} \\
32 & 40.2 & 69.20 \\
64 & 40.3 & 68.13 \\
128 & \textbf{40.4} & 68.06 \\
256 & 40.3 & 70.38 \\
\bottomrule
\end{tabular}
\column{0.48\textwidth}
\textbf{Segments $S$, selected rows}\par\medskip
\begin{tabular}{lrr}
\toprule
$S$ & Avg Acc & Avg PPL \\
\midrule
$4H$ & 40.3 & 72.46 \\
$2H$ & \textbf{40.7} & \textbf{62.03} \\
$H$ & 40.3 & 68.13 \\
$H/2$ & 40.4 & 62.09 \\
\bottomrule
\end{tabular}
\par\medskip
$H=$ attention head count.\par
More combinations do not ensure better prediction.
\end{columns}
\papersource{Paper, Tables 16--17, PDF pp. 24--25. Reported means; Pythia-160M, 30B tokens, hidden width 768.}
\end{frame}

\begin{frame}{Backup: smaller chunks trade compute for quality}
\centering
\begin{tabular}{lrrrr}
\toprule
Chunk size $k$ & 2 & 4 & 8 & 16 \\
\midrule
Avg Acc $\uparrow$ & \textbf{40.7} & 40.3 & 40.2 & 39.8 \\
Avg PPL $\downarrow$ & \textbf{59.80} & 68.13 & 65.81 & 69.70 \\
\bottomrule
\end{tabular}
\medskip\raggedright
\begin{itemize}
\item $k=2$ gives the best reported quality in this ablation.
\item Larger $k$ shortens the concept sequence and lowers its theoretical cost.
\item PPL is not monotonic; $k=8$ beats $k=4$ on the average here.
\item No measured runtime curve is supplied for these four settings.
\end{itemize}
\papersource{Paper, Table 18 and App. E.5, PDF pp. 22, 25. Pythia-160M, 30B tokens.}
\end{frame}

\begin{frame}{Backup: arithmetic checks preserve the evidence boundary}
\begin{itemize}
\item \textbf{Table 1:} Pythia-160M ConceptLM's four PPLs average 34.3175; reported Avg is 34.23. The ranking is unchanged.
\medskip
\item \textbf{Appendix E.4:} $64^{12}=2^{72}\approx4.7224\times10^{21}$, not $72\times10^{22}$. It still exceeds $50000^4=6.25\times10^{18}$.
\medskip
\item \textbf{Appendix C:} with $T/d=2$, the simplified FLOPs reduction is
\[
1-\frac{390+64.25\times2}{408+68\times2}=4.6875\%.
\]
\end{itemize}
\papersource{Paper, Table 1, PDF p. 7; Apps. C and E.4, pp. 20, 22. Calculations from printed values, not rerun experiments.}
\end{frame}

\begin{frame}{Backup: implementation checks clarify key ambiguities}
\begin{itemize}
\item \textbf{VQ boundary:} Pythia detaches pooled states before quantization.
\item \textbf{NCP target:} MSE uses the next pooled state with target-side detachment; predictor inputs retain gradients.
\item \textbf{Causal shift:} zero-prefix, repeat by $k$, then drop one leading position when \texttt{shift\_feature} is enabled.
\item \textbf{Mixture weights:} the inspected path multiplies raw head logits by codebook vectors; no softmax before this product.
\end{itemize}
\medskip
\small The README marks training code as pending and notes that Llama generation differs. These are static observations of public Pythia code, not claims about every original training run.
\papersource{Official repository, commit a0ab281286f5c0337c35de3181cc992c562eacaa; ConceptLM\_Pythiamodel.py, inspected 2026-09-28.}
\end{frame}

\end{document}