% Independent reading deck, based on arXiv:2609.10715v1.
% Compile with: python3 build_slides.py slides.en.tex
% Main talk: frames 1--23, about 25--30 minutes. Frames 24--29 are backup.
\documentclass[aspectratio=169,11pt]{beamer}
\usepackage[T1]{fontenc}
\usepackage[utf8]{inputenc}
\usepackage{lmodern}
\usepackage{amsmath,amssymb,graphicx,booktabs}
\usepackage{tikz}
\usetikzlibrary{arrows.meta,positioning}
\definecolor{AcademicBlue}{RGB}{26,57,86}
\definecolor{AcademicTeal}{RGB}{0,105,112}
\setbeamercolor{structure}{fg=AcademicBlue}
\setbeamercolor{frametitle}{fg=AcademicBlue,bg=white}
\setbeamercolor{title}{fg=AcademicBlue}
\setbeamercolor{alerted text}{fg=AcademicTeal}
\setbeamercolor{block title}{fg=white,bg=AcademicBlue}
\setbeamercolor{block body}{fg=black,bg=AcademicBlue!5}
\setbeamertemplate{navigation symbols}{}
\setbeamertemplate{itemize items}[circle]
\setbeamertemplate{footline}{%
  \hfill\usebeamerfont{page number in head/foot}%
  \insertframenumber\,/\,\inserttotalframenumber\hspace{4mm}\vspace{3mm}}
\setbeamersize{text margin left=8mm,text margin right=8mm}
\setbeamerfont{frametitle}{size=\Large,series=\bfseries}
\setbeamerfont{title}{size=\LARGE,series=\bfseries}
\setlength{\parskip}{3pt}
\newcommand{\papersource}[1]{\par\smallskip{\scriptsize\color{gray}#1\par}}
\newcommand{\sg}{\operatorname{sg}}
\newcommand{\NCP}{\mathrm{NCP}}
\newcommand{\NTP}{\mathrm{NTP}}
\newcommand{\VQ}{\mathrm{VQ}}
\hypersetup{pdftitle={NCP-ArchPreview: Independent Paper Reading},
  pdfauthor={Independent reading presentation},
  pdfsubject={arXiv:2609.10715v1}}
\title[NCP-ArchPreview]{NCP-ArchPreview}
\subtitle{Next Concept Prediction at Trillion-Token Scale}
\author{Paper authors: The Intern-NCP Team\\
  \small Shanghai AI Lab; LUMIA Lab, Shanghai Jiao Tong University}
\date{arXiv:2609.10715v1 \quad|\quad 28 September 2026}
\begin{document}

\begin{frame}
  \titlepage
  \vspace{-5mm}
  \begin{center}
    \alert{Predict multi-token latent representations while retaining token generation.}\\[2mm]
    {\small Independent paper reading \quad$\cdot$\quad 25--30 minutes + backup}
  \end{center}
\end{frame}

\begin{frame}{Can internal representations become prediction targets?}
  \begin{block}{Research question}
    Does explicit prediction of future token-span representations improve
    large-scale autoregressive pretraining?
  \end{block}
  \begin{itemize}
    \item NTP supervises individual tokens; longer-span structure is learned indirectly.
    \item NCP adds a predictive objective on a shorter, learned concept sequence.
    \item This report scales the \textbf{ConceptLM} approach to \textbf{8.94B}
      parameters and \textbf{5.73T} pretraining tokens, with NCP active from the start.
  \end{itemize}
  \papersource{Paper, Secs. 1 and 6.2, PDF pp. 1--3, 22. NCP and product-quantized concepts originate in the cited ConceptLM work.}
\end{frame}

\begin{frame}{A compressed middle path feeds a full token decoder}
  \begin{center}
  \begin{tikzpicture}[
    mod/.style={draw=AcademicBlue,thick,rounded corners=2mm,
      fill=AcademicBlue!5,align=center,text width=3.0cm,minimum height=1.5cm},
    arr/.style={-{Latex[length=2mm]},thick,AcademicBlue}]
    \node[mod] (e) at (0,0) {\textbf{Token Encoder}\\16 layers\\$T$ states};
    \node[mod,fill=AcademicTeal!9] (c) at (4.3,0) {\textbf{Concept Module}\\8 layers\\$T/4$ states};
    \node[mod] (d) at (8.6,0) {\textbf{Token Decoder}\\16 layers\\$T$ states};
    \draw[arr] (e) -- (c);
    \draw[arr] (c) -- (d);
    \node[align=center,font=\small] at (2.15,1.25) {mean pool\\$k=4$};
    \node[align=center,font=\small] at (6.45,1.25) {predict +\\repeat/shift};
    \draw[arr] (e.south) -- ++(0,-0.9) -| (d.south);
    \node[fill=white,font=\small] at (4.3,-1.65) {token states + cross-module residuals};
  \end{tikzpicture}
  \end{center}
  \begin{itemize}
    \item Only the Concept Module operates on the compressed sequence.
    \item IRC routes information across depths; CRC connects the three modules.
    \item The model still generates one token at a time.
  \end{itemize}
  \papersource{Teaching schematic based on Fig. 2, PDF p. 5; configuration: Table 10, p. 34.}
\end{frame}

\begin{frame}{Product quantization creates a compact concept vocabulary}
  \[
  \mathbf c_m=\frac1k\sum_{i=1}^{k}\mathbf h_{(m-1)k+i}
  \quad\longrightarrow\quad
  (\mathbf c_m^1,\ldots,\mathbf c_m^S)
  \]
  \[
  n_m^s=\arg\min_n\|\mathbf c_m^s-\mathbf e_n^s\|_2^2,
  \qquad \mathbf d_m^s=\mathbf e_{n_m^s}^s
  \]
  \begin{itemize}
    \item Hidden width $d=4096$; $k=4$ contextual token states per concept.
    \item $S=32$ segments; each has $N=128$ codewords of dimension 128.
    \item $128^{32}$ possible combinations require only $Nd=524{,}288$
      codebook parameters.
    \item Fixed token spans need not align with semantic boundaries.
  \end{itemize}
  \papersource{Paper, Eqs. 1--6, PDF pp. 4--5; Table 10, p. 34. Parameter count is calculated from the reported configuration.}
\end{frame}

\begin{frame}{Prediction uses soft codebook mixtures}
  \[
  \mathbf u_m=\operatorname{ConceptModule}(\mathbf c_{<m}),
  \qquad
  \boldsymbol\pi_m^s=\operatorname{softmax}
  \bigl(\operatorname{Head}_s(\mathbf u_m)\bigr)
  \]
  \[
  \hat{\mathbf c}_m^s=\sum_{n=1}^{N}\pi_{m,n}^s\mathbf e_n^s,
  \qquad
  \hat{\mathbf c}_m=\operatorname{concat}_s(\hat{\mathbf c}_m^s)
  \]
  \begin{block}{What discrete concept prediction means here}
    A finite codebook constrains the prediction, but the output is a
    differentiable weighted vector. Training uses continuous-target MSE.
  \end{block}
  \begin{itemize}
    \item No argmax or sampled codeword on the feedback path.
    \item Each segment lies in its codebook's convex hull (from the softmax formula).
  \end{itemize}
  \papersource{Paper, Eqs. 7--10 and 22, PDF pp. 6, 9. Convex-hull interpretation follows directly from the equations.}
\end{frame}

\begin{frame}{A causal shift prevents access to the target token span}
  \[
  \mathbf b_t=
  \begin{cases}
    \mathbf0,&1\le t<k,\\
    \hat{\mathbf c}_{\lfloor(t-k)/k\rfloor+2},&k\le t<T,
  \end{cases}
  \qquad \tilde{\mathbf h}_t=\mathbf h_t+\mathbf b_t
  \]
  \begin{center}
  \renewcommand{\arraystretch}{1.15}
  \begin{tabular}{cccc}
    \toprule
    Input position $t$ & Available history & Signal & Predicts\\
    \midrule
    1--3 & incomplete chunk & 0 & $x_2$--$x_4$\\
    4--7 & $\mathbf c_1$ & $\hat{\mathbf c}_2$ & $x_5$--$x_8$\\
    8--11 & $\mathbf c_{1:2}$ & $\hat{\mathbf c}_3$ & $x_9$--$x_{12}$\\
    \bottomrule
  \end{tabular}
  \end{center}
  \alert{At $t=4$, predict the next chunk from tokens 1--4; do not inject its true encoding.}
  \papersource{Paper, Eqs. 11--13, PDF p. 6. Table is a teaching expansion for $k=4$. Concept-to-decoder CRC requires the same causal alignment.}
\end{frame}

\begin{frame}{Three losses jointly train the two prediction levels}
  \[
  \begin{aligned}
  \mathcal L&=\mathcal L_\NTP+\alpha\mathcal L_\NCP+\beta\mathcal L_\VQ,\\
  \mathcal L_\NTP&=-\frac1{T-1}\sum_{t=1}^{T-1}
        \log p(x_{t+1}\mid\tilde{\mathbf h}_{\le t}),\\
  \mathcal L_\NCP&=\frac1{M-1}\sum_{m=2}^{M}
        \|\hat{\mathbf c}_m-\sg(\mathbf c_m)\|_2^2,\\
  \mathcal L_\VQ&=\frac1{MS}\sum_{m,s}
        \|\sg(\mathbf c_m^s)-\mathbf d_m^s\|_2^2.
  \end{aligned}
  \]
  \begin{itemize}
    \item VQ fits codewords; sg blocks target-side gradients.
    \item NCP updates the encoder via past concepts; NTP trains both paths.
  \end{itemize}
  \papersource{Paper, Eqs. 21--24, PDF pp. 8--9. Main-text convention shown; Algorithm 1 differs by a factor of $1/S$ in NCP normalization (backup).}
\end{frame}

\begin{frame}{Residual routing selects information across depth and scale}
  \begin{columns}[T,onlytextwidth]
    \begin{column}{0.48\textwidth}
      \begin{block}{IRC: within a module}
        \[
        \mathbf H_{\ell+1}
        =\sum_j w_{\ell,j}\mathbf X_{\ell,j}
        \]
      \end{block}
      \begin{itemize}
        \item Candidates are states from earlier depths.
        \item Weights depend on the current block output.
        \item Unnormalized, potentially signed.
        \item Initially selects the latest state.
      \end{itemize}
    \end{column}
    \begin{column}{0.48\textwidth}
      \begin{block}{CRC: across modules}
        \[
        \mathbf T_{\rm out}
        =\mathbf T+\mathbf D\odot
           \sum_j\alpha_j\operatorname{LN}(\mathbf S_j)
        \]
      \end{block}
      \begin{itemize}
        \item Source depths aligned by pooling/repetition.
        \item Target-conditioned softmax weights.
        \item Learned per-dimension scale $\mathbf D$.
        \item Encoder$\to$Concept/Decoder; Concept$\to$Decoder.
      \end{itemize}
    \end{column}
  \end{columns}
  \papersource{Paper, Eqs. 14--20, PDF pp. 7--8. Equations abbreviated for presentation; see Chinese notes for full notation.}
\end{frame}

\begin{frame}{The evidence comes from several distinct experimental regimes}
  \begin{center}
  \renewcommand{\arraystretch}{1.2}
  \begin{tabular}{lll}
    \toprule
    Question & Model / budget & Evidence\\
    \midrule
    Large-scale utility & 8.94B; 5.73T + 100B tokens & Table 1\\
    Capacity vs.\ computation & First 200B tokens & Fig. 3; Table 2\\
    Residual design & 1B; 150B tokens & Table 3\\
    Compute-optimal scaling & $10^{19}$--$10^{20}$ FLOPs & Figs. 4, 10\\
    Domain adaptation & Stage-1; Full / LoRA / VQ & Tables 4--7\\
    Complementarity with MTP & 3B; 150B tokens & Table 8\\
    Speculative drafting & 1.1B drafter; matched training & Table 9\\
    \bottomrule
  \end{tabular}
  \end{center}
  \alert{Overall AVG: 26 task scores, equally weighted. BPB is evaluated separately.}
  \papersource{Paper, Secs. 4--5, PDF pp. 10--21; aggregation: Appendix E, pp. 41--42.}
\end{frame}

\begin{frame}{Stage-1 reaches the baseline loss with 51.3\% of tokens}
  \begin{center}
  \includegraphics[width=0.83\linewidth,height=0.63\textheight,keepaspectratio]{figures/fig01a-stage1-loss.pdf}
  \end{center}
  \centering
  \textbf{Final loss gap: 0.091}\qquad
  \textbf{Reported token-convergence ratio: 1.95$\times$}
  \papersource{Paper, Fig. 1(a), PDF p. 2; Sec. 4.2.2, p. 10. Same training data; main models are not parameter-matched. This is not measured wall-clock speedup.}
\end{frame}

\begin{frame}{Downstream gains shrink after mid-training}
  \begin{center}
  \renewcommand{\arraystretch}{1.13}
  \begin{tabular}{lrr}
    \toprule
    Score & Stage-1 delta (pp) & Stage-2 delta (pp)\\
    \midrule
    Overall AVG & \textbf{+2.45} & \textbf{+0.59}\\
    MMLU aggregate & +2.58 & +1.82\\
    GSM8K & +5.99 & +3.34\\
    Math domain AVG & +3.75 & +1.76\\
    Code domain AVG & +2.64 & $-0.65$\\
    MC-Non-STEM AVG & +4.63 & +0.91\\
    HumanEval & +4.28 & $-3.69$\\
    \bottomrule
  \end{tabular}
  \end{center}
  \begin{itemize}
    \item Stage-1 overall: 46.59 $\to$ 49.04; Stage-2: 56.98 $\to$ 57.57.
    \item Lower mixture loss does not guarantee better performance in every domain.
  \end{itemize}
  \papersource{Paper, Table 1, PDF p. 11. Deltas are NCP minus released OLMo-3-7B; pp = percentage points.}
\end{frame}

\begin{frame}{Parameter and compute controls address different explanations}
  \begin{center}
  \renewcommand{\arraystretch}{1.3}
  \begin{tabular}{lcc}
    \toprule
    Configuration & Parameters & Computation\\
    \midrule
    Vanilla & $32P_{\rm blk}$ & $32F_{\rm blk}$\\
    Compute-aligned Vanilla & $34P_{\rm blk}$ & $34F_{\rm blk}$\\
    Size-aligned Vanilla & $40P_{\rm blk}$ & $40F_{\rm blk}$\\
    \textbf{NCP-ArchPreview} & $40P_{\rm blk}$ & $34F_{\rm blk}$\\
    \bottomrule
  \end{tabular}
  \end{center}
  \[
    F_{\rm NCP}\approx(16+8/4+16)F_{\rm blk}=34F_{\rm blk}
  \]
  \begin{itemize}
    \item Eight concept layers add capacity at about one-quarter sequence length.
    \item $34/40=85\%$: approximate compute relative to the size-aligned model.
  \end{itemize}
  \papersource{Paper, Table 2 and Sec. 4.3, PDF p. 13. Small components are omitted in this block accounting; comparisons cover the first 200B tokens.}
\end{frame}

\begin{frame}{Concept layers, routing, and NCP improve loss progressively}
  \begin{center}
  \includegraphics[width=0.98\linewidth,height=0.63\textheight,keepaspectratio]{figures/fig03-ablation.pdf}
  \end{center}
  \centering
  Full NCP beats the compute-aligned baseline and approaches the size-aligned one.
  \papersource{Paper, Fig. 3, PDF p. 12. Same recipe, first 200B tokens. Sequential additions do not establish independent, additive effects at every scale.}
\end{frame}

\begin{frame}{IRC + CRC gives the best loss in the 1B routing study}
  \begin{center}
  \renewcommand{\arraystretch}{1.12}
  \begin{tabular}{lrr}
    \toprule
    Variant & LM loss $\downarrow$ & Extra FLOPs\\
    \midrule
    No hierarchical residuals & 2.2588 & 0.000\%\\
    \textbf{IRC + CRC} & \textbf{2.2265} & +0.051\%\\
    IRC only & 2.2315 & +0.024\%\\
    IRC + input-level cross-module links & 2.2292 & +0.024\%\\
    IRC + all-stage softmax & 2.2295 & +0.024\%\\
    Block AttnRes ($S=4$) + cross-module links & 2.2409 & +0.026\%\\
    \bottomrule
  \end{tabular}
  \end{center}
  \begin{itemize}
    \item 150B training tokens; loss averaged over the final 200 logged steps.
    \item Analytical FLOPs exclude memory traffic and several routing overheads.
  \end{itemize}
  \papersource{Paper, Table 3 and Sec. 4.4, PDF pp. 14--15. Small FLOPs overhead is not a measured runtime or memory overhead.}
\end{frame}

\begin{frame}{Scaling fits favor NCP within the tested compute budgets}
  \begin{columns}[c,onlytextwidth]
    \begin{column}{0.61\textwidth}
      \includegraphics[width=\linewidth,height=0.70\textheight,keepaspectratio]{figures/fig04-scaling.pdf}
    \end{column}
    \begin{column}{0.36\textwidth}
      \begin{itemize}
        \item Search model/data allocation and hyperparameters per budget.
        \item Compare best validation loss.
        \item Author-reported efficiency: \textbf{1.74$\times$}.
      \end{itemize}
      \[
      C=F_{\rm tok}D
      \]
      {\small Analytical compute, not elapsed time.}
    \end{column}
  \end{columns}
  \papersource{Paper, Fig. 4, PDF p. 15; Appendix D, pp. 38--40. Extrapolation beyond the measured scaling ladder is not established.}
\end{frame}

\begin{frame}{Capability proxies reveal weaknesses hidden by mixture loss}
  \begin{center}
  \renewcommand{\arraystretch}{1.25}
  \begin{tabular}{lrrr}
    \toprule
    Stage-2 recipe & V1 & V2 & V3\\
    \midrule
    Function-generation NLL $\downarrow$ & \textbf{0.7330} & 0.7501 & 0.7598\\
    HumanEval $\uparrow$ & \textbf{45.60} & 42.19 & 39.96\\
    Competition-math NLL $\downarrow$ & \textbf{0.5742} & 0.5949 & 0.6075\\
    MATH-500 $\uparrow$ & \textbf{43.74} & 41.66 & 37.21\\
    \bottomrule
  \end{tabular}
  \end{center}
  \begin{itemize}
    \item Fixed teacher trajectories; source-balanced capability scores.
    \item Paper reports lower training loss from V1 to V3, but weaker capabilities.
    \item Only three matched-budget recipes: evidence for screening, not a universal predictor.
  \end{itemize}
  \papersource{Paper, Sec. 4.2.4, PDF p. 12; Appendix C, Table 11, p. 36. These auxiliary evaluation values must not be substituted into Table 1.}
\end{frame}

\begin{frame}{The 17M adaptation interface offers a domain-dependent tradeoff}
  \begin{center}
  \renewcommand{\arraystretch}{1.3}
  \begin{tabular}{lrrrr}
    \toprule
    Domain & Full gain & LoRA gain & VQ gain & VQ general $\Delta$\\
    \midrule
    Code AVG & $-1.61$ & +1.34 & \textbf{+2.65} & $-0.42$\\
    Math AVG & \textbf{+6.16} & +3.15 & +4.27 & +0.39\\
    TriviaQA EM & \textbf{+18.00} & +17.48 & +9.19 & +0.03\\
    \bottomrule
  \end{tabular}
  \end{center}
  \begin{itemize}
    \item VQ updates existing \textbf{codebooks and prediction heads}, about 17M parameters.
    \item LoRA also trains 17M parameters; Full trains the entire model.
    \item VQ is strongest on this code average, but freezing the backbone does not eliminate forgetting.
  \end{itemize}
  \papersource{Paper, Tables 4--7, PDF pp. 17--19. All gains are percentage points relative to each NCP Stage-1 adaptation baseline. General AVG uses five tasks.}
\end{frame}

\begin{frame}{VQ adaptation is faster and uses less memory in this setup}
  \begin{center}
  \includegraphics[width=0.98\linewidth,height=0.53\textheight,keepaspectratio]{figures/fig07-adaptation-efficiency.pdf}
  \end{center}
  \begin{itemize}
    \item At MBS 1 on 8 GPUs: VQ reaches 15,632 tokens/s/GPU.
    \item 1.50$\times$ LoRA throughput; 2.02$\times$ Full throughput.
    \item GPU type/capacity is unspecified; percentages are not absolute GB.
  \end{itemize}
  \papersource{Paper, Fig. 7, PDF p. 19. The right panel uses a different setup: MBS 2 on one GPU. These are adaptation measurements.}
\end{frame}

\begin{frame}{NCP and MTP are complementary in a 3B controlled study}
  \begin{center}
  \renewcommand{\arraystretch}{1.25}
  \begin{tabular}{lrr}
    \toprule
    Configuration & Mean LM loss $\downarrow$ & FLOPs ($10^{21}$)\\
    \midrule
    OLMo-3-3B + Residual & 2.3805 & 3.245\\
    OLMo-3-3B + Residual + MTP & 2.3739 & 3.750\\
    NCP-ArchPreview & 2.3707 & 3.227\\
    NCP-ArchPreview + MTP & \textbf{2.3664} & 3.731\\
    \bottomrule
  \end{tabular}
  \end{center}
  \begin{itemize}
    \item Residual-aligned baselines; first 150B tokens; one additional MTP layer.
    \item MTP predicts two positions ahead, with loss weight 0.3.
    \item Loss averages the \textbf{entire training interval}, excluding auxiliary losses.
  \end{itemize}
  \papersource{Paper, Sec. 5.2 and Table 8, PDF pp. 19--20. This does not establish a downstream gain for the full 8.94B model with MTP.}
\end{frame}

\begin{frame}{Concept conditioning increases speculative acceptance length}
  \begin{center}
  \renewcommand{\arraystretch}{1.13}
  \begin{tabular}{lrrr}
    \toprule
    Task & Baseline MAL & +Concept MAL & Reported gain\\
    \midrule
    GSM8K & 6.351 & 6.537 & +2.93\%\\
    MATH & 6.105 & 6.240 & +2.22\%\\
    HumanEval & 5.432 & 5.845 & +7.59\%\\
    MBPP & 5.844 & 6.099 & +4.37\%\\
    \textbf{Macro average} & \textbf{5.933} & \textbf{6.180} & \textbf{+4.17\%}\\
    \bottomrule
  \end{tabular}
  \end{center}
  \begin{itemize}
    \item 1.1B drafter, +0.04M parameters; 16-token proposal horizon.
    \item MAL counts committed tokens, including target correction/bonus when present.
    \item End-to-end latency improvement is not reported.
  \end{itemize}
  \papersource{Paper, Table 9 and Eqs. 26--27, PDF p. 21. Relative gains are as reported; rounded values yield small last-digit differences.}
\end{frame}

\begin{frame}{Per-head Q/K normalization stabilizes the Muon ablation}
  \begin{center}
  \includegraphics[width=0.98\linewidth,height=0.53\textheight,keepaspectratio]{figures/fig06-stability-ablation.pdf}
  \end{center}
  \begin{itemize}
    \item Layer-wise normalization permits head-level norm outliers.
    \item Per-head normalization suppresses logit and Q/K-norm growth here.
    \item \textbf{Main results still use layer-wise Q/K normalization.}
  \end{itemize}
  \papersource{Paper, Sec. 4.6 and Fig. 6, PDF pp. 16--17. Optimizer/normalization interaction is the authors' interpretation of the controlled comparison.}
\end{frame}

\begin{frame}{The strongest result is scalable joint token--concept learning}
  \begin{enumerate}
    \item \textbf{Feasibility at scale.}\\
      NCP works from the start of 8.94B / 5.73T pretraining while retaining
      token-level autoregressive generation.
    \item \textbf{Useful optimization and interfaces.}\\
      Controlled losses, lightweight adaptation, and speculative acceptance
      support several uses of the learned concept pathway.
    \item \textbf{The evidence has clear limits.}\\
      Smaller Stage-2 gains; no long-context training; no full-budget matched
      downstream study; no demonstrated semantic interpretability.
  \end{enumerate}
  \papersource{Synthesis of Secs. 4--8, PDF pp. 10--23. Distinguish tokens-to-loss, analytical FLOPs, adaptation throughput, and acceptance length.}
\end{frame}

\begin{frame}{Paper, materials, and scope of this reading}
  \textbf{The Intern-NCP Team.}\\
  \emph{NCP-ArchPreview Technical Report: Moving towards Latent Space
  Language Models through Next Concept Prediction.}

  \href{https://arxiv.org/abs/2609.10715v1}{arXiv:2609.10715v1}
  \quad Submitted 9 September 2026; read 28 September 2026.

  \begin{itemize}
    \item Read author TeX, all substantive sections and appendices; checked PDF figures/tables.
    \item Figures are copied from the versioned author sources.
    \item Architecture schematic and interpretations are for this presentation.
    \item No training reproduction or implementation audit was performed.
  \end{itemize}
  \papersource{Author-listed resources:
    \href{https://huggingface.co/collections/ArchSpace-Collection/ncp-archpreview}{model collection};
    \href{https://github.com/InternLM/lmdeploy}{LMDeploy};
    \href{https://github.com/LUMIA-Group/ncp_olmo_eval}{evaluation code}.
    Source links recorded from the paper; repositories were not audited.}
\end{frame}

\appendix

\begin{frame}{Backup: the NCP loss normalization needs implementation review}
  Main text, Eq. 22:
  \[
    \mathcal L_{\NCP}^{\rm text}
    =\frac1{M-1}\sum_{m=2}^{M}
    \|\hat{\mathbf c}_m-\sg(\mathbf c_m)\|_2^2
  \]
  Algorithm 1, line 21:
  \[
    \mathcal L_{\NCP}^{\rm alg}
    =\frac1{(M-1)S}\sum_{m=2}^{M}\sum_{s=1}^{S}
    \|\hat{\mathbf c}_m^s-\sg(\mathbf c_m^s)\|_2^2
    =\frac1S\mathcal L_{\NCP}^{\rm text}
  \]
  \begin{itemize}
    \item With standard squared norms and concatenation, the difference is $S=32$.
    \item The factor can be absorbed into $\alpha$, but $\alpha,\beta$ are not specified numerically.
    \item Inference feedback, incomplete chunks, and draft-state notation also need code-level verification.
  \end{itemize}
  \papersource{Paper, Eq. 22, PDF p. 9; Algorithm 1, p. 35; Sec. 2.3, p. 6; Eq. 26, p. 21.}
\end{frame}

\begin{frame}{Backup: architecture and optimizer configuration}
  \begin{center}
  \renewcommand{\arraystretch}{1.12}
  \begin{tabular}{ll}
    \toprule
    Token / concept / token layers & 16 / 8 / 16\\
    Hidden / FFN width & 4096 / 11008\\
    Attention heads / KV groups & 32 / 32\\
    Vocabulary / context & 100278 / 8192\\
    Token local window & 4096; full attention every fourth layer\\
    RoPE base / activation & 500000 / SwiGLU\\
    Normalization & RMSNorm, $\epsilon=10^{-6}$; layer-wise Q/K\\
    Precision / dropout & BF16 / 0\\
    PQ configuration & 32 segments; 128 words; 128 dimensions\\
    Optimizers & Moonlight Muon + AdamW for non-Muon weights\\
    Default learning rate & $6\times10^{-5}$; cosine schedule\\
    \bottomrule
  \end{tabular}
  \end{center}
  \papersource{Paper, Table 10, PDF p. 34; Sec. 3.6, p. 9. Tokenizer vocabulary and latent vocabulary are different objects.}
\end{frame}

\begin{frame}{Backup: multiple samples still estimate Pass@1}
  \begin{center}
  \renewcommand{\arraystretch}{1.05}
  \begin{tabular}{lrrl}
    \toprule
    Benchmark & Shots & Samples & Metric\\
    \midrule
    GSM8K & 8 & 1 & Pass@1\\
    GSM-Symbolic & 8 & 8 & Pass@1\\
    Minerva & 4 & 1 & Exact match\\
    MATH-500 & 4 & 32 & Pass@1\\
    BigCodeBench & 3 & 5 & Execution pass@1\\
    HumanEval & 3 & 32 & Execution pass@1\\
    DS-1000 & 3 & 5 & Execution pass@1\\
    MBPP & 3 & 32 & Execution pass@1\\
    MultiPL-E HumanEval / MBPP & 0 & 32 & Execution pass@1\\
    \bottomrule
  \end{tabular}
  \end{center}
  Global seed: 42. Samples are completions per problem, not inference batch size.
  \papersource{Paper, Table 13 and Appendix E, PDF pp. 39--42. Overall AVG excludes BPB, intermediate averages, and the aggregate MMLU row.}
\end{frame}

\begin{frame}{Backup: proxy scores retain a vector of capabilities}
  \[
    \ell_m(i)=-\frac1{|T_i|}
       \sum_{j\in T_i}\log p_m(y_{ij}\mid x_i,y_{i,<j})
  \]
  \begin{itemize}
    \item 177,202 fixed teacher trajectories, 63 sources, about 100M suffix tokens.
    \item Average within each source, then across sources in each capability.
    \item Four free-form capability NLLs agree with downstream recipe ordering.
    \item HellaSwag uses correct-vs-best-distractor choice margin instead.
  \end{itemize}
  \begin{center}
  \begin{tabular}{lrrr}
    \toprule
    HellaSwag & V1 & V2 & V3\\
    \midrule
    Margin $\uparrow$ & 0.09678 & 0.09932 & 0.10083\\
    Accuracy $\uparrow$ & 66.40 & 67.30 & 67.25\\
    \bottomrule
  \end{tabular}
  \end{center}
  \papersource{Paper, Appendix C, Eq. 28 and Table 11, PDF pp. 34--38. V2/V3 are treated as a leading tier; no individual data-source causal attribution.}
\end{frame}

\begin{frame}{Backup: adaptation averages conceal important task differences}
  \begin{center}
  \renewcommand{\arraystretch}{1.18}
  \begin{tabular}{lrrrr}
    \toprule
    NCP adaptation metric & Stage-1 & Full & LoRA & VQ\\
    \midrule
    HumanEval (code adaptation) & 31.27 & 43.27 & 36.49 & 33.94\\
    MBPP+ (code adaptation) & 29.63 & 7.14 & 21.96 & 31.22\\
    Code AVG & 30.04 & 28.43 & 31.38 & 32.69\\
    Math AVG & 30.56 & 36.72 & 33.71 & 34.83\\
    TriviaQA EM & 40.28 & 58.28 & 57.76 & 49.47\\
    \bottomrule
  \end{tabular}
  \end{center}
  \begin{itemize}
    \item Separate runs: Magicoder / Orca-Math / TriviaQA-RC.
    \item General AVG includes MMLU, ARC-E, HellaSwag, NQ Open, and SciQ.
    \item Adaptation checkpoints/evaluation values differ from the main results table.
  \end{itemize}
  \papersource{Paper, Secs. 4.1 and 5.1, Tables 5--7, PDF pp. 10, 18--19. Appendix E's prose omits SciQ from its list; tables include it.}
\end{frame}

\begin{frame}{Backup: the scaling ladder searches model--data allocation}
  \[
    C=F_{\rm tok}D
    \quad\text{rather than treating all parameters as active per token}
  \]
  \begin{center}
  \renewcommand{\arraystretch}{1.23}
  \begin{tabular}{lrr}
    \toprule
    Budget (FLOPs) & Best listed NCP loss & Encoder / Concept / Decoder\\
    \midrule
    $10^{19}$ & 2.759 & 4 / 4 / 4\\
    $3\times10^{19}$ & 2.599 & 8 / 4 / 8\\
    $5\times10^{19}$ & 2.537 & 8 / 8 / 8\\
    $8\times10^{19}$ & 2.476 & 8 / 4 / 8\\
    $10^{20}$ & 2.449 & 8 / 8 / 8\\
    \bottomrule
  \end{tabular}
  \end{center}
  Search learning rate and batch size on a representative model, then refine
  learning rates for other allocations at the same budget.
  \papersource{Paper, Appendix D, PDF p. 38; Table 12, p. 40. Best rows selected from the reported points. FLOPs accounting excludes embeddings.}
\end{frame}
\end{document}