\documentclass[aspectratio=169,11pt]{beamer}
\usepackage[T1]{fontenc}
\usepackage[utf8]{inputenc}
\usepackage{lmodern}
\usepackage{amsmath,amssymb,graphicx,booktabs,array}

\definecolor{AcademicBlue}{RGB}{26,57,86}
\definecolor{AcademicTeal}{RGB}{0,105,112}
\definecolor{AcademicGold}{RGB}{145,94,0}
\setbeamercolor{structure}{fg=AcademicBlue}
\setbeamercolor{frametitle}{fg=AcademicBlue,bg=white}
\setbeamercolor{title}{fg=AcademicBlue}
\setbeamercolor{alerted text}{fg=AcademicGold}
\setbeamercolor{block title}{fg=white,bg=AcademicBlue}
\setbeamercolor{block body}{fg=black,bg=AcademicBlue!5}
\setbeamertemplate{navigation symbols}{}
\setbeamertemplate{itemize items}[circle]
\setbeamertemplate{footline}{%
  \hfill\usebeamerfont{page number in head/foot}%
  \insertframenumber\,/\,\inserttotalframenumber\hspace{4mm}\vspace{3mm}}
\setbeamersize{text margin left=8mm,text margin right=8mm}
\setbeamerfont{frametitle}{size=\Large,series=\bfseries}
\setbeamerfont{title}{size=\LARGE,series=\bfseries}
\newcommand{\papersource}[1]{\par\smallskip{\scriptsize\color{gray}#1\par}}
\newcommand{\ncm}{\textbf{NCM}}
\newcommand{\genppl}{Gen.\ PPL}

\title[Next-Concept Modeling]{Next-Concept Modeling}
\subtitle{Fast, Parallel Coarse-to-Fine Text Generation}
\author[H. Kumbong et al.]{Paper by H. Kumbong, I. S. Khare, I. Mlauzi, et al.\\Research group paper discussion}
\date{\href{https://arxiv.org/abs/2610.14162v1}{arXiv:2610.14162v1} \quad 26 October 2026}

\begin{document}

\begin{frame}
  \titlepage
  \begin{center}
    \small Takeaway: generate a short-to-long hierarchy of discrete codes;
    parallelize within each level.
  \end{center}
\end{frame}

\begin{frame}{Can text be generated in a few \emph{cheap} parallel passes?}
  \begin{itemize}
    \item Autoregression requires roughly \(L\) sequential token decisions.
    \item Masked diffusion updates many positions at once, but repeats full-sequence work.
    \item Few-step distillation can reduce passes while retaining the full-sequence cost per pass.
  \end{itemize}
  \begin{block}{The NCM answer}
    Learn a hierarchy of discrete codes and generate one level per pass:
    \(K\ll L\), with \(\ell_k<L\) at the early levels.
  \end{block}
  \papersource{Paper, Secs.\ 1--2, PDF pp.\ 2--4.}
\end{frame}

\begin{frame}{A hierarchy changes both the schedule and the representation}
  \centering
  \includegraphics[width=0.96\linewidth,height=0.72\textheight,keepaspectratio]{figures/paradigms.pdf}
  \papersource{Paper, Fig.\ 2, PDF p.\ 2. NCM makes \(K\) serial level decisions; the \(\ell_k\) codes inside a level are sampled in parallel.}
\end{frame}

\begin{frame}{The two stages are learned separately}
  \centering
  \includegraphics[width=0.97\linewidth,height=0.70\textheight,keepaspectratio]{figures/method.pdf}
  \papersource{Paper, Fig.\ 4, PDF p.\ 5. Train the multiscale tokenizer first; freeze it before generator training.}
\end{frame}

\begin{frame}{Each level quantizes what the earlier levels missed}
  \[
    z=E_\phi(x)\in\mathbb{R}^{L\times d},\qquad
    u_k=\operatorname{Down}_k(z^{(k-1)})\in\mathbb{R}^{\ell_k\times d}
  \]
  \[
    r_k^i=\arg\min_j\|u_k^i-e_{k,j}\|_2^2,\qquad
    \hat z^{(k)}=\hat z^{(k-1)}+R_k(e_k(r_k))
  \]
  \begin{itemize}
    \item \(r_k\): \(\ell_k\) code indices; \(\ell_1<\cdots<\ell_K=L\).
    \item \(R_k\): maps short codebook vectors back to \(L\) latent positions.
    \item Later levels encode the residual \(z-\hat z^{(k)}\).
  \end{itemize}
  \papersource{Paper, Secs.\ 3.2--3.3, PDF pp.\ 6--7, Eqs.\ (3), (7)--(8).}
\end{frame}

\begin{frame}{Nested dropout forces useful partial reconstructions}
  \begin{itemize}
    \item Sample a cutoff \(k\); reconstruct text using only the first \(k\) levels.
    \item Token cross-entropy rewards information in coarse levels.
    \item VQ commitment and codebook losses keep the quantized latent close to the encoder latent.
  \end{itemize}
  \[
    \mathcal L_{\rm tok}=
    -\frac1L\sum_i\log p_\psi(x^i\mid\hat z^{(k)})
    +\beta\,\mathrm{MSE}(z,\mathrm{sg}[\hat z^{(k)}])
    +\mathrm{MSE}(\mathrm{sg}[z],\hat z^{(k)})
  \]
  \papersource{Paper, Sec.\ 3.3, PDF p.\ 7, Eq.\ (9). \(\mathrm{sg}\) denotes stop-gradient.}
\end{frame}

\begin{frame}{Within a level, codes are conditionally independent}
  \[
    p_\theta(r_{1:K})=
    \prod_{k=1}^K p_\theta(r_k\mid\hat z^{(k-1)})
    =
    \prod_{k=1}^K\prod_{i=1}^{\ell_k}
    p_\theta(r_k^i\mid\hat z^{(k-1)})
  \]
  \begin{block}{Operational consequence}
    One transformer call predicts all \(\ell_k\) code distributions.
    The next level waits for the current level's sampled codes.
  \end{block}
  \begin{itemize}
    \item Training: all level targets in one forward pass with teacher forcing.
    \item Generation: \(K\) sequential forward passes, then one decoder pass.
  \end{itemize}
  \papersource{Paper, Secs.\ 3.2 and 3.4, PDF pp.\ 6--7, Eqs.\ (4)--(5); Algorithms 1--2.}
\end{frame}

\begin{frame}{Corrupting context codes prepares the model for its own errors}
  \begin{columns}[T]
    \column{0.52\linewidth}
    \begin{itemize}
      \item Training replaces some conditioning codes with codes from the same level.
      \item Targets remain the clean codes.
      \item Moderate corruption greatly lowers TinyStories \genppl; too much loses signal.
    \end{itemize}
    \[
      \mathcal L_{\rm NCM}
      =-\sum_k w_k\sum_i\log p_\theta(r_k^i\mid\tilde z^{(k-1)})
    \]
    \column{0.45\linewidth}
    \centering
    \includegraphics[width=\linewidth,height=0.53\textheight,keepaspectratio]{figures/corruption_ablation.pdf}
  \end{columns}
  \papersource{Paper, Sec.\ 3.4, PDF p.\ 7, Eq.\ (10); Appendix F.3, Fig.\ 13, PDF p.\ 29.}
\end{frame}

\begin{frame}{Block-diagonal attention avoids repeated past-block work}
  \centering
  \includegraphics[width=0.95\linewidth,height=0.45\textheight,keepaspectratio]{figures/attention_masks.pdf}
  \vspace{2mm}
  \begin{tabular}{lccc}
    \toprule
    TinyStories attention & Block-causal & Two-level window & Block-diagonal\\
    \midrule
    \genppl\(\downarrow\) & 19.63 & 17.84 & \textbf{15.41}\\
    Time (s)\(\downarrow\) & 7.90 & 6.83 & \textbf{5.65}\\
    \bottomrule
  \end{tabular}
  \papersource{Paper, Appendix F.2, Fig.\ 11 and Table 9, PDF p.\ 28. Each input block already summarizes earlier levels.}
\end{frame}

\begin{frame}{A translation can be revised across levels}
  \centering
  \includegraphics[width=0.98\linewidth,height=0.66\textheight,keepaspectratio]{figures/trajectory.pdf}
  \papersource{Paper, Fig.\ 3, PDF p.\ 4; Appendix J.2, PDF p.\ 36. This is one example; intermediate BLEU is not monotonic (51.7 at step 8, 38.9 at step 9).}
\end{frame}

\begin{frame}{Evaluation spans generation, reasoning, and efficiency}
  \centering
  \begin{tabular}{lccc}
    \toprule
    Task & Target \(L\) & Levels \(K\) & Primary metric\\
    \midrule
    TinyStories & 128 & 12 & \genppl\ + entropy\\
    LM1B & 128 & 12 & \genppl\ + entropy\\
    OpenWebText & 1,024 & 32 & \genppl\ + entropy\\
    XSum summary & 48 & 8 & ROUGE-1/2/L\\
    WMT14 De--En & 64 & 11 & BLEU\\
    Sudoku & 89 & 12 & Fully solved boards\\
    \bottomrule
  \end{tabular}
  \vspace{3mm}
  \begin{itemize}
    \item Separate tokenizer and generator for each dataset.
    \item NFE counts generator evaluations; wall-clock time measures actual sampling.
  \end{itemize}
  \papersource{Paper, Sec.\ 4, PDF p.\ 8; Appendix B--C, PDF pp.\ 22--24.}
\end{frame}

\begin{frame}{The tokenizer nearly reconstructs all tokens}
  \centering
  \begin{tabular}{lrr}
    \toprule
    Dataset & Exact recovery (\%) & Codebook use (\%)\\
    \midrule
    OWT & 99.99 & 83.10\\
    LM1B & 99.86 & 89.31\\
    TinyStories & 99.98 & 90.12\\
    XSum & 99.79 & 86.20\\
    WMT14 & 98.79 & 61.89\\
    Sudoku (all difficulties) & 100.00 & 80--84\\
    \bottomrule
  \end{tabular}
  \vspace{5mm}
  \begin{block}{Why this matters}
    A digit error invalidates Sudoku. On natural text, the observed residual errors include entities, numerals, and word boundaries.
  \end{block}
  \papersource{Paper, Table 1, PDF p.\ 8; Appendix G, PDF p.\ 31. Full-hierarchy held-out reconstruction.}
\end{frame}

\begin{frame}{NCM reaches competitive quality with far fewer evaluations}
  \centering
  \includegraphics[width=0.96\linewidth,height=0.59\textheight,keepaspectratio]{figures/efficiency.pdf}
  \begin{itemize}
    \item LM1B: NCM \(93.05\) \genppl\ at 12 NFE; DUO \(97.30\) at 128.
    \item OWT: NCM \(83.12\) at 32 NFE; DUO \(77.69\) at 1,024. Read entropy alongside \genppl.
  \end{itemize}
  \papersource{Paper, Fig.\ 5, PDF p.\ 9; Table 8, PDF p.\ 26. \genppl\ is scored by GPT-2-large; lower is better.}
\end{frame}

\begin{frame}{The wall-clock advantage also reflects cheaper coarse steps}
  \centering
  \includegraphics[width=0.97\linewidth,height=0.63\textheight,keepaspectratio]{figures/latency.pdf}
  \begin{itemize}
    \item LM1B: \(11\times\) fewer NFE, about \(21\times\) faster than DUO.
    \item XSum: \(16\times\) fewer NFE, about \(31\times\) faster than AURORA-LM.
  \end{itemize}
  \papersource{Paper, Fig.\ 1, PDF p.\ 1; Sec.\ 4.6, PDF p.\ 12. H100 SXM 80GB, batch 16, 1,024 outputs.}
\end{frame}

\begin{frame}{Conditional generation: cheap samples enable MBR selection}
  \centering
  \begin{tabular}{llrrr}
    \toprule
    Task & Model / decoding & Score & NFE & Size\\
    \midrule
    XSum & AURORA-LM & ROUGE-L 26.6 & 128 & 130M\\
    XSum & \ncm & ROUGE-L 28.0 & 8 & 88M\\
    XSum & \ncm\ + MBR & ROUGE-L 29.6 & 128 & 88M\\
    \midrule
    WMT14 & E2D2 & BLEU 24.8 & 64 & 99M\\
    WMT14 & \ncm & BLEU 22.6 & 11 & 120M\\
    WMT14 & \ncm\ + MBR & BLEU 25.2 & 132 & 120M\\
    \bottomrule
  \end{tabular}
  \vspace{3mm}
  \begin{itemize}
    \item MBR selects the candidate most similar, on average, to the others.
    \item XSum uses 16 candidates; WMT14 uses 12 candidates.
  \end{itemize}
  \papersource{Paper, Table 2, PDF p.\ 10; Appendix F.4, PDF p.\ 30. Higher scores are better.}
\end{frame}

\begin{frame}{NCM solves more complete Sudoku boards at lower NFE}
  \centering
  \begin{tabular}{lrrrr}
    \toprule
    Model & NFE & Easy (\%) & Medium (\%) & Hard (\%)\\
    \midrule
    AR greedy & 128 & 14.6 & 5.1 & 1.0\\
    DUO & 128 & 96.3 & 84.7 & 58.4\\
    FMLM+ & 16 & 97.7 & 92.8 & 81.3\\
    \ncm & \textbf{12} & \textbf{99.5} & \textbf{96.5} & \textbf{86.1}\\
    \bottomrule
  \end{tabular}
  \vspace{5mm}
  \begin{block}{Mechanism proposed by the authors}
    Coarse levels represent the whole board; later levels can revise a partial solution.
  \end{block}
  \papersource{Paper, Sec.\ 4.4 and Table 3, PDF p.\ 10. Difficulty: 40 / 35 / 30 clues. Baseline scores are from prior work.}
\end{frame}

\begin{frame}{Training-token exposure is lower; model scaling looks promising}
  \begin{columns}[T]
    \column{0.50\linewidth}
    \begin{block}{LM1B total token positions}
      NCM tokenizer + generator: \(13.36\)B\\
      LM1B baselines: \(66\)B\\
      Approximately \(4.9\times\) fewer
    \end{block}
    \begin{itemize}
      \item OWT: \(127.22\)B vs.\ over \(500\)B.
      \item Positions include repeated data and padding.
      \item This is \emph{not} a FLOP comparison.
    \end{itemize}
    \column{0.47\linewidth}
    \centering
    \includegraphics[width=\linewidth,height=0.62\textheight,keepaspectratio]{figures/scaling.pdf}
  \end{columns}
  \papersource{Paper, Secs.\ 4.5--4.6, PDF pp.\ 11--12; Fig.\ 6(a), p.\ 11; Appendix C.2, p.\ 24. Scaling shares one tokenizer.}
\end{frame}

\begin{frame}{Hierarchy depth trades more passes for better quality}
  \centering
  \includegraphics[width=0.96\linewidth,height=0.62\textheight,keepaspectratio]{figures/levels_ablation.pdf}
  \begin{itemize}
    \item TinyStories: quality improves from 4 to 32 levels, with diminishing returns.
    \item Level-length schedule also matters: exponential and quadratic beat square-root and logarithmic schedules in this ablation.
  \end{itemize}
  \papersource{Paper, Appendix F.1, Figs.\ 9--10, PDF pp.\ 27--28. The ablation uses 10k generator steps without code corruption.}
\end{frame}

\begin{frame}{Fast sampling can be spent on more candidates}
  \begin{columns}[T]
    \column{0.52\linewidth}
    \begin{itemize}
      \item XSum: one NCM output uses 8 NFE.
      \item MBR with 16 candidates uses 128 NFE.
      \item ROUGE-L rises from 28.0 to 29.6.
      \item At matched sampling time, the tested NCM MBR points exceed the plotted baselines.
    \end{itemize}
    \column{0.45\linewidth}
    \centering
    \includegraphics[width=\linewidth,height=0.62\textheight,keepaspectratio]{figures/tts.pdf}
  \end{columns}
  \papersource{Paper, Sec.\ 4.5, Fig.\ 6(b), PDF p.\ 11; Table 2, p.\ 10.}
\end{frame}

\begin{frame}{The main open choices follow directly from the mechanism}
  \begin{enumerate}
    \item Fixed \(K\) and \(\ell_k\): even simple outputs traverse all levels.
    \item Tokenizer and generator are trained separately; reconstruction codes may not be ideal generation codes.
    \item A shown XSum summary changes a name and loan duration; the authors also note transition artifacts.
    \item Main generators are near 100M parameters; LM1B scaling reaches 725M.
  \end{enumerate}
  \begin{block}{Research direction}
    Test adaptive stopping and joint representation learning against exact reconstruction, quality, and wall-clock cost.
  \end{block}
  \papersource{Paper, Fig.\ 8, PDF p.\ 12; Appendix I, PDF p.\ 35; Fig.\ 6(a), p.\ 11.}
\end{frame}

\begin{frame}{Take-home: representation is the sampling schedule}
  \begin{enumerate}
    \item A residual VQ hierarchy gives short coarse levels and detailed fine levels.
    \item Each level is predicted in parallel; levels are generated in sequence.
    \item The reported gains combine fewer calls and lower average per-call cost.
    \item Quality comparisons require entropy, baseline protocol, and full-system cost.
  \end{enumerate}
  \vspace{4mm}
  {\small\textbf{Paper:} Hermann Kumbong, Ishan S. Khare, Iddah Mlauzi, Andrew Woen,
  Charles Ding, E. Kelly Buchanan, Simran Arora, Azalia Mirhoseini,
  Christopher R\'e. \emph{Next-Concept Modeling: Fast, Parallel Coarse-to-Fine Text Generation}.
  \href{https://arxiv.org/abs/2610.14162v1}{arXiv:2610.14162v1}.}
\end{frame}

\appendix

\begin{frame}{Backup: the clean objective has an ELBO interpretation}
  \[
    q(r_{1:K}\mid x)=\mathbf 1\{r_{1:K}=r^*_{1:K}(x)\}
  \]
  \[
    \mathcal L_{\rm ELBO}(x)
    =\log p_\psi(x\mid r^*_{1:K})
    +\sum_{k,i}\log p_\theta(r_k^{*i}\mid\hat z_*^{(k-1)})
    \leq\log p_{\theta,\psi}(x)
  \]
  \begin{itemize}
    \item Fixed tokenizer and decoder: only the generation term depends on \(\theta\).
    \item With clean conditioning and \(\alpha=0\), NCM's loss is its normalized negative.
    \item Code corruption and level weighting modify that clean objective.
  \end{itemize}
  \papersource{Paper, Appendix A, PDF pp.\ 21--22, Eqs.\ (16)--(18).}
\end{frame}

\begin{frame}{Backup: what the comparison does and does not equalize}
  \begin{itemize}
    \item Model-size tables report generator parameters; NCM has a separate frozen tokenizer.
    \item LM1B baselines share prior 66B-token checkpoints; TinyStories baselines are partly retrained by the authors.
    \item Conditional quality numbers mix prior reports and reproduced checkpoints. For unreleased checkpoints, same-architecture random weights are used for \emph{timing only}.
    \item Gen.\ PPL is a reference-model score, not the NCM marginal likelihood.
  \end{itemize}
  \papersource{Paper, Appendix C--E, PDF pp.\ 23--26; Appendix A, pp.\ 21--22.}
\end{frame}

\end{document}