\documentclass[aspectratio=169,11pt]{beamer}
\usepackage[T1]{fontenc}
\usepackage[utf8]{inputenc}
\usepackage{amsmath,amssymb,graphicx,booktabs}

\definecolor{AcademicBlue}{RGB}{26,57,86}
\definecolor{AcademicTeal}{RGB}{0,105,112}
\definecolor{TokenPink}{RGB}{190,35,112}
\definecolor{ByteOrange}{RGB}{221,82,0}
\definecolor{EotPurple}{RGB}{107,72,215}
\setbeamercolor{structure}{fg=AcademicBlue}
\setbeamercolor{frametitle}{fg=AcademicBlue,bg=white}
\setbeamercolor{title}{fg=AcademicBlue}
\setbeamercolor{alerted text}{fg=AcademicTeal}
\setbeamercolor{block title}{fg=white,bg=AcademicBlue}
\setbeamercolor{block body}{fg=black,bg=AcademicBlue!5}
\setbeamertemplate{navigation symbols}{}
\setbeamertemplate{itemize items}[circle]
\setbeamertemplate{footline}{\hfill\usebeamerfont{page number in head/foot}%
  \insertframenumber\,/\,\inserttotalframenumber\hspace{4mm}\vspace{3mm}}
\setbeamersize{text margin left=8mm,text margin right=8mm}
\setbeamerfont{frametitle}{size=\Large,series=\bfseries}
\setbeamerfont{title}{size=\LARGE,series=\bfseries}
\newcommand{\papersource}[1]{\par\smallskip{\scriptsize\color{gray}#1\par}}
\newcommand{\eot}{\texttt{<eot>}}

\title[Byte model distillation]{Breaking the Token Ceiling:\\Distilling Smaller, Stronger Byte Models}
\subtitle{Research group paper discussion}
\author{Paper by Kalyani Marathe, Artidoro Pagnoni, Tomasz Limisiewicz,\
Margaret Li, Mike Lewis, Luke Zettlemoyer, Srinivasan Iyer}
\date{\href{https://arxiv.org/abs/2609.12303v1}{arXiv:2609.12303v1}\quad 23 October 2026}

\begin{document}

\begin{frame}
  \titlepage
\end{frame}

\begin{frame}{A boundary marker enables exact byte targets}
  \begin{block}{Research question}
    Can a small byte model distill a tokenized teacher efficiently, and how does its quality scale with training compute?
  \end{block}
  \begin{itemize}
    \item \alert{End-Of-Token} adds a boundary symbol to the byte sequence, so one teacher forward pass supplies an exact target over the augmented alphabet.
    \item Byte models start behind token models at low training FLOPs and improve faster in the reported runs.
    \item A two-stage scaling fit predicts \textbf{52.4\%} six-task accuracy for distilled EOT versus \textbf{48.4\%} for distilled tokens at the asymptote.
  \end{itemize}
  \papersource{Marathe et al., Abstract and Secs. 2--6, PDF pp. 1, 4--16. The 52.4\% and 48.4\% values are extrapolations.}
\end{frame}

\begin{frame}{Token boundaries hide the next byte}
  A teacher distribution after a prefix may include all three BPE tokens:
  \begin{center}
  \begin{tabular}{lccc}
  \toprule
  Candidate token & \texttt{is} & \texttt{isu} & \texttt{isk}\\
  Teacher probability & 0.125 & 0.500 & 0.125\\
  \bottomrule
  \end{tabular}
  \end{center}
  \begin{itemize}
    \item After byte prefix \texttt{is}, \texttt{isu} proposes \texttt{u}; \texttt{isk} proposes \texttt{k}.
    \item \texttt{is} has already ended. Its next byte depends on a \emph{new} teacher token distribution.
    \item Re-running the teacher for each possible completed prefix is expensive.
  \end{itemize}
  \papersource{Paper, Sec. 2 and Fig. 2, PDF pp. 4--5. Probabilities are the paper's example.}
\end{frame}

\begin{frame}{Marginalize-It renormalizes the surviving paths}
  \begin{block}{Approximate next-byte target after \texttt{is}}
  \[
    P_{\mathrm{MI}}(u\mid\texttt{is})=\frac{0.5}{0.5+0.125}=0.8,
    \qquad P_{\mathrm{MI}}(k\mid\texttt{is})=0.2.
  \]
  \end{block}
  \begin{itemize}
    \item Aggregate teacher token probabilities by their next byte, conditioned on the gold byte prefix.
    \item The first byte is exact; later positions drop tokens that end at the current prefix.
    \item This provides a cheap approximate byte distillation baseline.
  \end{itemize}
  \papersource{Paper, Sec. 2.1 and Fig. 2, PDF pp. 4--5.}
\end{frame}

\begin{frame}{EOT turns termination into a predicted symbol}
  Append \eot{} to every teacher token's byte spelling: $\tilde b(t)=b(t)\Vert\langle\mathrm{eot}\rangle$.
  \[
    M(u)=\sum_{t:\,u\preceq\tilde b(t)}q(t\mid h),\qquad
    P(a\mid u,h)=\frac{M(ua)}{M(u)}.
  \]
  \begin{block}{The same example now keeps all probability mass}
  \[
    P(u,k,\langle\mathrm{eot}\rangle\mid\texttt{is})
      =\left(\frac{0.5}{0.75},\frac{0.125}{0.75},\frac{0.125}{0.75}\right)
      \approx(0.667,0.167,0.167).
  \]
  \end{block}
  The conversion exactly factorizes the teacher's distribution over \emph{boundary-marked} byte strings.
  \papersource{Paper, Sec. 2.2 and Fig. 2, PDF pp. 4--5. $u\preceq v$ denotes a string prefix.}
\end{frame}

\begin{frame}{The paper's B3 example shows the missing branch}
  \centering
  \includegraphics[width=0.98\linewidth,height=0.62\textheight,keepaspectratio]{figures/fig02-b3-crop.pdf}
  \par\smallskip
  \begin{minipage}{0.94\linewidth}
  \small Marginalize-It assigns $(0.8,0.2)$ to $(u,k)$; EOT assigns $(0.667,0.167,0.167)$ to $(u,k,\langle\mathrm{eot}\rangle)$.
  \end{minipage}
  \papersource{Paper, Fig. 2, PDF p. 5. Cropped from the original vector figure.}
\end{frame}

\begin{frame}{The study crosses three representations with two losses}
  \centering
  \begin{tabular}{lccr}
  \toprule
  Student representation & Cross-entropy & Teacher distillation & Vocab.\\
  \midrule
  \textcolor{TokenPink}{BPE tokens} & Token Sup. & Token Dist. (top-$k$) & 128,256\\
  \textcolor{ByteOrange}{UTF-8 bytes} & Byte Sup. & Marginalize-It Dist. & 260\\
  \textcolor{EotPurple}{Bytes + \eot} & EOT Sup. & EOT Dist. (exact) & 261\\
  \bottomrule
  \end{tabular}
  \par\bigskip
  \begin{itemize}
    \item Teacher: Llama 3-8B; training text: Llama-2 pretraining mixture.
    \item All students have the same 25-layer, $d=2048$ backbone and about 1.2846B layer parameters.
    \item Total parameters differ: 1.81B for tokens, 1.28B for either byte model.
  \end{itemize}
  \papersource{Paper, Sec. 3 and Table 1, PDF p. 6; Appendix A and Table 9, PDF p. 24.}
\end{frame}

\begin{frame}{EOT spends more compute per original token}
  \centering
  \begin{tabular}{lrrr}
  \toprule
  Student & FLOPs/unit & Units/BPE token & FLOPs/BPE token\\
  \midrule
  Token & $8.18\times10^9$ & 1.0 & $8.18\times10^9$\\
  Byte & $8.81\times10^9$ & 4.5 & $39.65\times10^9$\\
  Byte + \eot & $9.44\times10^9$ & 5.5 & $51.92\times10^9$\\
  \bottomrule
  \end{tabular}
  \par\bigskip
  \begin{itemize}
    \item EOT costs about \textbf{30.94\%} more student-training FLOPs than plain bytes for the same original text.
    \item Reported sequence lengths are 2,048 tokens, 9,216 bytes, and 11,264 byte/EOT units.
    \item A fixed FLOP budget therefore exposes each model to a different amount of text.
  \end{itemize}
  \papersource{Paper, Table 3, PDF p. 8; Appendix A, Table 9, PDF p. 24.}
\end{frame}

\begin{frame}{Eight benchmarks test three task families}
  \begin{itemize}
    \item \textbf{Multiple choice (4):} ARC-Easy, ARC-Challenge, HellaSwag, PIQA.
    \item \textbf{Generation (2):} MBPP and Natural Questions.
    \item \textbf{Translation (2):} FLORES English--German and German--English; reported in BLEU.
  \end{itemize}
  \begin{block}{The headline performance metric}
    Mean top-1 accuracy/error uses the \textbf{four choice + two generation} benchmarks. Translation does not enter the 52.4\% aggregate.
  \end{block}
  Validation bits per byte (BPB) is lower-is-better; task accuracy and BLEU are higher-is-better.
  \papersource{Paper, Secs. 3.3 and 6.4, PDF pp. 6, 14--15.}
\end{frame}

\begin{frame}{Token models lead early on validation BPB}
  \centering
  \includegraphics[width=0.92\linewidth,height=0.78\textheight,keepaspectratio]{figures/fig03-bpb-top.pdf}
  \papersource{Paper, Fig. 3 and Table 2, PDF pp. 7--8; learning rate $4\times10^{-3}$. Cropped from original Fig. 3.}
\end{frame}

\begin{frame}{Choice accuracy improves faster for byte students}
  \centering
  \includegraphics[width=0.94\linewidth,height=0.78\textheight,keepaspectratio]{figures/fig04-choice-flops.pdf}
  \papersource{Paper, Fig. 4, PDF p. 9; mean of four multiple-choice benchmarks, learning rate $4\times10^{-3}$.}
\end{frame}

\begin{frame}{Generation shows the same early token advantage}
  \centering
  \includegraphics[width=0.94\linewidth,height=0.78\textheight,keepaspectratio]{figures/fig05-generation-flops.pdf}
  \papersource{Paper, Fig. 5 and Sec. 5.2, PDF p. 10; learning rate $4\times10^{-3}$.}
\end{frame}

\begin{frame}{Translation keeps a large low-compute gap}
  \centering
  \includegraphics[width=0.94\linewidth,height=0.78\textheight,keepaspectratio]{figures/fig06-translation-flops.pdf}
  \papersource{Paper, Fig. 6 and Sec. 5.3, PDF p. 11; two translation directions, learning rate $4\times10^{-3}$.}
\end{frame}

\begin{frame}{A lower BPB can still give worse task accuracy}
  \begin{block}{Observed around $1.8\times10^{21}$ training FLOPs}
    All four byte variants have lower BPB than Token Supervised on validation set 1, yet their downstream scores are worse.
  \end{block}
  \begin{itemize}
    \item BPB scores the probability of the \emph{reference} sequence.
    \item Top-1 success also depends on the competitors' ranking.
    \item Tokenization and distillation change the BPB--task relationship; each model needs its own calibration curve.
  \end{itemize}
  \papersource{Paper, Secs. 4--6, PDF pp. 8--16; Appendix G, Table 24, PDF p. 50.}
\end{frame}

\begin{frame}{Two fitted laws connect training compute to task accuracy}
  For each model variant $m$ the authors fit:
  \[
    \underbrace{y_m(x)=b_m x^{a_m}+c_m}_{\text{validation BPB vs. FLOPs}},\qquad
    \underbrace{z_m(y)=\epsilon_m-k_m e^{-\gamma_m y}}_{\text{six-task error vs. BPB}}.
  \]
  \[
    \text{Predicted accuracy at compute }x
       =100\bigl[1-z_m(y_m(x))\bigr]\%.
  \]
  \begin{itemize}
    \item $x$: student training FLOPs; $c_m$: fitted BPB asymptote.
    \item Set $y=c_m$ for the reported asymptotic performance.
    \item The far right of each curve is a model-based extrapolation.
  \end{itemize}
  \papersource{Paper, Secs. 4 and 6.4--6.5, Eqns. (1)--(2), Tables 2, 4--5, PDF pp. 7--8, 14--16.}
\end{frame}

\begin{frame}{The fitted EOT asymptote leads by 4.0 points}
  \centering
  \begin{tabular}{lrr}
  \toprule
  Method, LR $4\times10^{-3}$ & BPB asymptote $c\downarrow$ & Predicted accuracy $\uparrow$\\
  \midrule
  Token Supervised & 0.9568 & 46.0\%\\
  Token Distilled & 0.9407 & 48.4\%\\
  Byte Supervised & 0.8967 & 51.2\%\\
  Marginalize-It Distilled & 0.9016 & 50.5\%\\
  EOT Supervised & 0.8891 & 51.6\%\\
  \textbf{EOT Distilled} & \textbf{0.8983} & \textbf{52.4\%}\\
  \bottomrule
  \end{tabular}
  \par\bigskip
  \begin{minipage}{0.94\linewidth}
  \small EOT Distilled exceeds Token Distilled by \textbf{4.0 percentage points}. EOT Supervised has a lower BPB than EOT Distilled but a lower predicted task score.
  \end{minipage}
  \papersource{Paper, Table 5 and Fig. 10, PDF pp. 15--16; six-task aggregate.}
\end{frame}

\begin{frame}{Equal-FLOP connections expose the crossover path}
  \begin{columns}[T]
    \begin{column}{0.65\linewidth}
      \centering
      \includegraphics[width=\linewidth,height=0.77\textheight,keepaspectratio]{figures/fig11-feather-plot.pdf}
    \end{column}
    \begin{column}{0.32\linewidth}
      \begin{itemize}
        \item \textcolor{TokenPink}{Squares: token distillation}
        \item \textcolor{ByteOrange}{Circles: Marginalize-It}
        \item \textcolor{EotPurple}{Stars: EOT distillation}
        \item Blue/green links connect equal FLOPs.
      \end{itemize}
    \end{column}
  \end{columns}
  \papersource{Paper, Fig. 11 and Sec. 7.1, PDF p. 17; LR $4\times10^{-3}$. Cropped from original Fig. 11.}
\end{frame}

\begin{frame}{EOT reaches the token asymptote in the fitted model}
  Target: Token Distilled's predicted asymptotic accuracy, \textbf{48.4\%}.
  \par\bigskip
  \centering
  \begin{tabular}{lrr}
  \toprule
  Byte distillation method & Required BPB & Predicted training FLOPs\\
  \midrule
  Marginalize-It & 0.9102 & $2.04\times10^{23}$\\
  End-Of-Token & 0.9145 & $6.33\times10^{22}$\\
  \bottomrule
  \end{tabular}
  \par\bigskip
  \begin{minipage}{0.91\linewidth}
  \small The authors invert the task law at 48.4\% accuracy, then invert the BPB--FLOPs law. These crossover budgets are extrapolations beyond the trained checkpoints.
  \end{minipage}
  \papersource{Paper, Sec. 7.2 and Table 6, PDF p. 18.}
\end{frame}

\begin{frame}{The one-sixth data claim comes from FLOP accounting}
  At EOT's fitted crossover, $6.33\times10^{22}$ FLOPs:
  \[
    \text{EOT text bytes}
      =6.70\times10^{12}\;\text{units}\times\frac{4.5}{5.5}
      =5.48\times10^{12}.
  \]
  \[
    \text{Token text-byte equivalent}
      =7.73\times10^{12}\;\text{tokens}\times4.5
      =3.48\times10^{13}.
  \]
  \begin{itemize}
    \item Ratio: $3.48\times10^{13}/5.48\times10^{12}=\mathbf{6.35\times}$ processed-text volume.
    \item Estimated logit storage: \textbf{5.19$\times$} smaller, assuming token top-$k\approx600$ with values and indices versus full byte logits.
  \end{itemize}
  \papersource{Paper, Table 7, PDF p. 18; storage assumptions in Appendix A, Table 10, PDF p. 26.}
\end{frame}

\begin{frame}{Current checkpoints and asymptotes tell different stories}
  \centering
  \begin{tabular}{lrll}
  \toprule
  Model & Six-task mean & Status & Source\\
  \midrule
  EOT Distilled & \textbf{52.4\%} & fitted asymptote & Table 8\\
  EOT Distilled & 44.6\% & largest trained checkpoint & Table 18\\
  Token Distilled & 44.4\% & largest trained checkpoint & Table 18\\
  Llama 3.2-1B & 45.9\% & evaluated open model & Table 8\\
  Gemma-3-1B-pt & 44.3\% & evaluated open model & Table 8\\
  Gemma 2B & 50.3\% & evaluated open model & Table 8\\
  \bottomrule
  \end{tabular}
  \par\bigskip
  \begin{minipage}{0.95\linewidth}
  \small The reported 6.5, 8.1, and 2.1 point leads over the open models compare EOT's \emph{predicted} 52.4\% with their \emph{measured} scores.
  \end{minipage}
  \papersource{Paper, Sec. 8, Table 8, PDF p. 19; Appendix C, Table 18, PDF p. 37.}
\end{frame}

\begin{frame}{The validation corpus moves the predicted crossover}
  \centering
  \begin{tabular}{lrrrr}
  \toprule
  Validation set & Token Dist. & EOT Dist. & EOT lead & EOT crossover FLOPs\\
  \midrule
  Set 1 & 48.4\% & 52.4\% & 4.0 pp & $6.33\times10^{22}$\\
  Set 2 & 48.7\% & 50.5\% & 1.8 pp & $1.69\times10^{23}$\\
  \bottomrule
  \end{tabular}
  \par\bigskip
  \begin{itemize}
    \item Both rows use the paper's LR $4\times10^{-3}$ fits and the same downstream tasks.
    \item The held-out text used for BPB calibration changes the fitted limit and crossover.
    \item Byte asymptotes also show sensitivity in the paper's leave-one-out analysis.
  \end{itemize}
  \papersource{Paper, Table 5, PDF p. 16; Appendix D--E, Fig. 21 and Tables 21--22, PDF pp. 38, 43--44.}
\end{frame}

\begin{frame}{What the study establishes}
  \begin{enumerate}
    \item A simple \eot{} boundary makes the teacher's token distribution exactly decomposable into augmented byte targets in one teacher forward pass.
    \item The measured student curves show an early token advantage and stronger later gains for byte students under the paper's training setup.
    \item The six-task lead, data ratio, and open-model comparison rely on fitted asymptotes and accounting assumptions. Byte inference cost remains a practical open question.
  \end{enumerate}
  \vfill
  \centering\href{https://arxiv.org/abs/2609.12303v1}{Marathe et al., arXiv:2609.12303v1}
  \papersource{Paper, Secs. 2--10 and Appendix D--E, PDF pp. 4--21, 38--45.}
\end{frame}

\end{document}