\documentclass[aspectratio=169,11pt]{beamer}
\usepackage[T1]{fontenc}
\usepackage[utf8]{inputenc}
\usepackage{amsmath,amssymb,graphicx,booktabs}

\definecolor{AcademicBlue}{RGB}{26,57,86}
\definecolor{AcademicTeal}{RGB}{0,105,112}
\definecolor{SoftBlue}{RGB}{232,240,247}
\setbeamercolor{structure}{fg=AcademicBlue}
\setbeamercolor{frametitle}{fg=AcademicBlue,bg=white}
\setbeamercolor{title}{fg=AcademicBlue}
\setbeamercolor{alerted text}{fg=AcademicTeal}
\setbeamercolor{block title}{fg=white,bg=AcademicBlue}
\setbeamercolor{block body}{fg=black,bg=SoftBlue}
\setbeamertemplate{navigation symbols}{}
\setbeamertemplate{itemize items}[circle]
\setbeamertemplate{footline}{%
  \hfill\usebeamerfont{page number in head/foot}%
  \insertframenumber\,/\,\inserttotalframenumber\hspace{4mm}\vspace{3mm}}
\setbeamersize{text margin left=8mm,text margin right=8mm}
\setbeamerfont{frametitle}{size=\Large,series=\bfseries}
\setbeamerfont{title}{size=\LARGE,series=\bfseries}
\newcommand{\papersource}[1]{\par\smallskip{\scriptsize\color{gray}#1\par}}
\newcommand{\strong}[1]{{\color{AcademicTeal}\bfseries #1}}

\title[Low-Bit N-Gram Embeddings]{Low-Bit N-Gram Embeddings\\for Language Models}
\subtitle{A research group paper discussion}
\author{Paper by Stephen Chung and Wenyu Du (DualverseAI)}
\date{\href{https://arxiv.org/abs/2610.31646v1}{arXiv:2610.31646v1} \quad 25 October 2026}

\begin{document}

\begin{frame}
  \titlepage
  \vfill
  \centering\small Takeaway: use fewer bits per value to keep, or expand, n-gram table capacity.
\end{frame}

\begin{frame}{Can precision buy more table addresses?}
  \begin{block}{Question}
    Can hashed n-gram memories keep their modeling benefit with fewer bits per stored value?
  \end{block}
  \begin{columns}[T,onlytextwidth]
    \begin{column}{0.32\textwidth}
      \strong{\Large 74--75\%}
      \par Table memory saved at 4 bits, with $>99\%$ of the original loss improvement retained.
    \end{column}
    \begin{column}{0.32\textwidth}
      \strong{\Large 122--126\%}
      \par Improvement with 4$\times$ as many rows at 2 bits, while using 52--54\% of baseline table memory.
    \end{column}
    \begin{column}{0.32\textwidth}
      \strong{\Large 47--55\%}
      \par Table memory saved in two existing large models, with $<1.2\%$ held-out perplexity change.
    \end{column}
  \end{columns}
  \vfill\papersource{Paper abstract and Secs. 3.2, 3.4, 3.6; PDF pp. 4--9. All percentages concern n-gram tables.}
\end{frame}

\begin{frame}{Hash collisions make row count valuable}
  \begin{itemize}
    \item N-gram keys are numerous; hashing maps them into a fixed set of learned rows.
    \item Fewer rows mean more unrelated n-grams share a whole embedding.
    \item Four tables, each with $m$ rows of width $d/4$, store about $md$ values.
  \end{itemize}
  \begin{block}{Scale of the issue}
    The 31M Llama-family backbone has $d=512$. At $m\approx262$k, its n-gram tables contain about \strong{134M values} before compression. The model name excludes these tables.
  \end{block}
  \vfill\papersource{Paper, Secs. 1--2 and App. B.1, Table 1; PDF pp. 1--3, 14. The 134M count is calculated from the reported dimensions.}
\end{frame}

\begin{frame}{Four lookups augment each token representation}
  \centering
  \includegraphics[width=0.97\linewidth,height=0.56\textheight,keepaspectratio]{figures/fig1_method.pdf}
  \[
    u_t=E[x_t]+\operatorname{Concat}_{j=1}^{4}
    T_j\!\left[h_j(x_{t-n_j+1},\ldots,x_t)\right]
  \]
  \small Two bigram tables and two trigram tables each return a $d/4$-wide vector.
  \vfill\papersource{Paper, Fig. 1, PDF p. 2; Eq. (1), PDF p. 3. Original figure.}
\end{frame}

\begin{frame}{A shared codebook plus one scale per row}
  \begin{columns}[T,onlytextwidth]
    \begin{column}{0.53\textwidth}
      \[
        \hat w_{ri}=s_r c_{q_{ri}},\qquad
        q_{ri}\in\{0,\ldots,2^b-1\}.
      \]
      \begin{itemize}
        \item $q_{ri}$: $b$-bit component index.
        \item $c_k$: 16-bit value shared by a table.
        \item $s_r$: one 16-bit scale per row.
      \end{itemize}
    \end{column}
    \begin{column}{0.44\textwidth}
      \centering
      \includegraphics[width=\linewidth,height=0.48\textheight,keepaspectratio]{figures/fig1-quant.pdf}
    \end{column}
  \end{columns}
  \begin{block}{Storage per $p$-component row}
    $16p\ \longrightarrow\ pb+16$ bits, plus a small shared codebook. For $p=128$, 2 bits use $272$ instead of $2048$ bits.
  \end{block}
  \papersource{Paper, Fig. 1(b), PDF p. 2; Eq. (2), PDF p. 3. Original figure panel.}
\end{frame}

\begin{frame}{Codebook and row scale solve different problems}
  \begin{enumerate}
    \item \textbf{Codebook:} compare uniform, normal-quantile, and Lloyd--Max values. Lloyd--Max fits normalized components by nearest-center assignment and mean updates.
    \item \textbf{Scale:} compare max-based, RMS, and error-fitted row scales.
    \item \textbf{Error-fitted:} test RMS and $\alpha s_r^{\max}$ for $\alpha=1.00,0.95,\ldots,0.30$; select the lowest BF16 reconstruction MSE.
  \end{enumerate}
  \begin{block}{Why the two levels?}
    The codebook captures the distribution of values across a table; $s_r$ captures each row's magnitude.
  \end{block}
  \vfill\papersource{Paper, Sec. 2 and App. A.2--A.3; PDF pp. 3, 13--14.}
\end{frame}

\begin{frame}{The baseline is the gain from adding tables}
  \begin{columns}[T,onlytextwidth]
    \begin{column}{0.53\textwidth}
      \begin{block}{From-scratch models}
        Train matched models with no tables and with 16-bit tables. Quantize the trained tables and measure validation BPB (lower is better).
      \end{block}
      \[
        R_b=\frac{L_{\varnothing}-L_b}{L_{\varnothing}-L_{16}}\times100\%.
      \]
    \end{column}
    \begin{column}{0.44\textwidth}
      \begin{block}{Existing large models}
        Keep the backbone fixed; compress FP8 tables; measure perplexity and task accuracy against the original model.
      \end{block}
      \small $R_b=100\%$ matches the 16-bit table; $R_b>100\%$ improves on it.
    \end{column}
  \end{columns}
  \vfill\papersource{Paper, Secs. 3.1, 3.4 and App. B.3--B.5; PDF pp. 4, 6, 15--16.}
\end{frame}

\begin{frame}{Five models, one n-gram lookup design}
  \centering
  \begin{tabular}{lrrr}
    \toprule
    Model family & Backbone parameters & Width $d$ & Rows/table \\
    \midrule
    nanochat & 26M & 384 & 262k, 1M \\
    nanochat & 50M & 512 & 262k \\
    GPT-2 & 30M & 512 & 262k \\
    Llama & 31M & 512 & 262k, 1M \\
    Llama & 322M & 960 & 262k, 1M \\
    \bottomrule
  \end{tabular}
  \par\vspace{2mm}
  \begin{itemize}
    \item ClimbMix training; held-out ClimbMix validation; three random seeds.
    \item About 500M training targets for small models, 1B for Llama 322M.
  \end{itemize}
  \vfill\papersource{Paper, Sec. 3.1 and App. B.1--B.2, Tables 1--2; PDF pp. 4, 14--15. Parameter counts exclude tables.}
\end{frame}

\begin{frame}{Four bits preserve nearly all of the table gain}
  \centering
  \includegraphics[width=0.95\linewidth,height=0.59\textheight,keepaspectratio]{figures/fig2_retention.pdf}
  \begin{itemize}
    \item 4 bits: 74--75\% table memory saved; 99.3--99.6\% gain retained.
    \item 3 bits: about 80\% saved; 97.1--98.1\% retained.
    \item 2 bits: 86--87\% saved; 85.9--90.1\% retained.
  \end{itemize}
  \papersource{Paper, Fig. 2 and App. C.1, Table 3; PDF pp. 5, 17. Ranges are for 262k rows/table. Original figure.}
\end{frame}

\begin{frame}{At two bits, scale fitting matters most}
  \begin{columns}[T,onlytextwidth]
    \begin{column}{0.52\textwidth}
      \centering
      \includegraphics[width=\linewidth,height=0.62\textheight,keepaspectratio]{figures/fig-design-scale.pdf}
    \end{column}
    \begin{column}{0.45\textwidth}
      \begin{itemize}
        \item Uniform + max scale: \strong{58.04\%} retained.
        \item Uniform + error-fitted: \strong{86.36\%}.
        \item With fitted scales, uniform, normal quantiles, and Lloyd--Max all lie near \strong{86.4--86.5\%}.
      \end{itemize}
    \end{column}
  \end{columns}
  \vfill\papersource{Paper, Fig. 3(a), PDF p. 5; 31M Llama, 262k rows/table. Original figure panel.}
\end{frame}

\begin{frame}{For small models, spend extra bits on bigrams}
  \begin{columns}[T,onlytextwidth]
    \begin{column}{0.58\textwidth}
      \centering
      \includegraphics[width=\linewidth,height=0.61\textheight,keepaspectratio]{figures/fig-design-mixed.pdf}
    \end{column}
    \begin{column}{0.39\textwidth}
      \begin{itemize}
        \item Same total bits $\Rightarrow$ approximately equal table memory.
        \item On 31M Llama, 4/3 bigram/trigram bits retain \strong{99.1\%}; 3/4 retain \strong{97.7\%}.
        \item Across five models, 4/3 retains 98.9--99.4\% and saves 77--78\% memory.
      \end{itemize}
    \end{column}
  \end{columns}
  \vfill\papersource{Paper, Fig. 3(b), PDF p. 5; full sweep in Fig. 8, PDF p. 20. Original figure panel.}
\end{frame}

\begin{frame}{FP8 tables in two large models also compress}
  \begin{columns}[T,onlytextwidth]
    \begin{column}{0.52\textwidth}
      \centering
      \includegraphics[width=\linewidth,height=0.61\textheight,keepaspectratio]{figures/fig-large-uniform.pdf}
    \end{column}
    \begin{column}{0.45\textwidth}
      \begin{itemize}
        \item Original DeepSeek-V4.1-Flash and Qwen3.8-Flash-Next tables are FP8.
        \item Uniform 4-bit: about half of table memory saved; validation perplexity changes by +1.26\% for DeepSeek and approximately 0 for Qwen.
        \item Uniform 3-bit: +5.06\% for DeepSeek, +0.52\% for Qwen.
      \end{itemize}
    \end{column}
  \end{columns}
  \vfill\papersource{Paper, Sec. 3.4 and Fig. 4(a), PDF p. 6; App. B.3, PDF p. 15. Original figure panel.}
\end{frame}

\begin{frame}{Mixed precision depends on the model}
  \begin{columns}[T,onlytextwidth]
    \begin{column}{0.56\textwidth}
      \centering
      \includegraphics[width=\linewidth,height=0.61\textheight,keepaspectratio]{figures/fig-large-mixed.pdf}
    \end{column}
    \begin{column}{0.41\textwidth}
      \begin{itemize}
        \item DeepSeek: 4/5/4 bits (bigram/trigram/four-gram), \strong{46.7\%} saved.
        \item Qwen: 4/3 bits (bigram/trigram), \strong{55.0\%} saved.
        \item DeepSeek favors an extra higher-order bit; Qwen favors an extra bigram bit.
      \end{itemize}
    \end{column}
  \end{columns}
  \vfill\papersource{Paper, Fig. 4(b), PDF p. 6; full grids in Figs. 9--10, PDF p. 21. Original figure panel.}
\end{frame}

\begin{frame}{Held-out perplexity changes stay below 1.2\%}
  \centering
  \begin{tabular}{llrrr}
    \toprule
    Model & Corpus & Original & Low-bit & Change \\
    \midrule
    DeepSeek & WikiText-2 & 2.7564 & 2.7870 & +1.110\% \\
    DeepSeek & PTB & 7.7730 & 7.7794 & +0.082\% \\
    DeepSeek & C4 & 7.9679 & 7.9584 & $-$0.119\% \\
    \addlinespace
    Qwen & WikiText-2 & 4.2270 & 4.2671 & +0.949\% \\
    Qwen & PTB & 4.0633 & 4.0558 & $-$0.183\% \\
    Qwen & C4 & 9.2329 & 9.2354 & +0.027\% \\
    \bottomrule
  \end{tabular}
  \par\vspace{2mm}
  \small DeepSeek 4/5/4 saves 46.7\%; Qwen 4/3 saves 55.0\% of FP8 table memory. Each corpus contributes 64 paired windows and 32,768 target tokens.
  \vfill\papersource{Paper, App. C.4, Table 6, PDF p. 22; evaluation protocol in App. B.3, PDF p. 15.}
\end{frame}

\begin{frame}{Downstream accuracy is largely preserved}
  \centering
  \includegraphics[width=0.91\linewidth,height=0.54\textheight,keepaspectratio]{figures/fig_downstream.pdf}
  \begin{itemize}
    \item Mean of three task differences: DeepSeek $-0.93$ percentage points; Qwen $-0.27$ points.
    \item All six task-level paired 95\% bootstrap intervals include zero.
  \end{itemize}
  \papersource{Paper, Fig. 5, PDF p. 7; App. C.4, Table 7, PDF p. 22. HellaSwag, ARC-Challenge, WinoGrande. Original figure.}
\end{frame}

\begin{frame}{Adaptation recovers aggressive compression}
  \centering
  \includegraphics[width=0.87\linewidth,height=0.55\textheight,keepaspectratio]{figures/fig_training.pdf}
  \begin{itemize}
    \item 1-bit post-compression: $-5.7\%$ retained; compress then fine-tune: \strong{65.3\%}.
    \item 1-bit quantized-forward pre-training: \strong{77.0\%} retained.
    \item Fine-tuning uses 381 extra updates for 262k-row tables, with all weights trainable.
  \end{itemize}
  \papersource{Paper, Fig. 6, PDF p. 8; App. C.5, Table 8, PDF p. 23; App. B.4, PDF pp. 15--16. Original figure.}
\end{frame}

\begin{frame}{More rows can beat the original table at lower cost}
  \centering
  \includegraphics[width=0.95\linewidth,height=0.57\textheight,keepaspectratio]{figures/fig3_rows.pdf}
  \par\smallskip\small At 1M rows/table and 2 bits: \strong{52--54\%} of the 262k-row, 16-bit table memory gives \strong{122--126\%} of its loss improvement; compress-then-fine-tune reaches \strong{129--145\%}.
  \vfill\papersource{Paper, Fig. 7, PDF p. 8; App. C.1, Table 4, PDF p. 18; App. C.5, Table 9, PDF p. 23. Original figure.}
\end{frame}

\begin{frame}{What the reported memory savings cover}
  \begin{itemize}
    \item \textbf{Stored n-gram tables:} indices, one 16-bit scale per row, and shared codebooks; the percentage is not whole-model memory.
    \item \textbf{Quantized training:} a BF16 master copy remains for gradient updates, so the storage ratio is not a training peak-memory ratio.
    \item \textbf{Deployment:} the paper reports model quality and table storage, without end-to-end throughput or decode-cost measurements.
    \item \textbf{Large-model evidence:} fixed backbones; three held-out corpora and three downstream tasks.
  \end{itemize}
  \vfill\papersource{Paper, Apps. B.3--B.5, PDF pp. 15--16; Sec. 3.4, PDF pp. 6--7.}
\end{frame}

\begin{frame}{Take-home: allocate bits and rows together}
  \begin{enumerate}
    \item \strong{4 bits} are a low-risk storage point in the five from-scratch models: about 75\% less table memory with $>99\%$ of the table gain.
    \item \strong{2 bits + more rows} can give better quality than smaller 16-bit tables at about half the table memory; fine-tune after compression.
    \item \strong{Mixed precision} should follow measured per-table sensitivity; DeepSeek and Qwen prefer different allocations.
  \end{enumerate}
  \vfill\papersource{Paper, Secs. 3.2--3.6, PDF pp. 4--9. Full paper: \href{https://arxiv.org/abs/2610.31646v1}{arXiv:2610.31646v1}.}
\end{frame}

\appendix
\begin{frame}{Backup: implementation of the low-bit representation}
  \begin{itemize}
    \item Hash keys: $k_t^{(2)}=x_t+(V+1)x_{t-1}$; $k_t^{(3)}=x_t+(V+1)x_{t-1}+(V+1)^2x_{t-2}$, with $V=8192$.
    \item Lloyd--Max samples up to 4M normalized components/table and runs 60 assignment/update iterations.
    \item Quantized-forward training keeps BF16 embeddings and passes gradients through a straight-through estimator.
    \item Codebook values and row scales are BF16; indices are packed at the selected width.
  \end{itemize}
  \vfill\papersource{Paper, Apps. A.1--A.3 and B.4; PDF pp. 13--16.}
\end{frame}

\begin{frame}{Backup: component-wise hashing at two bits}
  \centering
  \begin{tabular}{lrrr}
    \toprule
    Model & Whole row & Same column & Fully random \\
    \midrule
    nanochat 26M & 91.71 & 93.61 & 93.95 \\
    GPT-2 30M & 93.69 & 99.57 & 100.16 \\
    Llama 31M & 87.03 & 100.28 & 98.39 \\
    \bottomrule
  \end{tabular}
  \par\vspace{3mm}
  \small Values are \% of the improvement from conventional 16-bit tables. Per-component addresses improve means in these three 2-bit comparisons; the Llama same-column estimate has 10.54-point seed SD.
  \vfill\papersource{Paper, App. C.6, Table 10, PDF p. 24; three seeds, 262k rows/table.}
\end{frame}

\end{document}