\documentclass[aspectratio=169,11pt]{beamer}
\usepackage[T1]{fontenc}
\usepackage[utf8]{inputenc}
\usepackage{amsmath,amssymb,graphicx,booktabs}

\definecolor{AcademicBlue}{RGB}{26,57,86}
\definecolor{AcademicTeal}{RGB}{0,105,112}
\setbeamercolor{structure}{fg=AcademicBlue}
\setbeamercolor{frametitle}{fg=AcademicBlue,bg=white}
\setbeamercolor{title}{fg=AcademicBlue}
\setbeamercolor{alerted text}{fg=AcademicTeal}
\setbeamercolor{block title}{fg=white,bg=AcademicBlue}
\setbeamercolor{block body}{fg=black,bg=AcademicBlue!5}
\setbeamertemplate{navigation symbols}{}
\setbeamertemplate{itemize items}[circle]
\setbeamertemplate{footline}{%
  \hfill\usebeamerfont{page number in head/foot}%
  \insertframenumber\,/\,\inserttotalframenumber\hspace{4mm}\vspace{3mm}}
\setbeamersize{text margin left=8mm,text margin right=8mm}
\setbeamerfont{frametitle}{size=\Large,series=\bfseries}
\setbeamerfont{title}{size=\LARGE,series=\bfseries}
\newcommand{\papersource}[1]{\par\smallskip{\scriptsize\color{gray}#1\par}}

\title[MultiHashFormer]{MultiHashFormer:\\Hash-based Generative Language Models}
\subtitle{Research group paper discussion}
\author{Paper: Huiyin Xue, Atsuki Yamaguchi, Nikolaos Aletras}
\date{\href{https://arxiv.org/abs/2606.28057v2}{arXiv:2606.28057v2} \quad EMNLP 2026\\Reading presentation: 22 October 2026}

\begin{document}

\begin{frame}
  \titlepage
\end{frame}

\begin{frame}{A reversible hash interface enables token generation}
\begin{block}{Research question}
Can a causal LM share a fixed set of bucket embeddings across a growing vocabulary while still generating an unambiguous next token?
\end{block}
\begin{itemize}
  \item Encode each token as a \alert{unique multi-hash signature}.
  \item Decode its coordinates, then normalize over registered tokens.
  \item At 1B/3B, the main variant improves many benchmark scores; a 32K-to-48K vocabulary expansion adds no model parameters.
\end{itemize}
\papersource{Paper, Abstract; Secs. 1, 3, 5; PDF pp. 1--8.}
\end{frame}

\begin{frame}{A single hash bucket cannot recover a token}
\begin{columns}[T,onlytextwidth]
\column{0.48\textwidth}
\begin{block}{Single-ID hashing}
\[\texttt{cat}\mapsto 40,\qquad \texttt{map}\mapsto 40\]
Predicting bucket 40 leaves two possible words.
\end{block}
\column{0.48\textwidth}
\begin{block}{Multi-ID hashing}
\[\texttt{cat}\mapsto(12,40,56)\]
\[\texttt{map}\mapsto(99,40,3)\]
Coordinate collisions are allowed; the full signatures differ.
\end{block}
\end{columns}
\vspace{2mm}
\centering\alert{The output interface must be invertible on the registered vocabulary.}
\papersource{Paper, Sec. 1 and Fig. 1; PDF pp. 1--2. Example from the paper.}
\end{frame}

\begin{frame}{Signature capacity grows combinatorially}
\begin{itemize}
  \item $H$ independent MMH3 hashes; $B$ buckets per hash, with bucket 0 reserved for padding.
  \item For non-padding tokens, $\mathcal H_i(w)=\mathrm{MMH3}(w,\mathrm{seed}_i)\bmod(B-1)+1$.
  \item The authors resolve a full-signature collision by rehashing until an unused signature is assigned.
\end{itemize}
\begin{block}{Different scaling laws}
Non-padding signature combinations: $(B-1)^H$; learned bucket embeddings: $HBd$ parameters.
\end{block}
\papersource{Paper, Sec. 3.1; PDF p. 3. Capacity is an upper bound on possible signatures.}
\end{frame}

\begin{frame}{The interface surrounds a standard Transformer decoder}
\centering
\includegraphics[width=0.98\linewidth,height=0.69\textheight,keepaspectratio]{figures/fig-01-framework.png}
\vspace{1mm}
\begin{itemize}
  \item Hash Encoder combines the input buckets into one $d$-dimensional vector.
  \item Hash Decoder reconstructs the next token's signature after the backbone.
\end{itemize}
\papersource{Paper, Fig. 1; PDF p. 2. Three hash heads are illustrative; main experiments use H=3 or 4.}
\end{frame}

\begin{frame}{The encoder learns how to combine shared buckets}
\[
u_i=E^{(i)}_{\mathcal H_i(w),:},\quad
a_i=\sigma(u_iW_1)W_2,\quad
\tilde\alpha_i=\operatorname{softmax}(a)_i
\]
\[
e(w)=\left(\sum_{i=1}^{H}\tilde\alpha_i u_i\right)W_s
\]
\begin{itemize}
  \item $E^{(i)}\in\mathbb R^{B\times d}$; $W_1$ reduces to a 64-dimensional bottleneck in the experiments.
  \item The gate weights the buckets of the current token; the Transformer adds sequence context afterward.
\end{itemize}
\papersource{Paper, Secs. 3.1--3.2 and 4.1; PDF pp. 3, 5.}
\end{frame}

\begin{frame}{A soft cascade predicts the next token's coordinates}
\[
c^{(1)}=h_t,\qquad o^{(i)}=E^{(i)}c^{(i)},\qquad
p^{(i)}=\operatorname{softmax}(o^{(i)})
\]
\[
\bar e^{(i)}=E^{(i)\top}p^{(i)},\qquad
c^{(i+1)}=c^{(i)}+W_{\rm up}^{(i)\top}
\sigma\!\left(W_{\rm dn}^{(i)\top}
\begin{bmatrix}c^{(i)}\\\bar e^{(i)}\end{bmatrix}\right)
\]
\begin{itemize}
  \item Output projections reuse the input bucket embeddings.
  \item The next head receives a differentiable expectation over buckets, not a sampled ID.
\end{itemize}
\papersource{Paper, Sec. 3.3; PDF pp. 3--4.}
\end{frame}

\begin{frame}{Inference removes probability mass on invalid signatures}
\begin{block}{Training objective: target coordinates}
\[P_{\rm train}(w\mid c)=\prod_{i=1}^{H}p^{(i)}_{\mathcal H_i(w)},
\qquad \mathcal L=-\sum_i\log p^{(i)}_{\mathcal H_i(w)}.\]
\end{block}
\begin{block}{Inference distribution: registered tokens only}
\[P_{\rm inf}(w\mid c)=
\frac{\prod_i p^{(i)}_{\mathcal H_i(w)}}
{\sum_{w'\in\mathcal V_{\rm actl}}\prod_i p^{(i)}_{\mathcal H_i(w')}}.\]
\end{block}
\papersource{Paper, Sec. 3.4; PDF p. 4. The denominator scans the actual vocabulary.}
\end{frame}

\begin{frame}{The comparison includes a parameter-matched baseline}
\centering
\begin{tabular}{lrr}
\toprule
1B-scale model & Embedding & Decoder \\
\midrule
Standard & 65M & 0.9B \\
Standard+2L & 65M & 1.0B \\
MHF H3B10K & 65M & 0.9B \\
MHF H4B16K & 138M & 0.9B \\
\bottomrule
\end{tabular}
\vspace{3mm}
\begin{itemize}
  \item H3B10K directly matches Standard's reported parameter counts.
  \item H4B16K uses more embedding parameters; Standard+2L spends its extra budget on backbone depth.
\end{itemize}
\papersource{Paper, Sec. 4.1 and Table 1; PDF pp. 5--6. Counts are rounded as reported.}
\end{frame}

\begin{frame}{The study tests scaling, semantics, and adaptation}
\begin{tabular}{p{0.21\linewidth}p{0.70\linewidth}}
\toprule
Question & Protocol \\
\midrule
Core abilities & 10 tasks: LAMBADA, five commonsense tasks, three reading tasks, ReCoRD. \\
Rare words & Card-660: human similarity versus hidden-state cosine similarity. \\
Vocabulary growth & 6B Arabic/Chinese/Hindi + 2B English continual-training tokens; 32,768 to 48,122 tokens. \\
\bottomrule
\end{tabular}
\vspace{2mm}
\begin{itemize}
  \item 100M models train on 10B English tokens; 1B/3B models on 100B.
  \item Core benchmarks mix normalized accuracy, accuracy, and ReCoRD F1; 0-shot and 5-shot settings are specified per task.
\end{itemize}
\papersource{Paper, Secs. 4.2--4.4; PDF pp. 5--6.}
\end{frame}

\begin{frame}{MHF lifts LAMBADA accuracy at 1B and 3B}
\centering
\begin{tabular}{llr}
\toprule
Scale & Model & LAMBADA accuracy (\%) \\
\midrule
1B & Standard & 30.41 \\
1B & MHF H4B16K & \textbf{35.34} \\
\midrule
3B & Standard & 28.64 \\
3B & MHF H4B16K & \textbf{37.26} \\
\bottomrule
\end{tabular}
\vspace{4mm}
\begin{block}{Absolute score difference}
$+4.93$ percentage points at 1B; $+8.62$ percentage points at 3B.
\end{block}
\papersource{Paper, Sec. 5.1 and Table 1; PDF p. 6.}
\end{frame}

\begin{frame}{The 1B gains are broad, with an OBQA reversal}
\centering
\begin{tabular}{lrrrr}
\toprule
Model & ARC-E & COPA & OBQA & ReCoRD \\
\midrule
Standard & 59.85 & 64.00 & \textbf{31.60} & 63.20 \\
Standard+2L & 60.23 & 65.00 & 31.20 & 63.78 \\
MHF H3B10K & 61.95 & 65.00 & \textbf{34.00} & 64.51 \\
MHF H4B16K & \textbf{64.35} & \textbf{71.00} & 29.80 & \textbf{64.90} \\
\bottomrule
\end{tabular}
\vspace{3mm}
\begin{itemize}
  \item H4B16K beats Standard on 9/10 main tasks; OBQA is the exception.
  \item H3B10K beats Standard on 8/10 under equal reported parameters.
\end{itemize}
\papersource{Paper, Table 1; PDF p. 6. ReCoRD is F1; ARC-E and OBQA use normalized accuracy.}
\end{frame}

\begin{frame}{At 100M, deeper backbones often do better}
\centering
\begin{tabular}{lrr}
\toprule
Model & Main run LAMBADA & 3-seed mean (SD) \\
\midrule
Standard & 15.33 & 15.18 (0.4) \\
Standard+4L & \textbf{19.64} & \textbf{19.19 (0.6)} \\
MHF H3B10K & 16.40 & 16.67 (0.5) \\
MHF H4B16K & 18.79 & 18.80 (0.4) \\
\bottomrule
\end{tabular}
\vspace{3mm}
\begin{itemize}
  \item The authors attribute this to limited contextual disambiguation and representation capacity in small decoders.
  \item The three-seed appendix preserves the same LAMBADA ordering.
\end{itemize}
\papersource{Paper, Sec. 5.1/Table 1; PDF pp. 6--7. Appendix F/Table 10; PDF p. 18.}
\end{frame}

\begin{frame}{Rare-word similarity improves in the penultimate layer}
\centering
\begin{tabular}{lrr}
\toprule
1B model & Pearson $r$ & Spearman $\rho$ \\
\midrule
Standard & 0.26 & 0.25 \\
Standard+2L & 0.29 & 0.29 \\
MHF H3B10K & \textbf{0.32} & 0.29 \\
MHF H4B16K & 0.30 & \textbf{0.32} \\
\bottomrule
\end{tabular}
\vspace{3mm}
\begin{itemize}
  \item Card-660 correlates model cosine similarities with human ratings of rare-word pairs.
  \item These are representation correlations, not rare-word generation accuracy.
\end{itemize}
\papersource{Paper, Sec. 5.2 and Table 2; PDF p. 7. Penultimate hidden state shown.}
\end{frame}

\begin{frame}{Vocabulary growth adds zero MHF model parameters}
\centering
\begin{tabular}{lcc}
\toprule
 & Standard & MHF \\
\midrule
Vocabulary & 32,768 $\to$ 48,122 & 32,768 $\to$ 48,122 \\
Added embedding parameters & about 31M & \textbf{0M} \\
\bottomrule
\end{tabular}
\vspace{4mm}
\begin{block}{Observed post-adaptation outcomes}
H4B16K exceeds Standard on 12/22 tasks at 1B and 13/22 at 3B across Arabic, Hindi, Chinese, and English.
\end{block}
\papersource{Paper, Secs. 4.4, 5.3 and Fig. 2; PDF pp. 5--8. Both systems receive continual pre-training.}
\end{frame}

\begin{frame}{Added-language outcomes vary by task}
\centering
\includegraphics[width=0.99\linewidth,height=0.66\textheight,keepaspectratio]{figures/fig-02-added-languages.png}
\vspace{1mm}
\begin{itemize}
  \item Arabic, Hindi, and Chinese each contribute four tasks to Figure 2.
  \item Red/orange circles: Standard; blue squares: MHF. Darker shades denote 1B models.
\end{itemize}
\papersource{Paper, Fig. 2 and Sec. 5.3; PDF pp. 7--8. Cropped original panels; metrics vary by task.}
\end{frame}

\begin{frame}{English capabilities remain competitive after adaptation}
\centering
\includegraphics[width=0.94\linewidth,height=0.58\textheight,keepaspectratio]{figures/fig-02-english.png}
\vspace{1mm}
\begin{itemize}
  \item Authors count H3B10K and H4B16K as better or comparable on 6/10 and 9/10 English tasks, respectively.
  \item Red/orange circles: Standard; blue squares: MHF. Darker shades denote 1B models.
\end{itemize}
\papersource{Paper, Fig. 2 and Sec. 5.3; PDF pp. 7--8. Cropped original English panel.}
\end{frame}

\begin{frame}{Four IDs beat one under matched parameter counts}
\centering
\begin{tabular}{lrrr}
\toprule
LAMBADA accuracy (\%) & B4K & B8K & B16K \\
\midrule
H1: one ID & 4.29 & 8.77 & 14.30 \\
H4: four IDs & \textbf{30.27} & \textbf{30.47} & \textbf{30.91} \\
\bottomrule
\end{tabular}
\vspace{4mm}
\begin{itemize}
  \item H1 repeats its single value across four coordinates to match H4's parameters.
  \item Adding buckets alone leaves the token-identity ambiguity unresolved.
\end{itemize}
\papersource{Paper, Sec. 6 and Table 3; PDF p. 8. Ablation uses 1B models trained up to 20B tokens.}
\end{frame}

\begin{frame}{More buckets bring diminishing returns}
\centering
\includegraphics[width=0.83\linewidth,height=0.68\textheight,keepaspectratio]{figures/fig-03-hash-ablation.png}
\begin{itemize}
  \item Gray bars: embedding parameters; colored lines: mean task-group scores.
  \item H4B16K is the authors' chosen parameter/performance compromise.
\end{itemize}
\papersource{Paper, Fig. 3 and Sec. 6; PDF p. 8. 1B, approximately 20B training tokens.}
\end{frame}

\begin{frame}{Character-trigram LSH adds no consistent gain}
\centering
\begin{tabular}{lccccc}
\toprule
LSH : MMH3 heads & 0:4 & 1:3 & 2:2 & 3:1 & 4:0 \\
\midrule
LAMBADA accuracy (\%) & 30.91 & 30.60 & 30.80 & 31.40 & 31.36 \\
\bottomrule
\end{tabular}
\vspace{5mm}
\begin{itemize}
  \item The four-head configuration is held fixed while MMH3 heads are replaced.
  \item Performance does not rise monotonically with morphological hashing.
\end{itemize}
\papersource{Paper, Sec. 6 and Table 4; PDF p. 8. 1B analysis setting.}
\end{frame}

\begin{frame}{The rank argument is suggestive, not a measured cause}
\begin{itemize}
  \item A standard LM's logit matrix factors through dimension $d$, so its rank is at most $d$.
  \item MHF sums multiple coordinate-specific terms $C_iE_i^\top\Pi_i$; distinct terms may increase the attainable rank.
  \item Appendix A's printed $\min(B,Hd)$ bound does not follow directly from that sum with different $\Pi_i$.
  \item The main 1B/3B configurations have one training seed each; Figure 2 shows mixed task-level expansion results.
\end{itemize}
\papersource{Paper, Appendix A; PDF pp. 14--15. Limitations, PDF p. 9; Fig. 2, PDF p. 7.}
\end{frame}

\begin{frame}{Take-home messages}
\begin{enumerate}
  \item \alert{Full signatures solve token identity}: shared individual buckets can still support deterministic token recovery.
  \item \alert{The interface is useful at the tested larger scales}: 1B/3B improve many tasks; 100M is less favorable.
  \item \alert{Vocabulary parameters stay fixed under this expansion}: inference still scores registered tokens, and the unique-signature map must be maintained.
\end{enumerate}
\papersource{Paper, Secs. 3, 5, 6 and Appendix H; PDF pp. 3--8, 17.}
\end{frame}

\begin{frame}{Paper and primary evidence}
\begin{itemize}
  \item Xue, Yamaguchi, and Aletras. \emph{MultiHashFormer: Hash-based Generative Language Models}. EMNLP 2026; \href{https://arxiv.org/abs/2606.28057v2}{arXiv:2606.28057v2}.
  \item Architecture: Fig. 1, PDF p. 2. Main benchmarks: Table 1, PDF p. 6.
  \item Rare words and vocabulary expansion: Table 2/Fig. 2, PDF p. 7.
  \item Ablations: Tables 3--4/Fig. 3, PDF p. 8. Theory and extra checks: Appendix A/F/H, PDF pp. 14--18.
\end{itemize}
\end{frame}

\appendix

\begin{frame}{Backup: what rank bound follows from the decomposition?}
\[
S=\sum_{i=1}^{H}C_iE_i^\top\Pi_i,
\qquad C_i\in\mathbb R^{N\times d},\quad
E_i^\top\Pi_i\in\mathbb R^{d\times |\mathcal V|}.
\]
\[
\operatorname{rank}(S)\leq
\min\{N,|\mathcal V|,H\min(d,B)\}.
\]
\begin{itemize}
  \item Different $\Pi_i$ allow the terms to occupy different token subspaces.
  \item Row-wise softmax normalization adds a row-constant term; an upper bound alone does not show the attained rank.
\end{itemize}
\papersource{Reader's algebra from Paper, Appendix A.2; PDF p. 15.}
\end{frame}

\begin{frame}{Backup: decoding cost and source-text discrepancies}
\begin{itemize}
  \item Valid-token normalization scans $|\mathcal V_{\rm actl}|$ signatures. On one MI300X, Appendix H reports 97.38 ms/sample for 1B H4B16K versus 96.42 for Standard on LAMBADA.
  \item Sec. 5.1 attributes COPA 71.00 to H3B10K; Table 1 assigns it to H4B16K. The deck uses Table 1.
  \item Sec. 4.1 gives H3B10K $B=10{,}240$; Appendix E Table 9 gives $10{,}624$.
  \item Appendix G says the T-FREE comparison uses 20B tokens; Table 11's caption says 12B.
\end{itemize}
\papersource{Paper, Table 1/PDF p. 6; Sec. 4.1/PDF p. 5; Appendix E/G/H/PDF pp. 17--18.}
\end{frame}

\end{document}