\documentclass[aspectratio=169,11pt]{beamer}
\usepackage[T1]{fontenc}
\usepackage[utf8]{inputenc}
\usepackage{amsmath,amssymb,graphicx,booktabs}
\definecolor{AcademicBlue}{RGB}{26,57,86}
\definecolor{AcademicTeal}{RGB}{0,105,112}
\setbeamercolor{structure}{fg=AcademicBlue}
\setbeamercolor{frametitle}{fg=AcademicBlue,bg=white}
\setbeamercolor{title}{fg=AcademicBlue}
\setbeamercolor{alerted text}{fg=AcademicTeal}
\setbeamercolor{block title}{fg=white,bg=AcademicBlue}
\setbeamercolor{block body}{fg=black,bg=AcademicBlue!5}
\setbeamertemplate{navigation symbols}{}
\setbeamertemplate{itemize items}[circle]
\setbeamertemplate{footline}{\hfill\usebeamerfont{page number in head/foot}\insertframenumber\,/\,\inserttotalframenumber\hspace{4mm}\vspace{3mm}}
\setbeamersize{text margin left=8mm,text margin right=8mm}
\setbeamerfont{frametitle}{size=\Large,series=\bfseries}
\setbeamerfont{title}{size=\LARGE,series=\bfseries}
\setbeamertemplate{frametitle}{\vspace{3mm}\begin{beamercolorbox}[wd=\textwidth,sep=0pt]{frametitle}\usebeamerfont{frametitle}\strut\insertframetitle\strut\par\end{beamercolorbox}\vspace{1mm}}
\newcommand{\papersource}[1]{\par\smallskip{\scriptsize\color{gray}#1\par}}
\newcommand{\tc}{t_{\mathrm{canon}}}
\title[Tokenization robustness]{Language Models are not Equally Robust to Non-Canonical Tokenization across Languages}
\subtitle{Research group discussion \enspace$\cdot$\enspace 25--30 minutes}
\author{Paper by Poulami Ghosh and Preethi Jyothi\\IIT Bombay, India}
\date{arXiv:2607.26831v1 \enspace$\cdot$\enspace Reading: 13 November 2026}
\begin{document}

\begin{frame}
\titlepage
\end{frame}

\begin{frame}{The same string can trigger different multilingual behavior}
\begin{block}{Question}
Does a model preserve its task performance when the input string stays identical but its valid token segmentation changes?
\end{block}
\begin{itemize}
\item Study scope: 27 languages, six generation-based tasks, three model families.
\item English robustness does not generalize uniformly across languages.
\item English-only multi-tokenization training improves multilingual ARC accuracy.
\end{itemize}
\medskip
\href{https://arxiv.org/abs/2607.26831v1}{arXiv v1, 29 July 2026}.\\
Publication not found as of 2026-11-13.\\
Public code not found as of 2026-11-13.
\papersource{Paper, Secs. 1, 5--7; Tables 1 and 3, PDF pp. 1--2, 5--11. Status checked for this reading.}
\end{frame}

\begin{frame}{Token boundaries change the model's computation}
\begin{center}
\begin{tabular}{ll}
\toprule
Surface text & \texttt{abc}\\
\midrule
Possible token sequence 1 & \texttt{[abc]}\\
Possible token sequence 2 & \texttt{[ab] [c]}\\
Possible token sequence 3 & \texttt{[a] [b] [c]}\\
\bottomrule
\end{tabular}
\end{center}
\begin{itemize}
\item All sequences must use existing vocabulary entries and decode identically.
\item Embedding lookup, positions, and attention computations still change.
\item Tokenizer pruning or replacement can expose a model to unfamiliar segmentations.
\end{itemize}
\papersource{Teaching example with an assumed toy vocabulary; paper motivation in Secs. 1--2, PDF pp. 1--3.}
\end{frame}

\begin{frame}{Valid tokenizations define the perturbation space}
\[
\tc=T_{\mathrm{canon}}(x),\qquad
\mathcal T(x)=\{t\in V^*: \mathrm{decode}(t)=x\}.
\]
\[
f(t_1)\approx f(t_2),\qquad t_1,t_2\in\mathcal T(x)
\]
\begin{itemize}
\item $x$: a string; $V$: the original tokenizer vocabulary.
\item $\tc$: deterministic canonical segmentation; all alternatives preserve $x$.
\item $f$: model logits, representations, or task outputs.
\item The paper evaluates behavior through task scores and retrieval accuracy.
\end{itemize}
\papersource{Paper, Sec. 3, PDF pp. 3--4. Non-canonical tokenizations additionally require $t\ne\tc$.}
\end{frame}

\begin{frame}{Uniform paths require weighted edge sampling}
An MDD links string positions with valid token edges. Let $C(i)$ count suffix paths:
\[
C(m)=1,\qquad C(i)=\sum_{(i,j)\in E}C(j),\qquad
P(i\to j)=\frac{C(j)}{C(i)}.
\]
\begin{center}
\begin{tabular}{lrr}
\toprule
First token for \texttt{abc} & Valid completions & Edge probability\\
\midrule
\texttt{a} & 2: \texttt{[b,c]}, \texttt{[bc]} & $1/2$\\
\texttt{ab} & 1: \texttt{[c]} & $1/4$\\
\texttt{abc} & 1: empty suffix & $1/4$\\
\bottomrule
\end{tabular}
\end{center}
Each complete tokenization has probability $1/4$. Uniform outgoing edges would bias the path distribution.
\papersource{Teaching derivation of the paper's MDD sampling description; Sec. 3, PDF p. 4; original MDD examples in Fig. 2, PDF p. 5.}
\end{frame}

\begin{frame}{Perturb the user prompt while preserving its text}
\begin{center}
\begin{tabular}{lp{8.5cm}}
\toprule
Condition & Input representation\\
\midrule
Canonical & Default tokenizer output\\
Random & Uniformly sampled valid alternative segmentation; five seeds averaged\\
Character & Finest character-level condition used by the authors\\
\bottomrule
\end{tabular}
\end{center}
\begin{itemize}
\item Retokenize instructions, question content, and multiple-choice options.
\item Preserve the system prompt and special tokens.
\item The character unit and treatment of the canonical path need implementation clarification.
\end{itemize}
\papersource{Paper, Secs. 3--4, PDF pp. 3--5. Sampling formula uses $\mathcal U(\mathcal T(x))$ although the definition excludes $\tc$.}
\end{frame}

\begin{frame}{Three model families, six tasks, 27 selected languages}
\begin{itemize}
\item Llama-3.1-8B-Instruct, Qwen3-8B, gemma-3-12b-it.
\item Short answer: MGSM and MLQA.
\item Multiple choice: Multi-ARC, Multi-HellaSwag, Global-MMLU-Lite, Belebele.
\item Direct generation with lm-evaluation-harness; MC instructions require answer letters only.
\end{itemize}
\begin{block}{Scope of the comparison}
The study covers 27 selected languages. Exact task-by-model language coverage is not fully tabulated; the reported averages use the paper's aggregation.
\end{block}
\papersource{Paper, Sec. 4, PDF pp. 4--5; task details and language list in Tables 8--9, PDF p. 15.}
\end{frame}

\begin{frame}{Perturbation ranges differ across languages}
\[
\mathrm{FR}=\frac{\text{tokens}}{\text{words}},\qquad
\mathrm{LR}(t,x)=\frac{|t|}{|\tc|}
\]
\begin{center}
\begin{tabular}{lrr}
\toprule
Qwen tokenizer / language & Canonical FR & Character FR\\
\midrule
English & 1.35 & 5.16\\
Bengali & 6.66 & 6.68\\
Gujarati & 9.29 & 9.29\\
Telugu & 13.55 & 13.55\\
Chinese & 1.13 & 1.85\\
\bottomrule
\end{tabular}
\end{center}
A small score change can reflect a small perturbation space. Several near-character languages are analyzed separately in the appendix.
\papersource{Paper, Sec. 4 and Fig. 3(b), PDF pp. 4--5, 7; App. C, PDF pp. 13--14. FR values transcribed from Fig. 3.}
\end{frame}

\begin{frame}{Model robustness ranking changes with perturbation type}
\begin{center}
\begin{tabular}{lrrr}
\toprule
Model & Canonical score $\uparrow$ & Random drop $\downarrow$ & Character drop $\downarrow$\\
\midrule
Llama & 63.918 & 23.772\% & 40.960\%\\
Qwen & 64.748 & 11.391\% & 16.091\%\\
Gemma & 74.375 & 9.937\% & 20.928\%\\
\bottomrule
\end{tabular}
\end{center}
\begin{itemize}
\item Canonical scores and relative drops are averaged over the six tasks.
\item Gemma has the lowest random drop; Qwen has the lowest character drop.
\item Llama is most sensitive under both perturbations.
\end{itemize}
\papersource{Paper, Table 1, PDF p. 6. Values are relative losses, not percentage-point losses. Language coverage follows the paper.}
\end{frame}

\begin{frame}{Task type alone does not determine sensitivity}
\begin{center}
\begin{tabular}{lrrr}
\toprule
Random relative drop (\%) & Llama & Qwen & Gemma\\
\midrule
MGSM & 27.522 & 19.948 & 7.890\\
MLQA & 24.446 & 12.002 & 14.708\\
Global-MMLU-Lite & 20.371 & 8.957 & 9.766\\
Multi-ARC & 24.320 & 8.542 & 9.865\\
Multi-HellaSwag & 25.754 & 12.202 & 10.012\\
Belebele & 20.221 & 6.692 & 7.382\\
\bottomrule
\end{tabular}
\end{center}
Character perturbations cause larger drops in every model--task cell.\\
Short-answer tasks are often sensitive, but the pattern is not universal: Gemma's MGSM drop is below several MC drops.
\papersource{Paper, Sec. 5.1 and Table 1, PDF pp. 5--6. Task scores are not assumed to share one metric definition.}
\end{frame}

\begin{frame}{Llama degrades as tokenization becomes more granular}
\centering
\includegraphics[width=0.98\linewidth,height=0.65\textheight,keepaspectratio]{figures/fig-04a-arc-llama.pdf}\par
\raggedright
\medskip
The x-axis is sampled/canonical token count; colors identify languages. English is comparatively stable while several other languages decline.
\papersource{Original Fig. 4(a), Multilingual-ARC, PDF p. 7. Canonical is a separate reference point.}
\end{frame}

\begin{frame}{Nearly equal sequence lengths still leave performance gaps}
\begin{center}
\begin{tabular}{lrrr}
\toprule
Language & Canonical & LR .95--1.00 & LR 1.00--1.05\\
\midrule
Arabic & 78.30 & 74.73 & 73.00\\
English & 88.00 & -- & 87.90\\
Basque & 45.60 & 40.75 & 41.10\\
Hungarian & 78.90 & 73.33 & 71.10\\
French & 80.90 & -- & 81.30\\
Chinese & 86.90 & 87.38 & 86.69\\
\bottomrule
\end{tabular}
\end{center}
\begin{itemize}
\item Longer sequences alone do not explain the Arabic and Hungarian gaps.
\item English and Chinese are nearly unchanged; French slightly improves.
\item Table 6 does not explicitly identify the evaluated model.
\end{itemize}
\papersource{Paper, Sec. 5.2, PDF p. 8; Table 6, Multilingual-ARC, PDF p. 14.}
\end{frame}

\begin{frame}{Canonical fragmentation predicts additional vulnerability}
\[
\mathrm{logit}\,P(Y_r=1)=\beta_0+\beta_cY_c+\beta_f\mathrm{FR}_c
\]
\begin{center}
\begin{tabular}{lrr}
\toprule
Language & Canonical-correct coefficient & Fragmentation coefficient\\
\midrule
English & 3.416 & $-1.114$\\
Basque & 2.211 & $-0.138$\\
Italian & 2.692 & $-0.146$\\
Chinese & 4.718 & $-0.967$\\
\bottomrule
\end{tabular}
\end{center}
$Y_c,Y_r$: correct/incorrect under canonical and random inputs.\\
English FR: $p=.021$, controlling for canonical correctness.\\
This is an association; other languages' significance values are not tabulated.
\papersource{Teaching regression notation; paper, Sec. 5.1, PDF p. 6; Table 10, PDF p. 15.}
\end{frame}

\begin{frame}{Scaling improves robustness on the short-answer tasks}
\begin{center}
\begin{tabular}{llrrr}
\toprule
Task & Metric & Qwen 4B & Qwen 8B & Qwen 14B\\
\midrule
MLQA & Canonical score & 47.564 & 58.050 & 65.329\\
 & Random drop (\%) & 14.648 & 12.002 & 10.841\\
\midrule
MGSM & Canonical score & 44.267 & 45.911 & 45.289\\
 & Random drop (\%) & 21.085 & 19.948 & 16.208\\
\bottomrule
\end{tabular}
\end{center}
\begin{itemize}
\item MGSM baseline scores stay similar while perturbation loss falls.
\item Qwen 4B fails answer-format instructions on MC tasks.
\item Its negative MC drops cannot be read as clean reasoning improvements.
\end{itemize}
\papersource{Paper, Sec. 5.3 and Table 2, PDF pp. 8--9.}
\end{frame}

\begin{frame}{Tokenization also changes cross-lingual representations}
\begin{columns}[T]
\begin{column}{0.49\textwidth}
\centering Canonical inputs\\[1mm]
\includegraphics[width=\linewidth,height=0.46\textheight,keepaspectratio]{figures/fig-05a-retrieval-canonical.pdf}
\end{column}
\begin{column}{0.49\textwidth}
\centering Character inputs\\[1mm]
\includegraphics[width=\linewidth,height=0.46\textheight,keepaspectratio]{figures/fig-05c-retrieval-character.pdf}
\end{column}
\end{columns}
\medskip
\begin{itemize}
\item FLORES-200 sentence retrieval, cosine similarity, top-1 accuracy.
\item x-axis: layer; y-axis: acc@1. Axis maxima differ: 1.0 vs. 0.8.
\item A last-token representation probe shows changes beyond answer formatting.
\end{itemize}
\papersource{Original Fig. 5(a,c), Llama, PDF p. 8; method in Sec. 5.4, PDF pp. 8--9. Both sides use the stated tokenization.}
\end{frame}

\begin{frame}{Train on diverse tokenizations of the same text}
\begin{center}
\begin{tabular}{lp{9.5cm}}
\toprule
Scheme & Composition of ten copies\\
\midrule
Canonical & 10 copies of the same canonical segmentation\\
Mixed & 1 canonical + 1 character + 8 random alternatives\\
LR-Bucketed & 1 canonical + 1 character + 1 alternative from each of 8 length-ratio buckets\\
\bottomrule
\end{tabular}
\end{center}
\[
[1,1.25),\ [1.25,1.5),\ \ldots,\ [2.75,3.0]
\]
\begin{itemize}
\item Repeat the ten-copy pattern for $N=20$ or $30$.
\item Canonical($N\times$) controls for repeated exposure.
\item Bucketing spreads the alternative inputs across granularity levels.
\end{itemize}
\papersource{Paper, Sec. 6.1, PDF pp. 9--10. One-copy canonical, random, and character schemes are also evaluated.}
\end{frame}

\begin{frame}{English-only adaptation is evaluated across languages}
\begin{columns}[T]
\begin{column}{0.48\textwidth}
\begin{block}{Multilingual-ARC training}
1116 English training examples.\\
298 original validation examples.\\
894 validation inputs: three forms per example.
\end{block}
\end{column}
\begin{column}{0.48\textwidth}
\begin{itemize}
\item LoRA rank 16; alpha 32; dropout .1.
\item Targets: query, key, value projections.
\item Five epochs; maximum length 2048.
\item Learning rate $2.5\times10^{-6}$.
\end{itemize}
\end{column}
\end{columns}
\medskip
Table 3 matches Qwen3-8B's ARC baseline numerically, but the adapted model is not explicitly named there. Other tasks are not used to test augmentation transfer.
\papersource{Paper, Secs. 6.2--7, PDF pp. 10--11; Table 7, PDF p. 14. Model identification is inferred from Tables 1 and 3.}
\end{frame}

\begin{frame}{Bucketed training beats repeated canonical data}
\begin{center}
\begin{tabular}{lrrr}
\toprule
Training scheme & Canonical & Random & Character\\
\midrule
No fine-tuning & 78.3 & 71.8 & 66.0\\
Canonical 1$\times$ & 79.2 & 72.3 & 66.3\\
Canonical 30$\times$ & 81.2 & 73.7 & 67.9\\
Mixed 30$\times$ & 81.2 & 74.0 & 68.2\\
LR-Bucketed 30$\times$ & 81.8 & 74.8 & 69.2\\
\bottomrule
\end{tabular}
\end{center}
\begin{itemize}
\item Values: average multilingual ARC accuracy (\%).
\item Versus Canonical 30$\times$: +0.6 / +1.1 / +1.3 percentage points.
\item Repeating canonical data already helps; diversity adds a further gain.
\end{itemize}
\papersource{Paper, Table 3, PDF p. 10. Equal copy counts do not imply equal processed-token or compute budgets.}
\end{frame}

\begin{frame}{Higher perturbed accuracy need not close the relative gap}
\begin{center}
\begin{tabular}{lrr}
\toprule
Training scheme & Random drop (\%) $\downarrow$ & Character drop (\%) $\downarrow$\\
\midrule
No fine-tuning & 8.542 & 15.963\\
Canonical 30$\times$ & 9.405 & 16.490\\
Mixed 30$\times$ & 8.987 & 16.141\\
LR-Bucketed 30$\times$ & 8.637 & 15.548\\
\bottomrule
\end{tabular}
\end{center}
\begin{block}{Interpretation}
Bucketed training improves absolute accuracy and preserves robustness better than repeated canonical training. Its random relative gap is still slightly above the unadapted baseline.
\end{block}
\papersource{Paper, Table 3, PDF p. 10. Relative-drop columns are averages across languages.}
\end{frame}

\begin{frame}{The evidence distinguishes three effects}
\begin{enumerate}
\item \textbf{Segmentation}: equal decoded text still gives different task behavior.
\item \textbf{Length}: near-equal-length perturbations retain gaps in several languages.
\item \textbf{Representation}: sentence retrieval changes without generating an answer.
\end{enumerate}
\medskip
\begin{block}{Limits that affect interpretation}
Language coverage and perturbation strength differ; training gains are tested only on ARC. Vocabulary allocation, resource level, and training exposure are not independently manipulated.
\end{block}
\papersource{Paper, Secs. 5--7 and Limitations, PDF pp. 5--11; Tables 4--6, PDF pp. 13--14.}
\end{frame}

\begin{frame}{Evaluate the tokenizer interface across languages}
\begin{itemize}
\item Robustness observed in English is not a default multilingual property.
\item Report the language, perturbation type, and feasible tokenization range.
\item Use repeated canonical training as the augmentation baseline.
\item Track perturbed accuracy and the relative canonical gap separately.
\end{itemize}
\begin{block}{Discussion question}
Can diverse valid inputs improve robustness at a matched token/compute budget, and can the gains transfer beyond multilingual ARC?
\end{block}
\papersource{Takeaways based on Tables 1--3 and the paper's stated limitations, PDF pp. 6, 9--11.}
\end{frame}

\begin{frame}{References and paper status}
\begin{itemize}
\item Poulami Ghosh and Preethi Jyothi. \emph{Language Models are not Equally Robust to Non-Canonical Tokenization across Languages}.
\item \href{https://arxiv.org/abs/2607.26831v1}{arXiv:2607.26831v1}, submitted 29 July 2026.
\item \href{https://arxiv.org/pdf/2607.26831v1}{Versioned paper PDF} and \href{https://arxiv.org/src/2607.26831v1}{author TeX source}.
\end{itemize}
\medskip
Publication not found as of 2026-11-13.\\
Public code not found as of 2026-11-13.\\[3mm]
Companion analysis: \texttt{reading-notes.zh.md}.\\
Original figure assets and a local pdfLaTeX build helper accompany the deck.
\papersource{Publication/code searches were refreshed for this reading. This presentation discusses the arXiv v1 paper.}
\end{frame}

\appendix
\begin{frame}{Backup: small tokenization spaces limit perturbations}
\begin{center}
\begin{tabular}{lrrrr}
\toprule
 & \multicolumn{2}{c}{Log$_{10}$ tokenization count} & \multicolumn{2}{c}{ARC: Canonical $\to$ Random}\\
Language & Qwen & Gemma & Qwen & Gemma\\
\midrule
Bengali & 3.53 & 64.45 & 57.70$\to$52.64 & 65.61$\to$53.80\\
Gujarati & 1.12 & 59.73 & 54.30$\to$52.64 & 60.78$\to$56.62\\
Telugu & 17.07 & 56.19 & 42.70$\to$41.00 & 44.82$\to$40.32\\
\bottomrule
\end{tabular}
\end{center}
\begin{itemize}
\item Qwen's canonical tokenization is already near character level for these languages.
\item Armenian often has only one valid Qwen tokenization and is omitted.
\item Low sensitivity can reflect little room to change the segmentation.
\end{itemize}
\papersource{Paper, App. C, Tables 4--5, PDF pp. 13--14. Counts are on a log scale; no exponentiation of an averaged log is used.}
\end{frame}

\begin{frame}{Backup: average relative loss has its own denominator}
For $L$ languages, let $a_{c,\ell}$ and $a_{r,\ell}$ be canonical and random scores.
\[
\Delta_{r,\ell}=100\frac{a_{c,\ell}-a_{r,\ell}}{a_{c,\ell}}
\]
\[
\frac1L\sum_\ell\Delta_{r,\ell}
\ne100\frac{\bar a_c-\bar a_r}{\bar a_c}
\]
\begin{itemize}
\item Table 3 baseline: mean accuracy 78.3 canonical, 71.8 random.
\item Their ratio gives about 8.30\%; the reported mean relative loss is 8.542\%.
\item Do not reconstruct exact random scores from averaged canonical scores and averaged relative losses.
\end{itemize}
\papersource{Teaching explanation of aggregation; paper, Tables 1 and 3, PDF pp. 6, 10.}
\end{frame}

\begin{frame}{Backup: MGSM sensitivity also varies by language}
\centering
\includegraphics[width=0.98\linewidth,height=0.65\textheight,keepaspectratio]{figures/fig-08b-mgsm-qwen.pdf}\par
\raggedright
\medskip
Qwen: English remains comparatively stable, while Swahili and Thai decline strongly. Missing high-ratio points are not zero scores.
\papersource{Original Fig. 8(b), PDF p. 18. x-axis: sampled/canonical length ratio; y-axis: task performance.}
\end{frame}

\begin{frame}{Backup: control exposure before testing diversity}
\begin{center}
\begin{tabular}{lrrr}
\toprule
Training scheme & Canonical & Random & Character\\
\midrule
Canonical 10$\times$ & 81.1 & 73.6 & 67.7\\
Mixed 10$\times$ & 81.1 & 73.7 & 67.9\\
LR-Bucketed 10$\times$ & 81.1 & 73.8 & 68.0\\
Mixed 20$\times$ & 81.5 & 74.6 & 68.7\\
Mixed 30$\times$ & 81.2 & 74.0 & 68.2\\
LR-Bucketed 30$\times$ & 81.8 & 74.8 & 69.2\\
\bottomrule
\end{tabular}
\end{center}
\begin{itemize}
\item Canonical copies increase exposure without adding segmentation diversity.
\item Mixed 20$\times\to$30$\times$ is not monotonic; more copies alone do not guarantee a gain.
\item Five epochs over larger datasets also means more optimization steps.
\end{itemize}
\papersource{Paper, Table 3, PDF p. 10; interpretation distinguishes exposure from tokenization diversity.}
\end{frame}

\end{document}