% TIDE paper discussion. Compile with pdflatex, shell escape disabled.
% Figure crops use author-supplied vector PDFs; no experimental curves are redrawn.
\documentclass[aspectratio=169,11pt]{beamer}
\usepackage[T1]{fontenc}
\usepackage[utf8]{inputenc}
\usepackage{amsmath,amssymb,graphicx,booktabs,lmodern}
\definecolor{AcademicBlue}{RGB}{26,57,86}
\definecolor{AcademicTeal}{RGB}{0,105,112}
\setbeamercolor{structure}{fg=AcademicBlue}
\setbeamercolor{frametitle}{fg=AcademicBlue,bg=white}
\setbeamercolor{title}{fg=AcademicBlue}
\setbeamercolor{alerted text}{fg=AcademicTeal}
\setbeamercolor{block title}{fg=white,bg=AcademicBlue}
\setbeamercolor{block body}{fg=black,bg=AcademicBlue!5}
\setbeamertemplate{navigation symbols}{}
\setbeamertemplate{itemize items}[circle]
\setbeamertemplate{footline}{\hfill\usebeamerfont{page number in head/foot}\insertframenumber\,/\,\inserttotalframenumber\hspace{4mm}\vspace{3mm}}
\setbeamersize{text margin left=8mm,text margin right=8mm}
\setbeamerfont{frametitle}{size=\Large,series=\bfseries}
\setbeamerfont{title}{size=\LARGE,series=\bfseries}
\setbeamerfont{author}{size=\normalsize}
\newcommand{\papersource}[1]{\par\smallskip{\scriptsize\color{gray}#1\par}}
\newcommand{\RMS}{\operatorname{RMSNorm}}
\newcommand{\FFN}{\operatorname{FFN}}
\newcommand{\Attn}{\operatorname{Attn}}
\newcommand{\sm}{\operatorname{softmax}}
\newcommand{\E}{\mathbb{E}}
\newcommand{\norm}[1]{\left\lVert#1\right\rVert}
\title[TIDE]{TIDE: Every Layer Knows the Token\\Beneath the Context}
\subtitle{Token Identity Delivered Everywhere}
\author{Paper authors: Ajay Jaiswal, Lauren Hannah, Han-Byul Kim,\\Duc Hoang, Mehrdad Farajtabar, Minsik Cho (Apple)}
\date{Research discussion \textbullet\ arXiv:2605.06216v1\\Read on September 14, 2026}
\begin{document}

% 1 -- approx. 25 minutes for main slides 1--22.
\begin{frame}
\titlepage
\vspace{-5mm}
\centering\small A shared token memory supplies identity information at every depth.
\papersource{Independent reading slides based on the paper PDF and author TeX source.\\\href{https://arxiv.org/abs/2605.06216v1}{arxiv.org/abs/2605.06216v1}; all page locators use 1-based PDF pages.}
\end{frame}

% 2
\begin{frame}{TIDE gives the residual stream a direct identity input}
\begin{block}{Central idea}
Learn several static vectors for each token ID, then let every layer decide how much of this memory to add to its contextual computation.
\end{block}
\begin{itemize}
\item \textbf{Motivation:} rare tokens get fewer lookup updates; some token pairs become hard to distinguish at depth.
\item \textbf{Evidence:} lower validation loss, larger gains on rare tokens, and higher mean zero-shot scores.
\item \textbf{Tradeoff:} substantial extra embedding capacity, with relatively small measured decoding overhead.
\end{itemize}
\papersource{Paper, Sections 1, 3--4, PDF pp. 1--2, 5--12; Appendix I, p. 23.}
\end{frame}

% 3
\begin{frame}{The diagnosis concerns update sparsity and representation loss}
\begin{columns}[T]
\begin{column}{.48\textwidth}
\begin{block}{Rare-token update sparsity}
A lookup row is accessed only when its token appears in the input batch.
\end{block}
\[
\kappa_v=1-(1-f_v)^{BT}
\]
For rare $v$, $\kappa_v\approx f_vBT$.
\smallskip

Small $f_v$ means few opportunities to learn token-specific parameters.
\end{column}
\begin{column}{.48\textwidth}
\begin{block}{Contextual collapse}
A pair of different tokens can have similar contextual hidden states.
\end{block}
\[
\norm{h_u-h_v}\leq\delta
\]
A fixed-Lipschitz FFN then has limited output separation for that pair.
\end{column}
\end{columns}
\medskip
\small $f_v$: token probability; $B$: batch size; $T$: sequence length. The probability formula assumes independent positions. Norm diagnostics alone do not prove semantic failure.
\papersource{Paper, Section 2, PDF pp. 2--4; Appendix C, p. 18. Lookup-only gradient model; tied output embeddings require separate treatment.}
\end{frame}

% 4
\begin{frame}{One shared memory serves all layers}
\centering
\includegraphics[width=\linewidth,height=.65\textheight,keepaspectratio]{figures/main_diagram.pdf}
\medskip
\begin{itemize}
\item Memory lookup uses the \textbf{token ID}; routing uses the \textbf{current context}.
\item Memory and FFN outputs meet through residual addition.
\end{itemize}
\papersource{Paper, Fig. 3, PDF p. 5. Original author figure; the formal normalization is specified on the next slides.}
\end{frame}

% 5
\begin{frame}{Each memory bank learns its own token table}
\[
E_k\in\mathbb{R}^{|\mathcal V|\times d_b},\qquad
M_k(v)=\RMS(E_k[v]),\quad k=1,\ldots,K
\]
\[
\mathbf M=\operatorname{Stack}_k M_k(x)
\in\mathbb{R}^{B\times T\times K\times d_b}
\]
\begin{itemize}
\item The $K$ tables have separate parameters and are shared across depth.
\item Compute $\mathbf M$ once per forward pass; reuse it at all layers.
\item Values are context independent at fixed weights; they are learned during training.
\item The presented additive update needs $d_b=d$; no projection for $d_b\ne d$ is specified.
\end{itemize}
\papersource{Paper, Eqs. (3.3)--(3.4), PDF pp. 5--6. $|\mathcal V|$: vocabulary size; $d$: model width; $d_b$: memory width.}
\end{frame}

% 6
\begin{frame}{The router makes the injected mixture context dependent}
\begin{align*}
\tilde h^\ell &=h^{\ell-1}+\Attn(\RMS(h^{\ell-1})), &
\tilde n^\ell&=\RMS(\tilde h^\ell),\\
\alpha^\ell&=\sm(W_r^\ell\tilde n^\ell), &
W_r^\ell&\in\mathbb{R}^{(K+1)\times d},\\
m^\ell(v)&=\sum_{k=1}^{K+1}\alpha_k^\ell M_k(v), &
M_{K+1}(v)&=0.
\end{align*}
\begin{block}{The layer update}
\centering$\displaystyle h^\ell=\tilde h^\ell+\FFN(\tilde n^\ell)+m^\ell(v)$
\end{block}
Null-bank mass suppresses memory. The zero slot has no embedding table, but adds a router output row per layer.
\papersource{Paper, Eqs. (3.1)--(3.2), (3.5)--(3.6), PDF pp. 5--6. The memory values are static; the routed contribution $m^\ell(v)$ is contextual.}
\end{frame}

% 7
\begin{frame}{The gradient result is a conditional lower bound}
Assume that whenever $v$ appears, \emph{every} bank satisfies
\[
\norm{\nabla_{e_v^{(k)}}\mathcal L_s}^{2}\geq G_{\min}^{2}>0.
\]
Then summing over banks and steps gives
\[
\E\!\left[\sum_{s=1}^{\tau}\sum_{k=1}^{K}
\norm{\nabla_{e_v^{(k)}}\mathcal L_s}^{2}\right]
\geq K\tau\kappa_vG_{\min}^{2}.
\]
\begin{itemize}
\item This is $K$ times the \textbf{assumed single-bank lower bound}.
\item Positive softmax weights do not ensure a uniform nonzero gradient: routing suppression and cancellation remain possible.
\item No input occurrence still means no lookup gradient. A $K$-fold empirical improvement over the baseline does not follow.
\end{itemize}
\papersource{Paper, Proposition 3.2, PDF pp. 7--8; Appendix F, p. 21. Appendix simplifies the router to active banks only. Qualifications: reading analysis.}
\end{frame}

% 8
\begin{frame}{A discrete identity input can bypass a local FFN constraint}
For a collapsed pair with target separation $C=\norm{g(u)-g(v)}$,
\[
\max_{w\in\{u,v\}}\norm{\FFN(h_w)-g(w)}
\geq\frac{C-L_{\FFN}\delta}{2}.
\]
\begin{itemize}
\item The bound is informative when $C>L_{\FFN}\delta$, for fixed upstream representations and a controlled Lipschitz constant.
\item $E_k[u]$ and $E_k[v]$ are separate rows: their difference need not vanish as $\delta\to0$.
\item This supplies \textbf{capacity to separate}; it does not prove optimization success or useful separation after routing.
\end{itemize}
\begin{block}{Normalization caveat}
For fixed RMSNorm gain, memory norms are bounded. The paper's ``arbitrary $C$'' construction needs additional scale freedom.
\end{block}
\papersource{Paper, Propositions 2.2 and 3.3, PDF pp. 4, 8; Appendices D/G, pp. 19--20, 22. Caveats: reading analysis.}
\end{frame}

% 9
\begin{frame}{Training compares a fixed backbone with added memory}
\begin{itemize}
\item \textbf{Model labels:} 750M, 1B, and 3B LLaMA-style backbone families in Table 2; TIDE adds $K$ embedding tables.
\item \textbf{Training:} DCLM; LLaMA-3.1 tokenizer, $128{,}256$ entries; sequence length $2{,}048$; Adam and cosine learning-rate schedule.
\item \textbf{Budgets:} Fig. 5 compares 200B tokens; Fig. 8 extends to 500B. Table 7 lists 400--500B for the general configuration.
\item \textbf{Evaluation:} WikiText-2, PubMed, DCLM perplexity; eight zero-shot benchmarks; frequency-bin and layer diagnostics.
\end{itemize}
\begin{block}{Comparison boundary}
The added memory changes total parameter capacity. These are not demonstrated equal-total-parameter or equal-training-compute comparisons.
\end{block}
\papersource{Paper, Table 2 / Fig. 8, PDF p. 10; Appendix L / Table 7, pp. 26--27. Checkpoint-to-budget mapping is incompletely documented.}
\end{frame}

% 10
\begin{frame}{Loss gains are concentrated on rare token types}
\begin{columns}[c]
\begin{column}{.66\textwidth}
\centering
\includegraphics[trim=858bp 0 0 0,clip,width=\linewidth,height=.70\textheight,keepaspectratio]{figures/rare_token_loss_350b.pdf}
\end{column}
\begin{column}{.31\textwidth}
\textbf{TIDE-8E-1B vs. Base}\par
Matched 200B tokens.
\medskip

Mean loss reduction:
\begin{itemize}
\item Rare: 0.504 nats.
\item Common: 0.104 nats.
\end{itemize}
\medskip

Rare/common ratio: $\approx4.8\times$.
\medskip

All deciles improve; reductions are not strictly monotone.
\end{column}
\end{columns}
\papersource{Paper, Fig. 5 right panel (cropped from original), PDF p. 8. Rare = bins 0--2; common = 7--9; bins contain similar numbers of token types, not occurrences.}
\end{frame}

% 11
\begin{frame}{Numeric token pairs gain separation at middle and late layers}
\begin{columns}[c]
\begin{column}{.61\textwidth}
\centering
\includegraphics[trim=481bp 0 459.4bp 0,clip,width=\linewidth,height=.70\textheight,keepaspectratio]{figures/tide_vs_base_l2norm.pdf}
\end{column}
\begin{column}{.36\textwidth}
\begin{itemize}
\item Numerical pairs gain separation from middle layers onward.
\item Other panels gain separation late; rare-domain pairs lose it at some early layers.
\item Raw $\ell_2$ is diagnostic; larger norms can increase it.
\end{itemize}
\end{column}
\end{columns}
\papersource{Paper, Fig. 6 numeric-token panels, PDF p. 9, cropped original. Top: raw pairwise $\ell_2$; bottom: TIDE $-$ Base. Template count differs: 150 in text vs. 250 in Fig. 2 caption.}
\end{frame}

% 12
\begin{frame}{Validation perplexity generally falls as memory capacity grows}
\centering
\includegraphics[width=\linewidth,height=.61\textheight,keepaspectratio]{figures/ppl_performance.pdf}
\begin{itemize}
\item Reported TIDE curves outperform the baseline on all three corpora.
\item Larger $K$ generally helps; small local departures from ordering exist.
\item The plot does not establish the claimed $2\times$ token efficiency for $K=2$--$4$.
\end{itemize}
\papersource{Paper, Fig. 8, PDF p. 10. Original figure; qualification based on plotted values. More training-efficient per token does not imply less total training compute.}
\end{frame}

% 13
\begin{frame}{The nominal 1B family reports a 2.3 pp average gain}
\centering
\renewcommand{\arraystretch}{1.20}
\begin{tabular}{lrrrrr}
\toprule
Model & PPL $\downarrow$ & BoolQ & HellaSwag & PIQA & Reported avg.\ $\uparrow$\\
\midrule
Base & 5.19 & 61.7 & 63.9 & 74.9 & 61.4\\
TIDE, $K=2$ & 4.97 & 68.7 & 64.9 & 75.5 & 62.6\\
TIDE, $K=8$ & 4.89 & 69.3 & 65.3 & 75.5 & 63.0\\
TIDE, $K=16$ & 4.78 & 69.7 & 65.3 & 75.9 & 63.3\\
TIDE, $K=24$ & \textbf{4.60} & 69.5 & 66.3 & 77.3 & \textbf{63.7}\\
\bottomrule
\end{tabular}
\medskip
\begin{itemize}
\item Reported eight-task average: $61.4\to63.7$ ($+2.3$ percentage points).
\item Re-averaging the published task entries gives $+2.2$ percentage points.
\item PPL improves by $11.4\%$; individual tasks need not improve with each $K$.
\end{itemize}
\papersource{Paper, Table 2, PDF p. 10; selected columns, transcribed. Scores in \%; BoolQ uses accuracy, HellaSwag/PIQA normalized accuracy. Source caption calls PPL ``LAMBDA''; dataset naming is ambiguous.}
\end{frame}

% 14
\begin{frame}{Average gains span three families, with task-level regressions}
\centering
\renewcommand{\arraystretch}{1.25}
\begin{tabular}{lrrrr}
\toprule
Backbone label & Base avg. & TIDE-8E avg. & Gain (pp) & PPL: Base $\to$ TIDE\\
\midrule
750M & 59.7 & 60.7 & +1.0 & $5.63\to5.18$\\
1B & 61.4 & 63.0 & +1.6 & $5.19\to4.89$\\
3B & 67.2 & 68.3 & +1.1 & $4.00\to3.86$\\
\bottomrule
\end{tabular}
\medskip
\begin{itemize}
\item Examples of regressions: 750M BoolQ $63.5\to63.0$; 3B SciQ $93.3\to93.2$.
\item Within the nominal 1B family, only HellaSwag and PIQA are nondecreasing across \emph{all} reported $K$, including Base.
\item No multi-seed variance or confidence intervals are reported for Table 2.
\end{itemize}
\papersource{Paper, Table 2, PDF p. 10. Exact score transcriptions; gains calculated from rounded source averages. Backbone labels exclude the added table capacity.}
\end{frame}

% 15
\begin{frame}{The last-layer router uses more active memory for rare tokens}
\begin{columns}[c]
\begin{column}{.47\textwidth}
\centering
\includegraphics[width=\linewidth,height=.72\textheight,keepaspectratio]{figures/router_last_layer_null.pdf}
\end{column}
\begin{column}{.49\textwidth}
\begin{itemize}
\item Null mass rises \emph{overall} from 0.530 in the rarest bin to 0.889 in the most frequent bin.
\item Active mass is therefore $0.470$ vs. $0.111$.
\item Small reversals appear between bins; the trend is not strictly monotone.
\item This is a last-layer observation; it does not establish routing behavior at every depth.
\end{itemize}
\medskip
\small Complementary bank representations are also suggested by cosine distances and nearest-neighbor examples.
\end{column}
\end{columns}
\papersource{Paper, Fig. 10 right panel, PDF p. 12 (original); Fig. 9, p. 11; Appendix K, pp. 25--26.}
\end{frame}

% 16
\begin{frame}{Memory dependence varies across depth}
\centering
\includegraphics[trim=0 0 0 515bp,clip,width=\linewidth,height=.51\textheight,keepaspectratio]{figures/layer_ablation_ppl.pdf}\par
\small\textcolor{blue}{Blue: WikiText-2}\quad \textcolor{red}{Red: DCLM}\quad \textcolor{AcademicTeal}{Teal: PubMed}\normalsize
\begin{itemize}
\item Zero one layer's memory at inference; retain the other injections.
\item Layer 0 removal is catastrophic; layer 1 costs $8.1$--$12.9\%$ PPL.
\item Middle layers have smaller marginal effects; layer 13 has a secondary peak.
\end{itemize}
\papersource{Paper, Fig. 11(b), PDF p. 22, cropped original; Appendix H, p. 23. This tests dependence of a trained checkpoint, not the optimal injection schedule after retraining.}
\end{frame}

% 17
\begin{frame}{Embedding memory adds billions of stored parameters}
\[
P_{\rm tables}=K|\mathcal V|d_b,\qquad
P_{\rm routers}=L(K+1)d.
\]
\centering
\renewcommand{\arraystretch}{1.20}
\begin{tabular}{rrrr}
\toprule
$K$ & Added table parameters & FP16 storage & 4-bit storage\\
\midrule
2 & 0.525B & 1.051 GB & 0.263 GB\\
8 & 2.101B & 4.203 GB & 1.051 GB\\
24 & 6.304B & 12.608 GB & 3.152 GB\\
\bottomrule
\end{tabular}
\medskip
\begin{itemize}
\item Calculated for $|\mathcal V|=128{,}256$, $d_b=d=2{,}048$; table storage only, decimal GB, excluding metadata.
\item A ``TIDE-8E-1B'' label therefore does \textbf{not} mean 1B total parameters.
\item Static lookup permits caching/offloading in principle; runtime activations and training optimizer state also require memory.
\end{itemize}
\papersource{Paper, Section 3.2 / Fig. 4, PDF p. 6; Appendix J, pp. 23--24. Counts calculated from the reported vocabulary and width.}
\end{frame}

% 18
\begin{frame}{Measured decoding overhead ranges from 1.4\% to 21.1\%}
\centering
\renewcommand{\arraystretch}{1.20}
\begin{tabular}{lrr}
\toprule
Model (nominal 1B family) & ms/token $\downarrow$ & Increase vs. Base\\
\midrule
Base & 11.085 & ---\\
TIDE, $K=2$ & 11.236 & 1.4\%\\
TIDE, $K=4$ & 11.854 & 6.9\%\\
TIDE, $K=8$ & 12.688 & 14.5\%\\
TIDE, $K=16$ & 12.901 & 16.4\%\\
TIDE, $K=24$ & 13.422 & 21.1\%\\
\bottomrule
\end{tabular}
\medskip
\begin{itemize}
\item Reported on one B200 GPU, averaged over 5,120 generated tokens.
\item Table 4 does not document an SSD-offload benchmark; these timings cannot validate its end-to-end cost.
\end{itemize}
\papersource{Paper, Table 4 / Appendix I, PDF p. 23. Times transcribed; relative overhead calculated as (TIDE/Base $-1$)$\times100\%$.}
\end{frame}

% 19
\begin{frame}{8-bit table quantization changes perplexity very little}
\centering
\renewcommand{\arraystretch}{1.25}
\begin{tabular}{lrrrr}
\toprule
Table precision & Size & WikiText-2 & PubMed & DCLM\\
\midrule
16-bit & 4.20 GB & 10.088 & 11.100 & 16.108\\
8-bit & 2.10 GB & 10.089 & 11.100 & 16.113\\
4-bit & 1.05 GB & 10.263 & 11.277 & 16.343\\
\bottomrule
\end{tabular}
\medskip
\begin{itemize}
\item Results are for the memory tables of TIDE-8E-1B; lower PPL is better.
\item At 4 bits, PPL rises by $1.73\%$, $1.59\%$, and $1.46\%$, respectively.
\item This supports a useful storage tradeoff at this checkpoint, with a small measurable quality cost.
\end{itemize}
\papersource{Paper, Table 5 / Appendix J.1, PDF p. 24. Exact table values; percentage changes calculated relative to 16-bit.}
\end{frame}

% 20
\begin{frame}{Low-rank compression trades quality for storage}
\begin{columns}[c]
\begin{column}{.63\textwidth}
\centering
\includegraphics[trim=711bp 0 0 0,clip,width=\linewidth,height=.66\textheight,keepaspectratio]{figures/rank_reduction_ppl.pdf}
\par\small\textcolor{blue}{Blue: WikiText-2}\quad\textcolor{red}{Red: DCLM}\\\textcolor{AcademicTeal}{Teal: PubMed}
\end{column}
\begin{column}{.34\textwidth}
\begin{itemize}
\item Uniform SVD on eight tables: $r=\lceil(1-p)d_b\rceil$.
\item 10--30\% rank reduction has small PPL impact.
\item At 50\%, PPL rises roughly 10--20\%.
\item At 90\%, PPL rises by 484--657\%.
\end{itemize}
\end{column}
\end{columns}
\papersource{Paper, Fig. 12(b) / Appendix J.2, PDF pp. 24--25. Cropped original; the 50\% range is a visual estimate. $p$ is the rank reduction fraction.}
\end{frame}

% 21
\begin{frame}{Next tests should separate architecture from extra capacity}
\begin{enumerate}
\item \textbf{Capacity and compute controls:} compare equal-total-parameter and equal-training-FLOP alternatives, including a single enlarged table.
\item \textbf{Causal diagnostics:} measure normalized separation, gradient magnitudes, and the effect of token-ID intervention.
\item \textbf{System validation:} report SSD latency, cache hit rates, activation memory, and throughput under realistic batches.
\item \textbf{Reproducibility:} resolve layer-count and training-budget inconsistencies, and report multiple seeds and checkpoint details.
\end{enumerate}
\medskip
These are proposed follow-up experiments. They are not results reported in the paper.
\papersource{Reading analysis based on Sections 3--4 and Appendices H--M. Author limitations include storage, longer training, instruction tuning/RLHF, and bank interpretability (PDF p. 26).}
\end{frame}

% 22
\begin{frame}{The useful contribution is a persistent token-indexed side input}
\begin{block}{Take-home message}
A learned static memory, selected by contextual routers, is a promising way to preserve token-specific capacity across transformer depth.
\end{block}
\begin{itemize}
\item \textbf{Supported:} rare-token loss benefits, improved mean benchmarks, and a quantizable memory component.
\item \textbf{Qualified:} the theoretical gradient claim is conditional; storage increases substantially; causal and systems claims need stronger controls.
\item \textbf{Discussion:} how much of the benefit comes from repeated identity access, and how much from added token-specific capacity?
\end{itemize}
\papersource{Ajay Jaiswal et al. \emph{TIDE: Every Layer Knows the Token Beneath the Context}. arXiv:2605.06216v1, 2026.\\\href{https://arxiv.org/abs/2605.06216v1}{Versioned paper}; original PDF and author TeX source read. Main talk ends here; five backup slides follow.}
\end{frame}

\appendix
% 23
\begin{frame}{Backup: the null-bank proof needs a realizable constant logit}
With memory norms bounded by $C_M$, the algebra is valid:
\[
\norm{m^\ell(v)}\leq(1-\alpha_{K+1}^\ell)C_M.
\]
If active logits are zero and the null logit equals $s$,
\[
1-\alpha_{K+1}^\ell=\frac{K}{K+e^s}\longrightarrow0.
\]
\begin{itemize}
\item The stated router is bias-free: $z=W_r^\ell\tilde n^\ell$.
\item One fixed linear map cannot generally impose the same positive null logit for all possible hidden states; e.g., $n$ and $-n$.
\item A bias/constant feature or a suitable restricted hidden-state domain would justify the construction. The paper does not establish this condition.
\end{itemize}
\papersource{Paper, Proposition 3.1, PDF p. 7; Appendix E, pp. 20--21. Gap identified in reading analysis; it concerns the stated router-based proof, not impossibility of suppressing memory by other parameter settings.}
\end{frame}

% 24
\begin{frame}{Backup: occurrence counts are not update-step counts}
For 200B training tokens, $B=8$, and $T=2{,}048$,
\[
\tau=\frac{200\times10^9}{8\times2{,}048}\approx1.22\times10^7.
\]
The expected number of steps containing token $v$ is
\[
\E[N_v]=\tau[1-(1-f_v)^{BT}]\leq\tau.
\]
\begin{itemize}
\item Table 1 lists $1.66\times10^9$ ``non-zero gradient updates'' for a common token with $f_v=8.3\times10^{-3}$.
\item That value is the expected \emph{occurrence count} $200\mathrm B\,f_v$; it exceeds the number of optimizer steps.
\item The approximation $\tau f_vBT$ is valid only when $f_vBT\ll1$. It works for rare tokens, and fails for that common token.
\end{itemize}
\papersource{Paper, Table 1 / Eq. (2.2), PDF p. 3; Appendix C, p. 18. Arithmetic and interpretation checked in this reading.}
\end{frame}

% 25
\begin{frame}{Backup: a softmax does not guarantee $K$ times gradient energy}
\begin{block}{A simplified counterexample to an unconditional claim}
Let $m=\frac1K\sum_{k=1}^{K}e_k$ and let the upstream gradient be $g=\nabla_m\mathcal L$. Then
\[
\nabla_{e_k}\mathcal L=\frac{g}{K},\qquad
\sum_{k=1}^{K}\norm{\nabla_{e_k}\mathcal L}^{2}
=\frac{\norm{g}^{2}}{K}.
\]
\end{block}
\begin{itemize}
\item Every route is positive, but total squared gradient decreases as $K$ grows in this example.
\item Therefore a $K$-independent lower bound $G_{\min}$ is a substantive assumption, not a consequence of positive routing alone.
\item This toy example isolates routing; it is not a simulation of trained TIDE.
\end{itemize}
\papersource{Teaching derivation by the reader; relates to Proposition 3.2 and Appendix F, PDF pp. 7--8, 21. It illustrates the limitation of an unconditional interpretation.}
\end{frame}


% 26
\begin{frame}{Backup: two banks capture half of the rare-token gain}
Selected cross-entropy values (nats; lower is better):
\medskip
\centering
\renewcommand{\arraystretch}{1.25}
\begin{tabular}{lrrrrrr}
\toprule
$K$ & 0 & 2 & 4 & 8 & 16 & 24\\
\midrule
Rare & 6.671 & 6.454 & 6.361 & 6.353 & 6.254 & 6.250\\
Common & 2.967 & 2.940 & 2.928 & 2.929 & 2.893 & 2.892\\
\bottomrule
\end{tabular}
\medskip
\begin{itemize}
\item $K=0\to24$: rare loss falls by 0.421 nats; common loss by 0.075.
\item $K=2$ captures $51.5\%$ of the total rare gain; $K=16\to24$ adds only 0.004 nats.
\item The common curve has a small $K=4\to8$ reversal.
\end{itemize}
\medskip
{\small\textbf{Different evaluation views.} Fig. 5: fixed $K=8$, 200B tokens, individual deciles. Fig. 7: varying $K$, grouped bins. Keep their values separate.\par}
\papersource{Paper, Fig. 7, PDF p. 9; Section 4.1, pp. 10--11. Values transcribed from the figure; differences and ratios calculated in this reading.}
\end{frame}

% 27
\begin{frame}{Backup: banks differ from the input table in neighbor structure}
\begin{block}{Embedding-space distance}
For TIDE-8E-1B, mean row-wise cosine distance between the primary table $E$ and each $M_k$ ranges from 0.649 to 0.990.
\end{block}
\begin{itemize}
\item The measured coordinates differ from $E$; distance alone does not establish useful new semantics.
\end{itemize}
\begin{block}{Nearest-neighbor probe}
Top-10 cosine-neighbor sets have lower Jaccard overlap with $E$ for rare tokens than common tokens. For \texttt{asynchronously}, examples include \texttt{callback} and \texttt{caching} in memory banks.
\end{block}
\begin{itemize}
\item Different neighbors suggest complementary information, but there is no blinded semantic-quality assessment in this probe.
\end{itemize}
\vspace{-1mm}
\papersource{Paper, Fig. 9, PDF p. 11; Fig. 13, p. 25; Table 6, p. 26 (Section 4.2 / Appendix K). Interpretation: reading analysis.}
\end{frame}

\end{document}