% Compile with pdflatex. Replace the demonstration content before delivery.
\documentclass[aspectratio=169,11pt]{beamer}
\usepackage[T1]{fontenc}
\usepackage{lmodern}
\usepackage[utf8]{inputenc}
\usepackage{amsmath,amssymb,graphicx,booktabs}

\definecolor{AcademicBlue}{RGB}{26,57,86}
\definecolor{AcademicTeal}{RGB}{0,105,112}
\setbeamercolor{structure}{fg=AcademicBlue}
\setbeamercolor{frametitle}{fg=AcademicBlue,bg=white}
\setbeamercolor{title}{fg=AcademicBlue}
\setbeamercolor{alerted text}{fg=AcademicTeal}
\setbeamercolor{block title}{fg=white,bg=AcademicBlue}
\setbeamercolor{block body}{fg=black,bg=AcademicBlue!5}
\setbeamertemplate{navigation symbols}{}
\setbeamertemplate{itemize items}[circle]
\setbeamertemplate{footline}{%
  \hfill\usebeamerfont{page number in head/foot}%
  \insertframenumber\,/\,\inserttotalframenumber\hspace{4mm}\vspace{3mm}}
\setbeamersize{text margin left=8mm,text margin right=8mm}
\setbeamerfont{frametitle}{size=\Large,series=\bfseries}
\setbeamerfont{title}{size=\LARGE,series=\bfseries}
\newcommand{\papersource}[1]{\par\smallskip{\scriptsize\color{gray}#1\par}}

\title[Bank of Values]{Do Value Vectors in Deep Layers Need Context from the Residual Stream?}
\subtitle{Bank of Values (BoV) --- research paper discussion}
\author{Paper: Muyu He, Yuchen Liu, Qingya Huang, Li Zhang}
\date{arXiv:2606.02780v5 (23 August 2026)\\Reading: 17 September 2026}
\begin{document}
\begin{frame}\titlepage\end{frame}
\begin{frame}{Deep-layer values can be a token content bank}
\begin{block}{Main result}
Replace the value projection in the last third of layers with a layer-specific token lookup. Keep contextual queries and keys.
\end{block}\begin{itemize}
\item 780M-scale, 100B tokens: BPB $0.671\to0.663$; CORE $0.334\to0.351$.
\item 16 of 21 tasks improve; 5 regress.
\item Evidence covers 135M/780M scales; deployment gains remain conditional.
\end{itemize}
\papersource{Paper, Secs. 4--5; Tables 2 and 6, PDF pp. 6, 13.}
\end{frame}
\begin{frame}{The question is what deep values need to carry}
\begin{itemize}
\item Queries and keys determine which positions receive attention.
\item Values supply the content mixed by those attention weights.
\item Earlier work adds token information, or shares first-layer values across all layers.
\item BoV isolates depth, value source, scaling, and addition versus replacement.
\end{itemize}
\papersource{Paper, Secs. 1--3, PDF pp. 1--4.}
\end{frame}
\begin{frame}{Context survives through queries, keys, and residuals}
\[
\mathbf Q=\mathbf x_i\mathbf W_Q,\quad \mathbf K=\mathbf x_i\mathbf W_K,
\quad \mathbf V=\mathbf x_i\mathbf W_V
\]
\[
\mathbf O^{(h)}=\operatorname{softmax}\!\left(
\frac{\mathbf Q^{(h)}\mathbf K^{(h)\top}}{\sqrt{d_h}}\right)\mathbf V^{(h)}
\]\begin{itemize}
\item $\mathbf x_i$: contextual residual input; $d_h=d/H$.
\item $\mathbf x_0=\mathrm{RMSNorm}(\mathbf e)$: original token embedding.
\item BoV changes only the value source in selected layers. Causal masking is omitted above.
\end{itemize}
\papersource{Paper, Sec. 2, Eqs. (1)--(2), PDF p. 3.}
\end{frame}
\begin{frame}{BoV gives each deep layer its own table}
\begin{center}\includegraphics[width=.96\linewidth,height=.66\textheight,keepaspectratio]{figures/arch_figure.pdf}\end{center}
\papersource{Paper, Fig. 1, PDF p. 2. Original figure; only the last third of layers use BoV.}
\end{frame}
\begin{frame}{Depth restriction is essential in the 135M study}
\begin{center}\begin{tabular}{llr}\toprule
Value source & Target / coefficient & BPB $\downarrow$\\\midrule
Standard $\mathbf V$ & Baseline & 0.854\\
Shared $\mathbf V_1$ & All layers / fixed 1 & 0.872\\
Shared $\mathbf V_1$ & Last 4 / fixed 1 & 0.849\\
Shared $\mathbf V_1$ & Last 4 / learned & 0.847\\
Layer-specific $\mathbf x_0\mathbf W_V$ & Last 4 / learned & \textbf{0.845}\\
$\mathbf V+\mathbf x_0\mathbf W_V$ & Last 4 / learned & \textbf{0.845}\\\bottomrule
\end{tabular}\end{center}
Adding contextual values gives no gain in the last pair.

\papersource{Paper, Table 1, PDF p. 3. All runs: $1.50\times10^{18}$ FLOPs. These are projection variants, not the BoV lookup table.}
\end{frame}
\begin{frame}{Learned scales favor the context-free component}
\begin{center}\includegraphics[width=.96\linewidth,height=.48\textheight,keepaspectratio]{figures/d12_value_coeffs.pdf}\end{center}\begin{itemize}
\item Horizontal axis: layer indices 8--11 (zero-based).
\item Scales are unbounded and can be negative.
\item Interpretation: combine this pattern with ablations; scale magnitude alone is not a causal contribution measure.
\end{itemize}
\papersource{Paper, Fig. 2, PDF p. 4; final bullet is reading analysis.}
\end{frame}
\begin{frame}{BoV learns values directly as parameters}
\[
\mathbf E_v\in\mathbb R^{|\mathcal V|\times d},\qquad
\boxed{\mathbf v_p=\gamma_v\mathbf E_v[i_p]}
\]\begin{itemize}
\item $i_p$: token ID at position $p$; $\gamma_v$: learned per-layer scalar.
\item One independent table for each of the last $L/3$ layers; remove their $\mathbf W_V$.
\item Initialize each row from $\mathrm{RMSNorm}(\mathbf e_i)\mathbf W_V$.
\item Then train table entries directly; the initial factorization need not persist.
\end{itemize}
\papersource{Paper, Secs. 4--5, Eqs. (3)--(5), PDF pp. 5--6.}
\end{frame}
\begin{frame}{A fixed value does not imply a fixed interpretation}
\begin{block}{Teaching example, not a paper experiment}
The token ``bank'' has the same lookup value in two contexts at the same BoV layer. Its contextual keys, attention weights, and final residual state can still differ.
\end{block}\begin{itemize}
\item Repeated IDs reuse content vectors.
\item Context determines how those vectors are selected and combined.
\item Earlier layers still compute standard contextual values.
\end{itemize}
\papersource{Derived from the data flow in Sec. 4 and Fig. 1, PDF pp. 2, 5.}
\end{frame}
\begin{frame}{Decode keeps keys and IDs, then gathers values}
\begin{itemize}
\item Training / prefill: look up the current token sequence.
\item Decode: retain historical K and token IDs at target layers.
\item Gather historical V for one layer, compute attention, release temporary V.
\item History gathering moves $O(Td)$ data; it is not zero-cost memory access.
\item Host offload / prefetch is proposed, without measured serving throughput.
\end{itemize}
\papersource{Paper, Fig. 3 and Sec. 4, PDF pp. 5--6.}
\end{frame}
\begin{frame}{Memory savings require an explicit accounting model}
\[
\Delta_{\mathrm{KV}}=\frac L3 d\bigl(T-|\mathcal V|\bigr)
\]\begin{itemize}
\item Paper equation counts elements; multiply by bytes per element.
\item One-sequence break-even: $T=|\mathcal V|$ under the simplified model.
\item Tables cost fixed memory; temporary V and K caching still exist.
\item Batching, sliding windows, KV width, and offload change the accounting.
\end{itemize}
\papersource{Paper, Eq. (6), PDF p. 5. Accounting qualifications: reading analysis.}
\end{frame}
\begin{frame}{The memory figure has a denominator ambiguity}
\begin{center}\includegraphics[width=.96\linewidth,height=.46\textheight,keepaspectratio]{figures/memsave_ablation_ab.pdf}\end{center}\[
\underbrace{\frac{\Delta_{\mathrm{KV}}}{2LTd}}_{\text{fraction of total K+V cache}}
=\frac16\left(1-\frac{|\mathcal V|}{T}\right)\to16.7\%
\]
The left panel says ``V cache'', but its asymptote matches total K+V.

\papersource{Paper, Fig. 5(a), PDF p. 8; denominator check derived from Eq. (6). Qwen3 is a vocabulary-size example, not a tested model.}
\end{frame}
\begin{frame}{Three budgets answer different questions}
\begin{center}\begin{tabular}{lll}\toprule
Model scale & Controlled budget & Training tokens\\\midrule
135M & $1.50\times10^{18}$ FLOPs & Baseline: 1.97B\\
780M & $3.91\times10^{19}$ FLOPs & Baseline: 8.19B\\
 & & BoV: 8.39B\\
780M & 100B tokens & 100B each\\\bottomrule
\end{tabular}\end{center}\begin{itemize}
\item Sequence length 2048; vocabulary 32,768; bf16.
\item 12/24 layers, width 768/1536, 6/12 attention and KV heads.
\item Muon for matrices; AdamW for other parameters, including the BoV table.
\end{itemize}
\papersource{Paper, Secs. 3, 5; Table 8, PDF pp. 3--4, 6--7, 14.}
\end{frame}
\begin{frame}{CORE is a centered aggregate, not raw accuracy}
\[
c_j=\frac{\mathrm{acc}_j-r_j}{1-r_j},\qquad
\mathrm{CORE}=\frac1{21}\sum_{j=1}^{21}c_j
\]\begin{itemize}
\item 21 tasks; equal weight after centering against a task baseline $r_j$.
\item Validation BPB: lower is better; held-out ClimbMix, about 41.9M tokens.
\item CORE $+0.017$ is a score change, not raw accuracy $+1.7$ percentage points.
\item No multi-seed uncertainty or significance tests are reported.
\end{itemize}
\papersource{Paper, evaluation setup, PDF pp. 6--7. Task-specific baselines are not tabulated.}
\end{frame}
\begin{frame}{At 100B tokens, BoV improves aggregate quality}
\begin{center}\begin{tabular}{lrrr}\toprule
Metric & Baseline & BoV & Difference\\\midrule
BPB $\downarrow$ & 0.671 & 0.663 & $-0.008$\\
CORE $\uparrow$ & 0.334 & 0.351 & $+0.017$\\
CommonsenseQA & 0.437 & 0.523 & $+0.086$\\
SQuAD & 0.506 & 0.531 & $+0.025$\\
CoQA & 0.342 & 0.372 & $+0.030$\\\bottomrule
\end{tabular}\end{center}
Reported training FLOPs: $4.78\times10^{20}\to4.66\times10^{20}$, approximately 2.51\% lower.

\papersource{Paper, Table 2 and training setup, PDF p. 6. Task entries reproduce reported scores.}
\end{frame}
\begin{frame}{Aggregate gains hide substantial task regressions}
\begin{center}\begin{tabular}{lrrr}\toprule
Task & Baseline & BoV & Difference\\\midrule
Repeat-Copy & 0.000 & 0.094 & $+0.094$\\
Dyck & 0.087 & 0.154 & $+0.067$\\
LSAT-AR & 0.300 & 0.187 & $-0.113$\\
CS Algorithms & 0.442 & 0.414 & $-0.028$\\
Winograd & 0.766 & 0.740 & $-0.026$\\\bottomrule
\end{tabular}\end{center}
16 improvements and 5 regressions across 21 tasks. Symbolic reasoning does not improve uniformly.

\papersource{Paper, Table 6, PDF p. 13; 780M-scale, 100B tokens.}
\end{frame}
\begin{frame}{At equal FLOPs, BoV also sees more tokens}
\begin{center}\begin{tabular}{lrr}\toprule
Value variant & BPB $\downarrow$ & CORE $\uparrow$\\\midrule
Standard attention & 0.722 & 0.260\\
BoV & \textbf{0.714} & \textbf{0.272}\\
$\mathbf x_0\mathbf W_V$, last third & 0.719 & 0.259\\
$\mathbf V_1$, last third & 0.718 & 0.242\\
$\mathbf V_1$, all layers & 0.741 & 0.210\\\bottomrule
\end{tabular}\end{center}
BoV: 8.39B versus baseline 8.19B tokens (about 2.44\% more). Parameterization, optimization, and data exposure are not fully disentangled.

\papersource{Paper, Table 3 and Sec. 5.2, PDF p. 7; baseline token count: Table 8, p. 14.}
\end{frame}
\begin{frame}{Training curves support both tested model scales}
\begin{center}\includegraphics[width=.96\linewidth,height=.53\textheight,keepaspectratio]{figures/valbpb_ab.pdf}\end{center}\begin{itemize}
\item Axes: training tokens versus validation BPB.
\item Equal FLOPs can produce different final token counts.
\item These curves do not measure wall-clock training speed.
\end{itemize}
\papersource{Paper, Fig. 4, PDF p. 6.}
\end{frame}
\begin{frame}{The additive competitor is close, with mixed wins}
\begin{center}\begin{tabular}{lrr}\toprule
Metric & Additive $\mathbf V+\mathbf E_v[i]$ & BoV\\\midrule
BPB $\downarrow$ & \textbf{0.712} & 0.714\\
CORE $\uparrow$ & 0.268 & \textbf{0.272}\\
SQuAD & \textbf{0.439} & 0.416\\
BoolQ & 0.594 & \textbf{0.635}\\
Winograd & \textbf{0.692} & 0.645\\\bottomrule
\end{tabular}\end{center}\begin{itemize}
\item Across 21 tasks: BoV wins 10, loses 8, ties 3.
\item The additive baseline also differs in layer selection and bounded, input-dependent gating.
\end{itemize}
\papersource{Paper, Sec. 5.2, Table 7, PDF pp. 7, 13. Counts computed from the table.}
\end{frame}
\begin{frame}{Layer-specific tables and learned scales matter}
\begin{center}\includegraphics[width=.96\linewidth,height=.47\textheight,keepaspectratio]{figures/memsave_ablation_ab.pdf}\end{center}\begin{itemize}
\item Right panel: applying BoV to every layer hurts most.
\item Sharing tables or fixing the scale also increases loss.
\item Retaining contextual V gives only a small loss reduction.
\end{itemize}
\papersource{Paper, Sec. 5.3 and Fig. 5(b), PDF pp. 7--8. Exact endpoints and uncertainty are not tabulated.}
\end{frame}
\begin{frame}{Sparse access still adds stored parameters}
\begin{align*}
\text{Tables: }&8\times32768\times1536=402.7\text{M elements}\\
\text{Removed projections: }&8\times1536^2=18.9\text{M elements}\\
\text{Net addition: }&\approx383.8\text{M parameters}
\end{align*}\begin{itemize}
\item Derived for the 24-layer setting; excludes a few scalars.
\item bf16 tables alone occupy 768 MiB.
\item The 780M scale label is not an equal-total-parameter comparison.
\end{itemize}
\papersource{Reading derivation from Sec. 4 and Table 8, PDF pp. 5, 14; not a reported memory measurement.}
\end{frame}
\begin{frame}{Evidence is promising within a narrow tested scope}
\begin{itemize}
\item Established: useful deep-layer value replacement at 135M/780M scales.
\item Not established: larger-model scaling or long-context task quality.
\item Not measured: decode throughput, offload latency, and full training-memory tradeoffs.
\item Next tests: matched parameters/tokens/FLOPs, multiple seeds, task regressions, and serving benchmarks.
\end{itemize}
\papersource{Paper, Limitations, PDF p. 9. Proposed tests are reading analysis.}
\end{frame}
\begin{frame}{Discussion: what should a deep value represent?}
\begin{block}{Takeaway}
Contextual routing can combine token-specific content effectively in deep layers. The value path need not mirror the query/key paths in the tested setting.
\end{block}\begin{itemize}
\item The useful design is depth-specific, layer-specific, and trainable.
\item Quality improves on average; task tradeoffs remain.
\item Memory and speed claims require a concrete workload and implementation.
\end{itemize}
\papersource{Paper, Secs. 3--5 and 7, PDF pp. 3--9.}
\end{frame}
\begin{frame}{Sources and reproducibility notes}
Muyu He, Yuchen Liu, Qingya Huang, and Li Zhang.\par\medskip
\emph{Do Value Vectors in Deep Layers Need Context from the Residual Stream?}\par
\href{https://arxiv.org/abs/2606.02780v5}{arXiv:2606.02780v5}, 23 August 2026.
\par\medskip
\href{https://github.com/RiddleHe/nanochat}{Author code: github.com/RiddleHe/nanochat}
\par\medskip
Read: author TeX, all substantive sections and appendices, and the matching 14-page PDF. Original figures reused. Experiments were not rerun.
\par\medskip
Configuration caveat: Sec. 5 states matrix LR 0.02; Table 8 lists 0.0283 for 780M and describes scaling. Table 8 covers the shorter budget, not a complete 100B recipe.

\papersource{All page locators use PDF file pages, starting at 1.}
\end{frame}
\end{document}