% Compile with pdflatex. Replace the demonstration content before delivery.
\documentclass[aspectratio=169,11pt]{beamer}
\usepackage[T1]{fontenc}
\usepackage[utf8]{inputenc}
\usepackage{amsmath,amssymb,graphicx,booktabs}

\definecolor{AcademicBlue}{RGB}{26,57,86}
\definecolor{AcademicTeal}{RGB}{0,105,112}
\setbeamercolor{structure}{fg=AcademicBlue}
\setbeamercolor{frametitle}{fg=AcademicBlue,bg=white}
\setbeamercolor{title}{fg=AcademicBlue}
\setbeamercolor{alerted text}{fg=AcademicTeal}
\setbeamercolor{block title}{fg=white,bg=AcademicBlue}
\setbeamercolor{block body}{fg=black,bg=AcademicBlue!5}
\setbeamertemplate{navigation symbols}{}
\setbeamertemplate{itemize items}[circle]
\setbeamertemplate{footline}{%
  \hfill\usebeamerfont{page number in head/foot}%
  \insertframenumber\,/\,\inserttotalframenumber\hspace{4mm}\vspace{3mm}}
\setbeamersize{text margin left=8mm,text margin right=8mm}
\setbeamerfont{frametitle}{size=\Large,series=\bfseries}
\setbeamerfont{title}{size=\LARGE,series=\bfseries}
\newcommand{\papersource}[1]{\par\smallskip{\scriptsize\color{gray}#1\par}}

\title[CCQ]{Don't Read Everything}
\subtitle{A Curvature-Conditioned Query for Linear Attention}
\author{Paper: Dong Le, Thong Nguyen, Cong-Duy Nguyen, Anh-Tuan Luu}
\date{arXiv:2606.01294v2 (30 August 2026)\\Reading presentation: 16 September 2026}
\begin{document}
\begin{frame}
\titlepage
\vspace{-3mm}
\begin{center}\small Memory-aware query contraction improves several retrieval results,\\but the training-time mechanism remains unresolved.\end{center}
\end{frame}
\begin{frame}{The read step is a separate design choice}

\[
S_t=\mathrm{Update}(S_{t-1};k_t,v_t,\ldots),\qquad o_t=S_tq_t
\]
\begin{columns}[T]
\begin{column}{0.47\textwidth}
\begin{block}{Write-side changes}
Gating controls retention.\\[2mm]
Delta updates control overwriting.
\end{block}
\end{column}
\begin{column}{0.47\textwidth}
\begin{block}{CCQ changes the read}
Use the distribution of past keys\\to modify the current query.
\end{block}
\end{column}
\end{columns}
\medskip
\textbf{Question:} Can a fixed-size statistic reduce background interference?

\papersource{Paper, Secs. 1--3.1, PDF pp. 1--3.}
\end{frame}
\begin{frame}{Softmax contributes a useful local geometry}

\[
A_t(q)=\log\sum_{j\le t}e^{k_j^\top q},\qquad
\nabla^2 A_t(q)=\operatorname{Cov}_{p(q)}[k]
\]
\[
\nabla^2 A_t(0)=\underbrace{\frac1t\sum_j k_jk_j^\top-\mu_t\mu_t^\top}_{\Sigma_t},
\qquad \mu_t=\frac1t\sum_j k_j
\]
\begin{itemize}
\item At $q=0$, all keys have probability $1/t$.
\item High covariance eigenvalues identify directions of key variation.
\item The shared softmax denominator does not change key rankings.
\end{itemize}

\papersource{Paper, Eqs. (5), (8), (12); Secs. 3.1--3.3, PDF pp. 2--4; Appendix A, p. 11.}
\end{frame}
\begin{frame}{CCQ chooses one gradient step on a quadratic penalty}

\[
R_t(q)=\tfrac12q^\top\Sigma_tq,\qquad
\lambda_t=\sigma(W_\lambda q_t+b_\lambda)
\]
\begin{block}{Clean the query, then read}
\[
q_t^{\mathrm{clean}}=(I-\lambda_t\Sigma_t)q_t,
\qquad o_t=S_tq_t^{\mathrm{clean}}
\]
\end{block}
\begin{itemize}
\item One covariance--query product; the write recurrence is unchanged.
\item This operator is a design choice, not a unique Taylor consequence.
\item The identity at $q=0$ does not bound the remainder at trained queries.
\end{itemize}

\papersource{Paper, Eqs. (7)--(10), PDF p. 3; Appendix A.3, pp. 11--12. Normalization made explicit on slide 7.}
\end{frame}
\begin{frame}{Retrieval improves only under a geometric condition}

\[
s_{tj}^{\mathrm{CCQ}}=k_j^\top q_t-\lambda_t k_j^\top\Sigma_tq_t
\]
For $\Sigma_t\approx\rho uu^\top$, set
$\alpha=k_\star^\top u$ and $\beta=k_d^\top u$.
\begin{block}{Change in target-minus-distractor margin}
\[
\Delta\approx\lambda\rho(\beta-\alpha)(u^\top q)
\]
\end{block}
\begin{itemize}
\item The sign of $(\beta-\alpha)(u^\top q)$ decides help versus harm.
\item A zero margin change need not mean the query is unchanged.
\item High variance is not universally equivalent to irrelevant information.
\end{itemize}

\papersource{Paper, Eq. (11), p. 3; Appendix H, Eq. (26), p. 16. Boundary qualification: reading analysis.}
\end{frame}
\begin{frame}{Unit keys and a bounded gate make the step contractive}

\[
\|k_j\|_2=1\quad\Longrightarrow\quad
0\preceq\Sigma_t\preceq\bar C_t\preceq I
\]
\[
\sigma_i(\Sigma_t)\in[0,1],\quad\lambda_t\in(0,1)
\quad\Longrightarrow\quad 1-\lambda_t\sigma_i\in(0,1]
\]
\begin{itemize}
\item Query norm cannot increase; no eigendirection is annihilated.
\item Dividing the accumulated second moment by $t$ is essential.
\item This is a bound on the read correction, not a retrieval guarantee.
\end{itemize}
\begin{block}{Teaching example (not an experiment)}
$\Sigma=\operatorname{diag}(0.8,0.1),\ \lambda=0.5$:
$\ (1,1)/\sqrt2\mapsto(0.6,0.95)/\sqrt2$.
\end{block}

\papersource{Paper, Sec. 3.3, p. 5; Appendix G, Eq. (23), pp. 15--16.}
\end{frame}
\begin{frame}{Inference adds two running statistics per head}

Normalize $q_t,k_t$ to $\bar q_t,\bar k_t$, then include the current key:
\[
\begin{aligned}
\bar C_t&=\bar C_{t-1}+t^{-1}(\bar k_t\bar k_t^\top-\bar C_{t-1}),\\
\mu_t&=\mu_{t-1}+t^{-1}(\bar k_t-\mu_{t-1}),\\
q_t^{\mathrm{clean}}&=\bar q_t-\lambda_t
[\bar C_t\bar q_t-\mu_t(\mu_t^\top\bar q_t)].
\end{aligned}
\]
\begin{itemize}
\item Gate: linear projection of the pre-split, unnormalized query.
\item Cache: $d_vd_k+d_k^2+d_k$ numbers per head.
\item Here $d_k=256$, $d_v=512$: about 50\% more recurrent state.
\end{itemize}

\papersource{Paper, Algorithm 2, p. 13; Appendices D--E, p. 14. Gate starts near 0.01.}
\end{frame}
\begin{frame}{The probed needles avoid high-variance subspaces}

\centering
\includegraphics[width=0.97\linewidth,height=0.47\textheight,keepaspectratio]{figures/cov_validation_depth.pdf}
\par\medskip
\raggedright
\begin{itemize}
\item $\Delta_\mu=$ unique-needle alignment minus distractor alignment.
\item Negative head averages at 36/36 Qwen layers and 21/21 GDN layers.
\item Rare-token filtering and unmatched model pairs limit causal claims.
\end{itemize}

\papersource{Paper, Fig. 1, p. 5; Appendix I, pp. 16--20. Solid: unique tokens; dashed: all needle-span tokens.}
\end{frame}
\begin{frame}{Training is matched within each model scale}

\centering
\begin{tabular}{lcc}
\toprule
 & 500M & 1.3B\\
\midrule
Training tokens & 15B & 40B\\
Hidden size & 1024 & 2048\\
GLA / GDN family layers & 21 & 21\\
Global batch (tokens) & 262,144 & 524,288\\
\bottomrule
\end{tabular}
\par\medskip\raggedright
\begin{itemize}
\item FineWeb-Edu subset; 32K BPE; 4,096-token training window.
\item From scratch, AdamW, bfloat16, one 8-H200 node.
\item Controls: Transformer, Mamba2, GLA, GLA-Hedgehog, Gated DeltaNet.
\end{itemize}

\papersource{Paper, Sec. 4.1 and Table 1, pp. 5--6; Appendix D and Table 6, p. 14. GDN = Gated DeltaNet.}
\end{frame}
\begin{frame}{Four evaluations measure different capabilities}

\begin{description}
\item[LM + reasoning] WikiText / LAMBADA PPL and seven-task zero-shot accuracy.
\item[S-NIAH] Pass-key, number, UUID retrieval; 1K--8K. UUID only to 4K.
\item[Length extrapolation] Token PPL on six corpora; 4K--20K in 2K steps.
\item[LongBench] Fourteen English tasks at 1.3B; task-specific official metrics.
\end{description}
\medskip
\alert{LongBench Avg mixes F1, ROUGE-L, accuracy and code similarity.}
\par\medskip
Main quality tables do not report multi-seed uncertainty.

\papersource{Paper, Sec. 4; Appendix F, pp. 14--15. Seven-task Avg excludes LAMBADA.}
\end{frame}
\begin{frame}{Language modelling gains depend on the backbone}

\centering
\begin{tabular}{llcc}
\toprule
Scale & Backbone & Wiki PPL $\downarrow$ & 7-task Avg $\uparrow$\\
\midrule
500M & GLA & $27.27\to26.17$ & $48.54\to49.73$\\
500M & GDN & $26.33\to25.59$ & $48.98\to49.87$\\
1.3B & GLA & $18.80\to16.92$ & $55.75\to56.10$\\
1.3B & GDN & $17.98\to18.01$ & $55.64\to56.24$\\
\bottomrule
\end{tabular}
\par\medskip\raggedright
\begin{itemize}
\item Arrows compare the backbone with its CCQ variant.
\item 1.3B GLA: Wiki PPL falls by 1.88 (10\% relative).
\item 1.3B GDN: average accuracy rises, but Wiki PPL is slightly worse.
\end{itemize}

\papersource{Paper, Table 2, p. 7. Avg values use a percentage scale; no significance claim.}
\end{frame}
\begin{frame}{Several retrieval gains are large; others reverse}

\centering
\begin{tabular}{llrrr}
\toprule
Scale / backbone & Task / length & Base & CCQ & $\Delta$ pp\\
\midrule
500M / GLA & pass-key / 2K & 26.2 & 97.6 & +71.4\\
500M / GDN & number / 8K & 24.8 & 50.8 & +26.0\\
1.3B / GLA & pass-key / 4K & 47.8 & 96.8 & +49.0\\
1.3B / GLA & number / 8K & 31.2 & 60.6 & +29.4\\
1.3B / GDN & UUID / 4K & 6.0 & 46.6 & +40.6\\
\midrule
500M / GLA & UUID / 1K & 39.8 & 11.0 & $-28.8$\\
1.3B / GDN & pass-key / 8K & 99.8 & 98.2 & $-1.6$\\
\bottomrule
\end{tabular}
\par\medskip\raggedright
\alert{All models fail on UUID beyond 4K; those 8K results are not tabulated.}

\papersource{Paper, Table 3, p. 8. Accuracy in percent; pp = percentage points. Selected gains and counterexamples.}
\end{frame}
\begin{frame}{500M: CCQ improves many extrapolation curves}

\centering
\includegraphics[width=0.94\linewidth,height=0.72\textheight,keepaspectratio]{figures/length_extrap_500M.pdf}
\par
\small Axes are scaled separately. Transformer and GLA-Hedgehog are omitted as off-range.

\papersource{Paper, Fig. 3, p. 6. Token-level PPL; lower is better. No numerical values digitized from curves.}
\end{frame}
\begin{frame}{1.3B: the figure does not show universal improvement}

\centering
\includegraphics[width=0.94\linewidth,height=0.67\textheight,keepaspectratio]{figures/length_extrap_1B.pdf}
\par\small
CCQ-GDN (orange) is slightly worse than GDN (blue) on GovReport, QMSum and Qasper.

\papersource{Paper, Fig. 4, p. 7. This conflicts with the all-panels improvement claim in Sec. 4.4, p. 6.}
\end{frame}
\begin{frame}{LongBench averages improve, with task-level regressions}

\centering
\begin{tabular}{lrr}
\toprule
1.3B model & Avg & Change from backbone\\
\midrule
GLA & 14.07 & ---\\
CCQ-GLA & 15.77 & +1.70\\
GDN & 13.80 & ---\\
CCQ-GDN & 14.78 & +0.98\\
\bottomrule
\end{tabular}
\par\medskip\raggedright
\begin{itemize}
\item CCQ-GLA: TREC $24.75\to39.50$; GovReport $7.46\to9.11$.
\item CCQ-GDN: TriviaQA $21.01\to28.28$, but HotpotQA $5.77\to3.39$.
\item RepoBench-P: GLA 26.81 exceeds CCQ-GLA 26.37, despite bolding.
\end{itemize}

\papersource{Paper, Table 4, p. 8. Mixed-metric average; best-value markings and prose require checking.}
\end{frame}
\begin{frame}{Centring is not necessary for the observed gains}

\centering
\begin{tabular}{lrrr}
\toprule
1.3B GDN variant & Wiki PPL $\downarrow$ & Avg $\uparrow$ & UUID 4K $\uparrow$\\
\midrule
No CCQ & 17.98 & 55.64 & 6.0\\
Centred, learned gate & 18.01 & 56.24 & 46.6\\
Uncentred, learned gate & 17.30 & 56.20 & 55.6\\
Centred, fixed $\lambda=0.1$ & 17.99 & 55.85 & 20.0\\
\bottomrule
\end{tabular}
\par\medskip\raggedright
\begin{itemize}
\item Uncentred second moments win on WikiText and UUID 4K.
\item A learned gate beats fixed 0.1 on UUID, but strength is not matched.
\item Exact Hessian correspondence is motivation, not established necessity.
\end{itemize}

\papersource{Paper, Appendix E, p. 14; Table 7, p. 15. Final bullet: reading interpretation.}
\end{frame}
\begin{frame}{Turning CCQ off at inference preserves most retrieval gains}

\begin{block}{Intervention on the trained CCQ-Gated DeltaNet checkpoint}
Disabling the correction changes accuracy by at most:
\begin{itemize}
\item 2.2 percentage points: pass-key, 4K--100K;
\item 1.6 percentage points: number / UUID, 4K--16K.
\end{itemize}
\end{block}
\begin{itemize}
\item The advantage over a separately trained backbone largely remains.
\item This weakens a purely immediate read-time explanation.
\item It does not identify the training mechanism or prove all tasks are safe to disable.
\end{itemize}

\papersource{Paper, Appendix E, ``Inference-time intervention, p. 14. Interpretation distinguished from observation.}
\end{frame}
\begin{frame}{Decode throughput is a material implementation cost}

\centering
\begin{tabular}{lcc}
\toprule
1.3B, base $\to$ CCQ & GLA & GDN\\
\midrule
Train, k tokens/s/GPU & $41.6\to34.8$ & $45.1\to37.8$\\
Peak training memory, GiB & $87.5\to99.9$ & $81.5\to93.6$\\
4K prefill, ms & $198\to217$ & $189\to204$\\
Decode, tokens/s & $454\to298$ & $457\to297$\\
\bottomrule
\end{tabular}
\par\medskip\raggedright
\begin{itemize}
\item About 16\% less training throughput and 35\% less decode throughput.
\item Inference measurements: one H200, batch size 8.
\item Recurrent state: 64 to 96 MiB/sequence in the fp32 implementation.
\end{itemize}

\papersource{Paper, Appendix C, pp. 13--14; Appendix D, p. 14. Current unfused decode path, not projected performance.}
\end{frame}
\begin{frame}{The covariance can disagree with the effective memory}

\begin{itemize}
\item \textbf{Query mismatch:} $\Sigma_t$ is uniform over past keys; the exact Hessian uses $p(q_t)$ weights.
\item \textbf{Retention mismatch:} CCQ keeps undecayed statistics while the backbone forgets or overwrites.
\item \textbf{Task mismatch:} relevant frequent patterns may lie in high-variance directions.
\item \textbf{Scale limit:} trained only at 500M/1.3B, on GLA and GDN, up to 40B tokens.
\end{itemize}
\medskip
The Qwen 4B probe is a diagnostic, not a trained 4B CCQ model.

\papersource{Paper, Limitations, p. 9; Appendix J, p. 18; Appendix I, pp. 16--18.}
\end{frame}
\begin{frame}{What is established, and what should be tested next?}

\begin{block}{Established within the reported scope}
A recurrent query correction with a spectral bound, and several large matched-backbone retrieval improvements after training.
\end{block}
\begin{block}{Still unresolved}
How much comes from the online correction versus representations learned during CCQ training?
\end{block}
\begin{itemize}
\item Next: matched train-on/off and test-on/off experiments.
\item Add normalization-only and matched-strength gate controls.
\item Measure quality and speed when disabling CCQ at deployment.
\end{itemize}

\papersource{Synthesis of Secs. 3--4 and Appendix E. Proposed controls are reading analysis, not completed experiments.}
\end{frame}
\begin{frame}{Source and coverage}

\textbf{Dong Le, Thong Nguyen, Cong-Duy Nguyen, Anh-Tuan Luu.}\\[2mm]
\emph{Don't Read Everything: A Curvature-Conditioned Query for Linear Attention.}\\[2mm]
arXiv:2606.01294v2, revised 30 August 2026.\\
\url{https://arxiv.org/abs/2606.01294v2}
\medskip
\begin{itemize}
\item Read the version-matched author TeX and 20-page PDF, including Appendices A--J.
\item Figures reproduced from the author's source package; all locators use PDF pages.
\item Results are reported evidence, not an independent training reproduction.
\end{itemize}

\papersource{Reading date: 16 September 2026. Companion Chinese notes include detailed evidence checks.}
\end{frame}
\appendix
\begin{frame}{Backup: a shape-consistent chunkwise formulation}

Store queries and keys as rows. Let $L$ be the causal mask and $m_i$ the prefix key sum at position $t_i$.
\[
\begin{aligned}
P_{\mathrm{prev}}&=\bar Q S^K_{\mathrm{prev}},\\
P_{\mathrm{intra}}&=(\bar Q\bar K^\top\odot L)\bar K,\\
(\bar Cq)_i&=(P_{\mathrm{prev}}+P_{\mathrm{intra}})_i/t_i,\\
(\mathrm{mean\ correction})_i&=m_i(m_i^\top\bar q_i)/t_i^2.
\end{aligned}
\]
\begin{itemize}
\item Update chunk-prefix $\sum kk^\top$ and $\sum k$ after the chunk.
\item Mean correction is rowwise; avoid cross-token mixing from shorthand.
\end{itemize}

\papersource{Paper, Appendix B and Algorithm 1, pp. 12--13. Row-oriented restatement for clarity.}
\end{frame}
\begin{frame}{Backup: training throughput preserves linear scaling}

\centering
\includegraphics[width=0.96\linewidth,height=0.64\textheight,keepaspectratio]{figures/ccq_throughput.pdf}
\par\small
32K tokens per step; length and batch size vary together.\\
Controlled sweep values differ from full-run median throughput.

\papersource{Paper, Fig. 5, p. 15; full-run medians in Appendix C, p. 13.}
\end{frame}
\begin{frame}{Backup: pooled geometry is supportive, not causal}

\centering
\includegraphics[width=0.96\linewidth,height=0.50\textheight,keepaspectratio]{figures/cov_validation_projection.pdf}
\par\medskip\raggedright
\begin{itemize}
\item Red: unique needles; purple: distractors. Lower means less top-16 energy.
\item GDN gap is $-0.207$ when pooled, versus $-0.209$ in the depth summary.
\item Rare-token selection and missing query-sign conditions limit inference.
\end{itemize}

\papersource{Paper, Fig. 2, p. 5; Appendix I, pp. 16--18. Last bullet: reading interpretation.}
\end{frame}
\begin{frame}{Backup: qualify several claims using the displayed evidence}

\begin{itemize}
\item \textbf{Length PPL:} Fig. 4 contradicts the all-panels 1--2.5 PPL improvement claim for both variants.
\item \textbf{RepoBench-P:} GLA 26.81 exceeds CCQ-GLA 26.37 in Table 4.
\item \textbf{Mamba2 comparison:} CCQ-GDN gains $14.78-13.13=1.65$, not 2.64; 2.64 is CCQ-GLA's gain.
\item \textbf{Boundary case:} equal target/distractor suppression leaves the margin unchanged, not necessarily the query.
\end{itemize}
\medskip
Use numerical entries and explicit equations instead of best-value bolding or broad prose.

\papersource{Paper, Secs. 3.2, 4.4--4.5, pp. 4, 6--7; Table 4, p. 8; Appendix H, p. 16.}
\end{frame}
\end{document}