% Compile with pdflatex. Replace the demonstration content before delivery.
\documentclass[aspectratio=169,11pt]{beamer}
\usepackage[T1]{fontenc}
\usepackage[utf8]{inputenc}
\usepackage{amsmath,amssymb,graphicx,booktabs}
\usepackage{lmodern}

\definecolor{AcademicBlue}{RGB}{26,57,86}
\definecolor{AcademicTeal}{RGB}{0,105,112}
\setbeamercolor{structure}{fg=AcademicBlue}
\setbeamercolor{frametitle}{fg=AcademicBlue,bg=white}
\setbeamercolor{title}{fg=AcademicBlue}
\setbeamercolor{alerted text}{fg=AcademicTeal}
\setbeamercolor{block title}{fg=white,bg=AcademicBlue}
\setbeamercolor{block body}{fg=black,bg=AcademicBlue!5}
\setbeamertemplate{navigation symbols}{}
\setbeamertemplate{itemize items}[circle]
\setbeamertemplate{footline}{%
  \hfill\usebeamerfont{page number in head/foot}%
  \insertframenumber\,/\,\inserttotalframenumber\hspace{4mm}\vspace{3mm}}
\setbeamersize{text margin left=8mm,text margin right=8mm}
\setbeamerfont{frametitle}{size=\Large,series=\bfseries}
\setbeamerfont{title}{size=\LARGE,series=\bfseries}
\newcommand{\papersource}[1]{\par\smallskip{\scriptsize\color{gray}#1\par}}

\title[WavePrune]{WavePrune: One period is often enough for RoPE}
\subtitle{Channel-wise truncation, length extrapolation, and GPU execution}
\author{Paper: Guancheng Du, Luotian Huang, Shaowen Wang,\\Si Li, Kaifeng Lyu --- Tsinghua University}
\date{Reading seminar \quad 11 November 2026\\\href{https://arxiv.org/abs/2610.06963v1}{arXiv:2610.06963v1 (3 October 2026)}}
\begin{document}
\begin{frame}
\titlepage
\end{frame}

\begin{frame}{The key claim: repeated RoPE rotations can distract attention}
\begin{block}{Idea}
Keep each 2D RoPE channel's QK contribution only within its first rotation period.
\end{block}
\begin{itemize}
\item Training-free: HELMET improves on 4 of 5 models; Qwen3-8B: $35.7\to40.0$.
\item Pretraining: better extrapolation from 2048 to 4096 tokens.
\item A100 attention kernel, 32K, $d_h=128$: $1.15\times$ prefill, $1.24\times$ decode.
\end{itemize}
\medskip
{\small Publication not found as of 2026-11-11. \href{https://arxiv.org/abs/2610.06963}{Preprint record}.\\
\href{https://github.com/ducatiqwq/WavePrune}{Official code available}; full README forthcoming.}
\papersource{Paper, Abstract; Table 1, PDF p. 8; Table 11, p. 32. This deck is a reading presentation.}
\end{frame}

\begin{frame}{Translation invariance does not require periodic repetition}
\[
f_c=\theta^{-2c/d_h},\qquad w_c=\frac{2\pi}{f_c}
\]
For fixed query/key content, channel $c$ contributes
\[
\ell_c(\delta)=m_c^Qm_c^K\cos(\delta f_c+\omega_c),\qquad\delta=i-j.
\]
{\small $m_c^Q,m_c^K$: channel magnitudes; $\omega_c$: initial relative angle.}
\begin{itemize}
\item A common position shift preserves $\delta$: the useful invariance.
\item Advancing $\delta$ by roughly $w_c$ repeats the phase: aliasing.
\item Many frequencies create approximate recurrences across channels.
\end{itemize}
\papersource{Paper, Sec. 1--2, PDF pp. 1--3. $f_c$: radians/token; $w_c$: token distance.}
\end{frame}

\begin{frame}{Repeated phases create distant ``ghost'' sub-diagonals}
\begin{columns}[T]
\column{0.5\textwidth}\centering
\textbf{Original}\\
\includegraphics[width=\linewidth,height=0.53\textheight,keepaspectratio]{figures/attention-before.jpg}
\column{0.5\textwidth}\centering
\textbf{WavePrune}\\
\includegraphics[width=\linewidth,height=0.53\textheight,keepaspectratio]{figures/attention-after.jpg}
\end{columns}
\smallskip
Keys on the horizontal axis, queries on the vertical axis.\\
Parallel bright lines at distant lags lose strength after truncation.
\papersource{Paper, Fig. 1, PDF p. 2; Qwen3-8B layer 0 head 20; mean over 100 PG19 sequences, length 4096.}
\end{frame}

\begin{frame}{WavePrune masks channel contributions before softmax}
For causal positions $j\le i$:
\[
L_{i,j}^{\mathrm{WP}}=\frac{1}{\sqrt{d_h}}
\sum_{c=0}^{d_h/2-1}
\varphi_i^{(c)}(q_i)^\top\varphi_j^{(c)}(k_j)
\,\mathbf{1}_{|i-j|\le w_c}.
\]
\[
A=\operatorname{softmax}(L^{\mathrm{WP}}),\qquad Y=AV.
\]
\begin{itemize}
\item $\varphi_i^{(c)}$: 2D channel $c$ rotated at position $i$.
\item High frequencies stop contributing early; low frequencies reach farther.
\item A pruned term becomes zero; causal future logits remain $-\infty$.
\item Scaling stays $1/\sqrt{d_h}$; no learned pruning parameters.
\end{itemize}
\papersource{Paper, Sec. 3.1, Eq. (6), PDF p. 4. Use the model's actual, possibly rescaled frequencies.}
\end{frame}

\begin{frame}{The mask varies across distance and channel index}
\centering
\includegraphics[width=0.94\linewidth,height=0.57\textheight,keepaspectratio]{figures/mask.pdf}
\par\medskip
\raggedright
Blue: retained contributions. Hatched: pruned contributions.\\
Long-distance token pairs can still interact through low-frequency channels.
\papersource{Paper, Fig. 2, PDF p. 4. A zero logit still participates in softmax; this differs from a token window.}
\end{frame}

\begin{frame}{Theorem 1: even constant content has infinitely many near-maxima}
\begin{block}{Conditions and conclusion}
If $q_i=q$ and $k_i=k$ at every position, define
$L^\star=\sup_{\Delta\ge0}L_\Delta$.
For any $\tau>0$, infinitely many lags satisfy
\[
L_{i,i-\Delta}\ge L^\star-\tau,\qquad i>\Delta.
\]
\end{block}
\begin{enumerate}
\item Logits are a finite weighted sum of cosines.
\item Wrapped phases live in a bounded multidimensional box.
\item Pigeonhole gives arbitrarily close phase returns and logit recurrences.
\end{enumerate}
\papersource{Paper, Theorem 1, Eq. (7), PDF p. 5; direct proof, Appendix A, p. 18.}
\end{frame}

\begin{frame}{The random extension relates recurrence to channel sensitivity}
Under symmetric i.i.d. Q/K distributions (the proof uses independence):
\[
\Pr\!\left(L_{i,i-\Delta}-L_{i,i-\Delta-P}\le2\tau\right)\ge\tfrac14
\]
for infinitely many $P$, all $\Delta\ge0$ and $i>\Delta+P$.
\begin{itemize}
\item The bound is on a \emph{one-sided} logit gap.
\item Fewer sensitive mean-vector channels give a smaller recurrence bound.
\item A competing logit enlarges the softmax denominator and dilutes the target.
\end{itemize}
\medskip
Real hidden states depend on context; empirical attention maps establish that recurrences occur at practical lengths.
\papersource{Paper, Appendix B.2--B.4, PDF pp. 19--24. Sensitivity definitions and proof details in backup.}
\end{frame}

\begin{frame}{FlashWavePrune groups 16 scalar dimensions per MMA unit}
\begin{itemize}
\item Exact channel predicates disrupt a standard QK GEMM.
\item Group 16 scalar dimensions, corresponding to 8 two-dimensional RoPE channels.
\item Use the largest threshold in each group:
\end{itemize}
\[
d'_g=\max_{k\in\text{group }g}d_k.
\]
\begin{block}{Why this works on Tensor Cores}
Compute a standard partial GEMM per group, then apply one shared distance mask. Most groups are entirely skipped or retained.
\end{block}
The maximum threshold relaxes the exact mask. Most experiments use this approximation.
\papersource{Paper, Sec. 3.3, Eq. (8), PDF pp. 6--7. $d_k$: wavelength threshold of dimension $k$'s channel.}
\end{frame}

\begin{frame}{Two split points isolate the expensive boundary work}
For a query/key block pair, distances span $[l_D,r_D]$.
\medskip
\begin{tabular}{p{0.25\linewidth}p{0.28\linewidth}p{0.34\linewidth}}
\toprule
Group & Condition & Execution \\
\midrule
Fully masked & $d'_g<l_D$ & Skip QK; skip K load in decode \\[1ex]
Critical & $l_D\le d'_g<r_D$ & Group GEMM + mask \\[1ex]
Fully retained & $d'_g\ge r_D$ & Standard suffix GEMM \\
\bottomrule
\end{tabular}
\medskip
\begin{itemize}
\item Monotone thresholds give \texttt{splitL} and \texttt{splitR}.
\item Specialize reachable split pairs to control branching and code size.
\item Softmax and AV retain the standard FlashAttention path.
\end{itemize}
\papersource{Paper, Sec. 3.3, Fig. 5, PDF p. 7; Appendix C.1, p. 25.}
\end{frame}

\begin{frame}{Evaluation separates quality, learning, and kernel efficiency}
\begin{tabular}{p{0.24\linewidth}p{0.39\linewidth}p{0.24\linewidth}}
\toprule
Question & Setting & Evidence \\
\midrule
Drop-in quality & Five LLMs; 64K/128K HELMET & Task scores $\uparrow$ \\[1ex]
Learning benefit & GPT-2, 117M--774M; train 2K, test 4K & Validation loss $\downarrow$ \\[1ex]
Mechanism & Tetranacci and NIAH & Loss + attention \\[1ex]
GPU efficiency & A100; bf16; GQA 32/8; batch 32 & FA2/WP latency $\uparrow$ \\
\bottomrule
\end{tabular}
\medskip
Baseline: the same model or architecture with original RoPE attention.\\
WavePrune uses group approximation unless otherwise specified.
\papersource{Paper, Sec. 4, PDF pp. 7--10; Appendices D, F, G. 2K/4K denote 2048/4096 tokens here.}
\end{frame}

\begin{frame}{HELMET improves on four models, with a clear Llama exception}
\centering
\begin{tabular}{lrrr}
\toprule
Model & Baseline & WavePrune & Change \\
\midrule
Ministral3-3B & 39.1 & 39.7 & $+0.6$ \\
Qwen3-8B & 35.7 & 40.0 & $+4.3$ \\
Gemma3-12B & 38.2 & 38.7 & $+0.5$ \\
Qwen3-32B & 55.1 & 56.3 & $+1.2$ \\
Llama3.1-8B & 38.2 & 35.2 & $-3.0$ \\
\bottomrule
\end{tabular}
\par\medskip\raggedright
Six task groups, averaged over 64K and 128K; higher is better.\\
Qwen3-8B: Recall $+6.0$, ICL $+10.0$, Cite $-0.5$ points.\\
An average gain can include individual task regressions.
\papersource{Paper, Table 1, PDF p. 8. Adapted HELMET protocol; Qwen thinking disabled, YaRN above 32K.}
\end{frame}

\begin{frame}{The evaluation protocol matters for interpreting the scores}
\begin{itemize}
\item Six groups: Recall, RAG, Re-rank, Cite, LongQA, ICL.
\item NarrativeQA uses ROUGE-1 F1; summarization is omitted.
\item Output caps increase to 200 for RULER MV and 300 for JSON KV.
\item Table 1 reports cross-task standard deviations.
\end{itemize}
\begin{block}{Complementary general-task results}
Eight-benchmark mean scores rise modestly on the four reported models:\\
Ministral $47.47\to48.10$; Qwen3-8B $63.77\to64.09$;\\
Gemma $55.77\to56.15$; Qwen3-32B $52.05\to52.91$.
\end{block}
\papersource{Paper, Appendix D, PDF pp. 27--28; Table 7, p. 31. Table 7 does not include Llama3.1-8B.}
\end{frame}

\begin{frame}{Pretraining with WavePrune improves 2K-to-4K extrapolation}
\centering
\includegraphics[width=0.98\linewidth,height=0.54\textheight,keepaspectratio]{figures/pretraining.pdf}
\par\medskip\raggedright
OpenWebText; GPT-2 with RoPE base $10^4$; train length 2048.\\
Horizontal: training step. Vertical: validation loss at length 4096.\\
Blue improves extrapolation at all three model sizes.
\papersource{Paper, Fig. 6, PDF p. 9; Appendix G, p. 29. 64K training steps; about 15.7B training tokens.}
\end{frame}

\begin{frame}{The best result uses WavePrune in both training and evaluation}
\centering
\begin{tabular}{llrr}
\toprule
Pretraining & Evaluation & Medium loss & Large loss \\
\midrule
RoPE & RoPE & 3.8564 & 3.5618 \\
RoPE & WavePrune & 3.3314 & 3.1797 \\
WavePrune & RoPE & 4.3849 & 4.5162 \\
WavePrune & WavePrune & \textbf{3.0953} & \textbf{2.9536} \\
\bottomrule
\end{tabular}
\par\medskip\raggedright
Validation length 4096; lower is better.
\begin{itemize}
\item Inference-only integration already helps.
\item Learning with the rule gives an additional benefit.
\item Removing the rule after learning hurts: the model adapts to it.
\item At length 2048, matched train/eval losses remain comparable.
\end{itemize}
\papersource{Paper, Table 10, PDF p. 32. GPT2-medium 345M; GPT2-large 774M.}
\end{frame}

\begin{frame}{Controlled tasks test local dilution and distant interference}
\centering
\includegraphics[width=0.85\linewidth,height=0.48\textheight,keepaspectratio]{figures/synthetic.pdf}
\par\medskip\raggedright
\begin{itemize}
\item Tetranacci: predict a sum of the previous four terms, modulo 1000.
\item NIAH: recall a seven-digit number hidden in irrelevant text.
\item GPT2-base; train 2048, validate 4096; both losses improve.
\end{itemize}
\papersource{Paper, Sec. 4.3; Fig. 7, PDF p. 10; Appendix F, pp. 28--29. Each dataset: 500K samples, 99:1 split.}
\end{frame}

\begin{frame}{Attention probes support both proposed mechanisms}
\begin{block}{Local dependencies: Tetranacci}
Attention to neighbors within 15 tokens, averaged across 12 layers:\\
Baseline: $0.161\to0.123$, retention $76.7\%$.\\
WavePrune: $0.179\to0.148$, retention $82.8\%$.
\end{block}
\begin{block}{Long-range retrieval: NIAH}
Across four illustrated samples, the first retrieval heads attend to the target more consistently with WavePrune.\\
Sample losses: WP $0.030,0.005,0.003,0.001$;\\
baseline $6.328,4.734,10.840,5.767$.
\end{block}
The NIAH table contains inactive WavePrune heads too; the improvement is in the overall retrieval pattern.
\papersource{Paper, Tables 13--14, PDF p. 33. Retention percentages are reported by the paper from unrounded values.}
\end{frame}

\begin{frame}{Llama reveals when high-frequency truncation removes useful signals}
\begin{block}{Observed failure}
Llama3.1-8B, 32K NIAH: layer 16 head 21 retrieves a middle-context needle through a channel of wavelength about 571 tokens.
\end{block}
\begin{itemize}
\item The useful query--key distance greatly exceeds that wavelength.
\item WavePrune removes exactly this long-range signal.
\item Larger thresholds, fewer channels/layers, and ProLong continuation did not recover the baseline.
\end{itemize}
\medskip
\alert{Frequency is a scale prior; learned computation can cross that scale.}
\papersource{Paper, Appendix E, PDF p. 28; Table 1, p. 8. HELMET mean: $38.2\to35.2$.}
\end{frame}

\begin{frame}{One period is a useful default; grouping has an accuracy regime}
\centering
\includegraphics[width=0.88\linewidth,height=0.68\textheight,keepaspectratio]{figures/alpha.pdf}
\par\smallskip\raggedright
$w_c=\alpha\pi/f_c$; $\alpha=2$ means one period. Near this default, the two implementations are close; aggressive pruning separates them.
\papersource{Paper, Fig. 8a, PDF p. 11. Qwen3-8B, 32K, six-task HELMET average; exact peaks slightly below 2.}
\end{frame}

\begin{frame}{Benefits span several frequency groups, then saturate}
\centering
\includegraphics[width=0.88\linewidth,height=0.68\textheight,keepaspectratio]{figures/channels.pdf}
\par\smallskip\raggedright
Truncate the first $n$ highest-frequency channels. Beyond 40, remaining group windows exceed 32K and truncation has no further effect.
\papersource{Paper, Fig. 8b, PDF p. 11; Table 12, p. 32. Qwen3-8B, 32K, ICL only; grouped implementation.}
\end{frame}

\begin{frame}{Kernel speedups depend on length and head dimension}
\centering
\begin{tabular}{lrrrr}
\toprule
Phase & 8K & 16K & 32K & 64K \\
\midrule
Prefill, $d_h=128$ & $1.10\times$ & $1.13\times$ & $1.15\times$ & $1.19\times$ \\
Decode, $d_h=128$ & $1.13\times$ & $1.20\times$ & $1.24\times$ & $1.27\times$ \\
Prefill, $d_h=64$ & $1.02\times$ & $1.05\times$ & $1.09\times$ & $1.10\times$ \\
Decode, $d_h=64$ & $1.04\times$ & $1.05\times$ & $1.05\times$ & $1.05\times$ \\
\bottomrule
\end{tabular}
\par\medskip\raggedright
A100; bf16; batch 32; GQA 32 query heads / 8 KV heads.\\
Speedup = FA2 latency / WavePrune latency.
\begin{itemize}
\item QK FLOP savings favor prefill; fewer K reads also help decode.
\item At 128K, H128 decode reaches $1.28\times$; both prefill kernels OOM.
\item These are attention-kernel measurements; KV eviction is future work.
\end{itemize}
\papersource{Paper, Sec. 4.4, PDF pp. 9--10; Table 11, p. 32.}
\end{frame}

\begin{frame}{Forward wins require careful specialization; backward is harder}
\begin{itemize}
\item At 32K, removing split specialization makes H128 prefill $1.47\times$ slower than the final kernel.
\item Removing suffix-only QK computation makes it $1.46\times$ slower.
\item Removing suffix-only K loads makes H128 decode $1.14\times$ slower; H64 has a $0.99\times$ exception.
\end{itemize}
\medskip
\centering
\begin{tabular}{lrrr}
\toprule
Backward & 8K & 16K & 32K \\
\midrule
Overhead vs. FA2 & $+8.58\%$ & $+3.81\%$ & $+0.18\%$ \\
\bottomrule
\end{tabular}
\par\medskip\raggedright
Backward needs masks before MMA and shares work across $dQ,dK$; full split-pair specialization causes code and local-memory costs.
\papersource{Paper, Appendix C, Tables 3--4, PDF pp. 25--27.}
\end{frame}

\begin{frame}{The contribution connects a positional mechanism to executable sparsity}
\begin{enumerate}
\item A per-frequency finite-support rule can preserve relative positions while reducing periodic distractions.
\item Quality improves for several LLMs and small-model extrapolation; Llama exposes a concrete failure mode.
\item A grouped approximation translates QK sparsity into practical GPU gains.
\end{enumerate}
\begin{block}{Next experiments suggested by these results}
Head-specific thresholds that preserve retrieval signals; larger-scale pretraining; actual KV-capacity reclamation.
\end{block}
\papersource{Paper, Sec. 6, PDF pp. 11--12; Appendix E. Future experiments are proposals, not reported results.}
\end{frame}

\begin{frame}{References and relation to concurrent work}
\textbf{Primary paper}\\
Du, Huang, Wang, Li, and Lyu (2026).\\
\emph{WavePrune: One period is often enough for RoPE}.\\
\href{https://arxiv.org/abs/2610.06963v1}{arXiv:2610.06963v1} \quad
\href{https://github.com/ducatiqwq/WavePrune}{Official implementation}
\medskip
\begin{itemize}
\item Complements frequency rescaling such as YaRN; uses updated frequencies.
\item Concurrent ATFlash also uses wavelength windows and grouping.
\item Authors distinguish WavePrune by a one-period default, ghost-diagonal analysis, and pretraining experiments.
\end{itemize}
\papersource{Paper, Sec. 5, PDF pp. 10--11. Publication not found as of 2026-11-11; code files present, full README pending.}
\end{frame}

\appendix
\begin{frame}{Backup: the recurrence proof uses phase returns}
\[
L_\Delta=\frac1{\sqrt{d_h}}\sum_c m_c^Qm_c^K\cos(\Delta f_c+\omega_c)
\]
\[
|L_{\Delta+P}-L_\Delta|\le
\frac1{\sqrt{d_h}}\sum_c m_c^Qm_c^K|\operatorname{wrap}(Pf_c)|.
\]
\begin{itemize}
\item Partition the bounded phase box into finitely many small cells.
\item Infinitely many positions share one cell; their differences give infinitely many almost-periods.
\item Start within $\tau/2$ of the supremum and shift by a $\tau/2$ almost-period.
\end{itemize}
\papersource{Paper, Appendix A, PDF p. 18. Constant Q/K; a raw full-dimensional bound can be extremely large.}
\end{frame}

\begin{frame}{Backup: sparsity refines the recurrence bound}
\[
t_c=\frac{2\pi m_c^Qm_c^K}{\sqrt{d_h}},\qquad
\sigma_\tau(t)=\min_{\|z-t\|_1\le\tau}\|z\|_0
\]
\[
P_{\min}\le\prod_{i=1}^{\sigma_\tau(t)}
\left\lceil\frac{\sigma_\tau(t)t_i^s}{\tau}\right\rceil.
\]
\begin{itemize}
\item $m_c^Q,m_c^K$: channel magnitudes of mean vectors; $t_i^s$: sorted sensitivities.
\item Absorb weak channels into the error budget; finely cover strong channels.
\item Four reflected inner products average to the mean-vector inner product, giving a probability lower bound of $1/4$.
\end{itemize}
\papersource{Paper, Appendix B.2--B.4, PDF pp. 19--24. Visual contribution sparsity need not equal sensitivity sparsity.}
\end{frame}

\begin{frame}{Backup: a competing logit limits softmax concentration}
Let $L_t$ be the target logit and $L_g$ a ghost competitor. If
$L_t-L_g\le2\tau$, then
\[
A_t\le\frac{e^{L_t}}{e^{L_t}+e^{L_g}}
=\frac1{1+e^{L_g-L_t}}
\le\frac1{1+e^{-2\tau}}.
\]
Taking $\tau=\tfrac12\log((1-\epsilon)/\epsilon)$ gives
\[
\Pr(A_{i,i-\Delta}\le1-\epsilon)\ge\tfrac14
\]
once a suitable competing lag is available.
\papersource{Paper, Corollary 1, Appendix B.2.3, PDF p. 20. $0<\epsilon<1/2$; same distributional conditions as Theorem 2.}
\end{frame}

\begin{frame}{Backup: native Qwen3 group thresholds explain the plateau}
\centering
\begin{tabular}{lrrrr}
\toprule
Group end channel & 7 & 15 & 23 & 31 \\
Wavelength (tokens) & 28 & 160 & 900 & 5063 \\
\midrule
Group end channel & 39 & 47 & 55 & 63 \\
Wavelength (tokens) & 28472 & 160114 & 900388 & 5063255 \\
\bottomrule
\end{tabular}
\par\medskip\raggedright
Values rounded down, native RoPE base $10^6$.\\
At 32K, the groups ending at channels 47, 55, and 63 retain all causal distances. Activating truncation there changes nothing.
\papersource{Paper, Table 12, PDF p. 32. With frequency rescaling, thresholds must be recomputed.}
\end{frame}
\end{document}