% Research discussion deck based on arXiv:2601.22156v1. % Self-authored from the arxiv-paper-reader Beamer template. % Build: python3 build_slides.py slides.en.tex \documentclass[aspectratio=169,11pt]{beamer} \usepackage[T1]{fontenc} \usepackage[utf8]{inputenc} \usepackage{lmodern} \usepackage{amsmath,amssymb,graphicx,booktabs} \pdfstringdefDisableCommands{\def\translate#1{#1}} \definecolor{AcademicBlue}{RGB}{26,57,86} \definecolor{AcademicTeal}{RGB}{0,105,112} \definecolor{ResultOrange}{RGB}{181,103,0} \setbeamercolor{structure}{fg=AcademicBlue} \setbeamercolor{frametitle}{fg=AcademicBlue,bg=white} \setbeamercolor{title}{fg=AcademicBlue} \setbeamercolor{alerted text}{fg=AcademicTeal} \setbeamercolor{block title}{fg=white,bg=AcademicBlue} \setbeamercolor{block body}{fg=black,bg=AcademicBlue!5} \setbeamertemplate{navigation symbols}{} \setbeamertemplate{itemize items}[circle] \setbeamertemplate{footline}{% \hspace{4mm}{\scriptsize\color{gray}HypeNet + HALO \quad | \quad arXiv:2601.22156v1}% \hfill\usebeamerfont{page number in head/foot}% \insertframenumber\,/\,\inserttotalframenumber\hspace{4mm}\vspace{3mm}} \setbeamersize{text margin left=8mm,text margin right=8mm} \setbeamerfont{frametitle}{size=\Large,series=\bfseries} \setbeamerfont{title}{size=\LARGE,series=\bfseries} \newcommand{\papersource}[1]{\par\smallskip{\scriptsize\color{gray}#1\par}} \newcommand{\R}{\mathbb R} \newcommand{\Norm}{\operatorname{Norm}} \newcommand{\RoPE}{\operatorname{RoPE}} \newcommand{\KL}{D_{\mathrm{KL}}} \newcommand{\fig}[2]{\includegraphics[width=#2\linewidth,height=0.59\textheight,keepaspectratio]{figures/#1}} \title[HypeNet + HALO]{Hybrid Linear Attention Done Right} \subtitle{Efficient Distillation and Effective Architectures for Extremely Long Contexts} \author{Research discussion of Chen et al.} \date{Paper: 29 January 2026 \quad Discussion: 15 October 2026} \begin{document} \begin{frame}[plain] \vspace{0pt} {\LARGE\bfseries\color{AcademicBlue}Hybrid Linear Attention\\Done Right\par} \vspace{3mm} {\large Efficient Distillation and Effective Architectures\\for Extremely Long Contexts\par} \vspace{5mm} {\small Paper authors: Yingfa Chen$^\ast$, Zhen Leng Thai$^\ast$, Zihan Zhou, Zhu Zhang,\\ Xingyu Shen, Shuo Wang, Chaojun Xiao, Xu Han, Zhiyuan Liu\par} \vspace{2mm} {\small $^\ast$Equal contribution \quad | \quad arXiv:2601.22156v1, 29 Jan 2026\par} \begin{block}{The idea} Reuse a Transformer, retain selected global attention layers,\\ and put rotational position information in recurrent layers. \end{block} \papersource{Research discussion, 15 Oct 2026. Main talk: slides 1--22 (about 25 minutes); backup: 23--27.} \papersource{\href{https://arxiv.org/abs/2601.22156v1}{Paper: arxiv.org/abs/2601.22156v1}} \end{frame} \begin{frame}{Conversion must preserve long-distance retrieval} \begin{columns}[T,onlytextwidth] \begin{column}{0.48\textwidth} \begin{block}{Why convert?} Reuse pretrained Q/K/V/O weights and knowledge instead of paying for full pretraining. \end{block} \begin{itemize} \item Prior conversion budgets: about 20B--400B tokens for several major methods. \item HALO: about \alert{2.3B additional training tokens}. \end{itemize} \end{column} \begin{column}{0.48\textwidth} \begin{block}{What can go wrong?} Short-task scores recover, while the model loses access to distant details. \end{block} \begin{itemize} \item Fixed recurrent states compress history. \item A small number of full-attention layers preserve direct access. \end{itemize} \end{column} \end{columns} \vspace{3mm} \alert{Two design questions: which layers stay, and how should each type encode position?} \papersource{Paper, Secs. 1--3, PDF pp. 1--4; Table 1, p. 2. SMART is a lower-budget exception, reported as $>$7B.} \end{frame} \begin{frame}{HALO separates local adaptation from global recovery} \centering \fig{fig-02.pdf}{0.99} \begin{itemize} \item First align candidate RNNs; then select attention layers using inference. \item Distill the assembled hybrid and finetune on longer sequences. \end{itemize} \papersource{Paper, Fig. 2 and Sec. 4, PDF pp. 4--5. Dashed boxes: no training; solid boxes: training.} \end{frame} \begin{frame}{Three training stages use about 2.3B tokens} \centering \begin{tabular}{lrrl} \toprule Stage & Tokens & Context & Learning rate \\ \midrule 1. Layer alignment & 320M & 512 & $10^{-3}\rightarrow10^{-5}$ \\ 2. End-to-end KD & 1B & 512 & $\eta_2\rightarrow10^{-5}$ \\ 3. Long-context FT & 1B & 16K & $10^{-5}$, constant \\ \bottomrule \end{tabular} \vspace{3mm} \[ \mathcal L_1^{(l)} =\operatorname{MSE}\!\left( \mathbf Y_{\rm teacher}^{(l)}, \operatorname{RNN}^{(l)}(\mathbf X^{(l-1)})\right) \] \[ \mathcal L_2=\KL\!\left(f_{\rm orig}(\mathbf X)\Vert f_{\rm hybrid}(\mathbf X)\right) \] \begin{itemize} \item Stage 1 trains only candidate RNN layers; Stage 2 trains the hybrid. \item FineWeb-Edu, BF16, AdamW; $\eta_2=10^{-4}$ for the 2B student. \end{itemize} \papersource{Paper, Eqs. (7), (10), PDF pp. 4--5; Tables 5--6, p. 14. Stage 3 loss is not explicitly specified.} \end{frame} \begin{frame}{Keep layers whose replacement especially harms recall} \[ s_i= \frac{R_{\max}-\mathcal R(M^{(i)})} {C_{\max}-\mathcal C(M^{(i)})+10^{-6}}, \qquad \mathcal I_{\rm attn}=\underset{i}{\operatorname{Top}\!-\!k}(s_i) \] \begin{itemize} \item $M^{(i)}$: replace \emph{only} teacher layer $i$ with its aligned RNN. \item $R_{\max},C_{\max}$: best recall / CSR across these replacements. \item High score: large recall drop, relatively small CSR drop. \end{itemize} \begin{block}{For Qwen3-1.7B} Keep 7 of 28 full-attention layers: $\{3,21,2,9,25,6,8\}$ in importance order, using zero-based indices. \end{block} {\small This is a single-layer heuristic; a near-zero denominator can amplify score noise.} \papersource{Paper, Sec. 4.3, Eqs. (8)--(9), PDF p. 4; Table 7, p. 15. General rule stated as $k=\lfloor L/4\rfloor$.} \end{frame} \begin{frame}{HyPE places RoPE in the recurrent layers} \begin{columns}[T,onlytextwidth] \begin{column}{0.48\textwidth} \begin{block}{Recurrent layers} \large RoPE on Q and K\\[2mm] Fixed-size state\\[2mm] Decaying local context \end{block} \end{column} \begin{column}{0.48\textwidth} \begin{block}{Full-attention layers} \large NoPE\\[2mm] Explicit KV history\\[2mm] Position-dependent logit scaling \end{block} \end{column} \end{columns} \vspace{3mm} \begin{itemize} \item Local order is represented through recurrent dynamics and RoPE. \item Global attention avoids RoPE extrapolation at long distances. \item Its inputs still contain order information from lower layers. \end{itemize} \papersource{Paper, Sec. 5.1 and Fig. 3, PDF p. 5. The local/global division is the authors' design explanation.} \end{frame} \begin{frame}{The default RNN is fixed-decay Lightning Attention} \[ \begin{aligned} \mathbf q_t&=\Norm(\mathbf x_t\mathbf W_q),& \mathbf k_t&=\Norm(\mathbf x_t\mathbf W_k),& \mathbf v_t&=\mathbf x_t\mathbf W_v,\\ \widetilde{\mathbf q}_t&=\RoPE_t(\mathbf q_t),& \widetilde{\mathbf k}_t&=\RoPE_t(\mathbf k_t)/\sqrt{d_h},\\[1mm] \mathbf S_t&=\gamma_h\mathbf S_{t-1} +\widetilde{\mathbf k}_t^\top\mathbf v_t,& \mathbf o_t&=\widetilde{\mathbf q}_t\mathbf S_t . \end{aligned} \] \[ \gamma_h=\exp(-2^{-8h/H}),\qquad h=1,\ldots,H. \] \begin{itemize} \item Row vectors $\mathbf q,\mathbf k,\mathbf v\in\R^{1\times d_h}$; state $\mathbf S_t\in\R^{d_h\times d_h}$. \item The forget rate depends on the head, not the input token. \item Output normalization and a sigmoid gate precede the output projection. \end{itemize} \papersource{Paper, Appendix A, Eqs. (18)--(19), PDF pp. 13--14. Head index is omitted on vectors and states.} \end{frame} \begin{frame}{Fixed decay gives a concrete local memory scale} \[ \mathbf o_t=\sum_{i=1}^t \gamma_h^{\,t-i} \bigl(\widetilde{\mathbf q}_t\widetilde{\mathbf k}_i^\top\bigr)\mathbf v_i \qquad (\mathbf S_0=0). \] \begin{block}{A useful derivation} The distance for a $1/e$ decay is \[ \tau_h=\frac{1}{-\ln\gamma_h}=2^{8h/H} \quad\Longrightarrow\quad \tau_{\max}=256\text{ tokens}. \] \end{block} \begin{itemize} \item At a 4,096-token lag, even the slowest head has factor $e^{-16}$. \item RoPE provides relative position within this decaying direct path. \item Global attention and depth can still carry distant information. \end{itemize} \papersource{Teaching derivation from paper, Eqs. (18)--(19), PDF pp. 13--14. This is not a hard network-wide window.} \end{frame} \begin{frame}{NoPE attention needs sharper logits as context grows} \[ \alpha_{t,i}= \frac{\exp(s_t\mathbf q_t\mathbf k_i^\top/\sqrt{d_h})} {\sum_{j=1}^{t}\exp(s_t\mathbf q_t\mathbf k_j^\top/\sqrt{d_h})}, \qquad s_t=\frac{\ln(t+a)}{\ln a}. \] \begin{itemize} \item Apply at inference; choose $a$ after training using document loss. \item For HypeNet-2B, $a=500$: $s_{512}\approx1.11$, $s_{256K}\approx2.01$. \item Scale Q before the attention operator; no new attention kernel is needed. \end{itemize} \begin{block}{Why logarithmic growth is plausible} With one relevant logit gap $\Delta$ and $n$ equal distractors: $p_{\rm relevant}=1/(1+n e^{-s\Delta})$. Keeping $p$ stable requires $s\Delta$ to grow with $\ln n$. \end{block} \papersource{Paper, Eq. (11), p. 5; Table 6, p. 14. Bottom block: teaching model. Current-query index corrects Eq. (17).} \end{frame} \begin{frame}{The architecture also changes normalization and heads} \centering \begin{tabular}{p{0.29\linewidth}p{0.62\linewidth}} \toprule Modification & Purpose / implementation \\ \midrule RNN QK normalization & Control Q/K scale before rotation and state updates. \\[2mm] RNN GQA $\to$ MHA & Copy KV projections to independently trained heads. \\[2mm] Output gates & Normalize mixer output; multiply by sigmoid gate before O projection. \\[2mm] No short convolution & Added convolution failed for 1.7B conversion and did not help 8B. \\ \bottomrule \end{tabular} \vspace{4mm} \alert{More parameters, less KV traffic:} the authors report roughly 10\% parameter growth. \papersource{Paper, Sec. 5.2, PDF p. 6; Appendix A, p. 13; Appendix D, p. 14. Attention output gates are added at Stage 2.} \end{frame} \begin{frame}{Separate conversion quality from architecture potential} \centering \begin{tabular}{p{0.27\linewidth}p{0.30\linewidth}p{0.30\linewidth}} \toprule & Qwen3 conversion & Training from scratch \\ \midrule Model scale & 1.7/4/8B teachers;\newline 2/5/9B students & About 500M \\ Training tokens & About 2.3B added & 20B \\ Training context & 512, then 16K & 4K \\ Attention placement & HALO selection & 1 attention + 3 RNNs \\ Main question & Affordable transfer & Length generalization \\ \bottomrule \end{tabular} \vspace{3mm} \begin{itemize} \item CSR: seven zero-shot downstream tasks; higher is better. \item NIAH: RULER Single-1/2/3; unspecified NIAH denotes their mean. \item Long-context Qwen3 evaluation uses YaRN beyond 40,960 tokens. \end{itemize} \papersource{Paper, Secs. 6.1--6.3, PDF pp. 6--7; Appendix E.2, p. 16; Appendix H, pp. 18--20.} \end{frame} \begin{frame}{At 256K, retrieval gains vary across the three tasks} \centering \setlength{\tabcolsep}{10pt} \begin{tabular}{lrrrr} \toprule Model & Train tokens & Single-1 & Single-2 & Single-3 \\ \midrule Qwen3-1.7B + YaRN & --- & 17.0 & 19.2 & 19.0 \\ Jet-Nemotron-2B & 400B & 0.0 & 0.0 & 0.0 \\ KL-LS-GDN-3B & 25B & 14.8 & 10.4 & 11.0 \\ \textbf{HypeNet-2B} & \textbf{2.3B} & \textbf{99.8} & \textbf{86.2} & \textbf{48.8} \\ \bottomrule \end{tabular} \vspace{5mm} \begin{itemize} \item Very large gains at extreme lengths with a smaller conversion budget. \item Single-3 remains difficult: 48.8\% is far from perfect recall. \item At 64K Single-3, the teacher still wins: 98.4 vs.\ 72.6. \end{itemize} \papersource{Paper, Table 2, PDF p. 7. Accuracy (\%), higher is better. These are published-model comparisons.} \papersource{Table 2 single-task means do not exactly match Table 4 aggregates; keep the two tables separate (backup slide 27).} \end{frame} \begin{frame}{Layer selection changes recall much more than CSR} \centering \begin{tabular}{lrrr} \toprule Selection inside HALO & CSR & NIAH 128K & NIAH 256K \\ \midrule \textbf{HALO score} & \textbf{55.9} & \textbf{79.9} & \textbf{74.3} \\ Jet-Nemotron rule & 55.0 & 63.7 & 56.2 \\ KL-LS rule & 55.3 & 58.3 & 44.3 \\ Uniform across depth & 54.0 & 61.9 & 50.9 \\ Uniform in latter half & 55.8 & 39.2 & 40.4 \\ \bottomrule \end{tabular} \vspace{4mm} \begin{itemize} \item Same conversion pipeline; swap the selection rule. \item HALO vs.\ latter-half uniform: only 0.1 CSR points apart, but \alert{33.9 points} apart on 256K retrieval. \item Teacher CSR is 58.5: the student retains a 2.6-point gap. \end{itemize} \papersource{Paper, Table 4 and Sec. 6.5, PDF p. 8. Full Jet-Nemotron / KL-LS training procedures are not rerun here.} \end{frame} \begin{frame}{QK normalization and global NoPE matter most at 128K} \centering \begin{tabular}{lrr} \toprule Converted 2B configuration & CSR & NIAH 128K \\ \midrule \textbf{Full HypeNet} & \textbf{55.9} & \textbf{79.9} \\ Remove RNN RoPE & 53.8 & 47.9 \\ Add attention RoPE & 55.8 & 19.7 \\ Remove RNN QK norm & 55.3 & 17.3 \\ Keep RNN shared KV & 55.8 & 83.5 \\ Remove RNN output gate & 55.6 & 74.5 \\ Remove attention output gate & 55.4 & 80.9 \\ \bottomrule \end{tabular} \vspace{3mm} \begin{itemize} \item Tiny CSR changes can hide 60+ point retrieval losses. \item Independent KV heads and attention gating do not win at every length. \end{itemize} \papersource{Paper, Table 3, PDF p. 8. Same Qwen3-1.7B conversion setting; scores in percent.} \end{frame} \begin{frame}{From scratch, HyPE reaches 64 times the training length} \centering \fig{fig-04.pdf}{0.98} \begin{itemize} \item About 500M parameters, 20B tokens, 4K training context. \item \alert{93.5\% NIAH at 256K}; this is a separate from-scratch model. \end{itemize} \papersource{Paper, Fig. 4 and Sec. 6.3, PDF p. 7. Red line: training length; orange solid line: complete HypeNet.} \end{frame} \begin{frame}{Lightning has the strongest extrapolation in this setup} \centering \fig{fig-05.pdf}{0.98} \begin{itemize} \item Authors' hypothesis: fixed decay gives more stable length behavior. \item QK norms and other module settings also differ across variants. \end{itemize} \papersource{Paper, Fig. 5, PDF p. 7; discussion, p. 8; Table 15, p. 20. All variants omit short convolutions.} \end{frame} \begin{frame}{Dynamic scaling sustains the longest-range performance} \centering \fig{fig-08.pdf}{0.98} \begin{itemize} \item No scaling fails early; constant $s=1.5$ delays the decline. \item Position-dependent $s_t=\log_a(t+a)$ maintains high accuracy. \end{itemize} \papersource{Paper, Appendix F.1, PDF pp. 16--17; Fig. 8, p. 17. From-scratch, 4K-context setting.} \end{frame} \begin{frame}{Less KV memory offsets the extra parameters} \centering \includegraphics[width=0.94\linewidth,height=0.50\textheight,keepaspectratio]{figures/fig-01-tradeoff.pdf} \par{\small Black: Qwen3 \qquad \textcolor{ResultOrange}{Orange: HypeNet}} \vspace{2mm} \begin{itemize} \item At 128K, HypeNet offers a better quality--memory--throughput tradeoff. \item For the 2B student, only 7 of 28 layers keep growing KV caches. \item Full attention remains: prefill still contains a quadratic term. \end{itemize} \papersource{Paper, Fig. 1, left/center panels, PDF p. 2. Original vector panels cropped; legend restated above. BF16, batch 1.} \end{frame} \begin{frame}{At 512K: 3.0x faster decode, 3.4x faster prefill} \begin{columns}[T,onlytextwidth] \begin{column}{0.49\textwidth} \centering {\bfseries Decode: milliseconds / output token\par} \vspace{2mm} \includegraphics[width=\linewidth,height=0.47\textheight,keepaspectratio]{figures/fig-01-decode.pdf} \end{column} \begin{column}{0.49\textwidth} \centering {\bfseries Prefill: microseconds / input token\par} \vspace{2mm} \includegraphics[width=\linewidth,height=0.47\textheight,keepaspectratio]{figures/fig-06.pdf} \end{column} \end{columns} \vspace{4mm} \begin{itemize} \item Single A800, batch 1, BF16; Qwen3-1.7B vs.\ HypeNet-2B. \item Qwen3 runs out of memory at 1M; HypeNet has runtime measurements. \item The reported retrieval evaluations extend to 256K. \end{itemize} \papersource{Paper, Fig. 1 right, p. 2; Fig. 6 and Sec. 6.6, PDF pp. 8--9; efficiency setup, p. 7.} \end{frame} \begin{frame}{Training is affordable; selection still costs inference} \centering \begin{tabular}{lrr} \toprule Component & Token budget & A800 GPU-hours \\ \midrule Stage 1 & 320M training & 10.0 \\ Layer selection & 234M inference & Not reported \\ Stage 2 & 1B training & 43.4 \\ Stage 3 & 1B training & 37.7 \\ \midrule Three training stages & About 2.3B & 91.1 \\ \bottomrule \end{tabular} \vspace{4mm} \begin{itemize} \item Selection evaluates 28 variants on about 8.36M tokens each. \item The 91.1-hour sum excludes the unspecified selection runtime. \item These budgets reuse a pretrained teacher; searches add further cost. \end{itemize} \papersource{Paper, Table 8 and Appendix E, PDF p. 15; Qwen3-1.7B conversion. Reported FLOPs have an unresolved factor mismatch.} \end{frame} \begin{frame}{The strongest evidence is for synthetic retrieval} \begin{itemize} \item \textbf{General ability:} CSR drops from 58.5 to 55.9 for the 2B conversion. \item \textbf{Retrieval:} 256K Single-3 reaches 48.8\%, with substantial room left. \item \textbf{Behavior:} FineWeb-Edu conversion may weaken instruction following and post-training alignment. \item \textbf{Efficiency:} speedups are measured at batch 1 on an A800; the hybrid still contains full attention. \item \textbf{Reproduction:} v1 has inconsistent layer counts and cross-table aggregates; avoid mixing those entries. \end{itemize} \begin{block}{A useful next test} Evaluate real long-document tasks and recover instruction behavior; isolate fixed decay under matched module configurations. \end{block} \papersource{Paper, Tables 2--4, pp. 7--8; Limitations, p. 9; Tables 6--7, pp. 14--15. Next test: reader's analysis.} \end{frame} \begin{frame}{Three lessons for designing converted hybrids} \begin{enumerate} \item \textbf{Choose what to preserve.}\\ Retrieval-sensitive layer selection protects useful global access. \vspace{3mm} \item \textbf{Assign position information by function.}\\ Local recurrent RoPE and global NoPE work together with dynamic scaling. \vspace{3mm} \item \textbf{Evaluate the full tradeoff.}\\ Low conversion cost and long-input speed come with measurable quality and behavior constraints. \end{enumerate} \vspace{3mm} \papersource{Yingfa Chen et al., Hybrid Linear Attention Done Right: Efficient Distillation and Effective Architectures for Extremely Long Contexts.} \papersource{\href{https://arxiv.org/abs/2601.22156v1}{arXiv:2601.22156v1} \quad Author-linked code: \href{https://github.com/THUNLP/hybrid-linear-attention}{THUNLP/hybrid-linear-attention}.} \end{frame} \appendix \begin{frame}{Backup: longer layer alignment is not always better} \centering \begin{tabular}{lrrr} \toprule Stage 1 tokens & CSR & NIAH 16K & NIAH 128K \\ \midrule 100M & 55.2 & 80.9 & 79.9 \\ 320M & 55.4 & 88.2 & 80.9 \\ 625M & 55.4 & 90.0 & 75.6 \\ 1.3B & 55.1 & 81.0 & 61.2 \\ \bottomrule \end{tabular} \vspace{5mm} \begin{itemize} \item More local output matching does not ensure a better final hybrid. \item At 128K, 1.3B alignment tokens score 19.7 points below 320M. \item Compare within this ablation: its reference row differs from the main model. \end{itemize} \papersource{Paper, Table 9, Stage 1 ablations, and Appendix F.1.1, PDF p. 17. Scores in percent.} \end{frame} \begin{frame}{Backup: the learning rate changes the quality balance} \centering \begin{tabular}{lrr} \toprule Stage 2 maximum LR & CSR & NIAH 128K \\ \midrule $10^{-5}$ & 46.9 & 60.7 \\ $3\times10^{-5}$ & 55.5 & 54.9 \\ $10^{-4}$ & 56.4 & 68.7 \\ $3\times10^{-4}$ & 46.0 & 36.1 \\ $10^{-3}$ & 36.8 & 75.1 \\ \bottomrule \end{tabular} \vspace{4mm} \begin{itemize} \item $10^{-4}$ gives the best CSR in this group. \item Optimizing only long-context recall would favor a different setting. \item These are Table 9 ablations, not the final-model scores in Tables 3--4. \end{itemize} \papersource{Paper, Table 9, Stage 2 ablations, and Appendix F.1.1, PDF p. 17.} \end{frame} \begin{frame}{Backup: Lightning also has an efficient kernel} \centering \fig{fig-07.pdf}{0.98} \begin{itemize} \item Lightning is close to sliding-window attention with window 512. \item This is a mixer benchmark, separate from whole-model speedups. \end{itemize} \papersource{Paper, Fig. 7, PDF p. 16; Appendix E.1. Single A800-80GB, BF16; vertical axis is logarithmic (units as labeled).} \end{frame} \begin{frame}{Backup: interface compatibility does not ensure convergence} \[ \mathbf S_t=\mathbf F_t\mathbf S_{t-1}+\mathbf k_t^\top\mathbf v_t, \qquad \mathbf y_t=\mathbf q_t\mathbf S_t\mathbf W_o^\top. \] \begin{itemize} \item Q/K/V/O or equivalent projections support weight transfer into several recurrent families. \item \textbf{Mamba2:} the paper uses a multi-head adaptation to inherit attention projections. \item \textbf{KDA:} Stage 2 gradients became infinite after a few steps; reducing the learning rate did not fix the reported run. \item \textbf{Module comparison:} QK normalization, activations and gate parameterizations differ; all tested from-scratch RNNs omit short convolution. \end{itemize} \papersource{Paper, Appendix G, PDF pp. 17--18; Tables 10--11, p. 18; Table 15, p. 20.} \end{frame} \begin{frame}{Backup: use each v1 result under its own stated setting} \begin{itemize} \item \textbf{Aggregates:} Table 2 HypeNet means are 80.8 at 128K and 78.3 at 256K; Table 4 reports 79.9 and 74.3. \item \textbf{Layer counts:} Table 6 reverses the 2B RNN/attention counts. Larger-model totals are inconsistent; the 4B-teacher row in Table 7 boxes 8 layers despite the stated $k=\lfloor36/4\rfloor=9$. \item \textbf{Attention formula:} Eq. (17)'s denominator should use the current query $\widetilde{\mathbf q}_t$, not $\widetilde{\mathbf q}_j$. \item \textbf{Training details:} Stage 3's loss is unspecified; Table 8's total FLOPs do not equal tokens times its reported FLOPs/token. \end{itemize} \vspace{2mm} {\small The notes retain the original table values and label the softmax correction.} \papersource{Paper, Tables 2 and 4, pp. 7--8; Eq. (17), p. 13; Tables 5--8, pp. 14--15. Arithmetic comparisons: reader's analysis.} \end{frame} \end{document}