% Based on the arxiv-paper-reader Beamer template. Compile with pdfLaTeX.
\documentclass[aspectratio=169,11pt]{beamer}
\usepackage[T1]{fontenc}
\usepackage[utf8]{inputenc}
\usepackage{lmodern}
\usepackage{amsmath,amssymb,graphicx,booktabs}
\definecolor{AcademicBlue}{RGB}{26,57,86}
\definecolor{AcademicTeal}{RGB}{0,105,112}
\setbeamercolor{structure}{fg=AcademicBlue}
\setbeamercolor{frametitle}{fg=AcademicBlue,bg=white}
\setbeamercolor{title}{fg=AcademicBlue}
\setbeamercolor{alerted text}{fg=AcademicTeal}
\setbeamercolor{block title}{fg=white,bg=AcademicBlue}
\setbeamercolor{block body}{fg=black,bg=AcademicBlue!5}
\setbeamertemplate{navigation symbols}{}
\setbeamertemplate{itemize items}[circle]
\setbeamertemplate{footline}{\hfill\usebeamerfont{page number in head/foot}\insertframenumber\,/\,\inserttotalframenumber\hspace{4mm}\vspace{3mm}}
\setbeamersize{text margin left=8mm,text margin right=8mm}
\setbeamerfont{frametitle}{size=\Large,series=\bfseries}
\setbeamerfont{title}{size=\LARGE,series=\bfseries}
\newcommand{\papersource}[1]{\par\smallskip{\scriptsize\color{gray}#1\par}}
\newcommand{\fig}[2][4.6cm]{\begin{center}\includegraphics[width=0.98\linewidth,height=#1,keepaspectratio]{figures/#2.pdf}\end{center}}
\title[Hybrid Position]{Mechanics of Long-Context Hybrid Models}
\subtitle{Part 1.1: From Hybrid Attention to Hybrid Position\\Research group reading: mechanisms, interventions, and evidence}
\author{Paper authors: Xiaoran Liu, Ziwei He, Xipeng Qiu}
\date{arXiv:2610.10114v1 $\mid$ October 10, 2026}

\begin{document}

\begin{frame}
\titlepage
\vspace{-5mm}
\begin{center}\color{AcademicTeal}Local positional control and global retrieval need different recipes\\for inference extrapolation and continued training.\end{center}
\end{frame}

\begin{frame}{Two ways to obtain a longer context}
\begin{columns}[T]
\begin{column}{.48\textwidth}
\begin{block}{Length extrapolation}
Train at 4K, then evaluate at 8K--64K.\\[2mm]
\textbf{No weight updates.}\\[2mm]
Question: can the learned computation survive a longer input?
\end{block}
\end{column}
\begin{column}{.48\textwidth}
\begin{block}{Context extension}
Continue training at 32K for 5B tokens.\\[2mm]
\textbf{Weights are updated.}\\[2mm]
Question: does the model learn to use additional distant context?
\end{block}
\end{column}
\end{columns}
\medskip
64K remains beyond the training length even after 32K extension.
\papersource{Paper, Secs. 1 and 3.1, PDF pp. 2--4.}
\end{frame}

\begin{frame}{A controlled comparison, with two training stages}
\centering
\begin{tabular}{lcc}
\toprule
 & Initial pretraining & Continual pretraining \\
\midrule
Context length & 4K & 32K \\
Training tokens & 50B & 5B \\
Corpus & DCLM-Baseline-1.0 & LongWanjuan \\
Batch tokens & 1M & 0.5M \\
Peak learning rate & $10^{-3}$ & $10^{-3}$ \\
\bottomrule
\end{tabular}
\medskip
\begin{itemize}
\item Model sizes: 376M, 776M, 1B; additional 3B verification.
\item MHA; Llama 3 tokenizer; default efficient:NoPE ratio = 3:1.
\item Retrieval: RULER. State tracking: BABILong. Modeling: PG19.
\end{itemize}
\papersource{Paper, Sec. 3.1, p. 4; Appendix A and Table 2, p. 32. Length and corpus both change across stages.}
\end{frame}

\begin{frame}{The global branch is paired with positional bias}
\centering
\begin{tabular}{lll}
\toprule
Branch & Direct access & Positional structure \\
\midrule
NoPE full attention & All past tokens & No explicit RoPE \\
RoPE full attention & All past tokens & Rotary positions \\
SWA & Recent window & RoPE + local mask \\
GLA / GDN & Recurrent state & Data-dependent gates \\
\bottomrule
\end{tabular}
\medskip
\begin{columns}[T]
\begin{column}{.48\textwidth}
\begin{block}{Layer-wise hybrid (LH)}
Three efficient layers, then one full NoPE layer, repeated.
\end{block}
\end{column}
\begin{column}{.48\textwidth}
\begin{block}{Head-wise hybrid (HH)}
Efficient and full NoPE heads collaborate within each layer.
\end{block}
\end{column}
\end{columns}
\papersource{Paper, Sec. 3.1, p. 4. Default SWA window: 128. Full NoPE still stores a length-dependent KV cache.}
\end{frame}

\begin{frame}{A recurrent gate is an implicit position mechanism}
For GLA, an interaction between positions $s$ and $t$ contains
\[
q_t\left(\prod_{i=s+1}^{t}\operatorname{Diag}(\boldsymbol\alpha_i)\right)k_s^\top.
\]
\begin{itemize}
\item $q_t,k_s$: query and key; $\boldsymbol\alpha_i$: data-dependent gate.
\item Distance changes how many gate factors are accumulated.
\item No explicit position embedding does not imply no position bias.
\end{itemize}
\begin{block}{Teaching example: a constant scalar gate}
$0.99^{100}\approx0.366$, while $0.99^{1000}\approx4.3\times10^{-5}$.
\end{block}
\papersource{Paper, Sec. 6.2, p. 21. Scalar numerical example is an explanatory simplification.}
\end{frame}

\begin{frame}{The SWA advantage reverses after continued training}
\fig[4.9cm]{seesaw-776m}
\papersource{Redrawn from Tables 7 and 11, pp. 43 and 47. 776M, 3:1 LH. All averages 12 benchmark--length cells.}
\end{frame}

\begin{frame}{NoPE heads locate globally but often aggregate broadly}
\fig[4.55cm]{fig12b-tidal}
Each point is a head: entropy on the horizontal axis, top-64 hit rate on the vertical axis. Rows: 376M / 776M.
\papersource{Original Fig. 12(b), p. 13. Columns: pure RoPE, then RoPE:NoPE = 3:1, 1:1, 1:3. Colors also encode retrieval-score bins.}
\end{frame}

\begin{frame}{The best mixture depends on the attention family}
\centering
\begin{tabular}{lccc}
\toprule
776M HH; reported All score & 3:1 & 1:1 & 1:3 \\
\midrule
RoPE-NoPE + EME & \textbf{30.4} & 25.7 & 22.3 \\
SWA-NoPE + Log & 31.6 & \textbf{35.2} & 31.9 \\
\bottomrule
\end{tabular}
\medskip
\begin{itemize}
\item More global heads do not give monotonically better results.
\item In 1:1 HH at 4K, perturbing positional heads hurts more than perturbing NoPE heads.
\item Authors' interpretation: global coarse localization needs concentrated processing by other heads.
\end{itemize}
\papersource{Paper, Sec. 4, Figs. 10--13, pp. 11--13; Table 17, p. 52. Ratios are positional:NoPE.}
\end{frame}

\begin{frame}{A Jacobian makes attention concentration comparable}
For softmax attention, holding queries and keys fixed:
\[
o_t=\sum_s\alpha_{t,s}v_s,\qquad
J_{t,s}=\frac{\partial o_t}{\partial v_s}=\alpha_{t,s}I,
\qquad \|J_{t,s}\|_2=\alpha_{t,s}.
\]
Normalize the same nonnegative quantity for other attention operators:
\[
\widetilde\alpha_{t,s}=\frac{\|J_{t,s}\|_2}{\sum_u\|J_{t,u}\|_2},
\qquad
\widetilde H_t=-\frac{\sum_s\widetilde\alpha_{t,s}\log\widetilde\alpha_{t,s}}{\log L}.
\]
\begin{itemize}
\item $\|\cdot\|_2$: spectral norm; $L$: context length; nonzero denominator.
\item Recovers softmax attention weights without changing LA computation.
\end{itemize}
\papersource{Paper, Eq. (1), Sec. 6.1, p. 18. Normalization is written out from the accompanying description.}
\end{frame}

\begin{frame}{GLA and GDN behave like position-biased partners}
\fig[4.55cm]{fig21b-la-position}
LA heads tend toward lower hit rates and lower entropy than global NoPE heads, with differences between GLA and GDN.
\papersource{Original Fig. 21(b), p. 19. 3:1 LH; rows 376M / 776M. Axes: normalized entropy and top-64 hit rate at 4K.}
\end{frame}

\begin{frame}{SWA can learn local prediction during long training}
\fig[3.7cm]{fig14a-swa-ppl}
\begin{itemize}
\item After continual pretraining, short-context PPL improves sharply.
\item Longer-context positions improve less, especially in LH models.
\end{itemize}
\papersource{Original Fig. 14(a), p. 15. Solid: LH; dashed: HH. Orange: direct; red: Log; blue: continual pretraining. PPL: lower is better.}
\end{frame}

\begin{frame}{Parameter changes support a local-optimization trap}
\[
r_Q=\frac{\|W'_Q-W_Q\|_2}{\|W_Q\|_2},
\qquad
r_K=\frac{\|W'_K-W_K\|_2}{\|W_K\|_2}.
\]
\begin{itemize}
\item Prime denotes parameters after long-context continued training.
\item Ratios are computed per head, then averaged; HH groups are separated.
\item SWA parameters change more than NoPE parameters, yet the SWA window remains 128 tokens.
\end{itemize}
\begin{block}{Intervention suggested by the observation}
Expand the branch receiving those updates; emphasize long dependencies in the training objective.
\end{block}
\papersource{Paper, Sec. 5.1 and Fig. 15, pp. 14--16. Parameter displacement supports this interpretation; it is not a direct gradient-budget measurement.}
\end{frame}

\begin{frame}{A larger window plus LongCE improves SWA extension}
\fig[4.8cm]{swa-rescue}
\papersource{Redrawn from Table 15, p. 50. SWA-NoPE-LH, 32K training. >4K averages eight RULER/BABILong cells, including 64K.}
\end{frame}

\begin{frame}{Window choice should depend on the training stage}
\centering
\begin{tabular}{lrrrr}
\toprule
776M SWA-NoPE-LH & 128 & 512 & 2K & 4K \\
\midrule
4K training: score $>$4K & 20.4 & \textbf{22.2} & 18.6 & 17.7 \\
32K training: score $>$4K & 24.8 & 28.4 & 30.6 & \textbf{31.1} \\
32K training: All & 29.4 & 32.6 & \textbf{34.0} & 33.7 \\
\bottomrule
\end{tabular}
\medskip
\begin{itemize}
\item Short training: overly large windows weaken extrapolation.
\item Long training: overly small windows restrict extension benefits.
\item The reported 2K optimum concerns All, not every metric.
\end{itemize}
\papersource{Paper, Sec. 5.2; Tables 13--14, p. 49. Short-stage rows use direct extrapolation. Extension also adjusts the RoPE base.}
\end{frame}

\begin{frame}{EME applies complementary changes to both branches}
\begin{columns}[T]
\begin{column}{.48\textwidth}
\begin{block}{Position-biased branch}
Restrict direct attention to the training-length range.\\[2mm]
RoPE $\rightarrow$ sliding-window softmax\\[1mm]
GLA/GDN $\rightarrow$ sliding-window LA
\end{block}
\end{column}
\begin{column}{.48\textwidth}
\begin{block}{Global NoPE branch}
Keep global access and increase logit scale.\\[2mm]
$c(L)=\log L/\log L_0$ for $L\ge L_0$\\[1mm]
$\alpha=\operatorname{softmax}(c(L)z)$
\end{block}
\end{column}
\end{columns}
\medskip
At $L_0=4096$, $L=65536$, the logit multiplier is $4/3$.
\papersource{Paper, Secs. 3.2 and 6.2, pp. 4 and 20--23. EME: Extrapolation based on the Matthew Effect. Formula expresses the stated log-base rule.}
\end{frame}

\begin{frame}{SWLA reuses LA kernels on overlapping windows}
\fig[3.65cm]{swla-overlap}
\begin{itemize}
\item Independent window states; concatenate only the retained outputs.
\item Retained local prefixes span 3073--4096 tokens: an approximation.
\end{itemize}
\papersource{Teaching diagram based on Sec. 6.2, p. 20. Window = 4096; stride = 1024. It is not an original paper figure.}
\end{frame}

\begin{frame}{Log correction becomes more useful after SWLA}
\fig[4.7cm]{eme-ablation-776m}
\papersource{Redrawn from Table 7, p. 43. 776M, 3:1 LH, trained at 4K. Same >4K aggregation for all four variants.}
\end{frame}

\begin{frame}{The 100\% result is specific to SK1 at 776M}
\centering
\begin{tabular}{lrrrr}
\toprule
64K accuracy (\%) & SK1 direct & SK1 EME & SK2 EME & SK3 EME \\
\midrule
GLA-NoPE-LH & 0 & \textbf{100} & 59 & 64 \\
GLA-NoPE-HH & 0 & \textbf{100} & 31 & 34 \\
GDN-NoPE-LH & 99 & \textbf{100} & 90 & 53 \\
GDN-NoPE-HH & 92 & \textbf{100} & 35 & 16 \\
\bottomrule
\end{tabular}
\medskip
\begin{itemize}
\item All models were trained at 4K: 16$\times$ length extrapolation.
\item GDN already reaches 100\% on SK1 with Log alone.
\item Harder SK2/SK3 cases reveal the remaining performance gaps.
\end{itemize}
\papersource{Paper, Table 1, p. 23. These are RULER subtask accuracies, not overall long-context accuracy.}
\end{frame}

\begin{frame}{LongBench supports the trend with configuration effects}
\centering
\begin{tabular}{lcc}
\toprule
3B, LH; LongBench Avg. & 4K training + Log & 32K training \\
\midrule
SWA-NoPE & \textbf{20.1} & 23.7 \\
GLA-NoPE & 17.7 & \textbf{25.2} \\
GDN-NoPE & 18.1 & 24.9 \\
\bottomrule
\end{tabular}
\medskip
\begin{itemize}
\item Input cap: 31,500 tokens; output cap: 500 tokens.
\item ICL tasks give clearer evidence than difficult QA/summarization.
\item At 776M LH, SWA is not initially ahead on the overall average.
\end{itemize}
\papersource{Paper, Appendix A.1, pp. 32--34; Tables 18--19 and 22, pp. 53--54 and 57.}
\end{frame}

\begin{frame}{Efficiency depends on the implementation and workload}
\fig[3.1cm]{fig27a-throughput}
\begin{itemize}
\item Training throughput improves, especially at longer contexts.
\item FP32 LA states and FP16 SWA-128 caches have similar sizes here.
\item Overlapping SWLA repeats roughly $W/S=4$ times the LA work; these plots do not separately benchmark that EME implementation.
\end{itemize}
\papersource{Original Fig. 27(a), p. 24; Sec. 6.3. 8 H200 GPUs; TGS = tokens/GPU/s. SWLA work estimate is derived from the overlap recipe.}
\end{frame}

\begin{frame}{Three boundaries matter for using these findings}
\begin{enumerate}
\item \textbf{Scale and task.} Up to 3B, 50B initial tokens, pretraining tasks. A 3B GLA-HH EME model scores only 5\% on 64K SK1.
\medskip
\item \textbf{Layer layout.} The seesaw persists in a YOCO-like ordering, but EME is less effective in that ordering.
\medskip
\item \textbf{Cost.} Global NoPE retains growing KV memory; SWLA overlap adds work. Quality gains need their own system measurements.
\end{enumerate}
\papersource{Paper, Limitations, p. 25; Appendix A.2, pp. 34--36; Table 16, p. 51; Sec. 6.2, p. 20.}
\end{frame}

\begin{frame}{Choose the recipe for the stage and the failure mode}
\begin{block}{For training-free LA-NoPE extrapolation}
Test a bounded LA range together with stronger NoPE logits. Evaluate multiple retrieval tasks and actual inference cost.
\end{block}
\begin{block}{For long-context training of SWA hybrids}
Revisit window size, RoPE base, and the long-dependency objective together. Inspect PPL by context length.
\end{block}
\medskip
\textbf{Mechanistic takeaway:} global localization and concentrated processing are complementary roles whose balance changes with training and inference conditions.
\papersource{Synthesis of Secs. 4--6. These practical interpretations retain the paper's model and protocol conditions.}
\end{frame}

\begin{frame}{Paper and discussion}
\textbf{Xiaoran Liu, Ziwei He, Xipeng Qiu.}\\[2mm]
\emph{Mechanics of Long-Context Hybrid Models Part 1.1: From Hybrid Attention to Hybrid Position.}\\[2mm]
\href{https://arxiv.org/abs/2610.10114v1}{arXiv:2610.10114v1}, October 7, 2026.\\[5mm]
All experimental values in this presentation come from this version. Original panels and redrawn tables are identified on their slides.\\[5mm]
\color{AcademicTeal}\textbf{Discussion:} Can a more efficient SWLA implementation retain the same gains? Can targeted branch updates test the local-learning explanation more directly?
\papersource{End of main presentation: 23 slides, approximately 25 minutes. Backup slides follow.}
\end{frame}

\appendix

\begin{frame}{Backup: model and optimizer configuration}
\centering
\begin{tabular}{lrrrr}
\toprule
 & 376M & 776M & 1B & 3B \\
\midrule
Hidden size & 1024 & 1536 & 2048 & 3072 \\
Intermediate size & 3584 & 5376 & 7168 & 10752 \\
Layers & 8 & 12 & 16 & 24 \\
Query / KV heads & 8 / 8 & 12 / 12 & 16 / 16 & 24 / 24 \\
H200 GPUs & 8 & 8 & 8 & 16 \\
\bottomrule
\end{tabular}
\medskip
\begin{itemize}
\item Vocabulary 128,256; BFloat16 training; AdamW and cosine decay.
\item Initial: weight decay 0.1; warm-up 0.5B of 50B tokens.
\item Continued: weight decay 0; warm-up 1B of 5B tokens.
\end{itemize}
\papersource{Paper, Appendix A and Table 2, p. 32. Training throughput comparison excludes 3B to keep the GPU count fixed.}
\end{frame}

\begin{frame}{Backup: what the extended distribution measures}
\begin{block}{GLA recurrence: explanatory expansion}
\[
S_t=D_tS_{t-1}+k_t^\top v_t,
\qquad
o_t=\sum_{s=0}^{t}\left[q_t\left(\prod_{i=s+1}^{t}D_i\right)k_s^\top\right]v_s.
\]
\end{block}
\begin{itemize}
\item $S_t\in\mathbb R^{d_k\times d_v}$; $q,k,v$ are row vectors.
\item A scalar signed coefficient $a$ gives $\|aI\|_2=|a|$.
\item Example: coefficients $(+2,-2)$ normalize to $(0.5,0.5)$; sign cancellation disappears.
\item Normalized entropy also depends on the denominator $\log L$.
\end{itemize}
\papersource{Explanatory derivation from the GLA gate product in Sec. 6.2, p. 21, and Eq. (1), p. 18. Interpret as value-path sensitivity.}
\end{frame}

\begin{frame}{Backup: the reported averages weight length cells}
\centering
\begin{tabular}{lll}
\toprule
Reported column & Included scores & Number of cells \\
\midrule
RULER & 4K, 8K, 16K, 32K, 64K & 5 \\
BABILong & 0K, 2K, 4K, 8K, 16K, 32K, 64K & 7 \\
$\leq$4K & RULER 4K; BABILong 0K/2K/4K & 4 \\
$>$4K & Both benchmarks at 8K/16K/32K/64K & 8 \\
All & All listed benchmark--length scores & 12 \\
\bottomrule
\end{tabular}
\medskip
\[
\mathrm{All}\approx\frac{5\,\mathrm{RULER\ Avg.}+7\,\mathrm{BABILong\ Avg.}}{12}.
\]
The approximation sign accounts for displayed rounding. These are not two equally weighted benchmark averages.
\papersource{Paper, table headers and entries in Tables 7--15, pp. 43--50.}
\end{frame}

\begin{frame}{Backup: SWLA and Log solve different parts of retrieval}
\centering
\begin{tabular}{lrrr}
\toprule
776M GLA-NoPE-LH; 64K accuracy (\%) & SK1 & SK2 & SK3 \\
\midrule
Direct & 0 & 0 & 0 \\
Log & 0 & 0 & 0 \\
SWLA & 97 & 0 & 3 \\
SWLA + Log & \textbf{100} & \textbf{59} & \textbf{64} \\
\bottomrule
\end{tabular}
\par\medskip
SWLA can rescue the easiest subtask while the harder subtasks still need effective global aggregation.\\[3mm]
At 32K SK1, the same EME model scores 96\%: 100\% at 64K does not mean 100\% at every tested length.
\papersource{Paper, Table 1, p. 23. Trained at 4K; no additional training for these interventions.}
\end{frame}

\begin{frame}{Backup: the headline does not hold at every scale}
\centering
\begin{tabular}{lrrr}
\toprule
GLA-NoPE-HH + EME; 64K accuracy (\%) & SK1 & SK2 & SK3 \\
\midrule
376M & 100 & 49 & 37 \\
776M & 100 & 31 & 34 \\
1B & 64 & 68 & 66 \\
3B & 5 & 100 & 73 \\
\bottomrule
\end{tabular}
\par\medskip
The intervention is useful across settings, but extrapolation quality depends strongly on the task and model. Larger scale does not guarantee a higher SK1 score here.
\papersource{Paper, Table 1, p. 23; Table 16, p. 51. All models initially trained at 4K on 50B tokens.}
\end{frame}

\begin{frame}{Backup: additional checks and unresolved choices}
\centering
\begin{tabular}{lccc}
\toprule
776M, YOCO-like LH; All & SWA-NoPE & GLA-NoPE & GDN-NoPE \\
\midrule
Short training + Log & 30.6 & 21.7 & 25.6 \\
Long continued training & 30.6 & 36.1 & 31.4 \\
\bottomrule
\end{tabular}
\medskip
\begin{itemize}
\item YOCO-like here means placing efficient layers at the bottom.
\item Four learning rates retain the main seesaw trend.
\item Smaller SWA RoPE bases and dropping RoPE during continued training show no consistent optimum across configurations.
\end{itemize}
\papersource{Paper, Appendix A.2--A.3, pp. 34--37; Table 24, p. 59. EME is less effective in the YOCO-like ordering.}
\end{frame}

\end{document}