% Based on the arxiv-paper-reader Beamer template.
% Research discussion: approximately 25 minutes; frames after References are backup.
\documentclass[aspectratio=169,11pt]{beamer}
\usepackage[T1]{fontenc}
\usepackage[utf8]{inputenc}
\usepackage{amsmath,amssymb,graphicx,booktabs,array}
\newcolumntype{L}[1]{>{\raggedright\arraybackslash}p{#1}}
\definecolor{AcademicBlue}{RGB}{26,57,86}
\definecolor{AcademicTeal}{RGB}{0,105,112}
\setbeamercolor{structure}{fg=AcademicBlue}
\setbeamercolor{frametitle}{fg=AcademicBlue,bg=white}
\setbeamercolor{title}{fg=AcademicBlue}
\setbeamercolor{alerted text}{fg=AcademicTeal}
\setbeamercolor{block title}{fg=white,bg=AcademicBlue}
\setbeamercolor{block body}{fg=black,bg=AcademicBlue!5}
\setbeamertemplate{navigation symbols}{}
\setbeamertemplate{itemize items}[circle]
\setbeamertemplate{footline}{\hfill\usebeamerfont{page number in head/foot}\insertframenumber\,/\,\inserttotalframenumber\hspace{4mm}\vspace{3mm}}
\setbeamersize{text margin left=8mm,text margin right=8mm}
\setbeamerfont{frametitle}{size=\Large,series=\bfseries}
\setbeamerfont{title}{size=\LARGE,series=\bfseries}
\newcommand{\papersource}[1]{\par\smallskip{\scriptsize\color{gray}#1\par}}
\newcommand{\R}{\mathbb R}
\newcommand{\E}{\mathbb E}
\newcommand{\diag}{\operatorname{diag}}
\newcommand{\srope}{Selective RoPE}
\title[Selective RoPE]{Selective Rotary Position Embedding}
\subtitle{Content-dependent phase for sequence memory}
\author{Paper by Sajad Movahedi, Timur Carstensen, Arshia Afzal,\\Frank Hutter, Antonio Orvieto, and Volkan Cevher}
\date{ICLR 2026 \enspace$\cdot$\enspace arXiv:2511.17388v3\\Research discussion prepared on 13 October 2026}

\begin{document}
\begin{frame}
\titlepage
\vspace{-4mm}
\begin{center}
\alert{Learn how far to rotate at each token; retain the backbone's forgetting.}
\end{center}
\end{frame}

\begin{frame}{Memory needs control over amplitude and phase}
\centering
\includegraphics[width=.91\linewidth,height=.49\textheight,keepaspectratio]{figures/fig-01-overview.pdf}
\begin{itemize}
\item Forget gates control how much history survives.
\item Fixed RoPE advances a clock determined by token index.
\item \srope{} makes the phase increment depend on content.
\end{itemize}
\papersource{Paper, Fig. 1, PDF p. 2. Geometric illustration; general diagonal scaling need not preserve direction.}
\end{frame}

\begin{frame}{A fixed state compresses every previous key--value pair}
\[
S_t=S_{t-1}A_t+v_tk_t^\top,\qquad o_t=S_tq_t
\]
\[
o_t=\sum_{\tau=1}^t v_\tau
\left[k_\tau^\top\left(\prod_{\kappa=\tau+1}^{t}A_\kappa\right)q_t\right]
\]
\begin{itemize}
\item $S_t\in\R^{d_v\times d_k}$; $q_t,k_t\in\R^{d_k}$; $v_t\in\R^{d_v}$.
\item A positive diagonal gate rescales channels of the stored state.
\item Input-dependent rotations add conditional phase changes and pairwise mixing.
\end{itemize}
\papersource{Paper, Eq. (3), PDF p. 3. Product starts at $\tau+1$, consistent with the recurrence.}
\end{frame}

\begin{frame}{The RoPE trick implements complex rotation in real arithmetic}
\[
R(\omega)=\begin{bmatrix}\cos\omega&-\sin\omega\\\sin\omega&\cos\omega\end{bmatrix},
\qquad \widetilde q=q_1+iq_2
\]
\[
\Re\{\overline{\widetilde k}\,e^{i\omega}\widetilde q\}=k^\top R(\omega)q
\]
\begin{block}{Move the rotation to query and key}
\[
k_\tau^\top R^{t-\tau}q_t=(R^\tau k_\tau)^\top(R^tq_t).
\]
\end{block}
Two real channels form one complex channel; storage degrees of freedom are unchanged.
\papersource{Paper, Eqs. (5)--(7), PDF pp. 3--4; App. A.1, pp. 17--18. Consistent conjugation shown.}
\end{frame}

\begin{frame}{Learn angle increments and accumulate them along the sequence}
\[
\omega_t=\operatorname{Conv1D}(W_\omega q)_t,
\qquad \psi_{t,j}=\Theta_j\sum_{\kappa=1}^{t}\omega_{\kappa,j}
\]
\[
C_t=\operatorname{blockdiag}_{j}R(\psi_{t,j}),\qquad
q'_t=C_tq_t,\quad k'_t=C_tk_t
\]
\begin{itemize}
\item $W_\omega$ is learned; $\Theta_j$ scales the phase of channel pair $j$.
\item The same cumulative rotation acts on the current query and key.
\item Feed $q'_t,k'_t,v_t$ to the existing attention backbone.
\end{itemize}
\papersource{Paper, Fig. 4 and Sec. 4.1, PDF p. 7. Core path; gates and stabilization add further components.}
\end{frame}

\begin{frame}{The RoPE trick also works with cumulative learned phase}
Define $R_t=C_{t-1}^\top C_t$ on fixed two-dimensional channel pairs.
\[
R_{\tau+1:t}=C_\tau^\top C_t
\]
\[
k_\tau^\top R_{\tau+1:t}q_t=(C_\tau k_\tau)^\top(C_tq_t)
\]
\begin{itemize}
\item Constant increments recover ordinary RoPE.
\item Learned increments let intervening content change the phase difference.
\item Decoding keeps a running phase and short-convolution history.
\end{itemize}
\begin{block}{When a forget gate is present}
Preprocessing remains usable. Equivalence to a simple $\Lambda_tR_t$ transition additionally requires compatible, commuting gates.
\end{block}
\papersource{Paper, Eq. (13), PDF p. 6. Gate condition derived in the backup slides.}
\end{frame}

\begin{frame}{Softmax has an RFF representation with rotations}
For $\omega\sim\mathcal N(0,I)$,
\[
e^{q^\top k}=e^{(\|q\|^2+\|k\|^2)/2}
\E_\omega\!\left[\cos\bigl(\omega^\top(q-k)\bigr)\right].
\]
With equal query norms, each random-feature state has a transition
\[
\overline R_t=\diag\!\left(e^{i\Omega^\top(q_t-q_{t-1})}\right).
\]
\begin{itemize}
\item Finite random features approximate the exponential kernel.
\item Softmax also needs its normalization denominator.
\item Constant query norm removes an additional real exponential factor.
\end{itemize}
\papersource{Paper, Sec. 3.1 and App. A.2, PDF pp. 4, 18--20. Corrected conjugate/phase-difference convention.}
\end{frame}

\begin{frame}{Content-dependent phase can implement a parity clock}
The RFF increments telescope:
\[
\sum_{\kappa=\tau+1}^t\Omega^\top(q_\kappa-q_{\kappa-1})
=\Omega^\top(q_t-q_\tau).
\]
This structural representation alone adds no explicit positional bias.
\begin{block}{Teaching example: a content-dependent parity clock}
For a binary input $b_t$, choose $\omega_t=\pi b_t$. Then
\[
\cos\psi_t=\cos\!\left(\pi\sum_{\kappa\le t}b_\kappa\right)
=(-1)^{\sum_{\kappa\le t}b_\kappa}.
\]
Input 0 keeps the phase; input 1 flips it.
\end{block}
\papersource{Path independence: paper, Sec. 3.1, PDF p. 4. Parity construction is a teaching example.}
\end{frame}

\begin{frame}{Decay supplies a time window; rotation supplies phase}
\centering
\includegraphics[width=.98\linewidth,height=.36\textheight,keepaspectratio]{figures/fig-03-window.pdf}
\[
s_n(t)=\int_0^t e^{(-\alpha_n+i\omega_n)(t-u)}k_nv(u)\,du
\]
\begin{itemize}
\item $\alpha_n>0$ attenuates old inputs; $\omega_n$ creates oscillatory phase.
\item Smooth windows suppress spectral sidelobes in the illustrated signal.
\item This motivates forgetting; the analysis uses fixed diagonal SSM coefficients.
\end{itemize}
\papersource{Paper, Fig. 3 and Sec. 3.2, PDF p. 5; App. A.4, pp. 21--22. Hann and causal exponential windows differ.}
\end{frame}

\begin{frame}{Stable optimization requires more than learnable angles}
\begin{tabular}{@{}L{.38\linewidth}L{.57\linewidth}@{}}
\toprule
Component & Role reported in the paper\\
\midrule
Normalized $x_t$ for angle generation & Replaces query input to reduce instability\\[2mm]
Weight normalization or rank 8/16 MLP & Further stabilizes the phase projection\\[2mm]
SiLU after convolution & Improves the language-model baseline\\[2mm]
Sigmoid phase gate & Controls rotation strength; improves some results\\[2mm]
Learnable bias & Helps MAD in combination; limited LM benefit\\
\bottomrule
\end{tabular}
\medskip
\par\medskip
Norm-preserving rotations do not guarantee easy optimization of their angles.
\papersource{Paper, Secs. 4.1 and 4.3, PDF pp. 7--9. Fig. 4 omits these full configuration details.}
\end{frame}

\begin{frame}{Evaluate recall, state tracking, and language modeling}
\begin{tabular}{@{}L{.22\linewidth}L{.41\linewidth}L{.31\linewidth}@{}}
\toprule
Experiment & Question & Reported setting\\
\midrule
MQAR & Retrieve multiple key--value pairs & Length 512; 64 pairs\\[2mm]
MAD & Which memory operations improve? & Six task families\\[2mm]
Copying & Generalize exact copying beyond training & Train lengths 2--64\\[2mm]
State tracking & Learn input-conditioned state changes & Train length 128\\[2mm]
Language modeling & Transfer to pretraining and zero-shot tasks & 370M--1.3B models\\
\bottomrule
\end{tabular}
\medskip
\par\medskip
Within each backbone: NoPE vs. RoPE vs. \srope{}.
\papersource{Paper, Sec. 4, PDF pp. 7--9; App. B, pp. 27--29.}
\end{frame}

\begin{frame}{MQAR gains at width 128 do not extend to width 64}
\begin{columns}[c]
\begin{column}{.62\textwidth}
\includegraphics[width=\linewidth,height=.64\textheight,keepaspectratio]{figures/fig-06-mqar.pdf}
\end{column}
\begin{column}{.35\textwidth}
\begin{itemize}
\item At width 128, Selective RoPE leads the GLA variants.
\item By width 256, all three approach the ceiling.
\item At width 64, Selective RoPE is worse.
\end{itemize}
\end{column}
\end{columns}
\papersource{Paper, Fig. 6, PDF p. 7. Sequence length 512; 64 key--value pairs.}
\end{frame}

\begin{frame}{MAD gains depend on the phase gate and bias}
\centering
\begin{tabular}{@{}lrrrr@{}}
\toprule
GLA variant & IC recall & Memory & Noisy recall & Avg.\\
\midrule
NoPE & 87.3 & 38.7 & 87.6 & 65.9\\
RoPE & 92.6 & \textbf{61.4} & 91.9 & 72.5\\
Selective baseline & 94.0 & 57.1 & 91.7 & 72.0\\
\quad + phase gate & \textbf{96.6} & 56.9 & 94.3 & 72.3\\
\quad + bias & 95.0 & 61.3 & 91.2 & 72.7\\
\quad + gate and bias & 95.9 & 60.4 & \textbf{95.0} & \textbf{73.2}\\
\bottomrule
\end{tabular}
\medskip
\begin{itemize}
\item Best configuration: +7.3 percentage points over NoPE; +0.7 over RoPE.
\item The selective baseline alone trails RoPE on the average.
\item RoPE still leads on Memorize and Selective Copy.
\end{itemize}
\papersource{Selected columns from paper, Table 1, PDF p. 7. All scores higher is better.}
\end{frame}

\begin{frame}{Copying extrapolates better, but accuracy still decays}
\begin{columns}[c]
\begin{column}{.43\textwidth}
\centering
\includegraphics[width=\linewidth,height=.64\textheight,keepaspectratio]{figures/fig-05-copying.pdf}
\end{column}
\begin{column}{.54\textwidth}
\begin{itemize}
\item Copy the entire input after a copy marker.
\item Training lengths: 2--64; dashed line marks the boundary.
\item Selective RoPE decays more slowly than both alternatives.
\item Near length 160, its accuracy is already roughly 0.5.
\end{itemize}
\end{column}
\end{columns}
\papersource{Paper, Fig. 5, PDF p. 7; Table 7, p. 29. Configuration lists evaluation to 512; the plotted range is shorter.}
\end{frame}

\begin{frame}{State-tracking extrapolation depends on the backbone}
\centering
\includegraphics[width=.98\linewidth,height=.51\textheight,keepaspectratio]{figures/fig-07-state-tracking.pdf}
\begin{itemize}
\item $S_2$: one-layer models; $A_3$: two-layer DeltaNet. Train length: 128.
\item DeltaNet on $S_2$ generalizes most strongly toward length 512.
\item GLA, Transformer, and $A_3$ DeltaNet show distinct failure lengths.
\end{itemize}
\papersource{Paper, Fig. 7, PDF p. 8; App. B.2, pp. 27--28. Curves use the paper's Accuracy metric.}
\end{frame}

\begin{frame}{Language-model comparisons require matching the full setting}
\begin{itemize}
\item FineWeb; context 4096; Mistral 7B tokenizer with 32,000 tokens.
\item GLA/FoX 370M: 10B tokens; GLA 1.3B: 26B tokens.
\item Choose learning rates using 4M held-out tokens; evaluate zero-shot.
\item Table 3 adds a $\sim$400M Gated DeltaNet sweep with a stable variant.
\end{itemize}
\begin{block}{Read metrics separately}
Wiki and LAMBADA perplexity: lower is better.\par
Task accuracies: higher is better; HellaSwag and ARC-c use normalized accuracy.
\end{block}
\papersource{Paper, Sec. 4.3 and Table 3, PDF pp. 8--9; Table 5, p. 27. FoX results use output norm OFF.}
\end{frame}

\begin{frame}{Small linear models gain accuracy without uniform PPL gains}
\centering
\begin{tabular}{@{}llrrr@{}}
\toprule
Backbone & Encoding & Wiki PPL $\downarrow$ & LMB PPL $\downarrow$ & Avg. $\uparrow$\\
\midrule
GLA 370M & NoPE & \textbf{25.72} & \textbf{15.83} & 46.8\\
 & RoPE & 26.51 & 16.32 & 46.5\\
 & Selective & 25.98 & 16.33 & \textbf{47.1}\\
\midrule
Gated DeltaNet & NoPE & \textbf{25.82} & 16.27 & 46.6\\
$\sim$400M & RoPE & 25.92 & 16.79 & 46.7\\
 & Selective$^\dagger$ & 26.13 & \textbf{15.93} & \textbf{47.5}\\
\bottomrule
\end{tabular}
\medskip
\par\medskip
Against NoPE: +0.3 points for GLA; +0.9 points for Gated DeltaNet.\par
Wiki perplexity increases in both backbones.
\papersource{Paper, Table 3, PDF p. 9. All train on 10B tokens. $^\dagger$stable\_selective\_rope; averages as reported.}
\end{frame}

\begin{frame}{FoX and larger GLA expose different metric trade-offs}
\centering
\begin{tabular}{@{}llrrr@{}}
\toprule
Backbone & Encoding & Wiki PPL $\downarrow$ & LMB PPL $\downarrow$ & Avg. $\uparrow$\\
\midrule
FoX 370M & NoPE & 25.52 & 17.76 & 45.3\\
10B tokens & RoPE & \textbf{25.29} & 17.71 & 45.1\\
 & Selective & 33.87 & \textbf{17.29} & \textbf{46.1}\\
\midrule
GLA 1.3B & NoPE & 18.18 & 8.59 & \textbf{55.2}\\
26B tokens & RoPE & 18.50 & 8.88 & 54.4\\
 & Selective & \textbf{17.87} & \textbf{8.50} & 54.6\\
\bottomrule
\end{tabular}
\medskip
\par\medskip
FoX: Wiki PPL worsens substantially. GLA 1.3B: PPL improves, average accuracy drops versus NoPE.
\papersource{Paper, Table 3, PDF p. 9. FoX Avg. values are reported values and do not match the six displayed task means; see backup.}
\end{frame}

\begin{frame}{The selective baseline needs additional components to improve}
\centering
\begin{tabular}{@{}lrr@{}}
\toprule
GLA 370M / 10B tokens & Avg-PPL $\downarrow$ & Avg-Acc $\uparrow$\\
\midrule
NoPE & \textbf{20.78} & 46.8\\
RoPE & 21.42 & 46.5\\
Selective baseline & 21.27 & 45.8\\
\quad + SiLU & 21.14 & 46.7\\
\quad + phase gate & 21.16 & \textbf{47.1}\\
\bottomrule
\end{tabular}
\medskip
\begin{itemize}
\item The phase-gated row improves accuracy by 1.3 points over the baseline.
\item NoPE retains the lowest average perplexity.
\item Optimization and parameterization are part of the method's benefit.
\end{itemize}
\papersource{Paper, Table 2, PDF p. 8. Avg-PPL matches the arithmetic mean of Wiki and LAMBADA PPL.}
\end{frame}

\begin{frame}{The RFF temperature theorem needs a correction}
\begin{columns}[t]
\begin{column}{.48\textwidth}
\begin{block}{Paper's Theorem 1}
For unit queries and keys, with $\xi=q^\top k$:
\[
\sigma^2=\frac{1-\xi}{1+\xi}.
\]
The proof starts with $q-k$ but later uses terms involving $1+\xi$.
\end{block}
\end{column}
\begin{column}{.48\textwidth}
\begin{block}{Reading analysis}
The consistent phase-difference kernel has expectation
\[
e^{1-\sigma^2(1-\xi)}.
\]
As $D\to\infty$, matching $e^\xi$ requires $\sigma^2=1$ for $\xi<1$.
\end{block}
\end{column}
\end{columns}
\medskip
The proposed temperature spectrum remains a design heuristic; its stated optimality is not established by this derivation.
\papersource{Paper, App. A.3, PDF pp. 20--21. Independent derivation and finite-$D$ expression in backup.}
\end{frame}

\begin{frame}{Learned phase is the central contribution}
\begin{enumerate}
\item \textbf{Mechanism:} accumulate learned angle increments and rotate queries and keys before attention.
\medskip
\item \textbf{Evidence:} strong changes in recall and state tracking; language-model gains depend on architecture, scale, and metric.
\medskip
\item \textbf{Practical choice:} include normalization, phase gating, and matched NoPE/RoPE baselines in an ablation.
\end{enumerate}
\begin{block}{Open empirical questions}
Real-language length extrapolation and end-to-end runtime overhead remain unquantified in the reported v3 comparisons.
\end{block}
\papersource{Paper, Secs. 4--6, PDF pp. 7--10. Interpretation based on the reported results.}
\end{frame}

\begin{frame}{References and discussion materials}
\textbf{Sajad Movahedi, Timur Carstensen, Arshia Afzal,}\par
\textbf{Frank Hutter, Antonio Orvieto, and Volkan Cevher.}\par
\medskip
\emph{Selective Rotary Position Embedding.} ICLR 2026.\par
arXiv:2511.17388v3, revised 23 June 2026.
\medskip
\begin{itemize}
\item Paper: \href{https://arxiv.org/abs/2511.17388v3}{arxiv.org/abs/2511.17388v3}
\item Author code link: \href{https://github.com/timurcarstensen/selective-rope}{github.com/timurcarstensen/selective-rope}
\item Companion Chinese notes: \texttt{reading-notes.zh.md}
\end{itemize}
\medskip
Original figures are reproduced from the paper and its source package.\par
Algebraic corrections and teaching examples are marked as reading analysis.
\end{frame}

\appendix
\begin{frame}{Backup: the finite-feature error retains a dependence on $D$}
Reading analysis. Let $\xi=q^\top k<1$, $\|q\|=\|k\|=1$, and
\[
\widehat K_D=\frac{e}{D}\sum_{j=1}^D\cos\bigl(\omega_j^\top(q-k)\bigr),
\quad\omega_j\sim\mathcal N(0,\sigma^2I).
\]
\[
\begin{aligned}
\E[(\widehat K_D-e^\xi)^2]
={}&\frac{e^2}{2D}\left(1-e^{-2\sigma^2(1-\xi)}\right)^2\\
&+\left(e^{1-\sigma^2(1-\xi)}-e^\xi\right)^2.
\end{aligned}
\]
\begin{itemize}
\item First term: sampling variance. Second term: squared bias.
\item At $\xi=1/2$, the paper's $\sigma^2=1/3$ gives mean $2.301$; target $1.649$.
\item $\sigma=1$ matches the target mean exactly.
\end{itemize}
\papersource{Independent derivation for the consistent real-part RFF estimator; compare App. A.3, PDF pp. 20--21.}
\end{frame}

\begin{frame}{Backup: diagonal gates require a change of coordinates}
Run the existing gate in the rotated coordinates:
\[
\widehat S_t=\widehat S_{t-1}\Lambda_t+v_t(C_tk_t)^\top,
\qquad o_t=\widehat S_tC_tq_t.
\]
Set $S_t=\widehat S_tC_t$. Then
\[
S_t=S_{t-1}\underbrace{C_{t-1}^\top\Lambda_t C_t}_{A_t^{\rm eff}}
+v_tk_t^\top.
\]
\begin{itemize}
\item With a shared scalar gate inside each 2D pair, this reduces to $\Lambda_tR_t$.
\item General per-channel gates need not commute with the rotations.
\item In either case, $\|A_t^{\rm eff}\|_2=\|\Lambda_t\|_2$.
\end{itemize}
\papersource{Reading analysis of query--key preprocessing; relates to Sec. 3.3, PDF p. 6.}
\end{frame}

\begin{frame}{Backup: FoX averages do not match the displayed scores}
\centering
\begin{tabular}{@{}lrr@{}}
\toprule
FoX 370M / 10B & Reported Avg. & Mean of six shown accuracies\\
\midrule
NoPE & 45.3 & 46.18\\
RoPE & 45.1 & 45.93\\
Selective RoPE & 46.1 & 46.97\\
\bottomrule
\end{tabular}
\medskip
\begin{itemize}
\item Six columns: LAMBADA, PIQA, HellaSwag, WinoGrande, ARC-e, ARC-c.
\item The text calls Avg. a core multiple-choice macro-average, but the discrepancy is not resolved.
\item Ranking is unchanged; cite the aggregation issue with absolute averages.
\end{itemize}
\papersource{Paper, Table 3, PDF p. 9. Right column recomputed from displayed values; left column preserved verbatim.}
\end{frame}

\begin{frame}{Backup: related mechanisms fit distinct transition structures}
\begin{tabular}{@{}L{.23\linewidth}L{.71\linewidth}@{}}
\toprule
Method & Interpretation in this paper\\
\midrule
RFF / RFA & Input-dependent unit-modulus phase in a kernel representation\\[2mm]
PRF / Performer & Real exponential factor; not necessarily contractive in this representation\\[2mm]
CosFormer & A shared frequency $\pi/(2M)$ with sine/cosine state components\\[2mm]
ALiBi / FoX & Fixed / input-dependent exponential decay of softmax weights\\[2mm]
DeltaNet & Generalized Householder updates mix directions; not always pure rotations\\
\bottomrule
\end{tabular}
\papersource{Paper, App. A.5--A.7 and Table 4, PDF pp. 22--27. Interpretations do not equate the complete architectures.}
\end{frame}

\begin{frame}{Backup: synthetic protocols use different budgets and seeds}
\begin{tabular}{@{}L{.2\linewidth}L{.74\linewidth}@{}}
\toprule
Task & Reported configuration\\
\midrule
MAD & 66 task settings $\times$ 6 optimizer settings = 396 training runs per model configuration\\[2mm]
State tracking & 2M train / 0.5M eval sequences; 100 epochs; batch 4096; LR $10^{-3}$; five seeds\\[2mm]
Copying & Vocabulary 26; LR $5\times10^{-5}$; seed 42; train lengths 2--64; evaluation configured to 512\\[2mm]
MQAR & Eight learning-rate choices; endpoints are described inconsistently in the appendix\\
\bottomrule
\end{tabular}
\medskip
\par\medskip
Copying confidence bands are shown, but their construction is not specified in the accompanying text.
\papersource{Paper, App. B.2 and Tables 6--7, PDF pp. 27--29.}
\end{frame}
\end{document}