\documentclass[aspectratio=169,11pt]{beamer}
\usepackage[T1]{fontenc}
\usepackage[utf8]{inputenc}
\usepackage{lmodern}
\usepackage{amsmath,amssymb,graphicx,booktabs,array}

\definecolor{AcademicBlue}{RGB}{26,57,86}
\definecolor{AcademicTeal}{RGB}{0,105,112}
\setbeamercolor{structure}{fg=AcademicBlue}
\setbeamercolor{frametitle}{fg=AcademicBlue,bg=white}
\setbeamercolor{title}{fg=AcademicBlue}
\setbeamercolor{alerted text}{fg=AcademicTeal}
\setbeamercolor{block title}{fg=white,bg=AcademicBlue}
\setbeamercolor{block body}{fg=black,bg=AcademicBlue!5}
\setbeamertemplate{navigation symbols}{}
\setbeamertemplate{itemize items}[circle]
\setbeamertemplate{footline}{\hfill\usebeamerfont{page number in head/foot}\insertframenumber\,/\,\inserttotalframenumber\hspace{4mm}\vspace{3mm}}
\setbeamersize{text margin left=8mm,text margin right=8mm}
\setbeamerfont{frametitle}{size=\Large,series=\bfseries}
\newcommand{\papersource}[1]{\par\smallskip{\scriptsize\color{gray}#1\par}}
\newcommand{\keypoint}[1]{\begin{block}{}#1\end{block}}
\newcommand{\pope}{PoPE}

\begin{document}

\begin{frame}[plain]
\vfill
{\color{AcademicBlue}\LARGE\bfseries Decoupling the ``What'' and ``Where''\\[1mm] with Polar Coordinate Positional Embedding\par}
\vspace{5mm}
{\large PoPE: a research group discussion\par}
\vspace{4mm}
{\normalsize Anand Gopalakrishnan, R\'obert Csord\'as, J\"urgen Schmidhuber, Michael C. Mozer\par}
\vspace{5mm}
{\small \href{https://arxiv.org/abs/2509.10534v3}{arXiv:2509.10534v3} \quad
\href{https://proceedings.mlr.press/v306/gopalakrishnan26a.html}{Published at ICML 2026 (PMLR 306)} \quad
\href{https://github.com/agopal42/pope}{Official code}\par}
\vspace{2mm}
{\small Reading date: 28 October 2026\par}
\vfill
\end{frame}

\begin{frame}{The research question: can attention separate what from where?}
\keypoint{A query may need to match a token's \alert{content} and its \alert{relative position} independently.}
\begin{itemize}
\item RoPE gives each two-dimensional content feature an initial phase, then rotates it by position.
\item PoPE assigns content to nonnegative magnitudes and position to phases.
\item The paper tests this bias with indirect indexing, sequence modeling, and length extrapolation.
\end{itemize}
\papersource{Paper, Secs. 1--4, PDF pp. 1--7.}
\end{frame}

\begin{frame}{RoPE encodes relative position through paired rotations}
For query position $t$ and key position $s$, a $d$-dimensional head has $d/2$ two-dimensional components:
\[
 a^{\mathrm{RoPE}}_{ts}
 =\sum_{c=1}^{d/2}[R(t\theta_c)q_{tc}]^\top [R(s\theta_c)k_{sc}]
 =\sum_{c=1}^{d/2}q_{tc}^\top R((s-t)\theta_c)k_{sc}.
\]
\begin{itemize}
\item $R(\alpha)$ is a planar rotation; $\theta_c$ is a channel frequency.
\item The position term depends on $s-t$, so shifting both tokens preserves the score pattern.
\end{itemize}
\papersource{Paper, Sec. 2, Eq. (1), PDF p. 2.}
\end{frame}

\begin{frame}{In polar form, RoPE lets content shift the position peak}
\[
 a^{\mathrm{RoPE}}_{ts}
 =\sum_{c=1}^{d/2}\mu_{q,tc}\mu_{k,sc}
 \cos\!\left((s-t)\theta_c+\phi_{k,sc}-\phi_{q,tc}\right).
\]
\begin{itemize}
\item $\mu_{q,tc},\mu_{k,sc}$ are content-dependent magnitudes.
\item $\phi_{q,tc},\phi_{k,sc}$ are also content-dependent phases.
\item Their difference changes which offset maximizes a channel's response.
\end{itemize}
\keypoint{The paper's hypothesis: this coupling can obstruct rules that require separate content and position matches.}
\papersource{Paper, Sec. 2, Eq. (2), PDF p. 2; derivation in App. A.1, p. 12.}
\end{frame}

\begin{frame}{Indirect indexing exposes the required operation}
\begin{block}{Teaching example}
\texttt{ABCDE, D, -2} $\rightarrow$ \texttt{B}: find the unique anchor \texttt{D}, move two positions left, read the target.
\end{block}
\begin{enumerate}
\item Match the anchor by \alert{content}.
\item Shift from that anchor by the requested \alert{offset}.
\item Predict only the target character.
\end{enumerate}
\vspace{1mm}
The actual dataset uses 20--40 distinct letters and offsets from $-15$ to $+15$.
\papersource{Paper, Sec. 4, PDF p. 4; data generation in App. B.1, p. 13. The example above is illustrative.}
\end{frame}

\begin{frame}{PoPE starts each channel on the real axis}
\centering
\includegraphics[width=0.94\linewidth,height=0.62\paperheight,keepaspectratio]{figures/fig-01-rope-vs-pope.pdf}
\papersource{Original paper, Fig. 1, PDF p. 3. Left: RoPE pairs content dimensions. Right: PoPE maps each scalar to a positive magnitude before the position rotation.}
\end{frame}

\begin{frame}{PoPE factorizes the attention contribution}
\[
\tilde q_{tc}=\underbrace{\sigma(q_{tc})}_{\text{content magnitude}}e^{it\omega_c},\qquad
\tilde k_{sc}=\underbrace{\sigma(k_{sc})}_{\text{content magnitude}}e^{i(s\omega_c+\delta_c)},
\]
\[
\boxed{a^{\mathrm{PoPE}}_{ts}=\sum_{c=1}^{d}\sigma(q_{tc})\sigma(k_{sc})
\cos\!\left((s-t)\omega_c+\delta_c\right)}
\]
\begin{itemize}
\item $\sigma(x)=\log(1+e^x)$ ensures a nonnegative magnitude.
\item $\omega_c$ is fixed; $\delta_c$ is learned per head/channel and shared across positions.
\item One complex channel per real feature: $d$ frequencies rather than $d/2$.
\end{itemize}
\papersource{Paper, Sec. 3, Eqs. (3)--(6), PDF pp. 2--3.}
\end{frame}

\begin{frame}{A fixed phase bias can encode directional offsets}
\[
 g_c(\Delta)=\cos(\Delta\omega_c+\delta_c),\qquad \Delta=s-t.
\]
\begin{itemize}
\item If $\delta_c=0$, then $g_c(-\Delta)=g_c(\Delta)$ for an individual channel.
\item A learned nonzero $\delta_c$ allows different responses to left and right offsets.
\item Crucially, $\delta_c$ does not vary with the current key/query content.
\item The authors constrain $\delta_c\in[-2\pi,0]$; zero initialization helps length extrapolation.
\end{itemize}
\papersource{Paper, Sec. 3, Eq. (6), PDF p. 3. The symmetry statement is a direct deduction from the equation.}
\end{frame}

\begin{frame}{Complex attention has a measurable compute cost}
\[
 a^{\mathrm{PoPE}}_{ts}=\sum_{c=1}^{d}
 \big(x_{q,tc}x_{k,sc}+y_{q,tc}y_{k,sc}\big).
\]
\begin{itemize}
\item A custom Triton/FlashAttention-style kernel computes the real part without a complex score matrix.
\item The appendix estimates extra forward FLOPs as $L/(12D+2L)$ of one layer's dominant FLOPs.
\item At model width $D=1280$: $5.88\%$ at $L=1024$; $17.39\%$ at $L=4096$.
\item The current general implementation also increases complex Q/K traffic; in-kernel rotation could reduce it.
\end{itemize}
\papersource{Paper, Sec. 3, Eq. (10), PDF p. 4; App. A.2, pp. 12--13. FLOPs are analytical estimates.}
\end{frame}

\begin{frame}{Evaluation links each experiment to one claim}
\renewcommand{\arraystretch}{1.25}
\begin{tabular}{>{\raggedright\arraybackslash}p{0.27\linewidth}>{\raggedright\arraybackslash}p{0.36\linewidth}>{\raggedright\arraybackslash}p{0.24\linewidth}}
\toprule
Question & Data / setup & Metric / direction \\
\midrule
Can it compose content and offset? & Indirect indexing; 3 seeds & Target accuracy $\uparrow$ \\
Does it model ordered sequences? & JSB, MAESTRO, human genome & Test NLL $\downarrow$ \\
Does it scale on language? & OpenWebText; 124M--774M & Val. perplexity $\downarrow$ \\
Does it help downstream? & Six zero-shot tasks & Mean accuracy $\uparrow$ \\
Does it extrapolate? & PG-19, up to $10\times$ context & Test perplexity $\downarrow$ \\
\bottomrule
\end{tabular}
\papersource{Paper, Sec. 4, PDF pp. 4--7; App. B, pp. 13--15. Main RoPE/PoPE pairs share model and training settings.}
\end{frame}

\begin{frame}{PoPE solves the diagnostic indexing task}
\begin{center}
\renewcommand{\arraystretch}{1.3}
\begin{tabular}{lc}
\toprule
Position method & Test accuracy (\%, mean $\pm$ SD) \\
\midrule
RoPE & $11.16\pm2.45$ \\
RoPE, half channels & $28.50\pm6.10$ \\
ALiBi, half heads & $27.35\pm10.77$ \\
\textbf{PoPE} & $\mathbf{94.82\pm2.91}$ \\
\bottomrule
\end{tabular}
\end{center}
\keypoint{PoPE exceeds full RoPE by 83.66 percentage points on this targeted task.}
\papersource{Paper, Table 1, PDF p. 4; additional baselines in App. C, Table 9, p. 15. Three seeds.}
\end{frame}

\begin{frame}{Sequence modeling improves in music and genomics}
\begin{center}
\renewcommand{\arraystretch}{1.3}
\begin{tabular}{lcc}
\toprule
Dataset & RoPE test NLL & PoPE test NLL \\
\midrule
JSB Chorales & 0.5081 & \textbf{0.4889} \\
MAESTRO & 1.501 & \textbf{1.486} \\
Human Reference Genome & 4.217 & \textbf{4.152} \\
\bottomrule
\end{tabular}
\end{center}
\begin{itemize}
\item Lower is better; these are the reported best test NLL values.
\item Music uses up to 2,048 tokens; genome uses up to 1,000 tokens.
\end{itemize}
\papersource{Paper, Tables 2--3, PDF pp. 4--5; sequence lengths in App. B.3, p. 15.}
\end{frame}

\begin{frame}{OpenWebText gains persist from 124M to 774M}
\begin{center}
\renewcommand{\arraystretch}{1.3}
\begin{tabular}{lcc}
\toprule
Parameters & RoPE val. perplexity & PoPE val. perplexity \\
\midrule
124M & 21.55 & \textbf{21.33} \\
253M & 18.88 & \textbf{18.55} \\
774M & 15.85 & \textbf{15.45} \\
\bottomrule
\end{tabular}
\end{center}
\keypoint{The direction of the perplexity improvement is consistent across the three reported model sizes.}
\papersource{Paper, Table 4, PDF p. 5. OpenWebText pretraining uses 1,024-token contexts.}
\end{frame}

\begin{frame}{Ablations: the positive magnitude map matters most}
\begin{center}
\renewcommand{\arraystretch}{1.22}
\begin{tabular}{lcc}
\toprule
PoPE variant & 124M PPL & 253M PPL \\
\midrule
No softplus & 21.57 & 18.93 \\
ReLU instead of softplus & 21.55 & 18.90 \\
No learned $\delta$ & 21.42 & 18.57 \\
\textbf{Full PoPE} & \textbf{21.33} & \textbf{18.55} \\
\bottomrule
\end{tabular}
\end{center}
\begin{itemize}
\item All values are OpenWebText validation perplexity; lower is better.
\item Removing $\delta$ costs 0.09 PPL at 124M and 0.02 at 253M.
\end{itemize}
\papersource{Paper, Table 5, PDF p. 5.}
\end{frame}

\begin{frame}{Downstream mean accuracy improves modestly}
\begin{center}
\renewcommand{\arraystretch}{1.28}
\begin{tabular}{lccc}
\toprule
Parameters & RoPE mean (\%) & PoPE mean (\%) & Difference (pp) \\
\midrule
124M & 45.33 & \textbf{46.19} & +0.86 \\
253M & 48.76 & \textbf{48.78} & +0.02 \\
774M & 51.80 & \textbf{52.46} & +0.66 \\
\bottomrule
\end{tabular}
\end{center}
\begin{itemize}
\item Mean of six zero-shot task accuracies; not every individual task improves.
\item At 124M, RoPE remains higher on LAMBADA, HellaSwag, and PIQA.
\end{itemize}
\papersource{Paper, Table 6, PDF p. 6. pp = percentage points.}
\end{frame}

\begin{frame}{Length extrapolation compares distinct adaptation budgets}
\begin{itemize}
\item OpenWebText pretraining context: \textbf{1,024 tokens}.
\item PG-19 test contexts: 1,024 to \textbf{10,240 tokens}.
\item \textbf{RoPE} and \textbf{PoPE}: evaluated without long-context fine-tuning.
\item \textbf{YaRN}: RoPE frequency interpolation plus 500 fine-tuning steps at 4,096 tokens.
\item \textbf{PoPE+ft}: same 4,096-token, 500-step fine-tuning, without frequency interpolation.
\end{itemize}
\keypoint{The figure separates the out-of-the-box PoPE result from both fine-tuned alternatives.}
\papersource{Paper, Sec. 4, PDF pp. 5--7, Fig. 2. Fine-tuning uses OpenWebText; evaluation uses PG-19.}
\end{frame}

\begin{frame}{PoPE remains flatter out to 10$\times$ context length}
\centering
\includegraphics[width=0.94\linewidth,height=0.70\paperheight,keepaspectratio]{figures/fig-02-length-extrapolation.pdf}
\papersource{Original paper, Fig. 2, PDF p. 7. Lower perplexity is better. Green: PoPE zero-shot; blue: PoPE+ft; orange: YaRN; red: RoPE. Line styles denote model size.}
\end{frame}

\begin{frame}{Frequency probes support a broader-use interpretation}
\begin{tabular}{p{0.18\linewidth}p{0.72\linewidth}}
\toprule
Model & Observed channel-norm pattern \\
\midrule
RoPE & Higher norms in a sparse subset of frequency channels. \\
PoPE & More distributed use, including higher frequencies except in the first layer. \\
\bottomrule
\end{tabular}
\vspace{4mm}
\begin{itemize}
\item Measured across layers for 124M models; the appendix shows a similar 253M pattern.
\item PoPE has twice as many frequency rows because each real feature is rotated separately.
\item These plots describe usage, while task results establish performance.
\end{itemize}
\papersource{Paper, Fig. 3, PDF p. 8; App. D, Figs. 4--5, p. 16.}
\end{frame}

\begin{frame}{Takeaways and practical trade-offs}
\begin{enumerate}
\item \textbf{Mechanism:} PoPE fixes position-only phases and puts content in positive magnitudes.
\item \textbf{Evidence:} a large indexing gain; lower reported modeling losses; strong PG-19 extrapolation to 10,240 tokens.
\item \textbf{Scope:} downstream mean gains can be small (0.02 pp at 253M), with mixed task-level results.
\item \textbf{Cost:} complex attention adds arithmetic and memory traffic in the reported implementation.
\end{enumerate}
\papersource{Paper, Secs. 3--4, PDF pp. 2--8; App. A.2, pp. 12--13.}
\end{frame}

\begin{frame}{Paper and code}
\begin{itemize}
\item Gopalakrishnan, Csord\'as, Schmidhuber, Mozer. \emph{Decoupling the ``What'' and ``Where'' With Polar Coordinate Positional Embedding}. ICML 2026, PMLR 306:36197--36214.\\[2mm]
\href{https://proceedings.mlr.press/v306/gopalakrishnan26a.html}{Official PMLR proceedings record}
\item Paper version read: \href{https://arxiv.org/abs/2509.10534v3}{arXiv:2509.10534v3} (18 PDF pages).
\item Author implementation: \href{https://github.com/agopal42/pope}{agopal42/pope}.
\end{itemize}
\end{frame}

\appendix
\begin{frame}{Backup: experimental configurations}
\begin{center}
\renewcommand{\arraystretch}{1.16}
\begin{tabular}{lccc}
\toprule
Dataset & Layers & Context & Training steps \\
\midrule
Indirect indexing & 8 & 40 & 100,000 \\
JSB & 6 & 2,048 & 3,000 \\
MAESTRO & 6 & 2,048 & 60,000 \\
Human genome & 16 & 1,000 & 100,000 \\
OpenWebText & 12 / 16 / 36 & 1,024 & 100,000 \\
\bottomrule
\end{tabular}
\end{center}
\papersource{Paper, App. B.2--B.3, Tables 7--8, PDF pp. 14--15. OpenWebText entries correspond to 124M/253M/774M.}
\end{frame}

\begin{frame}{Backup: the v3 printed frequency differs from code}
\begin{itemize}
\item The v3 PDF prints PoPE's frequency after Eq. (4) as $\theta^{(c-1)/d}$.
\item The \href{https://github.com/agopal42/pope/blob/main/position.py\#L1070-L1074}{author implementation} computes $1/\theta^{(c-1)/d}$ with base $\theta=10{,}000$.
\item The v3 PDF's clamp sentence on p. 3 names $\theta_c$; the \href{https://github.com/agopal42/pope/blob/main/position.py\#L1153-L1160}{implementation} clamps $\delta_c$.
\item Our mechanism slides denote the fixed frequency by $\omega_c$ and discuss the intended bias clamp.
\end{itemize}
\papersource{Paper, Sec. 3, PDF pp. 2--3; linked official code.}
\end{frame}

\end{document}