\documentclass[aspectratio=169,11pt]{beamer}
\usepackage[T1]{fontenc}
\usepackage[utf8]{inputenc}
\usepackage{lmodern}
\usepackage{amsmath,amssymb,graphicx,booktabs}

\definecolor{AcademicBlue}{RGB}{30,56,85}
\definecolor{AcademicTeal}{RGB}{0,105,112}
\setbeamercolor{structure}{fg=AcademicBlue}
\setbeamercolor{frametitle}{fg=AcademicBlue,bg=white}
\setbeamercolor{title}{fg=AcademicBlue}
\setbeamercolor{alerted text}{fg=AcademicTeal}
\setbeamertemplate{navigation symbols}{}
\setbeamertemplate{itemize items}[circle]
\setbeamertemplate{footline}{\hfill\insertframenumber\,/\,\inserttotalframenumber\hspace{4mm}\vspace{3mm}}
\setbeamersize{text margin left=8mm,text margin right=8mm}
\setbeamerfont{frametitle}{size=\Large,series=\bfseries}
\setbeamerfont{title}{size=\LARGE,series=\bfseries}
\newcommand{\papersource}[1]{\par\smallskip{\scriptsize\color{gray}#1\par}}
\newcommand{\CA}{\textsc{CA}}
\newcommand{\CAI}{\textsc{Continuum Attention}}

\title[Continuum Attention]{Continuum Attention: Exact Retrieval over a Continuous Memory}
\subtitle{Piecewise-linear key/value trajectories and exponential moments}
\author{Afshin Khadangi}
\date{arXiv:2610.20586v2 \quad|\quad 9 November 2026}

\begin{document}
\begin{frame}
  \titlepage
  \vspace{-5mm}
  \begin{center}
    \footnotesize
    Publication not found as of 2026-11-09.
    \href{https://arxiv.org/abs/2610.20586v2}{arXiv record}\\
    \href{https://github.com/akhadangi/continuum-attention}{Official code: Metal and CUDA implementation}
  \end{center}
  \papersource{Paper identity and status checked 2026-11-09.}
\end{frame}

\begin{frame}{CA replaces point lookup with integration along memory}
  \begin{itemize}
    \item Softmax attention scores a query against keys at discrete token positions.
    \item CA joins adjacent keys and values into continuous piecewise-linear trajectories.
    \item A query creates an exponential relevance density and integrates values over its visible prefix.
  \end{itemize}
  \begin{center}
    \large\textbf{The decoder stays the same; the retrieval rule changes.}
  \end{center}
  \papersource{Paper, Sec. 1, PDF pp. 1--2.}
\end{frame}

\begin{frame}{Each query retrieves from a normalized continuous density}
  \[
  \mathbf{k}_j(u)=(1-u)\mathbf{k}_{j-1}+u\mathbf{k}_{j},\qquad
  \mathbf{v}_j(u)=(1-u)\mathbf{v}_{j-1}+u\mathbf{v}_{j}
  \]
  \[
  s_i(t)=\frac{\gamma}{\sqrt{d_h}}\mathbf{q}_i^\top\mathbf{k}(t),
  \quad p_i(t)=\frac{e^{s_i(t)}}{Z_i},
  \quad \mathbf{o}_i=\int_{-1/2}^{i+1/2}p_i(t)\mathbf{v}(t)\,dt
  \]
  \centering
  \includegraphics[width=0.86\linewidth,height=0.42\textheight,keepaspectratio]{figures/fig1_overview.pdf}
  \papersource{Paper, Sec. 2.1--2.2, Eqs. (2)--(8), PDF pp. 2--3; Fig. 1, p. 1.}
\end{frame}

\begin{frame}{Affine scores make every segment integral exact}
  For segment $j$, let $a=s_i(j-1)$ and $\Delta=s_i(j)-s_i(j-1)$:
  \[
  J_n(\Delta)=\int_0^1u^n e^{\Delta u}\,du,\quad
  M_j=e^aJ_0(\Delta)
  \]
  \[
  \mathbf U_j=e^a\big[(J_0-J_1)\mathbf v_{j-1}+J_1\mathbf v_j\big]
  =M_j\big[(1-\mu)\mathbf v_{j-1}+\mu\mathbf v_j\big],
  \quad \mu=\frac{J_1}{J_0}
  \]
  \begin{itemize}
    \item $J_0$ and $J_1$ are sufficient for the forward mass and value numerator.
    \item $\mu$ moves the effective retrieval point toward the endpoint with higher score.
    \item Closed forms have finite limits at $\Delta=0$.
  \end{itemize}
  \papersource{Paper, Sec. 2.3, Eqs. (10)--(18), PDF pp. 3--4.}
\end{frame}

\begin{frame}{Half-token caps preserve discrete attention at equal scores}
  \[
  Z_i^{\mathrm{cap}}=\tfrac12e^{s_i(0)}+\tfrac12e^{s_i(i)},\quad
  \mathbf U_i^{\mathrm{cap}}=\tfrac12e^{s_i(0)}\mathbf v_0+
  \tfrac12e^{s_i(i)}\mathbf v_i
  \]
  \[
  \mathbf o_i=\frac{\mathbf U_i^{\mathrm{cap}}+\sum_{j=1}^{i}\mathbf U_{ij}}
  {Z_i^{\mathrm{cap}}+\sum_{j=1}^{i}M_{ij}}
  \]
  \begin{itemize}
    \item Equal visible scores give every token equal total mass: uniform causal attention.
    \item In general, CA is a convex combination of original values: weights are nonnegative and sum to one.
  \end{itemize}
  \papersource{Paper, Sec. 2.4, Eqs. (19)--(22), PDF p. 5.}
\end{frame}

\begin{frame}{The analytic backward pass needs moments only through order two}
  \begin{itemize}
    \item Score endpoint gradients combine adjacent segment contributions and the half-token caps.
    \item Query and key gradients follow from the complete endpoint-score gradients.
    \item Value gradients use the final normalized effective weights.
    \item The learned positive gain $\gamma=e^\rho$ has an analytic scalar gradient.
  \end{itemize}
  \begin{alertblock}{Stable evaluation}
    Near $\Delta=0$, evaluate $J_n$ with a series; elsewhere shift scores by the visible maximum before exponentiation.
  \end{alertblock}
  \papersource{Paper, Secs. 2.5--2.6, Eqs. (23)--(35), PDF pp. 5--6.}
\end{frame}

\begin{frame}{CA is exact, but it remains quadratic attention}
  \begin{columns}[T,onlytextwidth]
    \column{0.48\textwidth}
      \begin{block}{Computation}
        Segment integrals are independent and can be tiled across GPU query prefixes. No quadrature or sequential scan over memory is needed.
      \end{block}
    \column{0.48\textwidth}
      \begin{block}{Cost}
        Arithmetic is $\mathcal O(T^2d_h)$, as in dense softmax. The paper's CUDA path materializes intermediate attention tiles.
      \end{block}
  \end{columns}
  \vspace{2mm}
  \begin{center}\alert{The method changes the inductive bias, not the asymptotic complexity.}\end{center}
  \papersource{Paper, Sec. 2.6, PDF p. 6; Sec. 3.6, PDF pp. 10--11.}
\end{frame}

\begin{frame}{A five-seed study isolates continuous integration from temperature}
  \begin{itemize}
    \item TinyStories byte LM: six layers, width 256, four heads, contexts 192 and 512.
    \item Eight attention settings × five paired seeds × two contexts = 80 runs.
    \item Fixed softmax $\tau=0.30$ (B$_{0.30}$) is parameter matched to fixed-gain CA $\tau=0.30$ (P).
    \item Learned-gain controls test whether results depend on optimizing the score scale.
  \end{itemize}
  \papersource{Paper, Appendix A, PDF p. 15; development protocol and results, pp. 16--18.}
\end{frame}

\begin{frame}{Matched-temperature CA lowers NLL at both context lengths}
  \begin{center}
    \begin{tabular}{lrr}
      \toprule
      Configuration & $T=192$ & $T=512$ \\
      \midrule
      Softmax B$_{0.30}$ & 1.029903 & 1.174585 \\
      CA P, fixed $\tau=0.30$ & \textbf{1.009888} & \textbf{1.126540} \\
      \midrule
      Improvement (nats/byte) & 0.020016 & 0.048045 \\
      Paired 95\% CI & [0.013779, 0.026253] & [0.033504, 0.062587] \\
      \bottomrule
    \end{tabular}
  \end{center}
  \begin{itemize}
    \item CA wins all five paired seeds at both lengths.
    \item Learned-gain H vs. G has a CI that includes zero at 192, but not at 512.
  \end{itemize}
  \papersource{Paper, Appendix A.1 / Table A.1, PDF p. 16; paired comparisons, Table A.3, p. 18.}
\end{frame}

\begin{frame}{The larger byte-level model has a matched training recipe}
  \begin{itemize}
    \item 101.2M parameters; 256-byte vocabulary; 2,048-byte context; eight layers, 16 heads.
    \item 26.2B FineWeb-Edu bytes, 50K steps, batch 524,288 bytes per step.
    \item Four NVIDIA V100 GPUs, FP16 training, three-seed evaluation.
    \item Baselines: Transformer, FastWeight, Byte Latent Transformer, and AU-Net.
  \end{itemize}
  \papersource{Paper, Sec. 3.1, PDF pp. 7--8.}
\end{frame}

\begin{frame}{CA reaches the selected loss target with fewer training bytes}
  \centering
  \includegraphics[width=0.98\linewidth,height=0.56\textheight,keepaspectratio]{figures/fig2_training_results.png}
  \begin{itemize}
    \item At about 1.10 training BPB: CA uses 8.9B bytes; Transformer uses 15.4B (1.74$\times$ data efficiency).
    \item At 8.9B bytes, CA BPB is 4.8\% lower; it uses 10.4 GB/GPU vs. FastWeight's 24.6 GB.
  \end{itemize}
  \papersource{Paper, Sec. 3.2 / Fig. 2, PDF p. 8; threshold and memory comparisons are specific to this run.}
\end{frame}

\begin{frame}{The largest language-model gain is multilingual}
  \centering
  \includegraphics[width=0.92\linewidth,height=0.57\textheight,keepaspectratio]{figures/fig3_domains.pdf}
  \begin{itemize}
    \item FLORES macro BPB: CA 1.8915 vs. AU-Net 2.0077, averaged across three seeds.
    \item CA is competitive on FineWeb-Edu and PG-19, but is weakest on Stack-smol code.
  \end{itemize}
  \papersource{Paper, Sec. 3.3 / Fig. 3 and Table 1, PDF p. 9. Lower BPB is better.}
\end{frame}

\begin{frame}{FLORES improvements are positive in all eight languages}
  \centering
  \includegraphics[width=0.94\linewidth,height=0.55\textheight,keepaspectratio]{figures/fig4_flores.pdf}
  \begin{itemize}
    \item Macro BPB is 1.8915; relative improvement is 5.8\% over the strongest macro baseline.
    \item Even the smallest gain, Bengali, is positive (+0.7\% vs. Transformer).
  \end{itemize}
  \papersource{Paper, Sec. 3.4 / Fig. 4 and Table 2, PDF p. 10; macro averages languages uniformly.}
\end{frame}

\begin{frame}{CA wins two multiple-choice tasks, not the overall mean}
  \centering
  \includegraphics[width=0.92\linewidth,height=0.48\textheight,keepaspectratio]{figures/fig5_multiple_choice.pdf}
  \begin{itemize}
    \item Highest scores: HellaSwag 30.70\% and ARC-Challenge 23.08\%.
    \item Four-task mean: 37.01\%, slightly below Transformer 37.02\% and FastWeight 37.09\%.
  \end{itemize}
  \papersource{Paper, Sec. 3.5 / Fig. 5 and Table 3, PDF p. 11. Differences in the plot are percentage points.}
\end{frame}

\begin{frame}{Byte perturbation results show specific robustness trade-offs}
  \centering
  \includegraphics[width=0.92\linewidth,height=0.52\textheight,keepaspectratio]{figures/fig6_robustness.pdf}
  \begin{itemize}
    \item CA leads on clean, repeated-byte, uppercase, and random-case prompts.
    \item FastWeight is stronger on byte deletion and antspeak; CA falls from 30.70\% clean to 25.76\% antspeak.
  \end{itemize}
  \papersource{Paper, Sec. 3.5 / Fig. 6 and Table 4, PDF pp. 11--12.}
\end{frame}

\begin{frame}{The GPU kernel is verified, while end-to-end speed remains open}
  \begin{itemize}
    \item V100 isolated operator, $B=2$, $H=16$, $T=2{,}048$, $d_h=64$: 16.60 ms forward + backward.
    \item Comparators: 23.47 ms conventional tiled; 19.52 ms fused score/gradient in a same-run comparison.
    \item Causality tests at lengths through 2,048: future-token perturbations leave earlier outputs unchanged; tested future gradients are zero.
    \item Full-model one-V100 inference: about $7.8\times10^4$ input bytes/s at batch 8.
  \end{itemize}
  \begin{alertblock}{Interpretation}
    These are operator-level timing and verification results, not a demonstrated end-to-end training speedup.
  \end{alertblock}
  \papersource{Paper, Sec. 3.6, PDF pp. 10--11; small-model numerical preflight, Appendix A.4 / p. 18.}
\end{frame}

\begin{frame}{What CA establishes, and what the experiments leave open}
  \begin{columns}[T,onlytextwidth]
    \column{0.49\textwidth}
      \textbf{Supported here}
      \begin{itemize}
        \item Exact causal integration with normalized, nonnegative weights.
        \item Matched-temperature NLL gains in five-seed small-model tests.
        \item Strong FLORES BPB across all eight evaluated languages.
      \end{itemize}
    \column{0.49\textwidth}
      \textbf{Current trade-offs}
      \begin{itemize}
        \item Quadratic arithmetic; intermediate attention tiles are materialized.
        \item Lower Stack-smol code performance and uneven task results.
        \item Larger-model and longer-context generalization is untested.
      \end{itemize}
  \end{columns}
  \vspace{2mm}
  \begin{center}\large\textbf{Take-away:} interpolate memory, integrate exactly, and measure the systems cost separately.\end{center}
  \papersource{Paper, Secs. 3--5, PDF pp. 7--12; full identity: \href{https://arxiv.org/abs/2610.20586v2}{arXiv:2610.20586v2}.}
\end{frame}

\end{document}