% Editable research presentation. Build with the accompanying build_slides.py.
% Main talk: frames 1--22, approximately 25 minutes. Frames 23--25 are backup.
\documentclass[aspectratio=169,11pt]{beamer}
\usepackage[T1]{fontenc}
\usepackage[utf8]{inputenc}
\usepackage{amsmath,amssymb,graphicx,booktabs}

\definecolor{AcademicBlue}{RGB}{26,57,86}
\definecolor{AcademicTeal}{RGB}{0,105,112}
\setbeamercolor{structure}{fg=AcademicBlue}
\setbeamercolor{frametitle}{fg=AcademicBlue,bg=white}
\setbeamercolor{title}{fg=AcademicBlue}
\setbeamercolor{alerted text}{fg=AcademicTeal}
\setbeamercolor{block title}{fg=white,bg=AcademicBlue}
\setbeamercolor{block body}{fg=black,bg=AcademicBlue!5}
\setbeamertemplate{navigation symbols}{}
\setbeamertemplate{itemize items}[circle]
\setbeamertemplate{footline}{%
  \hfill\usebeamerfont{page number in head/foot}%
  \insertframenumber\,/\,\inserttotalframenumber\hspace{4mm}\vspace{3mm}}
\setbeamersize{text margin left=8mm,text margin right=8mm}
\setbeamerfont{frametitle}{size=\Large,series=\bfseries}
\setbeamerfont{title}{size=\LARGE,series=\bfseries}
\newcommand{\papersource}[1]{\par\smallskip{\scriptsize\color{gray}#1\par}}
\newcommand{\R}{\mathbb{R}}
\newcommand{\CODE}{\operatorname{CODE}}
\newcommand{\LAVO}{\operatorname{LAVO}}
\newcommand{\softmax}{\operatorname{softmax}}
\newcommand{\diag}{\operatorname{diag}}
\renewcommand{\arraystretch}{1.17}
\pdfstringdefDisableCommands{\def\translate#1{#1}}

\title[LAVO]{Linear Attention via Orthogonal Memory}
\subtitle{Fixed-size global statistics with local detail}
\author[Paper authors: Zhang et al.]{Paper authors: Jun Zhang, Shuyang Jiang, Jiangtao Feng,\\
Lin Zheng, and Lingpeng Kong}
\date{arXiv:2312.11135v1 (18 December 2023)\\
Research reading presentation \textbar\ 13 October 2026}

\begin{document}

\begin{frame}
  \titlepage
  \centering
  \small\href{https://arxiv.org/abs/2312.11135v1}{arxiv.org/abs/2312.11135v1}
\end{frame}

\begin{frame}{Can each new token read a fixed-size history?}
  \begin{block}{The design goal}
    Keep the cost per new token bounded as the context grows,
    while retaining useful global information and local detail.
  \end{block}
  \begin{itemize}
    \item Compress history into $r$ memory vectors.
    \item Preserve a local window of width $w$.
    \item Use local relative positions to support length extrapolation.
  \end{itemize}
  \medskip
  \alert{The tradeoff: stable long-context behavior with a restricted global memory.}
  \papersource{Paper, Secs. 1--3, PDF pp. 1--5.}
\end{frame}

\begin{frame}{Causality exposes two different efficiency problems}
  \begin{columns}[T,onlytextwidth]
    \column{0.48\textwidth}
    \begin{block}{Growing collections of summaries}
      Fixed-size chunks of length $l$ leave more summaries to read:
      \[
        \sum_{t=1}^n O(t/l)=O(n^2/l).
      \]
      The paper uses causal LongShort as an example.
    \end{block}
    \column{0.48\textwidth}
    \begin{block}{Large recurrent states}
      Kernel attention can retain linear recurrence:
      \[
        S_t=S_{t-1}+\phi(k_t)\otimes v_t.
      \]
      State dimensions and saved training states increase practical cost.
    \end{block}
  \end{columns}
  \medskip
  LAVO chooses a small, incrementally updated statistic for the entire history.
  \papersource{Paper, Sec. 2, PDF pp. 2--3; Appendix A, Eqs. (7)--(11), PDF p. 15.}
\end{frame}

\begin{frame}{CODE compresses tokens into orthogonal directions}
  \centering
  \includegraphics[width=0.97\linewidth,height=0.57\textheight,keepaspectratio]{figures/fig-01-code.pdf}
  \medskip
  \begin{itemize}
    \item Project token features onto $r$ basis directions.
    \item Average each coordinate, then scale its basis vector.
    \item A query attends to these $r$ memory slots.
  \end{itemize}
  \papersource{Original Fig. 1, PDF p. 2; CODE definition in Sec. 3.1, PDF pp. 3--4.}
\end{frame}

\begin{frame}{Each memory slot is a scaled basis vector}
  Context $X\in\R^{n\times d}$; basis $B\in\R^{r\times d}$; coefficients $H\in\R^r$.
  \[
    H=\frac1n\sum_{t=1}^n Bx_t,
    \qquad \widetilde X=B\odot H=\diag(H)B.
  \]
  \[
    \LAVO(q_t,X)=\softmax(q_t^\top\widetilde X^\top)\widetilde X.
  \]
  \begin{itemize}
    \item Row-wise broadcasting gives memory slot $i$: $H_i b_i$.
    \item Softmax is retained; it operates over $r$ slots.
    \item The compressed memory supplies both keys and values.
  \end{itemize}
  \papersource{Paper, Eq. (3) and Sec. 3.1, PDF pp. 3--4. The diagonal form is an equivalent notation.}
\end{frame}

\begin{frame}{The global state stores projected means}
  \begin{block}{Direct consequence of the CODE equation}
    \[
      H=B\bar{x},\qquad \bar{x}=\frac1n\sum_t x_t.
    \]
    For a given basis, all context dependence enters through $r$ scalars.
  \end{block}
  \begin{itemize}
    \item Different histories with the same feature mean share this memory.
    \item Orthogonality separates directions; it does not prevent information loss.
    \item Local preprocessing and earlier layers can encode structure in $x_t$.
  \end{itemize}
  \papersource{Reading analysis derived from Eq. (3), PDF p. 3. This statement concerns CODE on fixed input features.}
\end{frame}

\begin{frame}{An online mean makes the causal state easy to update}
  \[
    H_t=\frac{(t-1)H_{t-1}+Bx_t}{t}
       =H_{t-1}+\frac{Bx_t-H_{t-1}}{t}.
  \]
  \[
    \widetilde X_{\le t}=\diag(H_t)B,
    \qquad
    F_{\mathrm{global},t}
      =\softmax(q_t^\top\widetilde X_{\le t}^{\top})\widetilde X_{\le t}.
  \]
  \begin{itemize}
    \item Each step updates $r$ coefficients and reads $r$ slots.
    \item New information has weight $1/t$ in the mean.
    \item Basic CODE has no content-dependent write or forgetting gate.
  \end{itemize}
  \papersource{Paper, Eq. (4), PDF p. 4. The update interpretation is a direct algebraic consequence.}
\end{frame}

\begin{frame}{Local attention supports both detail and compression}
  \centering
  \includegraphics[width=0.72\linewidth,height=0.29\textheight,keepaspectratio]{figures/fig-02-context.pdf}
  \begin{itemize}
    \item \textbf{Local branch:} extend windows into previous context and mask
          to a bounded, causal neighborhood.
    \item \textbf{Global branch:} aggregate within historical windows using local
          attention before applying CODE.
  \end{itemize}
  \[
    F_t=\tfrac12\left(F_{\mathrm{local},t}+F_{\mathrm{global},t}\right).
  \]
  \papersource{Paper, Sec. 3.2 and original Fig. 2, PDF p. 4. The authors motivate preprocessing by reduced feature variance.}
\end{frame}

\begin{frame}{Relative positions are embedded inside local attention}
  \centering
  \includegraphics[width=0.80\linewidth,height=0.32\textheight,keepaspectratio]{figures/fig-03-position.pdf}
  \[
    A_{ij}\leftarrow A_{ij}+P_{j-i+w-1},\qquad P\in\R^{2w-1}.
  \]
  \begin{itemize}
    \item Local offsets remain bounded when total context length grows.
    \item Position parameters use $O(w)$ space; applying biases costs $O(nw)$.
    \item $O(w)$ describes the bias table, not all training activations.
  \end{itemize}
  \papersource{Paper, Sec. 3.3 and original Fig. 3, PDF pp. 4--5.}
\end{frame}

\begin{frame}{Cross attention reads a compressed source sequence}
  Source $X\in\R^{n\times d}$ and target $Y\in\R^{m\times d}$:
  \[
    Q=YW_q,\qquad \widetilde X=\CODE(X),
  \]
  \[
    \LAVO(Q,X)=\softmax(Q\widetilde X^\top)\widetilde X.
  \]
  \begin{itemize}
    \item Compress the source once; each target query reads $r$ slots.
    \item Remove local features: source and target lack a natural local alignment.
    \item Total compression and retrieval cost: $O(rd(n+m))$.
  \end{itemize}
  \papersource{Paper, Sec. 3.4, Eqs. (5)--(6), PDF p. 5. Query multiplication is written consistently with row-vector shapes.}
\end{frame}

\begin{frame}{Linear cost requires fixed $r$ and $w$}
  \centering
  \begin{tabular}{l@{\hspace{1.8em}}l}
    \toprule
    Computation & Cost including feature dimension $d$ \\
    \midrule
    Basis projection and pooling & $O(nrd)$ \\
    Reading global memory & $O(nrd)$ \\
    Local attention & $O(nwd)$ \\
    Total attention computation & $O(nd(r+w))$ \\
    \bottomrule
  \end{tabular}\par
  \medskip
  \begin{itemize}
    \item Streaming state and local cache stay bounded with history length.
    \item Training activations still grow with sequence length.
    \item The paper suppresses $d$; projection layers have additional cost.
  \end{itemize}
  \papersource{Paper, Secs. 3.1--3.3, PDF pp. 3--5; explicit dimension and state accounting is reading analysis.}
\end{frame}

\begin{frame}{The central test is PG-19 length extrapolation}
  \begin{columns}[T,onlytextwidth]
    \column{0.48\textwidth}
    \begin{block}{Language-model setup}
      \begin{itemize}
        \item 12-layer GPT-2 backbone
        \item Hidden size 768; 12 heads
        \item GPT-2 BPE; 50,257 tokens
        \item LAVO: 122.91M parameters
        \item $r=64$ memory directions
      \end{itemize}
    \end{block}
    \column{0.48\textwidth}
    \begin{block}{Protocol and comparisons}
      \begin{itemize}
        \item Train: 8,192 context length
        \item Test: 8,192 / 12,288 / 16,384
        \item PPL: lower is better
        \item 125K steps; batch size 16
        \item Speed/memory: 1$\times$80GB A100
      \end{itemize}
    \end{block}
  \end{columns}
  \papersource{Paper, Tables 1--2 and Sec. 4.1, PDF pp. 5--6. A separate experiment trains at 2,048 tokens.}
\end{frame}

\begin{frame}{LAVO trades in-range quality for stable extrapolation}
  \centering
  \begin{tabular}{lrrr}
    \toprule
    Model & PPL at 8,192 $\downarrow$ & 12,288 $\downarrow$ & 16,384 $\downarrow$ \\
    \midrule
    FlashAttention & 16.02 & 40.33 & 94.34 \\
    FA + RoPE & \textbf{15.09} & 20.11 & 34.84 \\
    LongShort & 15.52 & \textbf{19.17} & 26.06 \\
    ABC & 31.53 & 86.78 & 147.43 \\
    Local & 19.73 & 21.24 & 22.16 \\
    LAVO & 19.43 & 19.41 & \textbf{19.40} \\
    \bottomrule
  \end{tabular}\par
  \medskip
  \begin{itemize}
    \item At training length, full attention has substantially better PPL.
    \item LAVO changes by only $-0.03$ from 8K to 16K: essentially stable.
  \end{itemize}
  \papersource{Paper, Table 2, PDF p. 6. Selected rows; all models here train at length 8,192.}
\end{frame}

\begin{frame}{The speed advantage emerges at longer sequences}
  \centering
  \begin{tabular}{lrrr}
    \toprule
    Model & 8K speedup $\uparrow$ & 16K speedup $\uparrow$ & 16K memory (MB) $\downarrow$ \\
    \midrule
    Vanilla & 1.00$\times$ & 1.00$\times$ & 25,747 \\
    FlashAttention & \textbf{14.00}$\times$ & 18.70$\times$ & \textbf{9,133} \\
    FA + RoPE & 12.40$\times$ & 16.96$\times$ & 9,193 \\
    Local & 5.00$\times$ & 9.10$\times$ & 11,909 \\
    LAVO & 11.49$\times$ & \textbf{22.00}$\times$ & 9,337 \\
    \bottomrule
  \end{tabular}\par
  \medskip
  At 16K, LAVO is about $1.18\times$ faster than FlashAttention in this table,
  with slightly higher memory use. FlashAttention is faster at 8K.
  \papersource{Paper, Table 2, PDF p. 6; 1$\times$80GB A100, relative to vanilla at each length.
  The prose says 20.3$\times$; this slide follows the table's 22.0$\times$. Timing scope is not fully specified.}
\end{frame}

\begin{frame}{Training at 2K supports stable evaluation up to 128K}
  \centering
  \includegraphics[width=0.91\linewidth,height=0.55\textheight,keepaspectratio]{figures/fig-04-unbounded.pdf}
  \begin{itemize}
    \item Separate run: train at 2,048; batch size 64; test on one 80GB A100.
    \item Larger local windows help; gains from length largely plateau after 16K.
    \item This measures language-model PPL, not distant fact retrieval.
  \end{itemize}
  \papersource{Paper, Sec. 4.3 and original Fig. 4, PDF p. 8. Authors report reaching 128K without IO optimization.}
\end{frame}

\begin{frame}{Speech results depend on the attention pattern}
  LJSpeech with Transformer-TTS; MCD and MSD measure acoustic distortion.\par
  \medskip
  \centering
  \begin{tabular}{llrr}
    \toprule
    Pattern & Model & MCD $\downarrow$ & MSD $\downarrow$ \\
    \midrule
    Causal & Vanilla & 4.095 & 2.199 \\
           & S4D & 4.030 & 2.189 \\
           & LAVO & \textbf{4.010} & \textbf{2.179} \\
    \midrule
    Noncausal & LongShort & \textbf{3.913} & \textbf{2.136} \\
              & LAVO & 3.964 & 2.155 \\
    \bottomrule
  \end{tabular}\par
  \medskip
  LAVO leads the causal comparison and ranks second in the noncausal table.
  \papersource{Paper, Tables 3 and 5, PDF p. 7; task description in Sec. 4.1, PDF p. 6. Selected rows.}
\end{frame}

\begin{frame}{Local structure explains much of the summary gain}
  Multi-News; maximum input / summary lengths: 4,096 / 400.\par
  \medskip
  \centering
  \begin{tabular}{llrrr}
    \toprule
    Pattern & Model & R-1 $\uparrow$ & R-2 $\uparrow$ & R-L $\uparrow$ \\
    \midrule
    Causal & S4D & \textbf{34.90} & \textbf{6.65} & \textbf{31.98} \\
           & Local & 33.50 & 6.27 & 30.74 \\
           & LAVO & 34.16 & 6.16 & 31.16 \\
    \midrule
    Noncausal & Vanilla & 34.61 & 6.35 & 31.66 \\
              & Local & 38.50 & 10.54 & 35.39 \\
              & LAVO & \textbf{39.01} & \textbf{10.77} & \textbf{35.70} \\
    \bottomrule
  \end{tabular}\par
  \smallskip
  Noncausal R-1: $+4.40$ over vanilla, but $+0.51$ over local.
  Causal R-2: local exceeds LAVO.
  \papersource{Paper, Tables 4 and 6, PDF p. 7. Selected rows; numeric values take precedence over the table's R-2 underlining.}
\end{frame}

\begin{frame}{Compressed source memory also helps cross attention}
  \begin{columns}[T,onlytextwidth]
    \column{0.48\textwidth}
    \begin{block}{Point cloud completion}
      PCN dataset; PoinTr backbone.
    \end{block}
    \centering
    \begin{tabular}{lrr}
      \toprule
      Metric & Vanilla & LAVO \\
      \midrule
      CDL1 $\downarrow$ & 7.425 & \textbf{7.200} \\
      CDL2 $\downarrow$ & 0.231 & \textbf{0.216} \\
      F-score $\uparrow$ & 0.779 & \textbf{0.794} \\
      \bottomrule
    \end{tabular}
    \column{0.48\textwidth}
    \begin{block}{Time-series forecasting}
      Informer; averaged evaluation settings.
    \end{block}
    \centering
    \begin{tabular}{lrr}
      \toprule
      Metric & Vanilla & LAVO \\
      \midrule
      ETT MSE $\downarrow$ & 1.138 & \textbf{1.123} \\
      ETT MAE $\downarrow$ & 0.775 & \textbf{0.765} \\
      Weather MSE $\downarrow$ & 0.478 & \textbf{0.444} \\
      Weather MAE $\downarrow$ & 0.508 & \textbf{0.492} \\
      \bottomrule
    \end{tabular}
  \end{columns}
  \medskip
  LAVO leads all reported metrics against vanilla, ABC, Performer, and cosFormer.
  \papersource{Paper, Tables 7--8, PDF p. 7; Sec. 4.2, PDF p. 8. ETT averages three datasets; both datasets average univariate/multivariate settings.}
\end{frame}

\begin{frame}{Context dissection has the larger ablation effect}
  \centering
  \begin{tabular}{lrrr}
    \toprule
    Configuration & R-1 $\uparrow$ & R-2 $\uparrow$ & R-L $\uparrow$ \\
    \midrule
    LAVO & \textbf{39.19} & \textbf{10.82} & \textbf{35.95} \\
    Without EPE & 39.01 & 10.77 & 35.70 \\
    Without EPE and CD & 35.29 & 6.80 & 32.29 \\
    \bottomrule
  \end{tabular}\par
  \medskip
  \begin{itemize}
    \item EPE: embedded position encoding; CD: context dissection.
    \item The final row \textbf{retains local features}; it removes dissection.
    \item R-1 losses: $0.18$ for EPE, then $3.72$ for CD conditional on no EPE.
  \end{itemize}
  \papersource{Paper, Table 9, PDF p. 8; Sec. 4.3, PDF p. 9.
  Caption/metrics identify summarization, but the prose references the LM setup.
  The full protocol is inconsistent; these values are presented as the table reports them.}
\end{frame}

\begin{frame}{LRA gains are concentrated in three tasks}
  LAVO uses $r=32$ and $w=256$ for all five tasks.\par
  \medskip
  \centering
  \begin{tabular}{lrr@{\hspace{1.8em}}l}
    \toprule
    Task / score $\uparrow$ & LAVO & Best in table & Best model \\
    \midrule
    Text & \textbf{65.28} & 65.28 & LAVO \\
    ListOps & \textbf{40.57} & 40.57 & LAVO \\
    Retrieval & 80.17 & 82.06 & Skyformer \\
    Pathfinder & \textbf{72.14} & 72.14 & LAVO \\
    Image & 40.55 & 48.87 & Reformer \\
    Average & \textbf{59.74} & 59.74 & LAVO \\
    \bottomrule
  \end{tabular}\par
  \medskip
  Average margin: $+2.37$ over vanilla; $+0.35$ over Skyformer.
  \papersource{Paper, Appendix C, Table 11, PDF p. 16.}
\end{frame}

\begin{frame}{A useful tradeoff: bounded statistics plus local detail}
  \begin{enumerate}
    \item \textbf{Mechanism.} Average projections provide a compact recurrent
          state; softmax reads a fixed number of memory slots.
    \medskip
    \item \textbf{Evidence.} PG-19 PPL stays stable beyond training length;
          cross-attention and other sequence tasks also benefit.
    \medskip
    \item \textbf{Tradeoff.} Mean pooling loses distinctions between histories;
          local structure is essential, and in-range LM quality is weaker.
  \end{enumerate}
  \medskip
  \alert{Next question: can selective writes preserve rare distant information
  at the same state budget?}
  \papersource{Synthesis of Secs. 3--4. Author limitation: short-sequence speed trails FlashAttention (Sec. 6, PDF p. 9).
  The proposed next question is reading analysis.}
\end{frame}

\begin{frame}{Paper reference and discussion guide}
  \textbf{Jun Zhang, Shuyang Jiang, Jiangtao Feng, Lin Zheng, Lingpeng Kong.}
  \medskip

  \emph{Linear Attention via Orthogonal Memory.}
  arXiv:2312.11135v1, 18 December 2023.
  \medskip

  \href{https://arxiv.org/abs/2312.11135v1}{https://arxiv.org/abs/2312.11135v1}
  \medskip
  \begin{itemize}
    \item Mechanism: Sec. 3, PDF pp. 3--5.
    \item Language modeling and 128K: Table 2 and Fig. 4, PDF pp. 6, 8.
    \item Additional evidence: Tables 3--11, PDF pp. 7--8, 15--16.
  \end{itemize}
  \papersource{Figures reproduced from the author's v1 TeX source. Tables are selected and reformatted from the paper.
  Algebraic interpretations are labeled as reading analysis. Backup slides follow.}
\end{frame}

\appendix

\begin{frame}{Backup: orthogonal slots can still hide a collision}
  Take $B=I_2$ and two contexts:
  \[
    X^{(a)}=\begin{bmatrix}2&0\\0&2\end{bmatrix},
    \qquad
    X^{(b)}=\begin{bmatrix}1&1\\1&1\end{bmatrix}.
  \]
  Both give $\bar x=(1,1)^\top$, $H=(1,1)^\top$, and $\widetilde X=I_2$.
  \[
    \LAVO(q,X^{(a)})=\LAVO(q,X^{(b)})\quad\text{for every fixed }q.
  \]
  \begin{itemize}
    \item Memory rows are orthogonal, yet the contexts are indistinguishable.
    \item The full model may distinguish them through local/contextual features.
  \end{itemize}
  \papersource{Teaching example derived from Eq. (3), PDF p. 3; not an experiment from the paper.}
\end{frame}

\begin{frame}{Backup: simulated generation has a separate speed table}
  Dummy inputs; recurrent evaluation from prefix length 1 to the target length.\par
  \medskip
  \centering
  \begin{tabular}{lrrr}
    \toprule
    Model & 8,192 $\uparrow$ & 12,288 $\uparrow$ & 16,384 $\uparrow$ \\
    \midrule
    Vanilla & 1.00 & 1.00 & 1.00 \\
    Local & 3.35 & 6.86 & 10.34 \\
    LongShort & 0.70 & 1.71 & 2.94 \\
    FlashAttention & \textbf{7.94} & \textbf{13.10} & 15.01 \\
    FA + RoPE & 5.73 & 10.28 & 12.66 \\
    LAVO & 3.71 & 10.80 & \textbf{20.81} \\
    \bottomrule
  \end{tabular}\par
  \medskip
  Implementation-sensitive speedups relative to vanilla; S4D is omitted.
  At 16K, LAVO / FlashAttention $\approx 1.39$.
  \papersource{Paper, Appendix B, Table 10, PDF p. 15. These values are distinct from Table 2 and are not end-to-end serving throughput.}
\end{frame}

\begin{frame}{Backup: three details that affect interpretation}
  \begin{block}{Maintaining the basis}
    Orthogonal initialization is described, but freezing $B$ versus training it
    with an orthogonality constraint is not clearly specified.
  \end{block}
  \begin{block}{Attributing the improvement}
    There is no controlled orthogonal-versus-nonorthogonal memory ablation.
    Table 9 removes CD only after EPE has already been removed.
  \end{block}
  \begin{block}{Matching the reported configurations}
    Table 6's LAVO row equals Table 9's ``without EPE'' row.
    Table 9's caption and its surrounding protocol description disagree.
  \end{block}
  \papersource{Paper, Sec. 3.1, PDF p. 3; Tables 6 and 9, PDF pp. 7--8; Sec. 4.3, PDF p. 9.}
\end{frame}

\end{document}