\documentclass[aspectratio=169,11pt]{beamer}
\usepackage[T1]{fontenc}
\usepackage[utf8]{inputenc}
\usepackage{amsmath,amssymb,graphicx,booktabs}
\usepackage{hyperref}

\definecolor{AcademicBlue}{RGB}{26,57,86}
\definecolor{AcademicTeal}{RGB}{0,105,112}
\setbeamercolor{structure}{fg=AcademicBlue}
\setbeamercolor{frametitle}{fg=AcademicBlue,bg=white}
\setbeamercolor{title}{fg=AcademicBlue}
\setbeamercolor{alerted text}{fg=AcademicTeal}
\setbeamertemplate{navigation symbols}{}
\setbeamertemplate{itemize items}[circle]
\setbeamertemplate{footline}{%
  \hfill\usebeamerfont{page number in head/foot}%
  \insertframenumber\,/\,\inserttotalframenumber\hspace{4mm}\vspace{3mm}}
\setbeamersize{text margin left=8mm,text margin right=8mm}
\setbeamerfont{frametitle}{size=\Large,series=\bfseries}
\setbeamerfont{title}{size=\LARGE,series=\bfseries}
\newcommand{\papersource}[1]{\par\smallskip{\scriptsize\color{gray}#1\par}}
\hypersetup{colorlinks=true,urlcolor=AcademicTeal,linkcolor=AcademicBlue}

\title[Transformer-FSQ]{Transformer-FSQ}
\subtitle{Discrete Latents for Extreme KV-Cache Compression}
\author{Andras Szecsenyi \and Thomas L. Lee \and Vaisakh Shaj\\
Elliot J. Crowley \and Amos Storkey}
\institute{The University of Edinburgh}
\date{arXiv:2611.09598v1 \quad 10 November 2026}

\begin{document}
\begin{frame}
  \titlepage
  \vspace{-4mm}
  \begin{center}
  \scriptsize
  Publication not found as of 10 Nov 2026
  \quad\textbar\quad
  \href{https://github.com/ASzecsenyi/transformer-fsq}{Code forthcoming: author-linked repository is empty}
  \end{center}
\end{frame}

\begin{frame}{The KV cache can exceed the model on local hardware}
  \begin{columns}[T]
    \column{0.53\linewidth}
      \begin{itemize}
        \item KV grows linearly; Qwen3-8B cache uses 144 KiB/token across all layers.
        \item Qwen3-8B-Base weights occupy 15.3 GiB in half precision.
        \item On a 16 GiB GPU, about 0.74 GiB remains for cache and runtime.
        \item That fits only about 5.4k uncompressed tokens before activations.
      \end{itemize}
    \column{0.43\linewidth}
      \begin{block}{The target}
        Extend context capacity by shrinking the \emph{per-token state}, while keeping standard Transformer attention.
      \end{block}
      \begin{center}
        \textbf{256$\times$ compression}\\[1mm]
        \Large 1.39M estimated tokens
      \end{center}
      This is a capacity estimate, not a million-token quality test.
  \end{columns}
  \papersource{Paper, §1, PDF p. 1, Fig. 1; Table 3, PDF p. 22.}
\end{frame}

\begin{frame}{FSQ stays useful after other cache methods collapse}
  \centering
  \includegraphics[width=0.96\linewidth,height=0.60\textheight,keepaspectratio]{figures/ruler_vs_compression.pdf}
  \begin{itemize}
    \item At $\leq 5\times$, VecInfer and Palu are often stronger.
    \item At $12\times$--$97.5\times$, Transformer-FSQ has the highest evaluated RULER score across these Qwen3 sizes.
  \end{itemize}
  \papersource{Paper, Fig. 2, PDF p. 2. RULER: 13 tasks, 50 items each, 8,192-token context.}
\end{frame}

\begin{frame}{Compress once, before K/V expansion}
  \begin{columns}[T]
    \column{0.46\linewidth}
      \begin{block}{Standard attention}
        Residual state $x_t$
        \[
          x_t \rightarrow W_kx_t,\ W_vx_t
          \rightarrow \text{store K and V}
        \]
        Each KV head's floating-point keys and values remain in the cache.
      \end{block}
    \column{0.50\linewidth}
      \begin{block}{Transformer-FSQ}
        Residual state $x_t$
        \[
          x_t \rightarrow E x_t \rightarrow Q_{\mathrm{FSQ}}
          \rightarrow \text{one packed code }z_t
        \]
        All KV heads reconstruct their K/V from the same code.
      \end{block}
  \end{columns}
  \vspace{2mm}
  \centering
  \alert{The bottleneck represents the shared state behind K and V, not separate post-hoc quantizers.}
  \papersource{Paper, §4, PDF p. 5; architecture cross-check: Fig. 4, PDF p. 6.}
\end{frame}

\begin{frame}{A wide product code uses four bits per coordinate}
  \small
  For $x_t\in\mathbb{R}^{C}$, encoder $E\in\mathbb{R}^{G\times C}$ produces $G$ latent coordinates:
  \[
    z_t=Q_{\mathrm{FSQ}}(E x_t)
    \in \mathcal L_1\times\cdots\times\mathcal L_G .
  \]
  With $L$ levels per coordinate:
  \[
    \tilde u=h\tanh\!\left(u+\operatorname{atanh}(\delta/h)\right)-\delta,
    \qquad
    \hat u=\tilde u+\operatorname{sg}\!\left(\lfloor\tilde u\rceil-\tilde u\right).
  \]
  \begin{columns}[T]
    \column{0.50\linewidth}
      \begin{itemize}
        \item Main experiments: $L=16$, hence $b=\lceil\log_2L\rceil=4$ bits.
        \item Cache size: $\lceil Gb/8\rceil$ bytes per token per layer.
      \end{itemize}
    \column{0.46\linewidth}
      \begin{itemize}
        \item The $L^G$ grid is implicit; no codebook table or nearest-neighbour lookup.
        \item Straight-through gradients make rounding trainable; the forward pass still uses rounded codes.
      \end{itemize}
  \end{columns}
  \papersource{Paper, §3 and §4.1, PDF pp. 4–5, Eq. (2)–(4). For $L=16$: $h=7.5075$, $\delta=0.5$.}
\end{frame}

\begin{frame}{Attention pools codes before decoding values}
  \[
    \hat k_t=\operatorname{RoPE}_t\!\left(N_k(W_kz_t)\right),
    \qquad
    \hat v_t=W_vz_t
  \]
  \begin{columns}[T]
    \column{0.50\linewidth}
      \begin{block}{Values}
        \[
          \sum_t a_t^{(h)}W_vz_t
          =W_v\!\left(\sum_ta_t^{(h)}z_t\right)
        \]
        Pool the compact codes, then apply $W_v$ once.
      \end{block}
    \column{0.46\linewidth}
      \begin{block}{Keys}
        RoPE depends on each cached position $t$.
        \[
          q^\top\operatorname{RoPE}_t(N_k(W_kz_t))
        \]
        Reconstruct keys at each position before scoring.
      \end{block}
    \end{columns}
    \vspace{1mm}
    This asymmetry saves value-side work, but key reconstruction is the main added arithmetic.
  \papersource{Paper, §4.1 and §4.3, PDF pp. 6–7, Eq. (5)–(6).}
\end{frame}

\begin{frame}{PCA and ridge regression convert a pretrained model}
  \begin{enumerate}
    \item Capture 32,768 ClimbMix residual tokens from the original model.
    \item Initialize encoder rows from PCA directions:
      \[
        E_{i,:}=\frac{u_{i\bmod C}^{\top}}{3\sigma_{i\bmod C}}.
      \]
    \item Quantize projected activations; fit a ridge decoder:
      \[
        D=(Z^\top Z+\lambda I_G)^{-1}Z^\top X,\quad \lambda=1.
      \]
    \item Compose $D$ with original K/V projections, then fine-tune only the inserted encoder/decoder.
  \end{enumerate}
  \begin{block}{Training budget}
    Backbone frozen; ClimbMix; 0.25 tokens per original parameter for reported checkpoints.
  \end{block}
  \papersource{Paper, §4.2 and Appendix C.2–C.3, PDF pp. 6–7, 22–23, Eq. (7)–(8).}
\end{frame}

\begin{frame}{The comparison prices each cache by actual bytes}
  \begin{columns}[T]
    \column{0.51\linewidth}
      \textbf{Models and baselines}
      \begin{itemize}
        \item Qwen3-0.6B, 1.7B, 8B Base
        \item PyramidKV: token eviction
        \item KIVI: scalar quantization
        \item VecInfer: vector quantization
        \item Palu: low-rank projection
        \item TransMLA: latent attention
      \end{itemize}
    \column{0.45\linewidth}
      \textbf{Evidence}
      \begin{itemize}
        \item RULER: 13 tasks, 50 items, 8,192 tokens
        \item LongBench: narrativeqa, gov\_report, trec
        \item Decode: full-model tokens/s vs native SDPA
        \item Charge metadata, windows, rotary state, and auxiliary cache bytes
      \end{itemize}
  \end{columns}
  \papersource{Paper, §4.4–5.1 and Appendix C.4–C.5, PDF pp. 7–8, 22–24.}
\end{frame}

\begin{frame}{RULER: FSQ leads where compression gets extreme}
  \centering
  \includegraphics[width=0.96\linewidth,height=0.69\textheight,keepaspectratio]{figures/ruler_vs_compression.pdf}
  \papersource{Paper, Fig. 2 and §5.2 RQ1, PDF pp. 2, 9–10. Scores use up to 3 seeds; Qwen3-8B uses 1.}
\end{frame}

\begin{frame}{Retrieval quality still falls as the cache gets smaller}
  \begin{table}
    \centering
    \begin{tabular}{lrrr}
      \toprule
      Model & 12$\times$ & 32$\times$ & 97.5$\times$\\
      \midrule
      Qwen3-0.6B & 0.39 & 0.22 & 0.04\\
      Qwen3-1.7B & 0.43 & 0.26 & 0.08\\
      Qwen3-8B   & 0.61 & 0.49 & 0.23\\
      \bottomrule
    \end{tabular}
  \end{table}
  \begin{itemize}
    \item Uncompressed means: 0.72, 0.77, and 0.92, respectively.
    \item Single-key retrieval survives deepest; multi-key distractors and variable tracking decline earlier.
  \end{itemize}
  \papersource{Paper, Tables 9–11, PDF pp. 37–39. Scores are RULER means across 13 tasks.}
\end{frame}

\begin{frame}{LongBench transfers the gain to summaries, not every task}
  \centering
  \includegraphics[width=0.95\linewidth,height=0.56\textheight,keepaspectratio]{figures/longbench_govreport_vs_compression.pdf}
  \begin{itemize}
    \item Qwen3-8B gov\_report: 12.5 ROUGE-L at 32$\times$, 11.4 at 97.5$\times$; Palu scores 7.0 at 102.4$\times$.
    \item On Qwen3-8B narrativeqa, PyramidKV reaches 18.8 F1 at 26.6$\times$; FSQ scores 8.5 at 32$\times$.
  \end{itemize}
  \papersource{Paper, Fig. 5, PDF p. 9; Table 14, p. 42. Qwen3-8B uses one seed.}
\end{frame}

\begin{frame}{Fused decoding offsets key reconstruction at 32$\times$}
  \centering
  \includegraphics[width=0.96\linewidth,height=0.64\textheight,keepaspectratio]{figures/decode_panels.pdf}
  \begin{itemize}
    \item Qwen3-8B: 1.01$\times$ native SDPA at 32$\times$ (95\% interval 0.98–1.05).
    \item At 97.5$\times$: 1.54$\times$ (1.16–1.88); at 409.6$\times$: 1.63$\times$ (1.20–2.26).
  \end{itemize}
  \papersource{Paper, Fig. 6 and Appendix D.5, PDF pp. 11, 29–30. Full-model decode on NVIDIA L40S.}
\end{frame}

\begin{frame}{The kernel reads codes and never writes reconstructed K/V}
  \begin{enumerate}
    \item Read packed codes from HBM and unpack to registers.
    \item Reconstruct keys with $W_k$ on tensor cores; apply key norm and RoPE on chip.
    \item Compute scores and online softmax; accumulate attention-weighted codes.
    \item Merge partials and apply $W_v$ once, after pooling.
  \end{enumerate}
  \begin{block}{At 32$\times$}
    128 B/token/layer of packed codes replace 4,096 B/token/layer of half-precision K/V.
  \end{block}
  \papersource{Paper, Appendix D.4, PDF pp. 27–28, Fig. 13. The figure's stored bytes do not equal all repeated head reads.}
\end{frame}

\begin{frame}{Eviction and FSQ can extend projected cache capacity}
  \centering
  \includegraphics[width=0.95\linewidth,height=0.55\textheight,keepaspectratio]{figures/capability_frontier_ruler_8k.pdf}
  \begin{itemize}
    \item At RULER $\geq0.30$: 3.80M projected tokens vs 2.23M for shared-head PyramidKV (1.7$\times$, 95\% interval 1.0–2.3).
    \item Uses a shared-head PyramidKV variant and extrapolates quality measured at 8,192 tokens.
  \end{itemize}
  \papersource{Paper, Appendix D.3, PDF pp. 26–29, Fig. 11. Capacity is projected from 8K RULER.}
\end{frame}

\begin{frame}{Training through the grid beats post-hoc quantization}
  \begin{columns}[T]
    \column{0.58\linewidth}
      \begin{table}
        \centering
        \begin{tabular}{lrr}
          \toprule
          Method & Width & RULER\\
          \midrule
          Continuous latent & 40 & 0.001\\
          Post-hoc FSQ & 160 & 0.136\\
          Transformer-FSQ (STE) & 160 & \textbf{0.177}\\
          Trained min--max 4-bit & 128 & 0.151\\
          \bottomrule
        \end{tabular}
      \end{table}
      80 B/token/layer (51×), Qwen3-0.6B; mean over 3 seeds.
    \column{0.38\linewidth}
      \begin{itemize}
        \item A wide low-bit latent carries more coordinates than a narrow FP16 latent.
        \item At 16 levels, RULER also exceeds 4- and 256-level grids at both tested budgets.
      \end{itemize}
  \end{columns}
  \papersource{Paper, Appendix E.3, PDF pp. 32, 34–35, Tables 7–8. Qwen3-0.6B, 3 seeds.}
\end{frame}

\begin{frame}{FSQ adapts to a 4-bit weight backbone}
  \begin{center}
    \begin{tabular}{lcc}
      \toprule
      Qwen3-8B setting & RULER & Change\\
      \midrule
      Uncompressed, 16-bit weights & 0.922 & --\\
      Uncompressed, NF4 weights & 0.913 & $-0.009$\\
      FSQ 32$\times$, then quantize weights to NF4 & 0.382 & $-0.111$\\
      FSQ 32$\times$, fine-tuned on frozen NF4 & 0.464 & $-0.029$ vs 16-bit FSQ\\
      \bottomrule
    \end{tabular}
  \end{center}
  Fine-tuning the discrete cache bottleneck on the quantized backbone recovers 74\% of the post-hoc loss.
  \papersource{Paper, §5.2 RQ5, PDF pp. 10–11. NF4 quantizes decoder linear layers in groups of 64.}
\end{frame}

\begin{frame}{The evidence supports a regime-specific conclusion}
  \begin{columns}[T]
    \column{0.49\linewidth}
      \begin{block}{Where it works}
        \begin{itemize}
          \item RULER 8K: strongest evaluated method at 12$\times$--97.5$\times$.
          \item gov\_report: competitive at 32$\times$--97.5$\times$.
          \item 32$\times$ Qwen3-8B decode: matches native SDPA.
        \end{itemize}
      \end{block}
    \column{0.47\linewidth}
      \begin{block}{What remains open}
        \begin{itemize}
          \item VecInfer/Palu can be better at moderate compression.
          \item PyramidKV is stronger on 8B narrativeqa.
          \item No code/math/many-shot reasoning test; RULER uses 8K.
          \item 8B quality results use one training seed.
        \end{itemize}
      \end{block}
  \end{columns}
  \papersource{Paper, §5–7, PDF pp. 8–12; Appendix D and G, pp. 25–29, 37–42.}
\end{frame}

\begin{frame}{Takeaways}
  \begin{enumerate}
    \item Cache a shared discrete state before K/V expansion; use the same code for all KV heads.
    \item Train through the low-bit grid so its coordinates fit the inference representation.
    \item The strongest evidence is at extreme compression, with an attention-aware fused decoder.
  \end{enumerate}
  \vspace{3mm}
  \begin{block}{Paper}
    \href{https://arxiv.org/abs/2611.09598v1}{Szecsenyi et al., arXiv:2611.09598v1 (2026)}
  \end{block}
  \papersource{Author-linked repository, currently empty: \href{https://github.com/ASzecsenyi/transformer-fsq}{transformer-fsq}.}
\end{frame}

\appendix
\begin{frame}{Backup: the grid size is an allocation choice}
  \begin{center}
    \begin{tabular}{lrrr}
      \toprule
      Levels per coordinate & 144 B/token & 80 B/token & Bits/coordinate\\
      \midrule
      4 & 0.178 & 0.104 & 2\\
      16 & \textbf{0.232} & \textbf{0.177} & 4\\
      256 & 0.161 & 0.035 & 8\\
      \bottomrule
    \end{tabular}
  \end{center}
  Each cache budget is fixed; changing level count changes the available latent width.
  \papersource{Paper, Appendix E.3, PDF p. 35, Table 8.}
\end{frame}

\begin{frame}{Backup: capacity and accuracy are counted by different protocols}
  \begin{itemize}
    \item RULER accuracy is measured at a fixed 8,192-token context.
    \item Cache bytes include all per-token data and metadata; ratios are priced at 8,192 tokens.
    \item The 16 GiB frontier maps those fixed-context scores to an estimated number of retained tokens.
    \item For the stacked capacity curve, prefill peak memory sets the ceiling.
  \end{itemize}
  \papersource{Paper, §4.4 and Appendix D.3, PDF pp. 7, 26–29, Fig. 11.}
\end{frame}

\begin{frame}{Backup: main positioning choices}
  \begin{itemize}
    \item Post-projection FSQ keeps the pretrained model's key normalization and RoPE path.
    \item FSQ-pre moves RoPE before the encoder, but query-key normalization breaks relative-position preservation in the reported probe.
    \item Per-head codes and nonlinear encoders worsened validation BPB; a shared code and linear encoder performed better.
    \item At moderate ratios, repeating PCA directions can create redundant code coordinates and parameter overhead.
  \end{itemize}
  \papersource{Paper, §4.3 and Appendix E.1–E.2, E.7–E.8, PDF pp. 7, 30–35.}
\end{frame}

\begin{frame}{References}
  \begin{itemize}
    \item Andras Szecsenyi, Thomas L. Lee, Vaisakh Shaj, Elliot J. Crowley, and Amos Storkey. \emph{Transformer-FSQ: Discrete Latents for Extreme KV-Cache Compression}. arXiv:2611.09598v1, 2026.
      \href{https://arxiv.org/abs/2611.09598v1}{arXiv page}.
    \item Qwen3-0.6B-Base, Qwen3-1.7B-Base, and Qwen3-8B-Base are evaluated using the paper's reported model configurations.
    \item RULER and LongBench scores follow the benchmark protocols described in the paper, §5.1 and Appendix C.5.
  \end{itemize}
  \vfill
  \papersource{Publication not found as of 10 Nov 2026. Code forthcoming: \href{https://github.com/ASzecsenyi/transformer-fsq}{author-linked repository}.}
\end{frame}
\end{document}