\documentclass[aspectratio=169,11pt]{beamer}
\usepackage[T1]{fontenc}
\usepackage[utf8]{inputenc}
\usepackage{amsmath,amssymb,graphicx,booktabs}

\definecolor{AcademicBlue}{RGB}{26,57,86}
\definecolor{AcademicTeal}{RGB}{0,105,112}
\definecolor{WarmOrange}{RGB}{176,91,45}
\setbeamercolor{structure}{fg=AcademicBlue}
\setbeamercolor{frametitle}{fg=AcademicBlue,bg=white}
\setbeamercolor{title}{fg=AcademicBlue}
\setbeamercolor{alerted text}{fg=AcademicTeal}
\setbeamercolor{block title}{fg=white,bg=AcademicBlue}
\setbeamercolor{block body}{fg=black,bg=AcademicBlue!5}
\setbeamertemplate{navigation symbols}{}
\setbeamertemplate{itemize items}[circle]
\setbeamertemplate{footline}{%
  \hfill\usebeamerfont{page number in head/foot}%
  \insertframenumber\,/\,\inserttotalframenumber\hspace{4mm}\vspace{3mm}}
\setbeamersize{text margin left=8mm,text margin right=8mm}
\setbeamerfont{frametitle}{size=\Large,series=\bfseries}
\newcommand{\papersource}[1]{\par\smallskip{\scriptsize\color{gray}#1\par}}
\newcommand{\highlight}[1]{{\color{AcademicTeal}\bfseries #1}}

\begin{document}

\begin{frame}[plain]
\vspace{8mm}
{\LARGE\bfseries\color{AcademicBlue} PACE: Periodic and Channel-Aligned Encoding\\[2mm]
of Four-Bit Recurrent States\par}
\vspace{5mm}
{\large Paper by Patrick Haller, Filipe Laitenberger, and Alan Akbik\par}
\vspace{3mm}
{\normalsize Research group paper reading \quad 27 October 2026\par}
\vfill
{\small \href{https://arxiv.org/abs/2610.29545v1}{arXiv:2610.29545v1} (21 Oct 2026)\par}
{\small Publication record: not found as of 27 Oct 2026.\par}
{\small Public official code: not found as of 27 Oct 2026.\par}
\end{frame}

\begin{frame}{Four bits become viable when rounding follows the kernel}
\begin{block}{One-sentence takeaway}
PACE rounds a recurrent state once per short decode buffer, groups scales within
value channels, and reserves 8 bits for error-persistent heads.
\end{block}
\begin{itemize}
  \item \highlight{Temporal}: fewer state writes mean fewer injected rounding errors.
  \item \highlight{Spatial}: key-axis groups avoid mixing large and small value columns.
  \item \highlight{Selective}: 8-bit heads protect the errors that decay slowly.
\end{itemize}
\papersource{Paper, Abstract and Sec.~1, PDF pp.~1--2; Fig.~2, PDF p.~3.}
\end{frame}

\begin{frame}{Concurrent decoding makes fixed-size state expensive}
\begin{itemize}
  \item Each gated-delta head stores a matrix \(S_t\in\mathbb R^{d_k\times d_v}\)
        for every active sequence.
  \item At 64 concurrent sequences, fp32 states account for 10--41\% of
        weights-plus-states memory on the evaluated models.
  \item Four-bit storage creates room for more sequences, provided quality survives
        repeated state updates.
\end{itemize}
\vspace{3mm}
\begin{block}{The failure mode}
Per-token decode materializes and rounds the \emph{same} recurrent state
after every generated token.
\end{block}
\papersource{Paper, Sec.~1, PDF p.~2. The memory share is the authors' 64-sequence estimate.}
\end{frame}

\begin{frame}{The gated-delta update exposes two useful structures}
\[
S_t=a_t(I-\beta_t k_tk_t^\top)S_{t-1}+\beta_t k_tv_t^\top
    =a_tS_{t-1}+k_tu_t^\top
\]
\[
u_t=\beta_t(v_t-a_tS_{t-1}^\top k_t),\qquad o_t=S_t^\top q_t .
\]
\begin{itemize}
  \item \(a_t\in(0,1)\): decay; \(\beta_t\): write strength.
  \item \(k_t\in\mathbb R^{d_k}\), \(u_t\in\mathbb R^{d_v}\): an outer-product update.
  \item A persistent large coordinate of \(u_t\) repeatedly writes one
        \highlight{value column} of \(S_t\).
\end{itemize}
\papersource{Paper, Sec.~2.1, Eqs.~(2)--(3), PDF p.~3.}
\end{frame}

\begin{frame}{Prefill already rounds once per chunk; decode can too}
\centering
\includegraphics[width=.96\linewidth,height=.53\textheight,keepaspectratio]{figures/timeline.pdf}
\par
\vspace{1mm}
\begin{itemize}
  \item Prefill period \(P=256\) tokens in every benchmark evaluation.
  \item Unbuffered decode: \(dP=1\); PACE headline configuration: \(dP=8\).
\end{itemize}
\papersource{Original paper Fig.~2a, PDF p.~3; Sec.~3.1, PDF p.~5.}
\end{frame}

\begin{frame}{Longer periods let old rounding errors decay}
\[
  E_h\approx \frac{e_h}{1-a_h^{2dP}}
\]
\begin{itemize}
  \item \(e_h\): energy injected by one rounding in head \(h\).
  \item \(a_h\): representative head decay; \(dP\): decode rounding interval.
  \item Fewer injections and more decay between them reduce retained error.
  \item A head with \(a_h\) near 1 holds errors longer and merits more precision.
\end{itemize}
\begin{block}{Reading the expression}
It uses independent rounding errors and a mean-decay proxy for
token-varying \(a_t\); it is the paper's ranking intuition.
\end{block}
\papersource{Paper, Sec.~3.2, Eq.~(5), PDF p.~5.}
\end{frame}

\begin{frame}{A buffer advances the packed state only at flushes}
\centering
\includegraphics[width=.96\linewidth,height=.70\textheight,keepaspectratio]{figures/pipeline.pdf}
\par
\papersource{Original paper Fig.~2b, PDF p.~3. A: period; B: buffer; C: scale groups; D: head selection.}
\end{frame}

\begin{frame}{The memory metric includes the decode buffer}
\[
\underbrace{b+\frac{16}{g}}_{\text{symmetric state, bits/value}}
\quad +\quad
\underbrace{dP\,
\frac{b_kd_k/\gamma+b_vd_v+32}{d_kd_v}}_{\text{buffer, bits/value}}
\]
\begin{itemize}
  \item \(b\): state bits; \(g\): group width; \(b_k,b_v\): buffered key/value bits.
  \item \(\gamma\): value heads sharing one key head.
  \item Uniform INT4, \(g=32\), int8 buffer, \(dP=8\):
        \(4.50+0.78=\boxed{5.28}\) resident bits/value.
\end{itemize}
\papersource{Paper, Sec.~5.1, Eq.~(6), PDF p.~6; Appendix A, PDF p.~15.}
\end{frame}

\begin{frame}{Large state entries form value columns}
\centering
\includegraphics[width=.96\linewidth,height=.57\textheight,keepaspectratio]{figures/axis.pdf}
\par
\begin{itemize}
  \item Dashed value-axis groups cross columns and have larger max/RMS.
  \item Solid key-axis groups stay within one column and use finer steps
        for typical entries.
\end{itemize}
\papersource{Original paper Fig.~3, PDF p.~4; Sec.~4.1, PDF p.~6.}
\end{frame}

\begin{frame}{Permuting the delta value removes the column advantage}
\centering
\begin{tabular}{lrrrr}
\toprule
Value/key max/RMS, \(g=128\) & 4B & 9B & 27B & 35B-A3B\\
\midrule
Original state & 1.588 & 1.555 & 1.601 & 1.871\\
\(u_t\) channels permuted/token & 0.776 & 0.760 & 0.808 & 0.963\\
\(k_t\) entries permuted/token & 2.265 & 2.220 & 2.261 & 2.580\\
\bottomrule
\end{tabular}
\par
\vspace{5mm}
\begin{itemize}
  \item A ratio \(>1\) means value-axis groups have the worse tail statistic.
  \item Permuting \(u_t\), while preserving its values, reverses that ranking.
\end{itemize}
\papersource{Paper, Sec.~4.2, PDF p.~6; Appendix C, Table~4, PDF p.~16.}
\end{frame}

\begin{frame}{PACE chooses precision at two granularities}
\begin{columns}[T]
\begin{column}{.49\textwidth}
\begin{block}{Within each head}
Group \(g\) adjacent entries along the key axis, all in one value column.
Round to nearest with a shared fp16 scale.
\end{block}
\end{column}
\begin{column}{.49\textwidth}
\begin{block}{Across heads}
Estimate retained error \(E_h\) on eight held-out 2,048-token windows;
keep the highest-ranked quarter of heads per layer at 8 bits.
\end{block}
\end{column}
\end{columns}
\vspace{5mm}
The head ranking is computed once per checkpoint; model weights are not fine-tuned.
\papersource{Paper, Secs.~5.2--5.3, PDF p.~7.}
\end{frame}

\begin{frame}{The two PACE variants spend different memory}
\centering
\begin{tabular}{lcc}
\toprule
 & PACE-4 & PACE-4/8\\
\midrule
State heads & all 4 bit & 75\% 4 bit; 25\% 8 bit\\
Key-axis group \(g\) & 64 & 32\\
State grid & symmetric & affine, integer zero point\\
Decode buffer \(k/v\) & int8 / int8 & int4 / bf16\\
Resident bits/value & \highlight{5.03} & \highlight{6.82}\\
\bottomrule
\end{tabular}
\par
\vspace{5mm}
For PACE-4/8, the mixed state payload is 5.66 bits/value;
the rest is buffer cost.
\papersource{Paper, Sec.~5.3, PDF p.~7; Appendix E, PDF p.~17; Table~1, PDF p.~9.}
\end{frame}

\begin{frame}{Evaluation: four hybrids and five headline tasks}
\begin{itemize}
  \item Qwen3.5-4B, 9B, 27B; Qwen3.6-35B-A3B.
  \item GSM8K and MATH-500: exact match; MMLU-Pro: 560-question subset.
  \item IFEval: 541 prompts, strict accuracy; RULER-32k:
        variable-tracking accuracy.
  \item Headline \texttt{avg}: unweighted mean of \emph{all five} scores.
        All tasks are zero-shot with greedy decoding and thinking disabled.
\end{itemize}
\begin{block}{Baseline detail}
DAMP INT8 rounds every token at 9.90 bits/value.
The authors reimplement Q-Mamba DSQ without its published fine-tuning step.
\end{block}
\papersource{Paper, Sec.~6, PDF p.~8; Table~1, PDF p.~9.}
\end{frame}

\begin{frame}{Eight-token rounding improves the uniform INT4 tradeoff}
\centering
\begin{tabular}{llrrr}
\toprule
Model & Decode period & 3-task avg & Resident bits & IFEval\\
\midrule
9B & \(dP=1\) & 78.1 & 4.60 & 61.4\\
9B & \(dP=8\) & 82.9 & 5.28 & 71.3\\
9B & \(dP=32\) & 85.6 & 7.64 & 76.9\\
\midrule
35B-A3B & \(dP=1\) & 70.7 & 4.60 & 54.7\\
35B-A3B & \(dP=8\) & 77.3 & 5.28 & 66.2\\
35B-A3B & \(dP=32\) & 78.4 & 7.64 & 68.8\\
\bottomrule
\end{tabular}
\par
\vspace{3mm}
Three-task average = GSM8K, IFEval, RULER-32k;
prefill remains \(P=256\).
\papersource{Paper, Sec.~7.1, PDF p.~8; Appendix A, Table~3, PDF p.~15.}
\end{frame}

\begin{frame}{A longer buffer has a visible memory cost}
\centering
\includegraphics[width=.88\linewidth,height=.54\textheight,keepaspectratio]{figures/period_cost.pdf}
\par
\vspace{1mm}
\begin{itemize}
  \item The buffer term grows linearly with \(dP\).
  \item At INT4, \(dP=8\) recovers much of the \(dP=32\) gain
        with one quarter of its buffer length.
\end{itemize}
\papersource{Original paper Fig.~4, right panel, PDF p.~7; Appendix A, Table~3, PDF p.~15.}
\end{frame}

\begin{frame}{PACE-4/8 approaches DAMP INT8 at 69\% of its bits}
\centering
\begin{tabular}{lrrrr}
\toprule
Five-task average & 4B & 9B & 27B & 35B-A3B\\
\midrule
DAMP INT8, 9.90 bits & 80.9 & 83.7 & 87.4 & 86.3\\
PACE-4/8, 6.82 bits & 79.1 & 83.7 & 87.3 & 85.0\\
PACE-4, 5.03 bits & 79.6 & 80.0 & 84.2 & 77.3\\
Q-Mamba DSQ, 5.03 bits & 75.2 & 72.8 & 78.9 & 71.2\\
\bottomrule
\end{tabular}
\par
\vspace{4mm}
\begin{itemize}
  \item PACE-4 beats the post-hoc Q-Mamba DSQ comparison at equal bits.
  \item The 4B PACE-4/8 average is 1.8 points below DAMP INT8.
\end{itemize}
\papersource{Paper, Table~1, PDF p.~9. Average covers all five headline tasks.}
\end{frame}

\begin{frame}{The average hides an IFEval gap}
\centering
\begin{tabular}{lrrr}
\toprule
IFEval strict accuracy & 9B & 27B & 35B-A3B\\
\midrule
DAMP INT8 & 83.4 & 88.0 & 83.0\\
PACE-4/8 & 81.3 & 86.1 & 81.0\\
Difference, percentage points & -2.1 & -1.9 & -2.0\\
\bottomrule
\end{tabular}
\par
\vspace{6mm}
PACE-4/8's headline parity is about the \emph{five-task mean};
it does not imply parity on every task.
\papersource{Paper, Table~1 and Sec.~7.2, PDF p.~9.}
\end{frame}

\begin{frame}{Key-axis grouping supplies a large ablation gain}
\centering
\begin{tabular}{lrrr}
\toprule
INT4, \(g=32\), 5.28 bits & IFEval 4B & IFEval 27B & PPL 4B\\
\midrule
Value-axis groups & 73.75 & 75.05 & 8.158\\
Key-axis groups & \highlight{80.59} & \highlight{78.37} & \highlight{7.719}\\
\bottomrule
\end{tabular}
\par
\vspace{6mm}
\begin{itemize}
  \item IFEval: higher is better; WikiText-103 perplexity: lower is better.
  \item At equal resident bits, grouping within columns helps on both models.
\end{itemize}
\papersource{Paper, Table~2, PDF pp.~9--10.}
\end{frame}

\begin{frame}{Head allocation and buffer precision remain task-sensitive}
\begin{itemize}
  \item At 6.44 bits, head allocation plus key-axis groups improves
        4B IFEval versus DAMP INT4: 81.52 vs.\ 79.48.
  \item On 27B IFEval the same ablation is lower: 78.00 vs.\ 79.85.
  \item With the PACE-4/8 state, 9B IFEval is 82.4 for
        \(\text{int4 key}/\text{bf16 value}\), but only 22.7 when
        the buffer widths are reversed.
\end{itemize}
\begin{block}{Interpretation}
Buffered values enter the state write directly; key errors are partly
filtered by L2 normalization after key quantization.
\end{block}
\papersource{Paper, Table~2, PDF p.~10; Appendix D, Table~5, PDF pp.~16--17.}
\end{frame}

\begin{frame}{State drift falls as the rounding period grows}
\centering
\includegraphics[width=.96\linewidth,height=.68\textheight,keepaspectratio]{figures/drift.pdf}
\par
\papersource{Original paper Fig.~5, PDF p.~10. Drift = \(\|S_q-S\|_F/\|S\|_F\), median over recurrent layers;
35B-A3B corroboration in Appendix B, Fig.~7, PDF p.~16.}
\end{frame}

\begin{frame}{Periodic INT4 keeps the decode-buffer speedup}
\centering
\includegraphics[width=.96\linewidth,height=.65\textheight,keepaspectratio]{figures/throughput.pdf}
\par
\vspace{1mm}
At batch 128, TPOT is 18.6\% lower than per-token bf16;
at batch 1, it is 1.4\% higher.
\papersource{Original paper Fig.~6, PDF p.~11. Qwen3.5-4B, H100 PCIe, 512-token context;
uniform INT4 state and int8 buffer, not full PACE-4/8.}
\end{frame}

\begin{frame}{What the evidence supports}
\begin{enumerate}
  \item \highlight{Less frequent rounding}: improved low-bit scores and
        lower state drift across tested Qwen hybrids.
  \item \highlight{Channel-aligned groups}: the delta-value permutation
        and grouping ablation explain a distinct four-bit gain.
  \item \highlight{Practical tradeoff}: PACE-4/8 saves resident bits with
        near-DAMP-INT8 mean scores on the three larger backbones.
\end{enumerate}
\vspace{3mm}
\begin{block}{Current boundary}
Only Qwen gated-delta hybrids were studied; prefill period is fixed at 256.
Throughput times uniform INT4, without PACE-4/8's grouping and head allocation.
\end{block}
\papersource{Paper, Sec.~8 and Limitations, PDF pp.~11--12.}
\end{frame}

\begin{frame}{Source and discussion pointers}
\begin{itemize}
  \item Haller, Laitenberger, Akbik.
        \href{https://arxiv.org/abs/2610.29545v1}
        {PACE: Periodic and Channel-Aligned Encoding of Four-Bit Recurrent States},
        arXiv:2610.29545v1, 2026.
  \item Main results: Table 1, PDF p.~9; mechanisms: Eqs.~(5)--(6)
        and Figs.~2--3, PDF pp.~3--6.
  \item Current publication and code status:
        no verifiable formal venue or public official implementation found
        as of 27 October 2026.
\end{itemize}
\vfill
\begin{block}{Question for follow-up}
Does the value-column structure persist in other recurrent model families,
and which prefill period best balances long-context accuracy with cost?
\end{block}
\end{frame}

\end{document}