% Self-authored reading presentation; adapted from the arxiv-paper-reader template.
% Main discussion: frames 1--22, about 25 minutes. Frames 23--29 are backup.
% Compile: python3 build_slides.py slides.en.tex
\documentclass[aspectratio=169,11pt]{beamer}
\usepackage[T1]{fontenc}
\usepackage[utf8]{inputenc}
\usepackage{lmodern}
\usepackage{amsmath,amssymb,graphicx,booktabs}

\definecolor{AcademicBlue}{RGB}{26,57,86}
\definecolor{AcademicTeal}{RGB}{0,105,112}
\definecolor{WarmRed}{RGB}{155,54,42}
\setbeamercolor{structure}{fg=AcademicBlue}
\setbeamercolor{frametitle}{fg=AcademicBlue,bg=white}
\setbeamercolor{title}{fg=AcademicBlue}
\setbeamercolor{alerted text}{fg=AcademicTeal}
\setbeamercolor{block title}{fg=white,bg=AcademicBlue}
\setbeamercolor{block body}{fg=black,bg=AcademicBlue!5}
\setbeamertemplate{navigation symbols}{}
\setbeamertemplate{itemize items}[circle]
\setbeamertemplate{footline}{%
  \hfill\usebeamerfont{page number in head/foot}%
  \insertframenumber\,/\,\inserttotalframenumber\hspace{4mm}\vspace{3mm}}
\setbeamersize{text margin left=8mm,text margin right=8mm}
\setbeamerfont{frametitle}{size=\Large,series=\bfseries}
\setbeamerfont{title}{size=\LARGE,series=\bfseries}
\setlength{\tabcolsep}{5pt}
\renewcommand{\arraystretch}{1.18}
\newcommand{\papersource}[1]{\vfill\par{\scriptsize\color{gray}#1\par}}
\newcommand{\cQ}{\mathcal Q}
\newcommand{\R}{\mathbb R}
\newcommand{\F}{\mathbb F}
\title[Associative Algebra Layers]{Change the Product,\newline Keep the Parameters}
\subtitle{Associative Algebra Layers for Transformers}
\author{Paper authors: Ilya Koziev and Ivan Oseledets\\[2mm]
\small Independent paper-reading presentation}
\date{arXiv:2609.32814v1 \quad $\cdot$ \quad 19 October 2026}

\begin{document}

\begin{frame}
  \titlepage
  \vspace{-4mm}
  \begin{center}
    \small A cheaper interaction rule retains the weight bank,\newline
    with a measured quality--throughput trade-off in a small LM.
  \end{center}
\end{frame}

\begin{frame}{The multiplication rule is an architecture choice}
  \begin{columns}[T,onlytextwidth]
    \begin{column}{0.47\textwidth}
      \begin{block}{Fast matrix multiplication}
        Keep the target $XW$ fixed.\par\medskip
        Find a cheaper exact algorithm.\par\medskip
        Example: Strassen uses 7 products for a $2\times2$ block product.
      \end{block}
    \end{column}
    \begin{column}{0.49\textwidth}
      \begin{block}{This paper}
        Learn with $Y=\mu_{\mathcal A}(X,W)$.\par\medskip
        Choose a cheaper interaction table.\par\medskip
        Keep all learned weight slots, while changing which products occur.
      \end{block}
    \end{column}
  \end{columns}
  \medskip
  \alert{Question: how much computation can be removed before quality suffers?}
  \papersource{Paper, Secs. 1--2, PDF pp. 1--3.}
\end{frame}

\begin{frame}{Two groups retain four weights with six products}
  \[
    X=\begin{bmatrix}A&U\\V&D\end{bmatrix},\qquad
    W=\begin{bmatrix}W_a&W_u\\W_v&W_d\end{bmatrix}
  \]
  \[
    X\star_2 W=
    \begin{bmatrix}
      AW_a&AW_u+UW_d\\
      VW_a+DW_v&DW_d
    \end{bmatrix}
  \]
  \begin{itemize}
    \item Ordinary multiplication also has $UW_v$ and $VW_u$ on the diagonal.
    \item The new table removes these two interactions; every weight block remains.
    \item One token evaluates one row type: 3 block products instead of 4.
  \end{itemize}
  \papersource{Paper, Eqs. (1)--(3), PDF pp. 1--2. Blocks denote compatible ordinary GEMMs.}
\end{frame}

\begin{frame}{The graph product attains its bilinear-rank bound}
  A vertex is a diagonal slot; an edge is an off-diagonal slot.
  \[
    e_i e_i=e_i,\qquad e_i a_{ij}=a_{ij},\qquad a_{ij}e_j=a_{ij}.
  \]
  All other basis products vanish, including every edge--edge product.
  \begin{block}{Upper and lower bounds meet}
    \[
      R(\cQ_G)\ge 2\dim\cQ_G-t(\cQ_G)
      =2(|V|+|E|)-|V|=|V|+2|E|.
    \]
    The explicit rule uses exactly $|V|+2|E|$ products.
  \end{block}
  For the complete directed graph:
  \[
    \dim\cQ_q=q^2,\qquad \boxed{R(\cQ_q)=2q^2-q}.
  \]
  \papersource{Paper, Secs. 2.2--3, Theorem 3.2, PDF pp. 3--4; proof in App. H.1, pp. 21--22.
  $R$: scalar bilinear rank; $t$: number of maximal two-sided ideals.}
\end{frame}

\begin{frame}{A token selects a row of the interaction table}
  \[
    X\in\F^{M\times K},\quad W\in\F^{K\times N},\quad
    W_{jk}\in\F^{s_j\times r_k},\quad i=\tau(p).
  \]
  \begin{block}{Type-$i$ projection}
    \[
      y_i=x_iW_{ii},\qquad y_j=x_iW_{ij}+x_jW_{jj}\quad(j\ne i).
    \]
  \end{block}
  \begin{itemize}
    \item Uses every diagonal block and the off-diagonal blocks in source row $i$.
    \item The full bank has $KN$ learned scalars, covered jointly across types.
    \item $\tau(p)$ is a fixed position schedule; each projection is token-local.
  \end{itemize}
  \papersource{Paper, Sec. 4, Eqs. (16)--(21), PDF pp. 4--5. $M$: token rows; $K,N$: feature widths.}
\end{frame}

\begin{frame}{Full matrix rank can coexist with restricted interactions}
  \begin{columns}[c,onlytextwidth]
    \begin{column}{0.43\textwidth}
      Teaching example, type 1:
      \[
        (y_1,y_2)=(x_1,x_2)
        \underbrace{\begin{bmatrix}
          W_{11}&W_{12}\\0&W_{22}
        \end{bmatrix}}_{T_1(W)}.
      \]
    \end{column}
    \begin{column}{0.52\textwidth}
      \begin{itemize}
        \item Invertible diagonal blocks make $T_1(W)$ full rank.
        \item Yet $x_2$ cannot directly affect $y_1$.
        \item Another row type exposes $W_{21}$.
      \end{itemize}
    \end{column}
  \end{columns}
  \bigskip
  \begin{block}{What associativity supplies}
    Compatible linear maps compose within the same family:
    \[
      \Phi_{W_2}\circ\Phi_{W_1}=\Phi_{W_1\star W_2}.
    \]
    Equal parameter storage still allows a different function class.
  \end{block}
  \papersource{Teaching representation derived from Eqs. (16)--(17). Paper, Secs. 2.1 and 4, PDF pp. 3--5.}
\end{frame}

\begin{frame}{The trained recursive law differs from direct four-group algebra}
  \begin{center}
  \begin{tabular}{lccc}
    \toprule
    Law & Slots & Displayed products & Projection fraction\\
    \midrule
    $\cQ_2$ & 4 & 6 & $3/4$\\
    $\cQ_4$ & 16 & 28 & $7/16$\\
    $\cQ_2^{\otimes2}$ & 16 & 36 & $9/16$\\
    \bottomrule
  \end{tabular}
  \end{center}
  \begin{itemize}
    \item $\cQ_4$ has exact rank 28; the recursive law has rank in $[28,36]$.
    \item Recursion retains some interactions through two intermediate steps.
    \item The LM pilot uses $\cQ_2^{\otimes2}$: \alert{43.75\% fewer FFN MACs}.
  \end{itemize}
  \papersource{Paper, Sec. 3.2, PDF p. 4; App. C, pp. 12--13. Product counts refer to the full block table.}
\end{frame}

\begin{frame}{Quadratic arithmetic requires a growing number of groups}
  Equal-width rectangular projections:
  \[
    C_{\rm proj}=\rho_qMKN,\qquad \rho_q=\frac{2q-1}{q^2}.
  \]
  For square operands, $n=qb$ with physical block size $b$:
  \[
    C(n,q)=(2q^2-q)(n/q)^3.
  \]
  \begin{columns}[T,onlytextwidth]
    \begin{column}{0.46\textwidth}
      \begin{block}{Fixed $q$}
        $C(n,q)=\Theta(n^3)$\par\medskip
        A constant-factor reduction.
      \end{block}
    \end{column}
    \begin{column}{0.50\textwidth}
      \begin{block}{Fixed $b$, $q=n/b$}
        $C=2bn^2-b^2n=\Theta(n^2)$\par\medskip
        The interaction law changes with $n$.
      \end{block}
    \end{column}
  \end{columns}
  \papersource{Paper, Sec. 5.1, Eqs. (22)--(24), PDF p. 5; App. H.2, p. 22. Counts are MACs; storage remains $n^2$.}
\end{frame}

\begin{frame}{GPU efficiency determines where subdivision should stop}
  Type-specific GEMMs have approximate dimensions
  \[
    (M/q)\times(K/q)\times(N/q).
  \]
  \begin{itemize}
    \item Small blocks reduce arithmetic but may lose utilization or waste padded tiles.
    \item Fused endpoint accumulation and a two-stage schedule offer different reuse.
    \item One-token decoding uses a dedicated vector-reduction kernel.
  \end{itemize}
  \begin{block}{Compute-dominated crossover model}
    \[
      \frac{\rho_q}{\eta_q}+\delta_q<1,
      \qquad \eta_q=\frac{P_a(q)}{P_d}.
    \]
    $\eta_q$: relative effective compute rate; $\delta_q$: overhead / dense time.
  \end{block}
  \papersource{Paper, Secs. 5.2--5.3, Eqs. (25)--(26), PDF pp. 5--6.
  A deployed architecture fixes $q$; kernel tiles and schedules may adapt.}
\end{frame}

\begin{frame}{The same row action supports training and cached decoding}
  \begin{columns}[T,onlytextwidth]
    \begin{column}{0.48\textwidth}
      \begin{block}{Parallel training}
        Group token rows by $\tau(p)$.\par\medskip
        Evaluate permitted rectangular GEMMs.\par\medskip
        $dX$ and $dW$ reuse the same support:
        \[C_{\rm train}=3C_{\rm fwd}.\]
      \end{block}
    \end{column}
    \begin{column}{0.48\textwidth}
      \begin{block}{Autoregressive decoding}
        Compute only the new token row.\par\medskip
        Preserve the absolute-position type.\par\medskip
        Standard triangular attention and full KV cache remain valid.
      \end{block}
    \end{column}
  \end{columns}
  \medskip
  \alert{Causality follows from token-local projections and the temporal mask.}
  \papersource{Paper, Sec. 6, Algorithm 1, PDF pp. 6--7; App. B, p. 12; App. H.3, p. 22.
  The three-pass count covers matrix-product MACs.}
\end{frame}

\begin{frame}{The evidence comes from three separate settings}
  \begin{center}
  \begin{tabular}{p{2.4cm}p{4.5cm}p{5.3cm}}
    \toprule
    Setting & Object & What it measures\\
    \midrule
    Theory & Direct $\cQ_q$ family & Exact rank and arithmetic scaling\\[1mm]
    H200 kernels & Synthetic BF16 inputs; public layer dimensions & Projection, FFN, and gradient latency\\[1mm]
    Small LM & Recursive $\cQ_2^{\otimes2}$; 110.6M parameters & Generation throughput and task quality\\
    \bottomrule
  \end{tabular}
  \end{center}
  \medskip
  \begin{itemize}
    \item Kernel calibration is separated from alternating timing trials.
    \item No Qwen or DeepSeek model weights are loaded in the shape benchmarks.
    \item Large-$q$ kernel results have no matching trained-model quality study.
  \end{itemize}
  \papersource{Paper, Sec. 7, PDF pp. 7--8; Apps. E--F, pp. 15--16.}
\end{frame}

\begin{frame}{Pure projections accelerate across several large shapes}
  \begin{center}
  \begin{tabular}{lrrrr}
    \toprule
    Shape & $M$ & Selected $q$ & Warm & Flushed\\
    \midrule
    Qwen3-1.7B & 2048 & 32 & $3.04\times$ & $2.84\times$\\
    Qwen3-4B & 2048 & 32 & $3.14\times$ & $2.94\times$\\
    Qwen3-8B & 2048 & 32 & $5.04\times$ & $4.52\times$\\
    Qwen3-30B expert & 2048 & 16 & $1.57\times$ & $1.30\times$\\
    DeepSeek-V3 expert & 2048 & 16 & $2.91\times$ & $2.47\times$\\
    Qwen3-32B & 32768 & 256 & $11.51\times$ & $11.70\times$\\
    \bottomrule
  \end{tabular}
  \end{center}
  \medskip
  BF16, H200 NVL; speedup = dense kernel time / algebraic kernel time.\par
  These timings isolate projections; MoE dispatch and communication are excluded.
  \papersource{Selected rows of paper Table 1, PDF p. 8. $q$ minimizes warm latency in the tested grid;
  256 is the largest tested value for Qwen3-32B.}
\end{frame}

\begin{frame}{The best group count depends on the number of token rows}
  \begin{columns}[c,onlytextwidth]
    \begin{column}{0.70\textwidth}
      \includegraphics[width=\linewidth,height=5.2cm,keepaspectratio]{figures/q-sweep.pdf}
    \end{column}
    \begin{column}{0.27\textwidth}
      $5120\to25600$\par\smallskip
      Qwen3-32B shape\par\bigskip
      Best tested $q$:\par\smallskip
      $M=2048$: 32\par
      $M=8192$: 128\par
      $M=32768$: 256\par\bigskip
      More groups can increase latency.
    \end{column}
  \end{columns}
  \papersource{Redrawn from paper Table 12, PDF p. 17. H200 NVL, BF16, warm medians; logarithmic latency axis.}
\end{frame}

\begin{frame}{Forward gains can disappear in the gradient kernels}
  \begin{center}
  \begin{tabular}{lrrr}
    \toprule
    Shape & Full FFN forward & Up: fwd+$dX$+$dW$ & Down: fwd+$dX$+$dW$\\
    \midrule
    Qwen3-1.7B & $2.55\times$ & $1.27\times$ & $1.76\times$\\
    Qwen3-8B & $4.17\times$ & $1.96\times$ & $2.56\times$\\
    Qwen3-30B expert & $1.74\times$ & \textcolor{WarmRed}{$0.65\times$} & \textcolor{WarmRed}{$0.57\times$}\\
    DeepSeek-V3 expert & $3.74\times$ & $1.79\times$ & $1.35\times$\\
    \bottomrule
  \end{tabular}
  \end{center}
  \medskip
  \begin{itemize}
    \item All rows: $M=2048$, $q=32$, warm replay on H200 NVL.
    \item The small expert's down projection rises from 38.5 to 67.6 $\mu$s.
    \item Only forward schedules are calibrated; gradient tiles are fixed.
  \end{itemize}
  \papersource{Selected rows of paper Tables 9--10, PDF p. 16; protocol in App. E, p. 15.
  Full FFN and individual projection training are different workloads.}
\end{frame}

\begin{frame}{Fixed leaf blocks show near-quadratic latency over four sizes}
  \begin{columns}[c,onlytextwidth]
    \begin{column}{0.70\textwidth}
      \includegraphics[width=\linewidth,height=5.2cm,keepaspectratio]{figures/square-scaling.pdf}
    \end{column}
    \begin{column}{0.27\textwidth}
      $q=n/b$ grows.\par\bigskip
      For $b=128$:\par
      endpoint slope $\approx2.01$.\par\bigskip
      At $n=32768$:\par
      $39.85\times$ warm speedup.\par\bigskip
      Synthetic products; model quality is untested here.
    \end{column}
  \end{columns}
  \papersource{Redrawn from paper Table 13, PDF p. 18; discussion in App. F, p. 16.
  Both axes logarithmic. Ratios use the paper's reported values.}
\end{frame}

\begin{frame}{The LM pilot changes only the FFN multiplication law}
  \begin{columns}[T,onlytextwidth]
    \begin{column}{0.48\textwidth}
      \begin{block}{Shared architecture}
        110,631,168 parameters\par
        12 pre-norm decoder blocks\par
        $D=768$, $F=1536$, 32 heads\par
        Bias-free SwiGLU\par
        Context: 1536 tokens\par
        GPT-2 vocabulary: 50,257
      \end{block}
    \end{column}
    \begin{column}{0.48\textwidth}
      \begin{block}{Shared training recipe}
        One run per model, from scratch\par
        12.288B token slots each\par
        100,000 steps on 2 H100s\par
        AdamW, peak LR $3\times10^{-4}$\par
        Same public instruction mixture\par
        BF16 computation; FP32 parameters
      \end{block}
    \end{column}
  \end{columns}
  \medskip
  Dense attention, embeddings, and LM head in both models.\par
  Algebraic FFN: $\cQ_2^{\otimes2}$, projection MAC fraction $9/16$.
  \papersource{Paper, Sec. 7.2, PDF pp. 7--8; Table 4 and App. D.1, p. 13.}
\end{frame}

\begin{frame}{Both models learn, while the algebraic loss stays higher}
  \centering
  \includegraphics[width=0.96\linewidth,height=5.65cm,keepaspectratio]{figures/fig-01-eval-loss.png}
  \par\smallskip
  Both losses decrease; the algebraic curve remains higher.
  \papersource{Paper Fig. 1, PDF p. 13. Original author image; no exact endpoint loss values are tabulated.}
\end{frame}

\begin{frame}{Realized generation throughput improves by about 7\%}
  \begin{center}
  \begin{tabular}{lrrr}
    \toprule
    Domain & Dense tok/s & Algebraic tok/s & Relative gain\\
    \midrule
    Code & 987.8 & 1054.5 & 6.8\%\\
    Math & 991.4 & 1060.6 & 7.0\%\\
    QA & 958.6 & 1033.1 & 7.8\%\\
    WMT ru--en & 985.9 & 1047.1 & 6.2\%\\
    \bottomrule
  \end{tabular}
  \end{center}
  \begin{itemize}
    \item 100 prompts/domain; three retained runs after one warm-up.
    \item Includes tokenization and sampling; both use CUDA-graph decoding.
    \item Response lengths differ: Code 195.5 vs. 182.4 tokens; QA 92.6 vs. 110.8.
  \end{itemize}
  \papersource{Paper Table 2, PDF p. 8; Tables 5--7 and App. D.2, pp. 14--15.
  Tabulated means imply a 6.93\% average gain. Fixed-length timings would isolate the layer effect.}
\end{frame}

\begin{frame}{All three downstream scores fall in the algebraic model}
  \begin{center}
  \begin{tabular}{lrrr}
    \toprule
    Metric (percent; higher is better) & Dense & Algebraic & Gap (pp)\\
    \midrule
    GSM8K flexible EM & $3.18\pm0.48$ & $2.20\pm0.40$ & $-0.98$\\
    MBPP pass@1 & $6.60\pm1.11$ & $3.80\pm0.86$ & $-2.80$\\
    IFEval loose accuracy & 34.53 & 31.06 & $-3.47$\\
    \bottomrule
  \end{tabular}
  \end{center}
  \smallskip
  $\pm$ denotes evaluation standard error, not variation across training runs.
  \begin{block}{A metric-label ambiguity in v1}
    Main Table 3 says \emph{instance-level}; Appendix Table 8 says
    \emph{instruction-level}. Values above follow the appendix.\par
    The exact IFEval aggregation key remains unclear from the paper.
  \end{block}
  \papersource{Paper Table 3, PDF p. 9; Table 8, p. 15. GSM8K: 5-shot; MBPP/IFEval: zero-shot.
  Gaps use rounded appendix percentages.}
\end{frame}

\begin{frame}{FFN arithmetic savings are diluted by the rest of the model}
  With dense attention and algebraic FFN projections:
  \[
    C_{\rm dec}=\underbrace{2DH_q+2DH_{kv}}_{\text{attention projections}}
    +\underbrace{\rho(g+1)DF}_{\text{FFN}}
    +\underbrace{2NH_q}_{\text{attention contractions}}.
  \]
  Pilot: $H_q=H_{kv}=D$, $F=2D$, $g=2$, $\rho=9/16$.
  \[
    \frac{\text{algebraic block projection MACs}}{\text{dense block projection MACs}}
    =\frac{4D^2+(9/16)6D^2}{10D^2}
    =\frac{59}{80}=73.75\%.
  \]
  \begin{itemize}
    \item FFN MAC reduction: 43.75\%; block projection reduction: 26.25\%.
    \item Attention contractions, LM head, memory traffic, and sampling add cost.
    \item Parameter storage and KV-cache size remain unchanged.
  \end{itemize}
  \papersource{Paper, Sec. 6.3, Eq. (29), PDF p. 7; Sec. 9, p. 9. $N$: cached positions; $H_q,H_{kv}$: head widths.}
\end{frame}

\begin{frame}{The contribution is a structured efficiency--quality trade-off}
  \begin{block}{Established by the paper}
    A rank-certified interaction family, a causal row implementation,
    and measurable kernel gains. A recursive small LM trains successfully
    and generates faster, with lower task scores.
  \end{block}
  \begin{block}{What changes the practical verdict}
    Large-$q$ quality, full-model training time, and equal-quality cost
    have not been established. The two LMs have one training run each.
  \end{block}
  \medskip
  \textbf{Discussion proposal:} compare direct and recursive four-group laws
  under one recipe, then measure fixed-length latency and cost at matched quality.
  \papersource{Paper, Sec. 9--10, PDF p. 9. Discussion proposal is reader analysis.}
\end{frame}

\begin{frame}{Paper and navigation}
  \textbf{Ilya Koziev and Ivan Oseledets}\par\smallskip
  \emph{Change the Product, Keep the Parameters:\newline
  Associative Algebra Layers for Transformers}\par\medskip
  arXiv:2609.32814v1, 26 September 2026\par
  \href{https://arxiv.org/abs/2609.32814v1}{https://arxiv.org/abs/2609.32814v1}
  \bigskip
  \begin{itemize}
    \item Core construction and costs: Secs. 2--6; proofs in Apps. A--C and H.
    \item Main evidence: Sec. 7; detailed results in Apps. D--F.
    \item Proposed attention extension: App. G; data description: App. I.
  \end{itemize}
  \papersource{All empirical data in this deck come from this version. PDF locators count from page 1.
  The next seven slides are optional technical backup.}
\end{frame}

\appendix

\begin{frame}{Backup: associativity follows from commuting endpoint actions}
  Write an element as diagonal slots $d$ and edge slots $n$:
  \[
    (d,n)\star(d',n')=(dd',\,dn'+nd').
  \]
  The diagonal component of either bracketing is $dd'd''$; the edge component is
  \[
    dd'n''+dn'd''+nd'd''.
  \]
  \begin{itemize}
    \item Left diagonal multiplication acts at the source vertex.
    \item Right diagonal multiplication acts at the target vertex.
    \item These actions commute; the identity is $(\mathbf 1,0)$.
  \end{itemize}
  \papersource{Paper, App. A, Eq. (30), PDF p. 12. For the graph proof and exact rank, see App. H.1, pp. 21--22.}
\end{frame}

\begin{frame}{Backup: backward uses the same interaction support}
  For a token of type $i$, with output gradient $\bar y_j$:
  \begin{align*}
    \bar x_i &\mathrel{+}=\bar y_iW_{ii}^{\top}
      +\sum_{j\ne i}\bar y_jW_{ij}^{\top},&
    \bar x_j&\mathrel{+}=\bar y_jW_{jj}^{\top}\quad(j\ne i),\\[2mm]
    \bar W_{ii}&\mathrel{+}=x_i^{\top}\bar y_i,&
    \bar W_{ij}&\mathrel{+}=x_i^{\top}\bar y_j\quad(j\ne i),\\[2mm]
    \bar W_{jj}&\mathrel{+}=x_j^{\top}\bar y_j\quad(j\ne i).
  \end{align*}
  Unequal-width exact count:
  \[
    C_i=\sum_j s_jr_j+s_i(N-r_i),\qquad C_{\rm train}=3\sum_i M_i C_i.
  \]
  \papersource{Paper, App. B, Eqs. (31)--(35), PDF p. 12; Sec. 4, Eqs. (18)--(20), p. 4.}
\end{frame}

\begin{frame}{Backup: recursive endpoints and the actual position schedule}
  Types and group indices are $L$-bit strings:
  \[
    y_\delta=\sum_{\beta:\ \beta_k\in\{\alpha_k,\delta_k\}}x_\beta W_{\beta\delta}.
  \]
  \begin{itemize}
    \item Hamming distance $h(\alpha,\delta)$ gives $2^h$ intermediate groups.
    \item At $L=2$: full-table count $4\cdot1+8\cdot2+4\cdot4=36$.
    \item Equal-width per-token fraction: $(3/4)^L$.
  \end{itemize}
  \begin{block}{Pilot schedule, zero-based positions}
    \[
      \tau(p)=\left\lfloor\frac{p\bmod768}{192}\right\rfloor.
    \]
    Consecutive 192-token intervals share a type; kernel microbenchmarks use cyclic types.
  \end{block}
  \papersource{Paper, App. C, Eqs. (36)--(37), PDF pp. 12--13; App. D.5, p. 15.}
\end{frame}

\begin{frame}{Backup: structured attention scores require support-aware scaling}
  This is a proposed extension; the LM pilot keeps attention dense.
  \[
    F(i,j)=\{i,j\},\quad d_h=qm,\quad
    s_{pu}=\frac{\sum_{a\in F(i,j)}\langle Q_p^{(a)},K_u^{(a)}\rangle}
                   {\sqrt{|F(i,j)|m}}.
  \]
  \begin{itemize}
    \item $|F(i,j)|=1$ for matching types; otherwise 2.
    \item Scaling assumes selected coordinates are initially independent, centered,
      and unit variance.
    \item Standard causal softmax and dense $PV$ follow the score calculation.
  \end{itemize}
  \papersource{Paper, App. G.1, Eqs. (38)--(44), PDF pp. 18--19.
  $i,j$ are the query/key position types; $m$ is the width of one feature group.}
\end{frame}

\begin{frame}{Backup: structured scores still require the full KV cache}
  A future query type may access a different group of the same stored key.
  \[
    K^{\rm cache},V^{\rm cache}\in\R^{N_p\times n_{kv}\times d_h},\qquad
    \text{storage}=2N_pH_{kv}.
  \]
  If the cache contains $n_i$ keys matching query type $i$:
  \[
    C_{QK}^{(i)}=\frac{H_q}{q}(2N_p-n_i),\qquad C_{PV}=N_pH_q.
  \]
  \begin{center}
  \begin{tabular}{lccc}
    \toprule
    Law & Score fraction & Forward ceiling & Training ceiling\\
    \midrule
    $\cQ_4$ & $7/16$ & $1.391\times$ & $1.474\times$\\
    $\cQ_2^{\otimes2}$ & $9/16$ & $1.280\times$ & $1.333\times$\\
    \bottomrule
  \end{tabular}
  \end{center}
  \papersource{Paper, App. G.2, Eqs. (52)--(61), PDF pp. 20--21; Table 14, p. 20.
  Arithmetic ceilings for attention contractions; dense $PV$ retained, with score recomputation in training.}
\end{frame}

\begin{frame}{Backup: generation and training use distinct protocols}
  \begin{columns}[T,onlytextwidth]
    \begin{column}{0.48\textwidth}
      \begin{block}{Training data}
        Public instruction mixture.\par\medskip
        Exact query--response deduplication.\par\medskip
        Benchmark near-duplicate checks via 3-character shingles; Jaccard $>0.80$ removed.\par\medskip
        Extra MBPP solution checks for code responses.
      \end{block}
    \end{column}
    \begin{column}{0.48\textwidth}
      \begin{block}{Evaluation}
        Throughput: temperature 0.8, top-$p$ 0.95, top-$k$ 50; up to 256 tokens.\par\medskip
        Quality: harness 0.4.9.1, batch 1, BF16; up to 512 tokens.\par\medskip
        Timing CIs use 3 retained runs; task SEs use evaluation examples.
      \end{block}
    \end{column}
  \end{columns}
  \papersource{Paper, Apps. D.2--D.4, PDF p. 14; App. I, pp. 23--24.
  The dataset manifest counts imported records, not consumed token proportions.}
\end{frame}

\begin{frame}{Backup: full-FFN gains vary with cache reuse}
  Fixed $q=32$, complete SwiGLU forward; speedups are warm / flushed.
  \begin{center}
  \begin{tabular}{lcc}
    \toprule
    Shape & $M=1$ & $M=2048$\\
    \midrule
    Qwen3-1.7B & $3.45\times/2.46\times$ & $2.55\times/2.45\times$\\
    Qwen3-4B & $4.74\times/3.21\times$ & $2.78\times/2.68\times$\\
    Qwen3-8B & $6.64\times/4.12\times$ & $4.17\times/3.92\times$\\
    Qwen3-30B-A3B expert & $1.71\times/1.42\times$ & $1.74\times/1.54\times$\\
    DeepSeek-V3 expert & $3.94\times/2.75\times$ & $3.74\times/3.40\times$\\
    \bottomrule
  \end{tabular}
  \end{center}
  \begin{itemize}
    \item Both methods merge gate/up and use the same activation kernel.
    \item At $M=1$, warm graph replay repeatedly uses one row type.
    \item These are isolated FFNs on synthetic BF16 tensors, on H200 NVL.
  \end{itemize}
  \papersource{Paper, Table 9, PDF p. 16; measurement protocol in App. E, p. 15.}
\end{frame}

\end{document}