% Self-authored reading presentation based on the arxiv-paper-reader template.
% Build: python3 build_slides.py slides.en.tex
\documentclass[aspectratio=169,11pt]{beamer}
\usepackage[T1]{fontenc}
\usepackage[utf8]{inputenc}
\usepackage{lmodern}
\usepackage{amsmath,amssymb,graphicx,booktabs}
\usepackage{tikz}
\usetikzlibrary{arrows.meta,positioning}

\definecolor{AcademicBlue}{RGB}{26,57,86}
\definecolor{AcademicTeal}{RGB}{0,105,112}
\definecolor{Warm}{RGB}{171,87,36}
\setbeamercolor{structure}{fg=AcademicBlue}
\setbeamercolor{frametitle}{fg=AcademicBlue,bg=white}
\setbeamercolor{title}{fg=AcademicBlue}
\setbeamercolor{alerted text}{fg=AcademicTeal}
\setbeamercolor{block title}{fg=white,bg=AcademicBlue}
\setbeamercolor{block body}{fg=black,bg=AcademicBlue!5}
\setbeamertemplate{navigation symbols}{}
\setbeamertemplate{itemize items}[circle]
\setbeamertemplate{footline}{%
  \hspace{8mm}{\color{gray}\scriptsize MS-GLA\quad |\quad arXiv:2609.35664v1}%
  \hfill\usebeamerfont{page number in head/foot}%
  \insertframenumber\,/\,\inserttotalframenumber\hspace{4mm}\vspace{3mm}}
\setbeamersize{text margin left=8mm,text margin right=8mm}
\setbeamerfont{frametitle}{size=\Large,series=\bfseries}
\setbeamerfont{title}{size=\LARGE,series=\bfseries}
\newcommand{\papersource}[1]{\par\smallskip{\scriptsize\color{gray}#1\par}}
\newcommand{\paper}[1]{\papersource{Paper: #1.}}
\newcommand{\repo}{https://github.com/prasoondev/msgla/blob/45fa2abc4e3efe2ffc79641473dde0984dbcab91}
\newcommand{\ms}{MS-GLA}
\newcommand{\diag}{\operatorname{Diag}}
\hypersetup{pdftitle={MS-GLA: Research Paper Discussion},pdfauthor={Paper reading presentation}}
\pdfstringdefDisableCommands{\def\translate#1{#1}}

\title[MS-GLA]{MS-GLA: Multi-Scale Gated Linear Attention}
\subtitle{Addressing Representational Bottlenecks\\via Multi-Temporal Resolution}
\author{Paper authors: Prasoon Dev, Anirudh Sankar, Vasudeva Varma}
\institute{IIIT Hyderabad\quad |\quad COLM 2026}
\date{Research paper discussion\quad |\quad 12 October 2026}

\begin{document}

\begin{frame}
  \titlepage
  \vspace{-4mm}
  \begin{center}
    \alert{Distribute a fixed recurrent-state budget across update timescales.}\\[2mm]
    {\small \href{https://arxiv.org/abs/2609.35664v1}{arXiv:2609.35664v1}\quad |\quad 20--30 minute discussion}
  \end{center}
\end{frame}

\begin{frame}{The question: can one update rate serve every timescale?}
  \begin{block}{The architectural bottleneck}
    Every GLA head updates at every token, while a fixed-size state must retain both local detail and longer-range information.
  \end{block}
  \begin{itemize}
    \item Input-dependent gates control retention at the current update rate.
    \item MS-GLA also changes \alert{how often each group of heads updates}.
    \item Coarse summaries trade temporal detail for fewer recurrent updates.
  \end{itemize}
  \paper{Secs. 1--2, PDF pp. 1--4}
\end{frame}

\begin{frame}{Three branches view the same sequence at different rates}
  \centering
  \begin{tikzpicture}[x=1cm,y=1cm,>=Latex,
      b/.style={draw=AcademicBlue,rounded corners,fill=AcademicBlue!4,
      minimum height=0.8cm,align=center,font=\small}]
    \node[b,minimum width=1.1cm] (x) at (0,0) {$\mathbf X$};
    \node[b,minimum width=2.2cm] (a) at (2.6,1.45) {$s=1$: identity};
    \node[b,minimum width=2.2cm] (b) at (2.6,0) {$s=2$: average};
    \node[b,minimum width=2.2cm] (c) at (2.6,-1.45) {$s=4$: average};
    \node[b,minimum width=1.65cm] (g1) at (5.3,1.45) {GLA: 2 heads};
    \node[b,minimum width=1.65cm] (g2) at (5.3,0) {GLA: 1 head};
    \node[b,minimum width=1.65cm] (g4) at (5.3,-1.45) {GLA: 1 head};
    \node[b,minimum width=1.75cm] (u1) at (8,1.45) {Native output};
    \node[b,minimum width=1.75cm] (u2) at (8,0) {Causal hold};
    \node[b,minimum width=1.75cm] (u4) at (8,-1.45) {Causal hold};
    \node[b,minimum width=1.5cm,fill=AcademicTeal!10] (f) at (10.8,0) {Weighted\\fusion};
    \foreach \p in {a,b,c}{\draw[->] (x.east)--(\p.west);}
    \draw[->] (a)--(g1);\draw[->] (b)--(g2);\draw[->] (c)--(g4);
    \draw[->] (g1)--(u1);\draw[->] (g2)--(u2);\draw[->] (g4)--(u4);
    \foreach \p in {u1,u2,u4}{\draw[->] (\p.east)--(f.west);}
    \node[font=\small,align=center,text=AcademicTeal] at (10.8,-1.55) {Weights from\\unpooled $\mathbf x_t$};
    \draw[->,AcademicTeal] (10.8,-1.1)--(f.south);
  \end{tikzpicture}
  \par\medskip
  All branches run; the fusion is dense over a small set of scales.
  \papersource{Teaching diagram based on paper, Sec. 3, PDF pp. 4--5.}
\end{frame}

\begin{frame}{GLA writes associations and decays the previous state}
  \[
  \mathbf S_t=\diag(\alpha_t)\mathbf S_{t-1}
      +\mathbf k_t^{\mathsf T}\mathbf v_t,
  \qquad \mathbf o_t=\mathbf q_t\mathbf S_t
  \]
  \begin{columns}[T,onlytextwidth]
    \begin{column}{.48\textwidth}
      \begin{block}{State and read/write vectors}
        $\mathbf S_t\in\mathbb R^{d_k\times d_v}$\\[1mm]
        $\mathbf q_t,\mathbf k_t\in\mathbb R^{1\times d_k}$\\[1mm]
        $\mathbf v_t,\mathbf o_t\in\mathbb R^{1\times d_v}$
      \end{block}
    \end{column}
    \begin{column}{.48\textwidth}
      \begin{itemize}
        \item $\alpha_t$: input-dependent, row-wise retention.
        \item $\mathbf k_t^{\mathsf T}\mathbf v_t$: new association.
        \item State size stays fixed as the sequence grows.
      \end{itemize}
    \end{column}
  \end{columns}
  \paper{Sec. 2.1, Eqs. (2), (5)--(6), PDF pp. 2--3}
\end{frame}

\begin{frame}{Pooling changes the time axis before GLA is applied}
  \[
  \mathbf X_i^{(s)}=
  \frac{\sum_{j=0}^{s-1}\mathbf X_{is+j}\mathbf M_{is+j}}
       {\max\!\left(1,\sum_{j=0}^{s-1}\mathbf M_{is+j}\right)}
  \]
  \begin{itemize}
    \item Input: $\mathbf X\in\mathbb R^{L\times d}$; mask $\mathbf M\in\{0,1\}^L$.
    \item Non-overlapping blocks; output length $\lceil L/s\rceil$, width $d$.
    \item Right padding handles incomplete blocks; the mask excludes padding.
  \end{itemize}
  \begin{block}{Causality requires delayed release}
    The average of $x_0,x_1,x_2,x_3$ cannot affect predictions at positions 0, 1, or 2.
  \end{block}
  \paper{Sec. 3.1, Eq. (8), PDF p. 4}
\end{frame}

\begin{frame}{The head budget is redistributed across scales}
  \[
    \sum_{s\in\mathcal S}h_s=H,
    \qquad \sum_s h_s d_kd_v=H d_kd_v
  \]
  \begin{center}
  \begin{tabular}{lcc}
    \toprule
    Variant & Scales & Heads per scale\\
    \midrule
    Two-scale & $\{1,2\}$ or $\{1,4\}$ & $[2,2]$\\
    Coarse-only & $\{2,4\}$ & $[2,2]$\\
    Main model & $\{1,2,4\}$ & $[2,1,1]$\\
    Four-scale & $\{1,2,4,8\}$ & $[1,1,1,1]$\\
    \bottomrule
  \end{tabular}
  \end{center}
  Independent projections and gates; per-head state dimensions stay fixed.
  \par\smallskip
  \alert{State equality is exact under these dimensions; total parameters include small branch and fusion overheads.}
  \papersource{Paper: Secs. 3.2, 4.1, PDF pp. 4--6. Parameter qualification: author code @ 45fa2abc4e3e.}
\end{frame}

\begin{frame}{A coarse output becomes visible only at the block end}
  \begin{center}
  \renewcommand{\arraystretch}{1.45}
  \begin{tabular}{lcccccccc}
    \toprule
    Position $t$ & 0&1&2&3&4&5&6&7\\
    Input & $x_0$&$x_1$&$x_2$&$x_3$&$x_4$&$x_5$&$x_6$&$x_7$\\
    $s=4$ output & 0&0&0&$y_0$&$y_0$&$y_0$&$y_0$&$y_1$\\
    \bottomrule
  \end{tabular}
  \end{center}
  \[
    \widehat{\mathbf Y}^{(s)}_t=
    \begin{cases}
      \mathbf 0,&t<s-1,\\
      \mathbf Y^{(s)}_{\lfloor(t-s+1)/s\rfloor},&t\ge s-1.
    \end{cases}
  \]
  Repeat each coarse output $s$ times, then shift right by $s-1$.
  \papersource{Paper: Sec. 3.3, PDF p. 5. Timeline and equivalent index formula are teaching aids.}
\end{frame}

\begin{frame}{The current token chooses the mixture of branch outputs}
  \[
    \mathbf w_t=\operatorname{Softmax}
      (\mathbf W_{\mathrm{fuse}}\mathbf x_t+\mathbf b_{\mathrm{fuse}}),
    \qquad
    \mathbf O_t=\sum_{s\in\mathcal S}w_{t,s}\widehat{\mathbf Y}^{(s)}_t
  \]
  \begin{itemize}
    \item $\mathbf W_{\mathrm{fuse}}\in\mathbb R^{|\mathcal S|\times d}$; the router reads unpooled $\mathbf x_t$.
    \item Each branch projects back to width $d$ before fusion.
    \item Weights are nonnegative and sum to 1 at each position.
  \end{itemize}
  \begin{block}{A different role from MoM}
    MS-GLA mixes temporal resolutions. MoM routes token updates to a sparse subset of memories.
  \end{block}
  \paper{Secs. 2.2, 3.3; Eqs. (10)--(11), PDF pp. 3, 5}
\end{frame}

\begin{frame}{A slower update clock can extend the retention horizon}
  \begin{block}{Teaching approximation: constant scalar retention $0<a<1$}
    After $D$ original tokens, old information is multiplied approximately by
    \[
      \underbrace{a^D}_{s=1}\qquad\text{versus}\qquad
      \underbrace{a^{D/s}}_{\text{scale }s}.
    \]
  \end{block}
  \[
    T_{1/2}^{(s)}\approx s\,\frac{\ln(1/2)}{\ln a}
  \]
  \begin{itemize}
    \item Assumes the same per-update retention; learned gates need not match.
    \item Pooling loses detail and introduces delay, motivating a fine branch.
  \end{itemize}
  \papersource{Reader derivation for intuition, based on paper Eq. (6) and Secs. 3.1--3.3.}
\end{frame}

\begin{frame}{Shorter branch sequences preserve chunkwise training}
  \begin{columns}[T,onlytextwidth]
    \begin{column}{.49\textwidth}
      \begin{block}{Training}
        Each branch applies the existing GLA chunkwise algorithm to a contiguous sequence of length $\lceil L/s\rceil$.
      \end{block}
      Idealized head-update count for $[2,1,1]$:
      \[
        \frac{2L+L/2+L/4}{4L}=68.75\%.
      \]
    \end{column}
    \begin{column}{.47\textwidth}
      \begin{block}{Streaming decoding}
        Keep recurrent state, pending tokens, and the most recent output for each branch.
      \end{block}
      Update a coarse branch only when its block is complete; otherwise hold its output.
    \end{column}
  \end{columns}
  \par\smallskip
  \alert{Update counts exclude projections, MLPs, fusion, and execution overhead.}
  \papersource{Paper: Sec. 3.4, PDF p. 5. Decode details: author code, ms\_gla\_layer.py @ 45fa2abc4e3e.}
\end{frame}

\begin{frame}{The experiment compares six approximately 340M models}
  \begin{itemize}
    \item 24 layers, hidden width 1024, four total heads; GLA is the baseline.
    \item FineWeb-Edu; context 2,048; batch 32; accumulation 1; seed 42.
    \item Language modeling, zero-shot tasks, recall, position curves, routing, and training cost.
  \end{itemize}
  \begin{block}{Training-budget records disagree}
    Main text: all models use 7B tokens. Appendix B:
    \[
      \begin{aligned}
      \mathrm{GLA}:&\quad68{,}664\times32\times2{,}048\approx4.50\mathrm B,\\
      \text{MS-GLA}:&\quad106{,}813\times32\times2{,}048\approx7.00\mathrm B.
      \end{aligned}
    \]
  \end{block}
  Both public Table 1 CSVs point to \texttt{step-106813}; the appendix may be stale. Actual run metadata is needed to resolve this.
  \papersource{Paper: Sec. 4, PDF pp. 5--6; Appendix B, p. 13. Author result CSVs @ 45fa2abc4e3e.}
\end{frame}

\begin{frame}{Three scales give the lowest mean perplexity}
  \begin{center}
  \begin{tabular}{lrrr}
    \toprule
    Model & Wikitext PPL $\downarrow$ & LAMBADA PPL $\downarrow$ & Mean PPL $\downarrow$\\
    \midrule
    GLA &19.63&22.26&20.94\\
    $\{1,2\}$ &19.25&20.99&20.12\\
    $\{1,4\}$ &\textbf{17.43}&21.69&19.56\\
    $\{2,4\}$ &21.66&31.47&26.56\\
    $\{1,2,4\}$ &18.33&\textbf{19.59}&\textbf{18.96}\\
    $\{1,2,4,8\}$ &18.80&20.18&19.49\\
    \bottomrule
  \end{tabular}
  \end{center}
  \begin{itemize}
    \item Mean PPL: $20.94\rightarrow18.96$, a \alert{9.46\% relative reduction}.
    \item Four-task mean score: $41.94\rightarrow42.85$ (+0.91 pp).
    \item $\{1,4\}$ wins Wikitext; the advantage depends on the task.
  \end{itemize}
  \paper{Sec. 5.1, Table 1, PDF pp. 6--7; means use the reported table values}
\end{frame}

\begin{frame}{Recall improves by 1.40 F1 points in the three-task mean}
  \begin{center}
  \begin{tabular}{lrrrr}
    \toprule
    Model & SWDE & FDA & SQuAD & Mean F1 $\uparrow$\\
    \midrule
    GLA &7.86&5.52&8.84&7.41\\
    $\{1,2\}$ &9.26&5.95&9.79&8.33\\
    $\{1,4\}$ &8.28&\textbf{6.06}&10.32&8.22\\
    $\{2,4\}$ &5.96&5.42&8.21&6.53\\
    $\{1,2,4\}$ &\textbf{9.30}&5.98&\textbf{11.15}&\textbf{8.81}\\
    $\{1,2,4,8\}$ &7.59&5.27&11.05&7.97\\
    \bottomrule
  \end{tabular}
  \end{center}
  \[
    \frac{8.81-7.41}{7.41}\times100\%\approx\alert{18.9\%}
  \]
  Relative gain in the mean; absolute F1 remains low. Four scales reduce SWDE and FDA below GLA despite strong SQuAD.
  \paper{Sec. 5.2, Table 2, PDF p. 7; all task columns report F1, higher is better}
\end{frame}

\begin{frame}{Position curves show lower PPL, with shared fluctuations}
  \centering
  \includegraphics[width=.96\linewidth,height=.68\textheight,keepaspectratio]{figures/fig-01-long-context.png}
  \par\smallskip
  \begin{minipage}{.97\textwidth}
    PG19: lower curves, without sustained baseline drift.\\
    SlimPajama: all variants rise at late positions; coarse-only is worst.
  \end{minipage}
  \paper{Fig. 1, PDF p. 8; original author figure, not redrawn}
\end{frame}

\begin{frame}{Document position does not establish the available context}
  \begin{block}{Public evaluator: independent sliding windows}
    Default \texttt{max\_ctx = 2048}; each window calls\\[1mm]
    \texttt{model(chunk, use\_cache=False)}\\[1mm]
    with no recurrent state passed from the previous window.
  \end{block}
  \begin{itemize}
    \item A document can reach 32K while each prediction sees at most 2K.
    \item Saved curve JSONs omit the actual window/stride settings.
    \item Default settings cannot establish $15\times$ continuous-state extrapolation; the exact run settings remain unresolved.
  \end{itemize}
  \alert{Needed: a full long-sequence forward pass or state-carrying chunks.}
  \papersource{Paper: Sec. 5.3, PDF pp. 7--8. Supplement: \href{\repo/benchmark_msgla/eval_msgla_perplexity.py}{author evaluator} and README @ 45fa2abc4e3e.}
\end{frame}

\begin{frame}{The router's regional means stay almost unchanged}
  \begin{center}
  \begin{tabular}{lrrr}
    \toprule
    Region of sequence & $s=1$ & $s=2$ & $s=4$\\
    \midrule
    First 10\% &52.64&18.41&28.95\\
    Middle 10\% &52.91&18.34&28.76\\
    Last 10\% &52.57&18.41&29.03\\
    \bottomrule
  \end{tabular}
  \end{center}
  \begin{itemize}
    \item Weights in percent for $\{1,2,4\}$; first-to-last shift $\leq0.08$ pp.
    \item Fine resolution has the largest mean weight in variants using $s=1$.
    \item Coarse-only routing is also stable, despite poor model quality.
  \end{itemize}
  \begin{block}{Interpretation}
    Stable regional averages support sustained average scale usage. They do not measure per-token routing diversity or causal contribution.
  \end{block}
  \paper{Sec. 5.4, Table 3, PDF p. 8}
\end{frame}

\begin{frame}{The main model retains 92\% of training throughput}
  \begin{center}
  \begin{tabular}{lrr}
    \toprule
    Model & Tokens/second $\uparrow$ & Peak memory, GiB $\downarrow$\\
    \midrule
    GLA &31,989&70.2\\
    $\{1,2\}$ &27,619&86.7\\
    $\{1,2,4\}$ &29,437&80.4\\
    $\{1,2,4,8\}$ &27,201&89.2\\
    $\{2,4\}$ &34,123&71.5\\
    \bottomrule
  \end{tabular}
  \end{center}
  \begin{itemize}
    \item Main configuration: \alert{7.98\% less throughput; 14.53\% more peak memory}.
    \item Fixed recurrent-state size does not fix training activation memory.
    \item Total training times require reconciled token counts and timing scope.
  \end{itemize}
  \paper{Sec. 5.5, Table 4, PDF p. 9; single RTX PRO 6000 Max-Q, Appendix B}
\end{frame}

\begin{frame}{Scale coverage competes with branch capacity}
  \begin{columns}[T,onlytextwidth]
    \begin{column}{.48\textwidth}
      \begin{block}{The strongest ablation pattern}
        Removing $s=1$ worsens mean PPL and recall F1:
        \[
          18.96\rightarrow26.56
        \]
        \[
          8.81\rightarrow6.53
        \]
        $\{1,2,4\}$ versus $\{2,4\}$.
      \end{block}
    \end{column}
    \begin{column}{.48\textwidth}
      \begin{block}{Adding a scale changes two things}
        $\{1,2,4\}:[2,1,1]$\\[1mm]
        $\{1,2,4,8\}:[1,1,1,1]$
      \end{block}
      The fine branch loses a head while $s=8$ is added. The experiment does not isolate these effects.
    \end{column}
  \end{columns}
  \par\smallskip
  A token-level pathway is valuable in the tested designs; more scales are not automatically better.
  \paper{Secs. 6.1--6.2, Tables 1--2, PDF pp. 7, 9--10}
\end{frame}

\begin{frame}{The design is useful; the strongest claims need sharper evidence}
  \begin{enumerate}
    \item \textbf{Contribution:} a concrete multi-rate GLA layer with fixed head/state budget and causal output alignment.
    \item \textbf{Reported benefit:} lower mean PPL and higher recall F1 for a fine-plus-coarse allocation, with modest training overhead.
    \item \textbf{Evidence limits:} one seed, GLA-only direct comparison, training-record inconsistency, and unresolved long-context protocol.
  \end{enumerate}
  \begin{block}{Do next}
    Reconcile checkpoint budgets; evaluate continuous long-context state; separate head allocation, pooling, and fusion in ablations.
  \end{block}
  \paper{Secs. 7--8, Appendix A, PDF pp. 10, 13; protocol qualifications from the preceding evidence}
\end{frame}

\begin{frame}{Sources for the paper discussion}
  \textbf{Primary paper}\par\smallskip
  Prasoon Dev, Anirudh Sankar, Vasudeva Varma.\par
  \emph{MS-GLA: Multi-Scale Gated Linear Attention for Addressing Representational Bottlenecks via Multi-Temporal Resolution.}\par
  arXiv:2609.35664v1, 28 September 2026. PDF marked COLM 2026.\par
  \href{https://arxiv.org/abs/2609.35664v1}{https://arxiv.org/abs/2609.35664v1}
  \bigskip

  \textbf{Author implementation and supplementary records}\par\smallskip
  \href{https://github.com/prasoondev/msgla}{github.com/prasoondev/msgla}, pinned commit \texttt{45fa2abc4e3e}.\par
  Layer/config files, Table 1 CSVs, long-document evaluators and saved curves. Accessed 12 October 2026.
  \bigskip

  Code observations supplement the paper; a repository snapshot does not identify every experimental run that produced it.
\end{frame}

\appendix

\begin{frame}{Backup: zero-shot gains are task dependent}
  \begin{center}
  \small
  \begin{tabular}{lrrrrr}
    \toprule
    Model & LAMBADA & PIQA & HellaSwag & Winogrande & Mean\\
    \midrule
    GLA &19.33&64.20&33.94&50.28&41.94\\
    $\{1,2\}$ &19.68&65.18&34.89&49.17&42.23\\
    $\{1,4\}$ &19.89&63.87&\textbf{35.49}&51.07&42.58\\
    $\{2,4\}$ &16.15&65.18&34.17&\textbf{52.57}&42.01\\
    $\{1,2,4\}$ &\textbf{20.69}&\textbf{65.83}&35.01&49.88&\textbf{42.85}\\
    $\{1,2,4,8\}$ &20.34&65.34&34.53&49.01&42.31\\
    \bottomrule
  \end{tabular}
  \end{center}
  \begin{itemize}
    \item All values are percentage scores; higher is better.
    \item Public CSVs identify HellaSwag as \texttt{acc\_norm}.
    \item Mean covers all four tasks, including Winogrande.
    \item $\{2,4\}$ leads Winogrande despite its poor language-model PPL.
  \end{itemize}
  \paper{Table 1, PDF p. 7; metric clarification from pinned author CSVs}
\end{frame}

\begin{frame}{Backup: optimization and hardware settings}
  \begin{center}
  \begin{tabular}{ll}
    \toprule
    Setting & Reported value\\
    \midrule
    Optimizer & AdamW, $\epsilon=10^{-15}$, weight decay 0.1\\
    Peak learning rate & $3\times10^{-4}$\\
    Schedule & 3,400 warmup steps; cosine to ratio 0.1\\
    Gradient clipping & Maximum norm 1.0; skip non-finite steps\\
    Tokens per step & $32\times2,048=65,536$; accumulation 1\\
    Random seed & 42; one training run per configuration\\
    Tokenizer & \texttt{fla-hub/transformer-1.3B-100B}\\
    Hardware & One NVIDIA RTX PRO 6000 Max-Q\\
    Framework & \texttt{flash-linear-attention} + \texttt{flame}\\
    \bottomrule
  \end{tabular}
  \end{center}
  \paper{Appendices A--B, PDF pp. 13--14; token-budget inconsistency discussed in the main slides}
\end{frame}

\begin{frame}{Backup: matrix-state equality leaves small extra costs}
  \[
    d_k=\frac{1024\times0.5}{4}=128,
    \qquad d_v=\frac{1024\times1}{4}=256
  \]
  \[
    \text{state elements per layer}=4\times128\times256=131{,}072
  \]
  \begin{itemize}
    \item Branch $s$ has $h_s$ such states; their total is unchanged.
    \item Independent low-rank gate input projections add parameters.
    \item A three-scale fusion projection adds $3(1024+1)=3,075$ parameters per layer.
    \item Cached decoding also holds incomplete blocks and previous outputs.
  \end{itemize}
  \papersource{Derived from \href{\repo/flame/configs/ms_gla_340M.json}{author config}, ms\_gla\_layer.py and GLA layer source @ 45fa2abc4e3e.}
\end{frame}

\begin{frame}{Backup: wall-clock time and ``TFLOPs'' need context}
  \begin{center}
  \small
  \begin{tabular}{lrrrr}
    \toprule
    Model & ``TFLOPs'' & Tokens/s & GiB & Training time\\
    \midrule
    GLA &91.2&31,989&70.2&3d 1:49:25\\
    $\{1,2\}$ &78.9&27,619&86.7&2d 18:23:08\\
    $\{1,2,4\}$ &84.0&29,437&80.4&2d 22:48:41\\
    $\{1,2,4,8\}$ &77.8&27,201&89.2&2d 23:43:25\\
    $\{2,4\}$ &97.4&34,123&71.5&2d 9:11:04\\
    \bottomrule
  \end{tabular}
  \end{center}
  \begin{itemize}
    \item No $\{1,4\}$ cost row is reported.
    \item ``TFLOPs'' lacks a clear total/per-step/rate definition.
    \item Lower throughput with shorter total time is not a matched-work speedup without consistent budgets and timing scope.
  \end{itemize}
  \paper{Table 4, PDF p. 9; hardware issues noted in Appendix B, pp. 13--14}
\end{frame}

\begin{frame}{Backup: key code evidence and how to interpret it}
  \begin{description}
    \item[Training budget] \href{\repo/benchmark_gla/results/7B/ppl_acc/gla.csv}{GLA CSV} and
    \href{\repo/benchmark_msgla/results/7B/124/ppl_acc/msgla.csv}{MS-GLA CSV}
    both name \texttt{step-106813}; this qualifies the appendix discrepancy.
    \item[Available context] \href{\repo/benchmark_msgla/eval_msgla_perplexity.py}{MS-GLA evaluator} and
    \href{\repo/benchmark_gla/eval_gla_perplexity.py}{GLA evaluator}
    use independent windows; the default maximum is 2,048.
    \item[Run identification] \href{\repo/benchmark_msgla/results/7B/124/slimpg19/msgla_124_results.json}{Saved long-document curves}
    do not include window size or stride; exact Figure 1 settings are unresolved.
    \item[Parameter budget] \href{\repo/flame/custom_models/ms_gla/ms_gla_layer.py}{Layer code}
    preserves per-head state dimensions while constructing independent branch projections and a fusion layer.
  \end{description}
  \papersource{Supplementary evidence at author commit 45fa2abc4e3e; accessed 12 October 2026.}
\end{frame}

\end{document}