\documentclass[aspectratio=169,11pt]{beamer} \usepackage[T1]{fontenc} \usepackage[utf8]{inputenc} \usepackage{lmodern} \usepackage{amsmath,amssymb,graphicx,booktabs,array} \definecolor{AcademicBlue}{RGB}{26,57,86} \definecolor{AcademicTeal}{RGB}{0,105,112} \setbeamercolor{structure}{fg=AcademicBlue} \setbeamercolor{frametitle}{fg=AcademicBlue,bg=white} \setbeamercolor{title}{fg=AcademicBlue} \setbeamercolor{alerted text}{fg=AcademicTeal} \setbeamercolor{block title}{fg=white,bg=AcademicBlue} \setbeamercolor{block body}{fg=black,bg=AcademicBlue!5} \setbeamertemplate{navigation symbols}{} \setbeamertemplate{itemize items}[circle] \setbeamertemplate{footline}{% \hfill\usebeamerfont{page number in head/foot}% \insertframenumber\,/\,\inserttotalframenumber\hspace{4mm}\vspace{3mm}} \setbeamersize{text margin left=8mm,text margin right=8mm} \setbeamerfont{frametitle}{size=\Large,series=\bfseries} \setbeamerfont{title}{size=\LARGE,series=\bfseries} \newcommand{\papersource}[1]{\par\smallskip{\scriptsize\color{gray}#1\par}} \newcommand{\figfit}[2][0.64\textheight]{\includegraphics[width=\linewidth,height=#1,keepaspectratio]{#2}} \title[Samba]{Samba: Simple Hybrid State Space Models} \subtitle{For Efficient Unlimited Context Language Modeling} \author{Paper authors: Liliang Ren, Yang Liu, Yadong Lu,\\Yelong Shen, Chen Liang, Weizhu Chen} \date{ICLR 2025 \enspace | \enspace arXiv:2406.07522v3 (28 Feb 2025)\\Research discussion \enspace | \enspace 16 Oct 2026} \begin{document} \begin{frame} \titlepage \vspace{-5mm} \begin{center} \color{AcademicTeal}Recurrent compression + precise local access, with a fixed history cache. \end{center} \note{This is a reading presentation of the paper, organized for about 25 minutes plus discussion. The central question is whether a simple hybrid architecture can retain the efficiency of recurrence and the precision of attention. Throughout the talk, distinguish streaming computation, language-model perplexity, and exact retrieval. The paper gives different evidence for each.} \end{frame} \begin{frame}{The problem is how to represent history} \begin{center} \begin{tabular}{>{\raggedright\arraybackslash}p{0.21\linewidth}>{\raggedright\arraybackslash}p{0.32\linewidth}>{\raggedright\arraybackslash}p{0.34\linewidth}} \toprule Mechanism & History representation & Main trade-off \\ \midrule Full attention & Explicit KV for all past tokens & Direct access; growing cache and work \\[2mm] Mamba & Fixed-dimensional recurrent state & Efficient streaming; compressed detail \\[2mm] Sliding window attention & Explicit KV for recent tokens & Precise local access; bounded direct reach \\ \bottomrule \end{tabular} \end{center} \begin{block}{Samba's question} Can these memory mechanisms complement each other without a global attention layer? \end{block} \papersource{Paper, Secs. 1--2, PDF pp. 1--4.} \note{Full attention provides an addressable representation of the entire prefix. A recurrent state instead summarizes the past. A sliding window preserves exact recent features but cannot directly access arbitrary distant tokens. Stacking local layers can extend indirect reach, so a window size is not an absolute receptive-field bound for the entire model. Samba adds a recurrent route to carry information across time.} \end{frame} \begin{frame}{Samba alternates two mixers, each followed by an MLP} \centering \figfit[0.49\textheight]{figures/architecture.png} \begin{block}{One repeat: Mamba $\rightarrow$ MLP $\rightarrow$ SWA $\rightarrow$ MLP} 1.7B Samba: 12 Mamba + 12 SWA + 24 MLP sublayers.\\ Each sublayer has Pre-Norm and a residual connection. \end{block} \papersource{Paper, Fig. 1, PDF p. 3; Table 12, PDF p. 24. Original author figure.} \note{Read the figure from left to right to compare arrangements, and top to bottom to follow computation. The architecture is a layer-wise interleaving, not an intra-layer mixture. The term layer counts MLPs separately. Each mixer receives contextualized features from preceding layers, so the recurrent and local memories communicate through the residual stream. The Mamba-SWA-MLP baseline has a larger fraction of mixers and fewer MLPs.} \end{frame} \begin{frame}{Mamba makes retention and writing input-dependent} \[ \mathbf U=\operatorname{SiLU}(\operatorname{DepthwiseConv}(\mathbf XW_{\rm in})), \qquad \Delta=\operatorname{Softplus}(\mathbf UW_rW_q+\mathbf b) \] \[ \boxed{Z_{t,i,j}=e^{-\Delta_{t,i}e^{A_{i,j}}}Z_{t-1,i,j} +\Delta_{t,i}U_{t,i}B_{t,j}} \] \[ Y_{t,i}=\sum_j Z_{t,i,j}C_{t,j}+D_iU_{t,i}, \quad \mathbf B=\mathbf UW_b,\quad \mathbf C=\mathbf UW_c \] \begin{itemize} \item $i$: expanded channel; $j$: state coordinate; $\mathbf Z_t\in\mathbb R^{d_e\times d_s}$. \item $d_e=2d_m$, $d_s=16$; short convolution width $k=4$. \item Larger $\Delta$: stronger decay and a larger current-input write factor. \end{itemize} \papersource{Paper, Sec. 2.1.1, Eq. (1)--(2) and unnumbered recurrence, PDF p. 3. Elementwise expansion for clarity.} \note{The state update is written elementwise to make broadcasting and outer-product orientation unambiguous. A is a learned log-rate parameter; the decay multiplier lies between zero and one. Delta is positive but is not a probability distribution. B and C are also input-dependent. The final Mamba output is gated and projected back to model width. During training, a hardware-aware parallel scan computes these recurrences.} \end{frame} \begin{frame}{Local attention preserves detail that compression can lose} \[ \mathcal W_t=\{\max(1,t-w+1),\ldots,t\},\qquad w=2048 \] \[ o_t=\sum_{j\in\mathcal W_t} \operatorname{softmax}_{j\in\mathcal W_t} \left(\frac{q_t^\top k_j}{\sqrt{d_h}}\right)v_j \] \begin{itemize} \item RoPE within the window; base frequency 10,000. \item Direct access to recent KV; older information travels through states and layers. \item FlashAttention 2 for SWA; a separate SwiGLU MLP follows each mixer. \end{itemize} \begin{block}{Design intuition} Let attention handle precise local access, while recurrence carries information forward. \end{block} \papersource{Paper, Secs. 2.1.2--2.1.3, PDF p. 4. SWA equation is a teaching restatement; $q,k$ include RoPE.} \note{The original paper explains SWA in prose. This equation makes its causal mask explicit. A key nuance is that the values in local attention have already been transformed by earlier recurrent layers. The model therefore has access to contextualized local features rather than only raw recent token embeddings. The window size is also a systems choice motivated by the relative performance of optimized attention and scan kernels.} \end{frame} \begin{frame}{A fixed history cache permits continuing the stream} \begin{center} \begin{tabular}{lcc} \toprule Mixer & Mixing work for $n$ tokens & History stored per layer \\ \midrule Full attention & $O(n^2d_m)$ & $O(nh_{kv}d_h)$ \\[2mm] SWA & $O(nwd_m)$ & $O(wh_{kv}d_h)$ \\[2mm] Mamba update & $O(nd_ed_s)$ & $O(d_ed_s+kd_e)$ \\ \bottomrule \end{tabular} \end{center} \begin{itemize} \item Fix $w$, state size, width, and depth: total sequence work is linear in $n$. \item History storage per stream stops growing after the window fills. \item Finite state capacity still constrains what can be recalled exactly. \end{itemize} \papersource{Complexity explanation from Sec. 2, PDF pp. 2--4; common projections omitted. Training activations are separate.} \note{This is an analytical explanation of the architecture, not a measured memory table in the paper. The claim concerns history storage for one stream. Weights, training activations, batching and temporary buffers are additional costs. Asymptotic linearity alone does not establish good wall-clock speed. We will therefore inspect training and inference measurements separately.} \end{frame} \begin{frame}{Four regimes test different questions} \begin{center} \begin{tabular}{llll} \toprule Samba size & Data & Training tokens & Main purpose \\ \midrule 421M & SlimPajama & 20B & Architecture ablations \\[2mm] 1.3B & SlimPajama & 100B & Scaling and training speed \\[2mm] 1.7B & Phi-2 & 230B & Downstream tasks, long context \\[2mm] 3.8B & Phi-3 & 3.2T & Large-scale quality, post-training \\ \bottomrule \end{tabular} \end{center} \begin{itemize} \item All four use 4,096-token training sequences and a 2,048-token SWA window. \item Strongest architecture comparisons match data and recipes: TFM++ / Phi-3. \item Original and June post-training recipes produce distinct Samba models. \end{itemize} \papersource{Paper, Sec. 3, PDF pp. 4--8; Appendix B; Table 12, PDF p. 24.} \note{Avoid merging experiments with different sizes and data. The million-token perplexity curve and most speed measurements use the approximately 1.7B models, not the 3.8B model. The largest model is important for matched-data quality comparisons. Public models in the appendix are useful reference points, but differences in their training corpora prevent clean architecture attribution.} \end{frame} \begin{frame}{Matched 3.8B pretraining improves the average by 4.1 points} \begin{center} \begin{tabular}{lrrr} \toprule Base model metric $\uparrow$ & TFM++ & Samba & Difference \\ \midrule MMLU & 67.2 & 71.2 & +4.0 \\ HellaSwag & 76.6 & 77.4 & +0.8 \\ ARC-C & 53.8 & 55.7 & +1.9 \\ WinoGrande & 72.6 & 77.1 & +4.5 \\ TruthfulQA MC2 & 47.3 & 43.4 & $-3.9$ \\ GSM8K & 51.5 & \alert{69.6} & \alert{+18.1} \\ HumanEval & 51.8 & 54.9 & +3.1 \\ \midrule Average & 60.1 & \textbf{64.2} & +4.1 \\ \bottomrule \end{tabular} \end{center} Same 3.2T-token data and recipe. GSM8K gain: \textbf{18.1 percentage points}. \papersource{Paper, Table 7, PDF p. 20. MMLU/GSM8K: 5-shot; others: zero-shot. ARC-C/HellaSwag: character-normalized.} \note{This is the most informative large-scale base-model comparison. The average improves, with a particularly large math gain. TruthfulQA decreases, so the improvement is not uniform. The authors conjecture that attention frees recurrent states for other computations, including arithmetic. That mechanism is an interpretation; the table establishes the performance pattern under the matched training setup.} \end{frame} \begin{frame}{Post-training improves short tasks and GovReport} \begin{center} \begin{tabular}{lrrr} \toprule Metric $\uparrow$ & Phi-3-mini & Samba-3.8B-IT & Difference \\ \midrule MMLU & 68.8 & 71.9 & +3.1 \\[1mm] GSM8K & 82.5 & 87.6 & +5.1 \\[1mm] HumanEval & 58.5 & 62.8 & +4.3 \\[1mm] GovReport (ROUGE-L) & 14.4 & 18.9 & +4.5 \\[1mm] SQuALITY (ROUGE-L) & 21.6 & 21.2 & $-0.4$ \\ \bottomrule \end{tabular} \end{center} \begin{itemize} \item Original post-training recipe; MMLU 5-shot, GSM8K 8-shot CoT. \item Mean document lengths: GovReport 11,533; SQuALITY 7,974 tokens. \end{itemize} \papersource{Paper, Table 1, PDF p. 4; Appendix F, p. 23. HumanEval: zero-shot pass@1. Phi-3 row marked as report-derived.} \note{These are instruction-tuned models using the original recipe. The June models are a separate experiment in the backup slides. GovReport improves substantially, while SQuALITY decreases slightly. ROUGE-L measures reference overlap rather than exact factual retrieval. Evaluation uses greedy decoding, capped at 450 generated tokens for GovReport and 600 for SQuALITY.} \end{frame} \begin{frame}{The best hybrid arrangement depends on the task} \begin{center} \begin{tabular}{lrrrr} \toprule Phi-2 model & Avg. (15) & SQuAD & MMLU & GSM8K \\ \midrule Llama-3 1.6B & 51.17 & 74.88 & 43.84 & 32.68 \\ Mistral 1.6B & 51.12 & 72.82 & 43.54 & 32.45 \\ Mamba 1.8B & 52.31 & 67.66 & 45.28 & 32.07 \\ Mamba-SWA-MLP 1.6B & 53.77 & 76.73 & 47.39 & \textbf{44.05} \\ Mamba-MLP 1.9B & 51.38 & 63.86 & 43.68 & 27.52 \\ Samba 1.7B & \textbf{54.33} & \textbf{77.64} & \textbf{48.01} & 38.97 \\ \bottomrule \end{tabular} \end{center} \begin{itemize} \item Samba gains 9.98 SQuAD points over pure Mamba. \item Mamba-SWA-MLP wins on GSM8K by 5.08 percentage points. \end{itemize} \papersource{Paper, Table 2, PDF p. 5. All trained on 230B Phi-2 tokens; approximate parameter matching. GSM8K: 5-shot.} \note{The models occupy a 1.6 to 1.9B parameter range. Their shared training corpus makes architecture comparisons useful, while parameter counts are only approximately matched. The SQuAD pattern is consistent with explicit attention improving precise access to context. The alternative three-layer repeat wins on mathematics, WinoGrande and SIQA, showing that a single static arrangement is not best for every task.} \end{frame} \begin{frame}{SWA and RoPE are critical to the observed extrapolation} \begin{center} \begin{tabular}{lrrr} \toprule About 1.3B, SlimPajama 100B & PPL @4K $\downarrow$ & @8K $\downarrow$ & @16K $\downarrow$ \\ \midrule Llama-2 & 7.60 & 44.32 & 249.64 \\ Llama-2-SWA & 7.60 & 7.37 & 7.21 \\ Mamba & 7.47 & 7.26 & 7.15 \\ Sliding GLA (1.2B) & 7.58 & 7.35 & 7.19 \\ Sliding RetNet (1.4B) & 7.56 & 7.35 & 7.56 \\ Mamba-SWA-MLP & 7.37 & 7.16 & 7.00 \\ MLP2-SWA-MLP & 7.81 & 7.58 & 7.42 \\ Samba-NoPE & 7.33 & 20.40 & \alert{326.17} \\ Samba & \textbf{7.32} & \textbf{7.11} & \textbf{6.96} \\ \bottomrule \end{tabular} \end{center} Removing RoPE barely changes 4K PPL, but destroys length extrapolation. \papersource{Paper, Table 3, PDF p. 6. Training length 4,096; SWA window 2,048. Reported PPL fluctuation: about $\pm0.3\%$.} \note{The architecture ablations test several possible explanations. Replacing Mamba with MLP hurts quality. Replacing it with GLA or RetNet does not match Samba here. Most dramatically, the NoPE variant looks almost identical at training length and then fails at longer lengths. A local attention mask and a recurrent state alone therefore do not guarantee extrapolation in a trained network.} \end{frame} \begin{frame}{Fewer recurrent layers can make a hybrid faster to train} \begin{center} \begin{tabular}{lrr} \toprule About 1.3B, 4K sequences & Layers & Throughput ($10^5$ tokens/s) \\ \midrule Mamba & 48 & 17.8 \\[2mm] Mamba-SWA-MLP & 36 & 23.5 \\[2mm] Samba & 36 & \textbf{25.2} \\[2mm] Llama-2-SWA & 40 & 26.2 \\[2mm] MLP2-SWA-MLP & 36 & 26.6 \\ \bottomrule \end{tabular} \end{center} \begin{itemize} \item Samba / Mamba: $25.2/17.8\approx1.42\times$ training throughput. \item MLP-heavy designs use efficient dense operations, but removing recurrence hurts PPL. \end{itemize} \papersource{Paper, Table 3 and Sec. 3.2, PDF pp. 6--7. Measured on 64 H100 GPUs; ratio calculated from the table.} \note{Asymptotic complexity is not the same as throughput. Pure Mamba needs more recurrent layers at the same parameter scale, while Samba devotes half of its sublayers to MLPs. MLP2-SWA-MLP is fastest in the table but loses language-model quality. Samba offers a quality and speed trade-off rather than a universal training-speed win. Do not compare these H100 measurements directly with the smaller A100 experiments.} \end{frame} \begin{frame}{Proof-Pile prediction improves far beyond 4K training} \begin{columns}[c] \column{0.64\textwidth} \centering\figfit[0.65\textheight]{figures/proof-pile.pdf} \column{0.34\textwidth} \begin{itemize} \item Samba 1.7B stays stable up to 1M tokens. \item SWA-only Mistral also extrapolates, at higher PPL. \item Pure Mamba deteriorates gradually at the long end. \end{itemize} \medskip \alert{Prediction quality and exact retrieval are separate tests.} \end{columns} \papersource{Paper, Fig. 2a and Sec. 3.3, PDF pp. 7--8. Original figure. PPL evaluation uses a 4,096-token sliding-window approach.} \note{The horizontal axis doubles context length from 1K to 1M, and the vertical axis is perplexity. The curve is for the approximately 1.7B Phi-2 models. The evaluation's 4096-token sliding window is distinct from the internal 2048-token SWA window. The paper does not spell out all state-passing implementation details here. Read this as the reported prediction experiment, and do not turn it into a million-token exact-recall claim.} \end{frame} \begin{frame}{Inference gains grow when full attention must read more history} \begin{columns}[T] \column{0.49\textwidth} \centering\textbf{Prompt processing} \figfit[0.46\textheight]{figures/prefill.jpg} \par\alert{$3.73\times$ at 128K prompt length} \par\small Total input tokens fixed at 128K; batch varies. \column{0.49\textwidth} \centering\textbf{Autoregressive generation} \figfit[0.46\textheight]{figures/decoding.jpg} \par\alert{$3.64\times$ at 64K generation length} \par\small Batch size fixed at 16. \end{columns} \medskip Both ratios compare Samba 1.7B with Llama-3 1.6B. \papersource{Paper, Figs. 6 / 2b, PDF pp. 21 / 7; Sec. 3.3, p. 8: single A100, bfloat16, 10 measurements averaged. Original figures.} \note{Read each panel under its own batching protocol. Fixed total prompt tokens means the prompt-processing batch size decreases as length increases. Generation uses batch 16. These gains must not be multiplied into an end-to-end speedup. Mistral's SWA-only architecture is also stable and has higher prompt throughput than Samba in this figure. The result is an attractive quality-efficiency combination relative to full attention.} \end{frame} \begin{frame}{Task tuning enables 256K Passkey recall from 4K examples} \centering \figfit[0.45\textheight]{figures/passkey.pdf} \begin{itemize} \item Samba 1.7B (left) vs. Mistral 1.6B (right); 500 steps at 4K. \item Seven lengths, 11 insertion depths, five keys per cell: 385 tests. \item Samba reaches 100\% on the final grid; the base model lacks this behavior. \end{itemize} \papersource{Paper, Fig. 3, PDF p. 8; Appendix C, p. 22. Batch 2,048: nominal 4.19B input tokens over 500 steps (calculated).} \note{The heatmap tests a single random five-digit passkey inserted into a long document. Green denotes successful recall. Although training uses only 4K sequences, Samba can maintain the selected signal over much longer sequences. Each cell contains only five keys; the perfect-recall statement refers to this measured grid. The step count sounds small, but the very large batch corresponds to about 4.19B input tokens at full sequence length.} \end{frame} \begin{frame}{Many-item retrieval exposes a capacity and transfer gap} \begin{columns}[c] \column{0.63\textwidth} \centering\figfit[0.65\textheight]{figures/phonebook.pdf} \column{0.35\textwidth} \begin{itemize} \item 20--480 name--number pairs; random target query. \item Samba 3.8B: 100 Phonebook tuning steps at 4K improve extrapolation. \item Passkey-tuned 1.7B models transfer poorly to Phonebook. \end{itemize} \end{columns} \papersource{Paper, Fig. 4 and Sec. 3.4, PDF pp. 8--9. Original figure. Longer inputs also contain more items.} \note{This task increases both distance and the number of things to remember. The blue Phonebook-tuned Samba curve stays better than other models beyond the training length, but accuracy still falls. More importantly, successful Passkey tuning does not create a general-purpose associative memory. The Passkey-tuned smaller models remain weak once retrieval extends beyond their local window. This is the strongest reason to separate length and capacity extrapolation.} \end{frame} \begin{frame}{Full-attention hybrids trail SWA, with an important table nuance} \begin{center} \begin{tabular}{lr@{\hspace{7mm}}r@{\hspace{7mm}}r} \toprule Full-attention block indices & PPL @4K & @8K & @16K \\ \midrule 11 & 10.29 & 10.53 & 13.66 \\[1mm] 5 & 10.10 & 10.05 & 12.83 \\[1mm] 0 & 10.89 & 10.55 & \alert{10.63} \\[1mm] 1, 5 & 10.06 & 10.34 & 13.57 \\[1mm] Samba: SWA at odd indices & 10.06 & 9.65 & \textbf{9.57} \\ \bottomrule \end{tabular} \end{center} \begin{itemize} \item All reported full-attention variants have worse 16K PPL than Samba. \item The block-0 row does \emph{not} show the ``explosion'' described in the prose. \end{itemize} \papersource{Paper, Table 5 and Sec. 4, PDF p. 9. Approximately 430M; 20B SlimPajama tokens; 12 mixer+MLP blocks.} \note{The table supports the choice of SWA for the tested setup. It does not support the stronger wording that even attention at the first block causes perplexity to explode. That row is 10.89, 10.55, 10.63. Regardless of extrapolation quality, retaining a full-attention layer also retains a length-growing global attention cost. Distinguish that complexity argument from the empirical quality result.} \end{frame} \begin{frame}{Entropy patterns suggest specialization across layers} \begin{columns}[T] \column{0.49\textwidth} \centering\textbf{Attention entropy} \figfit[0.42\textheight]{figures/attention-entropy.pdf} \column{0.49\textwidth} \centering\textbf{S6 selection entropy} \figfit[0.42\textheight]{figures/selection-entropy.pdf} \end{columns} \begin{itemize} \item Samba attention is more concentrated in middle layers, broader at the ends. \item Its middle Mamba layers have more dispersed selection than Mamba-MLP. \item Authors' interpretation: local retrieval frees recurrence for other structure. \end{itemize} \papersource{Paper, Fig. 5, PDF p. 10; entropy definitions in Appendix E, p. 23. 100 GSM8K samples. Observational evidence.} \note{The left plot compares attention entropy between Samba and Mistral. The right compares a normalized gate statistic between Samba and Mamba-MLP. Selection entropy normalizes Delta over time per channel, rather than measuring how much information the state contains. These patterns support the specialization interpretation, but do not causally identify the function of each layer.} \end{frame} \begin{frame}{Three kinds of long-context evidence must stay separate} \begin{center} \begin{tabular}{>{\raggedright\arraybackslash}p{0.24\linewidth}>{\raggedright\arraybackslash}p{0.29\linewidth}>{\raggedright\arraybackslash}p{0.34\linewidth}} \toprule Capability & Evidence & Practical implication \\ \midrule Streaming computation & Fixed state + 2K KV window & History cache can remain bounded \\[2mm] Language prediction & 4K training; Proof-Pile up to 1M & Useful prediction survives long input \\[2mm] Exact retrieval & Tuned Passkey to 256K; harder Phonebook & Retention depends on task and memory load \\ \bottomrule \end{tabular} \end{center} \begin{block}{Research question} How can ordinary pretraining produce retrieval that transfers across tasks and item counts? \end{block} \papersource{Synthesis of Secs. 2--3 and Appendix H, PDF pp. 2--9, 25.} \note{The paper demonstrates that simple hybridization can improve quality while maintaining efficient long streams. Its central limitation is concrete: long-context prediction does not automatically supply arbitrary exact retrieval, and single-target tuning transfers poorly to many-item lookup. This suggests testing retrieval across both distance and item count, rather than optimizing a single maximum context-length headline.} \end{frame} \begin{frame}{Paper and discussion pointers} \textbf{Liliang Ren, Yang Liu, Yadong Lu, Yelong Shen, Chen Liang, Weizhu Chen.} \medskip \emph{Samba: Simple Hybrid State Space Models for Efficient Unlimited Context Language Modeling.} ICLR 2025; arXiv:2406.07522v3. \medskip \href{https://arxiv.org/abs/2406.07522v3}{arxiv.org/abs/2406.07522v3}\\ \href{https://github.com/microsoft/Samba}{github.com/microsoft/Samba} \begin{itemize} \item Architecture and recurrence: Fig. 1, PDF pp. 2--4. \item Matched-data quality: Tables 1, 2, 7, 8. \item Extrapolation and recall: Figs. 2--4, 6--8. \end{itemize} \papersource{Figures in this presentation are the authors' originals. Tables select and reformat reported entries.} \note{End of the main presentation, approximately 25 minutes. Backup slides cover the June post-training comparison, head allocation, training-length and short-convolution ablations, the Passkey learning curves, entropy definitions, and exact architecture dimensions.} \end{frame} \appendix \begin{frame}{Backup: June post-training is a separate comparison} \begin{center} \begin{tabular}{lrr} \toprule Chat-mode evaluation & Samba (June) 3.8B & Phi3 (June) 3.8B \\ \midrule MMLU, 5-shot & 69.0 & 67.2 \\ MMLU-Pro, zero-shot CoT & 47.9 & 46.5 \\ ARC-C, 10-shot & 87.8 & 86.8 \\ GPQA, zero-shot CoT & 29.5 & 29.0 \\ GSM8K, 8-shot CoT & 86.4 & 84.8 \\ HumanEval, zero-shot & 70.1 & 66.5 \\ MBPP, 3-shot & 71.7 & 70.0 \\ \midrule Average & \textbf{66.1} & 64.4 \\ \bottomrule \end{tabular} \end{center} Same data and recipe within this pair; do not merge with the original IT results. \papersource{Paper, Table 8, PDF p. 21.} \end{frame} \begin{frame}{Backup: fewer KV heads work well in the tested small models} \begin{center} \begin{tabular}{lrrrr} \toprule Configuration & Query heads & KV heads & Head dim. & PPL @4K \\ \midrule SWA & 12 & 2 & 128 & 11.11 \\ SWA & 6 & 1 & 256 & 11.09 \\ SWA & 12 & 1 & 128 & \textbf{10.89} \\ SWA & 12 & 4 & 128 & 11.11 \\ \midrule Samba & 12 & 2 & 128 & 10.09 \\ Samba & 6 & 1 & 256 & \textbf{9.99} \\ Samba & 12 & 1 & 128 & 10.07 \\ Samba & 12 & 4 & 128 & 10.02 \\ \bottomrule \end{tabular} \end{center} MLP width is increased to offset reduced KV parameters; about 430M total parameters. \papersource{Paper, Table 6, PDF p. 10. Best rows among the tested configurations, not a universal head-count rule.} \end{frame} \begin{frame}{Backup: longer training sequences do not always help} \begin{center} \begin{tabular}{rrrr} \toprule Training length & Batch size & PPL @4K & PPL @16K \\ \midrule 2,048 & 1,024 & 38.12 & 357.32 \\[1mm] 4,096 & 512 & \textbf{11.16} & \textbf{10.61} \\[1mm] 8,192 & 256 & 11.26 & 10.69 \\[1mm] 16,384 & 128 & 11.63 & 11.02 \\[1mm] 32,768 & 64 & 12.46 & 11.86 \\ \bottomrule \end{tabular} \end{center} \begin{itemize} \item Llama-2-SWA 438M; fixed window 2,048; about 2M tokens per step. \item Training length $2w$ is best here; length and batch size change together. \item At length $w$, SWA behaves as full attention throughout training. \end{itemize} \papersource{Paper, Appendix D, Table 9, PDF p. 22.} \end{frame} \begin{frame}{Backup: short convolution improves quality at a speed cost} \begin{center} \begin{tabular}{lrrr} \toprule Model & PPL @4K & PPL @16K & Speed ($10^5$ tokens/s) \\ \midrule Llama-2-SWA & 11.12 & 10.57 & 4.96 \\ \quad + short convolution & 10.83 & 10.31 & 4.69 \\ Sliding GLA & 10.43 & 9.92 & 4.94 \\ \quad + short convolution & 10.39 & 9.87 & 4.44 \\ Sliding RetNet & 10.38 & 9.87 & 4.32 \\ \quad + short convolution & 10.25 & 9.74 & 3.80 \\ \midrule Samba (Table 3 reference) & 10.06 & 9.57 & 4.46 \\ \bottomrule \end{tabular} \end{center} SC is added to q/k/v; in hybrids, only the linear-attention layers receive it. \papersource{Paper, Appendix D, Table 10, PDF p. 23; Table 3, p. 6. About 430M, measured on 8 A100 GPUs.} \end{frame} \begin{frame}{Backup: similar training loss can hide different recall strategies} \begin{columns}[T] \column{0.49\textwidth} \centering\textbf{4K training loss} \figfit[0.44\textheight]{figures/passkey-loss.pdf} \column{0.49\textwidth} \centering\textbf{Long-context retrieval accuracy} \figfit[0.44\textheight]{figures/passkey-learning.pdf} \end{columns} \begin{itemize} \item Both training losses approach zero quickly. \item Samba approaches perfect long-range recall around step 150. \item AdamW, peak LR $10^{-4}$, batch 2,048, 250 warm-up steps. \end{itemize} \papersource{Paper, Figs. 7--8, PDF pp. 21--22; Appendix C. Fig. 8 caption specifies 256K; protocol also describes seven lengths.} \end{frame} \begin{frame}{Backup: selection entropy measures normalized gates} For $h$ heads, sequence length $n$, and generation length $l$: \[ \mathcal H_a=-\frac{1}{lh}\sum_{i=1}^{h}\sum_{j=n-l+1}^{n} \sum_{k=1}^{n}A_{ijk}\log A_{ijk}. \] For Mamba, normalize each channel's gate over \emph{time}: \[ \Delta'_{ij}=\frac{\Delta_{ij}}{\sum_{r=1}^{n}\Delta_{rj}},\qquad \mathcal H_s=-\frac{1}{d_e}\sum_{j=1}^{d_e}\sum_{i=1}^{n} \Delta'_{ij}\log\Delta'_{ij}. \] \begin{itemize} \item $A_{ijk}$ here denotes attention probabilities, not the Mamba rate parameter. \item $\mathcal H_s$ measures temporal gate dispersion, not stored information capacity. \end{itemize} \papersource{Paper, Appendix E, PDF p. 23.} \end{frame} \begin{frame}{Backup: architecture dimensions across scales} \begin{center} \begin{tabular}{l@{\hspace{7mm}}r@{\hspace{5mm}}r@{\hspace{5mm}}r@{\hspace{5mm}}r} \toprule Samba & 421M & 1.3B & 1.7B & 3.8B \\ \midrule Total sublayers $N$ & 24 & 36 & 48 & 64 \\ Model width $d_m$ & 1536 & 2304 & 2048 & 2816 \\ MLP width $d_p$ & 4096 & 6144 & 8196 & 9984 \\ Query heads & 12 & 18 & 32 & 11 \\ KV heads & 12 & 18 & 4 & 1 \\ Vocabulary size & 32000 & 32000 & 50304 & 32064 \\ \bottomrule \end{tabular} \end{center} \begin{itemize} \item All: $d_e=2d_m$, $d_s=16$, $d_r=d_m/16$, convolution width 4. \item All: training length 4,096, attention window 2,048. \item Pure Mamba 1.8B is listed as 64 layers in Table 11, versus 48 in Sec. 2.1. \end{itemize} \papersource{Paper, Tables 11--12, PDF p. 24; Sec. 2.1.1, p. 3. Detailed baseline configuration follows Table 11.} \end{frame} \end{document}