% Based on the arxiv-paper-reader Beamer template.
% Build: python3 build_slides.py slides.en.tex
\documentclass[aspectratio=169,11pt]{beamer}
\usepackage[T1]{fontenc}
\usepackage[utf8]{inputenc}
\usepackage{amsmath,amssymb,graphicx,booktabs}
\definecolor{AcademicBlue}{RGB}{26,57,86}
\definecolor{AcademicTeal}{RGB}{0,105,112}
\setbeamercolor{structure}{fg=AcademicBlue}
\setbeamercolor{frametitle}{fg=AcademicBlue,bg=white}
\setbeamercolor{title}{fg=AcademicBlue}
\setbeamercolor{alerted text}{fg=AcademicTeal}
\setbeamercolor{block title}{fg=white,bg=AcademicBlue}
\setbeamercolor{block body}{fg=black,bg=AcademicBlue!5}
\setbeamertemplate{navigation symbols}{}
\setbeamertemplate{itemize items}[circle]
\setbeamertemplate{footline}{\hfill\usebeamerfont{page number in head/foot}\insertframenumber\,/\,\inserttotalframenumber\hspace{4mm}\vspace{3mm}}
\setbeamersize{text margin left=8mm,text margin right=8mm}
\setbeamerfont{frametitle}{size=\Large,series=\bfseries}
\setbeamerfont{title}{size=\LARGE,series=\bfseries}
\newcommand{\papersource}[1]{\par\smallskip{\scriptsize\color{gray}#1\par}}
\newcommand{\R}{\mathbb R}
\newcommand{\E}{\mathcal E}
\newcommand{\AT}{\mathcal L_{\mathrm{AT}}}
\title[LoLCATs]{LoLCATs: On Low-Rank Linearizing\ of Large Language Models}
\subtitle{Paper discussion: transfer local functions, then adapt the model}
\author{\small Paper by Michael Zhang, Simran Arora, Rahul Chalamala, Alan Wu,\\
Benjamin Spector, Aaryan Singhal, Krithik Ramesh, Christopher R\'e}
\date{\small arXiv:2410.10254v3 --- 5 March 2025\quad (ICLR 2025)\\
Reading prepared 28 September 2026\quad |\quad 25-minute main talk + backup}

\begin{document}

\begin{frame}
\titlepage
\vspace{-3mm}
\begin{center}\alert{Low-cost conversion is feasible; capability preservation remains selective.}\end{center}
\papersource{Independent reading presentation. Source: \url{https://arxiv.org/abs/2410.10254v3}.}
\end{frame}

\begin{frame}{The problem is the cost of converting a strong LLM}
\begin{columns}[T]
\begin{column}{0.48\textwidth}
\begin{block}{What we already have}
A pretrained Transformer with useful knowledge and language abilities.
\end{block}
\begin{itemize}
\item Full attention: growing KV cache.
\item Linear attention: fixed-size recurrent state.
\item A direct swap can break model quality.
\end{itemize}
\end{column}
\begin{column}{0.48\textwidth}
\begin{block}{What LoLCATs asks}
Can a close local approximation make cheap global adaptation sufficient?
\end{block}
\begin{itemize}
\item Preserve useful pretrained computation.
\item Train few additional parameters.
\item Scale conversion up to 405B.
\end{itemize}
\end{column}
\end{columns}
\par\medskip
Conversion token counts exclude the original model's pretraining investment.
\papersource{Paper, Secs. 1--2, PDF pp. 1--4.}
\end{frame}

\begin{frame}{First transfer attention; then adapt the whole network}
\centering
\includegraphics[width=0.98\linewidth,height=0.64\textheight,keepaspectratio]{figures/fig01-framework.pdf}
\begin{block}{Two different training targets}
Stage 1: attention output MSE.\qquad Stage 2: next-token cross-entropy.
\end{block}
\papersource{Original Fig. 1, PDF p. 2; Sec. 3.1, pp. 4--5. Frozen pretrained weights are reused.}
\end{frame}

\begin{frame}{Linear attention compresses history into a recurrent state}
For one head, $q_n,k_n,v_n\in\R^d$ and $\phi_q,\phi_k:\R^d\to\R^F$:
\[
S_n=S_{n-1}+\phi_k(k_n)v_n^\top,\qquad
z_n=z_{n-1}+\phi_k(k_n)
\]
\[
\boxed{\hat y_n=\frac{\phi_q(q_n)^\top S_n}{\phi_q(q_n)^\top z_n}}
\]
\begin{itemize}
\item $S_n\in\R^{F\times d}$ stores key--value associations; $z_n\in\R^F$ normalizes.
\item Fixed $F,d$: $O(Fd)$ attention work per generated token.
\item Attention state is independent of history length; model weights remain.
\end{itemize}
\papersource{Paper, Eqs. (2)--(3), PDF p. 4. $F$ denotes the final feature dimension in this presentation.}
\end{frame}

\begin{frame}{Output matching calibrates layers on teacher inputs}
\[
\AT=\operatorname{mean}_{\text{batch, layer, head, token, channel}}
\bigl(y-\hat y\bigr)^2
\]
\begin{itemize}
\item Freeze pretrained projections; train feature maps and mixing terms.
\item Compute teacher and student attention from the same layer input.
\item Pass \alert{teacher outputs} to subsequent layers during transfer.
\item Match $N\times d$ outputs, without materializing all $N\times N$ weights.
\end{itemize}
\begin{block}{What output matching supervises}
It encourages functional agreement on sampled inputs. Matching $AV$ does not uniquely identify the attention matrix $A$.
\end{block}
\papersource{Paper, Sec. 3.1 and Eq. (5), PDF pp. 4--5. Explicit mean is a clarified notation. Teacher attention still costs compute.}
\end{frame}

\begin{frame}{LoRA adjusts the converted model's own computation}
\[
W'=W+\frac{\alpha}{r}BA,\qquad
\mathcal L_{\mathrm{LM}}=-\sum_t\log P_{\Theta'}(u_{t+1}\mid u_{1:t})
\]
\begin{itemize}
\item Add LoRA to $W_q,W_k,W_v,W_o$; freeze original weights and feature maps.
\item Default 7B/8B recipe: rank $r=8$, $\alpha=16$, dropout $0$.
\item The student now supplies all hidden states: train against accumulated error.
\end{itemize}
\begin{block}{``Low-rank'' refers to the update}
The full pretrained matrices remain. This is parameter-efficient adaptation, not a low-rank compression of the entire LLM.
\end{block}
\papersource{Paper, Sec. 3.1, PDF p. 5; App. A.1 and B.4.1, pp. 18--19, 23--25.}
\end{frame}

\begin{frame}{Neither transfer nor LoRA alone is sufficient here}
\centering
\begin{tabular}{lrr}
\toprule
Llama 3 8B, pure linear attention & T2R & Hedgehog\\
\midrule
Random swap, before adjustment & 1539.39 & 2448.01\\
Transfer only, before LoRA & 79.33 & 60.86\\
No transfer + LoRA & 16.05 & 9.02\\
Transfer + LoRA & \alert{4.11} & \alert{3.90}\\
\bottomrule
\end{tabular}
\par\medskip
\begin{itemize}
\item Alpaca validation perplexity $\downarrow$; final rows use equal total steps.
\item Transfer + LoRA: 2 + 2 epochs; no transfer: 4 LoRA epochs.
\item Pure linear attention still struggles on downstream MMLU.
\end{itemize}
\papersource{Selected results from Table 2, PDF p. 6. Pure Hedgehog MMLU: Table 3, p. 9.}
\end{frame}

\begin{frame}{Keep local softmax; compress only the older history}
Let $\mathcal W_n$ be the recent window, and $c_n$ its maximum attention logit.
\[
U_n=\sum_{i\in\mathcal W_n}\gamma e^{q_n^\top k_i/\sqrt d-c_n}v_i,
\quad D_n=\sum_{i\in\mathcal W_n}\gamma e^{q_n^\top k_i/\sqrt d-c_n}
\]
\[
\boxed{\hat y_n=\frac{U_n+\phi_q(q_n)^\top S_{n-w}}
{D_n+\phi_q(q_n)^\top z_{n-w}}}
\]
\begin{itemize}
\item $\gamma$: learned mixing term; $S,z$: history outside the local window.
\item Combine unnormalized contributions, then normalize jointly.
\item Evicted local KV entries enter the recurrent state.
\end{itemize}
\papersource{Paper, Eq. (6), PDF p. 7, reorganized for presentation. Empty historical state is zero.}
\end{frame}

\begin{frame}{The deployed recipe uses a GPU-friendly terraced window}
\begin{columns}[T]
\begin{column}{0.33\textwidth}
\centering
\includegraphics[width=\linewidth,height=0.52\textheight,keepaspectratio]{figures/fig09-sliding.png}
\small Standard sliding window
\end{column}
\begin{column}{0.33\textwidth}
\centering
\includegraphics[width=\linewidth,height=0.52\textheight,keepaspectratio]{figures/fig09-terrace.png}
\small TK terraced window
\end{column}
\begin{column}{0.30\textwidth}
\small
Rows: queries\\Columns: keys
\par\medskip
Blue: local softmax\\Cyan: linear attention
\par\medskip
Default tile size: 64. Local coverage varies with query position.
\end{column}
\end{columns}
\par\medskip
Apply RoPE \emph{before} Hedgehog features. A 64-wide projection produces 128 features after positive/negative softmax concatenation.
\papersource{Original Fig. 9, PDF p. 18; Table 1, p. 5; Table 7, p. 19.}
\end{frame}

\begin{frame}{A local window reduces the hardest attention errors}
\centering
\includegraphics[width=0.80\linewidth,height=0.57\textheight,keepaspectratio]{figures/fig07-layer-mse.png}
\begin{itemize}
\item Horizontal axis: layer; vertical axis: attention output MSE $\downarrow$.
\item The local + linear layer reduces late-layer error substantially.
\item Lower MSE correlates with quality; it is not a universal task guarantee.
\end{itemize}
\papersource{Original Fig. 7, right panel, PDF p. 8; related entropy analysis: Figs. 5--6, pp. 7--8.}
\end{frame}

\begin{frame}{The small-model recipe needs little additional training}
\begin{columns}[T]
\begin{column}{0.49\textwidth}
\begin{block}{7B / 8B setup}
\begin{itemize}
\item Base Mistral 7B / Llama 3 8B.
\item About 50K cleaned Alpaca samples.
\item Packed length 1024; global batch 8.
\item AdamW; BF16; terrace size 64.
\end{itemize}
\end{block}
\end{column}
\begin{column}{0.47\textwidth}
\begin{block}{Training budget}
\begin{itemize}
\item Transfer LR: $10^{-2}$.
\item LoRA LR: $10^{-4}$; rank 8.
\item Reported conversion tokens: 40M.
\item One 40GB A100: about 6.5 hours.
\end{itemize}
\end{block}
\end{column}
\end{columns}
\par\medskip
\alert{Budget caveat:} Table 7's epoch/token entries conflict with the 40M total stated elsewhere. Exact reproduction requires clarification.
\papersource{Paper, Sec. 3.2, p. 6; App. A.1, pp. 18--19; App. B.5.1, p. 26.}
\end{frame}

\begin{frame}{Zero-shot averages recover; MMLU retains a large gap}
\centering
\begin{tabular}{lrrr}
\toprule
Model & 5-task avg. & MMLU & 6-task avg.\\
\midrule
Mistral 7B & 74.4 & 62.4 & 72.4\\
\quad SUPRA (100B conversion tokens) & 69.9 & 34.2 & 64.0\\
\quad LoLCATs (40M reported) & \alert{74.5} & \alert{51.4} & 70.7\\
\midrule
Llama 3 8B & 73.1 & 66.6 & 72.0\\
\quad Mamba2-Llama 3 (20B tokens) & 65.6 & 43.2 & 61.9\\
\quad LoLCATs (40M reported) & \alert{74.2} & \alert{52.8} & 70.7\\
\bottomrule
\end{tabular}
\par\medskip
MMLU gains over these conversion baselines: \alert{+17.2} and \alert{+9.6} points.\\
Remaining MMLU gaps to the original models: 11.0 and 13.8 points.
\papersource{Selected columns of Table 3, PDF p. 9. Scores in percent, higher is better. MMLU is 5-shot; other tasks are zero-shot.}
\end{frame}

\begin{frame}{Transfer and the local window work best together}
\centering
\begin{tabular}{lrrrr}
\toprule
& \multicolumn{2}{c}{Hedgehog} & \multicolumn{2}{c}{T2R}\\
Setting & Avg. & MMLU & Avg. & MMLU\\
\midrule
Swap + adjust & 44.20 & 23.80 & 38.84 & 23.20\\
+ transfer & 55.32 & 23.80 & 54.83 & 23.10\\
+ window only & 68.78 & 45.80 & 39.52 & 23.80\\
+ transfer + window & \alert{70.66} & \alert{52.77} & \alert{68.28} & \alert{40.70}\\
\bottomrule
\end{tabular}
\par\medskip
\begin{itemize}
\item Pure linear transfer helps the average, but barely helps MMLU.
\item A window alone is insufficient for T2R under this protocol.
\item Three-seed LoLCATs MMLU: $52.74\pm0.64$ (mean $\pm$ SD).
\end{itemize}
\papersource{Table 5, PDF p. 11; Table 11, p. 22. Llama 3 8B; Avg. includes all six tasks.}
\end{frame}

\begin{frame}{Higher feasible batch sizes unlock throughput gains}
\begin{columns}[T]
\begin{column}{0.55\textwidth}
\includegraphics[width=\linewidth,height=0.62\textheight,keepaspectratio]{figures/fig08-throughput.png}
\end{column}
\begin{column}{0.41\textwidth}
\small
Llama 3 8B, one 80GB H100.\\
128-token prompt; 4096 generated tokens.
\par\medskip
\begin{itemize}
\item FA2 OOM at batch 64.
\item LoLCATs: batch 2048, about 3000 tokens/s.
\item Hatched FA2 bars are estimates beyond OOM.
\item \alert{FA2 is faster at small shared batch sizes.}
\end{itemize}
\end{column}
\end{columns}
\papersource{Original Fig. 8, left panel, and Sec. 4.2, PDF p. 10. About 3x peak throughput is not a single-request latency claim.}
\end{frame}

\begin{frame}{Attention transfer improves 70B and 405B MMLU}
\centering
\begin{tabular}{lrrrr}
\toprule
Llama 3.1 & Original & No transfer & LoLCATs & Gap closed\\
\midrule
8B & 66.11 & 51.44 & 54.88 & 23.4\%\\
70B & 78.80 & 28.74 & \alert{67.70} & \alert{77.8\%}\\
405B & 82.98 & 33.86 & \alert{72.20} & \alert{78.1\%}\\
\bottomrule
\end{tabular}
\[
\text{Gap closed}=\frac{\text{LoLCATs}-\text{No transfer}}
{\text{Original}-\text{No transfer}}
\]
\begin{itemize}
\item MMLU 5-shot accuracy (\%). No-transfer baseline uses the same hybrid layer.
\item Remaining original-model gaps: 11.10 points (70B), 10.78 (405B).
\end{itemize}
\papersource{Table 6, PDF p. 11; baseline protocols in App. A.2--A.3, pp. 18--21. Gap percentages calculated from table values.}
\end{frame}

\begin{frame}{Block-wise transfer helps at 405B, with a storage cost}
\begin{columns}[T]
\begin{column}{0.48\textwidth}
\begin{block}{405B validation perplexity}
Joint transfer, 126 layers: \textbf{4.23}\\[2mm]
14 blocks of 9 layers: \alert{\textbf{3.21}}
\end{block}
\begin{itemize}
\item Both evaluated after LoRA adjustment.
\item Joint and block-wise training are similar at 8B / 70B.
\end{itemize}
\end{column}
\begin{column}{0.48\textwidth}
Teacher forcing permits precomputed block inputs.
\[
\text{Disk bytes}\approx 2TD\frac{M}{b}
\]
\small $T$: tokens; $D$: model width; $M$: layers; $b$: layers per block; 16-bit states.
\par\medskip
More blocks enable independent training, but require more stored hidden states.
\end{column}
\end{columns}
\papersource{Sec. 3.3.2, PDF pp. 8--9; Table 24, p. 35. Disk storage is a training cost, distinct from inference KV memory.}
\end{frame}

\begin{frame}{The 405B run adds attention-map supervision}
\begin{itemize}
\item Stage 1: \alert{$1000\times$ output MSE + $1\times$ attention-map CE}.
\item Stage 2: LoRA rank 4, alpha 8, dropout 0.5; FP16.
\item Data: 20M RedPajama tokens per stage; packed length 1024.
\item Transfer: 14 blocks, about 5 hours on one H100 per block.
\item LoRA: 16 hours on 24 H100s; about 454 GPU-hours in total.
\end{itemize}
\begin{block}{Interpretation}
The 405B result does not establish that output MSE alone suffices at every scale. The baseline also has a different allocation of training stages.
\end{block}
\papersource{App. A.3 and Table 9, PDF pp. 20--21. GPU-hours derived as 14 x 5 + 24 x 16; preparation/I/O not separately quantified.}
\end{frame}

\begin{frame}{Retrieval is not automatically preserved by conversion}
\centering
\begin{tabular}{lc}
\toprule
8K passkey retrieval, Llama 3 8B & Exact match\\
\midrule
LoLCATs, packed Alpaca conversion data & \textbf{0\% in every position bin}\\
LoLCATs, dedicated passkey conversion data & \alert{100\% in every position bin}\\
\bottomrule
\end{tabular}
\par\medskip
\begin{itemize}
\item Dedicated recipe: 10K examples, approximately 8192 tokens each.
\item Task-trained LoLCATs still fails at 10240 tokens in the reported plot.
\item This demonstrates learnability of a narrow task, not default long-context retention.
\end{itemize}
\papersource{Table 21, PDF p. 28; Fig. 10, p. 29. Data length and amount differ from the default conversion recipe.}
\end{frame}

\begin{frame}{Data choices trade general scores for targeted abilities}
\centering
\begin{tabular}{lrr}
\toprule
Llama 3 8B conversion data & 5-task avg. & MMLU\\
\midrule
Alpaca & \alert{74.2} & 52.8\\
Random RedPajama & 72.1 & 55.2\\
RedPajama, high effective sequence length & 70.9 & \alert{56.5}\\
\bottomrule
\end{tabular}
\par\medskip
\begin{itemize}
\item Packing short samples does not create genuine long-range dependencies.
\item Adding multiple-choice CQA to Alpaca: MMLU 52.8 $\to$ 54.5.
\item CQA alone: 43.9. Targeted format alone is insufficient.
\end{itemize}
\papersource{Tables 19--20, PDF p. 27; Table 22, p. 30. Effective sequence length is an attention-distance statistic.}
\end{frame}

\begin{frame}{The evidence supports efficient conversion within limits}
\begin{enumerate}
\item \textbf{Quality:} near-recovery on five common tasks; persistent MMLU gaps and data-sensitive retrieval.
\par\medskip
\item \textbf{Compute:} small additional training cost; original pretraining is inherited.
\par\medskip
\item \textbf{Inference:} strong batch-capacity benefits; speed depends on batch, workload, and implementation.
\par\medskip
\item \textbf{Scale:} 405B conversion works with block training and an extended distillation objective.
\end{enumerate}
\papersource{Synthesis of Tables 3, 6, 9, 21; Fig. 8; paper limitations, PDF pp. 11--12.}
\end{frame}

\begin{frame}{Transfer the function before adapting the network}
\begin{block}{Main lesson}
Reducing the architectural mismatch locally can make a small global parameter update surprisingly effective.
\end{block}
\begin{itemize}
\item Local softmax + recurrent history is easier to transfer than pure linear attention.
\item Conversion data should exercise the capabilities we need to retain.
\item Judge the converted model on quality, memory, and the actual serving workload.
\end{itemize}
\par\medskip
\alert{Discussion:} Can normalized layer losses reduce the need for cached block inputs at 405B?
\papersource{Reader synthesis and open question. Full paper: \url{https://arxiv.org/abs/2410.10254v3}. Main talk ends here.}
\end{frame}

\appendix

\begin{frame}{Backup: evaluation metrics and comparison roles}
\begin{itemize}
\item PiQA, ARC-easy, Winogrande: zero-shot accuracy.
\item ARC-challenge, HellaSwag: zero-shot length-normalized accuracy.
\item MMLU: 5-shot accuracy. Six-task average includes it; five-task average excludes it.
\item Prior conversion numbers are partly cited from original papers; not all uniformly rerun.
\item Comparisons to models pretrained from scratch inherit different data and training investments.
\end{itemize}
\par\medskip
405B small improvements over the original model on non-MMLU tasks lack reported multi-seed uncertainty.
\papersource{Tables 3--6, PDF pp. 9--11; multi-seed evidence is limited to Table 11, p. 22.}
\end{frame}

\begin{frame}{Backup: small LoRA updates can already be enough}
\begin{columns}[T]
\begin{column}{0.45\textwidth}
\centering
\begin{tabular}{lrr}
\toprule
Rank & 5-task avg. & MMLU\\
\midrule
4 & 74.5 & 50.1\\
8 & 74.2 & 52.8\\
128 & 73.0 & 53.4\\
256 & 72.7 & 52.1\\
Full FT & 73.7 & 52.1\\
\bottomrule
\end{tabular}
\end{column}
\begin{column}{0.51\textwidth}
Five-task average, rank 8:
\begin{itemize}
\item $W_q$ only: 72.68
\item $W_k$ only: 72.29
\item $W_v$ only: \alert{74.19}
\item $W_o$ only: \alert{74.09}
\item All projections: 74.24
\end{itemize}
\end{column}
\end{columns}
\par\medskip
Values and output projections recover much of the measured quality. Projection ablation does not report MMLU.
\papersource{Tables 16--17, PDF pp. 25--26. Full FT uses a different LR. Inconsistent six-task averages are not reproduced.}
\end{frame}

\begin{frame}{Backup: full-attention hybrids recover more MMLU}
\centering
\begin{tabular}{lrr}
\toprule
Llama 3 8B architecture & Conversion tokens & MMLU\\
\midrule
Original full attention & 0 & 66.6\\
First 16 layers remain full attention & 20M & \alert{65.8}\\
All layers converted & 40M reported & 52.8\\
\bottomrule
\end{tabular}
\par\medskip
\begin{itemize}
\item The 50\% hybrid uses attention transfer only, without Stage 2 LoRA.
\item Remaining full-attention layers still require a growing KV cache.
\item At 1B scale: LoLCATs also nearly recovers five-task averages, while MMLU declines (Llama 3.2 1B: 31.9 $\to$ 27.3).
\end{itemize}
\papersource{Tables 12--15, PDF pp. 23--24.}
\end{frame}

\begin{frame}{Backup: the first transfer steps have high value}
\centering
\setlength{\tabcolsep}{10pt}
\begin{tabular}{lrrr}
\toprule
Transfer steps & 0 & 200 & 1800\\
\midrule
Alpaca validation PPL $\downarrow$ & 3.211 & 2.802 & 2.776\\
RedPajama validation PPL $\downarrow$ & 61.305 & 10.459 & 9.358\\
Five-task average $\uparrow$ & 56.86 & 73.68 & 73.66\\
\bottomrule
\end{tabular}
\par\medskip
\begin{itemize}
\item Each run uses 2000 subsequent LoRA steps.
\item More transfer can improve PPL without monotonic task-score gains.
\item A separate larger-RedPajama study also finds this distinction.
\end{itemize}
\papersource{Table 23 and Figs. 12--13, PDF p. 31. Llama 3 8B.}
\end{frame}

\begin{frame}{Backup: training dynamics are suggestive, not causal proof}
\begin{itemize}
\item Fig. 14: LoRA can further reduce attention-output MSE, especially where the initial mismatch is large.
\par\medskip
\item \alert{Diagnostic exception:} those LoRA updates optimize MSE directly, not the default next-token objective.
\par\medskip
\item Figs. 15--17: with next-token training, transferred models' value/output updates look more like the softmax model's updates.
\par\medskip
\item Selected attention heatmaps show recovery of some distant interactions, but do not prove exact attention-weight recovery.
\end{itemize}
\papersource{App. B.7.1--B.7.2, PDF pp. 32--34; App. E.2, pp. 50--58 (examples inspected selectively).}
\end{frame}

\begin{frame}{Backup: the prefill kernel exploits GPU locality}
\begin{enumerate}
\item Load 64-token query/key/value tiles.
\par\medskip
\item One warpgroup handles local attention and memory transfers; another handles linear attention and state updates.
\par\medskip
\item Use tensor-core operations and keep recurrent state in registers.
\par\medskip
\item Combine branch contributions, normalize, and write outputs.
\end{enumerate}
\par\medskip
This is a \alert{prefill} implementation. Algorithmic complexity alone does not establish end-to-end speed.
\papersource{App. C.2, Algorithm 3, PDF pp. 43--44. Schematic summary; not executable kernel code.}
\end{frame}

\begin{frame}{Backup: several paper details need careful reproduction}
\begin{itemize}
\item Token budget: Table 7 implies 60M by its epoch entries; the main text reports 40M.
\item MMLU relative gains in Sec. 4.1 do not match Table 3. Use the verified +17.2 / +9.6 percentage-point gains.
\item Some appendix averages disagree with the listed task scores.
\item MSE, block-loss, and ESL equations contain index ambiguities.
\item Appendix C code is illustrative and includes incomplete or erroneous details; it should not be executed verbatim.
\end{itemize}
\papersource{PDF pp. 5, 8--9, 19, 25--26, 29, 37--42. Detailed checks and adopted interpretations are in the Chinese notes.}
\end{frame}

\begin{frame}{Backup: source material and a practical replication order}
\textbf{Michael Zhang et al.} LoLCATs: On Low-Rank Linearizing of Large Language Models. arXiv:2410.10254v3, 2025.
\par\medskip
\url{https://arxiv.org/abs/2410.10254v3}\\
Author-linked code: \url{https://github.com/HazyResearch/lolcats}
\par\medskip
\begin{enumerate}
\item Fix the model, tokenizer, attention variant, and evaluation settings.
\item Reproduce the two-stage ablation before scaling up.
\item Test MMLU, held-out PPL, and retrieval separately.
\item Audit realized token counts, masks, causal state updates, and LoRA scaling.
\end{enumerate}
\papersource{This reading uses the matching author TeX and PDF. No model training or current repository audit was performed.}
\end{frame}

\end{document}