% Reading presentation. Compile locally with pdfLaTeX via build_slides.py. \documentclass[aspectratio=169,11pt]{beamer} \usepackage[T1]{fontenc} \usepackage[utf8]{inputenc} \usepackage{amsmath,amssymb,graphicx,booktabs} \definecolor{AcademicBlue}{RGB}{26,57,86} \definecolor{AcademicTeal}{RGB}{0,105,112} \setbeamercolor{structure}{fg=AcademicBlue} \setbeamercolor{frametitle}{fg=AcademicBlue,bg=white} \setbeamercolor{title}{fg=AcademicBlue} \setbeamercolor{alerted text}{fg=AcademicTeal} \setbeamercolor{block title}{fg=white,bg=AcademicBlue} \setbeamercolor{block body}{fg=black,bg=AcademicBlue!5} \setbeamertemplate{navigation symbols}{} \setbeamertemplate{itemize items}[circle] \setbeamertemplate{footline}{% \hfill\usebeamerfont{page number in head/foot}% \insertframenumber\,/\,\inserttotalframenumber\hspace{4mm}\vspace{3mm}} \setbeamersize{text margin left=8mm,text margin right=8mm} \setbeamerfont{frametitle}{size=\Large,series=\bfseries} \setbeamerfont{title}{size=\LARGE,series=\bfseries} \newcommand{\papersource}[1]{\par\smallskip{\scriptsize\color{gray}#1\par}} \newcommand{\fig}[2]{\includegraphics[width=\linewidth,height=#2,keepaspectratio]{figures/#1.pdf}} \newcommand{\F}{\mathrm{F}} \title[Puro-2B]{Puro-2B: Affordable From-Scratch Pretraining} \subtitle{Poor Lab's Qwen2-1.5B Trained on RTX 5090 within \$5090} \author{\small Paper by Kairong Luo, Jiarui Cui, Yaorui Yin,\\ Shengqi Chen et al.\\[3pt] \scriptsize Tsinghua University and Pengcheng Laboratory\\ \scriptsize Research paper discussion} \date{\small arXiv:2608.27370v2 \quad Reading: 14 October 2026} \begin{document} \begin{frame} \titlepage \begin{center} \alert{A complete training recipe makes pretraining experiments affordable.} \end{center} \end{frame} \begin{frame}{Two budgets support two different claims} \begin{center} \begin{tabular}{lrrr} \toprule Checkpoint & Total tokens & GPU cost & Avg15 $\uparrow$\\ \midrule Puro UD, half Phase 2 & 918.83B & \$4.37K & 55.54\\ Puro CMA, canonical & $\approx$1.40T & \$6.89K & 57.81\\ \midrule Qwen2-1.5B & & & 55.14\\ Qwen2.5-1.5B & & & 60.73\\ \bottomrule \end{tabular} \end{center} \begin{itemize} \item The \$4.4K result is an observed checkpoint, not only a fitted crossing. \item The best model uses \$6.9K and remains 2.91 points below Qwen2.5. \item Qwen2 is the comparison target; Puro starts from random initialization. \end{itemize} \papersource{Paper, Table 1, p.6; Tables 4--5, pp.21--22; Tables 18--19, p.53. Avg15 recomputed from the displayed task scores.} \end{frame} \begin{frame}{The headline is a marginal accelerator cost} \[ \text{GPU cost}=\text{active GPU-hours}\times\text{normalized hourly rate} \] \begin{block}{Canonical accounting} $(6{,}009.46+16{,}504.95)$ GPU-hours $\times$ \$0.3061/hour $\approx$ \textbf{\$6,891}. \end{block} \begin{itemize} \item Starts after training shards have been materialized. \item Excludes data processing, proxy runs, ablations, failed runs, SFT, evaluation, averaging operations, and research labor. \item The 5090 rate is a local amortization-based estimate, not a broadly available public cloud offer. \end{itemize} \papersource{Paper, Sec.2.2, p.8; Table 2, p.9; Appendix B, pp.38--40. Fixed paper exchange rate: 6.8067 CNY/USD.} \end{frame} \begin{frame}{A 2B Transformer is trained in two phases} \begin{itemize} \item Qwen3-1.7B architecture; untied embedding and LM head give about 2B parameters. \item Sequence length 4,096; global batch 1,536; micro-batch 2. \end{itemize} \begin{center} \begin{tabular}{lrr} \toprule & Phase 1 & Full Phase 2\\ \midrule Scheduled tokens & 438.84B & 959.99B\\ RTX 5090 GPUs & 24 & 96\\ TP / PP / DP & 1 / 2 / 12 & 1 / 4 / 24\\ Base LR trajectory & Power decay & Linear decay + CMA tail\\ Active time equivalent & 10.43 days & 7.16 days\\ \bottomrule \end{tabular} \end{center} The full run uses about 700 tokens per parameter: a data-rich regime for a compact model. \papersource{Paper, Sec.2.1, pp.6--8; Table 8, p.40; Appendix A, p.38.} \end{frame} \begin{frame}{Consumer hardware requires communication work} \begin{center} \begin{tabular}{lrr} \toprule Measured communication & Before & After\\ \midrule One-way P2P bandwidth & 31.5 GB/s & 56 GB/s\\ Point-to-point latency & 14.3 $\mu$s & 0.4 $\mu$s\\ 8-GPU AllReduce bus bandwidth & 14.75 GB/s & 27.34 GB/s\\ 24-GPU AllReduce, GDR enabled & 8.87 GB/s & 19.93 GB/s\\ \bottomrule \end{tabular} \end{center} \begin{itemize} \item 32 GB per GPU, no NVLink; 400 Gbps InfiniBand across nodes. \item Reported performance uses topology-specific driver modifications. \item GDR modification details are undisclosed; stock drivers give a slower communication path. \end{itemize} \papersource{Paper, Sec.3.1.2, pp.9--10. AllReduce numbers are nccl-tests busbw.} \end{frame} \begin{frame}{Parallelism follows the communication bottleneck} \begin{itemize} \item Use DP and PP; avoid frequent per-layer TP collectives. \item Give the expensive LM-head stage fewer Transformer layers. \item Distribute intact Muon matrices by memory footprint; fill remaining space with Adam states. \end{itemize} \begin{center} \begin{tabular}{lrr} \toprule & 24 GPUs & 96 GPUs\\ \midrule Pipeline layers & $18\mid10$ & $9\mid9\mid9\mid1$\\ Median TFLOP/s/GPU & 238 & 192\\ \bottomrule \end{tabular} \end{center} \[ P_{\rm eff}=\left(\frac{0.72}{419}+\frac{0.28}{209.5}\right)^{-1}\!\approx327, \quad \mathrm{MFU}_{\rm P1}\approx\frac{238}{327}=73\%. \] \papersource{Paper, Sec.3.1.3, pp.10--11. MFU uses a mixed-precision peak, not a BF16-only peak.} \end{frame} \begin{frame}{FP8 accelerates linear layers from initialization} \begin{center} \begin{tabular}{ll} \toprule Part of training & Representation\\ \midrule Linear-layer Fprop / Dgrad / Wgrad & E4M3 operands\\ Activation and activation-gradient scales & Groups of 128 values\\ Weight scales & $128\times128$ blocks\\ Residual flow, outputs, checkpoint weights & BF16\\ Master state and gradient accumulation & FP32\\ \bottomrule \end{tabular} \end{center} \begin{itemize} \item Local scales reduce the effect of outliers on quantization resolution. \item Power-of-two scales execute through the Blackwell MXFP8 path. \item About 72\% of theoretical work uses FP8; saved inputs also use less memory. \end{itemize} \papersource{Paper, Sec.3.2, p.11; Fig.14, p.43. Core attention retains BF16 arithmetic with FP32 softmax.} \end{frame} \begin{frame}{Proxy continuations guide data selection} \begin{columns}[c,onlytextwidth] \begin{column}{0.64\textwidth} \centering\fig{proxy_benchmark_protocol}{5.3cm} \end{column} \begin{column}{0.33\textwidth} \begin{itemize} \item Common 0.6B checkpoint after 86B base tokens. \item 39 candidate slices; about 8.4B continuation tokens each. \item Candidate fraction rises to 80\%. \item Capability profiles inform manually chosen mixture weights. \end{itemize} \end{column} \end{columns} \papersource{Paper, Fig.9, p.18; Appendix I, pp.60--63. Proxy tasks include C-Eval/CMMLU and differ from the final 15-task suite.} \end{frame} \begin{frame}{Phase 2 emphasizes mathematics and specialist data} \begin{center} \begin{tabular}{lrr} \toprule Domain share & Phase 1 & Phase 2 pool\\ \midrule English & 73.2\% & 59.4\%\\ Mathematics & 7.2\% & 18.3\%\\ Chinese & 11.7\% & 9.4\%\\ Code & 7.9\% & 11.5\%\\ Instruction-formatted & 0.0\% & 1.3\%\\ \bottomrule \end{tabular} \end{center} \[ \underbrace{938.06\mathrm{B}}_{\text{Phase 2 pool}}+ \underbrace{21.93\mathrm{B}}_{\text{Phase 1 replay}}= \underbrace{959.99\mathrm{B}}_{\text{Phase 2 schedule}}. \] The 43.87B transition includes replay and an early range already inside the Phase 2 pool. \papersource{Paper, Fig.8 / Table 3, p.18; Table 20, p.55. Semantic Code includes MegaMath-Code. Rounded shares may not sum to 100\%.} \end{frame} \begin{frame}{MuonH directly schedules the relative update size} For a selected matrix $W_t$ and a Muon update $u_t$: \[ \rho_t(W)=\frac{\eta_t\|u_t\|_F}{\|W_t\|_F} \qquad\text{(ordinary Muon; radial decay excluded)}. \] Fix $R=\|W_0\|_F$ and normalize $\widehat u_t=u_t/\|u_t\|_F$: \[ \widetilde W_{t+1}=W_t-\eta_t^H R\widehat u_t, \qquad W_{t+1}=R\frac{\widetilde W_{t+1}}{\|\widetilde W_{t+1}\|_F}. \] \begin{block}{Effective LR becomes explicit} The pre-projection displacement relative to the fixed radius is $\eta_t^H$. Production MuonH groups use $\eta_t^H=10\eta_t^{\rm base}$. \end{block} Other parameters use AdamW. The motivation is approximate scale invariance of selected attention/MLP matrices. \papersource{Paper, Eqs.(1)--(4), p.12; Sec.3.3.1, p.13; Table 9, p.40.} \end{frame} \begin{frame}{Matching effective LR nearly matches final loss} \begin{center}\fig{hyperball_trace_alignment}{4.35cm}\end{center} \[ \eta_t^{\rm aligned}=\rho_t^H\frac{\|W_t\|_F}{\|u_t\|_F} \quad\Rightarrow\quad L_{\rm aligned}\approx3.030,\quad L_{\rm MuonH}\approx3.029. \] Ordinary-LR Muon ends near 3.073. The aligned control uses no fixed-radius projection. \papersource{Paper, Fig.5, p.14; Eq.(5), p.13. Matched 170M BF16 diagnostic; effective LR traces use selected MLP down-projections.} \end{frame} \begin{frame}{Higher peaks favor longer learning-rate decay} \begin{center}\fig{wsd_decay_ratio}{3.95cm}\end{center} \begin{itemize} \item In 0.6B WSD sweeps, the competitive region shifts toward longer decay as effective peak LR rises. \item Production combines an open-ended Phase 1 power schedule with a long Phase 2 linear decay. \end{itemize} \papersource{Paper, Fig.6, pp.14--15; Appendix F.2, pp.51--52. One run per sweep setting; broad minima, not a precise monotone optimum.} \end{frame} \begin{frame}{CMA coordinates ordering, learning rate, and averaging} \begin{center}\fig{cma_flow}{4.1cm}\end{center} \begin{itemize} \item Align 376 source-local token-percentile buckets to preserve the mixture. \item Put more preferred scored data later; unscored sources remain randomly ordered. \item At step 218k, hold base LR at $4.08\times10^{-5}$; average six late checkpoints. \end{itemize} \papersource{Paper, Fig.7, p.16 (vector crop); Appendix H, pp.54--56. Scores are not compared across sources.} \end{frame} \begin{frame}{Averaging helps the selected constant-LR branch} \begin{center} \begin{tabular}{lrr} \toprule Phase 2 recipe & Final checkpoint & Model average\\ \midrule Uniform + decay (UD) & 56.00 & 55.57\\ Curriculum + decay (CD) & 57.17 & 57.18\\ Curriculum + constant tail at 215k & 55.64 & 56.80\\ Curriculum + constant tail at 218k & 57.12 & \textbf{57.81}\\ \bottomrule \end{tabular} \end{center} \begin{itemize} \item Curriculum improves the decay endpoint by about 1.17 points. \item Averaging alone does not improve the uniform trajectory. \item Canonical CMA is the selected 218k branch with six-checkpoint averaging. \end{itemize} \papersource{Paper, Table 19, p.53; Sec.3.4, p.16; Appendix H.3, p.56. Avg15 in percent. Fig.13b uses 55.99 for UD; the ledger uses 56.00.} \end{frame} \begin{frame}{The headline score gives each benchmark equal weight} \begin{center} \begin{tabular}{lll} \toprule Panel & Tasks & Evaluation\\ \midrule Math / code & GSM8K, MATH, MBPP, HumanEval & Generation\\ Complex reasoning & MMLU-Pro, BBH & Generation\\ Knowledge / commonsense & Nine remaining tasks & Answer likelihood\\ \bottomrule \end{tabular} \end{center} \[ P=\frac1{15}\sum_b s_b =\frac{4\,\mathrm{Avg4}+11\,\mathrm{Avg11}}{15}. \] \begin{itemize} \item Fixed base checkpoints, prompts, snapshots, greedy decoding, and parsers. \item Where applicable, select the better CF/MCF formulation separately for each model. \item Chinese proxy tasks are absent from this headline suite. \end{itemize} \papersource{Paper, Sec.4.1, pp.20--21; Appendix A, p.38. MBPP means sanitized-MBPP.} \end{frame} \begin{frame}{Broad capability improves, with uneven task trade-offs} \begin{center} \begin{tabular}{lrrr} \toprule Base model & Math/code & Reasoning/knowledge & Avg15\\ \midrule Qwen2-1.5B & 40.29 & 60.54 & 55.14\\ Qwen2.5-1.5B & 47.52 & 65.53 & 60.73\\ Instella-3B & 33.41 & 63.13 & 55.20\\ Yulan-Mini-2.4B & 52.11 & 61.24 & 58.80\\ SmolLM3-3B & 58.99 & 68.35 & 65.85\\ Puro UD, \$4.4K & 38.61 & 61.70 & 55.54\\ Puro CMA, \$6.9K & \textbf{43.50} & \textbf{63.02} & \textbf{57.81}\\ \bottomrule \end{tabular} \end{center} The \$4.4K model exceeds Qwen2 in Avg15 while trailing it on the four math/code tasks. \papersource{Paper, Tables 4--5, pp.21--22. All scores in percent, higher is better. Avg15 recomputed from individual displayed scores.} \end{frame} \begin{frame}{Ladders separate optimization and precision benefits} \begin{columns}[c,onlytextwidth] \begin{column}{0.48\textwidth} \centering\fig{muonh_compute_scaling}{4.4cm} \end{column} \begin{column}{0.48\textwidth} \centering\fig{fp8_ladder_summary}{4.4cm} \end{column} \end{columns} \begin{itemize} \item MuonH: $1.19\times$ fitted compute-equivalent gain with FP8 fixed. \item FP8: about $1.34\times$ quality-adjusted speedup with MuonH fixed. \end{itemize} \papersource{Paper, Fig.10, p.23; Appendix D, pp.42--44. 0.17B--1.7B at TPP=20; throughput proxy from 1.7B. Transfer to production TPP $\approx$700 is an estimate.} \end{frame} \begin{frame}{The cost law is a fixed-model scale-down fit} \begin{columns}[c,onlytextwidth] \begin{column}{0.60\textwidth} \centering\fig{renminbi_scaling_law}{4.8cm} \end{column} \begin{column}{0.37\textwidth} \[ P=a+b\log_2(C-C_{\rm P1}) \] \begin{itemize} \item $C_{\rm P1}=\$1.84$K fixed. \item Five UD budgets fit the incremental-cost trend. \item CD: fitted UD equivalent \$11.36K ($1.65\times$). \item CMA: \$16.55K ($2.40\times$). \end{itemize} \end{column} \end{columns} \alert{Both equivalent budgets extrapolate beyond the largest observed UD budget of \$6.89K.} \papersource{Paper, Fig.2b, p.5; Sec.4.3.1, pp.23--24; Tables 18--19, p.53. Filled: actual cost; hollow: fitted equivalent.} \end{frame} \begin{frame}{The CMA advantage survives matched SFT} \begin{center} \begin{tabular}{lrrr} \toprule SFT setting and metric & UD-based & CMA-based & Gain (pp)\\ \midrule GSM8K-based: GSM8K & 66.89 & 68.66 & +1.77\\ Math/code + replay: GSM8K & 74.10 & 76.12 & +2.02\\ Tulu-3: Avg15 & 53.96 & 55.13 & +1.17\\ Tulu-3: IFEval & 41.71 & 43.07 & +1.36\\ \bottomrule \end{tabular} \end{center} \begin{itemize} \item Same data, order, optimization, and budget within each paired comparison. \item Three matched SFT data orders; pretraining checkpoints are fixed. \item GSM8K checks use zero-shot ChatML and flexible numeric extraction. \end{itemize} \papersource{Paper, Fig.11, p.25; Tables 13, 15, 17, pp.47--49. Scores in percent. IFEval is separate from Avg15.} \end{frame} \begin{frame}{Post-training gains are not uniform across tasks} \begin{columns}[c,onlytextwidth] \begin{column}{0.50\textwidth} \begin{center} \begin{tabular}{lr} \toprule Tulu-3 task & CMA $-$ UD (pp)\\ \midrule HellaSwag & +7.25\\ ARC-Challenge & +3.95\\ MATH & +3.79\\ BoolQ & $-4.61$\\ HumanEval & $-1.63$\\ BBH & $-1.06$\\ \bottomrule \end{tabular} \end{center} \end{column} \begin{column}{0.46\textwidth} \begin{itemize} \item Ten of fifteen tasks favor CMA after Tulu-3 SFT. \item The paired result concerns the complete CMA recipe. \item SFT decontamination does not audit overlap already present in pretraining. \end{itemize} \end{column} \end{columns} \papersource{Paper, Table 17, p.49; Appendix A, p.38; Sec.3.5, p.17.} \end{frame} \begin{frame}{Three boundaries shape how the result transfers} \begin{block}{Data evidence} No strict corpus-wide exact/near-duplicate audit against all proxy and final benchmarks. Late curriculum data may concentrate overlap. \end{block} \begin{block}{Scale and attribution} TPP=20 ladder factors are transferred to TPP $\approx$700. Hardware, FP8, MuonH, and CMA factors use different baselines and should not be multiplied as a measured speedup. \end{block} \begin{block}{Reproduction conditions} The accelerator-only boundary, local 5090 pricing, and communication adaptations determine the reported cost. Upstream data terms remain component-specific. \end{block} \papersource{Paper, Sec.2.2, p.8; Sec.4.3, pp.22--24; Appendices A/B/D/I.} \end{frame} \begin{frame}{What this recipe makes possible} \begin{enumerate} \item \textbf{Repeatable pretraining research:} useful 2B base models at a measured, explicitly bounded GPU budget. \item \textbf{Observable optimization:} effective LR helps explain training dynamics beyond the scalar schedule. \item \textbf{Joint data and schedule design:} ordering, late updates, and averaging should be studied together. \end{enumerate} \medskip \textbf{Next experiment (reading analysis):} a decontaminated pool with matched ordering $\times$ tail LR $\times$ averaging controls. \papersource{Luo et al., Puro-2B: Poor Lab's Qwen2-1.5B Trained on RTX 5090 within \$5090.\par \href{https://arxiv.org/abs/2608.27370v2}{arXiv:2608.27370v2}, 3 September 2026.\par Paper-listed artifacts: \href{https://huggingface.co/collections/thu-pacman/puro-2b}{Puro-2B collection}; \href{https://github.com/thu-pacman/Puro-Megatron}{Puro-Megatron}; \href{https://github.com/thu-pacman/Kaiyuan-Spark}{Kaiyuan-Spark}.} \end{frame} \appendix \begin{frame}{Backup: the two learning-rate coordinates} The production \emph{base} schedule in Phase 1 is \[ \eta_{\rm base}(k)=\begin{cases} 5\times10^{-3}k/1000,&0\le k\le1000,\\ 5\times10^{-4}+4.5\times10^{-3}\!\left(1+\frac{k-1000}{1000}\right)^{-1/2},&k>1000. \end{cases} \] Each independent UD Phase 2 run starts from the same endpoint: \[ \eta_{{\rm base},j}(s)=\eta_0+(\eta_{\min}-\eta_0)\frac{s}{S_j}, \quad \eta_0\approx1.04\times10^{-3},\quad \eta_{\min}=10^{-5}. \] \begin{itemize} \item $s$: samples consumed after the shared Phase 1 checkpoint. \item $S_j$: branch-specific decay budget; schedules differ by horizon. \item Wrapped matrices use $10\times$ base LR in production. Ladder and WSD multipliers differ. \end{itemize} \papersource{Paper, Eqs.(10)--(11), pp.40--41; Fig.13a, p.42; Appendices D/F.} \end{frame} \begin{frame}{Backup: curriculum coordinates and the exact average} For internal bin $j$ of source $c$, let $T_{c,j}$ be its token count: \[ p(x)=\frac{\sum_{it_{\rm warm}}^{t}\Delta q_k\,G(x_k(t)),\\ x_k(t)&=Cq_k^{-\gamma}[S_q(t)-S_q(k-1)],\\ G(x)&=1-(1+x)^{-\beta}. \end{aligned} \] \begin{itemize} \item A decay has $\Delta q_k<0$; its loss effect arrives gradually through $G$. \item Effective-LR input reduces held-out mean RMSE: 0.0265 to 0.0210. \item The simpler $G=1$ variant reaches 0.0207 in the same two-run diagnostic. \end{itemize} \papersource{Paper, Appendix F.1 / Fig.16, pp.49--50. Last 20\% of validation targets held out. $C$ here is a response coefficient, not FLOPs.} \end{frame} \begin{frame}{Backup: all four math/code scores matter} \begin{center} \begin{tabular}{lrrrr} \toprule Base model & GSM8K & MATH & MBPP & HumanEval\\ \midrule Qwen2-1.5B & 59.82 & 23.70 & 50.19 & 27.44\\ Qwen2.5-1.5B & 67.70 & 32.28 & 58.37 & 31.71\\ Puro UD, \$4.4K & 56.03 & 27.86 & 49.81 & 20.73\\ Puro CMA, \$6.9K & 59.67 & 30.30 & 52.92 & 31.10\\ \bottomrule \end{tabular} \end{center} \begin{itemize} \item GSM8K and MATH use 4-shot generation. \item Sanitized-MBPP and HumanEval use 3-shot generation. \item SFT GSM8K uses a different zero-shot chat protocol; compare paired SFT initializations within each setup. \end{itemize} \papersource{Paper, Table 4, p.21; Appendix E.1, p.46. All scores in percent, higher is better.} \end{frame} \end{document}