\documentclass[aspectratio=169,11pt]{beamer}
\usepackage[T1]{fontenc}
\usepackage[utf8]{inputenc}
\usepackage{amsmath,amssymb,graphicx,booktabs}

\definecolor{AcademicBlue}{RGB}{26,57,86}
\definecolor{AcademicTeal}{RGB}{0,105,112}
\setbeamercolor{structure}{fg=AcademicBlue}
\setbeamercolor{frametitle}{fg=AcademicBlue,bg=white}
\setbeamercolor{title}{fg=AcademicBlue}
\setbeamercolor{alerted text}{fg=AcademicTeal}
\setbeamercolor{block title}{fg=white,bg=AcademicBlue}
\setbeamercolor{block body}{fg=black,bg=AcademicBlue!5}
\setbeamertemplate{navigation symbols}{}
\setbeamertemplate{itemize items}[circle]
\setbeamertemplate{footline}{%
  \hfill\usebeamerfont{page number in head/foot}%
  \insertframenumber\,/\,\inserttotalframenumber\hspace{4mm}\vspace{3mm}}
\setbeamersize{text margin left=8mm,text margin right=8mm}
\setbeamerfont{frametitle}{size=\Large,series=\bfseries}
\setbeamerfont{title}{size=\LARGE,series=\bfseries}
\newcommand{\papersource}[1]{\par\smallskip{\scriptsize\color{gray}#1\par}}

\title[LoGRA]{LoGRA: Scaling LLM Reinforcement Learning with Low-Rank Gradient Sketches}
\subtitle{Paper discussion: method, evidence, and trade-offs}
\author[Zhang et al.]{\small Paper authors: Shaokun Zhang, Yifan Zhang, Jian Hu, Yueying Li,\\
Hao Zhang, Binfeng Xu, Jan Kautz, Yi Dong (NVIDIA)}
\date{\href{https://arxiv.org/abs/2610.06647v3}{arXiv:2610.06647v3} \quad 23 October 2026}

\begin{document}

\begin{frame}
  \titlepage
\end{frame}

\begin{frame}{A compact update makes a larger RL run feasible}
  \begin{block}{One-sentence takeaway}
    LoGRA accumulates low-rank gradient sketches, merges their updates into full weights, and scales each step with a predicted policy-KL budget.
  \end{block}
  \begin{itemize}
    \item At 7B, average training memory drops from \textbf{31.82 to 17.29 GiB/GPU} (45.7\%) with similar MATH-500 peak Pass@1.
    \item On the same eight-H100 node, dense Adam OOMs at 27B; LoGRA completes training.
    \item A separate 27B trajectory improves held-out Reasoning-Gym score beyond 1,100 logged steps.
  \end{itemize}
  \papersource{Paper, Abstract; Table 1, PDF p. 7; Fig. 4, PDF p. 7.}
\end{frame}

\begin{frame}{RL memory includes much more than the weights}
  \begin{columns}[T,onlytextwidth]
    \begin{column}{0.49\textwidth}
      \begin{block}{A 7B illustration}
        \begin{itemize}
          \item BF16 model weights: about 14 GB.
          \item Two FP32 Adam moment buffers: about 56 GB.
          \item Gradients and activations add further cost.
        \end{itemize}
      \end{block}
    \end{column}
    \begin{column}{0.49\textwidth}
      \begin{block}{The design question}
        Can the training signal be represented more compactly while still updating the original weight matrices?
      \end{block}
      \vspace{2mm}
      \textbf{LoGRA's target:} gradient accumulation, optimizer state, and update representation in the training--rollout loop.
    \end{column}
  \end{columns}
  \papersource{Paper, Sec. 1, PDF pp. 1--2.}
\end{frame}

\begin{frame}{Sketch a matrix gradient in a smaller column space}
  \[
    W,G\in\mathbb R^{d\times k},\qquad
    A\in\mathbb R^{r\times k},\qquad r\ll\min(d,k)
  \]
  \[
    \boxed{S=GA^\top\in\mathbb R^{d\times r}}
    \qquad\longrightarrow\qquad
    \boxed{\widehat G=SA=(GA^\top)A}
  \]
  \begin{itemize}
    \item $A$ is an unlearned random projection; default entries are independent $\pm 1/\sqrt r$.
    \item $S$ stores $dr$ numbers instead of $dk$; the reconstructed update has rank at most $r$.
    \item $\widehat G$ approximates $G$; the full-size weight matrix remains trainable.
  \end{itemize}
  \papersource{Paper, Sec. 2.1, Eqs. (1)--(3), PDF pp. 2--3.}
\end{frame}

\begin{frame}{The implementation accumulates sketches directly}
  \begin{block}{One linear-layer example}
    If a sample gives $G_i=g_i x_i^\top$, then
    \[
      S=\sum_i G_iA^\top=\sum_i g_i(Ax_i)^\top.
    \]
    Project the inputs before forming the weight-gradient product.
  \end{block}
  \begin{itemize}
    \item Hold $A$ fixed over microbatches; accumulate $S$ in FP32 without a dense $d\times k$ gradient buffer.
    \item For $d=k=4096$, $r=64$: the FP32 accumulator falls from \textbf{64 MiB to 1 MiB}.
    \item Weights, projections, activations, and other buffers still occupy memory.
  \end{itemize}
  \papersource{Paper, Secs. 2.1--2.2, PDF p. 3. Linear-layer algebra is an explanatory expansion of Eq. (2).}
\end{frame}

\begin{frame}{Low-rank increments are merged into full weights}
  \[
    U=\operatorname{Optimizer}(S),\qquad
    \boxed{W\leftarrow W-\alpha\eta UA}
  \]
  \begin{itemize}
    \item For sketch SGD, $U=S$; the main math experiments use RowAdam.
    \item Refreshing $A$ each step changes the update subspace; the sum of updates need not have rank $r$.
    \item In the proposed factorized synchronization, rollout regenerates $A$ from a seed and receives $\alpha\eta U$.
  \end{itemize}
  \begin{block}{Measurement scope}
    The main benchmark protocol reports \emph{full-weight policy synchronization}; Table 1 does not measure factorized communication savings.
  \end{block}
  \papersource{Paper, Sec. 2.2 and Algorithm 1, PDF pp. 3--4; Appendix D.1, PDF p. 16.}
\end{frame}

\begin{frame}{RowAdam retains inexpensive adaptivity}
  \[
    v_{t,i}=\beta_2v_{t-1,i}+(1-\beta_2)\frac1r
    \sum_{j=1}^{r}(S_t)_{ij}^{2},\qquad
    (\widetilde U_t)_{i,:}=\frac{(S_t)_{i,:}}{\sqrt{\widehat v_{t,i}}+\epsilon_t}
  \]
  \begin{itemize}
    \item One second-moment statistic per sketch row; $\beta_2=0.95$.
    \item Rescale $\widetilde U_t$ to match the Frobenius norm of $S_t$.
    \item No first-moment momentum: RowAdam is \emph{not} dense Adam with a compressed storage format.
  \end{itemize}
  \papersource{Paper, Appendix B, Eq. (15), PDF pp. 14--15.}
\end{frame}

\begin{frame}{Predicted KL measures policy sensitivity to an update}
  \[
    D_W=-\eta UA,\qquad
    \overline D_{\rm KL}(\pi_\theta\|\pi_{\theta+\alpha D})
      \approx \alpha^2 q(D)
  \]
  \[
    q(D)=\frac{1}{2|\mathcal P|T^2}
      \sum_{s\in\mathcal P}\operatorname{Var}_{v\sim p(s)}[\dot z_v(s)]
  \]
  \begin{itemize}
    \item $\mathcal P$: sampled next-token contexts; $p(s)=\mathrm{softmax}(z(s)/T)$.
    \item $\dot z_v(s)$: directional logit change along the proposed update $D$.
    \item The variance ignores a common shift of all token logits, which leaves softmax unchanged.
  \end{itemize}
  \papersource{Paper, Sec. 2.3, Eqs. (5)--(6), PDF pp. 4--5; Appendix A, PDF pp. 13--14.}
\end{frame}

\begin{frame}{A KL budget determines the global step multiplier}
  \[
    \boxed{\alpha=\min\left\{\alpha_{\max},\sqrt{\delta/q(D)}\right\}}
  \]
  \begin{itemize}
    \item $\delta$ is the allowed \emph{predicted average} next-token KL for a training step.
    \item If $q(D)=4\delta$, choose at most half the nominal update size.
    \item A cosine-annealed budget reduces allowed policy movement later in training.
    \item The approximation is local; realized KL can differ from the prediction.
  \end{itemize}
  \begin{block}{Implementation detail}
    The long 27B run uses a finite-difference probe based on direct KL evaluations rather than the JVP expression above.
  \end{block}
  \papersource{Paper, Sec. 2.3, Eq. (7), PDF p. 5; Sec. 3.3, PDF p. 7.}
\end{frame}

\begin{frame}{The main comparison uses a fixed eight-GPU allocation}
  \begin{columns}[T,onlytextwidth]
    \begin{column}{0.49\textwidth}
      \begin{block}{Training protocol}
        \begin{itemize}
          \item Qwen2.5-Math-1.5B/7B; Qwen3.8-27B.
          \item DAPO-Math-7.5K, 128 prompts/update.
          \item Four FSDP training H100s; four generation H100s.
          \item LoGRA: rank 256 + RowAdam + annealed KL budget.
          \item Dense: Adam + annealed learning rate.
        \end{itemize}
      \end{block}
    \end{column}
    \begin{column}{0.49\textwidth}
      \begin{block}{Reported metrics}
        \begin{itemize}
          \item Training-GPU memory: temporal mean of per-update peak allocation.
          \item Peak accuracy: best MATH-500 evaluation in updates 0--200.
          \item Throughput: completed updates per hour.
          \item Three seeds for completed configurations.
        \end{itemize}
      \end{block}
    \end{column}
  \end{columns}
  \papersource{Paper, Secs. 3.1--3.2, PDF p. 5; Table 1, PDF p. 7; Appendix C--D, PDF pp. 15--16.}
\end{frame}

\begin{frame}{The memory advantage grows with model scale}
  \centering
  \includegraphics[width=0.96\linewidth,height=0.65\textheight,keepaspectratio]{figures/table1_memory_comparison.pdf}
  \begin{itemize}
    \item Average memory: \textbf{21.8\% less at 1.5B}; \textbf{45.7\% less at 7B}.
    \item 27B dense Adam OOM is a capacity mark, not a measured dense-memory value.
  \end{itemize}
  \papersource{Original paper Fig. 1, PDF p. 1; numerical values in Table 1, PDF p. 7. Broken axis: 34--48 GiB.}
\end{frame}

\begin{frame}{Compressed updates preserve competitive learning}
  \centering
  \includegraphics[width=0.97\linewidth,height=0.65\textheight,keepaspectratio]{figures/flora_pass1.pdf}
  \begin{itemize}
    \item At 7B: peak Pass@1 \textbf{72.33\% LoGRA vs 72.48\% dense} over updates 0--200.
    \item At 1.5B: 67.87\% vs 63.77\%; dense 27B does not complete an update.
  \end{itemize}
  \papersource{Original paper Fig. 2, PDF p. 6; Table 1, PDF p. 7. Curves show three-seed mean $\pm$ sample SD.}
\end{frame}

\begin{frame}{Table 1 shows memory savings, not a large speedup}
  \centering
  \begin{tabular}{llrrr}
    \toprule
    Model & Method & Avg. GiB/GPU $\downarrow$ & Updates/h $\uparrow$ & Peak P@1 $\uparrow$ \\
    \midrule
    1.5B & Dense & 9.18 & 54.28 & 63.77\% \\
    1.5B & LoGRA & \textbf{7.18} & 55.44 & \textbf{67.87\%} \\
    7B & Dense & 31.82 & 25.28 & \textbf{72.48\%} \\
    7B & LoGRA & \textbf{17.29} & 25.73 & 72.33\% \\
    27B & Dense & \multicolumn{3}{c}{OOM at initial Adam-state allocation} \\
    27B & LoGRA & 51.54 & 6.99 & 71.52\% \\
    \bottomrule
  \end{tabular}
  \vspace{2mm}
  \begin{itemize}
    \item The 27B case has no measured dense throughput or memory ratio.
    \item Accuracy peaks use updates 0--200; memory and throughput summarize the full runs.
  \end{itemize}
  \papersource{Adapted from paper Table 1, PDF p. 7; protocol in Sec. 3.2, PDF p. 5.}
\end{frame}

\begin{frame}{A separate 27B run sustains learning beyond 1,100 steps}
  \centering
  \includegraphics[width=0.97\linewidth,height=0.62\textheight,keepaspectratio]{figures/long27_stability.pdf}
  \begin{itemize}
    \item Eight-task held-out macro score: \textbf{39.69\% $\rightarrow$ 62.94\%}; peak 65.52\% at step 1,060.
    \item Mean response length: 8,312 $\rightarrow$ 4,705 tokens; truncation: 34.72\% $\rightarrow$ 12.95\%.
  \end{itemize}
  \papersource{Original paper Fig. 4 and Sec. 3.3, PDF p. 7. One trajectory; 193 held-out problems.}
\end{frame}

\begin{frame}{Projection rank matters more than basis details here}
  \centering
  \includegraphics[width=0.97\linewidth,height=0.62\textheight,keepaspectratio]{figures/rank_refresh.pdf}
  \begin{itemize}
    \item Rank 4 $\rightarrow$ 256 improves Pass@1 by 3.02--3.95 percentage points.
    \item At rank 256, fixed and refreshed Rademacher score 68.50\% and 68.31\%: no consistent short-horizon refresh benefit.
  \end{itemize}
  \papersource{Original paper Fig. 5, PDF p. 8. (b,c) show gradient-reconstruction cosine, not task accuracy.}
\end{frame}

\begin{frame}{LoGRA trades some speed and Pass@1 for lower memory vs LoRA}
  \centering
  \begin{tabular}{lrrrr}
    \toprule
    Method & Avg. GiB $\downarrow$ & Peak GiB $\downarrow$ & Pass@1 $\uparrow$ & s/update $\downarrow$ \\
    \midrule
    LoRA & 13.21 & 13.38 & \textbf{70.45\%} & \textbf{10.63} \\
    LoGRA & \textbf{7.18} & \textbf{8.66} & 68.47\% & 11.31 \\
    \bottomrule
  \end{tabular}
  \vspace{4mm}
  \begin{itemize}
    \item Rank-256 Qwen2.5-Math-1.5B; three seeds per method.
    \item LoGRA Pass@4 is 81.80\% vs 81.40\% for LoRA.
    \item The time metric excludes generation and synchronization; optimizers and schedules differ.
  \end{itemize}
  \papersource{Adapted from paper Table 2 and Sec. 3.4.2, PDF p. 9; Appendix D.4, PDF p. 17.}
\end{frame}

\begin{frame}{KL control helps only with a well-chosen budget}
  \centering
  \includegraphics[width=0.98\linewidth,height=0.63\textheight,keepaspectratio]{figures/long_horizon.pdf}
  \begin{itemize}
    \item Tail Pass@1, steps 700--800: 70.30\% (none), 63.43\% (large), 71.03\% (small), 71.47\% (annealed).
    \item No-control also remains stable; two-seed runs do not isolate annealing alone.
  \end{itemize}
  \papersource{Original paper Fig. 6 and Sec. 3.4.3, PDF pp. 9--10. Curves smoothed; tail means use raw evaluations.}
\end{frame}

\begin{frame}{Read the evidence at its measured scope}
  \begin{enumerate}
    \item \textbf{Strongest result:} substantial training-GPU memory savings and 27B feasibility under the reported eight-H100 allocation.
    \item \textbf{Comparable learning:} 7B MATH-500 peak accuracy is close to dense Adam; 1.5B is higher under the reported recipes.
    \item \textbf{Protocol differences:} RowAdam vs dense Adam, and LoGRA vs LoRA uses different optimizers and schedules.
    \item \textbf{Unmeasured here:} the main system comparison uses full-weight synchronization, so its numbers do not quantify factorized communication savings.
  \end{enumerate}
  \papersource{Paper, Tables 1--2, PDF pp. 7, 9; Appendix B/D.1/D.4, PDF pp. 14--17.}
\end{frame}

\begin{frame}{Take-home messages}
  \begin{block}{Method}
    Directly accumulated $S=GA^\top$ supports a compact optimizer state and a low-rank increment to full weights.
  \end{block}
  \begin{block}{Control}
    Predicted KL scales the increment in policy space; conservative budgets perform best in the reported long-horizon ablation.
  \end{block}
  \begin{block}{Result}
    Up to 45.7\% lower average training memory in the main math experiments, plus a successful single-node 27B run where dense Adam OOMs.
  \end{block}
  \papersource{Paper, Secs. 2--3 and Conclusion, PDF pp. 2--10.}
\end{frame}

\appendix

\begin{frame}{Backup: why predicted KL is quadratic}
  \small
  For fixed context $s$, define $K_s(a)=D_{\rm KL}(p(0)\|p(a))$ with $p(a)=\pi_{\theta+aD}(\cdot\mid s)$.
  \[
    K_s(0)=0,\qquad K'_s(0)=0,\qquad
    K_s(a)=\tfrac12a^2K''_s(0)+O(a^3).
  \]
  For $p_v(a)=\operatorname{softmax}(z_v(a)/T)$,
  \[
    \left.\frac{d}{da}\log p_v(a)\right|_0
      =\frac{\dot z_v-\mathbb E_{u\sim p}\dot z_u}{T},\qquad
    K''_s(0)=T^{-2}\operatorname{Var}_{v\sim p}[\dot z_v].
  \]
  Average over sampled contexts to obtain $q(D)$ in Eq. (6).
  \papersource{Paper, Appendix A.1, Eqs. (8)--(13), PDF pp. 13--14.}
\end{frame}

\begin{frame}{Backup: experimental settings that affect interpretation}
  \begin{itemize}
    \item Main runs: three seeds $\{1,2,42\}$; BF16, activation checkpointing, no CPU offload.
    \item LoGRA rank 256, projection refreshed every update; predicted-KL budget $2\times10^{-4}\rightarrow 2\times10^{-5}$ over 300 actual updates.
    \item Main 27B math runs end at 200 updates without shortening the budget schedule.
    \item Long 27B Reasoning-Gym run: seed 42, SGD sketch, 16,384-token context; budget annealed over 1,000 merges.
    \item LoRA comparison uses 300 logged steps for LoRA and 300 actual updates for LoGRA.
  \end{itemize}
  \papersource{Paper, Appendix C--D, PDF pp. 15--17.}
\end{frame}

\end{document}