% Compile with pdflatex. Replace the demonstration content before delivery.
\documentclass[aspectratio=169,11pt]{beamer}
\usepackage[T1]{fontenc}
\usepackage{lmodern}
\usepackage[utf8]{inputenc}
\usepackage{amsmath,amssymb,graphicx,booktabs}

\definecolor{AcademicBlue}{RGB}{26,57,86}
\definecolor{AcademicTeal}{RGB}{0,105,112}
\setbeamercolor{structure}{fg=AcademicBlue}
\setbeamercolor{frametitle}{fg=AcademicBlue,bg=white}
\setbeamercolor{title}{fg=AcademicBlue}
\setbeamercolor{alerted text}{fg=AcademicTeal}
\setbeamercolor{block title}{fg=white,bg=AcademicBlue}
\setbeamercolor{block body}{fg=black,bg=AcademicBlue!5}
\setbeamertemplate{navigation symbols}{}
\setbeamertemplate{itemize items}[circle]
\setbeamertemplate{footline}{%
  \hfill\usebeamerfont{page number in head/foot}%
  \insertframenumber\,/\,\inserttotalframenumber\hspace{4mm}\vspace{3mm}}
\setbeamersize{text margin left=8mm,text margin right=8mm}
\setbeamerfont{frametitle}{size=\Large,series=\bfseries}
\setbeamerfont{title}{size=\LARGE,series=\bfseries}
\newcommand{\papersource}[1]{\par\smallskip{\scriptsize\color{gray}#1\par}}

\title[Parcae]{Parcae: Scaling Laws For\\Stable Looped Language Models}
\subtitle{Paper discussion: stability, training budgets, and inference saturation}
\author{Reading presentation\\[1mm]\small Paper: Hayden Prairie, Zachary Novack,\\Taylor Berg-Kirkpatrick, and Daniel Y. Fu}
\date{arXiv:2604.12946v1 (14 April 2026)\\Reading: 16 September 2026}
\begin{document}
\begin{frame}\titlepage\end{frame}
\begin{frame}{More compute per parameter can help---up to a plateau}
\begin{itemize}
\item Parcae stabilizes shared-depth language models with constrained state transmission and better depth sampling.
\item At 1.3B: Core improves by \textbf{2.99 points} at matched data and approximately matched parameters.
\item Training: allocate compute to \textbf{both recurrence and data}.
\item Inference: added recurrence approaches a \textbf{training-dependent loss floor}.
\end{itemize}\begin{block}{Scope}Quality comparisons and compute-matched comparisons answer different questions.\end{block}
\papersource{Secs. 4--5; Table 5, PDF p. 8; Eq. (4), p. 11.}
\end{frame}

\begin{frame}{Loop the middle block to reuse stored weights}
\[s\ \xrightarrow{\mathcal P}\ e,\qquad h_0\xrightarrow{\mathcal R(\cdot,e)\;\times T}h_T\xrightarrow{\mathcal C}p\]\begin{itemize}
\item Prelude encodes tokens; Recurrent updates hidden states; Coda predicts tokens.
\item The recurrent block uses the \textbf{same parameters} at every iteration.
\item Training samples depth; inference chooses a depth $T$.
\item Repeated hidden-state computation increases effective depth and compute.
\end{itemize}
\papersource{Sec. 2.1, PDF p. 3. This schematic is an instructional summary.}
\end{frame}

\begin{frame}{Uncontrolled recurrence can explode the residual state}
\begin{center}\includegraphics[width=.96\linewidth,height=.51\textheight,keepaspectratio]{figures/fig-02.pdf}\end{center}\begin{itemize}
\item Left: training loss. Right: recurrent state norm on a logarithmic axis.
\item Residual normalization helps, but stability remains sensitive to learning rate.
\end{itemize}
\papersource{Fig. 2, PDF p. 4; Table 2, p. 5. Original author figure.}
\end{frame}

\begin{frame}{A linear surrogate isolates the state transmission}
\[
h_{t+1}=\overline A h_t+\overline B e+\overline{\mathcal R}(h_t,e)
\]
\begin{itemize}
\item $\overline A$: state transmission; $\overline B$: input injection.
\item $\overline{\mathcal R}$: nonlinear residual contribution of Transformer blocks.
\item Dropping $\overline{\mathcal R}$ gives an LTI surrogate:
\end{itemize}
\[
h_T=\overline A^T h_0+\sum_{k=0}^{T-1}\overline A^k\overline B e.
\]
Here $\overline A^T$ is a matrix power. For the linear system, $\rho(\overline A)<1$ implies asymptotic stability.
\papersource{Sec. 3, Eq. (2), PDF p. 5; App. C, p. 22. Expansion is a teaching derivation.}
\end{frame}

\begin{frame}{Negative diagonal dynamics constrain linear memory}
\[
A=\operatorname{Diag}(-\exp(a)),\qquad
\overline A=\exp(\Delta A),\qquad \overline B=\Delta B
\]
\begin{itemize}
\item For $\Delta_i>0$, each diagonal entry of $\overline A$ lies in $(0,1)$.
\item ZOH for $\overline A$; Euler for $\overline B$.
\item Normalize the prelude output: $e=\mathrm{LN}(\mathcal P(s))$.
\item Predict with $p=\mathcal C(C h_T)$; $C$ is a readout projection.
\end{itemize}
\begin{block}{Implementation detail to resolve}
The paper does not specify how learned $\Delta$ is kept positive.
\end{block}
\papersource{Sec. 4.1, Eq. (3), PDF p. 6. Positive step sizes are required by this argument.}
\end{frame}

\begin{frame}{Parcae combines a linear constraint and nonlinear blocks}
\begin{center}\includegraphics[width=.96\linewidth,height=.57\textheight,keepaspectratio]{figures/fig-01.pdf}\end{center}\begin{center}Constrain state transmission, normalize input injection, retain learned nonlinear updates.\end{center}
\papersource{Fig. 1, PDF p. 2. Original author figure, trimmed as in the paper.}
\end{frame}

\begin{frame}{The stability argument has a precise boundary}
\begin{itemize}
\item The LTI surrogate omits nonlinear updates; it is not the full network Jacobian.
\item Spectral radius and spectral norm differ in general; they coincide for this diagonal design.
\item Constraining $\overline A$ alone does not prove global nonlinear convergence.
\end{itemize}\begin{block}{Teaching counterexample}\[h_{t+1}=0.5h_t+\overline{\mathcal R}(h_t),\qquad \overline{\mathcal R}(h_t)=h_t.\]The full update multiplies the state by $1.5$.\end{block}
\papersource{Interpretation of Secs. 2.2--4 and App. C; counterexample is this reading\textquotesingle s analysis.}
\end{frame}

\begin{frame}{Sample depth per sequence to better cover the objective}
\[
\min_\theta\;\mathbb E_{(x,y),\,T\sim\Lambda}[\ell(f_\theta(x;T),y)]
\]
\begin{itemize}
\item Sample $T_i$ independently within a micro-batch.
\item Align endings: delay sequence $i$ by $T_{\max}-T_i$ steps.
\item Backpropagate through the final $\min(T_i,\mu_{\mathrm{bwd}})$ updates.
\end{itemize}
\begin{block}{Example: depths 2, 4, 6}
Wait for 4, 2, 0 steps respectively; all sequences finish together. Waiting must preserve state.
\end{block}
\papersource{Sec. 4.2, PDF p. 6; App. E, Algorithm 2, p. 23. Example is instructional.}
\end{frame}

\begin{frame}{Sample full forward depth before truncating gradients}
\[
T\sim\Lambda(\mu_{\mathrm{rec}}),\quad
n=\max(T-\mu_{\mathrm{bwd}},0),\quad
k=\min(T,\mu_{\mathrm{bwd}})
\]
\begin{itemize}
\item Sampling a no-gradient prefix and adding a fixed suffix shifts the depth distribution.
\item The correction restores low-depth exposure and improves generalization across $T$.
\item Main runs use $\mu_{\mathrm{bwd}}=\lceil\mu_{\mathrm{rec}}/2\rceil$.
\item Main RDM baselines also receive the sampling-distribution correction.
\end{itemize}
\papersource{App. H--I, PDF pp. 27--30. The backward-depth rule is empirical, not proven optimal.}
\end{frame}

\begin{frame}{Three evaluation protocols separate three questions}
\begin{tabular}{p{.24\linewidth}p{.29\linewidth}p{.35\linewidth}}
\toprule
Protocol & Held fixed & Question \\\midrule
End-to-end quality & Parameter scale and tokens & Does looping improve quality per stored parameter? \\[2mm]
Training isoFLOP & Parameter scale and training FLOPs & How much recurrence versus data? \\[2mm]
Inference scaling & A trained model & How much does increasing $T$ help? \\\bottomrule
\end{tabular}
\par\medskip
\textbf{Important:} the first protocol does not match training or inference compute.
\papersource{Sec. 5, PDF pp. 7--11; App. O, pp. 38--40.}
\end{frame}

\begin{frame}{The Transformer comparison uses a shared data recipe}
\begin{itemize}
\item FineWeb-Edu; 32,768-token BPE; context 2,048; batch 256; BF16.
\item 140M / 370M / 770M / 1.3B: 11.2B / 29.6B / 61.6B / 104B training tokens.
\item Loop the middle third; train at $\mu_{\mathrm{rec}}=8$, $\mu_{\mathrm{bwd}}=4$.
\item Parcae adds 0.35--0.83\% parameters. Learning rates are selected for baselines and reused.
\item Core and Core-Extended are aggregate downstream scores; evaluation uses three seeds.
\end{itemize}
\papersource{Apps. M, P--R, PDF pp. 38--44. Evaluation seeds are not independent pretraining runs.}
\end{frame}

\begin{frame}{Parcae improves PPL over prior residual-norm RDMs}
\begin{center}\begin{tabular}{lrrrr}
\toprule
Scale / depth & \multicolumn{2}{c}{Validation PPL $\downarrow$} & \multicolumn{2}{c}{6-task Avg. $\uparrow$}\\
& RDM & Parcae & RDM & Parcae\\\midrule
100M / $T=16$ & 14.23 & \textbf{13.59} & 45.03 & \textbf{46.83}\\
350M / $T=8$ & 10.76 & \textbf{10.09} & 49.45 & \textbf{50.12}\\\bottomrule
\end{tabular}\end{center}
\begin{itemize}
\item Huginn data/tokenizer; this is separate from the nanochat setup.
\item 350M validation PPL reduction: $\approx6.23\%$ from table values.
\item Individual tasks can regress; average improvement is not universal.
\end{itemize}
\papersource{Table 3, PDF p. 7. Abstract says 6.3\%; text says 6.2\%. Values above follow the table.}
\end{frame}

\begin{frame}{Each stabilization addresses a different failure mode}
\begin{center}\begin{tabular}{lrrr}
\toprule
140M cumulative ablation & \multicolumn{3}{c}{Validation loss $\downarrow$}\\
& $T=1$ & $T=4$ & $T=8$\\\midrule
RDM & \multicolumn{3}{c}{Divergent training}\\
Constrained $\overline A$ & 8.99 & 3.15 & 2.97\\
+ Per-sequence sampling & 3.38 & 3.01 & 2.98\\
+ Prelude norm & \textbf{3.28} & \textbf{2.97} & \textbf{2.95}\\\bottomrule
\end{tabular}\end{center}
Constrained transmission enables training; depth sampling especially helps shallow inference; prelude normalization further improves loss.
\papersource{Table 4, PDF p. 7. Not every downstream metric improves monotonically.}
\end{frame}

\begin{frame}{Matched parameters and data yield better Core scores}
\begin{center}\begin{tabular}{lrrrr}
\toprule
Scale & \multicolumn{2}{c}{Core $\uparrow$} & \multicolumn{2}{c}{Validation PPL $\downarrow$}\\
& Transformer & Parcae & Transformer & Parcae\\\midrule
140M & 13.00 & 14.04 & 21.48 & 19.06\\
370M & 17.46 & 20.00 & 15.79 & 14.49\\
770M & 22.42 & 25.07 & 13.08 & 12.49\\
1.3B & 25.45 & \textbf{28.44} & 11.95 & \textbf{11.42}\\\bottomrule
\end{tabular}\end{center}
At 1.3B, Core: $25.45\pm0.08\to28.44\pm0.28$;\\
Core-Extended: $15.90\pm0.23\to17.08\pm0.09$.\\[1mm]
\alert{Compute is not matched; Parcae is evaluated at $T=8$.}
\papersource{Table 5, PDF p. 8. Original error bars retained; the paper does not specify SD versus SE.}
\end{frame}

\begin{frame}{87.5\% means closing a gap, not saving parameters}
\[
\frac{\mathrm{Core}_{770M\;Parcae}-\mathrm{Core}_{770M\;Transformer}}
{\mathrm{Core}_{1.3B\;Transformer}-\mathrm{Core}_{770M\;Transformer}}
=\frac{25.07-22.42}{25.45-22.42}\approx87.5\%.
\]
\begin{itemize}
\item This measures the fraction of the quality gap to the next larger baseline that is closed.
\item Actual parameter counts are about 777M versus 1,334M ($1.72\times$).
\item Cross-scale comparisons also use different token budgets.
\end{itemize}
\papersource{Sec. 5.1, PDF p. 7; Tables 5, 19, 21, pp. 8, 42, 44.}
\end{frame}

\begin{frame}{At fixed FLOPs, recurrence has an interior optimum}
\begin{center}\includegraphics[width=.96\linewidth,height=.55\textheight,keepaspectratio]{figures/fig-04.pdf}\end{center}\begin{center}Right: more recurrence eventually displaces too much training data.\end{center}
\papersource{Fig. 4, PDF p. 8. Left horizontal axis is training FLOPs. Original author figure.}
\end{frame}

\begin{frame}{Optimal recurrence and tokens both grow with budget}
\begin{center}\includegraphics[width=.96\linewidth,height=.43\textheight,keepaspectratio]{figures/fig-05.pdf}\end{center}\begin{center}\begin{tabular}{lcc}
\toprule
Scale & $\mu_{\mathrm{rec}}^*\propto C^{\gamma_\mu}$ & $D^*\propto C^{\gamma_D}$\\\midrule
140M & $\gamma_\mu=0.40$ & $\gamma_D=0.77$\\
370M & $\gamma_\mu=0.38$ & $\gamma_D=0.78$\\\bottomrule
\end{tabular}\end{center}
\papersource{Fig. 5, PDF p. 9. Finite-range empirical fits, not universal exponents.}
\end{frame}

\begin{frame}{Training loss is fitted using effective unfolded capacity}
\[
\widehat{\mathcal L}_{train}(\mu,D)=E+X\mathbf N(\mu)^{-x}+YD^{-y}
\]
\begin{itemize}
\item $\mathbf N(\mu)$ counts effective unrolled parameters, not stored weights.
\item Huber objective on log loss, optimized with L-BFGS.
\item Held-out loss prediction error: 1.3\% (140M), 0.8\% (370M).
\end{itemize}
\[
C_{\mathrm{FLOPs}}\approx(2\widehat N_1+6\widehat N_2)D
+\text{attention cost}
\]
$\widehat N_1$: no backward pass; $\widehat N_2$: forward and backward.
\papersource{Sec. 5.2, PDF p. 9; App. D, p. 22; App. K, p. 35.}
\end{frame}

\begin{frame}{Compute-matched downstream wins depend on budget}
\begin{center}\begin{tabular}{lrrrr}
\toprule
Scale & FLOPs ($10^{18}$) & $\mu^*$ & Loop Core & Fixed Core\\\midrule
140M & 2 & 2 & 9.0 & \textbf{10.5}\\
140M & 8 & 6 & 10.5 & \textbf{11.8}\\
140M & 16 & 8 & \textbf{14.6} & 13.0\\
140M & 64 & 10 & \textbf{16.2} & 15.0\\
370M & 32 & 4 & 15.2 & \textbf{16.8}\\
370M & 128 & 6 & \textbf{20.1} & 18.1\\\bottomrule
\end{tabular}\end{center}
\textbf{Fixed baseline here:} Parcae with $\mu=1$.\\
Lower validation loss does not guarantee better Core at every budget.
\papersource{Table 6, PDF p. 9; Table 16 with error bars, p. 40. Selected original rows.}
\end{frame}

\begin{frame}{Inference gains plateau near the training depth}
\begin{center}\includegraphics[width=.96\linewidth,height=.55\textheight,keepaspectratio]{figures/fig-07.pdf}\end{center}\begin{center}Four model scales; all trained with $\mu_{\mathrm{rec}}=8$.\end{center}
\papersource{Fig. 7, PDF p. 10. PPL decreases; aggregate task scores increase, then saturate.}
\end{frame}

\begin{frame}{Saturating exponentials fit inference curves well}
\begin{center}\includegraphics[width=.96\linewidth,height=.40\textheight,keepaspectratio]{figures/fig-08.pdf}\end{center}\[
\mathcal L(T)=\mathcal L_\infty+Z e^{-zT}
\]
$\mathcal L_\infty$: training-dependent floor; $Z$: gap scale; $z$: decay rate.\\
The exponential beats the tested power-law alternatives, including in depth extrapolation.
\papersource{Fig. 8, PDF p. 10; Table 12, p. 36. Caption says power laws; actual fit is exponential.}
\end{frame}

\begin{frame}{A unified law separates the floor from inference progress}
\[
\widehat{\mathcal L}_{unified}(T\mid\mu,D)
=\underbrace{E+X\mathbf N(\mu)^{-x}+YD^{-y}}_{\text{training-dependent floor}}
+\underbrace{Ze^{-zT/\mu}}_{\text{inference gap}}
\]
\begin{itemize}
\item $T/\mu$ conditions the decay on training depth.
\item Full held-out prediction error: \textbf{0.85--1.31\%}.
\item With an empirically measured floor: \textbf{0.10--0.17\%}.
\item The measured-floor variant is an oracle diagnostic, not a fully predictive fit.
\end{itemize}
\papersource{Eq. (4), PDF p. 11; App. L, pp. 36--38.}
\end{frame}

\begin{frame}{The main payoff is parameter efficiency with boundaries}
\begin{itemize}
\item \textbf{Established:} improved stability in tested runs; higher quality per parameter at matched tokens.
\item \textbf{Compute allocation:} recurrence and data should grow together in the tested high-budget regimes.
\item \textbf{Inference:} depth helps until a training-dependent plateau.
\item \textbf{Open:} full nonlinear stability, larger-scale laws, and joint optimization of parameters/data/depth.
\item \textbf{Deployment:} latency, throughput, KV cache, and activation memory need separate measurement.
\end{itemize}
\papersource{Sec. 6, PDF p. 11; evidence boundaries from this reading. Main talk: approximately 25 minutes.}
\end{frame}

\begin{frame}{Reference and reading provenance}
\textbf{Hayden Prairie, Zachary Novack, Taylor Berg-Kirkpatrick, Daniel Y. Fu.}\\[2mm]
\emph{Parcae: Scaling Laws For Stable Looped Language Models.}\\
arXiv:2604.12946v1, 14 April 2026.\\[3mm]
\url{https://arxiv.org/abs/2604.12946v1}
\begin{itemize}
\item Read the version-matched PDF and author TeX, including Apps. A--R.
\item Figures are original author assets. Selected tables preserve original values.
\item Teaching examples and critical interpretations are marked explicitly.
\item This presentation does not reproduce model training.
\end{itemize}
\end{frame}

\begin{frame}{Backup: per-sequence sampling especially helps low depth}
\begin{center}\begin{tabular}{llrrr}
\toprule
Scale & Sampling & PPL $T=1$ & PPL $T=8$ & PPL $T=16$\\\midrule
100M & Per-batch & 300.32 & 16.65 & 13.81\\
& Per-sequence & 70.47 & 14.08 & 13.59\\
350M & Per-batch & 167.61 & 10.40 & 10.24\\
& Per-sequence & 17.92 & 10.09 & 10.11\\\bottomrule
\end{tabular}\end{center}
\begin{itemize}
\item The paper reports 1.8\% extra pretraining wall-clock time for per-sequence sampling.
\item This is sampling overhead, not the cost of looping versus a Transformer.
\end{itemize}
\papersource{App. G, Table 8, PDF pp. 25--26.}
\end{frame}

\begin{frame}{Backup: late instability can originate in the input}
\begin{itemize}
\item An early 1.3B configuration develops late loss spikes and large state norms.
\item The authors inspect state-transition, injection, readout, and actual activation expansion.
\item The large jump occurs at the first injection; prelude activations are implicated.
\item Prelude normalization enables the reported stable 1.3B run.
\item Improved gradient flow is an author interpretation, not directly demonstrated.
\end{itemize}
\papersource{App. J, Figs. 17--24, PDF pp. 31--34.}
\end{frame}

\begin{frame}{Backup: depth extrapolation favors a saturating exponential}
Fit on $T\le\mu$; evaluate on held-out $T>\mu$.\\[3mm]
\begin{center}\begin{tabular}{lrrrr}
\toprule
& $\mathcal L_\infty+Ze^{-zT}$ & $\mathcal L_\infty+Z(1+T)^{-z}$ & $\mathcal L_\infty+ZT^{-z}$ & $ZT^{-z}$\\\midrule
140M & \textbf{3.18} & 21.41 & 43.99 & 397.90\\
370M & \textbf{2.29} & 18.51 & 38.68 & 369.83\\\bottomrule
\end{tabular}\end{center}
Average Huber loss, $\times10^{-7}$; lower is better.\\[2mm]
The result compares these candidates on the observed depth range. It does not establish a unique universal law.
\papersource{App. L, Table 12, PDF p. 36.}
\end{frame}

\begin{frame}{Backup: the floor explains most of the prediction gap}
\begin{center}\includegraphics[width=.96\linewidth,height=.53\textheight,keepaspectratio]{figures/fig-26.pdf}\end{center}\begin{center}Orange: full prediction. Blue: measured floor. Black dots: empirical loss.\end{center}
\papersource{App. L, Fig. 26, PDF p. 38. Both model sizes are held out from the isoFLOP fit.}
\end{frame}

\begin{frame}{Backup: reproduction requires resolving reporting issues}
\begin{itemize}
\item RDM learning rates differ by $10\times$ between two paragraphs of App. Q.1.
\item App. I gives conflicting fixed backward depths in a figure caption and the text/table.
\item App. O reports a data-point count inconsistent with its listed model/depth grid.
\item App. O says eight fit parameters; the final fixed-$\gamma$ equation shows seven.
\item Use primary table values for PPL improvements; some narrative percentages disagree.
\end{itemize}
\papersource{PDF pp. 7--8, 29, 40, 42--43. These discrepancies are documented in the Chinese notes.}
\end{frame}
\end{document}