% Self-authored reading presentation. Compile with pdfLaTeX.
\documentclass[aspectratio=169,11pt]{beamer}
\usepackage[T1]{fontenc}
\usepackage[utf8]{inputenc}
\usepackage{lmodern,array}
\usepackage{amsmath,amssymb,graphicx,booktabs}
\definecolor{AcademicBlue}{RGB}{26,57,86}
\definecolor{AcademicTeal}{RGB}{0,105,112}
\setbeamercolor{structure}{fg=AcademicBlue}
\setbeamercolor{frametitle}{fg=AcademicBlue,bg=white}
\setbeamercolor{title}{fg=AcademicBlue}
\setbeamercolor{alerted text}{fg=AcademicTeal}
\setbeamercolor{block title}{fg=white,bg=AcademicBlue}
\setbeamercolor{block body}{fg=black,bg=AcademicBlue!5}
\setbeamertemplate{navigation symbols}{}
\setbeamertemplate{itemize items}[circle]
\setbeamertemplate{footline}{\hfill\usebeamerfont{page number in head/foot}\insertframenumber\,/\,\inserttotalframenumber\hspace{4mm}\vspace{3mm}}
\setbeamersize{text margin left=8mm,text margin right=8mm}
\setbeamerfont{frametitle}{size=\Large,series=\bfseries}
\setbeamerfont{title}{size=\LARGE,series=\bfseries}
\newcommand{\papersource}[1]{\par\smallskip{\scriptsize\color{gray}#1\par}}
\newcommand{\fig}[2]{\begin{center}\includegraphics[width=\linewidth,height=#2,keepaspectratio]{figures/#1.pdf}\end{center}}
\newcommand{\R}{\mathbb R}
\title[DepthBench]{DepthBench}
\subtitle{Measuring How Residual Connections\newline Enable More Computational Depth}
\author{A research paper reading presentation}
\date{arXiv:2609.32534v1\quad |\quad 19 October 2026}
\hypersetup{pdftitle={DepthBench: Research Paper Reading},pdfauthor={Paper reading presentation}}

\begin{document}

\begin{frame}
\titlepage
\vspace{-5mm}
{\small Paper authors: Keyu Wang, Yangyi Huang, Jiale Kang,\\
David Gonz\'alez-Mart\'inez, Weiyang Liu, and Shiwei Liu.}
\begin{block}{Central finding}
Residual design changes whether trading width for depth improves model quality.
\end{block}
\papersource{Paper version: \href{https://arxiv.org/abs/2609.32534v1}{arxiv.org/abs/2609.32534v1}. Main discussion: slides 2--21; supporting details: slides 22--26.}
\end{frame}

\begin{frame}{Trainable depth can still be weakly utilized}
\[
h_\ell=h_{\ell-1}+F_\ell\bigl(\mathrm{LN}(h_{\ell-1})\bigr)
\]
\begin{itemize}
\item A direct residual path supports optimization of deep networks.
\item Later updates can become weak, similar, or interchangeable.
\item Adding layers at fixed width also adds parameters and compute.
\end{itemize}
\begin{block}{DepthBench's question}
At a similar parameter budget, which architectures benefit from a deeper, narrower shape?
\end{block}
\papersource{Paper, Secs. 1--2.1, PDF pp. 2--3; Eq. (1).}
\end{frame}

\begin{frame}{The controlled variable is width--depth allocation}
\[
r=\frac{d}{L},\qquad N\approx12Ld^2+2Vd,\qquad V=50280
\]
\begin{columns}[T]
\begin{column}{0.49\textwidth}
\begin{block}{Fixed total parameters}
\centering
$(L,d)=(16,1216)\rightarrow(70,640)$\\[2mm]
$r:76.0\rightarrow9.1$\\[2mm]
Backbone: $284\mathrm{M}\rightarrow345\mathrm{M}$
\end{block}
\end{column}
\begin{column}{0.49\textwidth}
\begin{block}{Fixed backbone control}
Hold the Transformer backbone near 300M.\\[2mm]
Embedding/head parameters shrink with width.\\[2mm]
Separates shape gains from backbone growth.
\end{block}
\end{column}
\end{columns}
\medskip
Approximate parameter matching; the formula describes the sub-1B MHA backbone.
\papersource{Paper, Sec. 2.2, Tables 3 and 7, PDF pp. 4--5, 21.}
\end{frame}

\begin{frame}{Ten variants change how a layer accesses its past}
\centering
\renewcommand{\arraystretch}{1.25}
\begin{tabular}{>{\raggedright\arraybackslash}p{2.7cm}>{\raggedright\arraybackslash}p{4.4cm}>{\raggedright\arraybackslash}p{5.6cm}}
\toprule
Family & Architectures & Main intervention\\
\midrule
Baseline & Pre-LN & One stream; fixed residual addition\\
Normalization & Sandwich-LN, LNS, DeepNorm, KEEL & Normalization, scaling, initialization\\
Multi-stream & HC, mHC & Learned read, write, and stream mixing\\
Cross-layer & Full/Block AttnRes, MoDA & Read earlier outputs or depth KV\\
\bottomrule
\end{tabular}
\medskip
\par Normalization adjusts scale; richer residual paths change history access.
\papersource{Paper, Table 1 and Appendix A, PDF pp. 4, 19--20. MoDA uses Pre-LN in this benchmark.}
\end{frame}

\begin{frame}{Full AttnRes retrieves individual earlier outputs}
\[
a_{\ell i}=\operatorname{softmax}_i
\left(w_\ell^\top\operatorname{RMSNorm}_{\mathrm{key},\ell}(v_i)\right),
\qquad z_\ell=\sum_{i<\ell}a_{\ell i}v_i
\]
\[
f_\ell=F_\ell\bigl(\operatorname{RMSNorm}_{\mathrm{pre},\ell}(z_\ell)\bigr)
\]
\begin{itemize}
\item Sources: the embedding and individual sublayer branch outputs.
\item Learned query $w_\ell\in\R^d$; token-dependent mixture weights.
\item Zero-initialized queries start from uniform averaging; no $1/\sqrt d$ score scaling.
\item The output head also reads a learned mixture of sources.
\end{itemize}
\papersource{Implementation restated from Appendix A, PDF p. 19. $z_\ell$ denotes the mixed input; normalization is fused in the implementation.}
\end{frame}

\begin{frame}{HC propagates history through interacting streams}
\[
H_\ell=H_{\ell-1}A_\ell+
F_\ell\bigl(\mathrm{LN}(H_{\ell-1}\alpha_\ell)\bigr)\beta_\ell^\top
\]
\begin{itemize}
\item $H\in\R^{d\times m}$: $m=4$ streams for each token.
\item $\alpha_\ell\in\R^m$: read; $A_\ell\in\R^{m\times m}$: mix; $\beta_\ell\in\R^m$: write.
\end{itemize}
\begin{block}{Explicit path from branch $i$ to a later branch $\ell$}
\[
w_{i\rightarrow\ell}=\beta_i^\top A_{i+1}\cdots A_{\ell-1}\alpha_\ell
\]
History is recursively transported and recombined across streams.
\end{block}
\papersource{Paper, Table 1, Sec. 4.2 and Appendix A, PDF pp. 4, 10, 20. Path coefficients describe realized residual transport, not the full network Jacobian.}
\end{frame}

\begin{frame}{Four suites test shape, scale, and budget effects}
\centering
\renewcommand{\arraystretch}{1.25}
\begin{tabular}{lll}
\toprule
Suite & Budget & Shape coverage\\
\midrule
Main comparison & $\sim$400M total & 10 variants at $L=16$--32; deeper extensions\\
Backbone control & $\sim$300M backbone & Configuration pool: $L=16$--84\\
Multiple scales & 200M--500M & Three aspect ratios near 28, 43, 76\\
Larger scale & $\sim$1.6B total & $(L,d)=(28,2048),(40,1728),(54,1504)$\\
\bottomrule
\end{tabular}
\medskip
\begin{itemize}
\item FineWeb-Edu; about 20 tokens/parameter: 8B tokens at 400M, 32B at 1.6B.
\item Training context 2048; global batch 512 sequences; AdamW.
\item Architecture-specific LR selection at 400M; $5\times10^{-4}$ at 1.6B.
\end{itemize}
\papersource{Paper, Sec. 2.3, Tables 2--4 and 7--9, PDF pp. 4--5, 21--22. The 1.6B suite uses GQA rather than MHA.}
\end{frame}

\begin{frame}{At 400M, the same shape change has opposite effects}
\begin{center}
$(16,1216)\ \longrightarrow\ (32,896)$\qquad at approximately fixed total size
\end{center}
\medskip
\centering
\renewcommand{\arraystretch}{1.5}
\begin{tabular}{lrrr}
\toprule
Architecture & Loss, $L=16$ & Loss, $L=32$ & Change\\
\midrule
Pre-LN & 2.759 & 2.782 & $+0.023$\\
Full AttnRes & 2.751 & 2.718 & $-0.033$\\
HC & 2.729 & 2.699 & $-0.030$\\
\bottomrule
\end{tabular}
\begin{block}{Shape preference depends on residual design}
Full AttnRes and HC improve between these endpoints; Pre-LN worsens. Validation loss is lower-is-better.
\end{block}
\papersource{Paper, Sec. 3, PDF pp. 5--6; changes calculated from the reported losses.}
\end{frame}

\begin{frame}{Fixed-backbone scaling has a preferred depth}
\fig{fig-02-backbone}{4.6cm}
\vspace{-2mm}
\begin{center}{\small Black: Pre-LN\quad Red: Full AttnRes\quad Purple: HC}\end{center}
\begin{itemize}
\item Leftward means deeper; total model size decreases at similar backbone size.
\item Full AttnRes favors $L\approx32$; HC worsens from 70 to 84 layers.
\end{itemize}
\papersource{Paper, Fig. 2(c), PDF p. 6. Original panel cropped; axes and configuration labels retained.}
\end{frame}

\begin{frame}{Full AttnRes has the more consistent scale evidence}
\fig{fig-03}{4.3cm}
\begin{itemize}
\item Full AttnRes benefits from deeper shapes across 200M--500M and at 1.6B.
\item HC: the widest 500M run diverges; the 1.6B curve has no clear depth gain.
\item Pre-LN also gains slightly from depth at 1.6B; the 400M trend is not universal.
\end{itemize}
\papersource{Paper, Fig. 3 and Sec. 3, PDF pp. 6--7. Optimization sensitivity is the authors' explanation for HC's instability.}
\end{frame}

\begin{frame}{The gains extend to domain-specific likelihood}
\fig{fig-04}{4.4cm}
\begin{itemize}
\item Each panel averages teacher-forced NLL over two tasks; lower is better.
\item Deep HC/Full AttnRes generally help, with non-monotonic intermediate points.
\item These are reference-sequence likelihoods, not generation pass rates.
\end{itemize}
\papersource{Paper, Fig. 4, PDF p. 7. Coding: MBPP/HumanEval; STEM: SciQ/GPQA; math: GSM8K/MATH-500.}
\end{frame}

\begin{frame}{Representations retain more layer-specific structure}
\[
d(h_\ell,h_{\ell+n})=\frac1\pi\arccos
\frac{h_\ell\cdot h_{\ell+n}}{\|h_\ell\|_2\|h_{\ell+n}\|_2}
\]
\fig{fig-05-selected}{3.25cm}
\vspace{-2mm}
\begin{itemize}
\item $x$: starting layer; $y$: layer offset. Yellow means similar directions.
\item Full AttnRes and HC show less smooth structure than Pre-LN.
\item This probe is learning-rate sensitive; distance alone is not utility.
\end{itemize}
\papersource{Paper, Eq. (2), Fig. 5, PDF p. 8. Selected original panels rearranged; shared colorbar retained. LR $2\times10^{-3}$ for this comparison.}
\end{frame}

\begin{frame}{Layer interventions probe dependence and ordering}
\begin{block}{Skip a layer: how much does a later update change?}
\[
C(s,\ell)=\frac{\|(h_{\ell+1}-h_\ell)-(\bar h_{\ell+1}-\bar h_\ell)\|_2}
{\|h_{\ell+1}-h_\ell\|_2}
\]
\end{block}
\begin{block}{Swap two layers: how much does the loss change?}
\[
P(\ell_1,\ell_2)=\frac{|\mathcal L(M)-\mathcal L(M_{\mathrm{swap}})|}{\mathcal L(M)}
\]
\end{block}
\begin{itemize}
\item $\bar h$ denotes hidden states after skipping layer $s$.
\item The absolute-value permutation score measures change, not its sign.
\end{itemize}
\papersource{Paper, Eqs. (3)--(4), PDF p. 9. Causal score measures relative update perturbation, not task accuracy.}
\end{frame}

\begin{frame}{Late layers are more dependent and less interchangeable}
Restrict layer pairs to the last $3/4$ of a 32-layer model.
\medskip
\begin{center}
\renewcommand{\arraystretch}{1.5}
\begin{tabular}{lrr}
\toprule
Architecture & Fraction with $C>0.45$ & Fraction with $P>0.15$\\
\midrule
Pre-LN & $\sim1\%$ & $\sim8\%$\\
Full AttnRes & $\sim16\%$ & $\sim46\%$\\
HC & $\sim7\%$ & $\sim35\%$\\
\bottomrule
\end{tabular}
\end{center}
\begin{itemize}
\item Removing early layers from the summary exposes differences in later computation.
\item These are thresholded \emph{pair fractions}, not percentages of effective layers.
\item Strong perturbation responses do not by themselves imply favorable shape scaling.
\end{itemize}
\papersource{Paper, Fig. 6 and Eqs. (5)--(8), PDF pp. 9--10. Approximate percentages reported in the text.}
\end{frame}

\begin{frame}{Full AttnRes retrieves; HC transports and recombines}
\fig{fig-07}{4.15cm}
\begin{itemize}
\item Pre-LN: explicit residual coefficients $w_{i\to\ell}=1$.
\item Full AttnRes: non-negative, normalized weights over stored sources.
\item HC: products of write, transport, and read maps; coefficients may be signed.
\end{itemize}
\papersource{Paper, Fig. 7 and Sec. 4.2, PDF pp. 10--11. Panels use different color scales; coefficient magnitude is not a common importance score.}
\end{frame}

\begin{frame}{Stronger cooperation does not make every layer essential}
\fig{fig-08}{4.0cm}
\begin{itemize}
\item ARC-Easy pruning: HC and Pre-LN depend strongly on the first layer.
\item Full AttnRes is sensitive over more layers, including the final layer.
\item LogitLens: Full AttnRes follows a less monotonic prediction trajectory.
\end{itemize}
\papersource{Paper, Fig. 8, PDF p. 11. KL is $D_{\mathrm{KL}}(p_{\mathrm{final}}\Vert p_{\mathrm{probe}})$, so it measures agreement with the final model prediction.}
\end{frame}

\begin{frame}{Block aggregation can weaken early sequential reuse}
\fig{fig-09}{3.65cm}
\begin{itemize}
\item Fixed eight blocks: sublayer block size $b=2L/8$ grows with depth.
\item Green: weight on the current running sum. Early near-zero spans grow to about 2, 5, and 8 layers.
\item Plausible mechanism: a new update is computed but receives little immediate use.
\end{itemize}
\papersource{Paper, Fig. 9, PDF p. 12; Appendix A, p. 19. Evidence is for this schedule; no isolated block-count intervention is reported.}
\end{frame}

\begin{frame}{mHC's residual transport concentrates stream directions}
\fig{fig-10}{3.65cm}
\begin{itemize}
\item Effective rank: HC $2.53$--$2.81$; mHC $1.44$--$1.65$.
\item mHC directions approach collinearity; smaller singular values decay.
\item For doubly stochastic $A$, $AU=UA=U$, with $U=\mathbf1\mathbf1^\top/m$.
\end{itemize}
\papersource{Paper, Fig. 10 and Eq. (10), PDF pp. 12--13. Convergence toward $U$ requires contraction of nonuniform directions. Intervals use document bootstrap.}
\end{frame}

\begin{frame}{Depth increases prefill costs at similar parameter size}
\fig{fig-11}{3.35cm}
\begin{itemize}
\item $r:76.0\to9.1$: about $3.0\to4.4$ TFLOPs/sequence; KV cache $>2\times$.
\item Full AttnRes has the largest peak memory; HC the highest measured GPU-hours.
\item Near-fixed $Ld^2$ does not fix attention $T^2Ld$ or cache $TLd$.
\end{itemize}
\papersource{Paper, Fig. 11, PDF p. 13. 400M, $T=4096$, BF16, A100-SXM4-40GB; batch 8 for memory/time. Time is per 1B prefill tokens, not training time.}
\end{frame}

\begin{frame}{The scope of control matters for interpretation}
\begin{itemize}
\item \textbf{Parameter overhead varies.} At $L=24$: Pre-LN 405.41M; Full AttnRes 405.51M; HC 405.70M; mHC 410.13M; MoDA 453.64M.
\medskip
\item \textbf{Optimization differs.} LNS needs a larger LR; HC becomes less stable at larger scales; some LR choices are transferred.
\medskip
\item \textbf{Shape and compute change together.} Matching parameters does not match prefill FLOPs, activation memory, or latency.
\medskip
\item \textbf{Diagnostics answer different questions.} Similarity, internal dependence, pruning sensitivity, and predictive quality need not rank models identically.
\end{itemize}
\papersource{Paper, Table 6, Appendix C, Secs. 3--4, PDF pp. 6--13, 20, 22--23. MoDA's overhead is about 11.9\% relative to Pre-LN.}
\end{frame}

\begin{frame}{Accessible history is part of the depth-scaling design}
\begin{enumerate}
\item \textbf{Residual design changes shape preference.}\\
Full AttnRes and HC make deeper shapes useful in the 400M benchmark.
\medskip
\item \textbf{History access has different realizations.}\\
Explicit retrieval and recursive multi-stream transport preserve differentiated routes.
\medskip
\item \textbf{The practical optimum also depends on cost.}\\
Choose depth, routing granularity, optimization, and hardware budget together.
\end{enumerate}
\begin{block}{Next experiments suggested by this reading}
Vary block count; isolate mHC constraints; compare at matched FLOPs or GPU-hours.
\end{block}
\papersource{Synthesis of the paper's results. Proposed experiments are reading discussion, not reported findings.}
\end{frame}

\appendix

\begin{frame}{Reference and author-provided resources}
\textbf{Keyu Wang, Yangyi Huang, Jiale Kang, David Gonz\'alez-Mart\'inez, Weiyang Liu, Shiwei Liu.}
\medskip

\emph{DepthBench: Measuring How Residual Connections Enable More Computational Depth.}
\medskip

arXiv:2609.32534v1, submitted 26 September 2026.
\begin{itemize}
\item \href{https://arxiv.org/abs/2609.32534v1}{Versioned paper on arXiv}
\item \href{https://github.com/keyu-wang-2002/DepthBench/tree/main}{DepthBench code repository}
\item \href{https://huggingface.co/aspect-ratio-scaling}{Author checkpoint collection}
\end{itemize}
\medskip
Figures are from the paper's v1 TeX source. Figure 2(c) is cropped; Figure 5 uses selected panels rearranged for readability.
\papersource{Paper identity and resource links: PDF p. 1. All page references count from the first PDF page.}
\end{frame}

\begin{frame}{Backup: normalization methods change different scales}
\begin{align*}
\text{Sandwich-LN:}\quad h_\ell
&=h_{\ell-1}+\mathrm{LN}_{\mathrm{out}}\bigl(F_\ell(\mathrm{LN}_{\mathrm{in}}(h_{\ell-1}))\bigr)\\[2mm]
\text{LNS:}\quad h_\ell
&=h_{\ell-1}+F_\ell\bigl(\ell^{-1/2}\mathrm{LN}(h_{\ell-1})\bigr)\\[2mm]
\text{DeepNorm:}\quad h_\ell
&=\mathrm{LN}\bigl(\alpha h_{\ell-1}+F_\ell(h_{\ell-1})\bigr)\\[2mm]
\text{KEEL:}\quad h_\ell
&=\mathrm{LN}_{\mathrm{post}}\bigl(\alpha h_{\ell-1}+F_\ell(\mathrm{LN}_{\mathrm{pre}}(h_{\ell-1}))\bigr)
\end{align*}
\begin{itemize}
\item LNS shares the block-index scale between attention and FFN.
\item DeepNorm: $\alpha=(2L)^{1/4}$, with depth-aware initialization.
\item KEEL: gain based on sublayer count, with a special first block.
\end{itemize}
\papersource{Paper, Table 1 and Appendix A, PDF pp. 4, 19. Generic sublayer equations; method-specific indexing/initialization follows Appendix A.}
\end{frame}

\begin{frame}{Backup: learning rate and probe definitions matter}
\begin{itemize}
\item 400M optimal LR: most methods $2\times10^{-3}$; DeepNorm $10^{-3}$; LNS $10^{-2}$.
\item Figure 5 instead fixes LR at $2\times10^{-3}$; LNS is suboptimal and DeepNorm is excluded after divergence.
\item Block AttnRes borrows Full AttnRes's LR. mHC LR sweeps use a near-uniform initialization; main results use GAP-8.
\end{itemize}
\begin{block}{LogitLens uses architecture-dependent states}
Pre-LN: current residual stream. HC: sum of four streams. Full AttnRes: the next layer's depth-query mixture.
\[
p_{\mathrm{probe}}^{(\ell)}=\operatorname{softmax}\bigl(W_{\mathrm{LM}}\operatorname{Norm}(h_\ell)\bigr)
\]
\end{block}
\papersource{Paper, Table 4; Sec. 4.2; Fig. 5 and Fig. 12 captions; Appendix C, PDF pp. 5, 8, 10, 22--23.}
\end{frame}

\begin{frame}{Backup: doubly stochastic does not imply collapse alone}
\[
U=\frac1m\mathbf1\mathbf1^\top,\qquad U^2=U
\]
\begin{block}{An explanatory example, not a measured model}
For $A=(1-\varepsilon)I+\varepsilon U$, $0<\varepsilon<1$,
\[
A^k=U+(1-\varepsilon)^k(I-U).
\]
The uniform direction survives; differences between streams contract.
\end{block}
\begin{itemize}
\item Identity and permutation matrices are also doubly stochastic, without this contraction.
\item Measured transport rank is not the rank of the full nonlinear network.
\end{itemize}
\papersource{Teaching derivation motivated by Eq. (10), PDF p. 13. The paper's explanation explicitly conditions on contraction of other directions.}
\end{frame}

\begin{frame}{Backup: the FLOP accounting explains the trade-off}
\[
F_{\mathrm{Pre-LN}}=L(8Td^2+6Td\,d_{\mathrm{ff}}+2T^2d)+F_{\mathrm{head}}
\]
\[
\begin{aligned}
\Delta F_{\mathrm{HC}}&=192TdL,\qquad
\Delta F_{\mathrm{mHC}}=480TdL\quad(m=4),\\
\Delta F_{\mathrm{Full}}&=4TdL(2L+3)\approx8TdL^2.
\end{aligned}
\]
\begin{itemize}
\item One multiply--add counts as two FLOPs; causal attention evaluates about half of query--key pairs.
\item $F_{\mathrm{head}}=2TdV$ for all-token logits, or $2dV$ for last-token logits.
\item Counts omit normalization, softmax, and other elementwise operations.
\item For fixed backbone $Ld^2$: $Ld\propto\sqrt L$, increasing KV and sequence attention costs.
\end{itemize}
\papersource{Paper, Appendix D, PDF pp. 23--24. The final proportionality is an explanatory fixed-backbone approximation.}
\end{frame}

\end{document}