% SimpleHC paper reading; compile with pdflatex and no shell escape.
\documentclass[aspectratio=169,11pt]{beamer}
\usepackage[T1]{fontenc}
\usepackage[utf8]{inputenc}
\usepackage{amsmath,amssymb,graphicx,booktabs}

\definecolor{AcademicBlue}{RGB}{26,57,86}
\definecolor{AcademicTeal}{RGB}{0,105,112}
\definecolor{SoftGray}{RGB}{244,247,249}
\setbeamercolor{structure}{fg=AcademicBlue}
\setbeamercolor{frametitle}{fg=AcademicBlue,bg=white}
\setbeamercolor{title}{fg=AcademicBlue}
\setbeamercolor{alerted text}{fg=AcademicTeal}
\setbeamercolor{block title}{fg=white,bg=AcademicBlue}
\setbeamercolor{block body}{fg=black,bg=SoftGray}
\setbeamertemplate{navigation symbols}{}
\setbeamertemplate{itemize items}[circle]
\setbeamertemplate{footline}{%
  \hfill\usebeamerfont{page number in head/foot}%
  \insertframenumber\,/\,\inserttotalframenumber\hspace{4mm}\vspace{3mm}}
\setbeamersize{text margin left=8mm,text margin right=8mm}
\setbeamerfont{frametitle}{size=\Large,series=\bfseries}
\newcommand{\papersource}[1]{\par\smallskip{\scriptsize\color{gray}#1\par}}
\newcommand{\evidence}[1]{\vfill\papersource{Source: Zhang et al., arXiv:2611.05984v1, #1.}}

\title[SimpleHC]{On the Design of Hyper-Connections:\\ Simplification, Stability, and Scalability}
\subtitle{A paper discussion on SimpleHC}
\date{arXiv:2611.05984v1 \quad Paper submitted 4 Nov 2026}

\begin{document}

\begin{frame}[plain]
  \centering
  {\LARGE\bfseries\color{AcademicBlue}On the Design of Hyper-Connections:\par}
  \smallskip
  {\LARGE\bfseries\color{AcademicBlue}Simplification, Stability, and Scalability\par}
  \medskip
  {\large SimpleHC: a research group paper discussion\par}
  \medskip
  {\small Xiangdong Zhang, Xiaohan Qin, Sunan Zou, Tuo Dai, Bizhao Shi, Zhuo Xia,\par
  Yebin Yang, Yizhou Chen, Yuliang Liu, Yu Cheng, Junchi Yan\par}
  \medskip
  {\small \href{https://arxiv.org/abs/2611.05984v1}{arXiv:2611.05984v1} \quad Submitted 4 Nov 2026\par}
  \smallskip
  {\small Publication not found as of 5 Nov 2026.\par}
  {\small \href{https://github.com/aHapBean/SimpleHC}{Paper-linked code repository} returned 404 on 5 Nov 2026.\par}
  \vfill
  {\small\color{AcademicTeal}\bfseries Takeaway: shared access to the expanded residual state cuts overhead while keeping mHC-level quality.\par}
\end{frame}

\begin{frame}{The question is which HC interactions justify their cost}
  \begin{block}{Problem}
    Hyper-Connections add read gates, write gates, and residual-stream mixing. Which interactions improve quality, and which state accesses dominate training overhead?
  \end{block}
  \begin{itemize}
    \item Study the three components separately on 15B-A1B MoE models.
    \item Keep learned read/write gates; replace mixing with identity transitions.
    \item Share state access across four sublayers, retaining sequential output contributions.
  \end{itemize}
  \evidence{Abstract and Sec. 1, PDF pp. 1--2}
\end{frame}

\begin{frame}{HC reads, transforms, and writes across $n$ residual streams}
  \begin{align*}
    u_l &= X_l^\top h_l^{\mathrm{pre}}, & o_l &= F_l(u_l),\\
    X_{l+1} &= H_l^{\mathrm{res}}X_l+h_l^{\mathrm{post}}o_l^\top.
  \end{align*}
  \begin{itemize}
    \item $X_l\in\mathbb R^{n\times C}$: $n$ residual streams of width $C$; $u_l,o_l\in\mathbb R^C$.
    \item Read/write gates select and distribute information; $H_l^{\mathrm{res}}$ mixes old streams.
    \item mHC conditions gates on the full $nC$ state and approximately constrains mixing to be doubly stochastic using Sinkhorn normalization.
  \end{itemize}
  \evidence{Sec. 2, PDF pp. 2--3}
\end{frame}

\begin{frame}{Two interaction groups reach mHC-level quality on 15B}
  \centering
  \begin{tabular}{lcc}
    \toprule
    15B-A1B design & Pile loss $\downarrow$ & 5-task mean $\uparrow$\\
    \midrule
    mHC: learned gates + Sinkhorn mixing & 1.659 & 55.0\\
    identityHC: learned gates + identity & 1.660 & 55.4\\
    Fixed gates + unconstrained mixing & 1.660 & 56.0\\
    Learned gates + unconstrained mixing & 1.657 & 56.2\\
    \bottomrule
  \end{tabular}
  \medskip
  \begin{block}{Design implication}
    Learned read/write gating and unconstrained mixing each work as an interaction group. SimpleHC keeps the gating group to avoid repeated residual-matrix products.
  \end{block}
  \evidence{Sec. 5.2, Table 2, PDF p. 7}
\end{frame}

\begin{frame}{Identity transitions avoid cumulative mixing paths}
  \[
    c_{j,k}=(h_j^{\mathrm{pre}})^\top
      \underbrace{H_{j-1}^{\mathrm{res}}\cdots H_{k+1}^{\mathrm{res}}}_{\Phi_{k+1\to j}}
      h_k^{\mathrm{post}}.
  \]
  \begin{itemize}
    \item $c_{j,k}$ weights an earlier output $o_k$ in a later input $u_j$.
    \item With $H^{\mathrm{res}}=I$, the path coefficient is just the read/write gate inner product.
    \item Unconstrained mixing showed larger sampled peak coefficients; Sinkhorn mixing offered little mean-score gain over identity in the 15B study.
  \end{itemize}
  \begin{block}{Choice}
    Use full-state, learned read/write gates with identity residual transitions.
  \end{block}
  \evidence{Sec. 6.1 and App. C--D, PDF pp. 10, 19--20}
\end{frame}

\begin{frame}{SimpleHC simplifies structure and groups execution}
  \centering
  \includegraphics[width=0.93\linewidth,height=0.72\textheight,keepaspectratio]{figures/method.pdf}
  \papersource{Original Fig. 2, PDF p. 3. The right panel keeps sublayers sequential but reads and writes the wide state at group boundaries.}
\end{frame}

\begin{frame}{Layer-specific gates share one group-entry input}
  \[
    \bar h_j^{\mathrm{pre}}=g_j^{\mathrm{pre}}(X_L),\qquad
    \bar h_j^{\mathrm{post}}=g_j^{\mathrm{post}}(X_L),
    \quad L\le j<L+K.
  \]
  \begin{itemize}
    \item Default $K=4$: two Transformer blocks, four attention/FFN sublayers.
    \item Each sublayer has independent read/write parameters; all gate mappings see $X_L$.
    \item This changes gate conditioning from the current $X_j$ to $X_L$. It makes all gates available before sequential sublayer execution.
  \end{itemize}
  \begin{block}{The sole approximation}
    Shared-input gates replace per-sublayer-state gates; output reconstruction below is exact for the chosen shared-input gates.
  \end{block}
  \evidence{Sec. 3.2, Eqs. (3)--(4), PDF pp. 4--5}
\end{frame}

\begin{frame}{Scalar corrections preserve earlier sublayer outputs}
  \begin{align*}
    u_j &= X_L^\top\bar h_j^{\mathrm{pre}}
       +\sum_{k=L}^{j-1}d_{j,k}o_k,\\
    d_{j,k} &= (\bar h_j^{\mathrm{pre}})^\top\bar h_k^{\mathrm{post}}.
  \end{align*}
  \begin{itemize}
    \item Batch the $K$ base reads $X_L^\top\bar h_j^{\mathrm{pre}}$ in one scan of $X_L$.
    \item Each $d_{j,k}$ is a scalar; its product with $o_k\in\mathbb R^C$ brings the earlier output into layer $j$.
    \item Execute $F_L,\ldots,F_{L+K-1}$ sequentially, then write all updates once:
  \end{itemize}
  \[
    X_{L+K}=X_L+\sum_{k=L}^{L+K-1}\bar h_k^{\mathrm{post}}o_k^\top.
  \]
  \evidence{Sec. 3.2, Algorithm 1 and Eqs. (5)--(6), PDF pp. 4--5}
\end{frame}

\begin{frame}{Grouped execution equals sequential execution for fixed shared gates}
  \begin{block}{Induction over sublayers}
    At group entry, both schedules read the same $X_L$. If prior outputs agree, expanding sequential residual additions gives exactly the base read plus scalar corrections; the next outputs agree as well.
  \end{block}
  \begin{itemize}
    \item The group-boundary write equals the sum of all sequential writes.
    \item Gradients agree for the same function if the dependence of gates on $X_L$ is retained.
    \item Floating-point reduction order may change values; this result does not equate shared-input gates with the original $g_j(X_j)$.
  \end{itemize}
  \evidence{App. B, PDF pp. 18--19}
\end{frame}

\begin{frame}{Grouping halves logical residual-state traffic at $n=K=4$}
  \[
    Q_{\mathrm{SimpleHC}}(K)=\frac{4nC}{K}+2C+
      \frac{C(K-1)(K+4)}{2K}.
  \]
  \centering
  \begin{tabular}{lc}
    \toprule
    Residual operations, per token per sublayer & Logical traffic\\
    \midrule
    Fused mHC / identityHC, $n=4$ & $18C$\\
    SimpleHC, $n=K=4$ & $9C$\\
    \bottomrule
  \end{tabular}
  \medskip
  \begin{itemize}
    \item Wide gate reads, base reads, and write-back are amortized over $K$ layers.
    \item The correction term grows with $K$; $K=4$ is the lowest among tested $K=2,4,8,16$.
  \end{itemize}
  \evidence{Sec. 4.3, Table 1 and Eqs. (9)--(10), PDF pp. 5--6; logical forward activation traffic only}
\end{frame}

\begin{frame}{Specialized kernels turn fewer accesses into lower step time}
  \begin{columns}[T,onlytextwidth]
    \column{0.48\textwidth}
      \begin{block}{Forward}
        \begin{itemize}
          \item Jointly project $K$ read/write gates and compute RMS statistics.
          \item Batch base reads; fuse scalar corrections.
          \item Write the expanded state once per group.
        \end{itemize}
      \end{block}
    \column{0.48\textwidth}
      \begin{block}{Backward}
        \begin{itemize}
          \item Reuse one wide gradient scan across $K$ outputs.
          \item Fuse projection and RMS input gradients.
          \item Pack small gate reductions into tensor-core products.
        \end{itemize}
      \end{block}
  \end{columns}
  \medskip
  \centering
  At $n=K=4$, one $4C\times32$ projection supplies eight layer-specific gate vectors.
  \evidence{Sec. 4.1--4.2, PDF pp. 5--6; implementations use DeepGEMM and Triton}
\end{frame}

\begin{frame}{Quality and efficiency use distinct experimental setups}
  \begin{columns}[T,onlytextwidth]
    \column{0.48\textwidth}
      \begin{block}{Component study}
        15B-A1B MoE, $n=4$.\\
        Pile test loss and five downstream tasks.
      \end{block}
      \begin{block}{Training-step efficiency}
        Separate eight-layer models, $C=1024$--6144.\\
        End-to-end step overhead vs. Vanilla.
      \end{block}
    \column{0.48\textwidth}
      \begin{block}{Quality at scale}
        30B-A2B MoE, same residual variants.\\
        Pile test loss and 14 downstream tasks.
      \end{block}
      \begin{block}{Stress stability}
        230B-A6B, 64 layers, 10$\times$ reference LR.\\
        Gradient and loss over 28k steps.
      \end{block}
  \end{columns}
  \smallskip
  Quality backbones use MLA and 8192-token sequences; main training uses Muon/AdamW.
  \evidence{Sec. 5.1 and App. A/E, PDF pp. 7, 16--18, 21--22}
\end{frame}

\begin{frame}{At $K=4$, independent gates and scalar corrections protect quality}
  \centering
  \begin{tabular}{lcc}
    \toprule
    15B-A1B configuration & Pile loss $\downarrow$ & 5-task mean $\uparrow$\\
    \midrule
    mHC, per-sublayer gates & 1.659 & 55.0\\
    identityHC, per-sublayer gates & 1.660 & 55.4\\
    SimpleHC, $K=4$, independent gates & 1.660 & 55.0\\
    Shared input, $K=8$ & 1.662 & 54.3\\
    $K=4$, one reused gate pair & 1.667 & 53.5\\
    $K=4$, no scalar corrections & 1.705 & 47.9\\
    \bottomrule
  \end{tabular}\par
  \smallskip
  \alert{Layer-specific gates and scalar corrections both matter for quality.}
  \evidence{Sec. 5.3, Table 3, PDF p. 8; no-correction variant still writes all outputs at group exit}
\end{frame}

\begin{frame}{At 30B, SimpleHC matches mHC on aggregate quality}
  \centering
  \begin{tabular}{lcccc}
    \toprule
    30B-A2B metric & Vanilla & mHC & identityHC & SimpleHC\\
    \midrule
    Pile loss $\downarrow$ & 1.579 & 1.561 & 1.561 & 1.561\\
    14-task mean $\uparrow$ & 59.3 & 62.9 & 62.8 & 63.1\\
    MATH $\uparrow$ & 38.4 & 38.4 & 38.9 & 41.5\\
    HumanEval $\uparrow$ & 39.6 & 47.6 & 48.2 & 45.1\\
    C3 $\uparrow$ & 88.1 & 89.6 & 89.6 & 91.4\\
    \bottomrule
  \end{tabular}
  \medskip
  \begin{itemize}
    \item Mean score is comparable; individual tasks have different winners.
    \item The result supports scaling quality from 15B to the tested 30B MoE setup.
  \end{itemize}
  \evidence{Sec. 5.4, Table 4, PDF p. 9}
\end{frame}

\begin{frame}{SimpleHC adds only 2.4\% step time at the largest tested width}
  \begin{columns}[T,onlytextwidth]
    \column{0.67\textwidth}
      \includegraphics[width=\linewidth,height=0.72\textheight,keepaspectratio]{figures/training.pdf}
    \column{0.30\textwidth}
      \begin{block}{$C=6144$}
        SimpleHC: 2.4\%\\[4pt]
        mHC: about 7.0\%\\[4pt]
        identityHC: about 7.0\%
      \end{block}
      \smallskip
      \small Separate eight-layer benchmark, efficient implementations.
  \end{columns}
  \evidence{Sec. 5.5, original Fig. 3(a), PDF p. 9; relative to Vanilla training step time}
\end{frame}

\begin{frame}{Saved activation storage falls by about 31\% in the paper's accounting}
  \centering
  \begin{tabular}{lrr}
    \toprule
    Eight layers, $C=2048$, 8192 tokens & mHC & SimpleHC\\
    \midrule
    Retained storage for backward & 3044.5 MiB & 2092.875 MiB\\
    Relative reduction & \multicolumn{2}{c}{31.26\%}\\
    \bottomrule
  \end{tabular}
  \medskip
  \begin{itemize}
    \item One group-entry wide state is retained rather than one entry per sublayer.
    \item Counts sublayer inputs/outputs, residual copies, base reads, and HC auxiliary tensors.
    \item Excludes attention/FFN internals, optimizer/model states, workspaces, and allocator reservations: this is \alert{not peak GPU memory}.
  \end{itemize}
  \evidence{Sec. 5.5 and App. A.2, Table 7, PDF pp. 9, 17--18; BF16, microbatch 1, no recomputation}
\end{frame}

\begin{frame}{Gates differ by sublayer yet are predictable from $X_L$}
  \begin{columns}[T,onlytextwidth]
    \column{0.65\textwidth}
      \includegraphics[width=\linewidth,height=0.71\textheight,keepaspectratio]{figures/gate-prediction.pdf}
    \column{0.32\textwidth}
      \begin{block}{Write-gate MAE}
        Trained predictor: 0.01034\\[4pt]
        Frozen mapping: 0.05044\\[4pt]
        Constant gate: 0.06893
      \end{block}
      \small Frozen 15B identityHC teacher; only independent, layer-specific predictors are trained.
  \end{columns}
  \evidence{Sec. 6.2, original Fig. 5(b), PDF p. 11; test values from App. D.3, PDF p. 21}
\end{frame}

\begin{frame}{SimpleHC stays stable in a combined 230B-A6B stress test}
  \begin{columns}[T,onlytextwidth]
    \column{0.67\textwidth}
      \includegraphics[width=\linewidth,height=0.72\textheight,keepaspectratio]{figures/gradient-stress.pdf}
    \column{0.30\textwidth}
      \begin{block}{Combined stress}
        64 layers\\[3pt]
        512 routed experts\\[3pt]
        10$\times$ reference LR
      \end{block}
      \small mHC spikes reach about $1.9\times10^6$; SimpleHC and identityHC stay bounded across 28k steps.
  \end{columns}
  \evidence{Sec. 6.3, original Fig. 6(b), PDF p. 11; setting and diagnostics in App. E, PDF pp. 21--23}
\end{frame}

\begin{frame}{The main benefit comes from shared access, with clear scope}
  \begin{enumerate}
    \item Learned read/write gates with identity transitions provide mHC-level quality in the tested 15B and 30B MoE models.
    \item Shared-input gates plus exact scalar corrections reduce logical residual traffic from $18C$ to $9C$ at $n=K=4$.
    \item Specialized execution yields 2.4\% extra step time at $C=6144$ in an eight-layer benchmark.
  \end{enumerate}
  \medskip
  \begin{block}{Interpret the evidence at its measured scope}
    The timing benchmark is eight layers; the 31\% storage figure is a selected activation accounting; the stress test combines several pressures.
  \end{block}
  \evidence{Secs. 3--6 and App. A/E, PDF pp. 3--12, 17--18, 21--23}
\end{frame}

\begin{frame}{Paper and status links}
  \begin{itemize}
    \item Xiangdong Zhang et al. \emph{On the Design of Hyper-Connections: Simplification, Stability, and Scalability}. \href{https://arxiv.org/abs/2611.05984v1}{arXiv:2611.05984v1}, submitted 4 Nov 2026.
    \item \href{https://arxiv.org/pdf/2611.05984v1}{Versioned paper PDF} and \href{https://arxiv.org/src/2611.05984v1}{author TeX source}.
    \item Publication not found as of 5 Nov 2026; the \href{https://github.com/aHapBean/SimpleHC}{paper-linked code repository} returned 404 on that date.
  \end{itemize}
  \medskip
  \begin{block}{Discussion prompt}
    Can group-entry conditioning remain accurate for larger $K$ or a different residual architecture without reintroducing frequent wide-state reads?
  \end{block}
\end{frame}

\appendix

\begin{frame}{Backup: fixed gates expose the role of each interaction group}
  \centering
  \begin{tabular}{lcc}
    \toprule
    15B-A1B configuration & Pile loss $\downarrow$ & 5-task mean $\uparrow$\\
    \midrule
    identityHC & 1.660 & 55.4\\
    Fixed read gate, learned write gate & 1.676 & 53.6\\
    Learned read gate, fixed write gate & 1.678 & 52.8\\
    Fixed gates + Sinkhorn mixing & 1.678 & 52.5\\
    Fixed gates + unconstrained mixing & 1.660 & 56.0\\
    \bottomrule
  \end{tabular}\par
  \medskip
  \small Fixed read weights are $1/n$ and fixed write weights are $1$; all comparisons use the table's stated gate inputs and mixing type.
  \evidence{Sec. 5.2, Table 2, PDF p. 7}
\end{frame}

\begin{frame}{Backup: the stress observations do not isolate one cause}
  \begin{itemize}
    \item In one recorded event, an FFN branch output became very large before write gating; a finite-iteration Sinkhorn matrix later had row sums $(2,1,0.5,0.5)$ and could further amplify it.
    \item A separate backward spike showed a strong input-dependent mixing-gradient path. The paper leaves mathematical sensitivity versus numerical backward effects unresolved.
    \item In standard 15B training, maximum post-warmup 1,000-step rolling gradient-norm standard deviation was 0.032 for SimpleHC, versus 0.206 for mHC and 0.248 for identityHC.
  \end{itemize}
  \evidence{Sec. 6.3 and App. E.2--E.3, PDF pp. 11, 22--23}
\end{frame}

\end{document}