% Reading seminar, compile with pdflatex.
\documentclass[aspectratio=169,11pt]{beamer}
\usepackage[T1]{fontenc}
\usepackage[utf8]{inputenc}
\usepackage{amsmath,amssymb,graphicx,booktabs}

\definecolor{AcademicBlue}{RGB}{26,57,86}
\definecolor{AcademicTeal}{RGB}{0,105,112}
\setbeamercolor{structure}{fg=AcademicBlue}
\setbeamercolor{frametitle}{fg=AcademicBlue,bg=white}
\setbeamercolor{title}{fg=AcademicBlue}
\setbeamercolor{alerted text}{fg=AcademicTeal}
\setbeamercolor{block title}{fg=white,bg=AcademicBlue}
\setbeamercolor{block body}{fg=black,bg=AcademicBlue!5}
\setbeamertemplate{navigation symbols}{}
\setbeamertemplate{itemize items}[circle]
\setbeamertemplate{footline}{%
  \hfill\usebeamerfont{page number in head/foot}%
  \insertframenumber\,/\,\inserttotalframenumber\hspace{4mm}\vspace{3mm}}
\setbeamersize{text margin left=8mm,text margin right=8mm}
\setbeamerfont{frametitle}{size=\Large,series=\bfseries}
\setbeamerfont{title}{size=\LARGE,series=\bfseries}
\newcommand{\papersource}[1]{\par\smallskip{\scriptsize\color{gray}#1\par}}


\title[Orthros]{Orthros}
\subtitle{Phase-Aware Heterogeneous Attention for Efficient Transformers}
\author{Paper: Yuhong Chou, Zehao Liu, Yuqi Pan, Qian Liu,\\Xianwei Chen, Zhongzhu Zhou, and Jibin Wu}
\date{Reading seminar \enspace|\enspace arXiv:2610.13735v1\\12 November 2026}
\newcommand{\paperfig}[2]{\centering\includegraphics[width=\linewidth,height=#2,keepaspectratio]{figures/#1}\par}
\begin{document}
\begin{frame}\titlepage\end{frame}
\begin{frame}{Build memory locally; read it globally only when predicting}

\begin{block}{Core idea}
A base stream builds full layerwise KV with sliding-window attention. Output-specific tokens query that KV globally.
\end{block}
\begin{itemize}
\item Prefill: one global query per layer, at the last prompt position.
\item Decode: a base/output token pair per generated step.
\item Strongest gains occur with long inputs and short outputs.
\end{itemize}
\small \href{https://arxiv.org/abs/2610.13735v1}{arXiv v1, 9 October 2026}. Accepted at NeurIPS 2026 per \href{https://www.zhongzhuzhou.org/zh/publication/}{coauthor page} and arXiv comments.\\
Official code not found as of 12 November 2026; \href{https://github.com/daiwk/auto-research/tree/main/src/auto_research/reproductions/orthros}{third-party core-mechanism implementation}.

\papersource{Paper, Abstract and Secs. 1/3, PDF pp. 1--7. Third-party implementation is a diagnostic, not full experimental reproduction.}
\end{frame}
\begin{frame}{Phase roles offer another way to allocate expensive attention}

\begin{tabular}{lp{9.3cm}}
\toprule
Full attention & Global reads at every prompt position produce quadratic prefill work.\\[4pt]
Efficient attention & Local windows or compressed states reduce work but can impair detailed recall.\\[4pt]
Fixed layer hybrid & Retains some global layers, so a quadratic term remains.\\[4pt]
Orthros & Keeps full historical KV; reduces the number of global queries during prefill.\\
\bottomrule
\end{tabular}\par
\medskip
The design targets \alert{context assimilation versus output prediction}, within each layer.

\papersource{Paper, Secs. 1--2, PDF pp. 1--4. Fixed-width/window asymptotic comparison.}
\end{frame}
\begin{frame}{The streams share weights and same-layer KV}

\paperfig{architecture.pdf}{4.7cm}
\begin{itemize}
\item Base $X$ provides $Q,K,V$; output $X_{\rm cross}$ provides global queries.
\item Both streams share output projection, normalization, and tokenwise FFNs.
\end{itemize}

\papersource{Paper, Fig. 1(a)--(b), PDF p. 5; Algorithms 1--2, PDF p. 6. SWA: local; CA: full causal cross-attention.}
\end{frame}
\begin{frame}{Lightweight query transforms specialize two shared paths}

\[
k_t=x_tW_K,\qquad v_t=x_tW_V
\]
\[
q_t=\alpha\odot(x_tW_Q)+\beta,\qquad
q'_t=\alpha'\odot(x'_tW_Q)+\beta'
\]
\begin{itemize}
\item $x_t,x'_t\in\mathbb R^{1\times d_{\rm model}}$; $q,k,v\in\mathbb R^{1\times d}$ in the single-head notation.
\item Shared large projections; separate learnable scale-offset vectors.
\item Only the base stream generates K/V. The output stream does not write cache entries.
\item $T_{\rm out}$ counts prediction positions, starting at the final prompt token.
\end{itemize}

\papersource{Paper, Sec. 3.1, PDF p. 4; Algorithm 1, PDF p. 6. Prime denotes output-specific representations.}
\end{frame}
\begin{frame}{The streams differ in their causal attention domains}

\[
\mathcal W_t=\{\max(1,t-w+1),\ldots,t\}
\]
\[
o_t=\operatorname{softmax}_{j\in\mathcal W_t}
\!\left(\frac{q_tk_j^\top}{\sqrt d}\right)V_{\mathcal W_t}
\]
\[
o'_t=\operatorname{softmax}_{j\le t}
\!\left(\frac{q'_tk_j^\top}{\sqrt d}\right)V_{1:t}
\]
\begin{itemize}
\item Local output for all base positions; global output only where predictions are required.
\item Default base window $w=128$; old KV remains available to global queries.
\item Final logits and language-model loss use the output stream.
\end{itemize}

\papersource{Paper, Eqs. (4)--(5), PDF p. 5; Algorithms 1--2, PDF p. 6. All reads remain causal.}
\end{frame}
\begin{frame}{Output states never affect the memory-building path}

\begin{block}{Consistency invariant}
Base states and their KV depend only on base states. Skipping unused output copies cannot change the stored KV.
\end{block}
\[
[A,B,C]\ \xrightarrow{\text{base path}}\ \{K_A,K_B,K_C;V_A,V_B,V_C\}
\]
\[
C'\ \xrightarrow{\text{global read}}\ D,
\qquad (D,D')\ \xrightarrow{\text{next step}}\ E
\]
\begin{itemize}
\item The prompt needs only $C'$ to predict the first output $D$.
\item At the next step, base $D$ updates cache and $D'$ predicts $E$.
\item This explains how training can use many output copies while prefill uses one.
\end{itemize}

\papersource{Teaching example derived from Paper Secs. 3.1--3.2, PDF pp. 4--7; not an original experiment.}
\end{frame}
\begin{frame}{One computation graph supports four processing regimes}

\paperfig{regimes.pdf}{3.8cm}
\begin{itemize}
\item PT: duplicate every position. SFT: duplicate response-prediction positions.
\item Prefill: duplicate the last prompt token. Decode: one new token and its copy.
\item Duplicated embeddings diverge into two representations through the layers.
\end{itemize}

\papersource{Paper, Fig. 1(c)--(f), PDF p. 5; Sec. 3.2, PDF pp. 6--7. Decode also reads historical KV.}
\end{frame}
\begin{frame}{Prefill is linear when only one output query is needed}

\[
C=\mathcal O(TwD)+\mathcal O(T_{\rm out}TD)
+\mathcal O((T+T_{\rm out})D^2)
\]
\begin{itemize}
\item $T$: base length; $D$: model width; $w$: fixed window.
\item Prefill, $T_{\rm out}=1$: linear in $T$ at fixed width and window.
\item $T_{\rm out}=o(T)$: subquadratic attention; $T_{\rm out}=\Theta(T)$: quadratic.
\item Pretraining duplicates all tokens and retains a quadratic global-read term.
\end{itemize}

\papersource{Paper, Sec. 3.3, PDF p. 7. A small fixed output/input ratio alone does not imply subquadratic asymptotics.}
\end{frame}
\begin{frame}{The savings concern query work, while KV stays complete}

\begin{itemize}
\item Every layer retains all historical base K/V for output retrieval.
\item Local SWA reads do not imply KV eviction.
\item PT and decode process extra output-specific states through shared FFNs and projections.
\item At low batch, shared weight reads may hide some extra arithmetic; behavior changes with the serving regime.
\end{itemize}
\medskip
\begin{block}{Trade-off}
More training/output-path computation in exchange for cheaper input processing and global history access.
\end{block}

\papersource{Paper, Secs. 3.3 and 5, PDF pp. 7, 12. No fixed-size cache compression is introduced.}
\end{frame}
\begin{frame}{Separate the quality experiments and their token budgets}

\begin{tabular}{lll}
\toprule
Experiment & Scale & Budget / protocol\\
\midrule
Scaling + SO ablation & 160M--6.8B & 10B tokens; length 2048\\
From scratch & 1.3B & 100B FineWeb-Edu tokens\\
Conversion & Llama3.1/Qwen3-8B & 80B tokens; length 32768\\
128K NIAH & Converted Qwen3-8B & Additional 20B ProLong tokens\\
\bottomrule
\end{tabular}\par
\medskip
\begin{itemize}
\item Orthros window: 128. The scratch-training SWA baseline uses 512.
\item Equal original-token counts do not equalize training FLOPs across the double-stream design.
\end{itemize}

\papersource{Paper, Secs. 4.1--4.4, PDF pp. 8--10; App. A, PDF pp. 19--20.}
\end{frame}
\begin{frame}{Scale-offset gains are strongest in the larger tested models}

\paperfig{orthus_ablation.png}{3.6cm}
\begin{itemize}
\item All scales see 10B tokens; the appendix reports six-task averages.
\item At 6.8B: SO 53.10, w/o SO 50.23, Transformer 50.31.
\item At 360M/780M, SO does not beat w/o SO; the improvement is not uniform.
\end{itemize}

\papersource{Paper, Fig. 2, PDF p. 8; precise values in Table 7, PDF p. 20. These are not the seven-task 100B-token scores.}
\end{frame}
\begin{frame}{Scratch training preserves competitive common-sense performance}

\centering
\begin{tabular}{lrrrrrr}
\toprule
Model & Transformer & Orthros & GDN & GLA & RetNet & SWA\\
\midrule
Avg. $\uparrow$ & 53.41 & \alert{53.98} & 53.29 & 51.90 & 51.96 & 53.22\\
\bottomrule
\end{tabular}\par
\medskip
\begin{itemize}
\item 1.3B, 100B tokens; seven common-sense tasks.
\item Orthros gains 0.57 percentage points in the mean.
\item OBQA improves: 40.20 vs. 37.80; PIQA is lower: 71.82 vs. 72.91.
\item This supports model quality at matched data, with extra training computation.
\end{itemize}

\papersource{Paper, Table 1 and Sec. 4.2, PDF p. 9. Scores follow each table column\textquotesingle s accuracy/normalized-accuracy convention.}
\end{frame}
\begin{frame}{Conversion should be compared with continued pretraining}

\centering
\begin{tabular}{lrrr}
\toprule
Eight-task average $\uparrow$ & Base & Base CPT & Orthros Conversion\\
\midrule
Llama3.1-8B & 65.47 & 67.00 & \alert{67.26}\\
Qwen3-8B & 67.87 & 67.55 & \alert{68.37}\\
\bottomrule
\end{tabular}\par
\medskip
\begin{itemize}
\item CPT and conversion use the same 80B original tokens.
\item Conversion gains vs. CPT: +0.26 and +0.82 percentage points.
\item MMLU is slightly below CPT for both families.
\item Token-matched controls isolate extra data, but not extra compute from token duplication.
\end{itemize}

\papersource{Paper, Table 2, PDF p. 9; training details in Table 6, PDF p. 20. Conversion requires training, not just inference-time replacement.}
\end{frame}
\begin{frame}{Global queries recover much of the recall lost by efficient baselines}

\centering
\begin{tabular}{lrrrrrr}
\toprule
Model & Transformer & Orthros & GDN & GLA & RetNet & SWA\\
\midrule
Avg. $\uparrow$ & 33.9 & \alert{34.4} & 28.4 & 27.1 & 25.7 & 28.8\\
\bottomrule
\end{tabular}\par
\medskip
\begin{itemize}
\item Six recall-intensive tasks, 1.3B scratch-trained models.
\item FDA: 24.1 vs. 16.4; SWDE: 39.9 vs. 36.7, against Transformer.
\item SQuAD is lower: 38.7 vs. 42.6; the average hides task differences.
\item NQ denotes NarrativeQA here, not Natural Questions.
\end{itemize}

\papersource{Paper, Table 3 and Sec. 4.4, PDF p. 10. Appendix A mentions 8B inconsistently; the main text and table specify 1.3B.}
\end{frame}
\begin{frame}{The converted model retrieves needles across 128K context}

\paperfig{orthros_niah.pdf}{4.2cm}
\begin{itemize}
\item Horizontal: context length. Vertical: insertion depth. Color: accuracy.
\item Qwen3-8B conversion, followed by 20B tokens of long-context training.
\item Near-perfect needle retrieval supports long-range access in this test.
\end{itemize}

\papersource{Paper, Fig. 3 and Sec. 4.4, PDF p. 10. This differs from the 1.3B recall-task comparison.}
\end{frame}
\begin{frame}{The 21.9x headline is an eight-GPU prefill result}

\paperfig{orthus_prefill.png}{3.6cm}
\begin{itemize}
\item Left: one GPU, up to 128K; 3.5$\times$ marked at 128K.
\item Right: eight GPUs with sequence parallelism, up to 1M; 21.9$\times$ marked at 1M.
\item FLA-style kernels; vertical axes in milliseconds. The serving experiment uses a separate setup.
\end{itemize}

\papersource{Paper, Fig. 4 and Sec. 4.5, PDF p. 11; App. A, PDF p. 19. Both models are 8B-scale.}
\end{frame}
\begin{frame}{End-to-end gains shrink as output generation grows}

\centering
\begin{tabular}{lrrrr}
\toprule
Input / output & Transformer E2E & Orthros E2E & TTFT gain & E2E gain\\
 & (seconds) & (seconds) & &\\
\midrule
128K / 128 & 84.911 & 24.647 & 4.54$\times$ & \alert{3.45$\times$}\\
127K / 1K & 137.715 & 76.345 & 4.40$\times$ & 1.80$\times$\\
120K / 8K & 548.357 & 488.830 & 4.46$\times$ & \alert{1.12$\times$}\\
\bottomrule
\end{tabular}\par
\medskip
\begin{itemize}
\item Single H100, SGLang, Triton attention, batch size 2.
\item First setup: TTFT drops from 77.249 s to 17.016 s.
\item Similar prefill savings contribute less once decoding dominates total time.
\end{itemize}

\papersource{Paper, Table 4, PDF p. 11. Gains calculated from reported values; input/output lengths follow the table headers.}
\end{frame}
\begin{frame}{Decode throughput stays comparable in the tested small batch}

\centering
\begin{tabular}{lrrrr}
\toprule
Input / output & \multicolumn{2}{c}{Output throughput} & \multicolumn{2}{c}{Decode-only throughput}\\
 & Transformer & Orthros & Transformer & Orthros\\
\midrule
128K / 128 & 3.0 & 10.4 & 33.7 & 33.3\\
127K / 1K & 14.9 & 26.8 & 33.2 & 33.7\\
120K / 8K & 29.9 & 33.5 & 34.1 & 34.6\\
\bottomrule
\end{tabular}\par
\medskip
\begin{itemize}
\item All rates in tokens/s, aggregated over batch=2.
\item Output throughput includes prefill; decode-only throughput excludes it.
\item Shared weight loads may hide extra arithmetic at small batch; larger-batch behavior is unestablished.
\end{itemize}

\papersource{Paper, Table 4, PDF p. 11; system interpretation in Secs. 3.3/5, PDF pp. 7, 12.}
\end{frame}
\begin{frame}{Phase heterogeneity can also be combined with layer hybrids}

\centering
\begin{tabular}{lrrr}
\toprule
Input / output & Linear Hybrid E2E & Orthros-H E2E & Gain\\
\midrule
128K / 128 & 28.130 s & 12.771 s & 2.20$\times$\\
127K / 1K & 46.308 s & 30.724 s & 1.51$\times$\\
120K / 8K & 189.302 s & 173.887 s & 1.09$\times$\\
\bottomrule
\end{tabular}\par
\medskip
\begin{itemize}
\item Orthros-H uses a 1:3 mixture of full/linear layers, with a GDN variant.
\item Compare it within the hybrid pair; standard Orthros retains full global reads during decoding.
\item This table reports speed; full matching quality results for Orthros-H are not supplied.
\end{itemize}

\papersource{Paper, Sec. 4.6 and Table 4, PDF p. 11. Single H100, SGLang, batch=2; gains calculated from reported values.}
\end{frame}
\begin{frame}{The design trades training work for cheaper input processing}

\begin{itemize}
\item \textbf{Training:} PT duplicates all token states; long output/training attention remains quadratic.
\item \textbf{Memory:} full historical KV grows with context length.
\item \textbf{MoE:} split token pairs can trigger different expert reads and break I/O sharing.
\item \textbf{Deployment:} batch size, KV bandwidth, and output length change the bottleneck.
\end{itemize}
\medskip
The target workload is \alert{prefill-heavy serving with strong global recall requirements}.

\papersource{Paper, Sec. 5, PDF p. 12. Agentic RL and broader serving configurations are future work.}
\end{frame}
\begin{frame}{A dependency separation makes the efficiency mechanism possible}

\begin{enumerate}
\item Build memory through a local base stream at every layer.
\item Query the full history through output-specific states only where needed.
\item Keep output states out of the memory-building dependency chain.
\item Measure quality, training compute, cache cost, and serving latency together.
\end{enumerate}
\medskip
Global access is preserved; the number of prefill global queries is reduced.

\papersource{Interpretation of Paper Secs. 3--5, PDF pp. 4--12.}
\end{frame}
\begin{frame}{References and implementation status}

\begin{itemize}
\item Chou, Liu, Pan, Liu, Chen, Zhou, and Wu.\\
\emph{Orthros: Phase-Aware Heterogeneous Attention for Efficient Transformers}.\\
\href{https://arxiv.org/abs/2610.13735v1}{arXiv:2610.13735v1}, 9 October 2026.
\item NeurIPS 2026 acceptance per \href{https://www.zhongzhuzhou.org/zh/publication/}{coauthor publication list} and arXiv comments.
\item Official code not found as of 12 November 2026.\\
\href{https://github.com/daiwk/auto-research/tree/main/src/auto_research/reproductions/orthros}{Third-party core mechanism} / \href{https://daiwk.github.io/auto-research/reproductions/2610.13735-orthros/README-en/}{scope description}.
\end{itemize}
\medskip
Figures come from the authors' v1 source/PDF. Calculated ratios are labeled. This deck is a reading seminar, not an author presentation.

\papersource{Publication and code checked 12 November 2026. Formal proceedings publication was not independently confirmed.}
\end{frame}
\appendix
\begin{frame}{Backup: duplicated positions keep their original causal meaning}

\[
Z=\operatorname{Concat}[X_{1:T},X_{T-T_{\rm out}+1:T}]
\]
\begin{itemize}
\item Appending copies is a tensor layout choice; each output copy keeps its original prediction position.
\item For a query at original position $t$, Eq. (5) allows only keys $j\le t$.
\item Prefill copies the final prompt token, which predicts the first response token.
\item At decode, current input length is one, but global key length includes all historical base tokens.
\end{itemize}
\medskip
The causal mask must respect these positions rather than the physical appended indices.

\papersource{Paper, Eqs. (4)--(5), PDF p. 5; Algorithms 1--2 and Sec. 3.2, PDF pp. 6--7. Implementation implication derived from the equations.}
\end{frame}
\begin{frame}{Backup: long generation restores a quadratic read term}

For prompt length $P$ and response length $R$:
\[
\sum_{r=1}^R(P+r)=RP+\frac{R(R+1)}2
\]
\begin{itemize}
\item This counts historical keys read by successive global output queries, up to endpoint conventions.
\item Prefill adds a final-prompt global query of length about $P$.
\item Fixed $R$: cost grows linearly with $P$ at fixed model/window size.
\item Large $R$: the response's own quadratic global-read term remains.
\end{itemize}

\papersource{Teaching expansion of Paper Sec. 3.3, PDF p. 7; endpoint constants omitted. No timing claim is inferred from FLOPs alone.}
\end{frame}
\begin{frame}{Backup: training protocols need distinct budgets}

\begin{tabular}{lll}
\toprule
 & Scaling / ablation & 8B conversion / CPT\\
\midrule
Original tokens & 10B & 80B\\
Sequence length & 2048 & 32768\\
Global token batch & 0.25M & 8M\\
Steps & 40K & 10K\\
Learning rate & $3\times10^{-4}$ & $3\times10^{-5}$\\
Warmup steps & 1500 & 500\\
Weight decay & .05 & .1\\
Seed & 42 & 42\\
\bottomrule
\end{tabular}\par
\medskip
The 1.3B scratch comparison separately specifies 100B tokens and a 2M-token batch. All use H100 80GB training hardware.

\papersource{Paper, Tables 5--6, PDF pp. 19--20; Sec. 4.2, PDF p. 9. Original tokens exclude duplicated token-state processing.}
\end{frame}
\begin{frame}{Backup: precise scale-offset results do not form a uniform ranking}

\centering
\begin{tabular}{lrrr}
\toprule
Size & Transformer & Orthros w/o SO & Orthros w/ SO\\
\midrule
160M & 41.83 & 41.96 & 42.08\\
360M & 44.58 & 44.45 & 44.44\\
780M & 46.39 & 46.67 & 46.48\\
1.3B & 48.32 & 47.58 & 48.74\\
2.7B & 50.87 & 50.03 & 51.83\\
6.8B & 50.31 & 50.23 & 53.10\\
\bottomrule
\end{tabular}\par
\medskip
\begin{itemize}
\item Six-task averages at 10B tokens; Social-IQa is absent from this table.
\item Table 1 uses seven tasks at 100B tokens. Keep these protocols separate.
\end{itemize}

\papersource{Paper, Table 7, PDF p. 20, and Table 1, PDF p. 9. Metrics follow each table\textquotesingle s column labels.}
\end{frame}
\end{document}