% Research reading based on arXiv:2609.36259v1. % Adapted from the arxiv-paper-reader Beamer template; compile with pdfLaTeX. \documentclass[aspectratio=169,11pt]{beamer} \usepackage[T1]{fontenc} \usepackage[utf8]{inputenc} \usepackage{amsmath,amssymb,graphicx,booktabs} \definecolor{AcademicBlue}{RGB}{26,57,86} \definecolor{AcademicTeal}{RGB}{0,105,112} \setbeamercolor{structure}{fg=AcademicBlue} \setbeamercolor{frametitle}{fg=AcademicBlue,bg=white} \setbeamercolor{title}{fg=AcademicBlue} \setbeamercolor{alerted text}{fg=AcademicTeal} \setbeamercolor{block title}{fg=white,bg=AcademicBlue} \setbeamercolor{block body}{fg=black,bg=AcademicBlue!5} \setbeamertemplate{navigation symbols}{} \setbeamertemplate{itemize items}[circle] \setbeamertemplate{footline}{\hfill\usebeamerfont{page number in head/foot}\insertframenumber\,/\,\inserttotalframenumber\hspace{4mm}\vspace{3mm}} \setbeamersize{text margin left=8mm,text margin right=8mm} \setbeamerfont{frametitle}{size=\Large,series=\bfseries} \setbeamerfont{title}{size=\LARGE,series=\bfseries} \newcommand{\papersource}[1]{\par\smallskip{\scriptsize\color{gray}#1\par}} \newcommand{\U}{\mathcal U} \newcommand{\Kb}{\overline K} \newcommand{\Vb}{\overline V} \newcommand{\softmax}{\operatorname{softmax}} \newcommand{\sg}{\operatorname{sg}} \newcommand{\LA}{\operatorname{ScalarGatedLA}} \newcommand{\diag}{\operatorname{Diag}} \title[CyFA]{CyFA: Linear Sequence Modeling\newline with Relative-Time-Partitioned Memory} \subtitle{Research paper discussion (25--30 minutes + backup)} \author[Reading presentation]{Paper authors: Yixiao Chen, Shuojin Yang, Shi-Min Hu\\[2mm]{\small Reading presentation prepared by Codex}} \date{arXiv:2609.36259v1 --- September 28, 2026\\[1mm]Reading date: October 2, 2026} \hypersetup{pdftitle={CyFA: Research paper discussion},pdfauthor={Codex; based on Chen, Yang, and Hu (2026)}} \begin{document} \begin{frame} \titlepage \vspace{-5mm} \begin{center}\small\color{AcademicTeal}Organize a fixed memory by model age; compute it with two scalar recurrences.\end{center} \end{frame} \begin{frame}{More history competes for the same recurrent state} \[ S_t=S_{t-1}+k_tv_t^\top, \qquad o_t=S_t^\top q_t=\sum_{i\leq t}v_i(k_i^\top q_t). \] \begin{itemize} \item A fixed matrix compresses every earlier association. \item Forget gates reduce persistence; Delta Rule updates correct the association addressed by a new key. \item Multiple useful associations can still interfere at retrieval. \end{itemize} \begin{block}{CyFA's question} Can the same state budget keep associations more distinguishable by organizing their temporal addresses? \end{block} \papersource{Paper, Secs. 1--2, PDF pp. 1--3; Eq. (1).} \end{frame} \begin{frame}{A learned clock moves old writes around fixed slots} \centering \includegraphics[width=\linewidth,height=.51\textheight,keepaspectratio]{figures/fig-01-cyclic-memory.pdf} \begin{itemize} \item New KV enters age zero; old KV moves by $\delta_t\in(0,1)$. \item The unrolled helix is a visualization: every cycle reuses the \alert{same} slots. \item Scalar decay controls how much old content survives. \end{itemize} \papersource{Paper, Fig. 1, PDF p. 2. Original figure; $R$ and write strength are omitted in the illustration.} \end{frame} \begin{frame}{Transport, forgetting, and writing have separate controls} \[ \lambda_t=\lambda_{t-1}+\delta_t,\qquad K_t\in\mathbb R^{m\times d_k},\quad V_t\in\mathbb R^{m\times d_v} \] \[ \begin{aligned} K_t&=\alpha_tP(\delta_t)K_{t-1}+\beta_te_0k_t^\top,\\ V_t&=\alpha_tP(\delta_t)V_{t-1}+\beta_te_0v_t^\top. \end{aligned} \] \begin{itemize} \item $P(\delta_t)$: joint cyclic movement of aligned K/V states. \item $\alpha_t$: scalar retention; $\beta_t$: current write strength. \item $e_0$: age-zero slot; $\lambda_t-\lambda_i$: model age of write $i$. \end{itemize} \[ o_t=(RV_t)^\top\softmax(RK_tq_t),\qquad R\in\mathbb R^{m\times m}. \] \papersource{Paper, Eqs. (14)--(18), PDF p. 5. $R$ is learned, initialized to $I$; conventional dot-product scaling omitted as in the paper.} \end{frame} \begin{frame}{Fractional shifts interpolate between integer slots} \[ P(\tau)=\Phi\U(\tau)\Phi^\top,\qquad \U(\tau)=1\oplus\bigoplus_{j=1}^{(m-1)/2} \operatorname{Rot}\!\left(\frac{2\pi j\tau}{m}\right). \] \begin{itemize} \item $\Phi$: orthogonal real Fourier basis; $m$ is odd. \item $P(1)$ is an exact cyclic shift; $P(m)=I$. \item $P(a)P(b)=P(a+b)$: clock increments add up. \end{itemize} \[ e_r^\top P(\tau)e_s= \frac{\sin\!\big(\pi(\tau+s-r)\big)} {m\sin\!\big(\pi(\tau+s-r)/m\big)}. \] \begin{block}{A geometric interpolation, not a probability distribution} Coefficients can be negative. Transport preserves norms, but repeated cycles still overlap. \end{block} \papersource{Paper, Eqs. (12)--(13), PDF p. 4; App. A.1--A.2, Eq. (39), PDF pp. 15--16.} \end{frame} \begin{frame}{Changing coordinates cancels the recurrent rotation} \[ b=\Phi^\top e_0,\qquad \Kb_t=\U(-\lambda_t)\Phi^\top K_t. \] \begin{block}{The cancellation} \centering\( \U(-\lambda_t)\U(\delta_t)=\U(-\lambda_{t-1}). \) \end{block} \[ a_t=\beta_t\U(-\lambda_t)b,\qquad \begin{aligned} \Kb_t&=\alpha_t\Kb_{t-1}+a_tk_t^\top,\\ \Vb_t&=\alpha_t\Vb_{t-1}+a_tv_t^\top. \end{aligned} \] \begin{itemize} \item Existing states now undergo only scalar decay. \item Clock dependence moves into write/read vectors. \item Exact reconstruction: $K_t=\Phi\U(\lambda_t)\Kb_t$, and likewise for $V_t$. \end{itemize} \papersource{Paper, Eqs. (19)--(25), PDF pp. 5--6. $a_t$ abbreviates the paper's temporal write vector.} \end{frame} \begin{frame}{Two scalar attention passes surround a slot softmax} \[ \{o'_t\}=\LA(q,k,a,\alpha),\qquad o'_t=\Kb_tq_t \] \[ F_t=R\Phi\U(\lambda_t),\qquad o''_t=F_t^\top\softmax(F_to'_t) \] \[ \{o_t\}=\LA(o'',a,v,\alpha),\qquad o_t=\Vb_t^\top o''_t \] \begin{itemize} \item Pass 1: content matches accumulate evidence in temporal addresses. \item Bridge: token-wise transformations on $m$-dimensional vectors. \item Pass 2: the address query retrieves associated values. \item Training uses clock prefix sums and chunked recurrences; decoding carries fixed states and clocks. \end{itemize} \papersource{Paper, Eq. (27), PDF p. 6; App. B, PDF pp. 20--21. Softmax is over memory slots, not all past tokens.} \end{frame} \begin{frame}{The mixer combines CyFA with established network components} \begin{columns}[c,onlytextwidth] \column{.53\textwidth} \centering \includegraphics[width=\linewidth,height=.74\textheight,keepaspectratio]{figures/fig-02-architecture.pdf} \column{.44\textwidth} \begin{itemize} \item ShortConv on Q/K/V. \item RMSNorm on Q/K and head outputs. \item Low-rank sigmoid output gate. \item Pre-norm backbone with SwiGLU. \item $m=127$ active slots; $m^\ast=128$ allocated rows. \end{itemize} \medskip The padding row is masked and stays zero; the cycle has \alert{127} slots. \end{columns} \papersource{Paper, Fig. 2 and Sec. 3.4, PDF p. 7; App. A.7, PDF p. 20.} \end{frame} \begin{frame}{A stop-gradient protects the backbone from clock gradients} \begin{columns}[c,onlytextwidth] \column{.55\textwidth} \[ \delta_t=\sigma(W_\delta\sg(x_t)+b_\delta) \] \[ \frac{\partial\mathcal L}{\partial\delta_s} =\sum_{t\ge s}\frac{\partial\mathcal L}{\partial\lambda_t} \] \begin{itemize} \item Each increment receives future clock-path gradients. \item Stop-gradient blocks this branch's gradient into $x_t$. \item $W_\delta,b_\delta$ still learn; forward values stay input dependent. \end{itemize} \column{.41\textwidth} \centering \includegraphics[width=\linewidth,height=.73\textheight,keepaspectratio]{figures/fig-03-clock-gradient.pdf} \end{columns} \papersource{Paper, Eq. (28), Fig. 3, PDF p. 7; Eq. (95), App. D.1, PDF pp. 23--24. Curves: first 5,000 steps.} \end{frame} \begin{frame}{The controlled comparisons match main recurrent state size} \centering \begin{tabular}{lrrr} \toprule Scale & Training tokens & CyFA parameters & State entries/layer\\ \midrule 400M & 15B & 389M & 262,144\\ 800M & 30B & 800M & 393,216\\ 1.4B & 100B & 1,394M & 524,288\\ \bottomrule \end{tabular} \medskip \[ \underbrace{4\cdot128\cdot(256+256)}_{\text{CyFA, 400M}} =\underbrace{4\cdot256\cdot256}_{\text{GDN/KDA, 400M}}. \] \begin{itemize} \item 24 layers, SlimPajama, Mistral tokenizer, initial 2K context. \item State count excludes ShortConv cache; total memory and parameter counts are not exactly equal. \item Public 1.4B checkpoints are a separate, unmatched reference group. \end{itemize} \papersource{Paper, Sec. 4 and App. C, PDF pp. 8, 22--23; Tables 6--7.} \end{frame} \begin{frame}{Evaluation separates recall, general quality, and efficiency} \begin{description} \item[General quality] Wiki/LAMBADA perplexity $\downarrow$; six zero-shot commonsense tasks, average score $\uparrow$. \item[Recall at 2K] FDA, SWDE, SQD, NQ, TQA, DROP; protocol from \texttt{prefix-linear-attention}; inputs truncated to 2K. \item[Long inputs] LongBench at 8K; RULER at 1K--8K. Main results follow 1,024 continuation steps at 8K. \item[Efficiency] End-to-end training tokens/s and core-operator latency are measured separately. \end{description} \medskip \alert{Metric boundary:} LongBench averages task scores; it is not one shared accuracy metric. No multi-seed error bars are reported. \papersource{Paper, Secs. 4.1--4.3, PDF pp. 8--10; App. C.2, PDF pp. 22--23.} \end{frame} \begin{frame}{Recall gains over KDA persist across three model scales} \centering \begin{tabular}{lrrr} \toprule Scale & KDA & CyFA & Gain (percentage points)\\ \midrule 400M & 26.31 & \alert{33.25} & +6.94\\ 800M & 31.18 & \alert{37.13} & +5.95\\ 1.4B & 37.73 & \alert{43.47} & +5.74\\ \bottomrule \end{tabular} \begin{block}{What this result supports} Better retrieval from a matched main-state budget across the reported training scales. \end{block} \begin{itemize} \item The average covers six real-world recall-intensive tasks at 2K. \item CyFA leads the recurrent models on every recall task at each scale. \item Full Transformers still lead the average at 400M and 800M. \end{itemize} \papersource{Paper, Table 2, PDF p. 8. Gains are computed from reported averages.} \end{frame} \begin{frame}{Field-value retrieval shows the largest 400M gains} \centering \begin{tabular}{lrrrr} \toprule Model & FDA & SWDE & SQD & Recall avg.\\ \midrule Transformer & 53.04 & 38.89 & 30.70 & 33.98\\ Mamba-3 & 24.89 & 27.84 & 29.53 & 26.46\\ KDA & 26.07 & 25.02 & 29.49 & 26.31\\ CyFA & \alert{42.60} & \alert{35.61} & \alert{34.63} & \alert{33.25}\\ \bottomrule \end{tabular} \medskip \begin{itemize} \item FDA: +16.53 points over the best other recurrent model, KDA. \item SWDE: +7.77 points over the best other recurrent model, Mamba-3. \item Many coexisting field-value pairs directly stress compressed associative memory. \end{itemize} \medskip The full Transformer remains stronger on FDA and SWDE at this scale. \papersource{Paper, Table 2 and Sec. 4.1, PDF pp. 8--9. All scores $\uparrow$; avg. uses all six recall tasks.} \end{frame} \begin{frame}{General quality is competitive, with specific trade-offs} \centering \begin{tabular}{llrrr} \toprule Scale & Model & Wiki PPL $\downarrow$ & Lamb. PPL $\downarrow$ & Common avg. $\uparrow$\\ \midrule 400M & KDA & 25.76 & 51.17 & 41.90\\ & CyFA & 25.85 & 48.17 & 42.47\\ 800M & KDA & 19.83 & 22.20 & 46.81\\ & CyFA & 19.51 & 18.93 & 48.16\\ 1.4B & KDA & 15.97 & 11.88 & 52.89\\ & CyFA & 15.47 & 12.77 & 53.19\\ \bottomrule \end{tabular} \medskip \begin{itemize} \item Commonsense averages improve at all three scales. \item Wiki PPL is slightly worse at 400M; LAMBADA PPL is worse at 1.4B. \item Small differences have no reported uncertainty estimates. \end{itemize} \papersource{Paper, Table 2, PDF p. 8. Selected rows transcribed; no uniform per-metric dominance is claimed.} \end{frame} \begin{frame}{LongBench improves after 8K continuation} \centering \begin{tabular}{llrrr} \toprule Scale & Model & Average & LCC & RBP\\ \midrule 800M & KDA & 15.47 & 41.47 & 39.06\\ & CyFA & \alert{17.19} & \alert{47.84} & \alert{43.59}\\ 1.4B & KDA & 20.02 & 46.34 & 42.56\\ & CyFA & \alert{21.64} & \alert{53.60} & \alert{46.40}\\ \bottomrule \end{tabular} \medskip \begin{itemize} \item Inputs truncated to 8K; 1,024 extra training steps at 8K. \item Both code tasks improve. At 1.4B, LCC gains 7.26 points. \item At 1.4B, CyFA trails KDA on HQA, TRE, and SAM. \end{itemize} \begin{block}{A counting inconsistency in the paper} Text says 15 tasks; the table has 14 task columns plus Avg. CyFA leads 10/14 at 800M and 11/14 at 1.4B. \end{block} \papersource{Paper, Table 3 and Sec. 4.2, PDF p. 9. Average recomputed over the 14 displayed tasks.} \end{frame} \begin{frame}{Multi-key retrieval improves, but long inputs still degrade} \centering \includegraphics[width=\linewidth,height=.51\textheight,keepaspectratio]{figures/fig-04-ruler-after.pdf} \begin{itemize} \item 800M models, after 8K continuation; x-axis: 1K--8K tokens. \item MK-NIAH-1 requires multiple associations to coexist: CyFA clearly leads GDN/KDA through 4K. \item At 8K, difficult retrieval remains weak; superiority is not uniform across all panels and lengths. \end{itemize} \papersource{Paper, Fig. 4, PDF p. 9. Original figure; the multi-key panel has a different y-axis scale. No exact points inferred from the plot.} \end{frame} \begin{frame}{End-to-end training reaches 1.37$\times$ KDA at 8K} \begin{columns}[c,onlytextwidth] \column{.47\textwidth} \centering \includegraphics[width=\linewidth,height=.72\textheight,keepaspectratio]{figures/fig-05-throughput.pdf} \column{.49\textwidth} \begin{itemize} \item 400M models on one RTX Pro 6000. \item Same tokens per batch: longer sequences use smaller batches. \item CyFA stays around 80--86K tokens/s from 2K to 32K. \item Faster than the Transformer from 8K onward. \item Mamba-2 is faster than CyFA among recurrent models shown. \end{itemize} \end{columns} \papersource{Paper, Fig. 5 and Sec. 4.3, PDF pp. 9--10. Training throughput, not autoregressive serving throughput.} \end{frame} \begin{frame}{Core operator latency is about half of KDA's} \centering \includegraphics[width=.89\linewidth,height=.53\textheight,keepaspectratio]{figures/fig-06-kernels.pdf} \medskip \begin{tabular}{lrrr} \toprule & KDA & CyFA & CyFA / KDA\\ \midrule Forward & 4.37 ms & 2.04 ms & 46.7\%\\ Backward & 11.18 ms & 5.40 ms & 48.3\%\\ \bottomrule \end{tabular} \papersource{Paper, Fig. 6, PDF p. 10. One RTX Pro 6000; 400M training shapes, sequence length 2K, batch 32. These ratios describe the core operators only.} \end{frame} \begin{frame}{Forgetting, adaptive clocks, and capacity all matter} \centering \begin{tabular}{lrrr} \toprule 400M variant & Lamb. PPL $\downarrow$ & Recall avg. $\uparrow$ & $\Delta$ recall\\ \midrule Default, $m=127$ & 48.17 & 33.25 & --\\ No forgetting ($\alpha=1$) & 115.89 & 28.05 & $-5.20$\\ Fixed increment per head & 51.98 & 30.32 & $-2.93$\\ Fixed readout ($R=I$) & 58.75 & 31.79 & $-1.46$\\ No QK normalization & 51.37 & 30.12 & $-3.13$\\ $m=63$ & 51.34 & 26.68 & $-6.57$\\ $m=255$ & 46.79 & 34.83 & $+1.58$\\ \bottomrule \end{tabular}\par \medskip Each row is a separate change to the default. More slots also increase the memory/computation budget.\par \medskip Additional trade-off: adding SiLU improves PPL but reduces recall to 32.26. \papersource{Paper, Table 4, PDF p. 10. Full table and ShortConv ablation are included in the Chinese notes.} \end{frame} \begin{frame}{Clock interventions support contextual alignment} \begin{itemize} \item 400M: 56/96 heads are near unit increments; 40/96 are input dependent. \item Interventions affect the 40 input-dependent heads; 16 held-out sequences. \end{itemize} \medskip \centering \begin{tabular}{lrr} \toprule Clock setting & NLL $\downarrow$ & $\Delta$NLL\\ \midrule Original & 2.6105 & --\\ Per-head mean & 2.6555 & +0.0450\\ Shifted by 257 tokens & 2.7344 & +0.1238\\ Set to one & 2.7486 & +0.1381\\ \bottomrule \end{tabular} \begin{block}{Interpretation} Shifting preserves the increment distribution but breaks alignment with inputs. Its larger penalty supports use of context-dependent timing. \end{block} \papersource{Paper, Sec. 4.5, PDF p. 10; App. D.2 and Table 8, PDF p. 24. A single-checkpoint intervention, not a universal semantic-importance measure.} \end{frame} \begin{frame}{The contribution is better organization within finite capacity} \begin{enumerate} \item \textbf{Representation:} a learned clock controls temporal spacing while K/V remain aligned. \item \textbf{Computation:} an exact coordinate change yields two scalar-decay recurrences. \item \textbf{Evidence:} recall improves at matched main-state sizes with competitive training throughput. \end{enumerate} \begin{block}{Boundaries to retain} Cycles still alias; difficult 8K retrieval degrades. Some metrics regress. No multi-seed uncertainty or serving-latency benchmark is reported. \end{block} \medskip \textbf{Discussion:} How should clock resolution and slot capacity be allocated across heads under a total memory and compute budget? \papersource{Synthesis of Secs. 3--4 and Apps. A, C--E. The discussion question is this reading's analysis, not a paper result.} \end{frame} \begin{frame}{Source, reading scope, and verification} Yixiao Chen, Shuojin Yang, and Shi-Min Hu.\par \emph{CyFA: Linear Sequence Modeling with Relative-Time-Partitioned Memory.}\par arXiv:2609.36259v1, September 28, 2026. \medskip \begin{itemize} \item \href{https://arxiv.org/abs/2609.36259v1}{Versioned arXiv page} \quad \href{https://arxiv.org/pdf/2609.36259v1}{Paper PDF} \item Author-linked code: \href{https://github.com/Chyxx/CyclicFlowAttention}{github.com/Chyxx/CyclicFlowAttention}. \item Reading covers all main sections and Appendices A--E; figures are copied from the matching author TeX source. \item Independent small FP64 check: state reconstruction error $<4\times10^{-15}$; output error $<8\times10^{-16}$. \item This is an equation check, not a reproduction of training, gradients, or the authors' kernels. \end{itemize} \papersource{All reported benchmark numbers come from this paper. LongBench task-count inconsistency is identified explicitly. Backup slides follow.} \end{frame} \appendix \begin{frame}{Backup: temporal overlap factors out from content similarity} Let $p_i(t)=P(\lambda_t-\lambda_i)e_0$ and $D_m(x)=\sin(\pi x)/[m\sin(\pi x/m)]$. \[ p_i(t)^\top p_j(t)=D_m(\lambda_i-\lambda_j) \] \[ \langle p_i(t)k_i^\top,p_j(t)k_j^\top\rangle_F =(k_i^\top k_j)D_m(\lambda_i-\lambda_j). \] \begin{itemize} \item Nonzero integer separations within one cycle are orthogonal. \item Fractional separations generally overlap; full-cycle separations alias. \item Later shared transport preserves pairwise profile geometry. \item This identity excludes scalar write/forget amplitudes and is not a recall guarantee. \end{itemize} \papersource{Paper, App. A.3, Proposition 4 and Corollary 1; Eqs. (54), (60), PDF pp. 17--18.} \end{frame} \begin{frame}{Backup: scalar decay preserves a matrix-multiply chunk form} For a chunk of $C$ tokens, define $\gamma_r=\prod_{j=1}^r\alpha_j$ and \[ \Gamma_{rs}=\begin{cases}\gamma_r/\gamma_s,&r\ge s,\\0,&r