% Research discussion deck, based on the arxiv-paper-reader Beamer template. % Main talk: frames 1--21, about 25 minutes. Frames 22--27 are backups. \documentclass[aspectratio=169,11pt]{beamer} \usepackage[T1]{fontenc} \usepackage[utf8]{inputenc} \usepackage{lmodern} \usepackage{amsmath,amssymb,graphicx,booktabs} \definecolor{AcademicBlue}{RGB}{26,57,86} \definecolor{AcademicTeal}{RGB}{0,105,112} \setbeamercolor{structure}{fg=AcademicBlue} \setbeamercolor{frametitle}{fg=AcademicBlue,bg=white} \setbeamercolor{title}{fg=AcademicBlue} \setbeamercolor{alerted text}{fg=AcademicTeal} \setbeamercolor{block title}{fg=white,bg=AcademicBlue} \setbeamercolor{block body}{fg=black,bg=AcademicBlue!5} \setbeamertemplate{navigation symbols}{} \setbeamertemplate{itemize items}[circle] \setbeamertemplate{footline}{\hfill\usebeamerfont{page number in head/foot}\insertframenumber\,/\,\inserttotalframenumber\hspace{4mm}\vspace{3mm}} \setbeamersize{text margin left=8mm,text margin right=8mm} \setbeamerfont{frametitle}{size=\Large,series=\bfseries} \setbeamerfont{title}{size=\LARGE,series=\bfseries} \newcommand{\papersource}[1]{\par\smallskip{\scriptsize\color{gray}#1\par}} \newcommand{\xx}{\mathbf{x}} \newcommand{\zz}{\mathbf{z}} \newcommand{\sg}{\operatorname{sg}} \newcommand{\fmlm}{FMLM$^\star$} \newcommand{\elf}{ELF$^\star$} \title[Fixed-point flows]{Self-conditioned Flow Map Language Models via Fixed-point Flows} \subtitle{Compress prediction refinement and flow evolution} \author{\small Paper by Jaehoon Yoo, Wonjung Kim, Floor Eijkelboom,\\ Chanhyuk Lee, Nicholas M. Boffi, Seunghoon Hong, and Jinwoo Kim} \institute{KAIST \quad University of Amsterdam \quad Carnegie Mellon University} \date{Research discussion \;|\; 14 October 2026\\ \href{https://arxiv.org/abs/2607.00714v2}{arXiv:2607.00714v2, revised 1 October 2026}} \begin{document} \begin{frame} \titlepage \end{frame} \begin{frame}{The method compresses two kinds of computation} \begin{itemize} \item Self-conditioning reuses a denoising estimate across flow steps. \item Interpret this estimate as an iterate of a \alert{local fixed-point solver}. \item Learn to jump both across refinement iterations and across flow time. \end{itemize} \centering \includegraphics[width=0.94\linewidth,height=0.42\textheight,keepaspectratio]{figures/fig-01-two-compressions.pdf} \par\smallskip {\small Blue: flow-map jump. \quad Green: fixed-point jump.} \papersource{Paper, Fig. 1 (bottom row), PDF p. 2.} \end{frame} \begin{frame}{A denoiser determines a continuous text flow} \[ I_t=(1-t)\xx_0+t\xx_1,\qquad \xx_0,\xx_1\in\mathbb R^{L\times d} \] \[ D_t(\xx)=\mathbb E[\xx_1\mid I_t=\xx], \qquad b_t(\xx)=\frac{D_t(\xx)-\xx}{1-t} \] \begin{block}{Generation by local updates} \[ \xx_{t_{i+1}}=\xx_{t_i}+(t_{i+1}-t_i)b_{t_i}(\xx_{t_i}) \] \end{block} \begin{itemize} \item $L$: sequence length; $d$: embedding dimension; $t$: flow time. \item Noise $\xx_0$ evolves to text embeddings, then a decoder produces tokens. \item A flow map $X_{s,t}$ learns an entire interval in one evaluation. \end{itemize} \papersource{Paper, Sec. 2, Eqs. (1)--(5), PDF pp. 2--3.} \end{frame} \begin{frame}{Self-conditioning trains an initial guess and a correction} \[ \zz^1=\hat D_t(I_t,\mathbf0),\qquad \zz^2=\hat D_t(I_t,\sg(\zz^1)) \] \[ \mathcal L_\mu=\tfrac12\mathbb E\|\zz^1-\xx_1\|^2 +\tfrac12\mathbb E\|\zz^2-\xx_1\|^2 \] \begin{itemize} \item $\sg$: stop gradient. Expectations here also include flow time. \item Both predictions target the same clean embedding. \item If $\zz\perp\xx_1\mid I_t$ and second moments are finite, the unrestricted population optimum is $D_t(I_t)$ on the training support. \end{itemize} \begin{block}{Why it can help in practice} A finite learned network can use its first prediction to improve its next one. \end{block} \papersource{Paper, Sec. 3.1, Eq. (7), Proposition 3.1; PDF p. 4 and Appendix A.1, p. 13.} \end{frame} \begin{frame}{At fixed time and state, prediction becomes an iteration} \[ \boxed{\zz^{j+1}=\hat D_t(\xx,\zz^j)} \qquad \zz^\star=\hat D_t(\xx,\zz^\star) \] \begin{block}{Sufficient condition: contraction on a nonempty closed set $O$} \[ f(O)\subseteq O,\quad \|f(\zz)-f(\zz')\|\leq\eta\|\zz-\zz'\|,\quad 0\leq\eta<1 \] \end{block} \[ \|\zz^j-\zz^\star\|\leq\eta^j\|\zz^0-\zz^\star\| \] \begin{itemize} \item $t$ and $\xx$ stay fixed; only the denoising estimate changes. \item Convergence is to the \alert{learned fixed point}; better generation is tested empirically. \end{itemize} \papersource{Paper, Eq. (10), Proposition 3.3, PDF pp. 4--5; Appendix A.3, pp. 15--17.} \end{frame} \begin{frame}{Warm starts reduce the work needed to find a fixed point} \begin{center} \begin{tabular}{lll} \toprule Sampling & Initial prediction at time $t_i$ & Inner work\\ \midrule Cold start & $\zz^0_{t_i}=\mathbf0$ & Iterate from scratch\\ Warm start & $\zz^0_{t_i}=\hat\zz^\star_{t_{i-1}}$ & Reuse previous estimate\\ Conventional SC & Warm start & One iteration\\ \bottomrule \end{tabular} \end{center} \[ j\geq\left\lceil \frac{\log(\|\zz^0-\zz^\star\|/\varepsilon)}{\log(1/\eta)} \right\rceil \] \begin{itemize} \item Bound applies when $0<\eta<1$ and initial error exceeds $\varepsilon$. \item If neighboring fixed points are close, the previous estimate is useful. \item With enough iterations in the contraction domain, both starts reach the same fixed point. \end{itemize} \papersource{Paper, Sec. 3.3, PDF p. 6; Proposition A.5 and Eq. (37), PDF p. 18.} \end{frame} \begin{frame}{Solving the inner problem removes the history variable} \[ D_t^\star(\xx):=\zz^\star, \qquad b_t^\star(\xx)=\frac{D_t^\star(\xx)-\xx}{1-t} \] \[ \dot\xx_t=b_t^\star(\xx_t),\qquad t<1 \] \begin{itemize} \item A unique fixed point makes the velocity depend only on $(t,\xx)$. \item In the ideal case $D^\star=D$, this recovers the true data flow. \item Where this ODE has unique solutions, it defines a flow map: \end{itemize} \[ X_{s,t}^\star=X_{u,t}^\star\circ X_{s,u}^\star \] \papersource{Paper, Propositions 3.4--3.5, Eqs. (13)--(20), PDF pp. 5--6. ODE uniqueness is a separate condition.} \end{frame} \begin{frame}{Fixed-point distillation replaces repeated refinement} \[ \hat D_t(\xx,\cdot) \ \xrightarrow{\text{iterate}}\ \zz^\star \ \xrightarrow{\text{distill}}\ D_t^\star(\xx) \] \begin{block}{Basic regression target} \[ \mathcal L(D^\star)=\int_0^1\mathbb E\|D_t^\star(I_t)-\zz^\star\|^2\,dt \] \end{block} \begin{itemize} \item Actual offline method: CDEQ consistency distillation, yielding \elf. \item Teacher targets: 20 Anderson-accelerated iterations. \item Inference: one student evaluation per flow step, with self-conditioning input set to zero. \end{itemize} \papersource{Paper, Eq. (18), PDF p. 6; Appendix B.2, PDF p. 21.} \end{frame} \begin{frame}{A two-time denoiser represents large flow-map jumps} \[ v_{s,t}(\xx)=\frac{X_{s,t}^\star(\xx)-\xx}{t-s}, \qquad \delta_{s,t}(\xx)=\xx+(1-s)v_{s,t}(\xx) \] \begin{block}{Recover the next flow state} \[ \boxed{X_{s,t}^\star(\xx)=\frac{1-t}{1-s}\xx+ \frac{t-s}{1-s}\delta_{s,t}(\xx)} \] \end{block} \begin{itemize} \item On the diagonal: $\delta_{t,t}(\xx)=D_t^\star(\xx)$. \item Off the diagonal: predict the effect of a finite time interval. \item At the endpoint extension: $X_{0,1}^\star(\xx_0)=\delta_{0,1}(\xx_0)$. \end{itemize} \papersource{Paper, Eqs. (21)--(24), PDF pp. 6--7. Formal identities use $t<1$; training also includes $(0,1)$.} \end{frame} \begin{frame}{Consistency makes one jump agree with two smaller jumps} \[ \mathcal T(\delta)_{s,u,t}(\xx) =\gamma\delta_{s,u}(\xx) +(1-\gamma)\delta_{u,t}(X_{s,u}^\star(\xx)) \] \[ \gamma=\frac{(1-t)(u-s)}{(1-u)(t-s)} \] \begin{block}{Two complementary losses} \[ \begin{aligned} \mathcal L(\delta)={}&\mathbb E\|\delta_{s,t}-\sg(\mathcal T(\delta)_{s,u,t})\|^2\\ &+\mathbb E\|\delta_{t,t}-D_t^\star\|^2 \end{aligned} \] \end{block} First term: compress flow evolution.\quad Second: anchor local denoising. \papersource{Paper, Eqs. (25)--(27), PDF p. 7. In implementation, the intermediate state is computed from the current student under stop-gradient (Appendix B.3, p. 22).} \end{frame} \begin{frame}{Offline and online routes supply the same kind of target} \begin{columns}[T,onlytextwidth] \column{0.48\textwidth} \begin{block}{Offline: two stages} \[ \text{ELF}\longrightarrow\text{ELF}^{\star}\longrightarrow\text{FMLM}^{\star} \] \begin{itemize} \item First learn the fixed-point denoiser. \item Use one \elf\ evaluation as the diagonal target. \end{itemize} \end{block} \column{0.48\textwidth} \begin{block}{Online: joint compression} \[ \text{ELF}\xrightarrow{K\ \mathrm{FPIs}}\zz^K \] \begin{itemize} \item Estimate the target during training. \item Train \fmlm\ without a separate \elf\ model. \end{itemize} \end{block} \end{columns} \bigskip \alert{$K$ counts teacher calls during training, including the initial zero-conditioned call.} \par\medskip Both final models generate with a two-time student. \papersource{Paper, Sec. 3.4, PDF p. 7; Appendix B.3, PDF p. 22.} \end{frame} \begin{frame}{Generation quality must be evaluated together with diversity} \begin{itemize} \item OpenWebText; sequence length 1024. \item gPPL $\downarrow$: generated text scored by pretrained GPT-2 Large. \item Mean per-sample unigram entropy: target is near data entropy, \alert{5.44 nats}. \item Analysis: 256 samples. Distillation evaluation: 1024 samples. \end{itemize} \begin{block}{Why the entropy column matters} Low gPPL can come from repetitive text. DFM + ESD at one step: \[ \text{gPPL}=5.33,\qquad \text{entropy}=0.26 \] \end{block} \papersource{Paper, experiment setup, PDF p. 7; Table 1, PDF p. 9. gPPL is an external evaluator score, not the generator's held-out likelihood.} \end{frame} \begin{frame}{The experiments separate mechanism tests from distillation} \begin{center} \begin{tabular}{p{0.27\linewidth}p{0.65\linewidth}} \toprule Question & Experimental comparison\\ \midrule Q1: fixed-point behavior & ELF / LangFlow at fixed $(t,\xx)$; vary refinement count.\\[3pt] Q2: initialization & Warm versus cold starts at 1 and 100 iterations.\\[3pt] Q3: remove SC state & \elf\ versus its ELF teacher, both with 32 flow steps.\\[3pt] Q4: few-step generation & \fmlm\ versus teacher and diffusion / flow-map baselines.\\ \bottomrule \end{tabular} \end{center} \medskip Distillation teacher: ELF-B variant with GPT-2 Large embeddings for tokenizer matching. Students: 5 epochs each, batch 512, 8 B200 GPUs. \papersource{Paper, Secs. 4.1--4.2, PDF pp. 8--9; Appendix B.2--B.3, pp. 21--22.} \end{frame} \begin{frame}{Q1: repeated predictions approach a stable reference} \begin{columns}[c,onlytextwidth] \column{0.48\textwidth} \includegraphics[width=\linewidth,height=0.66\textheight,keepaspectratio]{figures/fig-02a-convergence.pdf} \column{0.48\textwidth} \begin{itemize} \item Average over 256 examples and ten times $t=0,0.1,\ldots,0.9$. \item Damped iteration: \end{itemize} \[ \zz^{j+1}=0.3\zz^j+0.7\hat D_t(\xx_t,\zz^j) \] \begin{itemize} \item Reference: $\zz^\star\approx\zz^{200}$ from the same run. \item The trajectory supports stabilization; the final zero distance is by construction. \end{itemize} \end{columns} \papersource{Paper, Fig. 2(a), PDF p. 8; Appendix B.1, PDF p. 21.} \end{frame} \begin{frame}{Q1: more refinement improves gPPL with extra computation} \begin{center} \begin{tabular}{r r r r r} \toprule FPIs & ELF gPPL $\downarrow$ & Entropy & LangFlow gPPL $\downarrow$ & Entropy\\ \midrule 1 & 101.19 & 5.51 & 77.26 & 5.49\\ 2 & 66.50 & 5.42 & 57.99 & 5.42\\ 3 & 56.67 & 5.37 & 52.63 & 5.40\\ 4 & 50.25 & 5.33 & 49.37 & 5.38\\ 5 & 46.31 & 5.31 & 47.27 & 5.36\\ \bottomrule \end{tabular} \end{center} \begin{itemize} \item Warm starts, no damping; ELF: 32 flow steps; LangFlow: 64. \item Entropy declines modestly and remains near the data level of 5.44. \item Inner denoiser calls scale as $N\times K$: refinement is useful computation that distillation aims to compress. \end{itemize} \papersource{Paper, Table 2, PDF p. 22; Appendix B.1, p. 21. ELF sampling $\gamma=0$; LangFlow temperature $1.0$.} \end{frame} \begin{frame}{Q2: enough refinement closes the warm--cold start gap} \begin{columns}[c,onlytextwidth] \column{0.49\textwidth} \centering \includegraphics[width=\linewidth,height=0.58\textheight,keepaspectratio]{figures/fig-03-warm-cold-elf.pdf} \column{0.49\textwidth} \centering \includegraphics[width=\linewidth,height=0.58\textheight,keepaspectratio]{figures/fig-03-warm-cold-langflow.pdf} \end{columns} \medskip At 1 FPI, warm starts are better. At 100 FPIs, the frontiers overlap. \par\smallskip {\small ELF: 32 flow steps; LangFlow: 64. Damping $\alpha=0$ for 1 FPI and $0.3$ for 100 FPIs. Compare initialization within each budget.} \papersource{Paper, Fig. 3, PDF p. 8; Appendix B.1, PDF p. 21.} \end{frame} \begin{frame}{Q3--Q4: each compression retains much of the teacher's frontier} \begin{columns}[T,onlytextwidth] \column{0.48\textwidth} \centering\textbf{Remove the SC state} \par\medskip \includegraphics[width=\linewidth,height=0.49\textheight,keepaspectratio]{figures/fig-04-elf-star.pdf} \par\smallskip {\small \elf\ and ELF both use 32 steps; their frontiers are close.} \column{0.48\textwidth} \centering\textbf{Reduce flow steps} \par\medskip \includegraphics[width=\linewidth,height=0.49\textheight,keepaspectratio]{figures/fig-05-fmlm-star.pdf} \par\smallskip {\small 8-step \fmlm\ approaches 32-step ELF, with a gap at high entropy.} \end{columns} \medskip \papersource{Paper, Figs. 4--5, Sec. 4.2, PDF p. 9. Step reduction is reported; a 4x wall-clock speedup is not measured here.} \end{frame} \begin{frame}{Q4: \fmlm\ improves the few-step quality--diversity tradeoff} \begin{center} {\small Each cell: gPPL $\downarrow$ / entropy in nats. Data entropy: 5.44.} \medskip \setlength{\tabcolsep}{9pt} \begin{tabular}{l r r r} \toprule Model & 1 step & 2 steps & 4 steps\\ \midrule Duo + DCD & 5743.29 / 6.02 & 891.16 / 5.41 & 250.86 / 5.37\\ MDLM + SDTT & 1260.86 / 5.26 & 877.22 / 5.34 & 339.73 / 5.38\\ FMLM & 168.30 / 5.17 & 133.29 / 5.25 & 111.31 / 5.26\\ DFM + ESD & 5.33 / 0.26 & 108.91 / 5.15 & 77.08 / 5.27\\ \fmlm\ offline & \textbf{113.91 / 5.37} & \textbf{93.66 / 5.46} & \textbf{73.86 / 5.35}\\ \fmlm\ online & 117.89 / 5.39 & 98.96 / 5.47 & 80.26 / 5.44\\ \bottomrule \end{tabular} \end{center} \medskip \alert{Single-step gPPL drops 32.3\% versus FMLM, with entropy closer to data.} \papersource{Paper, Table 1 (selected rows), PDF p. 9. Online uses 8 training FPIs. Two-/four-step \fmlm\ use sampling $\gamma=0.75/1.0$ (Appendix B.3, p. 22).} \end{frame} \begin{frame}{Eight online FPIs offer a useful training-cost compromise} \begin{center} \begin{tabular}{l r r r r} \toprule Target route & 1-step gPPL & 2-step gPPL & 4-step gPPL & Time\\ \midrule Online, 1 FPI & 176.31 & 149.51 & 120.35 & 18 h\\ Online, 2 FPIs & 141.97 & 115.25 & 89.32 & 18 h\\ Online, 4 FPIs & 123.28 & 101.27 & 81.27 & 20 h\\ Online, 8 FPIs & 117.89 & 98.96 & 80.26 & 23 h\\ Online, 16 FPIs & 116.63 & 98.61 & 82.50 & 32 h\\ Offline & 113.91 & 93.66 & 73.86 & 82 h\\ \bottomrule \end{tabular} \end{center} \begin{itemize} \item 8 FPIs: about 72\% less reported training time than offline. \item Single-step gPPL is 3.5\% higher; entropy is 5.39 versus 5.37. \item 16 FPIs bring little improvement and worsen the 4-step result. \end{itemize} \papersource{Paper, Table 3, PDF p. 23. Times compare distillation routes on the reported hardware; teacher pretraining is not separately itemized.} \end{frame} \begin{frame}{Local refinement helps; semantic quality remains a limitation} \begin{block}{Evidence from fixed-state predictions} At $t=0.25$, three ELF iterations improve grammatical agreement and make references to people more contextually consistent. \end{block} \begin{block}{Evidence from complete generated samples} The 1-, 2-, and 4-step examples still contain malformed phrases, invented words, and topic drift, including in the 4-step output. \end{block} \begin{itemize} \item gPPL and unigram entropy capture only part of text quality. \item Results concern OpenWebText generation at length 1024. \item The method still relies on a separately trained teacher. \end{itemize} \papersource{Paper, Appendix D, Figs. 6--9, PDF pp. 23--27; Sec. 6, p. 10. Qualitative assessment of the shown samples is this reading's analysis.} \end{frame} \begin{frame}{The key contribution is a bridge from self-conditioning to flow maps} \begin{enumerate} \item \textbf{Interpretation:} historical predictions serve as initial guesses for a local fixed-point iteration. \medskip \item \textbf{Method:} distill both the local solver and the time evolution into a two-time denoiser. \medskip \item \textbf{Tradeoff:} offline gives the best reported quality; online substantially reduces distillation time. \end{enumerate} \bigskip \alert{Keep distinct: convergence to a learned fixed point, agreement with the Bayes target, and empirical generation quality.} \papersource{Yoo et al., Self-conditioned Flow Map Language Models via Fixed-point Flows.\newline \href{https://arxiv.org/abs/2607.00714v2}{arXiv:2607.00714v2}, 1 October 2026. Main talk ends here; technical backups follow.} \end{frame} \appendix \begin{frame}{Backup: contraction does not imply improving Bayes error} Proposition 3.3 bounds the denoising error by \[ \begin{aligned} \|\zz^j-D_t(\xx)\|\leq{}&\|\hat D_t(\xx,\zz^0)-D_t(\xx)\|\\ &+\frac{\eta-\eta^j}{1-\eta}\|\zz^1-\zz^0\|. \end{aligned} \] The extra term grows with $j$; the bound is not a monotonic-improvement result. \begin{block}{Teaching counterexample} Target $D=0$, map $f(z)=z/2+1$, initialization $z^0=0$: \[ 1,\ 1.5,\ 1.75,\ldots\longrightarrow2. \] The iteration contracts toward its fixed point while moving away from the target. \end{block} \papersource{Paper, Eqs. (11)--(12), PDF p. 5; Appendix A.3, pp. 15--17. Counterexample is a teaching construction.} \end{frame} \begin{frame}{Backup: low loss gives a weaker, scale-dependent contraction} Let $X$ be fixed, $f$ continuous, and $Z\sim\mu$ supported on a closed $O$. \[ \mathcal L(f)=p\mathbb E\|f(Z)-X\|^2+(1-p)\mathbb E\|f(f(Z))-X\|^2,\quad 0