% Self-authored presentation based on the arXiv paper-reader Beamer template.
% Build: python3 build_slides.py slides.en.tex
\documentclass[aspectratio=169,11pt]{beamer}
\usepackage[T1]{fontenc}
\usepackage[utf8]{inputenc}
\usepackage{amsmath,amssymb,graphicx,booktabs}
\definecolor{AcademicBlue}{RGB}{26,57,86}
\definecolor{AcademicTeal}{RGB}{0,105,112}
\setbeamercolor{structure}{fg=AcademicBlue}
\setbeamercolor{frametitle}{fg=AcademicBlue,bg=white}
\setbeamercolor{title}{fg=AcademicBlue}
\setbeamercolor{alerted text}{fg=AcademicTeal}
\setbeamercolor{block title}{fg=white,bg=AcademicBlue}
\setbeamercolor{block body}{fg=black,bg=AcademicBlue!5}
\setbeamertemplate{navigation symbols}{}
\setbeamertemplate{itemize items}[circle]
\setbeamertemplate{footline}{\hfill\usebeamerfont{page number in head/foot}\insertframenumber\,/\,\inserttotalframenumber\hspace{4mm}\vspace{3mm}}
\setbeamersize{text margin left=8mm,text margin right=8mm}
\setbeamerfont{frametitle}{size=\Large,series=\bfseries}
\setbeamerfont{title}{size=\LARGE,series=\bfseries}
\newcommand{\papersource}[1]{\par\smallskip{\scriptsize\color{gray}#1\par}}
\newcommand{\LF}[1]{\ensuremath{\mathrm{LexFrame}[#1]}}
\title[Lexical Reference Frames]{Lexical Reference Frames}
\subtitle{Interpreting LM Residual Streams in Native Semantics}
\author{Paper: Jian Gu, Hongyu Zhang, Chunyang Chen, Aldeida Aleti}
\date{arXiv:2610.22169v2 \quad $\vert$ \quad Research discussion, 12 November 2026}
\begin{document}

% Main talk: frames 1--22, approximately 25 minutes.
\begin{frame}
\titlepage
\vspace{-6mm}
\begin{center}
\alert{Read a hidden state through the layers that will use it.}\\[2mm]
{\small Publication not found as of 2026-11-12 \quad $\vert$ \quad
\href{https://github.com/jianguda/lexframe}{Official public code}}\\
{\scriptsize \href{https://arxiv.org/abs/2610.22169v2}{Paper v2, revised 18 October 2026}; presentation is an independent reading.}
\end{center}
\end{frame}

\begin{frame}{A fixed output direction can misread an intermediate state}
\begin{columns}[T]
\begin{column}{0.49\textwidth}
\begin{block}{What should ``this state means token $v$'' mean?}
How the remaining model uses the state, given its prefix.
\end{block}
\begin{itemize}
\item Logit lens: reuse the output frame.
\item Jacobian lens: one average map per layer.
\item Point-cloud geometry: describe where states vary.
\end{itemize}
\end{column}
\begin{column}{0.47\textwidth}
\vspace{3mm}
OLMo3, layer 16 of 32:\\[3mm]
\begin{tabular}{lr}
\toprule
Direction & Angle to native readout\\
\midrule
Logit lens & $75^\circ$\\
Jacobian lens & $77^\circ$\\
\LF{1} & $13^\circ$\\
\bottomrule
\end{tabular}
\vspace{4mm}

The same token has different local readout directions in different contexts.
\end{column}
\end{columns}
\papersource{Paper, Sec. 1, Fig. 1, PDF p. 2. Angles averaged over states and labels.}
\end{frame}

\begin{frame}{The vocabulary stays fixed; its local frame moves}
\centering
\includegraphics[width=0.94\linewidth,height=0.40\textheight,keepaspectratio]{figures/fig02-transport.pdf}
\begin{itemize}
\item Input embedding and output unembedding fix the two interfaces.
\item Between them, the lexical frame depends on the remaining layers and context.
\item A frame supplies coefficients to \alert{read}, and basis vectors to \alert{write}.
\end{itemize}
\papersource{Paper, Fig. 2, PDF p. 4. Author schematic: coloured lexical bases, black hidden state.}
\end{frame}

\begin{frame}{Native transport defines a state-specific lexical reading}
\[
 x_\ell(t)\ \xrightarrow[\text{prefix at layer }\ell\text{ given}]{T_\ell:\ \text{remaining model}}\ x_L(t)
 \ \xrightarrow{N,\ W}\ z_\ell
\]
\[
 z_\ell(v)=w_v^\top N(T_\ell(x_\ell(t))),\qquad
 c_v=z_v-\frac{1}{|V|}\sum_u z_u
\]
\begin{itemize}
\item $x_\ell(t)\in\mathbb R^d$: residual state; $N$: final normalization.
\item $W\in\mathbb R^{|V|\times d}$: unembedding, with row $w_v^\top$.
\item Native transport evolves the prefix normally after the cut.
\item The complete native reading equals the model output at every cut.
\end{itemize}
\papersource{Paper, Secs. 3.1--3.2, PDF pp. 3--4, Eq. (2).}
\end{frame}

\begin{frame}{Readout, basis and coefficient answer different questions}
\[
 J_\ell=\frac{\partial N(T_\ell)}{\partial x_\ell(t)},\qquad
 s_v=W^+e_v,\qquad \bar w=\frac{1}{|V|}\sum_v w_v
\]
\begin{align*}
\text{Readout: }& b_\ell(v)=J_\ell^\top(w_v-\bar w)
&&\text{local sensitivity}\\[2mm]
\text{Basis: }& s_\ell(v)=J_\ell^+s_v
&&\text{selective output move}\\[2mm]
\text{Coefficient: }&z_\ell(v)=w_v^\top N(T_\ell(x_\ell(t)))
&&\text{transported value}
\end{align*}
\begin{block}{Why gradient and basis differ}
The steepest increase can also raise correlated tokens. A pseudoinverse instead tries to realize a specified output change with a small state move.
\end{block}
\papersource{Paper, Sec. 3.2, PDF p. 4, Eqs. (1)--(2); App. C.1, PDF pp. 16--17.}
\end{frame}

\begin{frame}{Context variation bounds every fixed readout direction}
For one label, let $\hat b_i$ be native unit readouts across $m$ states.
\[
\frac1m\sum_i r^\top\hat b_i=r^\top\bar b\leq\|\bar b\|,\qquad
\|\bar b\|^2=\frac1m+\left(1-\frac1m\right)\hat\kappa.
\]
\begin{itemize}
\item $r$ is any fixed unit direction; $\hat\kappa$ is mean pairwise cosine.
\item With many states, the bound approaches $\sqrt{\kappa_\ell(v)}$.
\item At mid-depth, the label-averaged bound is \alert{0.30 / 0.35}.
\end{itemize}
\begin{block}{Scope of the result}
This bounds mean directional cosine, not output KL. A lens with state-dependent final normalization needs that extra qualification; the paper reports it below the bound empirically.
\end{block}
\papersource{Paper, Sec. 3.2, PDF p. 4; App. C.2, Eq. (5), PDF p. 17. Model order: SmolLM3 / OLMo3.}
\end{frame}

\begin{frame}{Order controls cross-position information paths}
\centering
\includegraphics[width=0.98\linewidth,height=0.50\textheight,keepaspectratio]{figures/fig03-orders.pdf}
\begin{itemize}
\item Order 0: state-only transport, using typical attention writes.
\item Order 1: target attends to a prefix evolved at order 0.
\item Order 2: prefix evolves at order 1; one more interaction is retained.
\end{itemize}
\alert{Order counts nested cross-position exchanges, not attention layers.}
\papersource{Paper, Sec. 3.3, Fig. 3, PDF p. 5. Order $\infty$ recovers native transport.}
\end{frame}

\begin{frame}{Typical attention retains input-dependent average behavior}
\[
\hat a_k(y)=\bar a_k+G_k(y-\bar y_k)
\]
\begin{itemize}
\item $y$: attention input at the current position.
\item $\bar y_k$, $\bar a_k$: calibration means of input and attention write.
\item $G_k$: layerwise ridge regression, fitted in closed form on unlabelled text.
\item The actual residual blocks, normalizations and MLPs remain in the transport.
\end{itemize}
\begin{block}{What is fitted?}
Attention outputs from their inputs. Neither output logits nor answer labels are training targets. The model weights stay fixed.
\end{block}
\papersource{Paper, Eq. (3), PDF p. 5; App. A.1, PDF p. 13. Ridge selected by two-fold cross-validation.}
\end{frame}

\begin{frame}{Finite orders become exact near the output}
\[
 h_k^n(t)=F_k\!\left(h_{k-1}^n(t),
 \mathrm{Att}_k\!\left(h_{k-1}^n(t);\{h_{k-1}^{n-1}(s)\}_{s<t}\right)\right)
\]
\begin{itemize}
\item Every stream starts from $h_\ell^n=x_\ell$; order 0 is the base case.
\item Induction: order $n$ agrees with the model for its first $n$ remaining layers.
\item Thus $L-\ell\leq n$ implies exact reading, readouts and basis.
\end{itemize}
\begin{block}{Where approximation is informative}
The complete native reading repeats the final output. Finite orders reveal what survives when selected future context interactions are withheld.
\end{block}
\papersource{Paper, App. C.2, Eq. (6), PDF p. 17. Reading order $n$ uses $n+1$ streams.}
\end{frame}

\begin{frame}{Evaluation separates distributions, directions and interventions}
\begin{columns}[T]
\begin{column}{0.49\textwidth}
\textbf{Models and data}
\begin{itemize}
\item SmolLM3-3B: 36 layers, tied embeddings.
\item OLMo3-7B: 32 layers, untied embeddings.
\item WikiText-103; 128-token sequences.
\item Calibration: 256 training sequences.
\item Reading test: 64 test sequences.
\end{itemize}
\end{column}
\begin{column}{0.47\textwidth}
\textbf{Comparisons and metrics}
\begin{itemize}
\item Logit lens; Jacobian lens fitted on 100 prompts; direction-only ridge probe.
\item KL(model $\Vert$ reading), in nats.
\item Top-1 agreement with model.
\item ECE against actual next tokens.
\item Native gradients and ablations as intervention references.
\end{itemize}
\end{column}
\end{columns}
\papersource{Paper, Sec. 4, PDF pp. 5--7; App. A.1, pp. 12--13. Means over layers 1 to $L-1$. Tuned lens omitted.}
\end{frame}

\begin{frame}{One interaction recovers native directions from mid-depth}
\begin{columns}[c]
\begin{column}{0.48\textwidth}
\centering
\includegraphics[width=\linewidth,height=0.70\textheight,keepaspectratio]{figures/fig07-directions.pdf}
\end{column}
\begin{column}{0.48\textwidth}
\textbf{At layers 18/36 and 16/32}\par\smallskip
\begin{tabular}{lcc}
\toprule
Cosine & Smol & OLMo\\
\midrule
Fixed-direction bound & 0.30 & 0.35\\
\LF{1} & \alert{0.97} & \alert{0.97}\\
\bottomrule
\end{tabular}
\vspace{4mm}
\begin{itemize}
\item Blue solid: order 1; dashed: order 2.
\item Grey dotted: coherence bound.
\item Lowest layers retain a larger approximation gap.
\end{itemize}
\end{column}
\end{columns}
\papersource{Paper, Fig. 7, PDF p. 14; Sec. 4.1, p. 6. Direction comparison over 64 frequency-ranked labels.}
\end{frame}

\begin{frame}{Most distribution error disappears at order 1}
\centering
\includegraphics[width=0.91\linewidth,height=0.43\textheight,keepaspectratio]{figures/fig04-readings.pdf}
\medskip
\begin{tabular}{lrrrrr}
\toprule
Layer-mean KL $\downarrow$ & Logit & Jacobian & Order 0 & Order 1 & Order 2\\
\midrule
SmolLM3 & 29.06 & 7.51 & 0.87 & \alert{0.16} & 0.06\\
OLMo3 & 5.66 & 6.19 & 0.93 & \alert{0.15} & 0.05\\
\bottomrule
\end{tabular}
\papersource{Paper, Fig. 4, PDF p. 6; Table 1, p. 7. Vertical axis is logarithmic; KL in nats.}
\end{frame}

\begin{frame}{Faithful readings also have useful probabilities}
\centering
\begin{tabular}{lrrrr}
\toprule
 & \multicolumn{2}{c}{Top-1 agreement $\uparrow$} & \multicolumn{2}{c}{ECE $\downarrow$}\\
\cmidrule(lr){2-3}\cmidrule(lr){4-5}
Method & Smol & OLMo & Smol & OLMo\\
\midrule
Logit lens & 0.04 & 0.15 & 0.632 & 0.107\\
Jacobian lens & 0.14 & 0.13 & 0.376 & 0.191\\
\LF{0} & 0.64 & 0.64 & 0.018 & 0.045\\
\LF{1} & \alert{0.86} & \alert{0.86} & 0.027 & 0.018\\
\LF{2} & 0.92 & 0.93 & 0.020 & 0.020\\
Full model & --- & --- & 0.018 & 0.023\\
\bottomrule
\end{tabular}
\medskip
\begin{itemize}
\item Top-1 compares to the model; ECE compares to actual next tokens.
\item At the embedding, order-1 agreement is only 0.42 / 0.39.
\item Closer model agreement need not monotonically improve ECE.
\end{itemize}
\papersource{Paper, Table 1 and Sec. 4.1, PDF pp. 6--7. ECE uses 15 bins.}
\end{frame}

\begin{frame}{A lexical distance predicts substitution effects better}
\begin{columns}[c]
\begin{column}{0.49\textwidth}
\textbf{Replace one real hidden state with another; finish the model.}
\medskip

Layer-mean rank correlation with the output effect:\par
\medskip
\begin{tabular}{lrr}
\toprule
Distance & Smol & OLMo\\
\midrule
Cosine & 0.27 & 0.28\\
Euclidean & 0.08 & 0.12\\
Geodesic & 0.09 & 0.07\\
Jacobian-lens reading & 0.28 & 0.24\\
\LF{1} reading KL & \alert{0.94} & \alert{0.95}\\
\bottomrule
\end{tabular}
\end{column}
\begin{column}{0.43\textwidth}
\centering
\includegraphics[width=\linewidth,height=0.59\textheight,keepaspectratio]{figures/fig05-tangent.pdf}
\end{column}
\end{columns}
\papersource{Paper, Table 2 and Fig. 5, PDF p. 7. Plot: squared readout share in 64 local/global principal directions; grey: random. 7,961 replacement pairs/layer.}
\end{frame}

\begin{frame}{The basis writes a token more selectively than its gradient}
\centering
\begin{tabular}{lrr}
\toprule
Direction & Smol & OLMo\\
\midrule
Fixed output basis & 0.45 & 0.80\\
Order-1 readout & 0.54 & 0.73\\
Order-1 basis & \alert{0.84} & \alert{0.97}\\
Native readout & 0.56 & 0.79\\
Native basis & \alert{0.98} & \alert{1.00}\\
\bottomrule
\end{tabular}
\medskip
\begin{itemize}
\item Move by $0.1\|x\|$; ask if the intended token gains \emph{most}.
\item A readout raises correlated tokens; an inverse transport targets a lexical output move.
\item Coefficients still require nonlinear transport: $b_\ell(v)^\top x_\ell$ is a slope-related pairing, not the token's logit.
\end{itemize}
\papersource{Paper, Table 3, Sec. 4.3, PDF p. 8. 64 states and 64 labels/layer; explicit FP32 Jacobian.}
\end{frame}

\begin{frame}{Basis steering reduces changes to other tokens}
\textbf{Task:} move the runner-up token above the current top token.
\[
\delta_b\propto b_\ell(u)-b_\ell(v),\qquad
\delta_s\propto s_\ell(u)-s_\ell(v)
\]
\centering
\begin{tabular}{llrrr}
\toprule
Model & Direction & Reach $\uparrow$ & Side $\downarrow$ & Clean $\uparrow$\\
\midrule
Smol & Order-1 readout & 0.92 & 0.23 & 0.61\\
 & Order-1 basis & 0.89 & 0.12 & \alert{0.71}\\
 & Native basis & 0.90 & 0.11 & 0.73\\
\midrule
OLMo & Order-1 readout & 0.94 & 0.20 & 0.64\\
 & Order-1 basis & 0.95 & 0.07 & \alert{0.82}\\
 & Native basis & 0.97 & 0.06 & 0.85\\
\bottomrule
\end{tabular}
\papersource{Paper, Table 5, PDF p. 9. Steps: $0.01$--$1\times\|x\|$. Side: other-token change at first flip, nats. Clean: fraction of all predictions flipped with Side $<0.1$. Effects evaluated in the complete model.}
\end{frame}

\begin{frame}{A finite difference can outperform the exact local gradient}
\textbf{Remove a component write $q$ at one position; keep its prefix.}
\[
 A^{(n)}=c_v^{(n)}(x)-c_v^{(n)}(x-q),\qquad
 A_{\rm linear}^{(n)}=b^{(n)}(v)^\top q
\]
\centering
\begin{tabular}{llrrr}
\toprule
Model & Attribution & Layer & Attention & MLP\\
\midrule
Smol & Order-1 difference & \alert{0.88} & 0.79 & \alert{0.88}\\
 & Native linear (AtP) & 0.62 & 0.75 & 0.63\\
\midrule
OLMo & Order-1 difference & \alert{0.90} & 0.85 & \alert{0.90}\\
 & Native linear (AtP) & 0.68 & 0.84 & 0.71\\
\bottomrule
\end{tabular}
\medskip

Approximate transport retains nonlinear finite effects, especially for MLP writes.
\papersource{Paper, Table 6 and Sec. 4.4, PDF p. 9. Correlation with actual centred-logit ablation effects; 64 sequences. Equations restate the intervention in compact notation.}
\end{frame}

\begin{frame}{The output frame becomes less adequate during training}
\begin{columns}[c]
\begin{column}{0.46\textwidth}
\centering
\includegraphics[width=\linewidth,height=0.68\textheight,keepaspectratio]{figures/fig06-training.pdf}
\end{column}
\begin{column}{0.50\textwidth}
\textbf{Mean KL: first trained $\to$ final}\par
\medskip
\begin{tabular}{lrr}
\toprule
 & Logit lens & Order 1\\
\midrule
Smol & $4.82\to29.06$ & $0.15\to0.16$\\
OLMo & $2.32\to5.66$ & $0.02\to0.15$\\
\bottomrule
\end{tabular}
\begin{itemize}
\item Darker curves: later checkpoints.
\item The plot compares native readouts at each layer to the final layer.
\item Every checkpoint is calibrated anew.
\end{itemize}
\end{column}
\end{columns}
\papersource{Paper, Fig. 6 and App. B, PDF pp. 13--14. Checkpoint-wise calibration; Smol's earliest checkpoint is already trained.}
\end{frame}

\begin{frame}{Predictions settle early, but this reader does not save compute}
\centering
\begin{tabular}{lrrrr}
\toprule
 & \multicolumn{2}{c}{Settled layer share} & \multicolumn{2}{c}{Threshold-skipped share}\\
\cmidrule(lr){2-3}\cmidrule(lr){4-5}
Method & Smol & OLMo & Smol & OLMo\\
\midrule
Logit lens & 0.03 & 0.11 & 0.00 & 0.04\\
Jacobian lens & 0.09 & 0.09 & 0.00 & 0.03\\
Order 0 & 0.58 & 0.58 & 0.23 & 0.20\\
Order 1 & \alert{0.82} & \alert{0.81} & \alert{0.45} & \alert{0.42}\\
Order 2 & 0.89 & 0.90 & 0.53 & 0.54\\
\bottomrule
\end{tabular}
\begin{itemize}
\item Settled: uninterrupted agreement back from the last layer (retrospective).
\item Exit threshold selected on half the sequences for 0.95 model agreement.
\end{itemize}
\alert{LexFrame still runs the remaining layers: skipped depth is not speedup.}
\papersource{Paper, Table 7, PDF p. 16; explicit computational qualification in App. B, p. 15.}
\end{frame}

\begin{frame}{Faithfulness has a concrete computational cost}
\begin{itemize}
\item Calibration: one forward over calibration text plus one ridge solve per layer.
\item Reading all layers of a sequence at order $n$ costs approximately
\[
\frac{(n+1)(L+1)}{2}\quad\text{full-forward equivalents.}
\]
\item Jacobian-lens fitting costs about $Pd$ backward passes; its later reading is cheap.
\item Basis construction needs regularized inverse-Jacobian solves; steering uses 30 LSMR steps.
\end{itemize}
\begin{block}{Practical role}
A faithful offline reference and analysis tool. A cheaper adaptive reader would be needed to turn early readable predictions into efficient inference.
\end{block}
\papersource{Paper, App. A.1, PDF p. 13; App. C.3, p. 17. Measured Jacobian fit cost: 18 / 44 times calibration plus test readings at orders 0--2.}
\end{frame}

\begin{frame}{Three results change how hidden states should be interpreted}
\begin{enumerate}
\item \textbf{Meaning is downstream and contextual.}\\
A native frame explains why one direction per layer misses intermediate readouts.
\medskip
\item \textbf{Values, gradients and inverse directions differ.}\\
Use transported coefficients to read, differences to attribute, and basis vectors to steer selectively.
\medskip
\item \textbf{Order makes future context interaction controllable.}\\
One interaction is already accurate from mid-depth in these models; it remains expensive.
\end{enumerate}
\papersource{Paper, Sec. 5, PDF p. 9; Apps. C--D, pp. 16--18.}
\end{frame}

\begin{frame}{Evidence scope and sources}
\begin{itemize}
\item Tested: two model families on WikiText-103; context extension to 1,024 tokens; runner-up flips and single-position write ablations.
\item Tuned lens is omitted, so no direct distribution comparison is established.
\item Hallucination detection, knowledge editing and distillation are proposed uses, without downstream evaluations here.
\end{itemize}
\medskip
\textbf{Primary sources}
\begin{itemize}
\item Gu, Zhang, Chen, Aleti (2026). \emph{Lexical Reference Frames: Interpreting LM Residual Streams in Native Semantics}.\\
\href{https://arxiv.org/abs/2610.22169v2}{arXiv:2610.22169v2} \quad
\href{https://github.com/jianguda/lexframe}{Official implementation}
\item Publication not found as of 2026-11-12; arXiv and author citation list a preprint.
\end{itemize}
\end{frame}

\appendix
\begin{frame}{Backup: regression matters more than a constant write}
\centering
\begin{tabular}{llrrr}
\toprule
Model & Typical attention & KL $\downarrow$ & Cosine $\uparrow$ & Attribution $\uparrow$\\
\midrule
Smol & Regression & \alert{0.16} & 0.89 & 0.88\\
 & Mean only & 1.26 & 0.73 & 0.67\\
 & Zero & 0.76 & 0.71 & 0.68\\
\midrule
OLMo & Regression & \alert{0.15} & 0.87 & 0.90\\
 & Mean only & 0.52 & 0.74 & 0.77\\
 & Zero & 0.47 & 0.74 & 0.77\\
\bottomrule
\end{tabular}
\begin{itemize}
\item All rows evaluate order 1; the substitution changes its order-0 prefix stream.
\item Three disjoint calibration texts: KL 0.160--0.163 / 0.146--0.151.
\item Cosine here is layer-averaged, unlike the mid-depth value of 0.97.
\end{itemize}
\papersource{Paper, Table 8, PDF p. 16; App. B. Point estimates in this ablation.}
\end{frame}

\begin{frame}{Backup: longer contexts degrade the finite orders gradually}
\begin{columns}[c]
\begin{column}{0.45\textwidth}
\centering
\includegraphics[width=\linewidth,height=0.68\textheight,keepaspectratio]{figures/fig09-context.pdf}
\end{column}
\begin{column}{0.51\textwidth}
\textbf{128 $\to$ 1,024 tokens: mean KL}\par
\medskip
\begin{tabular}{lrr}
\toprule
 & Order 1 & Order 2\\
\midrule
Smol & $0.16\to0.25$ & $0.06\to0.15$\\
OLMo & $0.15\to0.18$ & $0.05\to0.08$\\
\bottomrule
\end{tabular}
\begin{itemize}
\item Thick curves: 1,024; thin: 128 tokens.
\item Both plot axes are logarithmic.
\item 64 test sequences; calibration token budget held fixed.
\item Order 1 still matches the model's top token at most states.
\end{itemize}
\end{column}
\end{columns}
\papersource{Paper, Fig. 9 and App. B, PDF p. 15.}
\end{frame}

\begin{frame}{Backup: the direction gap fits an exponential across depth}
\begin{columns}[c]
\begin{column}{0.46\textwidth}
\centering
\includegraphics[width=\linewidth,height=0.68\textheight,keepaspectratio]{figures/fig08-relaxation.pdf}
\end{column}
\begin{column}{0.50\textwidth}
\begin{itemize}
\item Mean angle shrinks by 9.2\% / 10.5\% of the remaining angle per layer.
\item Exponential fit: $R^2=0.98$ / $0.99$.
\item Smol coherence plateau: layers 8--17, $\sqrt\kappa\approx0.29$.
\item Plateau absent at 2.5T training tokens, present from 8.1T; absent in all 9 OLMo checkpoints.
\end{itemize}
\medskip
{\small Author interpretation: overdamped relaxation. The empirical fit does not establish a physical dynamics model.}
\end{column}
\end{columns}
\papersource{Paper, Fig. 8 and App. B, PDF pp. 14--15.}
\end{frame}

\begin{frame}{Backup: redundant coordinates remain local after transport}
\[
 s_v=W^+e_v,\qquad \sum_v s_vw_v^\top=W^+W=I
 \quad\text{if $W$ has full column rank.}
\]
\[
\sum_v s_\ell(v)(J_\ell^\top w_v)^\top=J_\ell^+J_\ell
\]
\begin{itemize}
\item The vocabulary is larger than state dimension: a redundant frame.
\item The right side is a projection onto output-effective state directions (ignoring ridge).
\item A readout paired with the state measures a rescaling response:
\[
 b_\ell(v)^\top x_\ell=
 \left.\frac{d}{d\alpha}c_v(\alpha x_\ell)\right|_{\alpha=1}.
\]
\item Normalization breaks degree-one homogeneity: coefficient $\neq$ gradient pairing.
\end{itemize}
\papersource{Paper, App. C.1, Eq. (4), PDF p. 17. Dual relation uses uncentred rows; centring and ridge add qualifications.}
\end{frame}

\begin{frame}{Backup: both candidates appear before the final choice}
\textbf{OLMo3 prefix:} ``A review of Boulter 's performance in The''
\medskip
\begin{center}
\begin{tabular}{rlll}
\toprule
Layer & Logit lens & Jacobian lens & Order 1\\
\midrule
8 & nto & urga & Bill\\
12 & comparison & newline fragments & Times\\
16 & orem & Illustrated & \alert{Guardian}\\
20 & odor & Guardian & Guardian\\
24 & Guardian & Guardian & Guardian\\
32 & Guardian & Guardian & Guardian\\
\bottomrule
\end{tabular}
\end{center}
\begin{itemize}
\item Final model: Guardian 0.29; Times 0.22.
\item First among listed layers: order 1 at 16, Jacobian at 20, logit at 24.
\end{itemize}
\papersource{Paper, Table 9 and worked example, PDF p. 16. Selected rows transcribed; newline token displayed descriptively.}
\end{frame}

\begin{frame}{Backup: numerical and statistical protocols}
\begin{itemize}
\item Means exclude embedding and final layer: layers $1,\ldots,L-1$ equally weighted.
\item Fixed seeds and sampled positions; one run, deterministic up to rounding.
\item 95\% percentile bootstrap over sequences, at least 300 resamples.
\item Reported maximum table CI half-widths: 12\% of KL, 0.03 nats for steering Side, 0.04 otherwise; fitted-term ablation uses point estimates.
\item Released half precision for most gradients; FP32 explicit Jacobian in the composition test.
\end{itemize}
\[
\min_\delta\|J_\ell\delta-s_v\|^2+\lambda\|\delta\|^2,\qquad
\lambda=10^{-3}\frac{\|J_\ell^\top s_v\|^2}{\|s_v\|^2}.
\]
\papersource{Paper, App. A.1, PDF pp. 12--13. Basis solved in closed form for composition, with 30 LSMR iterations for steering.}
\end{frame}
\end{document}