\documentclass[aspectratio=169,11pt]{beamer}
\usepackage[T1]{fontenc}
\usepackage[utf8]{inputenc}
\usepackage{amsmath,amssymb,graphicx,booktabs}

\definecolor{AcademicBlue}{RGB}{26,57,86}
\definecolor{AcademicTeal}{RGB}{0,105,112}
\setbeamercolor{structure}{fg=AcademicBlue}
\setbeamercolor{frametitle}{fg=AcademicBlue,bg=white}
\setbeamercolor{title}{fg=AcademicBlue}
\setbeamercolor{alerted text}{fg=AcademicTeal}
\setbeamercolor{block title}{fg=white,bg=AcademicBlue}
\setbeamercolor{block body}{fg=black,bg=AcademicBlue!5}
\setbeamertemplate{navigation symbols}{}
\setbeamertemplate{itemize items}[circle]
\setbeamertemplate{footline}{%
  \hfill\usebeamerfont{page number in head/foot}%
  \insertframenumber\,/\,\inserttotalframenumber\hspace{4mm}\vspace{3mm}}
\setbeamersize{text margin left=8mm,text margin right=8mm}
\setbeamerfont{frametitle}{size=\Large,series=\bfseries}
\setbeamerfont{title}{size=\LARGE,series=\bfseries}
\newcommand{\papersource}[1]{\par\smallskip{\scriptsize\color{gray}#1\par}}
\newcommand{\LexFrame}{LexFrame}
\newcommand{\LexOne}{LexFrame[1]}
\newcommand{\LexZero}{LexFrame[0]}
\newcommand{\LexTwo}{LexFrame[2]}
\newcommand{\Native}{LexFrame[$\infty$]}

\title[Lexical Reference Frames]{Lexical Reference Frames}
\subtitle{Interpreting LM Residual Streams in Native Semantics}
\author{Jian Gu \and Hongyu Zhang \and Chunyang Chen \and Aldeida Aleti}
\date{arXiv:2610.22169v2 \quad|\quad 10 November 2026}

\begin{document}

\begin{frame}
  \titlepage
  \vspace{-1.1em}
  \begin{center}
    \small
    A model-native, context-aware reading of every residual-stream state\\[0.35em]
    Publication: unverified as of 2026-11-10
    \quad
    \href{https://arxiv.org/abs/2610.22169v2}{arXiv v2}
    \quad
    \href{https://github.com/jianguda/lexframe}{Official code}
  \end{center}
\end{frame}

\begin{frame}{The middle of a language model has no fixed vocabulary frame}
  \begin{columns}[T,onlytextwidth]
    \column{0.49\textwidth}
      \begin{block}{Fixed lens}
        One direction per token and layer.\\
        The same map reads every context.
      \end{block}
      \begin{itemize}
        \item Logit lens reuses the output frame.
        \item Jacobian lens averages over prompts.
      \end{itemize}
    \column{0.49\textwidth}
      \begin{block}{Point cloud}
        Describes which states are near or far.\\
        Does not tell how later layers use them.
      \end{block}
      \begin{itemize}
        \item Distances need not predict intervention effects.
        \item A hidden state needs a model-defined reading.
      \end{itemize}
  \end{columns}
  \vspace{0.6em}
  \begin{alertblock}{Question}
    What vocabulary directions do the remaining layers use to read this state in this context?
  \end{alertblock}
  \papersource{Paper, Sec. 1, PDF pp. 1--2.}
\end{frame}

\begin{frame}{A state's native frame is pulled back through the remaining layers}
  \begin{columns}[T,onlytextwidth]
    \column{0.62\textwidth}
      \includegraphics[width=\linewidth,height=0.58\textheight,keepaspectratio]{figures/reference-frame-transport.pdf}
    \column{0.36\textwidth}
      \begin{itemize}
        \item Input and output frames are fixed by the model's vocabulary.
        \item In between, later layers transport the state and attend to its prefix.
        \item Therefore the relevant token directions depend on layer and context.
      \end{itemize}
  \end{columns}
  \papersource{Paper, Fig. 2, PDF p. 4. Native transport keeps the prefix fixed while carrying the target state forward.}
\end{frame}

\begin{frame}{A frame has a reading side and a writing side}
  \[
    b_\ell(v)=\frac{\partial c_v}{\partial x_\ell(t)}
      =J_\ell^\top(w_v-\bar w)
    \qquad
    s_\ell(v)=J_\ell^{+}s_v
  \]
  {\centering\small \(J_\ell=\partial N(T_\ell)/\partial x_\ell(t)\); \(c_v\) is the centred output logit.\par}
  \begin{columns}[T,onlytextwidth]
    \column{0.49\textwidth}
      \begin{block}{Lexical readout}
        Local gradient of token \(v\)'s centred logit.\\
        Which small state change raises \(v\) fastest?
      \end{block}
    \column{0.49\textwidth}
      \begin{block}{Lexical basis}
        Smallest local move whose transported effect matches output basis \(s_v\).\\
        How can we write \(v\) selectively?
      \end{block}
  \end{columns}
  \vspace{0.3em}
  \centering
  \(\displaystyle z_\ell(v)=\langle w_v,N(T_\ell(x_\ell(t)))\rangle\)
  \quad: transported coefficient used to read the state.
  \papersource{Paper, Sec. 3.2, Eqs. (1)--(2), PDF p. 4; basis construction in Appendix C.1, p. 17.}
\end{frame}

\begin{frame}{Context variation gives every fixed lens a direction ceiling}
  \[
    \frac{1}{m}\sum_i r^\top \hat b_i
      = r^\top\bar b
      \leq \lVert\bar b\rVert
      \longrightarrow \sqrt{\kappa_\ell(v)}
  \]
  \begin{itemize}
    \item \(\hat b_i\): unit native readouts of one token across contexts.
    \item \(\kappa_\ell(v)\): mean cosine of those readouts across sequences.
    \item A state-independent direction cannot follow context-dependent rotation.
  \end{itemize}
  \begin{block}{Mid-depth result}
    Fixed-direction bound: 0.30 (SmolLM3) and 0.35 (OLMo3).\\
    \LexOne{} versus native readouts: cosine 0.97 in both models.
  \end{block}
  \papersource{Paper, Sec. 3.2 and Sec. 4.1, PDF pp. 4, 6; proof in Appendix C.2, p. 17. The bound is for direction similarity, not reading KL.}
\end{frame}

\begin{frame}{LexFrame order controls how much prefix attention is restored}
  \centering
  \includegraphics[width=0.98\linewidth,height=0.61\textheight,keepaspectratio]{figures/lexframe-orders.pdf}
  \begin{itemize}
    \item Order 0: only the target stream; later attention is replaced by a typical output.
    \item Order 1: target attends to order-0 prefix states.
    \item Order 2: prefix states exchange one more round; \LexFrame[\(\infty\)]{} is the full model.
  \end{itemize}
  \papersource{Paper, Fig. 3, PDF p. 5. Higher order means one more round of information exchange between positions.}
\end{frame}

\begin{frame}{Order 0 keeps the model's layers and estimates withheld attention}
  \[
    \hat a_k(y)=\bar a_k+G_k(y-\bar y_k)
  \]
  \begin{itemize}
    \item \(G_k\) is a per-layer ridge regression from attention input to attention output.
    \item Fit once on unlabelled calibration text; no target logits are used.
    \item \LexOne{} restores the target's real attention to the prefix, while prefix states remain order 0.
    \item The next order measures the contribution of one additional prefix exchange.
  \end{itemize}
  \begin{alertblock}{Reading the design}
    Every frame runs the remaining layers. The chosen order controls only how far context information travels between positions.
  \end{alertblock}
  \papersource{Paper, Sec. 3.3, Eq. (3), PDF p. 5; Appendix A.1, p. 13.}
\end{frame}

\begin{frame}{Higher order approaches the model, with a measurable compute cost}
  \begin{itemize}
    \item \LexFrame[\(n\)]{} is exact when at most \(n\) layers remain.
    \item \LexFrame[\(\infty\)]{} returns the model's output at every layer.
    \item Reading all layers at order \(n\) costs about \((n+1)(L+1)/2\) forward-pass units.
  \end{itemize}
  \begin{center}
    \begin{tabular}{ll}
      Calibration & one forward pass over unlabelled text\\
      Reading & \(n+1\) streams through the remaining layers
    \end{tabular}
  \end{center}
  \begin{block}{Trade-off}
    More attention rounds improve fidelity; one fixed matrix multiply is cheaper at read time, but a Jacobian lens also has a costly fit.
  \end{block}
  \papersource{Paper, Appendix C.2--C.3, PDF p. 17. Costs are counted in forward passes, independent of hardware.}
\end{frame}

\begin{frame}{Evaluation uses two open models and one controlled text source}
  \begin{columns}[T,onlytextwidth]
    \column{0.54\textwidth}
      \begin{itemize}
        \item SmolLM3-3B: 36 layers, tied input/output embeddings.
        \item OLMo3-7B: 32 layers, untied embeddings.
        \item WikiText-103; 256 calibration sequences and 64 test sequences, each 128 tokens.
        \item Jacobian lens fitted on 100 prompts; linear probe is a supervised reference.
      \end{itemize}
    \column{0.42\textwidth}
      \begin{block}{Metrics}
        KL: model distribution to reading, nats.\\
        Top-1: agreement with model prediction.\\
        ECE: calibration against the true next token.
      \end{block}
      \begin{itemize}
        \item Layer averages exclude embedding and output.
        \item One deterministic run; intervals bootstrap sequences.
      \end{itemize}
  \end{columns}
  \vspace{-0.2em}
  \papersource{Paper, Sec. 4 and Appendix A, PDF pp. 5--6, 13--14.}
\end{frame}

\begin{frame}{One attention round passes the fixed-lens ceiling}
  {\centering
  \includegraphics[width=0.92\linewidth,height=0.38\textheight,keepaspectratio]{figures/lexical-directions.pdf}\par}
  \vspace{0.25em}
  \begin{columns}[T,onlytextwidth]
    \column{0.49\textwidth}
      \begin{itemize}
        \item OLMo3 layer 16: logit lens is \(75^\circ\) from native; Jacobian lens \(77^\circ\); \LexOne{} \(13^\circ\).
        \item Four layers before output, lenses remain at least \(41^\circ\) away.
      \end{itemize}
    \column{0.49\textwidth}
      \begin{itemize}
        \item At mid-depth, \LexOne{} reaches cosine 0.97 versus the fixed-direction bound 0.30/0.35.
      \end{itemize}
  \end{columns}
  \papersource{Paper, Fig. 1 and Sec. 4.1, PDF pp. 2, 6. Direction angles in the figure are OLMo3-7B.}
\end{frame}

\begin{frame}{LexFrame[1] readings nearly reproduce the model distribution}
  {\centering
  \includegraphics[width=0.86\linewidth,height=0.45\textheight,keepaspectratio]{figures/reading-fidelity.pdf}\par}
  \begin{block}{Layer-averaged reading fidelity}
    \small
    KL, SmolLM3/OLMo3: logit lens 29.06/5.66; Jacobian lens 7.51/6.19; LexFrame[0] 0.87/0.93; \LexOne{} 0.16/0.15; \LexTwo{} 0.06/0.05.\\
    Top-token agreement: \LexOne{} 0.86/0.86; \LexTwo{} 0.92/0.93.
  \end{block}
  \papersource{Paper, Fig. 4 and Table 1, PDF pp. 6--7. Lower KL is closer to the model.}
\end{frame}

\begin{frame}{A worked example reveals a choice before the output layer}
  \begin{block}{Context and model output}
    “A review of Boulter's performance in The”\\
    Model: Guardian \(0.29\) \qquad runner-up: Times \(0.22\)
  \end{block}
  \centering
  \begin{tabular}{rlll}
    \toprule
    Layer & Logit lens & Jacobian lens & LexFrame[1]\\
    \midrule
    0  & “oretical” & -- & “New”\\
    12 & “comparison” & “\ldots” & “Times”\\
    16 & “orem” & “Illustrated” & “Guardian”\\
    20 & “odor” & “Guardian” & “Guardian”\\
    24 & “Guardian” & “Guardian” & “Guardian”\\
    \bottomrule
  \end{tabular}
  \begin{itemize}
    \item At layer 12, LexFrame exposes the runner-up; at layer 16, it reads the final choice.
    \item The two lenses reach “Guardian” only at layers 20 and 24.
  \end{itemize}
  \papersource{Paper, Appendix B, Table 9, PDF p. 16. A single-position example; aggregate results are in Tables 1 and 7.}
\end{frame}

\begin{frame}{State-space distance poorly predicts what substitution changes}
  \begin{columns}[T,onlytextwidth]
    \column{0.56\textwidth}
      \includegraphics[width=\linewidth,height=0.52\textheight,keepaspectratio]{figures/point-cloud-tangent.pdf}
    \column{0.41\textwidth}
      \begin{itemize}
        \item Replace one position's state with another real state from the same layer.
        \item No point-cloud distance correlates above 0.28 with the downstream output change.
        \item The divergence between \LexOne{} readings correlates at 0.94/0.95.
      \end{itemize}
      \begin{alertblock}{Why}
        The model's lexical readouts lie mostly outside the leading directions in which the state cloud varies.
      \end{alertblock}
  \end{columns}
  \vspace{-0.3em}
  \papersource{Paper, Sec. 4.2, Table 2 and Fig. 5, PDF p. 7; 7,961 substitution pairs per layer.}
\end{frame}

\begin{frame}{The direction that measures a token is not the one that writes it}
  \begin{itemize}
    \item Move each state by 10\% of its norm along a token direction.
    \item Count how often that token gains the most logit.
  \end{itemize}
  \centering
  \begin{tabular}{lrr}
    \toprule
    Direction & SmolLM3 & OLMo3\\
    \midrule
    LexFrame[1], readout & 0.54 & 0.73\\
    LexFrame[1], basis & 0.84 & 0.97\\
    Native frame, readout & 0.56 & 0.79\\
    Native frame, basis & 0.98 & 1.00\\
    \bottomrule
  \end{tabular}
  \begin{block}{Interpretation}
    The readout is a gradient. The basis is a local move whose output effect best matches the token's lexical direction.
  \end{block}
  \papersource{Paper, Sec. 4.3, Table 3, PDF p. 8. Values are shares of moves where the target token gains the most.}
\end{frame}

\begin{frame}{Intermediate-layer coefficients need the transport}
  \centering
  \begin{tabular}{lrrrr}
    \toprule
    State used for output coordinates & Smol KL & Smol Top-1 & OLMo KL & OLMo Top-1\\
    \midrule
    Layer state & 8.36 & 0.04 & 5.64 & 0.15\\
    Fixed map & 5.19 & 0.13 & 5.46 & 0.13\\
    LexFrame[1] transport & 0.16 & 0.87 & 0.15 & 0.84\\
    Native transport & 0.00 & 1.00 & 0.00 & 1.00\\
    \bottomrule
  \end{tabular}
  \vspace{1.0em}
  \begin{itemize}
    \item Coefficients are output unembedding scores after transport.
    \item A local gradient dotted with the layer state is not the amount of a token carried by that state.
  \end{itemize}
  \papersource{Paper, Sec. 4.3, Table 4 and Appendix C.1, PDF pp. 8, 17.}
\end{frame}

\begin{frame}{Lexical bases steer predictions with fewer side effects}
  \small
  \centering
  \begin{tabular}{llrrr}
    \toprule
    Model & Direction & Reach & Other-logit change & Clean\\
    \midrule
    SmolLM3 & LexFrame[1] readout & 0.92 & 0.23 & 0.61\\
    SmolLM3 & LexFrame[1] basis & 0.89 & 0.12 & 0.71\\
    SmolLM3 & Native basis & 0.90 & 0.11 & 0.73\\
    OLMo3 & LexFrame[1] readout & 0.94 & 0.20 & 0.64\\
    OLMo3 & LexFrame[1] basis & 0.95 & 0.07 & 0.82\\
    OLMo3 & Native basis & 0.97 & 0.06 & 0.85\\
    \bottomrule
  \end{tabular}
  \begin{itemize}
    \item Reach: share of runner-up tokens made top-1.
    \item Other-logit change: mean change to other tokens at the first flip, in nats.
    \item Clean: flips with other-token change below 0.1 nats.
  \end{itemize}
  \papersource{Paper, Sec. 4.4, Table 5, PDF p. 9.}
\end{frame}

\begin{frame}{Coefficient differences capture nonlinear downstream effects}
  \small
  \centering
  \begin{tabular}{llrrr}
    \toprule
    Model & Attribution & Layer & Attention & MLP\\
    \midrule
    SmolLM3 & LexFrame[1], difference & 0.88 & 0.79 & 0.88\\
    SmolLM3 & Native, linear & 0.62 & 0.75 & 0.63\\
    OLMo3 & LexFrame[1], difference & 0.90 & 0.85 & 0.90\\
    OLMo3 & Native, linear & 0.68 & 0.84 & 0.71\\
    \bottomrule
  \end{tabular}
  \begin{itemize}
    \item Reference effect: remove a component's write and run the remaining model.
    \item Difference of transported coefficients needs no linearization.
    \item The largest gap is for MLP writes, where downstream response is less linear.
  \end{itemize}
  \papersource{Paper, Sec. 4.4, Table 6, PDF p. 9. Entries are layer-averaged correlations with measured logit effects.}
\end{frame}

\begin{frame}{LexFrame closes the angle to native readouts gradually}
  \begin{columns}[T,onlytextwidth]
    \column{0.50\textwidth}
      \includegraphics[width=\linewidth,height=0.43\textheight,keepaspectratio]{figures/depth-relaxation.pdf}
    \column{0.47\textwidth}
      \begin{itemize}
        \item Mean angle shrinks by 9.2\% (SmolLM3) and 10.5\% (OLMo3) of what remains per layer.
        \item Exponential fit \(R^2=0.98/0.99\).
        \item The authors relate this pattern to exponential relaxation in an overdamped system.
      \end{itemize}
  \end{columns}
  \papersource{Paper, Appendix B, Fig. 8, PDF pp. 14--15. This is an empirical fit, not a derived Transformer-dynamics law.}
\end{frame}

\begin{frame}{Readings suggest early exits, but LexFrame itself does not save compute}
  \small
  \centering
  \begin{tabular}{llrr}
    \toprule
    Model & Reading & Settled-layer share & Exit skip share\\
    \midrule
    SmolLM3 & Logit lens & 0.03 & 0.00\\
    SmolLM3 & Jacobian lens & 0.09 & 0.00\\
    SmolLM3 & LexFrame[1] & 0.82 & 0.45\\
    OLMo3 & Logit lens & 0.11 & 0.04\\
    OLMo3 & Jacobian lens & 0.09 & 0.03\\
    OLMo3 & LexFrame[1] & 0.81 & 0.42\\
    \bottomrule
  \end{tabular}
  \begin{itemize}
    \item Threshold chosen for 95\% model agreement on half the sequences; evaluated on the other half.
    \item LexFrame transports the state through later layers to read it, so these figures measure potential depth, not achieved speedup.
  \end{itemize}
  \papersource{Paper, Appendix B, Table 7, PDF pp. 15--16.}
\end{frame}

\begin{frame}{The key fidelity checks survive longer context and fit ablations}
  \begin{columns}[T,onlytextwidth]
    \column{0.49\textwidth}
      \begin{block}{128 to 1,024 tokens}
        \begin{itemize}
          \item \LexOne{} KL: 0.16/0.15 to 0.25/0.18.
          \item \LexTwo{} KL: 0.06/0.05 to 0.15/0.08.
          \item Ranking of methods is unchanged.
        \end{itemize}
      \end{block}
    \column{0.49\textwidth}
      \begin{block}{Remove the fitted regression}
        \begin{itemize}
          \item Mean-only KL: 1.26 (SmolLM3), 0.52 (OLMo3).
          \item Full regression KL: 0.16/0.15.
          \item Readout cosine and attribution also decline.
        \end{itemize}
      \end{block}
  \end{columns}
  \papersource{Paper, Appendix B, Fig. 9 and Table 8, PDF pp. 15--16.}
\end{frame}

\begin{frame}{Takeaways and evidence boundaries}
  \begin{enumerate}
    \item A hidden state is best read in vocabulary directions induced by its remaining layers and context.
    \item One round of target-to-prefix attention largely closes the middle-layer gap to the native frame in these two models.
    \item Readout gradients and lexical basis moves answer different questions; transport is essential for intermediate coefficients.
  \end{enumerate}
  \begin{block}{What the experiments establish}
    Strong direction, reading, substitution, steering and attribution results on two open models and WikiText-103.
  \end{block}
  \begin{itemize}
    \item One deterministic run per setting; no independent retraining replication.
    \item Early-exit savings are potential skips, since LexFrame itself computes through later layers.
  \end{itemize}
  \papersource{Paper, Secs. 4--5 and Appendix A, PDF pp. 5--9, 13--14.}
\end{frame}

\end{document}