\documentclass[aspectratio=169,11pt]{beamer}
\usepackage[T1]{fontenc}
\usepackage[utf8]{inputenc}
\usepackage{amsmath,amssymb,graphicx,booktabs}
\usepackage{array}

\definecolor{AcademicBlue}{RGB}{26,57,86}
\definecolor{AcademicTeal}{RGB}{0,105,112}
\definecolor{SoftGray}{RGB}{87,96,105}
\setbeamercolor{structure}{fg=AcademicBlue}
\setbeamercolor{frametitle}{fg=AcademicBlue,bg=white}
\setbeamercolor{title}{fg=AcademicBlue}
\setbeamercolor{alerted text}{fg=AcademicTeal}
\setbeamercolor{block title}{fg=white,bg=AcademicBlue}
\setbeamercolor{block body}{fg=black,bg=AcademicBlue!5}
\setbeamertemplate{navigation symbols}{}
\setbeamertemplate{itemize items}[circle]
\setbeamertemplate{footline}{\hfill\usebeamerfont{page number in head/foot}\insertframenumber\,/\,\inserttotalframenumber\hspace{4mm}\vspace{3mm}}
\setbeamersize{text margin left=8mm,text margin right=8mm}
\setbeamerfont{frametitle}{size=\Large,series=\bfseries}
\setbeamerfont{title}{size=\LARGE,series=\bfseries}
\newcommand{\papersource}[1]{\par\smallskip{\scriptsize\color{SoftGray}#1\par}}
\newcommand{\keyterm}[1]{\textcolor{AcademicTeal}{\textbf{#1}}}
\newcolumntype{L}[1]{>{\raggedright\arraybackslash}p{#1}}

\title[Linear knowledge and transfer]{Knowledge That Is Linear Before Fine-Tuning\\Generalizes After It}
\subtitle{When can recalled facts support learning a new relation?}
\author{Paper by Faridoun Mehri and Hinrich Sch\"utze\\Research group reading presentation}
\date{\href{https://arxiv.org/abs/2610.22300v2}{arXiv:2610.22300v2} (19 Oct 2026)\quad\textbullet\quad Reading: 27 Oct 2026}

\begin{document}

\begin{frame}
  \titlepage
  \vspace{-3mm}
  \begin{center}
    \small \keyterm{Takeaway:} linear readability of old relations predicts transfer after fine-tuning.\\[2mm]
    \scriptsize No formal publication found as of 27 Oct 2026\quad\textbullet\quad
    \href{https://github.com/NightMachinery/active-inert-facts}{Official code and data}
  \end{center}
\end{frame}

\begin{frame}{A recalled fact may still fail to support new learning}
  \begin{block}{Research question}
    Which property of a pretrained model's knowledge predicts whether new knowledge built on it generalizes?
  \end{block}
  \begin{itemize}
    \item \keyterm{Recall:} the model can answer an existing relation, such as city $\to$ continent.
    \item \keyterm{Active knowledge:} the relation's shared structure supports a new task on unseen subjects.
    \item \keyterm{Hypothesis:} linearly readable answers in pretrained subject states support that transfer.
  \end{itemize}
  \papersource{Paper, Secs. 1--2, PDF pp. 1--4. ``Active'' and ``inert'' are task-relative operational terms.}
\end{frame}

\begin{frame}{Random relabeling isolates the old relation's structure}
  \centering
  \includegraphics[width=0.97\linewidth,height=0.70\textheight,keepaspectratio]{figures/fig1-concept.pdf}
  \papersource{Paper, Fig. 1, PDF p. 2. Original conceptual illustration; open circles are unseen subjects.}
\end{frame}

\begin{frame}{Train on some subjects; test on different subjects}
  \begin{columns}[T,onlytextwidth]
    \begin{column}{0.57\textwidth}
      \begin{block}{Constructed relation}
        \[R^{\pi}(u)=\pi(e_R(u)),\qquad \pi:\mathcal E\longrightarrow L\]
        $e_R(u)$: old answer; $L$: random labels; $\pi$: random bijection.
      \end{block}
      \begin{itemize}
        \item Europe $\mapsto$ crab; Asia $\mapsto$ zebra.
        \item Paris $\mapsto$ crab is seen in fine-tuning.
        \item Munich $\mapsto$ crab is tested only if Munich is held out.
      \end{itemize}
    \end{column}
    \begin{column}{0.39\textwidth}
      \begin{block}{Subject split}
        $T$: fine-tuning and probe subjects\\[2mm]
        $H$: held-out evaluation subjects
      \end{block}
      At least four $T$ and one $H$ subject per old answer. $H$ is never used to fit or select the probe.
    \end{column}
  \end{columns}
  \papersource{Paper, Sec. 3, PDF pp. 4--5. The mapping is withheld from the model.}
\end{frame}

\begin{frame}{The three measures answer different questions}
  \begin{tabular}{@{}L{0.22\linewidth}L{0.36\linewidth}L{0.36\linewidth}@{}}
    \toprule
    Measure & Model and subjects & What is scored \\
    \midrule
    Recall & Pretrained model; $T$ & Old answers under old relation prompts \\[1mm]
    Pre-linearity & Pretrained model; $T$ & Cross-validated linear probe of old answers, minus a majority baseline \\[1mm]
    Generalization & Fine-tuned model; $H$ & New labels under a held-out prompt template \\
    \bottomrule
  \end{tabular}
  \vspace{1mm}
  \begin{block}{Common metric: class-balanced candidate accuracy}
    Top-1 over candidate first tokens, averaged equally across answer classes. Chance is $1/|\mathcal E|$.
  \end{block}
  \papersource{Paper, Sec. 3, PDF p. 5. Evaluation is candidate-restricted first-token classification.}
\end{frame}

\begin{frame}{Pre-linearity probes a compact subject representation}
  \begin{columns}[T,onlytextwidth]
    \begin{column}{0.60\textwidth}
      \begin{block}{Probe input and model}
        \[
        z_k=\frac{v_k^{\top}(h-\mu)}{\sigma_k},\quad k=1,\ldots,16.
        \]
        \[
        p(e\mid u)=\operatorname{softmax}(Wz+b)_e.
        \]
      \end{block}
      \begin{itemize}
        \item $h$: state of the subject span's last token in an old-relation query.
        \item PCA gives 16 directions; whitening scales each to unit variance.
        \item Fit cross-entropy plus $10^{-2}\|W\|^2/2$.
      \end{itemize}
    \end{column}
    \begin{column}{0.36\textwidth}
      \begin{block}{Interpretation}
        Can a simple readout separate subjects by their old answers before any new-task fine-tuning?
      \end{block}
      The same dimension and regularization are fixed for every model and relation.
    \end{column}
  \end{columns}
  \papersource{Paper, Sec. 3, PDF p. 5. PCA and whitening are fitted inside each training fold.}
\end{frame}

\begin{frame}{Nested cross-validation keeps layer choice inside training data}
  \centering
  \includegraphics[width=0.94\linewidth,height=0.75\textheight,keepaspectratio]{figures/fig5-cross-validation.pdf}
  \papersource{Paper, Sec. 3, PDF pp. 5--6; Fig. 5, PDF p. 14. Original figure cropped from PDF.}
\end{frame}

\begin{frame}{The test covers 70 relations and 11 language models}
  \begin{columns}[T,onlytextwidth]
    \begin{column}{0.48\textwidth}
      \begin{tabular}{@{}l@{\hspace{3mm}}c@{\hspace{4mm}}l@{}}
        \toprule
        Source & $n$ & Example \\
        \midrule
        Hernandez & 11 & company $\to$ HQ \\
        Town & 6 & town $\to$ latitude band \\
        BEAR & 53 & place $\to$ continent \\
        \midrule
        Total & 70 & $\geq 5$ answers each \\
        \bottomrule
      \end{tabular}
    \end{column}
    \begin{column}{0.48\textwidth}
      \begin{itemize}
        \item 11 open-weight models, about 2B--27B parameters; base and post-trained checkpoints.
        \item Main labels: 179 curated animal names.
        \item Each relation has at least 25 fine-tuning subjects.
        \item Subject frequency uses Dolma v1.7 counts as a pretraining-exposure proxy.
      \end{itemize}
    \end{column}
  \end{columns}
  \papersource{Paper, Sec. 4, PDF pp. 6--8; Appendix E, PDF pp. 21--22.}
\end{frame}

\begin{frame}{All models learn the observed new-relation facts}
  \begin{block}{Fine-tuning recipe}
    Full-parameter updates; four fictional-story templates per training subject; cross-entropy only on the new label's first token; 20 epochs.
  \end{block}
  \begin{itemize}
    \item Each subject--label fact appears 80 times in total: 20 epochs $\times$ 4 wordings.
    \item Evaluation uses a fifth, held-out wording and unseen subjects $H$.
    \item Every reported model reaches 1.00 candidate accuracy on its training subjects.
  \end{itemize}
  \vspace{1mm}
  \alert{The remaining question is generalization to $H$, rather than fitting the observed mapping.}
  \papersource{Paper, Sec. 3, PDF p. 5; Appendix B and Table 3, PDF pp. 14--15.}
\end{frame}

\begin{frame}{Higher pre-linearity tracks higher held-out generalization}
  \centering
  \includegraphics[width=0.98\linewidth,height=0.70\textheight,keepaspectratio]{figures/fig2-scatter.pdf}
  \papersource{Paper, Fig. 2, PDF p. 6. One point per relation in each model; Pearson $r=0.87$--$0.92$ across 70 relations.}
\end{frame}

\begin{frame}{Four controls reduce the association but do not erase it}
  \begin{tabular}{@{}L{0.52\linewidth}cc@{}}
    \toprule
    Within-model association across relations & Pearson $r$ & Spearman $\rho$ \\
    \midrule
    Raw & 0.87--0.92 & 0.85--0.90 \\[1.5mm]
    Controlling for old-relation Recall & 0.79--0.86 & 0.79--0.87 \\[1.5mm]
    Jointly controlling for Recall, answer count, training-subject count and subject frequency & 0.70--0.82 & 0.75--0.85 \\
    \bottomrule
  \end{tabular}
  \vspace{4mm}
  \begin{block}{Statistical unit}
    Correlations are computed across the 70 relations separately within each of the 11 models. Intervals bootstrap relations, with controls refitted in each resample.
  \end{block}
  \papersource{Paper, Table 1, PDF p. 7; Appendix H, PDF p. 24. Numbers are coefficients, not accuracy gains.}
\end{frame}

\begin{frame}{Pre-linearity retains more signal than Recall}
  \begin{columns}[T,onlytextwidth]
    \begin{column}{0.48\textwidth}
      \centering
      \includegraphics[width=\linewidth,height=0.68\textheight,keepaspectratio]{figures/fig3a-predictors.pdf}
    \end{column}
    \begin{column}{0.48\textwidth}
      \begin{itemize}
        \item Pre-linearity $\mid$ Recall: Pearson $r=0.79$--$0.86$.
        \item Recall $\mid$ Pre-linearity: Pearson $r=0.40$--$0.58$.
        \item Recall carries some independent association, but less than Pre-linearity.
      \end{itemize}
    \end{column}
  \end{columns}
  \papersource{Paper, Sec. 5 and Fig. 3a, PDF pp. 8--9. Dots: partial correlations; bars: 95\% bootstrap intervals.}
\end{frame}

\begin{frame}{Changing the labels leaves the pattern intact}
  \begin{columns}[T,onlytextwidth]
    \begin{column}{0.48\textwidth}
      \centering
      \includegraphics[width=\linewidth,height=0.68\textheight,keepaspectratio]{figures/fig3b-label-sets.pdf}
    \end{column}
    \begin{column}{0.48\textwidth}
      \begin{itemize}
        \item Rerun all 70 relations with 13 other WordNet label categories.
        \item Pre-linearity--Generalization Pearson $r=0.91$--$0.93$.
        \item Relation ranking agrees with animal labels: $\rho=0.95$--$0.97$.
      \end{itemize}
      \vspace{2mm}
      \begin{block}{Scope}
        This category comparison is on MiniCPM5-2B-Base only.
      \end{block}
    \end{column}
  \end{columns}
  \papersource{Paper, Sec. 5 and Fig. 3b, PDF pp. 8--9; label sets in Table 7, PDF p. 22.}
\end{frame}

\begin{frame}{New mappings and seeds produce similar correlations}
  \begin{tabular}{@{}lccc@{}}
    \toprule
    MiniCPM5-2B-Base, 70 relations & Run 0 & Run 1 & Run 2 \\
    \midrule
    Pearson $r$ & 0.92 & 0.93 & 0.92 \\
    Spearman $\rho$ & 0.90 & 0.89 & 0.92 \\
    \bottomrule
  \end{tabular}
  \vspace{5mm}
  \begin{itemize}
    \item Each run redraws the answer-to-label bijection and the fine-tuning seed.
    \item On 17 Hernandez and Town relations, repeats across ten models yield Pearson $r=0.78$--$0.95$.
    \item Subject split, label subset and templates remain fixed across repeats.
  \end{itemize}
  \papersource{Paper, Table 2, PDF p. 9; Appendix D, Tables 4--5, PDF p. 20.}
\end{frame}

\begin{frame}{Wrong labels often match the probe's preferred old answer}
  \centering
  \includegraphics[width=0.98\linewidth,height=0.51\textheight,keepaspectratio]{figures/fig4-errors.pdf}
  \vspace{1mm}
  \begin{itemize}
    \item Probe top-3: 21.5\% of errors versus 8.8\% after shuffling wrong labels.
    \item Pretrained model top-3: 11.5\% versus 7.3\% shuffled.
  \end{itemize}
  \papersource{Paper, Fig. 4, PDF p. 9. Shares use equal split when explanations coincide; baselines are shuffled and chance errors.}
\end{frame}

\begin{frame}{The error analysis links the correlation to individual subjects}
  \begin{block}{Diagnostic sequence}
    Wrong new label $\xrightarrow{\pi^{-1}}$ old answer $\xrightarrow{\text{compare}}$ probe and pretrained-model rankings.
  \end{block}
  \begin{itemize}
    \item 37,571 held-out errors from 757 model--relation fine-tunes that made an error.
    \item Probe rankings receive more attribution than the old-relation model rankings.
    \item All considered explanations cover 48.3\% of errors, versus 26.7\% after shuffling.
  \end{itemize}
  \vspace{1mm}
  \alert{Interpretation: errors preferentially follow structure readable by the probe.}
  \papersource{Paper, Sec. 5, PDF p. 9; Appendix G, PDF pp. 23--24. Attribution depends on the named explanations and share rule.}
\end{frame}

\begin{frame}{The evidence supports a representation-based predictor}
  \begin{itemize}
    \item The old relation's linearly readable answer structure is measurable before fine-tuning.
    \item It predicts transfer to unseen subjects across diverse tested relations and models.
    \item Recall, relation size and a frequency proxy do not account for the full association.
  \end{itemize}
  \vspace{2mm}
  \begin{block}{Scope of the inference}
    English relations, arbitrary word labels, full-parameter fine-tuning and candidate-restricted first-token evaluation. Correlations and error alignment do not identify a unique causal mechanism.
  \end{block}
  \papersource{Paper, Secs. 3--6, PDF pp. 4--10. The scope statement is our interpretation of the reported design.}
\end{frame}

\begin{frame}{Take-home: measure usable structure, not just recall}
  \begin{enumerate}
    \item \keyterm{Design:} random relabeling turns an old relation into an unseen transfer test.
    \item \keyterm{Finding:} Pre-linearity predicts Generalization in all 11 models ($r=0.87$--$0.92$).
    \item \keyterm{Mechanistic clue:} wrong new labels align with the probe's old-answer ranking.
  \end{enumerate}
  \vspace{3mm}
  \begin{block}{Question for discussion}
    Would improving linear readability of an existing relation causally improve transfer to a new one?
  \end{block}
  \papersource{Paper, Table 1, PDF p. 7; Fig. 4, PDF p. 9. Discussion question is a proposed follow-up.}
\end{frame}

\begin{frame}{Paper and resources}
  \begin{itemize}
    \item Faridoun Mehri and Hinrich Sch\"utze. \emph{Knowledge That Is Linear Before Fine-Tuning Generalizes After It}. \href{https://arxiv.org/abs/2610.22300v2}{arXiv:2610.22300v2}, 2026.
    \item \href{https://arxiv.org/pdf/2610.22300v2}{Versioned paper PDF} and \href{https://github.com/NightMachinery/active-inert-facts}{official experiment code and data}.
    \item The arXiv record lists no journal reference or conference as of 27 Oct 2026.
  \end{itemize}
  \vspace{5mm}
  \centering\large Questions and discussion
\end{frame}

\appendix
\begin{frame}{Backup: exact model-level Pearson coefficients}
  \centering
  \begin{tabular}{@{}lcc@{}}
    \toprule
    Model & Raw $r$ & Joint-control $r$ \\
    \midrule
    G9v3-3B & .92 & .82 \\
    Gemma-4-E2B & .90 & .80 \\
    Granite-4.2-3B & .87 & .70 \\
    LFM2.5-2.6B-Base & .89 & .76 \\
    MiniCPM5-2B-Base & .92 & .82 \\
    Nanbeige4-3B-Base & .90 & .77 \\
    OLMo-3-1025-7B & .89 & .80 \\
    Phi-4-mini-instruct & .88 & .75 \\
    Qwen3.5-9B & .90 & .77 \\
    Qwen3.5-9B-Base & .87 & .72 \\
    Qwen3.8-27B & .90 & .79 \\
    \bottomrule
  \end{tabular}
  \papersource{Paper, Table 1, PDF p. 7. Joint controls: Recall, answer count, training-subject count and subject frequency.}
\end{frame}

\end{document}