% Self-authored paper discussion, based on the skill's beamer-template.tex.
% Main talk: frames 1--20, approximately 25 minutes. Frames 21--26 are backup.
% Author figures are reproduced without changing data; provenance: figures/manifest.json.
\documentclass[aspectratio=169,11pt]{beamer}
\usepackage[T1]{fontenc}
\usepackage[utf8]{inputenc}
\usepackage{amsmath,amssymb,graphicx,booktabs}
\definecolor{AcademicBlue}{RGB}{26,57,86}
\definecolor{AcademicTeal}{RGB}{0,105,112}
\definecolor{BurstZero}{RGB}{76,29,75}
\definecolor{BurstHalf}{RGB}{161,26,91}
\definecolor{BurstNine}{RGB}{232,63,63}
\definecolor{BurstOne}{RGB}{246,156,115}
\definecolor{VarHalf}{RGB}{97,31,83}
\definecolor{VarNine}{RGB}{203,27,79}
\definecolor{VarOne}{RGB}{245,136,96}
\pdfstringdefDisableCommands{\def\translate#1{#1}}
\setbeamercolor{structure}{fg=AcademicBlue}
\setbeamercolor{frametitle}{fg=AcademicBlue,bg=white}
\setbeamercolor{title}{fg=AcademicBlue}
\setbeamercolor{alerted text}{fg=AcademicTeal}
\setbeamercolor{block title}{fg=white,bg=AcademicBlue}
\setbeamercolor{block body}{fg=black,bg=AcademicBlue!5}
\setbeamertemplate{navigation symbols}{}
\setbeamertemplate{itemize items}[circle]
\setbeamertemplate{footline}{\hfill{\scriptsize Chan et al.\ (2022)\quad\insertframenumber/\inserttotalframenumber}\hspace{4mm}\vspace{3mm}}
\setbeamersize{text margin left=8mm,text margin right=8mm}
\setbeamerfont{frametitle}{size=\Large,series=\bfseries}
\setbeamerfont{title}{size=\LARGE,series=\bfseries}
\setlength{\abovedisplayskip}{6pt}
\setlength{\belowdisplayskip}{6pt}
\newcommand{\papersource}[1]{\par\smallskip{\scriptsize\color{gray}#1\par}}
\newcommand{\bigplot}[1]{\par\centering\includegraphics[width=.93\linewidth,height=.65\textheight,keepaspectratio]{figures/#1.pdf}\par}
\newcommand{\burstlegend}{\textcolor{BurstZero}{$p=0$}\quad\textcolor{BurstHalf}{$p=0.5$}\quad\textcolor{BurstNine}{$p=0.9$}\quad\textcolor{BurstOne}{$p=1$}}
\newcommand{\varlegend}{\textcolor{VarHalf}{$p=0.5$}\quad\textcolor{VarNine}{$p=0.9$}\quad\textcolor{VarOne}{$p=1$}}
\title[Data distributions and ICL]{Data Distributional Properties Drive\\Emergent In-Context Learning\\in Transformers}
\subtitle{How training data shape the strategy a model learns}
\author{Paper discussion of Chan et al.\ (2022)}
\date{arXiv:2205.05055v6\quad\textbar\quad Read on 29 September 2026}

\begin{document}

% 1: 1 minute.
\begin{frame}
\titlepage
\vspace{-5mm}
\begin{center}\small\alert{Local repetition and global rarity can favor learning from context.}\end{center}
\end{frame}

% 2: 1.5 minutes.
\begin{frame}{The same training answer can come from two strategies}
\begin{columns}[T]
\begin{column}{.47\textwidth}
\begin{block}{In-context learning (ICL)}
Use examples in the current sequence to assign a label to the query.
\end{block}
No weight updates during the test episode.
\end{column}
\begin{column}{.47\textwidth}
\begin{block}{In-weights learning (IWL)}
Recall an image--label association acquired through training updates.
\end{block}
No matching query class is needed in context.
\end{column}
\end{columns}
\medskip
\alert{Question:} Which training distributions favor each strategy?
\papersource{Paper, Sec. 1, PDF pp. 1--3. ICL ability itself is acquired through training.}
\end{frame}

% 3: 1.5 minutes.
\begin{frame}{A controlled image task isolates data properties}
\begin{columns}[c]
\begin{column}{.46\textwidth}
\includegraphics[width=\linewidth,height=.54\textheight,keepaspectratio]{figures/model.pdf}
\end{column}
\begin{column}{.50\textwidth}
\begin{itemize}
\item Omniglot: 1,623 classes, 20 images per class.
\item Eight image--label pairs, then one query: 17 tokens.
\item ResNet + label embeddings + causal Transformer.
\end{itemize}
\end{column}
\end{columns}
\[
\mathcal L(\theta)=-\log p_\theta(\ell_q\mid\mathcal C,u_q)
\]
\small Only the query label is supervised. $\mathcal C$: context; $u_q$: query image.
\papersource{Paper, Fig. 1a, p. 2; Sec. 2, p. 4. Equation is our notation for the stated loss.}
\end{frame}

% 4: 2 minutes. Explain the restricted candidate metric before any results.
\begin{frame}{Two probes separate context use from stored associations}
\begin{columns}[T]
\begin{column}{.48\textwidth}
\textbf{ICL: unseen image classes}
\medskip
\includegraphics[width=\linewidth,height=.23\textheight,keepaspectratio]{figures/eval-icl.pdf}
\begin{itemize}
\item Two classes, four examples each.
\item Random labels 0/1 per episode.
\item Compare only outputs 0 and 1; chance: 50\%.
\end{itemize}
\end{column}
\begin{column}{.48\textwidth}
\textbf{IWL: trained image classes}
\medskip
\includegraphics[width=\linewidth,height=.23\textheight,keepaspectratio]{figures/eval-iwl.pdf}
\begin{itemize}
\item Query class absent from context.
\item Training labels retained.
\item Full output space; usual chance: $1/1600$.
\end{itemize}
\end{column}
\end{columns}
\papersource{Paper, Fig. 1c--d, p. 2; Sec. 2.3, pp. 4--5. Labels 0/1 are familiar, not novel.}
\end{frame}

% 5: 1 minute.
\begin{frame}{Burstiness makes the query class locally available}
\begin{columns}[c]
\begin{column}{.58\textwidth}
\includegraphics[width=\linewidth,height=.57\textheight,keepaspectratio]{figures/train-sequences.pdf}
\end{column}
\begin{column}{.38\textwidth}
\begin{itemize}
\item Bursty: query class occurs three times in context.
\item Another class also occurs three times.
\item $p(\mathrm{bursty})$ mixes entire sequence types.
\end{itemize}
\end{column}
\end{columns}
\medskip
Default labels stay fixed across training sequences: either strategy can work.
\papersource{Paper, Fig. 1b, p. 2; Sec. 2.1, p. 4. Non-bursty sequences use uniform random sampling.}
\end{frame}

% 6: 1 minute.
\begin{frame}{More burstiness produces stronger ICL}
\centering
\small $p=P(\mathrm{bursty})$:\quad\burstlegend
\bigplot{burst-icl}
\normalsize Without burstiness, held-out ICL stays near chance.
\papersource{Paper, Fig. 2a, PDF p. 3. Mean with standard deviation; 5 runs. Chance: 50\%.}
\end{frame}

% 7: 1 minute.
\begin{frame}{The same intervention reduces parameter-based recall}
\bigplot{burst-iwl}
\normalsize Longer training can shift the balance toward IWL.
\papersource{Paper, Fig. 2b, PDF p. 3; Sec. 3.1, p. 5. Full-output IWL; query unsupported in context.}
\end{frame}

% 8: 1.5 minutes.
\begin{frame}{A larger class vocabulary favors ICL}
\centering
\small Training classes: 100 (purple), 1,600 (blue), 12,800 (teal).
\bigplot{classes-icl}
\normalsize $p(\mathrm{bursty})=0.9$; IWL moves in the opposite direction.
\papersource{Paper, Fig. 3, PDF p. 4. The 12,800-class augmentation may also introduce label ambiguity.}
\end{frame}

% 9: 1 minute.
\begin{frame}{Locally stable, globally variable labels strengthen ICL}
\bigplot{labels-icl}
\normalsize A class has 1, 2, 5, or 10 possible labels; one label is used within each sequence.
\papersource{Paper, Fig. 4, PDF p. 6; Sec. 3.1, pp. 5--6. Fixed burstiness: $p=0.9$.}
\end{frame}

% 10: 1.5 minutes. Selected intact panels, shared legend stated here.
\begin{frame}{Image variation can discourage memorization}
\centering
\small ICL on held-out classes; shared legend:\quad\varlegend
\medskip
\begin{columns}[c]
\begin{column}{.48\textwidth}\centering
\includegraphics[width=\linewidth,height=.57\textheight,keepaspectratio]{figures/variation-icl-zero.pdf}
\end{column}
\begin{column}{.48\textwidth}\centering
\includegraphics[width=\linewidth,height=.57\textheight,keepaspectratio]{figures/variation-icl-half.pdf}
\end{column}
\end{columns}
\medskip
\normalsize Full Omniglot is harder again: more variation is not always better.
\papersource{Paper, selected panels from Fig. 5a, PDF p. 6. 3 seeds; shading: standard deviation.}
\end{frame}

% 11: 1.5 minutes.
\begin{frame}{Zipf frequencies create a common head and a rare tail}
\begin{block}{Class rank controls sampling frequency}
\[
p(X=x)\propto x^{-\alpha},\qquad\alpha\geq0
\]
\end{block}
\begin{itemize}
\item $X$ is class rank; rank 1 is the most common.
\item $\alpha=0$: uniform. Larger $\alpha$: a more concentrated head.
\item Hypothesis: common classes support IWL; rare classes favor ICL.
\end{itemize}
\medskip
\textbf{Experiment:} 12,800 classes, $p(\mathrm{bursty})=0.9$; separately test IWL on the top 10 classes and the rest.
\papersource{Paper, Sec. 3.2, Eq. (1), PDF pp. 7--8. Hypothesis is a behavioral interpretation.}
\end{frame}

% 12: 2 minutes.
\begin{frame}{At $\alpha=1$, two diagnostic abilities coexist}
\begin{columns}[c]
\begin{column}{.49\textwidth}\centering
\textbf{Held-out ICL: two choices}\par\medskip
\includegraphics[width=\linewidth,height=.48\textheight,keepaspectratio]{figures/zipf-icl.pdf}
\end{column}
\begin{column}{.49\textwidth}\centering
\textbf{IWL: 10 common classes}\par\medskip
\includegraphics[width=\linewidth,height=.48\textheight,keepaspectratio]{figures/zipf-common.pdf}
\end{column}
\end{columns}
\medskip
\begin{itemize}
\item Rare-class IWL remains near chance across tested exponents.
\item $\alpha=1$ works well here; its optimality is not universal.
\end{itemize}
\papersource{Paper, Fig. 6c--e, p. 7. Mean over steps 400k--500k; 3 seeds; error bars: standard deviation.}
\end{frame}

% 13: 2 minutes. This qualification belongs in the main talk.
\begin{frame}{Full-output evaluation reveals a selection problem}
\begin{columns}[c]
\begin{column}{.48\textwidth}\centering
\textbf{ICL: all output labels}\par\medskip
\includegraphics[width=\linewidth,height=.48\textheight,keepaspectratio]{figures/zipf-multiclass.pdf}
\end{column}
\begin{column}{.48\textwidth}\centering
\textbf{Output is a context label}\par\medskip
\includegraphics[width=\linewidth,height=.48\textheight,keepaspectratio]{figures/zipf-context-output.pdf}
\end{column}
\end{columns}
\medskip
At $\alpha=1$, full-output ICL drops sharply: the model often selects a label outside the context.
\papersource{Paper, Appendix C.4, p. 16; Figs. 10e and 11, p. 18. Same model, different scoring.}
\end{frame}

% 14: 1.5 minutes. Explicitly restrict the inference to tested configurations.
\begin{frame}{Tested recurrent models fail to acquire comparable ICL}
\begin{columns}[c]
\begin{column}{.47\textwidth}\centering
\textbf{Transformer}\par\medskip
\includegraphics[width=\linewidth,height=.55\textheight,keepaspectratio]{figures/arch-transformer.pdf}
\end{column}
\begin{column}{.50\textwidth}\centering
\textbf{LSTM}\par\medskip
\includegraphics[width=\linewidth,height=.55\textheight,keepaspectratio]{figures/arch-lstm.pdf}
\end{column}
\end{columns}
\medskip
\begin{itemize}
\item Vanilla RNN also stays near chance (Fig. 7b, not shown).
\item 1,600 classes, $p=0.9$; colors identify runs, not matched settings.
\end{itemize}
\papersource{Paper, Fig. 7a,c and Sec. 3.3, p. 8. Chance: 50\%. Inference restricted to tested models.}
\end{frame}

% 15: 1 minute.
\begin{frame}{Reported parameter counts are not exactly matched}
\centering
\begin{tabular}{lrr}
\toprule
Depth & Transformer & LSTM \\
\midrule
12 layers & 831,479 & 627,959 \\
2 layers & 331,639 & 297,719 \\
\bottomrule
\end{tabular}
\medskip
\begin{itemize}
\item Main text and captions say parameter counts were matched.
\item Appendix C.1 gives the unequal counts above; RNN counts are not listed there.
\item Search: 15 runs per architecture/depth, 90 runs in total.
\end{itemize}
\papersource{Paper, Sec. 3.3, p. 8, versus Appendix C.1, p. 15. This discrepancy is our reading note.}
\end{frame}

% 16: 1 minute.
\begin{frame}{Data properties change the relative value of each strategy}
\centering
\begin{tabular}{p{.37\linewidth}p{.23\linewidth}p{.27\linewidth}}
\toprule
Intervention & ICL & IWL \\
\midrule
More local repetition & Increases & Decreases \\
More training classes & Increases & Decreases \\
More labels per class & Increases & Not plotted in Fig. 4 \\
Within-class variation & Often increases & Often decreases \\
Moderate Zipf skew & Can remain high & High on common classes \\
\bottomrule
\end{tabular}
\par\medskip
\alert{Interpretation:} context is useful when local evidence is reliable and globally memorizing every association is difficult.
\papersource{Synthesis of Figs. 2--6, pp. 3--7. Trends depend on conditions; variation is not monotonic.}
\end{frame}

% 17: 1 minute.
\begin{frame}{The evidence has four practical boundaries}
\begin{enumerate}
\item \textbf{Task:} short image sequences and query classification, rather than a language-model training experiment.
\medskip
\item \textbf{Metric:} two-choice ICL does not guarantee correct full-output predictions or novel-label generation.
\medskip
\item \textbf{Interventions:} rotations can add label ambiguity; image variation also changes visual difficulty.
\medskip
\item \textbf{Mechanism:} behavioral results do not identify the circuits implementing either strategy.
\end{enumerate}
\papersource{Paper, Secs. 2--4; Appendices B and C.4. Mechanistic boundary is our assessment.}
\end{frame}

% 18: 1 minute.
\begin{frame}{Three findings to retain}
\begin{block}{Data and architecture jointly shape ICL}
The Transformer architecture alone is insufficient in the tested distributions.
\end{block}
\begin{block}{Global rarity can coexist with local repetition}
Rare classes can still supply repeated evidence within a context window.
\end{block}
\begin{block}{Coexistence leaves output selection unresolved}
A model can do well on both probes yet often choose an irrelevant label.
\end{block}
\papersource{Paper, Sec. 4, pp. 8--10; Appendix C.4, p. 16. Conclusions scoped to this task.}
\end{frame}

% 19: optional discussion within the 20--30 minute slot.
\begin{frame}{Discussion: which control would clarify the mechanism?}
\begin{enumerate}
\item Hold class exposure and label vocabulary fixed while changing local repetition and tail frequencies separately.
\medskip
\item Change the frequency ranks of the two test labels; report both restricted and full-output accuracy.
\medskip
\item Match parameter count and compute more tightly; track when ICL yields to parameter recall during training.
\end{enumerate}
\medskip
\alert{These are proposed follow-ups, not experiments reported in the paper.}
\papersource{Discussion prompts from this reading, motivated by Secs. 3--4 and Appendix C.}
\end{frame}

% 20: end of main talk.
\begin{frame}{Reference and reading materials}
\small
Stephanie C. Y. Chan, Adam Santoro, Andrew K. Lampinen, Jane X. Wang, Aaditya K. Singh, Pierre H. Richemond, James L. McClelland, and Felix Hill.
\par\medskip
\textbf{Data Distributional Properties Drive Emergent In-Context Learning in Transformers.} NeurIPS 2022. arXiv:2205.05055v6, revised 17 November 2022.
\par\medskip
\href{https://arxiv.org/abs/2205.05055v6}{arxiv.org/abs/2205.05055v6}\par
\href{https://github.com/deepmind/emergent_in_context_learning}{Author code: deepmind/emergent\_in\_context\_learning}
\par\medskip
\normalsize
Read: full main text, Appendices A--C, and Figs. 1--11.\\
Materials: matched-version PDF, author TeX, and original figures.\\
This presentation is a reading analysis; no experiments were rerun.
\papersource{Author names follow the PDF. Venue information follows the arXiv record.}
\end{frame}

\appendix

% 21: backup.
\begin{frame}{Backup: training details and uncertainty}
\begin{itemize}
\item Transformer: 12 layers, width 64, 8 attention heads.
\item ResNet: 2 blocks per group; channels (16, 32, 32, 64); trained from scratch.
\item Adam, 500k steps, 16 TPU v2/v3 cores.
\item Warm up to $3\times10^{-4}$ at step 4,000, then inverse-square-root decay.
\item Figs. 5--6: 3 seeds. Other standard experiments: 5 runs. Error bars show standard deviation.
\item Architecture search: maximum learning rate $[10^{-5},0.1]$; warmup $[1,10000]$, log-uniform sampling.
\end{itemize}
\papersource{Paper, Appendices A and C.1, PDF p. 15. Search uses 90 runs, not a 225-point grid per model.}
\end{frame}

% 22: backup.
\begin{frame}{Backup: more classes also reduce IWL}
\bigplot{classes-iwl}
At fixed burstiness, the small vocabulary is much easier to memorize.
\papersource{Paper, Fig. 3b, PDF p. 4. Fixed $p(\mathrm{bursty})=0.9$; 5 runs.}
\end{frame}

% 23: backup.
\begin{frame}{Backup: image noise can suppress IWL}
\centering
\small IWL on trained classes; shared legend:\quad\varlegend
\medskip
\begin{columns}[c]
\begin{column}{.48\textwidth}\centering
\includegraphics[width=\linewidth,height=.57\textheight,keepaspectratio]{figures/variation-iwl-zero.pdf}
\end{column}
\begin{column}{.48\textwidth}\centering
\includegraphics[width=\linewidth,height=.57\textheight,keepaspectratio]{figures/variation-iwl-half.pdf}
\end{column}
\end{columns}
\medskip
\normalsize The effect is strongest at high burstiness; low-burstiness models still memorize.
\papersource{Paper, selected panels from Fig. 5b, PDF p. 6. 3 seeds; shading: standard deviation.}
\end{frame}

% 24: backup. One enlarged original panel instead of an unreadable full montage.
\begin{frame}{Backup: familiar classes can also be relabeled in context}
\bigplot{trained-burst}
Training classes are randomly mapped to 0/1 at evaluation. Trends resemble held-out ICL, with slightly higher performance.
\papersource{Paper, Fig. 9a, PDF p. 17; Appendix C.3, pp. 15--16. Other interventions are in Fig. 9b--d.}
\end{frame}

% 25: backup.
\begin{frame}{Backup: Transformers also perform well on IWL}
\begin{columns}[c]
\begin{column}{.48\textwidth}\centering
\textbf{Transformer}\par\medskip
\includegraphics[width=\linewidth,height=.57\textheight,keepaspectratio]{figures/arch-iwl-transformer.pdf}
\end{column}
\begin{column}{.48\textwidth}\centering
\textbf{Vanilla RNN}\par\medskip
\includegraphics[width=\linewidth,height=.57\textheight,keepaspectratio]{figures/arch-iwl-rnn.pdf}
\end{column}
\end{columns}
\medskip
Neither recurrent model has superior IWL; the LSTM comparison is in Fig. 8c. Each colored line represents a run.
\papersource{Paper, Fig. 8a--b, PDF p. 16; Appendix C.2, p. 15. Parameter-count caveat still applies.}
\end{frame}

% 26: backup; original teaching example, not experimental data.
\begin{frame}{Backup: a restricted correct answer can still lose globally}
Let the correct context label be 1, with logits
\[
z_0=1,\qquad z_1=2,\qquad z_7=5.
\]
\begin{columns}[T]
\begin{column}{.47\textwidth}
\begin{block}{Two-choice evaluation}
\[\arg\max_{j\in\{0,1\}}z_j=1\]
Correct: the model ranks the relevant pair well.
\end{block}
\end{column}
\begin{column}{.47\textwidth}
\begin{block}{Full-output evaluation}
\[\arg\max_j z_j=7\]
Incorrect: another familiar label wins.
\end{block}
\end{column}
\end{columns}
\papersource{Teaching example, not paper data. Illustrates Sec. 2.3, p. 5, and Appendix C.4, p. 16.}
\end{frame}

\end{document}