% Compile with pdflatex. Replace the demonstration content before delivery.
\documentclass[aspectratio=169,11pt]{beamer}
\usepackage[T1]{fontenc}
\usepackage{lmodern}
\usepackage[utf8]{inputenc}
\usepackage{amsmath,amssymb,graphicx,booktabs}

\definecolor{AcademicBlue}{RGB}{26,57,86}
\definecolor{AcademicTeal}{RGB}{0,105,112}
\setbeamercolor{structure}{fg=AcademicBlue}
\setbeamercolor{frametitle}{fg=AcademicBlue,bg=white}
\setbeamercolor{title}{fg=AcademicBlue}
\setbeamercolor{alerted text}{fg=AcademicTeal}
\setbeamercolor{block title}{fg=white,bg=AcademicBlue}
\setbeamercolor{block body}{fg=black,bg=AcademicBlue!5}
\setbeamertemplate{navigation symbols}{}
\setbeamertemplate{itemize items}[circle]
\setbeamertemplate{footline}{%
  \hfill\usebeamerfont{page number in head/foot}%
  \insertframenumber\,/\,\inserttotalframenumber\hspace{4mm}\vspace{3mm}}
\setbeamersize{text margin left=8mm,text margin right=8mm}
\setbeamerfont{frametitle}{size=\Large,series=\bfseries}
\setbeamerfont{title}{size=\LARGE,series=\bfseries}
\newcommand{\papersource}[1]{\par\smallskip{\scriptsize\color{gray}#1\par}}

\title[CKA]{Similarity of Neural Network\\Representations Revisited}
\subtitle{Centered kernel alignment for comparing learned representations}
\author{Paper: Kornblith, Norouzi, Lee, and Hinton\\Independent research reading}
\date{ICML 2019 \;|\; arXiv:1905.00414v4\\20 September 2026}
\begin{document}
\begin{frame}
\titlepage
\end{frame}
\begin{frame}{The question is which structure two networks share}
\begin{itemize}
\item Feed the same $n$ examples, in the same order, to two networks.
\item Compare $X\in\mathbb R^{n\times p_1}$ and $Y\in\mathbb R^{n\times p_2}$ after column centering.
\item Can we recover corresponding layers across seeds, widths, architectures, and training datasets?
\item Key idea: compare relations between examples, retaining relative variance across directions.
\end{itemize}
\papersource{Paper, Sec. 1 and Problem Statement, PDF pp. 1--2.}
\end{frame}
\begin{frame}{Invariance determines what similarity can see}
\begin{center}
\begin{tabular}{lll}\toprule
Transformation & Linear CKA & Motivation \\\midrule
Neuron permutation & Invariant & Coordinate symmetry \\
Orthogonal rotation & Invariant & Preserve inner products \\
Global scaling & Invariant & Remove overall amplitude \\
Per-feature scaling & Sensitive & Retain relative variance \\
Invertible linear map & Sensitive & Avoid excessive invariance \\\bottomrule
\end{tabular}\end{center}
\medskip
Preserving a network's output by reparameterization does not preserve gradient-descent dynamics.
\papersource{Paper, Sec. 2, PDF pp. 2--3; Table 1, p. 4.}
\end{frame}
\begin{frame}{Full sample-space rank makes strong invariance collapse}
\begin{block}{Theorem 1}
For $X,Y\in\mathbb R^{n\times p}$ with rank $n$, invariance to any invertible right multiplication implies
\[s(X,Z)=s(Y,Z)\quad\text{for every }Z.\]
\end{block}
\begin{itemize}
\item Complete rows to invertible $X',Y'$; set $A=(X')^{-1}Y'$. Then $XA=Y$.
\item Width $p\ge n$ alone is insufficient: the rank assumption matters.
\item Centered data live in $\mathbf1^\perp$: apply the argument to the effective $n-1$ dimensional sample space.
\end{itemize}
\papersource{Paper, Sec. 2.1, p. 2; Appendix A, p. 12. Centering clarification: reading analysis.}
\end{frame}
\begin{frame}{Compare sample geometry instead of matching neurons}
\[
X,Y\ \longrightarrow\ K=XX^T,\ L=YY^T\ \longrightarrow\ \text{normalized alignment}
\]
\[
\langle XX^T,YY^T\rangle_F=\operatorname{tr}(XX^TYY^T)=\|Y^TX\|_F^2.
\]
\begin{itemize}
\item $K_{ij}$ describes the relation between examples $i$ and $j$ in one network.
\item Both Gram matrices are $n\times n$, even when feature widths differ.
\item One scalar per layer pair; repeat across layers to obtain a heatmap.
\end{itemize}
\papersource{Paper, Sec. 3, Eq. (1), PDF p. 3.}
\end{frame}
\begin{frame}{CKA normalizes a centered kernel dependence statistic}
\[
H=I_n-\frac1n\mathbf1\mathbf1^T,\qquad
\operatorname{HSIC}(K,L)=\frac{\operatorname{tr}(KHLH)}{(n-1)^2}
\]
\[
\operatorname{CKA}(K,L)=\frac{\operatorname{HSIC}(K,L)}
{\sqrt{\operatorname{HSIC}(K,K)\operatorname{HSIC}(L,L)}}.
\]
\begin{itemize}
\item Equivalently: cosine similarity of $HKH$ and $HLH$.
\item Defined for nonzero centered Gram norms; $[0,1]$ for PSD kernels.
\item This is not mutual information; low linear CKA does not establish independence.
\end{itemize}
\papersource{Paper, Sec. 3, Eqs. (3)--(4), PDF pp. 3--4; domain clarification: reading analysis.}
\end{frame}
\begin{frame}{Linear CKA preserves scale ratios without an SVD}
\begin{block}{For column-centered activations}
\[
\operatorname{CKA}_{\rm linear}(X,Y)=
\frac{\|Y^TX\|_F^2}{\|X^TX\|_F\|Y^TY\|_F}.
\]
\end{block}
\begin{itemize}
\item Feature-space and sample-Gram computations give the same value.
\item RBF alternative: $k(x_i,x_j)=\exp(-\|x_i-x_j\|^2/(2\sigma^2))$.
\item Paper sets $\sigma$ to a fraction of median sample distance; this choice preserves global-scale invariance.
\item Fixed absolute RBF bandwidth does not have that guarantee.
\end{itemize}
\papersource{Paper, Sec. 3; Table 1, p. 4; Eq. (14), p. 5.}
\end{frame}
\begin{frame}{CCA and regression remove different amounts of scale}
\[
R^2_{\rm CCA}=\frac{\|Q_Y^TQ_X\|_F^2}{p_1},\qquad
R^2_{\rm LR}=\frac{\|Q_Y^TX\|_F^2}{\|X\|_F^2}.
\]
\begin{itemize}
\item $Q_X,Q_Y$: orthonormal bases of the feature column spaces.
\item CCA compares two whitened spaces; regression whitens the predictor side.
\item SVCCA truncates principal components before CCA (threshold 0.99 in the main comparison).
\item PWCCA weights canonical correlations by feature projections; it is generally asymmetric and not orthogonally invariant.
\end{itemize}
\smallskip
Full-column-rank notation above; use effective bases for rank-deficient inputs.
\papersource{Paper, Sec. 4, pp. 4--5; Appendix C, pp. 12--14.}
\end{frame}
\begin{frame}{CKA emphasizes agreement in high-variance directions}
\[
R^2_{\rm CCA}=\frac1{p_1}\sum_{i,j}\langle u_X^i,u_Y^j\rangle^2
\]
\[
\operatorname{CKA}_{\rm linear}=
\frac{\sum_{i,j}\lambda_X^i\lambda_Y^j\langle u_X^i,u_Y^j\rangle^2}
{\sqrt{\sum_i(\lambda_X^i)^2}\sqrt{\sum_j(\lambda_Y^j)^2}}.
\]
\begin{itemize}
\item $u^i$: unit eigenvector of a sample Gram matrix; $\lambda^i$: its eigenvalue.
\item CCA weights subspace overlap equally; CKA weights both sides by their spectra.
\item Benefit and limitation: low-variance differences contribute less.
\end{itemize}
\papersource{Paper, Sec. 5, Eqs. (13)--(14), PDF p. 5.}
\end{frame}
\begin{frame}{Same subspace can mean very different sample geometry}
\begin{block}{Teaching example, not a paper experiment}
Let $u,v$ be orthonormal, zero-mean sample vectors:
\[X=[10u,\ v],\qquad Y=[u,\ 10v].\]
\end{block}
\[
R^2_{\rm CCA}=1,\qquad
\operatorname{CKA}_{\rm linear}=\frac{100+100}{10000+1}\approx0.0200.
\]
The spaces coincide, but the dominant direction switches. CKA deliberately regards this as a large change.
\papersource{Teaching derivation from the paper's Eq. (14), p. 5.}
\end{frame}
\begin{frame}{The sanity check tests recovery of corresponding layers}
\begin{itemize}
\item Train 10 architecturally identical Tiny-10 CNNs from different initializations.
\item For each layer, find the most similar layer in each other network; exclude logits.
\item Average layer-identification accuracy. Activations use the CIFAR-10 training set.
\item Symmetrize PWCCA and regression as $S+S^T$.
\item This operational criterion is not a universal ground truth for representation similarity.
\end{itemize}
\papersource{Paper, Sec. 6.1, Fig. 2 and Table 2, PDF p. 6.}
\end{frame}
\begin{frame}{Linear CKA recovers CNN layers with 99.3\% accuracy}
\begin{columns}[c]
\begin{column}{0.48\textwidth}
\centering\includegraphics[width=\linewidth,height=0.70\textheight,keepaspectratio]{figures/tiny_cifar_comparison.pdf}
\end{column}
\begin{column}{0.50\textwidth}
\begin{tabular}{lr}\toprule
Index & Accuracy (\%)\\\midrule
CCA, mean squared & 10.6\\
SVCCA, mean squared & 15.1\\
PWCCA & 11.1\\
Linear regression & 45.4\\
Linear HSIC & 22.2\\
Linear CKA & 99.3\\
RBF CKA, 0.4 & 99.1\\\bottomrule
\end{tabular}
\medskip
\par Layer matching, not classification.\par
Linear and RBF 0.4 CKA are not significantly different under the paper's test.
\end{column}\end{columns}
\papersource{Paper, Fig. 2 and selected rows of Table 2, p. 6; jackknife z-test, p < 0.05.}
\end{frame}
\begin{frame}{Excessive depth can stop refining representations}
\centering
\includegraphics[width=0.94\linewidth,height=0.59\textheight,keepaspectratio]{figures/training_pathology_log_reg.pdf}
\par\smallskip Upper: within-network CKA. Lower: layer-wise logistic probes.\par The 8x-depth network plateaus before halfway through.
\papersource{Paper, Sec. 6.2, Fig. 3, PDF p. 7. Titles show individual-model accuracies.}
\end{frame}
\begin{frame}{Residual structure and depth shape layer correspondence}
\begin{columns}[c]
\begin{column}{0.49\textwidth}
\centering\includegraphics[width=\linewidth,height=0.46\textheight,keepaspectratio]{figures/resnets.pdf}
\par\smallskip ResNet-62: post-residual activations differ from block interiors.
\end{column}
\begin{column}{0.49\textwidth}
\centering\includegraphics[width=\linewidth,height=0.56\textheight,keepaspectratio]{figures/cross_arch.pdf}
\par\smallskip Cross-architecture bands suggest intermediate layers in deeper models.
\end{column}\end{columns}
\papersource{Paper, Sec. 6.2, Figs. 4--5, PDF p. 7; CIFAR-10 test inputs.}
\end{frame}
\begin{frame}{Wider networks agree more; early layers saturate first}
\centering
\includegraphics[width=0.94\linewidth,height=0.59\textheight,keepaspectratio]{figures/layers_paper.pdf}
\par\smallskip Left: comparison with width 4096. Right: same-width models.\par Colors denote conv1--conv8; bands are jackknife standard errors.
\papersource{Paper, Sec. 6.2, Fig. 6, PDF p. 7; 10 models, CIFAR-10 test inputs.}
\end{frame}
\begin{frame}{Training on different datasets still aligns early layers}
\centering
\includegraphics[width=0.94\linewidth,height=0.59\textheight,keepaspectratio]{figures/cross_dataset.pdf}
\par\smallskip Each comparison feeds identical inputs to both models.\par Left/right panels use CIFAR-10/CIFAR-100 test inputs.
\papersource{Paper, Sec. 6.3, Fig. 7, PDF p. 8; trained and untrained controls.}
\end{frame}
\begin{frame}{Shared directions are concentrated in the large spectrum}
\begin{columns}[c]
\begin{column}{0.51\textwidth}
\centering\includegraphics[width=\linewidth,height=0.69\textheight,keepaspectratio]{figures/eigs.png}
\end{column}
\begin{column}{0.47\textwidth}
\begin{itemize}
\item Left: compare scaling by $XX^T$ and $YY^T$ on $u_X^i$.
\item Right: cosine of the rotation under $YY^T$.
\item Large eigenvalues are toward the right of each panel.
\item The 64-unit pooling layer shares roughly 10 dimensions, near the class count.
\end{itemize}
\smallskip An observation, not a universal rank theorem.
\end{column}\end{columns}
\papersource{Paper, Sec. 6.4, Fig. 8, PDF p. 8.}
\end{frame}
\begin{frame}{Transformer results depend on where activations are taken}
\begin{center}
\begin{tabular}{lrrrr}\toprule
Index & Layer Norm & Scale & Attn/FFN & Residual\\\midrule
CCA, mean squared & 87.8 & 87.8 & 95.3 & 95.2\\
PWCCA & 88.5 & 88.9 & 96.1 & 87.0\\
Linear CKA & 78.6 & 95.6 & 86.0 & 73.6\\
RBF CKA, 0.4 & 92.3 & 96.5 & 89.1 & 98.1\\\bottomrule
\end{tabular}\end{center}
\begin{itemize}
\item Layer-identification accuracy (\%); all tested methods are non-trivial.
\item CCA ignores channel-wise scaling; CKA can reveal its consistent structure.
\item Linear CKA is not uniformly best, notably after residual addition.
\end{itemize}
\papersource{Paper, Appendix F.1, selected rows of Table F.1, p. 18; no significance markings reproduced.}
\end{frame}
\begin{frame}{Control experiments narrow the interpretation}
\begin{itemize}
\item Alternative SVCCA thresholds still miss the tested CNN layer structure.
\item Untrained networks also exhibit architecture-induced similarity; high CKA alone does not establish shared learned knowledge.
\item With and without batch normalization, representations remain related; hyperparameters were tuned separately.
\item Within-class CKA resembles all-example CKA, so between-class separation is not the only source of structure.
\end{itemize}
\papersource{Paper, Appendix F.2--F.4, Figs. F.3--F.7, PDF p. 19.}
\end{frame}
\begin{frame}{CKA is useful evidence, not functional equivalence}
\begin{itemize}
\item Interpretation depends on the input sample, kernel, and exact activation location.
\item High-variance weighting can underemphasize low-variance task-relevant differences.
\item Avoiding the invariance theorem does not guarantee freedom from finite-sample or high-dimensional estimation effects.
\item Heatmap similarity does not establish causal interchangeability, safe pruning, or equal out-of-distribution behavior.
\end{itemize}
\papersource{Paper, Sec. 7, p. 8; evidence boundaries and application implications: reading analysis.}
\end{frame}
\begin{frame}{Preserve sample geometry and report the conditions}
\begin{enumerate}
\item Choose invariances to match the question: CKA retains relative spectral scale.
\item Read the strong CNN correspondence result together with the Transformer exceptions.
\item Pair similarity with probes, spectral inspection, and interventions when making functional claims.
\end{enumerate}
\medskip
\textbf{Discussion:} Which differences matter for your task even when their variance is small?
\papersource{Synthesis of Secs. 2--7 and Appendix F; final question is a discussion prompt.}
\end{frame}
\begin{frame}{Reference and reading provenance}
\textbf{Similarity of Neural Network Representations Revisited}\par
Simon Kornblith, Mohammad Norouzi, Honglak Lee, Geoffrey Hinton.\par
ICML 2019; arXiv:1905.00414v4 (19 July 2019).
\medskip\par
\url{https://arxiv.org/abs/1905.00414v4}
\medskip\par
Read from the matching author TeX source and 20-page PDF.\par
Main text and Appendices A--F covered.\par
Figures reproduced from author assets and checked against the PDF.\par
Prepared as an independent reading presentation on 20 September 2026.
\papersource{Page references in this deck use 1-based PDF file pages.}
\end{frame}
\begin{frame}{Backup: regularization connects the similarity family}
\[
\widetilde Q_X=U_X\Sigma_X(\Sigma_X^2+\kappa_X I)^{-1/2}
\]
\[
\|\widetilde Q_Y^T\widetilde Q_X\|_F^2=
\sum_{i,j}\frac{\lambda_X^i\lambda_Y^j}
{(\lambda_X^i+\kappa_X)(\lambda_Y^j+\kappa_Y)}
\langle u_X^i,u_Y^j\rangle^2.
\]
\begin{itemize}
\item Zero regularization: recover CCA with its corresponding normalization.
\item One-sided large regularization: a suitable normalization recovers regression.
\item Both sides large: separable normalization converges to linear CKA.
\end{itemize}
Do not mix the normalization choices when taking these limits.
\papersource{Paper, Appendix D.1, PDF pp. 14--16.}
\end{frame}
\begin{frame}{Backup: a dimensional check on the regression identity}
\[
\operatorname{CKA}(XX^T,Q_YQ_Y^T)=
\frac{\|Q_Y^TX\|_F^2}{\|X^TX\|_F\sqrt{p_2}}
\]
\[
R^2_{\rm LR}=\operatorname{CKA}(XX^T,Q_YQ_Y^T)
\frac{\sqrt{p_2}\|X^TX\|_F}{\|X\|_F^2}.
\]
\begin{itemize}
\item The paper's Eq. (12) prints $\sqrt{p_1}$ in both PDF and TeX.
\item With $Q_Y^TQ_Y=I_{p_2}$, direct substitution gives $\sqrt{p_2}$: an apparent typo.
\item No difference for equal widths; use rank$(Y)$ for an effective rank basis.
\end{itemize}
\papersource{Reading derivation checking Paper, Eq. (12), PDF p. 5; not an author-issued correction.}
\end{frame}
\begin{frame}{Backup: protocols and accuracy numbers are not interchangeable}
\begin{itemize}
\item Fig. 3 individual Plain models: 94.1, 95.0, 93.2, 91.9\% classification accuracy.
\item Table E.2 means: 93.9, 94.8, 93.7, 91.3\% for Plain-10/18/34/66.
\item Transformer: WMT18 English--German; 10 encoders (45 pairs); 12 alternating attention/FFN sublayers.
\item Evaluation: 3,000 newstest2013 sentences, 75,804 tokens, excluding end-of-sentence tokens.
\item Main CNN matching and Appendix F.5 use training inputs; architecture/width plots use test inputs.
\end{itemize}
\papersource{Paper, Fig. 3, p. 7; Table E.2, p. 17; Appendix F.1, p. 18; F.5, p. 20.}
\end{frame}
\end{document}