\documentclass[aspectratio=169,11pt]{beamer}
\usepackage[T1]{fontenc}
\usepackage[utf8]{inputenc}
\usepackage{lmodern}
\usepackage{amsmath,amssymb,graphicx,booktabs}

\definecolor{AcademicBlue}{RGB}{26,57,86}
\definecolor{AcademicTeal}{RGB}{0,105,112}
\setbeamercolor{structure}{fg=AcademicBlue}
\setbeamercolor{frametitle}{fg=AcademicBlue,bg=white}
\setbeamercolor{title}{fg=AcademicBlue}
\setbeamercolor{alerted text}{fg=AcademicTeal}
\setbeamercolor{block title}{fg=white,bg=AcademicBlue}
\setbeamercolor{block body}{fg=black,bg=AcademicBlue!5}
\setbeamertemplate{navigation symbols}{}
\setbeamertemplate{itemize items}[circle]
\setbeamertemplate{footline}{%
  \hfill\usebeamerfont{page number in head/foot}%
  \insertframenumber\,/\,\inserttotalframenumber\hspace{4mm}\vspace{3mm}}
\setbeamersize{text margin left=8mm,text margin right=8mm}
\setbeamerfont{frametitle}{size=\Large,series=\bfseries}
\setbeamerfont{title}{size=\LARGE,series=\bfseries}
\newcommand{\papersource}[1]{\par\smallskip{\scriptsize\color{gray}#1\par}}

\title[LiSA]{LiSA: Linear Stochastic Attention for Long-Range Visual Modeling}
\subtitle{Multi-hop token interactions with linear sequence scaling}
\author{\texorpdfstring{Sahar Dastani, Fereshteh Shakeri, Samuel Barbeau, Kunal Mahatha,\\
Ismail Ben Ayed, Herve Lombaert, Christian Desrosiers}{Sahar Dastani, Fereshteh Shakeri, Samuel Barbeau, Kunal Mahatha, Ismail Ben Ayed, Herve Lombaert, Christian Desrosiers}}
\date{arXiv:2611.04216v2 \quad|\quad 4 November 2026}

\begin{document}

\begin{frame}
  \titlepage
  \vspace{-5mm}
  \centering
  \small
  \href{https://github.com/Sahardastani/lisa}{WACV 2027 author project; official code forthcoming}
\end{frame}

\begin{frame}{Can linear attention model indirect token relationships?}
  \begin{block}{The paper's question}
    Can a vision model add multi-hop context while keeping attention cost linear in the number of patches?
  \end{block}
  \begin{itemize}
    \item Softmax attention explicitly compares every pair of tokens: quadratic time and memory.
    \item Kernelized linear attention avoids the \(n\times n\) matrix, but aggregates each value through one direct query--key interaction.
    \item LiSA treats token similarities as transition probabilities and propagates information along short random walks.
  \end{itemize}
  \papersource{Paper, Sec. 1, PDF pp. 1--2.}
\end{frame}

\begin{frame}{Kernelized linear attention is a one-hop update}
  For positive feature maps \(Q,K\in\mathbb{R}^{n\times d_h}\):
  \[
    Y=Q(K^\top V).
  \]
  \begin{itemize}
    \item First aggregate keys and values into a \(d_h\times d\) summary.
    \item Each query reads that summary once: direct, first-order interaction.
    \item Indirect paths such as \(A^2V\) and \(A^3V\), for \(A=QK^\top\), are absent.
  \end{itemize}
  \begin{block}{LiSA's design target}
    Add useful transitive paths without forming token-by-token attention.
  \end{block}
  \papersource{Paper, Secs. 1 and 3.1, PDF pp. 2--3.}
\end{frame}

\begin{frame}{LiSA expands the receptive field through low-rank terms}
  \centering
  \includegraphics[width=0.97\linewidth,height=0.66\textheight,keepaspectratio]{figures/fig-01-higher-order.png}
  \papersource{Paper, Fig. 1, PDF p. 1. The right panel contrasts multi-hop terms with the \(n\times n\) softmax matrix.}
\end{frame}

\begin{frame}{The layer blends global diffusion with local detail}
  \centering
  \includegraphics[width=0.98\linewidth,height=0.72\textheight,keepaspectratio]{figures/fig-02-lisa-overview.png}
  \papersource{Paper, Fig. 2, PDF p. 4. The schematic shows a global transition branch and a masked \(3\times3\) local branch.}
\end{frame}

\begin{frame}{A random walk assigns weight to paths of different lengths}
  Normalize the positive query--key affinities into a row-stochastic matrix:
  \[
    S=\tilde QK^\top,\qquad
    \tilde Q=D^{-1}Q,\quad D=\operatorname{Diag}\bigl(Q(K^\top\mathbf 1)\bigr).
  \]
  Truncate the geometric walk after \(T\) terms:
  \[
    P^{(T)}=\frac{1-\beta}{1-\beta^T}
      \sum_{k=0}^{T-1}\beta^kS^k,\qquad
    Y_{\mathrm{global}}=P^{(T)}V.
  \]
  \begin{itemize}
    \item \(S^0=I\) retains the starting token; \(S^k\) propagates across \(k\) transitions.
    \item \(\beta\) controls continuation; \(T\) is the maximum retained path depth.
  \end{itemize}
  \papersource{Paper, Sec. 3.2, Eqs. (4)--(6), PDF pp. 3--4.}
\end{frame}

\begin{frame}{Low rank keeps the recurrence in feature space}
  Since \(S=\tilde QK^\top\), for \(k\geq1\):
  \[
    S^kV=\tilde Q(K^\top\tilde Q)^{k-1}K^\top V.
  \]
  \begin{itemize}
    \item The repeated core \(K^\top\tilde Q\) is only \(d_h\times d_h\).
    \item LiSA never needs to materialize the \(n\times n\) matrix \(S\).
    \item The paper reports \(O(Td_h^3+ndd_h)\); with small fixed \(T\), cost is linear in token count \(n\).
  \end{itemize}
  \begin{block}{Practical choice}
    Main experiments use \(T=3\); each additional step increases the propagation work.
  \end{block}
  \papersource{Paper, Sec. 3.2, Eq. (8) and Algorithm 1, PDF pp. 4--5.}
\end{frame}

\begin{frame}{Adaptive hop weights and a local branch protect detail}
  \begin{columns}[T]
    \column{0.51\linewidth}
      \textbf{Global branch}
      \[
        Y_g=\beta_0V+
        \sum_{k=0}^{T-2}\beta_{k+1}
        \tilde Q(K^\top\tilde Q)^kK^\top V.
      \]
      \begin{itemize}
        \item A small MLP predicts nonnegative step weights per token and head.
        \item Softmax makes the \(T\) weights sum to one.
      \end{itemize}
    \column{0.45\linewidth}
      \textbf{Local branch}
      \begin{itemize}
        \item Masked similarity only within a \(3\times3\) spatial neighborhood.
        \item Includes the token itself as a self-loop.
        \item Preserves sharp local structure while global diffusion carries long-range context.
      \end{itemize}
  \end{columns}
  \papersource{Paper, Sec. 3.2, Eq. (10), PDF p. 5; adaptive weights in Supp. Sec. 8, PDF p. 12.}
\end{frame}

\begin{frame}{A learned mixture balances global and local evidence}
  \[
    Y=(1-\alpha)Y_{\mathrm{global}}+\alpha Y_{\mathrm{local}},
    \qquad \alpha\in[0,1].
  \]
  \begin{itemize}
    \item Global-only and local-only variants are both weaker than the learned mixture in the reported ablation.
    \item Repeated averaging can oversmooth features as \(T\) grows.
    \item The local self-loop and branch retain neighboring detail while diffusion extends reach.
  \end{itemize}
  \begin{alertblock}{Modeling trade-off}
    More hops can connect distant regions, but excessive diffusion weakens token distinctions.
  \end{alertblock}
  \papersource{Paper, Sec. 3.2 and Table 6, PDF pp. 5, 7; failure cases in Supp. Sec. 9, PDF p. 12.}
\end{frame}

\begin{frame}{Evaluation spans classification and dense prediction}
  \begin{itemize}
    \item \textbf{ImageNet-1K:} DeiT, PVT, Swin, and BiFormer; Top-1 accuracy. Main classification models are trained from scratch for 300 epochs with AdamW and standard augmentation.
    \item \textbf{ADE20K:} Semantic FPN and UPerNet with ImageNet-initialized backbones; mIoU and mAcc.
    \item \textbf{COCO:} Mask R-CNN with PVT-T and Swin-T; 1\(\times\) schedule; box and mask AP.
    \item The paper reports eight A100/H100 GPUs for classification; the efficiency table does not state GPU or batch size.
  \end{itemize}
  \papersource{Paper, Sec. 4, PDF pp. 5--7; detailed efficiency table in Supp. Sec. 12, PDF p. 13.}
\end{frame}

\begin{frame}{Top-1 accuracy improves across four backbone families}
  \centering
  \begin{tabular}{lrrr}
    \toprule
    Backbone / input & Self-attention & FLatten & LiSA \\
    \midrule
    DeiT-T, \(224^2\) & 72.20 & 74.10 & \textbf{74.54} \\
    DeiT-T, \(384^2\) & 74.90 & 76.53 & \textbf{77.54} \\
    DeiT-S, \(224^2\) & 79.80 & 80.10 & \textbf{80.78} \\
    PVT-T, \(224^2\) & 75.10 & 77.80 & \textbf{78.53} \\
    PVT-S, \(224^2\) & 79.80 & 81.70 & \textbf{81.96} \\
    Swin-T, \(224^2\) & 81.30 & 82.10 & \textbf{82.25} \\
    \bottomrule
  \end{tabular}
  \begin{itemize}
    \item Gains over self-attention range from \(+0.95\) to \(+3.43\) percentage points in this selection.
    \item LiSA also exceeds FLatten on each listed row; the margin is smaller on Swin-T (\(+0.15\)).
  \end{itemize}
  \papersource{Paper, Table 1, PDF p. 6. All rows use the reported main setting \(T=3\).}
\end{frame}

\begin{frame}{The largest classification gains appear on PVT and higher resolution}
  \begin{columns}[T]
    \column{0.48\linewidth}
      \begin{block}{PVT-T at \(224^2\)}
        LiSA reaches \(78.53\%\) Top-1:
        \begin{itemize}
          \item \(+3.43\) points over PVT.
          \item \(+0.73\) points over FLatten.
          \item FLOPs: 2.12G, versus 1.94G for PVT and 2.00G for FLatten.
        \end{itemize}
      \end{block}
    \column{0.48\linewidth}
      \begin{block}{DeiT-T at \(384^2\)}
        LiSA reaches \(77.54\%\) Top-1:
        \begin{itemize}
          \item \(+2.64\) points over DeiT-T.
          \item \(+1.01\) points over FLatten.
          \item 3.75G FLOPs, below DeiT-T's 4.70G.
        \end{itemize}
      \end{block}
  \end{columns}
  BiFormer also improves: \(+1.0/+0.3/+0.7\) Top-1 points for T/S/B.
  \papersource{Paper, Tables 1--2, PDF p. 6. Gains are absolute percentage points.}
\end{frame}

\begin{frame}{ADE20K gains are strong on PVT-T, mixed against FLatten on Swin}
  \centering
  \begin{tabular}{llrrr}
    \toprule
    Backbone & Method & FLOPs & mIoU & mAcc \\
    \midrule
    PVT-T / S-FPN & PVT & 158G & 36.57 & 46.72 \\
    & FLatten & 169G & 37.21 & 48.95 \\
    & LiSA & 170G & \textbf{40.85} & \textbf{51.75} \\
    \midrule
    Swin-T / UPerNet & Swin & 945G & 44.51 & 55.61 \\
    & FLatten & 946G & 44.82 & \textbf{57.01} \\
    & LiSA & 947G & \textbf{45.03} & 56.40 \\
    \bottomrule
  \end{tabular}
  \begin{itemize}
    \item PVT-T vs. PVT: \(+4.28\) mIoU and \(+5.03\) mAcc points.
    \item Swin-T vs. FLatten: \(+0.21\) mIoU, but \(-0.61\) mAcc points.
  \end{itemize}
  \papersource{Paper, Table 3, PDF p. 6. mIoU and mAcc are higher-is-better metrics.}
\end{frame}

\begin{frame}{COCO detection gains are larger with the PVT-T backbone}
  \centering
  \begin{tabular}{lrrr}
    \toprule
    Backbone & FLOPs & Box AP & Mask AP \\
    \midrule
    PVT-T & 240G & 36.7 & 35.1 \\
    LiSA-PVT-T & 245G & \textbf{40.5} & \textbf{37.6} \\
    \midrule
    Swin-T & 267G & 43.7 & 39.8 \\
    LiSA-Swin-T & 269G & \textbf{44.5} & \textbf{40.3} \\
    \bottomrule
  \end{tabular}
  \begin{itemize}
    \item PVT-T: \(+3.8\) box AP and \(+2.5\) mask AP for 5G more FLOPs.
    \item Swin-T: \(+0.8\) box AP and \(+0.5\) mask AP for 2G more FLOPs.
    \item Both use Mask R-CNN, a 1\(\times\) schedule, and 1280\(\times\)800 input.
  \end{itemize}
  \papersource{Paper, Table 4, PDF p. 7. AP is averaged over COCO IoU thresholds.}
\end{frame}

\begin{frame}{Short walks capture most of the ablation gain}
  \begin{columns}[T]
    \column{0.55\linewidth}
      \textbf{Top-1 by truncation depth}

      \begin{tabular}{cr}
        \toprule
        \(T\) & Mean \(\pm\) std. (3 runs) \\
        \midrule
        1 & \(73.34\pm0.64\) \\
        2 & \(75.43\pm0.56\) \\
        3 & \(75.63\pm0.39\) \\
        4 & \(75.16\pm0.13\) \\
        5 & \(\mathbf{75.73\pm0.47}\) \\
        \bottomrule
      \end{tabular}
    \column{0.41\linewidth}
      \textbf{Learned global/local mix}

      {\small
      \begin{tabular}{lrrr}
        \toprule
        Model & Global & Local & Mix \\
        \midrule
        DeiT-T & 73.7 & 73.4 & \textbf{76.3} \\
        DeiT-S & 78.3 & 78.1 & \textbf{80.7} \\
        \bottomrule
      \end{tabular}
      }
  \end{columns}
  \vspace{2mm}
  Main experiments use \(T=3\) for lower compute; \(T=5\) is only slightly higher in this ablation.
  \papersource{Paper, Fig. 3 and Table 6, PDF p. 7; ablations use mini-ImageNet unless stated otherwise.}
\end{frame}

\begin{frame}{The proposed attention core contributes beyond FLatten add-ons}
  \centering
  \begin{tabular}{lrrrr}
    \toprule
    Core & No add-on & +F & +D & +F+D \\
    \midrule
    LA & 65.7 & 65.9 & 73.3 & 74.4 \\
    GL (LiSA) & \textbf{69.2} & \textbf{69.3} & \textbf{75.2} & \textbf{76.3} \\
    \bottomrule
  \end{tabular}
  \begin{itemize}
    \item F and D denote FLatten's focusing and depthwise-convolution components.
    \item GL beats LA with each matched auxiliary configuration.
    \item LiSA also improves linear-angular DeiT-T from 66.0 to 73.4, and Agent Attention by \(+2.37\) (T) / \(+2.01\) (S).
  \end{itemize}
  \papersource{Paper, Table 7 and Sec. 4.4, PDF p. 8; alternative plug-ins in PDF pp. 7--8.}
\end{frame}

\begin{frame}{Qualitative maps show the intended transitive behavior}
  \centering
  \includegraphics[width=0.98\linewidth,height=0.62\textheight,keepaspectratio]{figures/fig-04-transitive.png}
  \begin{itemize}
    \item Local attention stays near the query patch.
    \item LiSA response spreads to more of the bird as \(T\) grows from 3 to 7.
  \end{itemize}
  \papersource{Paper, Fig. 4b, PDF p. 8. This is a qualitative visualization, not an accuracy metric.}
\end{frame}

\begin{frame}{At long sequences, LiSA saves memory but trails FLatten}
  \begin{columns}[T]
    \column{0.49\linewidth}
      \centering
      \includegraphics[width=\linewidth,height=0.42\textheight,keepaspectratio]{figures/fig-05-latency.png}
      {\small Latency}
    \column{0.49\linewidth}
      \centering
      \includegraphics[width=\linewidth,height=0.42\textheight,keepaspectratio]{figures/fig-05-memory.png}
      {\small Peak GPU memory}
  \end{columns}
  \begin{itemize}
    \item At 16,385 tokens: 162 ms / 3.4 GB for LiSA, 92 ms / 1.6 GB for FLatten, and 899 ms / 50.3 GB for DeiT-T.
    \item At 197 tokens, LiSA latency is already 13.53 ms, versus 7.15 ms for FLatten and 3.50 ms for DeiT-T.
  \end{itemize}
  \papersource{Paper, Fig. 5, PDF p. 8; exact values: Supp. Table 13, p. 13. GPU and batch size are not stated there.}
\end{frame}

\begin{frame}{What the evidence supports}
  \begin{itemize}
    \item LiSA adds higher-order, transitive interactions to kernelized linear attention through a stochastic diffusion expansion.
    \item Low-rank algebra avoids an \(n\times n\) matrix; with short fixed \(T\), sequence scaling remains linear.
    \item The strongest reported benefits are on PVT-T, high-resolution DeiT-T, and PVT-T dense prediction.
    \item There is a real cost trade-off: more latency and memory than FLatten; larger diffusion depth can oversmooth; Swin ADE20K mAcc is lower than FLatten.
  \end{itemize}
  \begin{block}{Take-home}
    LiSA is a stronger linear-attention operator when a task benefits from multi-hop context and can absorb the added constant cost.
  \end{block}
  \papersource{Evidence: Tables 1--7, PDF pp. 6--8; Supp. Secs. 9--10 and Table 13, PDF pp. 12--13.}
\end{frame}

\begin{frame}{Paper status and links}
  \begin{itemize}
    \item \textbf{Paper:} \href{https://arxiv.org/abs/2611.04216v2}{arXiv:2611.04216v2}, revised 4 November 2026.
    \item \textbf{Venue:} Accepted at WACV 2027 based on author project and camera-ready materials; no official proceedings entry was found. See the \href{https://wacv.thecvf.com/Conferences/2027/AuthorGuides}{WACV 2027 author guide}.
    \item \textbf{Official code:} \href{https://github.com/Sahardastani/lisa}{author repository}; implementation marked “Code Coming Soon”.
  \end{itemize}
  \papersource{Status checked 9 November 2026. The arXiv version remains the source for all method and result claims in this deck.}
\end{frame}

\end{document}