% Research group presentation; compile with pdflatex and shell escape disabled.
\documentclass[aspectratio=169,11pt]{beamer}
\usepackage[T1]{fontenc}
\usepackage[utf8]{inputenc}
\usepackage{amsmath,amssymb,graphicx,booktabs}
\definecolor{AcademicBlue}{RGB}{26,57,86}
\definecolor{AcademicTeal}{RGB}{0,105,112}
\setbeamercolor{structure}{fg=AcademicBlue}
\setbeamercolor{frametitle}{fg=AcademicBlue,bg=white}
\setbeamercolor{title}{fg=AcademicBlue}
\setbeamercolor{alerted text}{fg=AcademicTeal}
\setbeamercolor{block title}{fg=white,bg=AcademicBlue}
\setbeamercolor{block body}{fg=black,bg=AcademicBlue!5}
\setbeamertemplate{navigation symbols}{}
\setbeamertemplate{itemize items}[circle]
\setbeamertemplate{footline}{%
  \hfill\usebeamerfont{page number in head/foot}%
  \insertframenumber\,/\,\inserttotalframenumber\hspace{4mm}\vspace{3mm}}
\setbeamersize{text margin left=8mm,text margin right=8mm}
\setbeamerfont{frametitle}{size=\Large,series=\bfseries}
\setbeamerfont{title}{size=\LARGE,series=\bfseries}
\newcommand{\papersource}[1]{\par\smallskip{\scriptsize\color{gray}#1\par}}
\newcommand{\pp}{\,\mathrm{pp}}
\title[Context length hurts]{Context Length Alone Hurts LLM Performance Despite Perfect Retrieval}
\subtitle{Reading of arXiv:2510.05381v1}
\author{Yufeng Du, Minyang Tian, Srikanth Ronanki, Subendhu Rongali,\\
Sravan Bodapati, Aram Galstyan, Azton Wells, Roy Schwartz,\\
Eliu A Huerta, Hao Peng}
\date{Research group discussion \quad 3 November 2026}
\begin{document}

\begin{frame}
  \titlepage
  \papersource{\href{https://arxiv.org/abs/2510.05381v1}{arXiv:2510.05381v1}, 18-page PDF. Page references in this talk use that PDF.}
\end{frame}

\begin{frame}{One sentence: retrieval is not the whole task}
  \begin{block}{Takeaway}
    A model can recite the relevant evidence from a long input yet solve the same task much less accurately than with a short input.
  \end{block}
  \begin{itemize}
    \item Controlled tests vary input length, filler content, evidence position, and attention visibility.
    \item A two-call retrieve-then-solve workflow improves selected GSM8K and RULER results.
  \end{itemize}
  \vfill
  \small Published in \href{https://aclanthology.org/2025.findings-emnlp.1264/}{Findings of EMNLP 2025}.
  Public implementation not found as of 3 November 2026.
  \papersource{Paper, Abstract and Secs. 3--5, PDF pp. 1, 4--9; \href{https://aclanthology.org/2025.findings-emnlp.1264/}{ACL Anthology}.}
\end{frame}

\begin{frame}{The question: does perfect recall imply effective use?}
  \begin{columns}[T]
    \begin{column}{0.48\linewidth}
      \begin{block}{Common decomposition}
        Find evidence in the input \(\longrightarrow\) use it to answer.
      \end{block}
      Retrieval-only tests measure the first step.
    \end{column}
    \begin{column}{0.48\linewidth}
      \begin{block}{This paper's test}
        Hold the underlying problem fixed. Increase surrounding input. Measure recall and solution accuracy separately.
      \end{block}
    \end{column}
  \end{columns}
  \vfill
  \alert{Research question:} can length hurt solving even when evidence remains retrievable?
  \papersource{Paper, Secs. 1 and 3.1, PDF pp. 2, 4--5. Retrieval here means reciting context, not external search.}
\end{frame}

\begin{frame}{A motivating failure: the conditions are present}
  \begin{block}{Short task}
    \(x_1=100,\quad x_2=x_1,\quad x_3=x_2+46\).
    Correct answer: \(x_3=146\).
  \end{block}
  \centering
  \includegraphics[width=0.96\linewidth,height=0.46\textheight,keepaspectratio]{figures/fig-01-failure-crop.png}
  \par\medskip
  \raggedright After about 25K spaces, the model states \(x_2=100\), then says \(x_2\) is unknown.
  \papersource{Crop of original paper Fig. 1, PDF p. 2. One illustrative case; aggregate evidence follows.}
\end{frame}

\begin{frame}{The benchmark inserts length into an unchanged problem}
  \[
    \underbrace{E}_{\text{evidence}}+
    \underbrace{D_L}_{\text{filler of length }L}+
    \underbrace{Q}_{\text{question}}
  \]
  \begin{itemize}
    \item Evidence is one contiguous span, initially at the beginning.
    \item Question is at the end; the short task supplies the \(L=0\) baseline.
    \item Filler progresses from essays to whitespace to masked positions.
  \end{itemize}
  \centering
  \includegraphics[width=0.79\linewidth,height=0.34\textheight,keepaspectratio]{figures/fig-02-design.png}
  \papersource{Paper, Sec. 3.1, Table 1 and original Fig. 2, PDF pp. 4--5.}
\end{frame}

\begin{frame}{Four tasks probe different uses of evidence}
  \centering
  \begin{tabular}{@{}lll@{}}
    \toprule
    Task & Evidence \(E\) & Question \(Q\) \\
    \midrule
    VarSum & Values of 50 integers & Sum three named variables \\
    GSM8K & Problem and reasoning steps & Math question \\
    MMLU & Problem description & Question and four options \\
    HumanEval & Function and docstring & Coding instruction \\
    \bottomrule
  \end{tabular}
  \vspace{0.8em}
  \begin{block}{Design intent}
    Make retrieval relatively easy so solution accuracy is visible as a separate bottleneck.
  \end{block}
  \papersource{Paper, Sec. 3.1 and Table 1, PDF p. 4; prompt examples, PDF pp. 13--14.}
\end{frame}

\begin{frame}{Exact-match recall and accuracy are separate tests}
  \[
    R_L=\frac{\#\text{ exact recitations of }(E,Q)}{N}\times100\%,
    \qquad
    A_L=\frac{\#\text{ correct answers}}{N}\times100\%.
  \]
  \[
    \Delta A_L=A_L-A_0,\qquad \Delta R_L=R_L-R_0
    \quad \text{(percentage points).}
  \]
  \begin{itemize}
    \item Any mismatch in the recitation scores zero for that retrieval test.
    \item The answer prompt does not require recitation before solving.
    \item These are two task-level rates, not a joint per-answer conditional rate.
  \end{itemize}
  \papersource{Teaching notation for Paper, Sec. 3.1, PDF pp. 4--5; appendix Table 6, PDF p. 16.}
\end{frame}

\begin{frame}{The controls separate candidate explanations}
  \centering
  \begin{tabular}{@{}lcl@{}}
    \toprule
    Condition & Input pattern & Main factor tested \\
    \midrule
    Essay filler & \(E\;D_L\;Q\) & Recall versus answer \\
    Whitespace & \(E\;D_L\;Q\) & Semantic distraction \\
    Adjacent evidence & \(D_L\;E\;Q\) & Evidence--question distance \\
    Masked filler & \(E\;[\mathrm{mask}]_L\;Q\) & Attention to filler content \\
    \bottomrule
  \end{tabular}
  \par\vspace{0.8em}
  Masking still shifts positions; adjacency still shifts the absolute position of \(E,Q\).
  \papersource{Paper, Secs. 3--4, original Figs. 2 and 4 and Table 3, PDF pp. 4--7.}
\end{frame}

\begin{frame}{Essay filler: accuracy falls faster than recall}
  \centering
  \includegraphics[width=0.95\linewidth,height=0.69\textheight,keepaspectratio]{figures/fig-03-essay.png}
  \papersource{Original paper Fig. 3, PDF p. 6. Blue: answer accuracy; orange dashed: exact-match retrieval. Horizontal axis is total input tokens.}
\end{frame}

\begin{frame}{At 30K, the recall--answer gap is substantial}
  \centering
  \begin{tabular}{@{}lccc@{}}
    \toprule
    Model / task & \(A_0 \to A_{30K}\) & \(R_0 \to R_{30K}\) & \(\Delta A\) \\
    \midrule
    Llama / VarSum & \(96.0\to11.0\%\) & \(100.0\to83.0\%\) & \(-85.0\pp\) \\
    Llama / MMLU & \(63.2\to39.0\%\) & \(97.0\to97.0\%\) & \(-24.2\pp\) \\
    Mistral / HumanEval & \(34.8\to0.0\%\) & \(81.5\to91.0\%\) & \(-34.8\pp\) \\
    \bottomrule
  \end{tabular}
  \vspace{0.9em}
  \begin{block}{Cleanest contrast}
    Llama/MMLU keeps 97\% exact-match retrieval while losing 24.2 percentage points of accuracy.
  \end{block}
  \papersource{Paper, appendix Table 6, PDF p. 16. 30K entries computed from each 0-token baseline plus the reported delta.}
\end{frame}

\begin{frame}{Whitespace reduces distraction, not the effect}
  \centering
  \includegraphics[width=0.93\linewidth,height=0.62\textheight,keepaspectratio]{figures/fig-04-whitespace.png}
  \vspace{0.2em}
  \begin{itemize}
    \item Left: \(E\), whitespace, \(Q\). At 30K, Llama/VarSum: \(96\%\to48\%\).
    \item Right: whitespace, \(E\), \(Q\). At 30K, Llama/VarSum: \(97\%\to77\%\).
  \end{itemize}
  \papersource{Original paper Fig. 4, PDF p. 7; appendix Tables 7--8, PDF p. 17. Horizontal axis counts inserted whitespace tokens.}
\end{frame}

\begin{frame}{Closed models vary, including clear exceptions}
  \centering
  \begin{tabular}{@{}lccc@{}}
    \toprule
    Model / task & 0 filler & 30K filler & Change \\
    \midrule
    GPT-4o / VarSum & \(100.0\%\) & \(100.0\%\) & \(0.0\pp\) \\
    GPT-4o / GSM8K & \(87.8\%\) & \(80.8\%\) & \(-7.0\pp\) \\
    Claude row / MMLU & \(82.2\%\) & \(14.6\%\) & \(-67.6\pp\) \\
    Gemini-2.0 / GSM8K & \(83.2\%\) & \(89.4\%\) & \(+6.2\pp\) \\
    \bottomrule
  \end{tabular}
  \par\vspace{0.8em}
  \raggedright No exact-match retrieval results are reported for these three models. The paper prose says Claude-3.7-Sonnet; Table 2 labels that row Claude-3.5.
  \papersource{Paper, Sec. 4.1 and Table 2, PDF p. 6; Limitations, PDF p. 9. Changes are percentage points.}
\end{frame}

\begin{frame}{Attention masking removes filler visibility}
  \centering
  \begin{tabular}{@{}lccc@{}}
    \toprule
    Model / task & 0 masked & 30K masked & Change \\
    \midrule
    Llama / HumanEval & \(57.3\%\) & \(7.3\%\) & \(-50.0\pp\) \\
    Llama / MMLU & \(62.8\%\) & \(41.7\%\) & \(-21.1\pp\) \\
    Mistral / VarSum & \(66.0\%\) & \(32.0\%\) & \(-34.0\pp\) \\
    Mistral / HumanEval & \(34.8\%\) & \(26.9\%\) & \(-7.9\pp\) \\
    \bottomrule
  \end{tabular}
  \par\vspace{0.8em}
  \raggedright The model attends to evidence and question, while their positions are farther apart.
  \papersource{Paper, Sec. 4.2 and Table 3, PDF p. 7. 30K accuracies computed from baselines and deltas.}
\end{frame}

\begin{frame}{The controls support a length-related failure}
  \begin{enumerate}
    \item Essay filler: recall can remain high while accuracy drops.
    \item Whitespace: meaningful distractor content is not required.
    \item Adjacent evidence: a growing \(E\)-to-\(Q\) distance is not required.
    \item Masking: attending to filler content is not required.
  \end{enumerate}
  \vspace{0.4em}
  \begin{block}{Interpretation}
    The study isolates length/position-related degradation. It does not identify a unique internal mechanism.
  \end{block}
  \papersource{Paper, Secs. 3--4 and Discussion, PDF pp. 5--9. Interpretation follows the reported controls.}
\end{frame}

\begin{frame}{Retrieve then Solve makes the second call short}
  \centering
  \includegraphics[width=0.88\linewidth,height=0.61\textheight,keepaspectratio]{figures/fig-05-mitigation.png}
  \vspace{0.2em}
  \begin{itemize}
    \item First call: recite relevant evidence from the long input.
    \item New call: answer from that evidence plus the original question.
  \end{itemize}
  \papersource{Original paper Fig. 5 and Sec. 5, PDF p. 8; RULER prompt, PDF pp. 14--15.}
\end{frame}

\begin{frame}{On GSM8K, the two-call workflow recovers accuracy}
  \centering
  \begin{tabular}{@{}lccccc@{}}
    \toprule
    Inserted essay tokens & 0 & 3,750 & 7,500 & 15,000 & 26,250 \\
    \midrule
    Direct answer & 70.6 & 49.3 & 43.4 & 41.6 & 35.5 \\
    Retrieve then Solve & 76.2 & 71.4 & 66.7 & 69.1 & 66.7 \\
    \bottomrule
  \end{tabular}
  \vspace{0.8em}
  \begin{block}{At 26,250 inserted tokens}
    Mistral-v0.3-7B-Instruct gains \(31.2\) percentage points: \(35.5\%\to66.7\%\).
  \end{block}
  The method gains \(5.6\) points with no filler, so the entire long-input gain cannot be assigned solely to shortening.
  \papersource{Paper, Sec. 5 and Table 4, PDF p. 8. Scores are answer accuracy in percent.}
\end{frame}

\begin{frame}{On RULER, the largest reported gain is 4 points}
  \centering
  \begin{tabular}{@{}lccc@{}}
    \toprule
    GPT-4o task / input & Direct & Two-call & Gain \\
    \midrule
    QA2 / 32K & \(68.4\%\) & \(72.4\%\) & \(+4.0\pp\) \\
    QA2 / 8K & \(71.4\%\) & \(74.0\%\) & \(+2.6\pp\) \\
    QA1 / 4K & \(90.4\%\) & \(92.2\%\) & \(+1.8\pp\) \\
    QA1 / 128K & \(88.2\%\) & \(88.2\%\) & \(0.0\pp\) \\
    \bottomrule
  \end{tabular}
  \par\vspace{0.8em}
  \raggedright First-step retrieval must be accurate; no open-model RULER result is reported.
  \papersource{Paper, Sec. 5 and Table 5, PDF p. 9; Limitations, PDF p. 9.}
\end{frame}

\begin{frame}{Evidence, scope, and a useful next experiment}
  \begin{itemize}
    \item \textbf{Evidence:} accuracy can fall despite near-stable exact-match recall, sparse filler, or masked filler.
    \item \textbf{Protocol:} recall and answering use separate prompts; closed-model recall is not reported.
    \item \textbf{Method:} the two-call strategy depends on retrieval and adds a model call.
    \item \textbf{Next experiment:} report item-level answer accuracy conditional on verified retrieval under a paired protocol.
  \end{itemize}
  \vfill
  \alert{Practical evaluation:} measure evidence access and final task success at the target input length.
  \papersource{Paper, Secs. 3--7 and Limitations, PDF pp. 4--9. Next experiment is our proposed extension.}
\end{frame}

\begin{frame}{Reference and publication record}
  \small
  Yufeng Du, Minyang Tian, Srikanth Ronanki, Subendhu Rongali,
  Sravan Babu Bodapati, Aram Galstyan, Azton Wells, Roy Schwartz,
  Eliu A Huerta, and Hao Peng. 2025.
  \emph{Context Length Alone Hurts LLM Performance Despite Perfect Retrieval.}
  Findings of the Association for Computational Linguistics: EMNLP 2025,
  pp. 23281--23298.
  \par\medskip
  \href{https://aclanthology.org/2025.findings-emnlp.1264/}{Official ACL Anthology entry}
  \quad
  \href{https://doi.org/10.18653/v1/2025.findings-emnlp.1264}{DOI}
  \quad
  \href{https://arxiv.org/abs/2510.05381v1}{Read arXiv v1}
  \par\medskip
  Publication verified on 3 November 2026. Public implementation not found as of that date.
  \papersource{Paper identity: arXiv v1, PDF p. 1; publication: ACL Anthology official record.}
\end{frame}

\appendix
\begin{frame}{Backup: reported deltas are percentage points}
  \begin{itemize}
    \item Appendix Tables 6--9 give absolute accuracy at zero filler and differences thereafter.
    \item Llama/MMLU with essay filler: \(A_0=63.2\%\), \(\Delta A_{30K}=-24.2\pp\).
    \item Therefore \(A_{30K}=39.0\%\); the reported 24.2 is not a relative 24.2\% reduction.
    \item Essay plot: total input tokens. Whitespace and masking tables: inserted-token counts.
  \end{itemize}
  \papersource{Paper, original Fig. 3, PDF p. 6; appendix Tables 6--9, PDF pp. 16--17.}
\end{frame}

\begin{frame}{Backup: three discrepancies in arXiv v1}
  \begin{itemize}
    \item Table 2 labels the Claude row Claude-3.5; Sec. 4.1 prose says Claude-3.7-Sonnet.
    \item Sec. 4.1 attributes Gemini/GSM8K \(+8.6\pp\) to 30K. Table 2 gives \(+8.6\pp\) at 15K and \(+6.2\pp\) at 30K.
    \item Sec. 4.2 compares Llama/HumanEval masking \(-50.0\pp\) with whitespace \(-19.4\pp\); the corresponding 30K between-evidence-and-question whitespace row gives \(-31.7\pp\).
  \end{itemize}
  \papersource{Paper, Secs. 4.1--4.2 and Table 2, PDF pp. 6--7; appendix Tables 7--8, PDF p. 17. Numeric slides follow the tables.}
\end{frame}
\end{document}