% ============================================================
% arXiv / Meta build of "Sparse Delta Memory".
% Uses the Meta FAIR pre-print template (fairmeta.cls) and shares the body
% and appendix content with neurips_2026.tex through body.tex and appendix.tex.
% Set this file as the main document to build the arXiv version.
% ============================================================
% arXiv: force PDF output (paper uses PDF figures) — must precede \documentclass.
\pdfoutput=1
\PassOptionsToPackage{dvipsnames}{xcolor} % so fairmeta's xcolor knows ForestGreen (used in macros)
\documentclass{fairmeta}

% --- Encoding ---
\usepackage[utf8]{inputenc}

% --- Math (not provided by fairmeta) ---
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{amsfonts}
\usepackage{mathtools}
\usepackage{bbm}
\usepackage{mathrsfs}
\usepackage{nicefrac}

% --- Floats, figures, algorithms ---
\usepackage{float}
\usepackage{wrapfig}
\usepackage{algorithm}
\usepackage[noend]{algpseudocode}

% --- TikZ (architecture figure in body.tex) ---
\usepackage{tikz}
\usetikzlibrary{positioning, arrows.meta, calc, decorations.pathreplacing, shapes.geometric}

% --- Misc ---
\usepackage{enumitem}
\usepackage{pifont}
\usepackage{xspace}
\usepackage{comment} % lets the \begin{comment} block inside macros.tex be skipped

\usepackage{palatino}
% palatino remaps \ttdefault to Courier (pcr), which renders \url in a heavy,
% wide monospace. Use the roman text font for URLs so they match the body text.
\urlstyle{rm}

% Float placement: mirror neurips_2026.sty so tables/figures sit near their
% references instead of being deferred to the end of the document
% (fairmeta inherits article's restrictive float defaults).
\renewcommand{\topfraction}{0.85}
\renewcommand{\bottomfraction}{0.4}
\renewcommand{\textfraction}{0.1}
\renewcommand{\floatpagefraction}{0.7}

% Shared math macros (\subg, \subr, \sz, ... used in the result tables)
\input{macros}

% Review/editing comment macros (kept in sync with neurips_2026.tex)
\newcommand{\rv}[1]{{\textcolor{blue}{[H: #1]}}}
\newcommand{\lc}[1]{{\textcolor{orange}{[L: #1]}}}
\newcommand{\mysp}{\phantom{0}}
% Eval-table (Table 2) citations: year-only so the first column doesn't wrap
% under author-year citations (full reference remains in the bibliography).
\newcommand{\evalcite}[1]{\citeyearpar{#1}}

% ============================================================
% Title / authors (fairmeta template)
% ============================================================
\title{Sparse Delta Memory: Scaling the State of Linear RNNs through Sparsity}
\subtitle{}

\author{Loïc Cabannes$^{1, 2}$}
\author{Pierre-Emmanuel Mazar\'e$^1$}
\author{Gergely Szilvasy$^1$}
\author{Matthijs Douze$^1$}
\author{Maria Lomeli$^1$}
\author{Ilze Amanda Auzina$^{1, 3}$}
\author{Justin Carpentier$^2$}
\author{Gabriel Synnaeve$^1$}
\author{Herv\'e J\'egou$^1$}

\affiliation{$^1$Meta FAIR}
\affiliation{$^2$Inria Paris \& ENS-PSL University}
\affiliation{$^3$University of Tübingen}

\date{\today}

\abstract{
Linear attention models allow a fixed state size and a fixed amount of compute per token. However, due to their limited state size, linear attention models fall behind in long-context recall compared to softmax-attention-based transformer architectures. Increasing the state size of linear attention improves recall performance but at the cost of higher FLOPs. In this work, we introduce \textbf{Sparse Delta Memory\,(SDM)}, an architecture that scales the hidden state of gated linear RNNs to orders of magnitude higher capacity using a sparse addressing scheme. SDM extends the Gated DeltaNet architecture by replacing the dense key-value outer product with sparse reads and writes to a large explicit memory.
We show that, under an isoFLOP constraint and with an identical number of parameters, a higher state memory capacity significantly improves performance on in-context learning and long-context retrieval tasks. Moreover, by learning the initial state of the SDM memory and therefore using it as a parametric memory, we show that the model further improves on a wide range of common-knowledge and reasoning tasks.
}

\metadata[Code]{\url{https://github.com/facebookresearch/sparse-delta-memory}}

\begin{document}

\maketitle

\input{body}

\bibliography{references}
\bibliographystyle{references}

\appendix

\input{appendix}

\end{document}