slideset_128_vcpus.tex

\documentclass{beamer}

\usepackage{tikz}
\usetikzlibrary{calc,shadows}
\usepackage{listings}
\usepackage{graphicx}
\usepackage{fontspec}



% ── Poppins font (Vates official typeface) ─────────────────────────
\setsansfont{Poppins}[
  Path           = assets/fonts/,
  Extension      = .ttf,
  UprightFont    = *-Regular,
  BoldFont       = *-Bold,
  ItalicFont     = *-Italic,
  BoldItalicFont = *-BoldItalic
]
\renewcommand{\familydefault}{\sfdefault}

% ── Vates brand colors ─────────────────────────────────────────────
\definecolor{VatesBlue}{HTML}{1A1B38}
\definecolor{VatesRed}{HTML}{BE1621}
\definecolor{VatesBeige}{HTML}{FFFCE4}

\lstset{numbers=left,
  basicstyle=\tiny\ttfamily,
  keywordstyle=\color{VatesBlue},
  breaklines=true,
  showtabs=false,
  showstringspaces=false,
  numbers=none,
}

% ── Theme base ────────────────────────────────────────────────────
\usetheme{CambridgeUS}

\setbeamercolor{palette primary}{fg=white,bg=VatesRed}
\setbeamercolor{palette secondary}{fg=white,bg=VatesBlue}
\setbeamercolor{palette tertiary}{fg=white,bg=VatesBlue}
\setbeamercolor{palette quaternary}{fg=white,bg=VatesBlue}

% Canvas and body text
\setbeamercolor{normal text}{fg=VatesBlue,bg=white}
\setbeamercolor{background canvas}{bg=white}
\setbeamercolor{structure}{fg=VatesRed}

% Title page elements
\setbeamercolor{title}{fg=VatesBlue}
\setbeamercolor{subtitle}{fg=VatesBlue}
\setbeamercolor{author}{fg=VatesBlue}
\setbeamercolor{date}{fg=VatesBlue}
\setbeamercolor{institute}{fg=VatesBlue}

% Frame title (same color as background so planet shows through)
\setbeamercolor{frametitle}{fg=white,bg=VatesBlue}
\setbeamercolor{framesubtitle}{fg=VatesBlue}

% Blocks
\setbeamercolor{block title}{fg=white,bg=VatesRed}
\setbeamercolor{block body}{fg=white,bg=VatesBlue}

% Lists
\setbeamercolor{item}{fg=VatesRed}
\setbeamercolor{subitem}{fg=VatesBeige}
\setbeamercolor{subsubitem}{fg=VatesBeige}

% TOC
\setbeamercolor{section in toc}{fg=white}
\setbeamercolor{subsection in toc}{fg=VatesBeige}

% Alerts
\setbeamercolor{alerted text}{fg=VatesRed}

% ── Overview slide helpers ─────────────────────────────────────────
\newcommand{\ovdone}[1]{\textcolor{gray}{#1}}
\newcommand{\ovcurrent}[1]{\textcolor{VatesRed}{\textbf{#1}}}

% No navigation clutter
\setbeamertemplate{navigation symbols}{}

% Remove top headline bar
\setbeamertemplate{headline}{}
% ── Footer bar ────────────────────────────────────────────────────
% Edit \footlinetext to customise the footer text
\newcommand{\footlinetext}{Scaling Xen HVM Guests Beyond 128 vCPUs}

\setbeamercolor{footline}{fg=white,bg=VatesBlue}
\setbeamertemplate{footline}{%
  \begin{beamercolorbox}[wd=\paperwidth,ht=0.45cm,dp=0.2cm,leftskip=0.4cm,rightskip=0.4cm]{footline}%
    \usebeamerfont{footline}\footlinetext\hfill\insertframenumber\,/\,\inserttotalframenumber%
  \end{beamercolorbox}%
}

% ── Background: white canvas + planet (top-right corner) ─────────
\setbeamertemplate{background}{%
  \begin{tikzpicture}
    \useasboundingbox (0,0) rectangle (\paperwidth,\paperheight);
  \end{tikzpicture}%
}

% ── Frametitle: blue band with stars + XCP-ng logo ────────────────
\setbeamertemplate{frametitle}{%
  \nointerlineskip%
  \begin{beamercolorbox}[wd=\paperwidth,ht=0.8cm,dp=0.25cm]{frametitle}%
    \tikz[overlay,baseline=0pt]{%
      % Stars scattered across the band
      \fill[white,opacity=0.70] (4.5cm, 0.60cm) circle (0.025cm);
      \fill[white,opacity=0.55] (5.8cm, 0.18cm) circle (0.020cm);
      \fill[white,opacity=0.80] (7.0cm, 0.72cm) circle (0.028cm);
      \fill[white,opacity=0.65] (8.4cm, 0.35cm) circle (0.022cm);
      \fill[white,opacity=0.90] (6.3cm, 0.78cm) circle (0.018cm);
      \fill[white,opacity=0.60] (9.6cm, 0.55cm) circle (0.025cm);
      \fill[white,opacity=0.75] (10.8cm,0.22cm) circle (0.020cm);
    }%
    \hspace{0.5cm}%
    \usebeamerfont{frametitle}\insertframetitle\strut%
  \end{beamercolorbox}%
}

% ── Title page: rounded blue title box with shadow and stars ───────
\setbeamertemplate{title page}{%
  \vbox{}%
  \vfill%
  \begin{center}
  \begin{tikzpicture}
    \node[
      fill=VatesBlue,
      rounded corners=8pt,
      drop shadow={fill=black!50, opacity=0.4, shadow xshift=3pt, shadow yshift=-4pt},
      text width=\dimexpr\paperwidth-2.5cm\relax,
      inner xsep=0.6cm,
      inner ysep=0.6cm,
      align=left
    ] (box) {%
      \usebeamerfont{title}\center\fontsize{16}{20}\selectfont\color{white}\inserttitle%
      \ifx\insertsubtitle\@empty\else%
        \par\vskip0.3em%
        {\usebeamerfont{subtitle}\color{white!80!VatesBlue}\insertsubtitle}%
      \fi%
    };
    % Stars inside the box
    \fill[white,opacity=0.70] ($(box.north west)+(2.0cm,-0.40cm)$) circle (0.025cm);
    \fill[white,opacity=0.55] ($(box.north west)+(4.0cm,-0.80cm)$) circle (0.020cm);
    \fill[white,opacity=0.80] ($(box.north east)+(-2.5cm,-0.30cm)$) circle (0.028cm);
    \fill[white,opacity=0.65] ($(box.north east)+(-1.5cm,-0.70cm)$) circle (0.022cm);
    \fill[white,opacity=0.90] ($(box.south east)+(-3.0cm, 0.45cm)$) circle (0.018cm);
    \fill[white,opacity=0.60] ($(box.south west)+(5.0cm,  0.50cm)$) circle (0.025cm);
    \fill[white,opacity=0.75] ($(box.south east)+(-1.5cm, 0.60cm)$) circle (0.020cm);
    \fill[white,opacity=0.65] ($(box.center)   +(3.0cm,  0.30cm)$) circle (0.022cm);
    \fill[white,opacity=0.50] ($(box.north west)+(6.0cm,-0.50cm)$) circle (0.018cm);
  \end{tikzpicture}
  \end{center}
  \vskip0.8em%
  \begin{beamercolorbox}[wd=\textwidth]{author}%
    \center\usebeamerfont{author}\insertauthor%
  \end{beamercolorbox}%
  \vskip0.2em%
  \begin{beamercolorbox}[wd=\textwidth]{date}%
    \center\usebeamerfont{date}\insertdate%
  \end{beamercolorbox}%
  \vfill%
}

% ── Logo: Vates red logo on every slide ───────────────────────────
\logo{\includegraphics[height=0.8cm]{assets/png-vates-fulllogo-red.png}}

% ── Title page details ────────────────────────────────────────────
\title{Scaling Xen HVM Guests\\Beyond 128 vCPUs}
\author{Julian Vetter}
\date{02.04.2026}

\begin{document}

\begin{frame}
    \titlepage
    \center Xen Summit - Grenoble
\end{frame}

\begin{frame}{Overview}
How can we achieve...
\center \includegraphics[width=.8\textwidth]{assets/toystory_meme}
\end{frame}

\begin{frame}{Overview}
What needs to be changed
\begin{itemize}
\item \ovcurrent{I/O request}
\item x2APIC Support
\begin{itemize}
    \item APIC \& vLAPIC
    \item MADT \& Boot sequence
    \item CPU topology
    \item MSI/RTE
\end{itemize}
\item More???
\end{itemize}
\end{frame}

\begin{frame}{I/O request}
    \center \includegraphics[width=.6\textwidth]{assets/ioreq_server_paper}
\end{frame}

\begin{frame}[fragile]{I/O request: 2 types of I/O requests}
\textbf{bufioreq} (buffered/asynchronous):
\begin{itemize}
    \item shared ring buffer (\texttt{buffered\_iopage\_t})
    \item 511 slots -> for small write-only PIO ops
    \item vCPU does not block
\end{itemize}
\textbf{ioreq} (synchronous): \textbf{one \texttt{ioreq\_t} slot per vCPU}
\begin{itemize}
    \item Guest vCPU traps on PIO/MMIO
    \item Xen writes request into the vCPU's slot
    \item Xen signals QEMU via event channel (vCPU blocks)
    \item QEMU handles the request, writes response back (vCPU resumes)
\end{itemize}
\end{frame}

\begin{frame}[fragile]{I/O request: The 128 vCPU Limit}
    \begin{itemize}
        \item One \texttt{ioreq\_t} slot per vCPU, all packed into a \textbf{single domain-heap page}
        \begin{lstlisting}[language=C]
/* public/hvm/ioreq.h */
struct shared_iopage {
    struct ioreq vcpu_ioreq[1];  /* variable-length, one per vCPU */
};
        \end{lstlisting}
        \item \texttt{sizeof(ioreq\_t)} = 32 bytes
        \item \texttt{PAGE\_SIZE / sizeof(ioreq\_t)} = 4096 / 32 = \textbf{128 slots}
        \item[] \textbf{$\Rightarrow$ Hard ceiling of 128 vCPUs per ioreq server}
        \item Allocation maps exactly one page
        \item Currently \textit{no} mechanism to map multiple pages
    \end{itemize}
\end{frame}

\begin{frame}{Overview}
What needs to be changed
\begin{itemize}
\item \ovdone{I/O request}
\item x2APIC Support
\begin{itemize}
    \item \ovcurrent{APIC \& vLAPIC}
    \item MADT \& Boot sequence
    \item CPU topology
    \item MSI/RTE
\end{itemize}
\item More???
\end{itemize}
\end{frame}

\begin{frame}{8 Bit IDs ought to be enough for anybody!}
\vfill
\begin{center}
\begin{tikzpicture}
    \node[
        text width=0.80\textwidth,
        inner xsep=0.8cm,
        inner ysep=0.6cm,
        align=center
    ] (box) {
        {\fontsize{36}{36}\selectfont\color{VatesRed}\textbf{``}}\\[-0.6em]
        {\large\itshape 640K ought to be enough for anybody!}\\[0.4em]
        {\fontsize{36}{36}\selectfont\color{VatesRed}\textbf{''}}\\[0.4em]
        {\small\textcolor{gray}{--- Bill Gates, 1981 (attributed)}}
    };
\end{tikzpicture}
\end{center}
\vfill
\end{frame}

\begin{frame}[fragile]{APIC and vLAPIC}
\textbf{Physical Local APIC (LAPIC)}
    \begin{itemize}
        \item One LAPIC per CPU: local interrupt delivery, IPIs, APIC timer
        \item Interrupt routing: hardware matches destination APIC ID
        \item Two modes:
        \begin{itemize}
            \item \textbf{xAPIC}: MMIO
            \item \textbf{x2APIC}: MSR-based
        \end{itemize}
    \end{itemize}
    \textbf{Xen vLAPIC}
    \begin{itemize}
        \item Guest xAPIC MMIO $\rightarrow$ Xen intercepts \& emulates
        \item Guest x2APIC MSR $\rightarrow$ \texttt{guest\_wrmsr\_x2apic()} / \texttt{guest\_rdmsr\_x2apic()}
    \end{itemize}
\end{frame}

\begin{frame}{x2APIC}
    \begin{itemize}
        \item APIC ID register is what the HW matches against for IRQ routing!
        \item APIC supports 8 Bit APIC IDs
        \item[] $\rightarrow$ \textbf{8Bit == 256 IDs $\rightarrow$ Enough for up to 256 vCPUs, right?!}
        \item No.
        \item \footnotesize{\texttt{vlapic\_set\_reg(vlapic, APIC\_ID, (v->vcpu\_id * 2) << 24);}}
        \item[] $\rightarrow$ \textbf{We need x2APIC support!}
    \end{itemize}
    \center
    \includegraphics[width=0.5\textwidth]{assets/apic_id_register}
\end{frame}

\begin{frame}{Overview}
What needs to be changed
\begin{itemize}
\item \ovdone{I/O request}
\item x2APIC Support
\begin{itemize}
    \item \ovdone{APIC \& vLAPIC}
    \item \ovcurrent{MADT \& Boot sequence}
    \item CPU topology
    \item MSI/RTE
\end{itemize}
\item More???
\end{itemize}
\end{frame}

\begin{frame}{Boot and CPU Enumeration}
\begin{enumerate}
\item[1] \textbf{BSP (Bootstrap Processor) starts up (APIC ID 0)}
    \begin{itemize}
        \item BSP executes the BIOS/UEFI and boot OS
        \item Other CPUs (APs) remain in halted/wait-for-SIPI
        \item BSP always APIC ID 0, so 8-bit xAPIC limitation isn't a problem for initial boot CPU
    \end{itemize}
\end{enumerate}
\end{frame}

\begin{frame}{Boot and CPU Enumeration}
    \begin{enumerate}
        \item[2] \textbf{Topology discovery via ACPI}
        \begin{itemize}
            \item BSP reads ACPI MADT (Multiple APIC Description Table), which has two relevant entry types:
            \begin{itemize}
                \item \textcolor{VatesRed}{Type 0} (APIC): 8-bit APIC ID
                \item \textcolor{VatesRed}{Type 9} (x2APIC): 32-bit APIC ID
            \end{itemize}
        \end{itemize}
        \begin{itemize}
            \item If any CPU has APIC ID $\geq$ 255 $\rightarrow$ Type 9 entry $\rightarrow$ Signal to the OS that x2APIC mode is required
            \item In virtualized environment $\rightarrow$ Obligation of XEN to set up registers in the proper way and craft ACPI MADT
            \item[] $\rightarrow$ \textbf{ACPI 4.0 x2APIC support missing in XEN}
        \end{itemize}
    \end{enumerate}
    \center \includegraphics[width=.4\textwidth]{assets/rsdp_table}
\end{frame}

\begin{frame}{Boot and CPU and Enumeration}
\begin{enumerate}
    \item[3] \textbf{BSP switches to x2APIC before waking APs}

    \begin{itemize}
        \item \textbf{Critical step:} Before sending INIT-SIPI-SIPI to APs with APIC ID $>$ 255, BSP must switch to x2APIC mode first, because in xAPIC mode, the ICR destination field is only 8 bits
        \item  In x2APIC mode, ICR is accessed via MSRs and has full 32-bit destination field.
    \end{itemize}
\end{enumerate}
\end{frame}

\begin{frame}{Boot and CPU and Enumeration}
\begin{enumerate}
    \item[4] \textbf{APs wake up and switch to x2APIC themselves}
    \begin{itemize}
        \item AP wakes at SIPI vector in xAPIC mode. If APIC ID $>$ 255, the 8-bit xAPIC ID register contains a truncated/invalid value
        \item The startup trampoline does \textbf{not} read the ID yet; it unconditionally sets the \texttt{EXTD} bit in \texttt{IA32\_APIC\_BASE} to switch to x2APIC mode
        \item Only after the switch does the AP read its real 32-bit ID
        \item Same trampoline binary runs on every AP regardless of ID
    \end{itemize}
\end{enumerate}
\end{frame}

\begin{frame}{Boot and CPU and Enumeration}
    \center
    \includegraphics[width=.6\textwidth]{assets/apic_icr_register}
\end{frame}

\begin{frame}{Boot and CPU and Enumeration}
    \center
    \includegraphics[width=.6\textwidth]{assets/x2apic_icr_register}
\end{frame}

\begin{frame}{Overview}
What needs to be changed
\begin{itemize}
\item \ovdone{I/O request}
\item x2APIC Support
\begin{itemize}
    \item \ovdone{APIC \& vLAPIC}
    \item \ovdone{MADT \& Boot sequence}
    \item \ovcurrent{CPU topology}
    \item MSI/RTE
\end{itemize}
\item More???
\end{itemize}
\end{frame}

\begin{frame}{CPU Topology}
    \begin{itemize}
        \item Xen assigns IDs as \texttt{vcpu\_id * 2}, so with $>$128 vCPUs, IDs exceed 255.
        \item Each AP reads its 32-bit x2APIC ID via \texttt{RDMSR(0x802)} after switching mode. But firmware must first \emph{enumerate} those CPUs before it can boot them.
        \item Firmware (OVMF/EDK2) uses CPUID leaf 0xB for this enumeration in x2APIC mode
        \item EBX=0 signals ``leaf unsupported'' $\rightarrow$ firmware falls back to CPUID leaf 0x1, which only has an 8-bit ID field $\rightarrow$ CPUs with ID $>$ 255 are invisible to firmware.
    \end{itemize}
\end{frame}

\begin{frame}{Boot and CPU and Enumeration}
    \center
    \includegraphics[width=.6\textwidth]{assets/cpuid_enumeration}
\end{frame}

\begin{frame}{Boot and CPU and Enumeration}
    \center
    \includegraphics[width=.6\textwidth]{assets/cpuid_discovery_0x1}
\end{frame}

\begin{frame}{Boot and CPU and Enumeration}
    \center
    \includegraphics[width=.6\textwidth]{assets/cpuid_discovery_0xb}
\end{frame}

\begin{frame}{Overview}
What needs to be changed
\begin{itemize}
\item \ovdone{I/O request}
\item x2APIC Support
\begin{itemize}
    \item \ovdone{APIC \& vLAPIC}
    \item \ovdone{MADT \& Boot sequence}
    \item \ovdone{CPU topology}
    \item \ovcurrent{MSI/RTE}
\end{itemize}
\item More???
\end{itemize}
\end{frame}

\begin{frame}{MSI / RTE — Mechanism \& Problem}
\begin{columns}[t]
    \column{.5\textwidth}
    \textbf{What is an MSI?}
    \begin{itemize}
        \item PCIe device triggers an interrupt by doing \textbf{DMA write} to a magic address in the \texttt{0xFEEx\_xxxx} range
        \item APIC sees this write and delivers interrupt to the CPU whose APIC ID is encoded in the address
        \item MSI address bits \texttt{[19:12]}: 8-bit destination APIC ID
        \item IO-APIC RTE: same concept for legacy interrupts, 8-bit dest in bits \texttt{[63:56]}
    \end{itemize}

    \column{.5\textwidth}
    \textbf{The Problem}
    \begin{itemize}
        \item Linux builds the MSI address from the target vCPU's APIC ID
        \item Linux detects it can't set interrupt affinity $\rightarrow$ marks vCPU as not interruptible $\rightarrow$ \textbf{refuses to bring it online}
        \item Same failure for IO-APIC RTEs
        \item[] $\Rightarrow$ Even if Xen creates 128+ vCPUs, the guest won't bring them up
    \end{itemize}
\end{columns}
\end{frame}

\begin{frame}{MSI / RTE — Solution}
\begin{columns}[t]
    \column{0.48\textwidth}
    \textbf{Extended destination ID}
    \begin{itemize}
        \item Xen announces \texttt{XEN\_HVM\_CPUID\_EXT\_DEST\_ID} via CPUID
        \item Linux sees the flag, switches to extended encoding
        \item Repurpose unused bits for 7 extra dest ID bits:
        \begin{itemize}
            \item MSI address bits \texttt{[11:5]}
            \item IO-APIC RTE bits \texttt{[55:49]}
        \end{itemize}
        \item \textbf{15-bit dest ID} $\rightarrow$ up to \textbf{32,768} APIC IDs addressable
    \end{itemize}

    \column{0.48\textwidth}
    \textbf{End-to-end flow}
    \begin{itemize}
        \item Linux programs MSI address with extended bits set
        \item Write intercepted by Xen's virtual PCI/MSI layer
        \item Xen's virtual IO-APIC and MSI decoder understand the extended encoding
        \item Interrupt delivered to correct vCPU
        \item Fully software-emulated / no physical hardware dependency
    \end{itemize}
\end{columns}
\end{frame}

\begin{frame}{MSI / RTE}
\begin{itemize}
    \item[] \footnotesize{\textit{Note: extended dest ID bits originate from Intel ICH2/ICH4 datasheets (2002) and were never incorporated into the PCIe MSI spec. All major chipsets carry the behaviour forward silently for compatibility. In Xen, the vIOAPIC and MSI handling are fully software-emulated, so this is independent of physical hardware.}}
\end{itemize}
\begin{columns}[c]
    \column{0.5\textwidth}\center
        \includegraphics[width=.8\linewidth]{assets/msi_ext_dest_ids}
    \column{0.5\textwidth}\center
        \includegraphics[width=.8\linewidth]{assets/linux_commit_dest_ids}
\end{columns}
\end{frame}

\begin{frame}{Overview}
What needs to be changed
\begin{itemize}
\item \ovdone{I/O request}
\item \ovdone{x2APIC Support}
\begin{itemize}
    \item \ovdone{APIC \& vLAPIC}
    \item \ovdone{MADT \& Boot sequence}
    \item \ovdone{CPU topology}
    \item \ovdone{MSI/RTE}
\end{itemize}
\item \ovcurrent{More???}
\end{itemize}
\end{frame}


\begin{frame}{More???}
    \begin{itemize}
        \item With the listed problems and according fixes I was able to boot a Linux guest with 130 vCPUs!
        \item I tried to start one with 180 vCPUs $\rightarrow$ failed. Guest crash somewhere with backtrace in XEN
        \item More work ahead!
    \end{itemize}
\end{frame}

\end{document}