Guard api/xml and api/json against OutOfMemoryError

core/src/main/java/hudson/model/Api.java

@@ -42,6 +42,7 @@
 import jenkins.model.Jenkins;
 import jenkins.security.SecureRequester;
 import jenkins.security.stapler.StaplerNotDispatchable;
+import jenkins.util.SystemProperties;
 import jenkins.util.xml.FilteredFunctionContext;
 import org.dom4j.CharacterData;
 import org.dom4j.Document;
@@ -108,6 +109,10 @@
                       @QueryParameter int depth) throws IOException, ServletException {
         setHeaders(rsp);
 
+        if (!checkHeapForApiResponse(rsp)) {
+            return;
+        }
+
         String[] excludes = req.getParameterValues("exclude");
 
         if (xpath == null && excludes == null) {
@@ -256,6 +261,9 @@
     private void doJsonImpl(StaplerRequest2 req, StaplerResponse2 rsp) throws IOException, ServletException {
         if (req.getParameter("jsonp") == null || permit(req)) {
             setHeaders(rsp);
+            if (!checkHeapForApiResponse(rsp)) {
+                return;
+            }
             rsp.serveExposedBean(req, bean, req.getParameter("jsonp") == null ? Flavor.JSON : Flavor.JSONP);
         } else {
             rsp.sendError(HttpURLConnection.HTTP_FORBIDDEN, "jsonp forbidden; implement jenkins.security.SecureRequester");
@@ -292,6 +300,9 @@
 
     private void doPythonImpl(StaplerRequest2 req, StaplerResponse2 rsp) throws IOException, ServletException {
         setHeaders(rsp);
+        if (!checkHeapForApiResponse(rsp)) {
+            return;
+        }
         rsp.serveExposedBean(req, bean, Flavor.PYTHON);
     }
 
@@ -304,6 +315,41 @@
         return false;
     }
 
+    /**
+     * Checks if there is sufficient heap memory before processing a potentially large API response.
+     * Large serialization operations (especially without the tree parameter) can cause OutOfMemoryError.
+     *
+     * @param rsp the response to send error to if heap is low
+     * @return true if it is safe to proceed, false if the request was rejected
+     * @see <a href="https://issues.jenkins.io/browse/JENKINS-75747">JENKINS-75747</a>
+     */
+    private boolean checkHeapForApiResponse(StaplerResponse2 rsp) throws IOException {
+        long minFreeBytes = SystemProperties.getLong(Api.class.getName() + ".minFreeMemoryBytes", 50L * 1024 * 1024);
+        if (minFreeBytes <= 0) {
+            return true;
+        }
+
+        Runtime rt = Runtime.getRuntime();
+        long maxMemory = rt.maxMemory();
+        long totalMemory = rt.totalMemory();
+        long freeMemory = rt.freeMemory();
+        long usedMemory = totalMemory - freeMemory;
+        long availableMemory = maxMemory - usedMemory;
+
+        if (availableMemory >= minFreeBytes) {
+            return true;
+        }
+
+        LOGGER.warning(() -> String.format(
+                "Rejecting API request due to low heap: available=%d, required=%d. Suggest using the tree parameter.",
+                availableMemory, minFreeBytes));
+
+        rsp.setStatus(HttpServletResponse.SC_SERVICE_UNAVAILABLE);
+        rsp.setContentType("text/plain;charset=UTF-8");
+        rsp.getWriter().print(Messages.Api_LowMemory());
+        return false;
+    }
+
     @Restricted(NoExternalUse.class)
     protected void setHeaders(StaplerResponse2 rsp) {
         rsp.setHeader("X-Jenkins", Jenkins.VERSION);

core/src/main/resources/hudson/model/Messages.properties

@@ -81,6 +81,7 @@ Api.MultipleMatch=XPath "{0}" matched {1} nodes. \
     Create XPath that only matches one, or use the "wrapper" query parameter to wrap them all under a root element.
 Api.NoXPathMatch=XPath {0} didn’t match
 Api.WrapperParamInvalid=The wrapper parameter can only contain alphanumeric characters or dash/dot/underscore. The first character has to be a letter or underscore.
+Api.LowMemory=Insufficient heap memory to process this API request. The server may run out of memory when serializing large responses. To avoid this, use the tree parameter to limit the data returned, e.g. tree=jobs[name],builds[number]. See the API documentation for details.
 
 BallColor.Aborted=Aborted
 BallColor.Disabled=Disabled

test/src/test/java/hudson/model/ApiTest.java

@@ -189,6 +189,36 @@ void escapedParameter() throws Exception {
                 page.getWebResponse().getContentAsString());
     }
 
+    @Test
+    @Issue("JENKINS-75747")
+    void lowHeapRejectsApiRequest() throws Exception {
+        // Force the OOM guard to reject by requiring more free memory than available
+        String prop = Api.class.getName() + ".minFreeMemoryBytes";
+        String original = System.getProperty(prop);
+        try {
+            System.setProperty(prop, String.valueOf(Long.MAX_VALUE));
+
+            JenkinsRule.WebClient wc = j.createWebClient();
+            wc.getOptions().setThrowExceptionOnFailingStatusCode(false);
+
+            Page xmlPage = wc.goTo("api/xml", null);
+            WebResponse xmlResponse = xmlPage.getWebResponse();
+            assertEquals(HttpServletResponse.SC_SERVICE_UNAVAILABLE, xmlResponse.getStatusCode());
+            assertThat(xmlResponse.getContentAsString(), containsString("tree parameter"));
+
+            Page jsonPage = wc.goTo("api/json", null);
+            WebResponse jsonResponse = jsonPage.getWebResponse();
+            assertEquals(HttpServletResponse.SC_SERVICE_UNAVAILABLE, jsonResponse.getStatusCode());
+            assertThat(jsonResponse.getContentAsString(), containsString("tree parameter"));
+        } finally {
+            if (original != null) {
+                System.setProperty(prop, original);
+            } else {
+                System.clearProperty(prop);
+            }
+        }
+    }
+
     @Test
     @Issue("SECURITY-1704")
     void project_notExposedToIFrame() throws Exception {