Guard api/xml and api/json against OutOfMemoryError

Add a heap check before processing potentially large API responses.
When free memory is below a configurable threshold (default 50MB),
the request is rejected with HTTP 503 and a message suggesting use
of the tree parameter to limit returned data.

- System property hudson.model.Api.minFreeMemoryBytes to configure
  or disable (set to 0)
- Applies to XML, JSON, and Python API endpoints
- Add ApiTest.lowHeapRejectsApiRequest to verify rejection behavior

JENKINS-75747

Made-with: Cursor

Author: sahilleth

core/src/main/java/hudson/model/Api.java

@@ -42,6 +42,7 @@
 import jenkins.model.Jenkins;
 import jenkins.security.SecureRequester;
 import jenkins.security.stapler.StaplerNotDispatchable;
+import jenkins.util.SystemProperties;
 import jenkins.util.xml.FilteredFunctionContext;
 import org.dom4j.CharacterData;
 import org.dom4j.Document;
@@ -108,6 +109,10 @@ public void doXml(StaplerRequest2 req, StaplerResponse2 rsp,
                       @QueryParameter int depth) throws IOException, ServletException {
         setHeaders(rsp);
 
+        if (!checkHeapForApiResponse(rsp)) {
+            return;
+        }
+
         String[] excludes = req.getParameterValues("exclude");
 
         if (xpath == null && excludes == null) {
@@ -256,6 +261,9 @@ public void doJson(StaplerRequest req, StaplerResponse rsp) throws IOException,
     private void doJsonImpl(StaplerRequest2 req, StaplerResponse2 rsp) throws IOException, ServletException {
         if (req.getParameter("jsonp") == null || permit(req)) {
             setHeaders(rsp);
+            if (!checkHeapForApiResponse(rsp)) {
+                return;
+            }
             rsp.serveExposedBean(req, bean, req.getParameter("jsonp") == null ? Flavor.JSON : Flavor.JSONP);
         } else {
             rsp.sendError(HttpURLConnection.HTTP_FORBIDDEN, "jsonp forbidden; implement jenkins.security.SecureRequester");
@@ -292,6 +300,9 @@ public void doPython(StaplerRequest req, StaplerResponse rsp) throws IOException
 
     private void doPythonImpl(StaplerRequest2 req, StaplerResponse2 rsp) throws IOException, ServletException {
         setHeaders(rsp);
+        if (!checkHeapForApiResponse(rsp)) {
+            return;
+        }
         rsp.serveExposedBean(req, bean, Flavor.PYTHON);
     }
 
@@ -304,6 +315,41 @@ private boolean permit(StaplerRequest2 req) {
         return false;
     }
 
+    /**
+     * Checks if there is sufficient heap memory before processing a potentially large API response.
+     * Large serialization operations (especially without the tree parameter) can cause OutOfMemoryError.
+     *
+     * @param rsp the response to send error to if heap is low
+     * @return true if it is safe to proceed, false if the request was rejected
+     * @see <a href="https://issues.jenkins.io/browse/JENKINS-75747">JENKINS-75747</a>
+     */
+    private boolean checkHeapForApiResponse(StaplerResponse2 rsp) throws IOException {
+        long minFreeBytes = SystemProperties.getLong(Api.class.getName() + ".minFreeMemoryBytes", 50L * 1024 * 1024);
+        if (minFreeBytes <= 0) {
+            return true;
+        }
+
+        Runtime rt = Runtime.getRuntime();
+        long maxMemory = rt.maxMemory();
+        long totalMemory = rt.totalMemory();
+        long freeMemory = rt.freeMemory();
+        long usedMemory = totalMemory - freeMemory;
+        long availableMemory = maxMemory - usedMemory;
+
+        if (availableMemory >= minFreeBytes) {
+            return true;
+        }
+
+        LOGGER.warning(() -> String.format(
+                "Rejecting API request due to low heap: available=%d, required=%d. Suggest using the tree parameter.",
+                availableMemory, minFreeBytes));
+
+        rsp.setStatus(HttpServletResponse.SC_SERVICE_UNAVAILABLE);
+        rsp.setContentType("text/plain;charset=UTF-8");
+        rsp.getWriter().print(Messages.Api_LowMemory());
+        return false;
+    }
+
     @Restricted(NoExternalUse.class)
     protected void setHeaders(StaplerResponse2 rsp) {
         rsp.setHeader("X-Jenkins", Jenkins.VERSION);

core/src/main/resources/hudson/model/Messages.properties

@@ -81,6 +81,7 @@ Api.MultipleMatch=XPath "{0}" matched {1} nodes. \
     Create XPath that only matches one, or use the "wrapper" query parameter to wrap them all under a root element.
 Api.NoXPathMatch=XPath {0} didn’t match
 Api.WrapperParamInvalid=The wrapper parameter can only contain alphanumeric characters or dash/dot/underscore. The first character has to be a letter or underscore.
+Api.LowMemory=Insufficient heap memory to process this API request. The server may run out of memory when serializing large responses. To avoid this, use the tree parameter to limit the data returned, e.g. tree=jobs[name],builds[number]. See the API documentation for details.
 
 BallColor.Aborted=Aborted
 BallColor.Disabled=Disabled

test/src/test/java/hudson/model/ApiTest.java

@@ -189,6 +189,36 @@ void escapedParameter() throws Exception {
                 page.getWebResponse().getContentAsString());
     }
 
+    @Test
+    @Issue("JENKINS-75747")
+    void lowHeapRejectsApiRequest() throws Exception {
+        // Force the OOM guard to reject by requiring more free memory than available
+        String prop = Api.class.getName() + ".minFreeMemoryBytes";
+        String original = System.getProperty(prop);
+        try {
+            System.setProperty(prop, String.valueOf(Long.MAX_VALUE));
+
+            JenkinsRule.WebClient wc = j.createWebClient();
+            wc.getOptions().setThrowExceptionOnFailingStatusCode(false);
+
+            Page xmlPage = wc.goTo("api/xml", null);
+            WebResponse xmlResponse = xmlPage.getWebResponse();
+            assertEquals(HttpServletResponse.SC_SERVICE_UNAVAILABLE, xmlResponse.getStatusCode());
+            assertThat(xmlResponse.getContentAsString(), containsString("tree parameter"));
+
+            Page jsonPage = wc.goTo("api/json", null);
+            WebResponse jsonResponse = jsonPage.getWebResponse();
+            assertEquals(HttpServletResponse.SC_SERVICE_UNAVAILABLE, jsonResponse.getStatusCode());
+            assertThat(jsonResponse.getContentAsString(), containsString("tree parameter"));
+        } finally {
+            if (original != null) {
+                System.setProperty(prop, original);
+            } else {
+                System.clearProperty(prop);
+            }
+        }
+    }
+
     @Test
     @Issue("SECURITY-1704")
     void project_notExposedToIFrame() throws Exception {