Pipeline shows disabled endpoints fix (#2881)

.github/workflows/build.yml

@@ -141,4 +141,5 @@ jobs:
         run: |
           chmod +x ./testing/test_webpages.sh
           chmod +x ./testing/test.sh
+          chmod +x ./testing/test_disabledEndpoints.sh
           ./testing/test.sh

exampleYmlFiles/docker-compose-latest-fat-endpoints-disabled.yml

@@ -0,0 +1,35 @@
+
+services:
+  stirling-pdf:
+    container_name: Stirling-PDF-Fat-Disable-Endpoints
+    image: docker.stirlingpdf.com/stirlingtools/stirling-pdf:latest-fat
+    deploy:
+      resources:
+        limits:
+          memory: 4G
+    healthcheck:
+      test: ["CMD-SHELL", "curl -f http://localhost:8080/api/v1/info/status | grep -q 'UP'"]
+      interval: 5s
+      timeout: 10s
+      retries: 16
+    ports:
+      - 8080:8080
+    volumes:
+      - ./stirling/latest/data:/usr/share/tessdata:rw
+      - ./stirling/latest/config:/configs:rw
+      - ./stirling/latest/logs:/logs:rw
+      - ../testing/allEndpointsRemovedSettings.yml:/configs/settings.yml:rw
+    environment:
+      DOCKER_ENABLE_SECURITY: "true"
+      SECURITY_ENABLELOGIN: "false"
+      PUID: 1002
+      PGID: 1002
+      UMASK: "022"
+      SYSTEM_DEFAULTLOCALE: en-US
+      UI_APPNAME: Stirling-PDF
+      UI_HOMEDESCRIPTION: Demo site for Stirling-PDF Latest-fat with  all Endpoints Disabled
+      UI_APPNAMENAVBAR: Stirling-PDF Latest-fat
+      SYSTEM_MAXFILESIZE: "100"
+      METRICS_ENABLED: "true"
+      SYSTEM_GOOGLEVISIBILITY: "true"
+    restart: on-failure:5

src/main/java/stirling/software/SPDF/config/EndpointConfiguration.java

@@ -45,6 +45,10 @@ public void disableEndpoint(String endpoint) {
         }
     }
 
+    public Map<String, Boolean> getEndpointStatuses() {
+        return endpointStatuses;
+    }
+
     public boolean isEndpointEnabled(String endpoint) {
         if (endpoint.startsWith("/")) {
             endpoint = endpoint.substring(1);

src/main/java/stirling/software/SPDF/config/EndpointInterceptor.java

@@ -23,7 +23,29 @@ public boolean preHandle(
             HttpServletRequest request, HttpServletResponse response, Object handler)
             throws Exception {
         String requestURI = request.getRequestURI();
-        if (!endpointConfiguration.isEndpointEnabled(requestURI)) {
+        boolean isEnabled;
+
+        // Extract the specific endpoint name (e.g: /api/v1/general/remove-pages -> remove-pages)
+        if (requestURI.contains("/api/v1") && requestURI.split("/").length > 4) {
+
+            String[] requestURIParts = requestURI.split("/");
+            String requestEndpoint;
+
+            // Endpoint: /api/v1/convert/pdf/img becomes pdf-to-img
+            if ("convert".equals(requestURIParts[3]) && requestURIParts.length > 5) {
+                requestEndpoint = requestURIParts[4] + "-to-" + requestURIParts[5];
+            } else {
+                requestEndpoint = requestURIParts[4];
+            }
+
+            log.debug("Request endpoint: {}", requestEndpoint);
+            isEnabled = endpointConfiguration.isEndpointEnabled(requestEndpoint);
+            log.debug("Is endpoint enabled: {}", isEnabled);
+        } else {
+            isEnabled = endpointConfiguration.isEndpointEnabled(requestURI);
+        }
+
+        if (!isEnabled) {
             response.sendError(HttpServletResponse.SC_FORBIDDEN, "This endpoint is disabled");
             return false;
         }

src/main/java/stirling/software/SPDF/controller/api/SettingsController.java

@@ -1,17 +1,20 @@
 package stirling.software.SPDF.controller.api;
 
 import java.io.IOException;
+import java.util.Map;
 
 import org.springframework.http.HttpStatus;
 import org.springframework.http.ResponseEntity;
 import org.springframework.stereotype.Controller;
+import org.springframework.web.bind.annotation.GetMapping;
 import org.springframework.web.bind.annotation.PostMapping;
 import org.springframework.web.bind.annotation.RequestBody;
 import org.springframework.web.bind.annotation.RequestMapping;
 
 import io.swagger.v3.oas.annotations.Hidden;
 import io.swagger.v3.oas.annotations.tags.Tag;
 
+import stirling.software.SPDF.config.EndpointConfiguration;
 import stirling.software.SPDF.config.InstallationPathConfig;
 import stirling.software.SPDF.model.ApplicationProperties;
 import stirling.software.SPDF.utils.GeneralUtils;
@@ -23,9 +26,13 @@
 public class SettingsController {
 
     private final ApplicationProperties applicationProperties;
+    private final EndpointConfiguration endpointConfiguration;
 
-    public SettingsController(ApplicationProperties applicationProperties) {
+    public SettingsController(
+            ApplicationProperties applicationProperties,
+            EndpointConfiguration endpointConfiguration) {
         this.applicationProperties = applicationProperties;
+        this.endpointConfiguration = endpointConfiguration;
     }
 
     @PostMapping("/update-enable-analytics")
@@ -41,4 +48,10 @@ public ResponseEntity<String> updateApiKey(@RequestBody Boolean enabled) throws
         applicationProperties.getSystem().setEnableAnalytics(enabled);
         return ResponseEntity.ok("Updated");
     }
+
+    @GetMapping("/get-endpoints-status")
+    @Hidden
+    public ResponseEntity<Map<String, Boolean>> getDisabledEndpoints() {
+        return ResponseEntity.ok(endpointConfiguration.getEndpointStatuses());
+    }
 }

src/main/resources/static/js/pipeline.js

@@ -153,22 +153,32 @@ document.getElementById("submitConfigBtn").addEventListener("click", function ()
 let apiDocs = {};
 let apiSchemas = {};
 let operationSettings = {};
+let operationStatus = {};
 
 fetchWithCsrf("v1/api-docs")
   .then((response) => response.json())
   .then((data) => {
     apiDocs = data.paths;
     apiSchemas = data.components.schemas;
-    let operationsDropdown = document.getElementById("operationsDropdown");
+    return fetchWithCsrf("api/v1/settings/get-endpoints-status")
+      .then((response) => response.json())
+      .then((data) => {
+        operationStatus = data;
+      })
+      .catch((error) => {
+        console.error("Error:", error);
+      });
+  })
+  .then(() => {
     const ignoreOperations = ["/api/v1/pipeline/handleData", "/api/v1/pipeline/operationToIgnore"]; // Add the operations you want to ignore here
-
+    let operationsDropdown = document.getElementById("operationsDropdown");
     operationsDropdown.innerHTML = "";
 
     let operationsByTag = {};
 
     // Group operations by tags
-    Object.keys(data.paths).forEach((operationPath) => {
-      let operation = data.paths[operationPath].post;
+    Object.keys(apiDocs).forEach((operationPath) => {
+      let operation = apiDocs[operationPath].post;
       if (!operation || !operation.description) {
         console.log(operationPath);
       }
@@ -209,13 +219,19 @@ fetchWithCsrf("v1/api-docs")
           }
           operationPathDisplay = operationPathDisplay.replaceAll(" ", "-");
           option.textContent = operationPathDisplay;
-          option.value = operationPath; // Keep the value with slashes for querying
-          group.appendChild(option);
+
+          if (!(operationPathDisplay in operationStatus)) {
+            option.value = operationPath; // Keep the value with slashes for querying
+            group.appendChild(option);
+          }
         });
 
         operationsDropdown.appendChild(group);
       }
     });
+  })
+  .catch((error) => {
+    console.error("Error:", error);
   });
 
 document.getElementById('deletePipelineBtn').addEventListener('click', function(event) {

testing/allEndpointsRemovedSettings.yml

@@ -0,0 +1,147 @@
+#############################################################################################################
+#                                         Welcome to settings file from                                     #
+#                      ____ _____ ___ ____  _     ___ _   _  ____       ____  ____  _____                   #
+#                     / ___|_   _|_ _|  _ \| |   |_ _| \ | |/ ___|     |  _ \|  _ \|  ___|                  #
+#                     \___ \ | |  | || |_) | |    | ||  \| | |  _ _____| |_) | | | | |_                     #
+#                      ___) || |  | ||  _ <| |___ | || |\  | |_| |_____|  __/| |_| |  _|                    #
+#                     |____/ |_| |___|_| \_\_____|___|_| \_|\____|     |_|   |____/|_|                      #
+#                                                                                                           #
+# Custom setting.yml file with all endpoints disabled to only be used for testing purposes                  #
+# Do not comment out any entry, it will be removed on next startup                                          #
+# If you want to override with environment parameter follow parameter naming SECURITY_INITIALLOGIN_USERNAME #
+#############################################################################################################
+
+
+security:
+  enableLogin: false # set to 'true' to enable login
+  csrfDisabled: false # set to 'true' to disable CSRF protection (not recommended for production)
+  loginAttemptCount: 5 # lock user account after 5 tries; when using e.g. Fail2Ban you can deactivate the function with -1
+  loginResetTimeMinutes: 120 # lock account for 2 hours after x attempts
+  loginMethod: all # Accepts values like 'all' and 'normal'(only Login with Username/Password), 'oauth2'(only Login with OAuth2) or 'saml2'(only Login with SAML2)
+  initialLogin:
+    username: '' # initial username for the first login
+    password: '' # initial password for the first login
+  oauth2:
+    enabled: false # set to 'true' to enable login (Note: enableLogin must also be 'true' for this to work)
+    client:
+      keycloak:
+        issuer: '' # URL of the Keycloak realm's OpenID Connect Discovery endpoint
+        clientId: '' # client ID for Keycloak OAuth2
+        clientSecret: '' # client secret for Keycloak OAuth2
+        scopes: openid, profile, email # scopes for Keycloak OAuth2
+        useAsUsername: preferred_username # field to use as the username for Keycloak OAuth2. Available options are: [email | name | given_name | family_name | preferred_name]
+      google:
+        clientId: '' # client ID for Google OAuth2
+        clientSecret: '' # client secret for Google OAuth2
+        scopes: email, profile # scopes for Google OAuth2
+        useAsUsername: email # field to use as the username for Google OAuth2. Available options are: [email | name | given_name | family_name]
+      github:
+        clientId: '' # client ID for GitHub OAuth2
+        clientSecret: '' # client secret for GitHub OAuth2
+        scopes: read:user # scope for GitHub OAuth2
+        useAsUsername: login # field to use as the username for GitHub OAuth2. Available options are: [email | login | name]
+    issuer: '' # set to any Provider that supports OpenID Connect Discovery (/.well-known/openid-configuration) endpoint
+    clientId: '' # client ID from your Provider
+    clientSecret: '' # client secret from your Provider
+    autoCreateUser: true # set to 'true' to allow auto-creation of non-existing users
+    blockRegistration: false # set to 'true' to deny login with SSO without prior registration by an admin
+    useAsUsername: email # default is 'email'; custom fields can be used as the username
+    scopes: openid, profile, email # specify the scopes for which the application will request permissions
+    provider: google # set this to your OAuth Provider's name, e.g., 'google' or 'keycloak'
+  saml2:
+    enabled: false # Only enabled for paid enterprise clients (enterpriseEdition.enabled must be true)
+    provider: '' # The name of your Provider
+    autoCreateUser: true # set to 'true' to allow auto-creation of non-existing users
+    blockRegistration: false # set to 'true' to deny login with SSO without prior registration by an admin
+    registrationId: stirling # The name of your Service Provider (SP) app name. Should match the name in the path for your SSO & SLO URLs
+    idpMetadataUri: https://dev-XXXXXXXX.okta.com/app/externalKey/sso/saml/metadata # The uri for your Provider's metadata
+    idpSingleLoginUrl: https://dev-XXXXXXXX.okta.com/app/dev-XXXXXXXX_stirlingpdf_1/externalKey/sso/saml # The URL for initiating SSO. Provided by your Provider
+    idpSingleLogoutUrl: https://dev-XXXXXXXX.okta.com/app/dev-XXXXXXXX_stirlingpdf_1/externalKey/slo/saml # The URL for initiating SLO. Provided by your Provider
+    idpIssuer: '' # The ID of your Provider
+    idpCert: classpath:okta.cert # The certificate your Provider will use to authenticate your app's SAML authentication requests. Provided by your Provider
+    privateKey: classpath:saml-private-key.key # Your private key. Generated from your keypair
+    spCert: classpath:saml-public-cert.crt # Your signing certificate. Generated from your keypair
+
+enterpriseEdition:
+  enabled: false # set to 'true' to enable enterprise edition
+  key: 00000000-0000-0000-0000-000000000000
+  SSOAutoLogin: false # Enable to auto login to first provided SSO
+  CustomMetadata:
+    autoUpdateMetadata: false # set to 'true' to automatically update metadata with below values
+    author: username # supports text such as 'John Doe' or types such as username to autopopulate with user's username
+    creator: Stirling-PDF # supports text such as 'Company-PDF'
+    producer: Stirling-PDF # supports text such as 'Company-PDF'
+
+legal:
+  termsAndConditions: https://www.stirlingpdf.com/terms-and-conditions # URL to the terms and conditions of your application (e.g. https://example.com/terms). Empty string to disable or filename to load from local file in static folder
+  privacyPolicy: https://www.stirlingpdf.com/privacy-policy # URL to the privacy policy of your application (e.g. https://example.com/privacy). Empty string to disable or filename to load from local file in static folder
+  accessibilityStatement: '' # URL to the accessibility statement of your application (e.g. https://example.com/accessibility). Empty string to disable or filename to load from local file in static folder
+  cookiePolicy: '' # URL to the cookie policy of your application (e.g. https://example.com/cookie). Empty string to disable or filename to load from local file in static folder
+  impressum: '' # URL to the impressum of your application (e.g. https://example.com/impressum). Empty string to disable or filename to load from local file in static folder
+
+system:
+  defaultLocale: en-US # set the default language (e.g. 'de-DE', 'fr-FR', etc)
+  googlevisibility: false # 'true' to allow Google visibility (via robots.txt), 'false' to disallow
+  enableAlphaFunctionality: false # set to enable functionality which might need more testing before it fully goes live (this feature might make no changes)
+  showUpdate: false # see when a new update is available
+  showUpdateOnlyAdmin: false # only admins can see when a new update is available, depending on showUpdate it must be set to 'true'
+  customHTMLFiles: false # enable to have files placed in /customFiles/templates override the existing template HTML files
+  tessdataDir: /usr/share/tessdata # path to the directory containing the Tessdata files. This setting is relevant for Windows systems. For Windows users, this path should be adjusted to point to the appropriate directory where the Tessdata files are stored.
+  enableAnalytics: true # set to 'true' to enable analytics, set to 'false' to disable analytics; for enterprise users, this is set to true
+  disableSanitize: false # set to true to disable Sanitize HTML; (can lead to injections in HTML)
+  datasource:
+    enableCustomDatabase: false # Enterprise users ONLY, set this property to 'true' if you would like to use your own custom database configuration
+    customDatabaseUrl: '' # eg jdbc:postgresql://localhost:5432/postgres, set the url for your own custom database connection. If provided, the type, hostName, port and name are not necessary and will not be used
+    username: postgres # set the database username
+    password: postgres # set the database password
+    type: postgresql # the type of the database to set (e.g. 'h2', 'postgresql')
+    hostName: localhost # the host name to use for the database url. Set to 'localhost' when running the app locally. Set to match the name of the container name of your database container when running the app on a server (Docker configuration)
+    port: 5432 # set the port number of the database. Ensure this matches the port the database is listening to
+    name: postgres # set the name of your database. Should match the name of the database you create
+  customPaths:
+    pipeline:
+      watchedFoldersDir: "" #Defaults to /pipeline/watchedFolders
+      finishedFoldersDir: "" #Defaults to /pipeline/finishedFolders
+    operations:
+      weasyprint: "" #Defaults to /opt/venv/bin/weasyprint
+      unoconvert: "" #Defaults to /opt/venv/bin/unoconvert
+
+
+
+ui:
+  appName: '' # application's visible name
+  homeDescription: '' # short description or tagline shown on the homepage
+  appNameNavbar: '' # name displayed on the navigation bar
+  languages: [] # If empty, all languages are enabled. To display only German and Polish ["de_DE", "pl_PL"]. British English is always enabled.
+
+endpoints: # All the possible endpoints are disabled
+  toRemove: [crop, merge-pdfs, multi-page-layout, overlay-pdfs, pdf-to-single-page, rearrange-pages, remove-image-pdf, remove-pages, rotate-pdf, scale-pages, split-by-size-or-count, split-pages, split-pdf-by-chapters, split-pdf-by-sections, add-password, add-watermark, auto-redact, cert-sign, get-info-on-pdf, redact, remove-cert-sign, remove-password, sanitize-pdf, validate-signature, file-to-pdf, html-to-pdf, img-to-pdf, markdown-to-pdf, pdf-to-csv, pdf-to-html, pdf-to-img, pdf-to-markdown, pdf-to-pdfa, pdf-to-presentation, pdf-to-text, pdf-to-word, pdf-to-xml, url-to-pdf, add-image, add-page-numbers, add-stamp, auto-rename, auto-split-pdf, compress-pdf, decompress-pdf, extract-image-scans, extract-images, flatten, ocr-pdf, remove-blanks, repair, replace-invert-pdf, show-javascript, update-metadata, filter-contains-image, filter-contains-text, filter-file-size, filter-page-count, filter-page-rotation, filter-page-size] # list endpoints to disable (e.g. ['img-to-pdf', 'remove-pages'])
+  groupsToRemove: [] # list groups to disable (e.g. ['LibreOffice'])
+
+metrics:
+  enabled: true # 'true' to enable Info APIs (`/api/*`) endpoints, 'false' to disable
+
+# Automatically Generated Settings (Do Not Edit Directly)
+AutomaticallyGenerated:
+  key: cbb81c0f-50b1-450c-a2b5-89ae527776eb
+  UUID: 10dd4fba-01fa-4717-9b78-3dc4f54e398a
+  appVersion: 0.44.3
+
+processExecutor:
+  sessionLimit: # Process executor instances limits
+    libreOfficeSessionLimit: 1
+    pdfToHtmlSessionLimit: 1
+    qpdfSessionLimit: 4
+    tesseractSessionLimit: 1
+    pythonOpenCvSessionLimit: 8
+    weasyPrintSessionLimit: 16
+    installAppSessionLimit: 1
+    calibreSessionLimit: 1
+  timeoutMinutes: # Process executor timeout in minutes
+    libreOfficetimeoutMinutes: 30
+    pdfToHtmltimeoutMinutes: 20
+    pythonOpenCvtimeoutMinutes: 30
+    weasyPrinttimeoutMinutes: 30
+    installApptimeoutMinutes: 60
+    calibretimeoutMinutes: 30
+    tesseractTimeoutMinutes: 30