Modular: rate limiter allows twice as many OPTIONS requests, since they can't be authenticated

Also tweaks rate limit parameters

Author: DougReeder

lib/appFactory.js

@@ -144,10 +144,10 @@ module.exports = async function ({ hostIdentity, jwtSecret, accountMgr, storeRou
 
   app.use([`${basePath}/`, `${basePath}/oauth`, `${basePath}/account`, `${basePath}/admin`], memorySession);
 
-  app.use((req, _res, next) => {
+  app.use(async (req, _res, next) => {
     if (req.session?.privileges?.STORE) {
       // refunds the points consumed by rate-limiting middleware
-      rateLimiterReward(req.ip, POINTS_UNAUTH_REQUEST - POINTS_AUTH_REQUEST);
+      await rateLimiterReward(req.ip, POINTS_UNAUTH_REQUEST - POINTS_AUTH_REQUEST);
     }
     next();
   });

lib/middleware/rateLimiterMiddleware.js

@@ -9,16 +9,18 @@
 // per-instance.)
 
 const { debuglog } = require('node:util');
+const errToMessages = require('../util/errToMessages');
 const { loggingMiddleware } = require('../logger');
 const { RateLimiterMemory, BurstyRateLimiter } = require('rate-limiter-flexible');
 
 const POINTS_AUTH_REQUEST = 1;
-const POINTS_UNAUTH_REQUEST = 50;
+const POINTS_UNAUTH_REQUEST = 35;
 // OAuth page requires 9 GETs + 1 POST
 // root page + login = 17 requests
 const MAX_BURST_POINTS = 19 * POINTS_UNAUTH_REQUEST;
 // requesting an invitation is 7 requests
-const SUSTAINED_POINTS_PER_SEC = 8 * POINTS_UNAUTH_REQUEST;
+// allows (10 - 1) * 35 = 315 authorized req/sec
+const SUSTAINED_POINTS_PER_SEC = 10 * POINTS_UNAUTH_REQUEST;
 
 // remotestorage.js appears to keep requesting until 10 failures
 
@@ -80,12 +82,17 @@ const rateLimiterMiddleware = async (req, res, next) => {
     await rateLimiter.consume(req.ip, POINTS_UNAUTH_REQUEST);
     next();
   } catch (err) {
-    if (debugEnabled) {
-      await new Promise(resolve => loggingMiddleware(req, res, resolve));
-      res.logNotes.add(`consumedPoints=${err.consumedPoints} msBeforeNext=${err.msBeforeNext}`);
+    if (err instanceof Error) {
+      errToMessages(err, res.logNotes);
+      res.status(500).end();
+    } else {
+      if (debugEnabled) {
+        await new Promise(resolve => loggingMiddleware(req, res, resolve));
+        res.logNotes.add(`consumedPoints=${err.consumedPoints} msBeforeNext=${err.msBeforeNext}`);
+      }
+      res.set({ 'Retry-After': Math.ceil(err.msBeforeNext / 1000) });
+      res.status(429).end();
     }
-    res.set({ 'Retry-After': Math.ceil(err.msBeforeNext / 1000) });
-    res.status(429).end();
   }
 };
 

lib/routes/S3_store_router.js

@@ -1100,7 +1100,7 @@ module.exports = function ({ endPoint = 'play.min.io', accessKey = 'Q3AM3UQ867SP
       // Upload allows you to pass ContentLength, but then fails for streams longer than about 5.1MB, with a message about XML.
 
       parallelUpload.on('httpUploadProgress', (progress) => { // TODO: does this add value?
-        console.debug(bucketName, s3Path, `part ${progress.part}   ${progress.loaded?.toLocaleString()} / ${progress.total?.toLocaleString()} bytes`);
+        console.debug(bucketName, s3Path, `: part ${progress.part}: ${progress.loaded?.toLocaleString()} / ${progress.total?.toLocaleString()} bytes`);
       });
 
       const uploadResponse = await parallelUpload.done();

lib/routes/storage_common.js

@@ -21,7 +21,12 @@ module.exports = function (hostIdentity, jwtSecret) {
   });
 
   router.options('/:username/*',
-    (_req, res, next) => { res.logLevel = 'debug'; next(); },
+    async (req, res, next) => {
+      // not authenticated, but no disk nor network access required, so we can allow more
+      await rateLimiterReward(req.ip, Math.round(POINTS_UNAUTH_REQUEST / 2));
+      res.logLevel = 'debug';
+      next();
+    },
     corsAllowPrivate,
     corsRS
   );
@@ -149,7 +154,7 @@ module.exports = function (hostIdentity, jwtSecret) {
         return [requiredScopeName, '*'].includes(grantedParts[0]) && grantedParts[1].startsWith(requiredPermission);
       })) {
         // refunds the points consumed by rate-limiter middleware
-        rateLimiterReward(req.ip, POINTS_UNAUTH_REQUEST - POINTS_AUTH_REQUEST);
+        await rateLimiterReward(req.ip, POINTS_UNAUTH_REQUEST - POINTS_AUTH_REQUEST);
         next();
       } else {
         return unauthorized(req, res, 403, 'insufficient_scope', requiredScope, `user has permissions '${grantedScopes}' but lacks '${requiredScope}'`);