1154 enhancement apiv2 return total amount of pages for paginated api calls (#1173)

* FEAT: Added total page size in meta data and get correct first page for pagination

* FEAT: made pagination default page size and maximum page size configurable through config

* Fixed bug in upgrade script

Author: jessevz

src/inc/apiv2/common/AbstractModelAPI.class.php

@@ -9,6 +9,7 @@
 use Middlewares\Utils\HttpErrorException;
 
 use DBA\AbstractModelFactory;
+use DBA\Aggregation;
 use DBA\JoinFilter;
 use DBA\Factory;
 use DBA\ContainFilter;
@@ -382,9 +383,11 @@ public static function getManyResources(object $apiClass, Request $request, Resp
     $aliasedfeatures = $apiClass->getAliasedFeatures();
     $factory = $apiClass->getFactory();
 
-    // TODO: Maximum and default should be configurable per server instance
     $defaultPageSize = 10000;
     $maxPageSize = 50000;
+    // TODO: if 0.14.4 release has happened, following parameters can be retrieved from config
+    // $defaultPageSize = SConfig::getInstance()->getVal(DConfig::DEFAULT_PAGE_SIZE);
+    // $maxPageSize = SConfig::getInstance()->getVal(DConfig::MAX_PAGE_SIZE);
 
     $pageAfter = $apiClass->getQueryParameterFamilyMember($request, 'page', 'after') ?? 0;
     $pageSize = $apiClass->getQueryParameterFamilyMember($request, 'page', 'size') ?? $defaultPageSize;
@@ -426,18 +429,28 @@ public static function getManyResources(object $apiClass, Request $request, Resp
     /* Include relation filters */
     $finalFs = array_merge($aFs, $relationFs);
 
+    $primaryKey = $apiClass->getPrimaryKey();
     //according to JSON API spec, first and last have to be calculated if inexpensive to compute 
     //(https://jsonapi.org/profiles/ethanresnick/cursor-pagination/#auto-id-links))
-    //if this query is too expensive for big tables, it should be removed
-    $max = $factory->minMaxFilter($finalFs, $apiClass->getPrimaryKey(), "MAX");
+    //if this query is too expensive for big tables, it can be removed
+    $agg1 = new Aggregation($primaryKey, Aggregation::MAX);
+    $agg2 = new Aggregation($primaryKey, Aggregation::MIN);
+    $agg3 = new Aggregation($primaryKey, Aggregation::COUNT);
+    $aggregation_results = $factory->multicolAggregationFilter($finalFs, [$agg1, $agg2, $agg3]);
+
+    $max = $aggregation_results[$agg1->getName()];
+    $min = $aggregation_results[$agg2->getName()];
+    $total = $aggregation_results[$agg3->getName()];
+
+    $totalPages = ceil($total / $pageSize);
 
     //pagination filters need to be added after max has been calculated
     $finalFs[Factory::LIMIT] = new LimitFilter($pageSize);
 
-    $finalFs[Factory::FILTER][] = new QueryFilter($apiClass->getPrimaryKey(), $pageAfter, '>', $factory);
+    $finalFs[Factory::FILTER][] = new QueryFilter($primaryKey, $pageAfter, '>', $factory);
     $pageBefore = $apiClass->getQueryParameterFamilyMember($request, 'page', 'before');
     if (isset($pageBefore)) {
-      $finalFs[Factory::FILTER][] = new QueryFilter($apiClass->getPrimaryKey(), $pageBefore, '<', $factory);
+      $finalFs[Factory::FILTER][] = new QueryFilter($primaryKey, $pageBefore, '<', $factory);
     }
 
     /* Request objects */
@@ -525,12 +538,8 @@ public static function getManyResources(object $apiClass, Request $request, Resp
       }
       // Build prev link 
       $prevId = $defaultSort == "DESC" ? $maxId : $minId;
-      if ($prevId != 1) { //only set previous page when its not the first page
+      if ($prevId != $min) { //only set previous page when its not the first page
         $prevParams = $selfParams;
-        //This scenario might return a link to an empty array if the elements with the lowest id are deleted, but this is allowed according
-        //to the json API spec https://jsonapi.org/profiles/ethanresnick/cursor-pagination/#auto-id-links
-        //We could also get the lowest id the same way we got the max, but this is probably unnecessary expensive.
-        //But pull request: https://github.com/hashtopolis/server/pull/1069 would create a cheaper way of doing this in a single query
         $prevParams['page']['before'] = $prevId;
         unset($prevParams['page']['after']);
         $linksPrev = $request->getUri()->getPath() . '?' .  urldecode(http_build_query($prevParams));
@@ -541,7 +550,7 @@ public static function getManyResources(object $apiClass, Request $request, Resp
     $firstParams = $request->getQueryParams();
     unset($firstParams['page']['before']);
     $firstParams['page']['size'] = $pageSize;
-    $firstParams['page']['after'] = 0;
+    $firstParams['page']['after'] = $min;
     $linksFirst = $request->getUri()->getPath() . '?' .  urldecode(http_build_query($firstParams));
     $links = [
         "self" => $linksSelf,
@@ -551,8 +560,9 @@ public static function getManyResources(object $apiClass, Request $request, Resp
         "prev" => $linksPrev,
       ];
 
+    $metadata = ["page" => ["total_pages" => $totalPages]];
     // Generate JSON:API GET output
-    $ret = self::createJsonResponse($dataResources, $links, $includedResources);
+    $ret = self::createJsonResponse($dataResources, $links, $includedResources, $metadata);
 
     $body = $response->getBody();
     $body->write($apiClass->ret2json($ret));

src/inc/defines/config.php

@@ -68,6 +68,8 @@ class DConfig {
   const HASH_MAX_LENGTH            = "hashMaxLength";
   const MAX_HASHLIST_SIZE          = "maxHashlistSize";
   const UAPI_SEND_TASK_IS_COMPLETE = "uApiSendTaskIsComplete";
+  const DEFAULT_PAGE_SIZE          = "defaultPageSize";
+  const MAX_PAGE_SIZE              = "maxPageSize";
 
   // Section: UI
   const TIME_FORMAT            = "timefmt";
@@ -272,6 +274,10 @@ public static function getConfigType($config) {
         return DConfigType::TICKBOX;
       case DConfig::HC_ERROR_IGNORE:
         return DConfigType::STRING_INPUT;
+      case DConfig::DEFAULT_PAGE_SIZE:
+        return DConfigType::NUMBER_INPUT; 
+      case DConfig::MAX_PAGE_SIZE:
+          return DConfigType::NUMBER_INPUT;
     }
     return DConfigType::STRING_INPUT;
   }
@@ -406,6 +412,10 @@ public static function getConfigDescription($config) {
         return "Also send 'isComplete' for each task on the User API when listing all tasks (might affect performance)";
       case DConfig::HC_ERROR_IGNORE:
         return "Ignore error messages from crackers which contain given strings (multiple values separated by comma)";
+      case DConfig::DEFAULT_PAGE_SIZE:
+        return "The default page size of items that are returned in API calls.";
+      case DConfig::MAX_PAGE_SIZE:
+        return "The maximum page size of items that are allowed to return in an API call.";
     }
     return $config;
   }

src/install/hashtopolis.sql

@@ -170,7 +170,10 @@ INSERT INTO `Config` (`configId`, `configSectionId`, `item`, `value`) VALUES
   (74, 4, 'agentUtilThreshold1', '90'),
   (75, 4, 'agentUtilThreshold2', '75'),
   (76, 3, 'uApiSendTaskIsComplete', '0'),
-  (77, 1, 'hcErrorIgnore', 'DeviceGetFanSpeed');
+  (77, 1, 'hcErrorIgnore', 'DeviceGetFanSpeed'),
+  (78, 3, 'defaultPageSize', '10000'),
+  (79, 3, 'maxPageSize', '50000');
+
 
 CREATE TABLE `ConfigSection` (
   `configSectionId` INT(11)      NOT NULL,

src/install/updates/update_v0.14.3_v0.14.x.php

@@ -0,0 +1,20 @@
+<?php /** @noinspection SqlNoDataSourceInspection */
+
+use DBA\Config;
+use DBA\Factory;
+use DBA\QueryFilter;
+
+if (!isset($PRESENT["v0.14.x_pagination"])) {
+  $qF = new QueryFilter(Config::ITEM, DConfig::DEFAULT_PAGE_SIZE, "=");
+  $item = Factory::getConfigFactory()->filter([Factory::FILTER => $qF], true);
+  if (!$item) {
+    $config = new Config(null, 3, DConfig::DEFAULT_PAGE_SIZE, '10000');
+    Factory::getConfigFactory()->save($config);
+  }
+  $qF = new QueryFilter(Config::ITEM, DConfig::MAX_PAGE_SIZE, "=");
+  $item = Factory::getConfigFactory()->filter([Factory::FILTER => $qF], true);
+  if (!$item) {
+    $config = new Config(null, 3, DConfig::MAX_PAGE_SIZE, '50000');
+    Factory::getConfigFactory()->save($config);
+  }
+}