Skip to content

Commit eed46ff

Browse files
jessevzjessevz
authored
Count (#1145)
* FEAT added count endpoint * Fixed bug in count endpoint when no filters have been provided * Fixed helper tests * FEAT made test for count endpoint * FEAT added possibility to join in count filters * FEAT added the possibility to create complex filters in count endpoint. By adding the possibility to expand and filter on expanded objects * Removed commented out code
1 parent bfd39df commit eed46ff

File tree

5 files changed

+200
-22
lines changed

5 files changed

+200
-22
lines changed

‎ci/apiv2/hashtopolis.py‎

Lines changed: 22 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -291,6 +291,19 @@ def delete(self, obj):
291291

292292
# TODO: Cleanup object to allow re-creation
293293

294+
def count(self, filter):
295+
self.authenticate()
296+
uri = self._api_endpoint + self._model_uri + "/count"
297+
headers = self._headers
298+
payload = {}
299+
if filter:
300+
for k, v in filter.items():
301+
payload[f"filter[{k}]"] = v
302+
303+
logger.debug("Sending GET payload: %s to %s", json.dumps(payload), uri)
304+
r = requests.get(uri, headers=headers, params=payload)
305+
self.validate_status_code(r, [200], "Getting count failed")
306+
return self.resp_to_json(r)['meta']
294307

295308
# Build Django ORM style django.query interface
296309
class QuerySet():
@@ -434,6 +447,11 @@ def get_first(cls):
434447
@classmethod
435448
def get(cls, **filters):
436449
return QuerySet(cls, filters=filters).get()
450+
451+
@classmethod
452+
def count(cls, **filters):
453+
return cls.get_conn().count(filter=filters)
454+
437455

438456
@classmethod
439457
def paginate(cls, **pages):
@@ -912,7 +930,7 @@ def import_cracked_hashes(self, hashlist, source_data, separator):
912930
'separator': separator,
913931
}
914932
response = self._helper_request("importCrackedHashes", payload)
915-
return response['data']
933+
return response['meta']
916934

917935
def get_file(self, file, range=None):
918936
payload = {
@@ -925,19 +943,19 @@ def recount_file_lines(self, file):
925943
'fileId': file.id,
926944
}
927945
response = self._helper_request("recountFileLines", payload)
928-
return File(**response['data'])
946+
return File(**response['meta'])
929947

930948
def unassign_agent(self, agent):
931949
payload = {
932950
'agentId': agent.id,
933951
}
934952
response = self._helper_request("unassignAgent", payload)
935-
return response['data']
953+
return response['meta']
936954

937955
def assign_agent(self, agent, task):
938956
payload = {
939957
'agentId': agent.id,
940958
'taskId': task.id,
941959
}
942960
response = self._helper_request("assignAgent", payload)
943-
return response['data']
961+
return response['meta']

‎ci/apiv2/test_count.py‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
from hashtopolis import HashType
2+
from utils import BaseTest
3+
4+
5+
class CountTest(BaseTest):
6+
model_class = HashType
7+
8+
def create_test_objects(self, **kwargs):
9+
objs = []
10+
for i in range(90000, 90100, 10):
11+
obj = HashType(hashTypeId=i,
12+
description=f"Dummy HashType {i}",
13+
isSalted=(i < 90050),
14+
isSlowHash=False).save()
15+
objs.append(obj)
16+
self.delete_after_test(obj)
17+
return objs
18+
19+
def test_count(self):
20+
model_objs = self.create_test_objects()
21+
model_count = len(model_objs)
22+
api_count = HashType.objects.count(hashTypeId__gte=90000, hashTypeId__lte=91000)['count']
23+
self.assertEqual(model_count, api_count)

‎src/dba/AbstractModelFactory.class.php‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -471,6 +471,10 @@ public function countFilter($options) {
471471
$query = $query . " FROM " . $this->getModelTable();
472472

473473
$vals = array();
474+
475+
if (array_key_exists('join', $options)) {
476+
$query .= $this->applyJoins($options['join']);
477+
}
474478

475479
if (array_key_exists("filter", $options)) {
476480
$query .= $this->applyFilters($vals, $options['filter']);
@@ -750,6 +754,21 @@ private function applyOrder($orders) {
750754
return " ORDER BY " . implode(", ", $orderQueries);
751755
}
752756

757+
private function applyJoins($joins) {
758+
$query = "";
759+
foreach ($joins as $join) {
760+
$joinFactory = $join->getOtherFactory();
761+
$localFactory = $this;
762+
if ($join->getOverrideOwnFactory() != null) {
763+
$localFactory = $join->getOverrideOwnFactory();
764+
}
765+
$match1 = $join->getMatch1();
766+
$match2 = $join->getMatch2();
767+
$query .= " INNER JOIN " . $joinFactory->getModelTable() . " ON " . $localFactory->getModelTable() . "." . $match1 . "=" . $joinFactory->getModelTable() . "." . $match2 . " ";
768+
}
769+
return $query;
770+
}
771+
753772
//applylimit is slightly different than the other apply functions, since you can only limit by a single value
754773
//the $limit argument is a single object LimitFilter object instead of an array of objects.
755774
private function applyLimit($limit) {

‎src/inc/apiv2/common/AbstractBaseAPI.class.php‎

Lines changed: 31 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
use DBA\AgentStat;
1616
use DBA\Assignment;
1717
use DBA\Chunk;
18+
use DBA\ComparisonFilter;
1819
use DBA\Config;
1920
use DBA\ConfigSection;
2021
use DBA\CrackerBinary;
@@ -875,14 +876,19 @@ protected function getPrimaryKey(): string
875876
}
876877
}
877878

879+
function getFilters(Request $request) {
880+
return $this->getQueryParameterFamily($request, 'filter');
881+
}
882+
878883
/**
879884
* Check for valid filter parameters and build QueryFilter
880885
*/
881-
protected function makeFilter(Request $request, array $features): array
886+
// protected function makeFilter(Request $request, array $features): array
887+
protected function makeFilter(array $filters, object $apiClass): array
882888
{
883-
$qFs = [];
884-
885-
$filters = $this->getQueryParameterFamily($request, 'filter');
889+
$qFs = [];
890+
$features = $apiClass->getAliasedFeatures();
891+
$factory = $apiClass->getFactory();
886892
foreach ($filters as $filter => $value) {
887893

888894
if (preg_match('/^(?P<key>[_a-zA-Z0-9]+?)(?<operator>|__eq|__ne|__lt|__lte|__gt|__gte|__contains|__startswith|__endswith|__icontains|__istartswith|__iendswith)$/', $filter, $matches) == 0) {
@@ -919,44 +925,52 @@ protected function makeFilter(Request $request, array $features): array
919925
switch($matches['operator']) {
920926
case '':
921927
case '__eq':
922-
array_push($qFs, new QueryFilter($remappedKey, $val, '='));
928+
$operator = '=';
923929
break;
924930
case '__ne':
925-
array_push($qFs, new QueryFilter($remappedKey, $val, '!='));
931+
$operator = '!=';
926932
break;
927933
case '__lt':
928-
array_push($qFs, new QueryFilter($remappedKey, $val, '<'));
934+
$operator = '<';
929935
break;
930936
case '__lte':
931-
array_push($qFs, new QueryFilter($remappedKey, $val, '<='));
937+
$operator = '<=';
932938
break;
933939
case '__gt':
934-
array_push($qFs, new QueryFilter($remappedKey, $val, '>'));
940+
$operator = '>';
935941
break;
936942
case '__gte':
937-
array_push($qFs, new QueryFilter($remappedKey, $val, '>='));
943+
$operator = '>=';
938944
break;
939945
case '__contains':
940-
array_push($qFs, new LikeFilter($remappedKey, "%" . $val . "%"));
946+
array_push($qFs, new LikeFilter($remappedKey, "%" . $val . "%", $factory));
941947
break;
942948
case '__startswith':
943-
array_push($qFs, new LikeFilter($remappedKey, $val . "%"));
949+
array_push($qFs, new LikeFilter($remappedKey, $val . "%", $factory));
944950
break;
945951
case '__endswith':
946-
array_push($qFs, new LikeFilter($remappedKey, "%" . $val));
952+
array_push($qFs, new LikeFilter($remappedKey, "%" . $val, $factory));
947953
break;
948954
case '__icontains':
949-
array_push($qFs, new LikeFilterInsensitive($remappedKey, "%" . $val . "%"));
955+
array_push($qFs, new LikeFilterInsensitive($remappedKey, "%" . $val . "%", $factory));
950956
break;
951957
case '__istartswith':
952-
array_push($qFs, new LikeFilterInsensitive($remappedKey, $val . "%"));
958+
array_push($qFs, new LikeFilterInsensitive($remappedKey, $val . "%", $factory));
953959
break;
954960
case '__iendswith':
955-
array_push($qFs, new LikeFilterInsensitive($remappedKey, "%" . $val));
961+
array_push($qFs, new LikeFilterInsensitive($remappedKey, "%" . $val, $factory));
956962
break;
957963
default:
958964
assert(False, "Operator '" . $matches['operator'] . "' not implemented");
959965
}
966+
967+
if ($operator) {
968+
if (array_key_exists($val, $features)) {
969+
array_push($qFs, new ComparisonFilter($remappedKey, $val, $operator, $factory));
970+
} else {
971+
array_push($qFs, new QueryFilter($remappedKey, $val, $operator, $factory));
972+
}
973+
}
960974
}
961975
return $qFs;
962976
}
@@ -1257,7 +1271,7 @@ protected static function getOneResource(object $apiClass, object $object, Reque
12571271

12581272
//Meta response for helper functions that do not respond with resource records
12591273
protected static function getMetaResponse(array $meta, Request $request, Response $response, int $statusCode=200) {
1260-
$ret = self::createJsonResponse($meta=$meta);
1274+
$ret = self::createJsonResponse(meta: $meta);
12611275
$body = $response->getBody();
12621276
$body->write(self::ret2json($ret));
12631277

‎src/inc/apiv2/common/AbstractModelAPI.class.php‎

Lines changed: 105 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -392,7 +392,8 @@ public static function getManyResources(object $apiClass, Request $request, Resp
392392
$aFs = [];
393393

394394
/* Generate filters */
395-
$qFs_Filter = $apiClass->makeFilter($request, $aliasedfeatures);
395+
$filters = $apiClass->getFilters($request);
396+
$qFs_Filter = $apiClass->makeFilter($filters, $apiClass);
396397
$qFs_ACL = $apiClass->getFilterACL();
397398
$qFs = array_merge($qFs_ACL, $qFs_Filter);
398399
if (count($qFs) > 0) {
@@ -559,6 +560,108 @@ public function get(Request $request, Response $response, array $args): Response
559560
return self::getManyResources($this, $request, $response);
560561
}
561562

563+
/**
564+
* Maps filters to the appropiate models based on their feautures.
565+
*
566+
* Helper function to get valid filters for the models. This is usefull when multiple objects
567+
* have been included and the correct filters need to be mapped to the correct objects.
568+
* Currently used to make complex filters for counting objects
569+
*
570+
* @param array $filters An associative array of filters where the key is the filter
571+
* name and the value is the filter value. Filters should match
572+
* the pattern `<field><operator>`, where `<operator>` can be
573+
* one of the supported suffixes (e.g., `__eq`, `__ne`).
574+
* @param array $models An array of model objects. Each model must have a `getFeatures()`
575+
* method that returns an associative array of model features.
576+
* The features should map filter keys to their respective
577+
* attributes or aliases.
578+
*
579+
* @return array An associative array mapping model classes to their respective valid filters.
580+
* The structure is:
581+
* [
582+
* ModelClassName => [
583+
* 'filter' => 'value',
584+
* ...
585+
* ],
586+
* ...
587+
* ]
588+
*
589+
* @throws HTException If a filter key does not match the expected format or is invalid.
590+
*/
591+
public function filterObjectMap(array $filters, array $models) {
592+
593+
$modelFilterMap = [];
594+
foreach ($filters as $filter => $value) {
595+
if (preg_match('/^(?P<key>[_a-zA-Z0-9]+?)(?<operator>|__eq|__ne|__lt|__lte|__gt|__gte|__contains|__startswith|__endswith|__icontains|__istartswith|__iendswith)$/', $filter, $matches) == 0) {
596+
throw new HTException("Filter parameter '" . $filter . "' is not valid");
597+
}
598+
599+
foreach($models as $model) {
600+
$features = $model->getFeatures();
601+
// Special filtering of _id to use for uniform access to model primary key
602+
$cast_key = $matches['key'] == '_id' ? array_column($features, 'alias', 'dbname')[$this->getPrimaryKey()] : $matches['key'];
603+
if (array_key_exists($cast_key, $features) == false) {
604+
continue; //not a valid filter for current model
605+
};
606+
$modelFilterMap[$model::class][$filter] = $value;
607+
break; //filter has been found for current model, so break to go to next filter
608+
}
609+
}
610+
return $modelFilterMap;
611+
}
612+
613+
/**
614+
* API entry point for retrieving count information of data
615+
*/
616+
public function count(Request $request, Response $response, array $args): Response
617+
{
618+
$this->preCommon($request);
619+
$factory = $this->getFactory();
620+
621+
//resolve all expandables
622+
$validExpandables = $this::getExpandables();
623+
$expands = $this->makeExpandables($request, $validExpandables);
624+
625+
$objects = [$factory->getNullObject()];
626+
//build join filters
627+
foreach ($expands as $expand) {
628+
$relation = $this->getToManyRelationships()[$expand];
629+
$objects[] = $this->getModelFactory($relation["relationType"])->getNullObject();
630+
$otherFactory = $this->getModelFactory($relation["relationType"]);
631+
$primaryKey = $this->getPrimaryKey();
632+
$aFs[Factory::JOIN][] = new JoinFilter($otherFactory, $relation["relationKey"], $primaryKey, $factory);
633+
}
634+
635+
$filters = $this->getFilters($request);
636+
$filterObjectMap = $this->filterObjectMap($filters, $objects);
637+
$qFs = [];
638+
foreach($filterObjectMap as $class => $cur_filters) {
639+
$relationApiClass = new ($this->container->get('classMapper')->get($class))($this->container);
640+
$current_qFs = $this->makeFilter($cur_filters, $relationApiClass);
641+
$qFs = array_merge($qFs, $current_qFs);
642+
}
643+
644+
if (count($qFs) > 0) {
645+
$aFs[Factory::FILTER] = $qFs;
646+
}
647+
648+
$count = $factory->countFilter($aFs);
649+
$meta = ["count" => $count];
650+
651+
$include_total = $request->getQueryParams()['include_total'];
652+
if ($include_total == "true") {
653+
$meta["total_count"] = $factory->countFilter([]);
654+
}
655+
656+
$ret = self::createJsonResponse(meta: $meta);
657+
658+
$body = $response->getBody();
659+
$body->write($this->ret2json($ret));
660+
661+
return $response->withStatus(200)
662+
->withHeader("Content-Type", 'application/vnd.api+json');
663+
}
664+
562665
/**
563666
* Get input field names valid for creation of object
564667
*/
@@ -1106,6 +1209,7 @@ static public function register($app): void
11061209

11071210
if (in_array("GET", $available_methods)) {
11081211
$app->get($baseUri, $me . ':get')->setname($me . ':get');
1212+
$app->get($baseUri . "/count", $me . ':count')->setname($me . ':count');
11091213
}
11101214

11111215
foreach ($me::getToOneRelationships() as $name => $relationship) {

0 commit comments

Comments
 (0)