Unlimit search result (#12)

* Allow setting limit to none to return all search results returned by apple api

* Allow setting limit to none to return all search results returned by apple api

* update .gitignore

* add multipage test

---------

Co-authored-by: Dale Wahl <[email protected]>

Author: ddxv

.gitignore

@@ -3,4 +3,8 @@ itunes_app_scraper/__pycache__
 __pycache__
 .pytest_cache
 log/*
-.idea/
+.idea/
+
+build/
+dist/
+*.egg-info/

features/scraper.feature

@@ -10,11 +10,6 @@ Feature: scraper initial
       When we search for result from mindful
       Then the scraper will return "15" results
 
-  Scenario: run a collections search
-     Given we have itunes scraper installed
-      When we search for the topic topfreeapplications
-      Then the scraper will return "50" results
-
   Scenario: run a developer search
      Given we have itunes scraper installed
       When we search for the developer 384434796
@@ -39,4 +34,4 @@ Feature: scraper initial
   Scenario: read app details for pokemon
      Given we have itunes scraper installed
       When we search for the app with id "1094591345"
-      Then the title is "Pokémon GO"
+      Then the title is "Pokémon GO"

itunes_app_scraper/scraper.py

@@ -30,8 +30,8 @@ def get_app_ids_for_query(self, term, num=50, page=1, country="nl", lang="nl"):
 		Retrieve suggested app IDs for search query
 
 		:param str term:  Search query
-		:param int num:  Amount of items to return per page, default 50
-		:param int page:  Amount of pages to return
+		:param int|None num:  Amount of items to return per page, default 50
+		:param int|None page:  Amount of pages to return
 		:param str country:  Two-letter country code of store to search in,
 		                     default 'nl'
 		:param str lang:  Language code to search with, default 'nl'
@@ -44,7 +44,10 @@ def get_app_ids_for_query(self, term, num=50, page=1, country="nl", lang="nl"):
 		url = "https://search.itunes.apple.com/WebObjects/MZStore.woa/wa/search?clientApplication=Software&media=software&term="
 		url += quote_plus(term)
 
-		amount = int(num) * int(page)
+		if num is None or page is None:
+			amount = None
+		else:
+			amount = int(num) * int(page)
 
 		country = self.get_store_id_for_country(country)
 		headers = {

scraper_test.py

@@ -48,3 +48,14 @@ def test_country_code_does_not_exist():
     scraper = AppStoreScraper()
     with pytest.raises(AppStoreException, match="Country code not found for XZ"):
         scraper.get_store_id_for_country('xz')
+
+def test_query_multiple_pages():
+    query = "game"
+    scraper = AppStoreScraper()
+    results = set()
+    for page in range(1,4):
+        page_results = scraper.get_app_ids_for_query(query, country="us", lang="en", page=page)
+        if page_results:
+            [results.add(x) for x in page_results]
+            assert len(results) > (page-1)*50
+    print(f"Total results for query {query}: {len(results)}")