New charts url (#9) replaces collections

* Add optional timeout option to all requests

* Add description of timeout option

* Since API already returns all apps for a developer, add function to return all results

* Update to new charts url

* Rever to tabs for consistency with old library

* Seems the collections need a new name to use charts API

* Now API only returns IDs

* Requires two letter country code, upper or lower

* Requires two letter country code, upper or lower

* merge fixes

* update tests

* fix get_app_ids_for_developers

* add collection test

---------

Co-authored-by: Dale Wahl <[email protected]>

Author: ddxv

itunes_app_scraper/scraper.py

@@ -89,29 +89,31 @@ def get_app_ids_for_collection(self, collection="", category="", num=50, country
 		if not collection:
 			collection = AppStoreCollections.TOP_FREE_IOS
 
-		country = self.get_store_id_for_country(country)
-		params = (collection, category, num, country)
-		url = "http://ax.itunes.apple.com/WebObjects/MZStoreServices.woa/ws/RSS/%s/%s/limit=%s/json?s=%s" % params
+		country = country.lower()
+		params = (country, category, collection, num)
+		url = "https://itunes.apple.com/WebObjects/MZStoreServices.woa/ws/charts?cc=%s&g=%s&name=%s&limit=%s" % params
+
 
 		try:
 			result = requests.get(url, timeout=timeout).json()
 		except json.JSONDecodeError:
 			raise AppStoreException("Could not parse app store response")
 
-		return [entry["id"]["attributes"]["im:id"] for entry in result["feed"]["entry"]]
+		return result["resultIds"]
 
-	def get_app_ids_for_developer(self, developer_id, country="nl", lang="", timeout=None):
+	def get_apps_for_developer(self, developer_id, country="nl", lang="", timeout=None):
 		"""
-		Retrieve App IDs linked to given developer
+		Retrieve Apps linked to given developer
 
 		:param int developer_id:  Developer ID
 		:param str country:  Two-letter country code for the store to search in.
 		                     Defaults to 'nl'.
 		:param str lang: Dummy argument for compatibility. Unused.
 		:param int timeout: Seconds to wait for response before stopping.
 
-		:return list:  List of App IDs linked to developer
+		:return list[dict]:  List of Apps linked to developer
 		"""
+		country = country.lower()
 		url = "https://itunes.apple.com/lookup?id=%s&country=%s&entity=software" % (developer_id, country)
 
 		try:
@@ -120,11 +122,31 @@ def get_app_ids_for_developer(self, developer_id, country="nl", lang="", timeout
 			raise AppStoreException("Could not parse app store response")
 
 		if "results" in result:
-			return [app["trackId"] for app in result["results"] if app["wrapperType"] == "software"]
+			return [app for app in result["results"] if app["wrapperType"] == "software"]
 		else:
 			# probably an invalid developer ID
 			return []
 
+	def get_app_ids_for_developer(self, developer_id, country="nl", lang="", timeout=None):
+		"""
+		Retrieve App IDs linked to given developer
+
+		:param int developer_id:  Developer ID
+		:param str country:  Two-letter country code for the store to search in.
+		                     Defaults to 'nl'.
+		:param str lang: Dummy argument for compatibility. Unused.
+		:param int timeout: Seconds to wait for response before stopping.
+
+		:return list:  List of App IDs linked to developer
+		"""
+		apps = self.get_apps_for_developer(developer_id, country=country, lang=lang, timeout=timeout)
+		if len(apps) > 0:
+			app_ids =[app["trackId"] for app in apps if app["wrapperType"] == "software"] 
+		else:
+			return []
+		return app_ids
+		
+
 	def get_similar_app_ids_for_app(self, app_id, country="nl", lang="nl", timeout=None):
 		"""
 		Retrieve list of App IDs of apps similar to given app

itunes_app_scraper/util.py

@@ -25,23 +25,18 @@ class AppStoreCollections:
 	"""
 	App store collection IDs
 
-	Borrowed from https://github.com/facundoolano/app-store-scraper. These are
+	Based on https://itunes.apple.com/WebObjects/MZStoreServices.woa/ws/genres?id=6000. These are
 	the various collections displayed in the app store, usually on the front
 	page.
 	"""
-	TOP_MAC = 'topmacapps'
-	TOP_FREE_MAC = 'topfreemacapps'
-	TOP_GROSSING_MAC = 'topgrossingmacapps'
+	TOP_FREE_MAC = 'freeMacAppsV2'
 	TOP_PAID_MAC = 'toppaidmacapps'
-	NEW_IOS = 'newapplications'
-	NEW_FREE_IOS = 'newfreeapplications'
-	NEW_PAID_IOS = 'newpaidapplications'
-	TOP_FREE_IOS = 'topfreeapplications'
-	TOP_FREE_IPAD = 'topfreeipadapplications'
-	TOP_GROSSING_IOS = 'topgrossingapplications'
-	TOP_GROSSING_IPAD = 'topgrossingipadapplications'
-	TOP_PAID_IOS = 'toppaidapplications'
-	TOP_PAID_IPAD = 'toppaidipadapplications'
+	TOP_FREE_IOS = 'freeAppsV2'
+	TOP_FREE_IPAD = 'freeIpadApplications'
+	TOP_GROSSING_IOS = 'appsByRevenue'
+	TOP_GROSSING_IPAD = 'ipadAppsByRevenue'
+	TOP_PAID_IOS = 'paidApplications'
+	TOP_PAID_IPAD = 'paidIpadApplications'
 
 class AppStoreCategories:
 	"""

scraper_test.py

@@ -58,4 +58,9 @@ def test_query_multiple_pages():
         if page_results:
             [results.add(x) for x in page_results]
             assert len(results) > (page-1)*50
-    print(f"Total results for query {query}: {len(results)}")
+    print(f"Total results for query {query}: {len(results)}")
+
+def test_get_app_ids_for_collection():
+    scraper = AppStoreScraper()
+    results = scraper.get_app_ids_for_collection(AppStoreCollections.TOP_FREE_IOS, country="us", lang="en")
+    assert len(results) > 0

util_test.py

@@ -15,7 +15,7 @@ def test_category_does_not_exist():
 
 def test_collection_exists():
     collection = AppStoreCollections()
-    assert collection.NEW_IOS == 'newapplications'
+    assert collection.TOP_FREE_IOS == 'freeAppsV2'
 
 def test_collection_does_not_exist():
     collection = AppStoreCollections()
@@ -24,5 +24,5 @@ def test_collection_does_not_exist():
 
 def test_app_utils():
     utils = AppStoreUtils()
-    json_object = json.loads(utils.get_entries(AppStoreCollections()))
-    assert "names" in json_object
+    collections = utils.get_entries(AppStoreCollections())
+    assert "TOP_FREE_IOS" in collections