Developers: Apps are list inside of results key

itunes_app_scraper/scraper.py

@@ -57,7 +57,10 @@ def get_app_ids_for_query(self, term, num=50, page=1, country="nl", lang="nl", t
 		}
 
 		try:
-			result = requests.get(url, headers=headers, timeout=timeout).json()
+			with requests.get(url, headers=headers, timeout=(10, 30) if timeout is None else (timeout,timeout), stream=True) as r:
+				r.raise_for_status()
+				content = r.content
+				result = json.loads(content)
 		except ConnectionError as ce:
 			raise AppStoreException("Cannot connect to store: {0}".format(str(ce)))
 		except json.JSONDecodeError:
@@ -95,7 +98,10 @@ def get_app_ids_for_collection(self, collection="", category="", num=50, country
 
 
 		try:
-			result = requests.get(url, timeout=timeout).json()
+			with requests.get(url, timeout=(10, 30) if timeout is None else (timeout,timeout), stream=True) as r:
+				r.raise_for_status()
+				content = r.content
+				result = json.loads(content)
 		except json.JSONDecodeError:
 			raise AppStoreException("Could not parse app store response")
 
@@ -117,7 +123,10 @@ def get_apps_for_developer(self, developer_id, country="nl", lang="", timeout=No
 		url = "https://itunes.apple.com/lookup?id=%s&country=%s&entity=software" % (developer_id, country)
 
 		try:
-			result = requests.get(url, timeout=timeout).json()
+			with requests.get(url, timeout=(10, 30) if timeout is None else (timeout,timeout), stream=True) as r:
+				r.raise_for_status()
+				content = r.content
+				result = json.loads(content)
 		except json.JSONDecodeError:
 			raise AppStoreException("Could not parse app store response")
 
@@ -141,7 +150,7 @@ def get_app_ids_for_developer(self, developer_id, country="nl", lang="", timeout
 		"""
 		apps = self.get_apps_for_developer(developer_id, country=country, lang=lang, timeout=timeout)
 		if len(apps) > 0:
-			app_ids =[app["trackId"] for app in apps if app["wrapperType"] == "software"] 
+			app_ids =[app["trackId"] for app in apps["results"] if app["wrapperType"] == "software"] 
 		else:
 			return []
 		return app_ids
@@ -171,7 +180,9 @@ def get_similar_app_ids_for_app(self, app_id, country="nl", lang="nl", timeout=N
 			"Accept-Language": lang
 		}
 
-		result = requests.get(url, headers=headers, timeout=timeout).text
+		with requests.get(url, headers=headers, timeout=(10, 30) if timeout is None else (timeout,timeout), stream=True) as r:
+			r.raise_for_status()
+			result = r.text
 		if "customersAlsoBoughtApps" not in result:
 			return []
 
@@ -227,13 +238,19 @@ def get_app_details(self, app_id, country="nl", lang="", add_ratings=False, flat
 		try:
 			if sleep is not None:
 				time.sleep(sleep)
-			result = requests.get(url, timeout=timeout).json()
+			with requests.get(url, timeout=(10, 30) if timeout is None else (timeout, timeout), stream=True) as r:
+				r.raise_for_status()
+				content = r.content
+				result = json.loads(content)
 		except Exception:
 			try:
 				# handle the retry here.
 				# Take an extra sleep as back off and then retry the URL once.
 				time.sleep(2)
-				result = requests.get(url, timeout=timeout).json()
+				with requests.get(url, timeout=(10, 30) if timeout is None else (timeout,timeout), stream=True) as r:
+					r.raise_for_status()
+					content = r.content
+					result = json.loads(content)
 			except Exception:
 				raise AppStoreException("Could not parse app store response for ID %s" % app_id)
 
@@ -334,13 +351,17 @@ def get_app_ratings(self, app_id, countries=None, sleep=1, timeout=None):
 			try:
 				if sleep is not None:
 					time.sleep(sleep)
-				result = requests.get(url, headers=headers, timeout=timeout).text
+				with requests.get(url, headers=headers, timeout=(10, 30) if timeout is None else (timeout,timeout), stream=True) as r:
+					r.raise_for_status()
+					result = r.text
 			except Exception:
 				try:
 					# handle the retry here.
 					# Take an extra sleep as back off and then retry the URL once.
 					time.sleep(2)
-					result = requests.get(url, headers=headers, timeout=timeout).text
+					with requests.get(url, headers=headers, timeout=(10, 30) if timeout is None else timeout, stream=True) as r:
+						r.raise_for_status()
+						result = r.text
 				except Exception:
 					raise AppStoreException("Could not parse app store rating response for ID %s" % app_id)