Skip to content

Commit abf5cfe

Browse files
committed
fix: Added User-Agent to all request
Error is throwing fine (closes #2)
1 parent 82d0eb2 commit abf5cfe

File tree

1 file changed

+10
-5
lines changed

1 file changed

+10
-5
lines changed

main.py

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,10 @@
66

77
output_dir = 'output'
88

9+
headers = {
10+
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36"
11+
}
12+
913

1014
def createOutputFolderIfNotExist():
1115
if not os.path.exists(output_dir):
@@ -40,7 +44,7 @@ def spider(url, directory):
4044

4145
# while page <= max_pages:
4246
# url = 'https://www.inssia.com/viewtopic.php?f=35&t=23XXX&start=' + str(page)
43-
sourcecode = requests.get(url)
47+
sourcecode = requests.get(url, headers=headers)
4448
plaintext = sourcecode.text
4549
soup = BeautifulSoup(plaintext, "lxml")
4650

@@ -54,20 +58,21 @@ def spider(url, directory):
5458

5559
filename = link.strip('/').rsplit('/', 1)[-1] # to get the correct file name
5660

57-
res = requests.get(link, stream=True) # use requests to get the content of the images
61+
res = requests.get(link, headers=headers, stream=True) # use requests to get the content of the images
5862

5963
if res.status_code == 200:
6064
with open(f'{directory}/{filename}', 'wb') as f:
6165
shutil.copyfileobj(res.raw, f)
6266
# f.write(image) # write the image into a file
6367
print(
64-
f'{colored(f" ---#{image_count} SUCCESS:", "green")} - Image successfully Downloaded: {colored(filename, "blue")}')
68+
f'{colored(f" ---#{image_count} SUCCESS:", "green")}'
69+
f' - Image successfully Downloaded: {colored(filename, "blue")}')
6570

6671
image_count += 1
6772
else:
68-
print(f'{colored(" ---ERROR:", "red")} - Image Could not be retrieved')
73+
print(f'{colored(" ---ERROR:", "red")} - Image Could not be retrieved: {colored(filename, "blue")}')
6974

70-
print(f'Total Images found on {url} is: {colored(image_count, "orange")}')
75+
print(colored(f"Total Images found on {url} is: {image_count}", "yellow"))
7176

7277

7378
if __name__ == '__main__':

0 commit comments

Comments
 (0)