Skip to content

Commit 3050eb2

Browse files
GermeyGermey
committed
add new feature
1 parent 019dad4 commit 3050eb2

4 files changed

Lines changed: 63 additions & 54 deletions

File tree

‎CHANGELOG.md‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,19 @@
11
# Gerapy Playwright Changelog
22

3+
## 0.2.0 (2021-12-28)
4+
5+
- New Feature: Add support for:
6+
- Specifying `channel` for launching
7+
- Specifying `executablePath` for launching
8+
- Specifying `slowMo` for launching
9+
- Specifying `devtools` for launching
10+
- Specifying `--disable-extensions` in args for launching
11+
- Specifying `--hide-scrollbars` in args for launching
12+
- Specifying `--no-sandbox` in args for launching
13+
- Specifying `--disable-setuid-sandbox` in args for launching
14+
- Specifying `--disable-gpu` in args for launching
15+
- Update: change `GERAPY_PLAYWRIGHT_SLEEP` default to 0
16+
317
## 0.1.2 (2021-12-28)
418

519
- Fix: Add retrying logic for PlaywrightError

‎example/requirements.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
gerapy-playwright
1+
gerapy-playwright==0.1.2

‎gerapy_playwright/downloadermiddlewares.py‎

Lines changed: 42 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -107,12 +107,10 @@ def from_crawler(cls, crawler):
107107
GERAPY_PLAYWRIGHT_DEFAULT_USER_AGENT)
108108
cls.headless = settings.get(
109109
'GERAPY_PLAYWRIGHT_HEADLESS', GERAPY_PLAYWRIGHT_HEADLESS)
110-
# cls.dumpio = settings.get(
111-
# 'GERAPY_PLAYWRIGHT_DUMPIO', GERAPY_PLAYWRIGHT_DUMPIO)
112-
# cls.ignore_https_errors = settings.get('GERAPY_PLAYWRIGHT_IGNORE_HTTPS_ERRORS',
113-
# GERAPY_PLAYWRIGHT_IGNORE_HTTPS_ERRORS)
114-
# cls.slow_mo = settings.get(
115-
# 'GERAPY_PLAYWRIGHT_SLOW_MO', GERAPY_PLAYWRIGHT_SLOW_MO)
110+
cls.channel = settings.get(
111+
'GERAPY_PLAYWRIGHT_CHANNEL', GERAPY_PLAYWRIGHT_CHANNEL)
112+
cls.slow_mo = settings.get(
113+
'GERAPY_PLAYWRIGHT_SLOW_MO', GERAPY_PLAYWRIGHT_SLOW_MO)
116114
# cls.ignore_default_args = settings.get('GERAPY_PLAYWRIGHT_IGNORE_DEFAULT_ARGS',
117115
# GERAPY_PLAYWRIGHT_IGNORE_DEFAULT_ARGS)
118116
# cls.handle_sigint = settings.get(
@@ -121,24 +119,22 @@ def from_crawler(cls, crawler):
121119
# 'GERAPY_PLAYWRIGHT_HANDLE_SIGTERM', GERAPY_PLAYWRIGHT_HANDLE_SIGTERM)
122120
# cls.handle_sighup = settings.get(
123121
# 'GERAPY_PLAYWRIGHT_HANDLE_SIGHUP', GERAPY_PLAYWRIGHT_HANDLE_SIGHUP)
124-
# cls.auto_close = settings.get(
125-
# 'GERAPY_PLAYWRIGHT_AUTO_CLOSE', GERAPY_PLAYWRIGHT_AUTO_CLOSE)
126-
# cls.devtools = settings.get(
127-
# 'GERAPY_PLAYWRIGHT_DEVTOOLS', GERAPY_PLAYWRIGHT_DEVTOOLS)
128-
# cls.executable_path = settings.get(
129-
# 'GERAPY_PLAYWRIGHT_EXECUTABLE_PATH', GERAPY_PLAYWRIGHT_EXECUTABLE_PATH)
130-
# cls.disable_extensions = settings.get('GERAPY_PLAYWRIGHT_DISABLE_EXTENSIONS',
131-
# GERAPY_PLAYWRIGHT_DISABLE_EXTENSIONS)
132-
# cls.hide_scrollbars = settings.get(
133-
# 'GERAPY_PLAYWRIGHT_HIDE_SCROLLBARS', GERAPY_PLAYWRIGHT_HIDE_SCROLLBARS)
134-
# cls.mute_audio = settings.get(
135-
# 'GERAPY_PLAYWRIGHT_MUTE_AUDIO', GERAPY_PLAYWRIGHT_MUTE_AUDIO)
136-
# cls.no_sandbox = settings.get(
137-
# 'GERAPY_PLAYWRIGHT_NO_SANDBOX', GERAPY_PLAYWRIGHT_NO_SANDBOX)
138-
# cls.disable_setuid_sandbox = settings.get('GERAPY_PLAYWRIGHT_DISABLE_SETUID_SANDBOX',
139-
# GERAPY_PLAYWRIGHT_DISABLE_SETUID_SANDBOX)
140-
# cls.disable_gpu = settings.get(
141-
# 'GERAPY_PLAYWRIGHT_DISABLE_GPU', GERAPY_PLAYWRIGHT_DISABLE_GPU)
122+
cls.devtools = settings.get(
123+
'GERAPY_PLAYWRIGHT_DEVTOOLS', GERAPY_PLAYWRIGHT_DEVTOOLS)
124+
cls.executable_path = settings.get(
125+
'GERAPY_PLAYWRIGHT_EXECUTABLE_PATH', GERAPY_PLAYWRIGHT_EXECUTABLE_PATH)
126+
cls.disable_extensions = settings.get('GERAPY_PLAYWRIGHT_DISABLE_EXTENSIONS',
127+
GERAPY_PLAYWRIGHT_DISABLE_EXTENSIONS)
128+
cls.hide_scrollbars = settings.get(
129+
'GERAPY_PLAYWRIGHT_HIDE_SCROLLBARS', GERAPY_PLAYWRIGHT_HIDE_SCROLLBARS)
130+
cls.mute_audio = settings.get(
131+
'GERAPY_PLAYWRIGHT_MUTE_AUDIO', GERAPY_PLAYWRIGHT_MUTE_AUDIO)
132+
cls.no_sandbox = settings.get(
133+
'GERAPY_PLAYWRIGHT_NO_SANDBOX', GERAPY_PLAYWRIGHT_NO_SANDBOX)
134+
cls.disable_setuid_sandbox = settings.get('GERAPY_PLAYWRIGHT_DISABLE_SETUID_SANDBOX',
135+
GERAPY_PLAYWRIGHT_DISABLE_SETUID_SANDBOX)
136+
cls.disable_gpu = settings.get(
137+
'GERAPY_PLAYWRIGHT_DISABLE_GPU', GERAPY_PLAYWRIGHT_DISABLE_GPU)
142138
cls.download_timeout = settings.get('GERAPY_PLAYWRIGHT_DOWNLOAD_TIMEOUT',
143139
settings.get('DOWNLOAD_TIMEOUT', GERAPY_PLAYWRIGHT_DOWNLOAD_TIMEOUT))
144140
# cls.ignore_resource_types = settings.get('GERAPY_PLAYWRIGHT_IGNORE_RESOURCE_TYPES',
@@ -176,35 +172,36 @@ async def _process_request(self, request, spider):
176172

177173
options = {
178174
'headless': self.headless,
175+
'args': [],
179176
}
180-
# if self.executable_path:
181-
# options['executablePath'] = self.executable_path
182-
# if self.ignore_https_errors:
183-
# options['ignoreHTTPSErrors'] = self.ignore_https_errors
184-
# if self.slow_mo:
185-
# options['slowMo'] = self.slow_mo
186-
# if self.ignore_default_args:
177+
if self.executable_path is not None:
178+
options['executablePath'] = self.executable_path
179+
if self.slow_mo is not None:
180+
options['slowMo'] = self.slow_mo
181+
if self.devtools is not None:
182+
options['devtools'] = self.devtools
183+
if self.channel is not None:
184+
options['channel'] = self.channel
185+
# if self.ignore_default_args is not None:
187186
# options['ignoreDefaultArgs'] = self.ignore_default_args
188187
# if self.handle_sigint:
189188
# options['handleSIGINT'] = self.handle_sigint
190189
# if self.handle_sigterm:
191190
# options['handleSIGTERM'] = self.handle_sigterm
192191
# if self.handle_sighup:
193192
# options['handleSIGHUP'] = self.handle_sighup
194-
# if self.auto_close:
195-
# options['autoClose'] = self.auto_close
196-
# if self.disable_extensions:
197-
# options['args'].append('--disable-extensions')
198-
# if self.hide_scrollbars:
199-
# options['args'].append('--hide-scrollbars')
200-
# if self.mute_audio:
201-
# options['args'].append('--mute-audio')
202-
# if self.no_sandbox:
203-
# options['args'].append('--no-sandbox')
204-
# if self.disable_setuid_sandbox:
205-
# options['args'].append('--disable-setuid-sandbox')
206-
# if self.disable_gpu:
207-
# options['args'].append('--disable-gpu')
193+
if self.disable_extensions is not None:
194+
options['args'].append('--disable-extensions')
195+
if self.hide_scrollbars is not None:
196+
options['args'].append('--hide-scrollbars')
197+
if self.mute_audio is not None:
198+
options['args'].append('--mute-audio')
199+
if self.no_sandbox is not None:
200+
options['args'].append('--no-sandbox')
201+
if self.disable_setuid_sandbox is not None:
202+
options['args'].append('--disable-setuid-sandbox')
203+
if self.disable_gpu is not None:
204+
options['args'].append('--disable-gpu')
208205

209206
# pretend as normal browser
210207
_pretend = self.pretend # get global pretend setting

‎gerapy_playwright/settings.py‎

Lines changed: 6 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -15,16 +15,14 @@
1515

1616
# playwright settings
1717
GERAPY_PLAYWRIGHT_HEADLESS = True
18+
GERAPY_PLAYWRIGHT_CHANNEL = None
1819
GERAPY_PLAYWRIGHT_EXECUTABLE_PATH = None
19-
GERAPY_PLAYWRIGHT_IGNORE_HTTPS_ERRORS = False
2020
GERAPY_PLAYWRIGHT_SLOW_MO = None
21-
GERAPY_PLAYWRIGHT_IGNORE_DEFAULT_ARGS = False
22-
GERAPY_PLAYWRIGHT_HANDLE_SIGINT = True
23-
GERAPY_PLAYWRIGHT_HANDLE_SIGTERM = True
24-
GERAPY_PLAYWRIGHT_HANDLE_SIGHUP = True
25-
GERAPY_PLAYWRIGHT_DUMPIO = False
21+
# GERAPY_PLAYWRIGHT_IGNORE_DEFAULT_ARGS = False
22+
# GERAPY_PLAYWRIGHT_HANDLE_SIGINT = True
23+
# GERAPY_PLAYWRIGHT_HANDLE_SIGTERM = True
24+
# GERAPY_PLAYWRIGHT_HANDLE_SIGHUP = True
2625
GERAPY_PLAYWRIGHT_DEVTOOLS = False
27-
GERAPY_PLAYWRIGHT_AUTO_CLOSE = True
2826
GERAPY_PLAYWRIGHT_PRETEND = True
2927

3028
# playwright args
@@ -41,5 +39,5 @@
4139
# ``manifest``, ``other``.
4240
GERAPY_PLAYWRIGHT_IGNORE_RESOURCE_TYPES = []
4341
GERAPY_PLAYWRIGHT_SCREENSHOT = None
44-
GERAPY_PLAYWRIGHT_SLEEP = 1
42+
GERAPY_PLAYWRIGHT_SLEEP = 0
4543
GERAPY_ENABLE_REQUEST_INTERCEPTION = False

0 commit comments

Comments
 (0)