Skip to content

Commit caf8891

Browse files
committed
Merge branch 'browser-settings-refactor' into dev
2 parents 0ad4090 + fbbe9cf commit caf8891

File tree

112 files changed

+3053
-1184
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

112 files changed

+3053
-1184
lines changed

.github/workflows/test-stack-reusable-workflow.yml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -292,8 +292,8 @@ jobs:
292292
293293
- name: Specific tests in built container for Selenium
294294
run: |
295-
docker run --rm -e "WEBDRIVER_URL=http://selenium:4444/wd/hub" --network changedet-network test-changedetectionio bash -c 'cd changedetectionio;pytest tests/fetchers/test_content.py && pytest tests/test_errorhandling.py'
296-
295+
docker run --name "changedet" --hostname changedet --rm -e "FLASK_SERVER_NAME=changedet" -e "WEBDRIVER_URL=http://selenium:4444/wd/hub" --network changedet-network test-changedetectionio bash -c 'cd changedetectionio; pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/fetchers/test_content.py'
296+
docker run --name "changedet" --hostname changedet --rm -e "FLASK_SERVER_NAME=changedet" -e "WEBDRIVER_URL=http://selenium:4444/wd/hub" --network changedet-network test-changedetectionio bash -c 'cd changedetectionio; pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/test_errorhandling.py'
297297
298298
# SMTP tests
299299
smtp-tests:

changedetectionio/__init__.py

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,6 @@
1010
from loguru import logger
1111
import getopt
1212
import logging
13-
import os
1413
import platform
1514
import signal
1615
import threading

changedetectionio/api/Import.py

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -154,11 +154,10 @@ def post(self):
154154
if extras['processor'] not in available:
155155
return f"Invalid processor '{extras['processor']}'. Available processors: {', '.join(available)}", 400
156156

157-
# Validate fetch_backend if provided
157+
# Validate fetch_backend if provided (legacy API compat — still accepted, stored as-is)
158158
if 'fetch_backend' in extras:
159159
from changedetectionio.content_fetchers import available_fetchers
160160
available = [f[0] for f in available_fetchers()]
161-
# Also allow 'system' and extra_browser_* patterns
162161
is_valid = (
163162
extras['fetch_backend'] == 'system' or
164163
extras['fetch_backend'] in available or
@@ -167,6 +166,14 @@ def post(self):
167166
if not is_valid:
168167
return f"Invalid fetch_backend '{extras['fetch_backend']}'. Available: system, {', '.join(available)}", 400
169168

169+
# Validate browser_profile if provided
170+
if 'browser_profile' in extras:
171+
from changedetectionio.model.browser_profile import get_builtin_profiles, RESERVED_MACHINE_NAMES
172+
store_profiles = self.datastore.data['settings']['application'].get('browser_profiles', {})
173+
known = set(get_builtin_profiles().keys()) | set(store_profiles.keys()) | {'system', None}
174+
if extras['browser_profile'] not in known:
175+
return f"Invalid browser_profile '{extras['browser_profile']}'. Available: {', '.join(str(k) for k in known)}", 400
176+
170177
# Validate notification_urls if provided
171178
if 'notification_urls' in extras:
172179
from wtforms import ValidationError

changedetectionio/api/Tags.py

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,9 @@ def queue_watches_background():
8585
# Create clean tag dict without Watch-specific fields
8686
clean_tag = {k: v for k, v in tag.items() if k not in watch_only_fields}
8787

88+
# fetch_backend is a legacy field superseded by browser_profile — omit from API response
89+
clean_tag.pop('fetch_backend', None)
90+
8891
return clean_tag
8992

9093
@auth.check_token

changedetectionio/api/Watch.py

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -105,6 +105,9 @@ def get(self, uuid):
105105
watch['viewed'] = watch_obj.viewed
106106
watch['link'] = watch_obj.link,
107107

108+
# fetch_backend is a legacy field superseded by browser_profile — omit from API response
109+
watch.pop('fetch_backend', None)
110+
108111
return watch
109112

110113
@auth.check_token

changedetectionio/blueprint/browser_steps/__init__.py

Lines changed: 12 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -208,28 +208,23 @@ async def start_browsersteps_session(watch_uuid):
208208
browsersteps_start_session = {'start_time': time.time()}
209209

210210
# Build proxy dict first — needed by both the CDP path and fetcher-specific launchers
211-
proxy_id = datastore.get_preferred_proxy_for_watch(uuid=watch_uuid)
211+
proxy_url = datastore.get_proxy_url_for_watch(uuid=watch_uuid)
212212
proxy = None
213-
if proxy_id:
214-
proxy_url = datastore.proxy_list.get(proxy_id, {}).get('url')
215-
if proxy_url:
216-
from urllib.parse import urlparse
217-
parsed = urlparse(proxy_url)
218-
proxy = {'server': proxy_url}
219-
if parsed.username:
220-
proxy['username'] = parsed.username
221-
if parsed.password:
222-
proxy['password'] = parsed.password
223-
logger.debug(f"Browser Steps: UUID {watch_uuid} selected proxy {proxy_url}")
213+
if proxy_url:
214+
from urllib.parse import urlparse
215+
parsed = urlparse(proxy_url)
216+
proxy = {'server': proxy_url}
217+
if parsed.username:
218+
proxy['username'] = parsed.username
219+
if parsed.password:
220+
proxy['password'] = parsed.password
221+
logger.debug(f"Browser Steps: UUID {watch_uuid} selected proxy {proxy_url}")
224222

225223
# Resolve the fetcher class for this watch so we can ask it to launch its own browser
226224
# if it supports that (e.g. CloakBrowser, which runs locally rather than via CDP)
227225
watch = datastore.data['watching'][watch_uuid]
228226
from changedetectionio import content_fetchers
229-
fetcher_name = watch.get_fetch_backend or 'system'
230-
if fetcher_name == 'system':
231-
fetcher_name = datastore.data['settings']['application'].get('fetch_backend', 'html_requests')
232-
fetcher_class = getattr(content_fetchers, fetcher_name, None)
227+
fetcher_class = content_fetchers.get_fetcher(watch.effective_browser_profile.fetch_backend)
233228

234229
browser = None
235230
playwright_context = None
@@ -241,7 +236,7 @@ async def start_browsersteps_session(watch_uuid):
241236
result = await fetcher_class.get_browsersteps_browser(proxy=proxy, keepalive_ms=keepalive_ms)
242237
if result is not None:
243238
browser, playwright_context = result
244-
logger.debug(f"Browser Steps: using fetcher-specific browser for '{fetcher_name}'")
239+
logger.debug(f"Browser Steps: using fetcher-specific browser for '{fetcher_class.__name__}'")
245240

246241
# Default: connect to the remote Playwright/sockpuppetbrowser via CDP
247242
if browser is None:

changedetectionio/blueprint/check_proxies/__init__.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,8 @@ def long_task(uuid, preferred_proxy):
4646
watch_uuid=uuid
4747
)
4848

49-
asyncio.run(update_handler.call_browser(preferred_proxy_id=preferred_proxy))
49+
update_handler.preferred_proxy_override = preferred_proxy
50+
asyncio.run(update_handler.call_browser())
5051
# title, size is len contents not len xfer
5152
except content_fetcher_exceptions.Non200ErrorCodeReceived as e:
5253
if e.status_code == 404:

changedetectionio/blueprint/imports/importer.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -175,9 +175,9 @@ def run(self,
175175
dynamic_wachet = str(data.get('dynamic wachet', '')).strip().lower() # Convert bool to str to cover all cases
176176
# libreoffice and others can have it as =FALSE() =TRUE(), or bool(true)
177177
if 'true' in dynamic_wachet or dynamic_wachet == '1':
178-
extras['fetch_backend'] = 'html_webdriver'
178+
extras['browser_profile'] = 'browser_chromeplaywright'
179179
elif 'false' in dynamic_wachet or dynamic_wachet == '0':
180-
extras['fetch_backend'] = 'html_requests'
180+
extras['browser_profile'] = 'direct_http_requests'
181181

182182
if data.get('xpath'):
183183
# @todo split by || ?

changedetectionio/blueprint/settings/__init__.py

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,9 @@
1515
def construct_blueprint(datastore: ChangeDetectionStore):
1616
settings_blueprint = Blueprint('settings', __name__, template_folder="templates")
1717

18+
from changedetectionio.blueprint.settings.browser_profile import construct_blueprint as construct_browser_profile_blueprint
19+
settings_blueprint.register_blueprint(construct_browser_profile_blueprint(datastore), url_prefix='/browsers')
20+
1821
@settings_blueprint.route("", methods=['GET', "POST"])
1922
@login_optionally_required
2023
def settings_page():
Lines changed: 200 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,200 @@
1+
import flask_login
2+
from flask import Blueprint, render_template, request, redirect, url_for, flash
3+
from flask_babel import gettext
4+
5+
from changedetectionio.store import ChangeDetectionStore
6+
from changedetectionio.auth_decorator import login_optionally_required
7+
8+
9+
def construct_blueprint(datastore: ChangeDetectionStore):
10+
settings_browser_profile_blueprint = Blueprint(
11+
'settings_browsers',
12+
__name__,
13+
template_folder="templates"
14+
)
15+
16+
def _render_index(browser_profile_form=None, editing_machine_name=None):
17+
from changedetectionio import forms
18+
from changedetectionio import content_fetchers as cf
19+
from changedetectionio.model.browser_profile import BrowserProfile, RESERVED_MACHINE_NAMES
20+
21+
# Only browser-capable fetchers are valid profile types
22+
fetcher_choices = cf.available_browser_fetchers()
23+
if browser_profile_form is None:
24+
browser_profile_form = forms.BrowserProfileForm()
25+
browser_profile_form.fetch_backend.choices = fetcher_choices
26+
27+
fetcher_supports_screenshots = {name: True for name, _ in fetcher_choices}
28+
fetcher_requires_connection_url = {name: True for name, cls in cf.FETCHERS.items()
29+
if getattr(cls, 'requires_connection_url', False)}
30+
31+
# Table shows default built-in profiles first, then user-created profiles
32+
store_profiles = datastore.data['settings']['application'].get('browser_profiles', {})
33+
user_profiles = dict(cf.DEFAULT_BROWSER_PROFILES)
34+
for machine_name, raw in store_profiles.items():
35+
try:
36+
user_profiles[machine_name] = BrowserProfile(**raw) if isinstance(raw, dict) else raw
37+
except Exception:
38+
pass
39+
40+
current_default = datastore.data['settings']['application'].get('browser_profile') or 'direct_http_requests'
41+
42+
return render_template(
43+
"browser_profiles.html",
44+
browser_profiles=user_profiles,
45+
browser_profile_form=browser_profile_form,
46+
reserved_browser_profile_names=RESERVED_MACHINE_NAMES,
47+
fetcher_choices=fetcher_choices,
48+
fetcher_supports_screenshots=fetcher_supports_screenshots,
49+
fetcher_requires_connection_url=fetcher_requires_connection_url,
50+
current_default_profile=current_default,
51+
editing_machine_name=editing_machine_name,
52+
)
53+
54+
@settings_browser_profile_blueprint.route("", methods=['GET'])
55+
@login_optionally_required
56+
def index():
57+
return _render_index()
58+
59+
@settings_browser_profile_blueprint.route("/<string:machine_name>/edit", methods=['GET'])
60+
@login_optionally_required
61+
def edit(machine_name):
62+
from changedetectionio import forms
63+
from changedetectionio.model.browser_profile import BrowserProfile, RESERVED_MACHINE_NAMES
64+
65+
if machine_name in RESERVED_MACHINE_NAMES:
66+
flash(gettext("Built-in browser profiles cannot be edited."), 'error')
67+
return redirect(url_for('settings.settings_browsers.index'))
68+
69+
store_profiles = datastore.data['settings']['application'].get('browser_profiles', {})
70+
raw = store_profiles.get(machine_name)
71+
if raw is None:
72+
flash(gettext("Browser profile not found."), 'error')
73+
return redirect(url_for('settings.settings_browsers.index'))
74+
75+
profile = BrowserProfile(**raw) if isinstance(raw, dict) else raw
76+
form = forms.BrowserProfileForm(data=profile.model_dump())
77+
return _render_index(browser_profile_form=form, editing_machine_name=machine_name)
78+
79+
@settings_browser_profile_blueprint.route("/save", methods=['POST'])
80+
@login_optionally_required
81+
def save():
82+
from changedetectionio import forms
83+
from changedetectionio import content_fetchers as cf
84+
from changedetectionio.model.browser_profile import BrowserProfile, RESERVED_MACHINE_NAMES
85+
86+
fetcher_choices = [(name, desc) for name, desc in cf.available_fetchers()]
87+
browser_profile_form = forms.BrowserProfileForm(formdata=request.form)
88+
browser_profile_form.fetch_backend.choices = fetcher_choices
89+
90+
if not browser_profile_form.validate():
91+
flash(gettext("Browser profile error: {}").format(
92+
'; '.join(str(e) for errs in browser_profile_form.errors.values() for e in errs)
93+
), 'error')
94+
return redirect(url_for('settings.settings_browsers.index'))
95+
96+
name = browser_profile_form.name.data.strip()
97+
machine_name = BrowserProfile.machine_name_from_str(name)
98+
99+
if machine_name in RESERVED_MACHINE_NAMES:
100+
flash(gettext("Cannot use reserved profile name '{}'. Please choose a different name.").format(name), 'error')
101+
return redirect(url_for('settings.settings_browsers.index'))
102+
103+
original_machine_name = request.form.get('original_machine_name', '').strip()
104+
store_profiles = datastore.data['settings']['application'].setdefault('browser_profiles', {})
105+
106+
if machine_name != original_machine_name and machine_name in store_profiles:
107+
flash(gettext("A browser profile named '{}' already exists.").format(name), 'error')
108+
return redirect(url_for('settings.settings_browsers.index'))
109+
110+
profile_data = {
111+
'name': name,
112+
'fetch_backend': browser_profile_form.fetch_backend.data,
113+
'browser_connection_url': browser_profile_form.browser_connection_url.data or None,
114+
'viewport_width': browser_profile_form.viewport_width.data or 1280,
115+
'viewport_height': browser_profile_form.viewport_height.data or 1000,
116+
'block_images': bool(browser_profile_form.block_images.data),
117+
'block_fonts': bool(browser_profile_form.block_fonts.data),
118+
'ignore_https_errors': bool(browser_profile_form.ignore_https_errors.data),
119+
'user_agent': browser_profile_form.user_agent.data or None,
120+
'locale': browser_profile_form.locale.data or None,
121+
'custom_headers': browser_profile_form.custom_headers.data or '',
122+
'is_builtin': False,
123+
}
124+
125+
try:
126+
BrowserProfile(**profile_data)
127+
except Exception as e:
128+
flash(gettext("Browser profile validation error: {}").format(str(e)), 'error')
129+
return redirect(url_for('settings.settings_browsers.index'))
130+
131+
# Handle rename: remove old key, cascade-update watches and tags
132+
if original_machine_name and original_machine_name != machine_name and original_machine_name in store_profiles:
133+
del store_profiles[original_machine_name]
134+
for watch in datastore.data['watching'].values():
135+
if watch.get('browser_profile') == original_machine_name:
136+
watch['browser_profile'] = machine_name
137+
for tag in datastore.data.get('settings', {}).get('application', {}).get('tags', {}).values():
138+
if tag.get('browser_profile') == original_machine_name:
139+
tag['browser_profile'] = machine_name
140+
141+
store_profiles[machine_name] = profile_data
142+
datastore.commit()
143+
flash(gettext("Browser profile '{}' saved.").format(name), 'notice')
144+
return redirect(url_for('settings.settings_browsers.index'))
145+
146+
@settings_browser_profile_blueprint.route("/<string:machine_name>/delete", methods=['GET'])
147+
@login_optionally_required
148+
def delete(machine_name):
149+
from changedetectionio.model.browser_profile import RESERVED_MACHINE_NAMES
150+
151+
if machine_name in RESERVED_MACHINE_NAMES:
152+
flash(gettext("Built-in browser profiles cannot be deleted."), 'error')
153+
return redirect(url_for('settings.settings_browsers.index'))
154+
155+
store_profiles = datastore.data['settings']['application'].get('browser_profiles', {})
156+
if machine_name not in store_profiles:
157+
flash(gettext("Browser profile not found."), 'error')
158+
return redirect(url_for('settings.settings_browsers.index'))
159+
160+
raw = store_profiles[machine_name]
161+
profile_name = raw.get('name', machine_name) if isinstance(raw, dict) else machine_name
162+
163+
for watch in datastore.data['watching'].values():
164+
if watch.get('browser_profile') == machine_name:
165+
watch['browser_profile'] = None
166+
167+
for tag in datastore.data.get('settings', {}).get('application', {}).get('tags', {}).values():
168+
if tag.get('browser_profile') == machine_name:
169+
tag['browser_profile'] = None
170+
171+
if datastore.data['settings']['application'].get('browser_profile') == machine_name:
172+
datastore.data['settings']['application']['browser_profile'] = None
173+
174+
del store_profiles[machine_name]
175+
datastore.commit()
176+
flash(gettext("Browser profile '{}' deleted.").format(profile_name), 'notice')
177+
return redirect(url_for('settings.settings_browsers.index'))
178+
179+
@settings_browser_profile_blueprint.route("/set-default", methods=['POST'])
180+
@login_optionally_required
181+
def set_default():
182+
from changedetectionio import content_fetchers as cf
183+
184+
machine_name = request.form.get('machine_name', '').strip()
185+
if not machine_name:
186+
flash(gettext("No profile specified."), 'error')
187+
return redirect(url_for('settings.settings_browsers.index'))
188+
189+
from changedetectionio.model.browser_profile import get_profile
190+
store_profiles = datastore.data['settings']['application'].get('browser_profiles', {})
191+
if get_profile(machine_name, store_profiles) is None:
192+
flash(gettext("Unknown browser profile '{}'.").format(machine_name), 'error')
193+
return redirect(url_for('settings.settings_browsers.index'))
194+
195+
datastore.data['settings']['application']['browser_profile'] = machine_name
196+
datastore.commit()
197+
flash(gettext("Default browser profile set to '{}'.").format(machine_name), 'notice')
198+
return redirect(url_for('settings.settings_browsers.index'))
199+
200+
return settings_browser_profile_blueprint

0 commit comments

Comments
 (0)