From b50226e6f7a07b5c4fbbc2566c3d66109cb22860 Mon Sep 17 00:00:00 2001 From: vringar Date: Tue, 6 Aug 2024 17:56:08 +0200 Subject: [PATCH 1/2] refactor(config): always enable extension --- openwpm/browser_manager.py | 5 +-- openwpm/config.py | 1 - openwpm/deploy_browsers/deploy_firefox.py | 51 +++++++++++------------ 3 files changed, 25 insertions(+), 32 deletions(-) diff --git a/openwpm/browser_manager.py b/openwpm/browser_manager.py index d9c65034d..2d929a611 100644 --- a/openwpm/browser_manager.py +++ b/openwpm/browser_manager.py @@ -739,10 +739,7 @@ def run(self) -> None: self.crash_recovery, ) - extension_socket: Optional[ClientSocket] = None - - if self.browser_params.extension_enabled: - extension_socket = self._start_extension(browser_profile_path) + extension_socket = self._start_extension(browser_profile_path) self.logger.debug( "BROWSER %i: BrowserManager ready." % self.browser_params.browser_id diff --git a/openwpm/config.py b/openwpm/config.py index 1898d3901..674d8dab6 100644 --- a/openwpm/config.py +++ b/openwpm/config.py @@ -77,7 +77,6 @@ class BrowserParams(DataClassJsonMixin): to customize behaviour of an individual browser """ - extension_enabled: bool = True cookie_instrument: bool = True js_instrument: bool = False js_instrument_settings: List[Union[str, dict]] = field( diff --git a/openwpm/deploy_browsers/deploy_firefox.py b/openwpm/deploy_browsers/deploy_firefox.py index 5e0647065..c4797b794 100755 --- a/openwpm/deploy_browsers/deploy_firefox.py +++ b/openwpm/deploy_browsers/deploy_firefox.py @@ -98,25 +98,24 @@ def deploy_firefox( # because status_queue is read off no matter what. status_queue.put(("STATUS", "Display", (display_pid, display_port))) - if browser_params.extension_enabled: - # Write config file - extension_config: Dict[str, Any] = dict() - extension_config.update(browser_params.to_dict()) - extension_config["logger_address"] = manager_params.logger_address - extension_config["storage_controller_address"] = ( - manager_params.storage_controller_address - ) - extension_config["testing"] = manager_params.testing - ext_config_file = browser_profile_path / "browser_params.json" - with open(ext_config_file, "w") as f: - json.dump(extension_config, f, cls=ConfigEncoder) - logger.debug( - "BROWSER %i: Saved extension config file to: %s" - % (browser_params.browser_id, ext_config_file) - ) + # Write config file + extension_config: Dict[str, Any] = dict() + extension_config.update(browser_params.to_dict()) + extension_config["logger_address"] = manager_params.logger_address + extension_config["storage_controller_address"] = ( + manager_params.storage_controller_address + ) + extension_config["testing"] = manager_params.testing + ext_config_file = browser_profile_path / "browser_params.json" + with open(ext_config_file, "w") as f: + json.dump(extension_config, f, cls=ConfigEncoder) + logger.debug( + "BROWSER %i: Saved extension config file to: %s" + % (browser_params.browser_id, ext_config_file) + ) - # TODO restore detailed logging - # fo.set_preference("extensions.@openwpm.sdk.console.logLevel", "all") + # TODO restore detailed logging + # fo.set_preference("extensions.@openwpm.sdk.console.logLevel", "all") # Configure privacy settings configure_firefox.privacy(browser_params, fo) @@ -153,15 +152,13 @@ def deploy_firefox( ), ) - # Add extension - if browser_params.extension_enabled: - # Install extension - ext_loc = os.path.join(root_dir, "../../Extension/openwpm.xpi") - ext_loc = os.path.normpath(ext_loc) - driver.install_addon(ext_loc, temporary=True) - logger.debug( - "BROWSER %i: OpenWPM Firefox extension loaded" % browser_params.browser_id - ) + # Install extension + ext_loc = os.path.join(root_dir, "../../Extension/openwpm.xpi") + ext_loc = os.path.normpath(ext_loc) + driver.install_addon(ext_loc, temporary=True) + logger.debug( + "BROWSER %i: OpenWPM Firefox extension loaded" % browser_params.browser_id + ) # set window size driver.set_window_size(*DEFAULT_SCREEN_RES) From 3488099ca00771fa20c83c0274e3f4b6bed54c0f Mon Sep 17 00:00:00 2001 From: vringar Date: Tue, 6 Aug 2024 18:08:23 +0200 Subject: [PATCH 2/2] refactor(extension): use settings page and local storage to configure extension --- Extension/bundled/manifest.json | 13 +++-- Extension/bundled/settings/settings.html | 10 ++++ Extension/src/feature.ts | 20 ++++---- openwpm/deploy_browsers/deploy_firefox.py | 58 +++++++++++++++-------- test/manual_test.py | 32 +++++++++---- 5 files changed, 90 insertions(+), 43 deletions(-) create mode 100644 Extension/bundled/settings/settings.html diff --git a/Extension/bundled/manifest.json b/Extension/bundled/manifest.json index aea14f5f6..b4e77f6ae 100644 --- a/Extension/bundled/manifest.json +++ b/Extension/bundled/manifest.json @@ -7,12 +7,15 @@ "scripts": ["feature.js"] }, "content_scripts": [], - "applications": { + "browser_specific_settings": { "gecko": { - "id": "openwpm@mozilla.org", - "strict_min_version": "60.0" + "id": "openwpm@mozilla.org" } }, + + "options_ui": { + "page": "settings/settings.html" + }, "content_security_policy": "script-src 'self' 'unsafe-eval'; object-src 'self';", "permissions": [ "", @@ -25,9 +28,9 @@ "alarms", "downloads", "tabs", - "dns", - "mozillaAddons" + "dns" ], + "experiment_apis": { "sockets": { "schema": "./privileged/sockets/schema.json", diff --git a/Extension/bundled/settings/settings.html b/Extension/bundled/settings/settings.html new file mode 100644 index 000000000..25d595982 --- /dev/null +++ b/Extension/bundled/settings/settings.html @@ -0,0 +1,10 @@ + + + + + + + +

THIS PAGE SHOULD ONLY BE USED BY SELENIUM TO PROGRAMATICALLY SET THE SETTINGS

+ + diff --git a/Extension/src/feature.ts b/Extension/src/feature.ts index e972671b5..ff8924433 100644 --- a/Extension/src/feature.ts +++ b/Extension/src/feature.ts @@ -8,15 +8,9 @@ import { NavigationInstrument } from "./background/navigation-instrument"; import * as loggingDB from "./loggingdb"; import { CallstackInstrument } from "./callstack-instrument"; -async function main() { - // Read the browser configuration from file - const filename = "browser_params.json"; - const raw_config = await browser.profileDirIO.readFile(filename); - let config: any; - if (raw_config) { - config = JSON.parse(raw_config); - console.log("Browser Config:", config); - } else { +async function setup(config: any) { + console.log("Browser Config:", config); + if(!config){ config = { navigation_instrument: true, cookie_instrument: true, @@ -104,4 +98,10 @@ async function main() { await browser.profileDirIO.writeFile("OPENWPM_STARTUP_SUCCESS.txt", ""); } -main(); +browser.storage.local.onChanged.addListener((changes) => { + const changedItems = Object.keys(changes); + if (!changedItems.includes("initialized")) { + return; + } + setup(changes.config?.newValue) +}); diff --git a/openwpm/deploy_browsers/deploy_firefox.py b/openwpm/deploy_browsers/deploy_firefox.py index c4797b794..ea3ae6519 100755 --- a/openwpm/deploy_browsers/deploy_firefox.py +++ b/openwpm/deploy_browsers/deploy_firefox.py @@ -4,12 +4,14 @@ import subprocess import tempfile from pathlib import Path +from time import sleep from typing import Any, Dict, Optional, Tuple from easyprocess import EasyProcessError from multiprocess import Queue from pyvirtualdisplay import Display from selenium import webdriver +from selenium.webdriver.common.by import By from selenium.webdriver.firefox.options import Options from selenium.webdriver.firefox.service import Service @@ -98,25 +100,6 @@ def deploy_firefox( # because status_queue is read off no matter what. status_queue.put(("STATUS", "Display", (display_pid, display_port))) - # Write config file - extension_config: Dict[str, Any] = dict() - extension_config.update(browser_params.to_dict()) - extension_config["logger_address"] = manager_params.logger_address - extension_config["storage_controller_address"] = ( - manager_params.storage_controller_address - ) - extension_config["testing"] = manager_params.testing - ext_config_file = browser_profile_path / "browser_params.json" - with open(ext_config_file, "w") as f: - json.dump(extension_config, f, cls=ConfigEncoder) - logger.debug( - "BROWSER %i: Saved extension config file to: %s" - % (browser_params.browser_id, ext_config_file) - ) - - # TODO restore detailed logging - # fo.set_preference("extensions.@openwpm.sdk.console.logLevel", "all") - # Configure privacy settings configure_firefox.privacy(browser_params, fo) @@ -159,6 +142,7 @@ def deploy_firefox( logger.debug( "BROWSER %i: OpenWPM Firefox extension loaded" % browser_params.browser_id ) + apply_extension_configuration(driver, browser_params, manager_params) # set window size driver.set_window_size(*DEFAULT_SCREEN_RES) @@ -172,3 +156,39 @@ def deploy_firefox( status_queue.put(("STATUS", "Browser Launched", int(pid))) return driver, browser_profile_path, display + + +def apply_extension_configuration( + driver: webdriver.Firefox, + browser_params: BrowserParamsInternal, + manager_params: ManagerParamsInternal, +) -> None: + # Write config file + extension_config: Dict[str, Any] = dict() + extension_config.update(browser_params.to_dict()) + extension_config["logger_address"] = manager_params.logger_address + extension_config["storage_controller_address"] = ( + manager_params.storage_controller_address + ) + extension_config["testing"] = manager_params.testing + config = json.dumps(extension_config, cls=ConfigEncoder) + driver.get("about:debugging#/runtime/this-firefox") + sleep(0.1) + # Very brittle way of finding the extension ID + # Might break if the about:debugging screen ever changes + extension_id = driver.find_element( + By.XPATH, "//span[@title='OpenWPM']/../section/dl/div[3]/dd" + ).text + driver.get(f"moz-extension://{extension_id}/settings/settings.html") + driver.execute_script( + f""" + browser.storage.local.set({{ + config: {config}, + initialized: true + }}); + """ + ) + logger.debug("BROWSER %i: Set extension configuration:", browser_params.browser_id) + + # TODO restore detailed logging + # fo.set_preference("extensions.@openwpm.sdk.console.logLevel", "all") diff --git a/test/manual_test.py b/test/manual_test.py index e6abf2e94..1d8d743e8 100644 --- a/test/manual_test.py +++ b/test/manual_test.py @@ -8,12 +8,15 @@ import click import IPython from selenium import webdriver +from selenium.webdriver.common.by import By from selenium.webdriver.firefox.firefox_binary import FirefoxBinary from selenium.webdriver.firefox.options import Options +from selenium.webdriver.firefox.service import Service from openwpm import js_instrumentation as jsi -from openwpm.config import BrowserParams +from openwpm.config import BrowserParams, BrowserParamsInternal, ManagerParamsInternal from openwpm.deploy_browsers import configure_firefox +from openwpm.deploy_browsers.deploy_firefox import apply_extension_configuration from openwpm.utilities.platform_utils import get_firefox_binary_path from .conftest import xpi @@ -102,11 +105,10 @@ def start_webdriver( """ firefox_binary_path = get_firefox_binary_path() - fb = FirefoxBinary(firefox_path=firefox_binary_path) server, thread = start_server() def register_cleanup(driver): - driver.get(BASE_TEST_URL) + # driver.get(BASE_TEST_URL) def cleanup_server(): print("Cleanup before shutdown...") @@ -131,14 +133,24 @@ def cleanup_server(): # fp.set_preference("extensions.@openwpm.sdk.console.logLevel", "all") configure_firefox.optimize_prefs(fo) - driver = webdriver.Firefox(firefox_binary=fb, options=fo) + fo.binary_location = firefox_binary_path + geckodriver_path = subprocess.check_output( + "which geckodriver", encoding="utf-8", shell=True + ).strip() + driver = webdriver.Firefox( + options=fo, + service=Service( + executable_path=geckodriver_path, + ), + ) + browser_params = BrowserParamsInternal() + if load_browser_params is True: # There's probably more we could do here # to set more preferences and better emulate # what happens in TaskManager. But this lets # us pass some basic things. - browser_params = BrowserParams() if browser_params_file is not None: with open(browser_params_file, "r") as f: browser_params.from_json(f.read()) @@ -146,15 +158,17 @@ def cleanup_server(): js_request_as_string = jsi.clean_js_instrumentation_settings(js_request) browser_params.js_instrument_settings = js_request_as_string - with open(browser_profile_path / "browser_params.json", "w") as f: - f.write(browser_params.to_json()) - if with_extension: # add openwpm extension to profile xpi() ext_xpi = join(EXT_PATH, "dist", "openwpm-1.0.zip") driver.install_addon(ext_xpi, temporary=True) - + manager_params = ManagerParamsInternal() + manager_params.logger_address = None + manager_params.storage_controller_address = None + manager_params.testing = True + browser_params.browser_id = 1 + apply_extension_configuration(driver, browser_params, manager_params) return register_cleanup(driver)