diff --git a/build_defs/defaults.bzl b/build_defs/defaults.bzl index 6d61e58c2..7b1c238c4 100644 --- a/build_defs/defaults.bzl +++ b/build_defs/defaults.bzl @@ -136,3 +136,7 @@ THIRD_PARTY_PY_GREENLET = [ THIRD_PARTY_PY_GOOGLE_GENAI = [ requirement("google-genai"), ] + +THIRD_PARTY_PY_WATCHDOG = [ + requirement("watchdog"), +] diff --git a/mesop/bin/BUILD b/mesop/bin/BUILD index a594ca39a..72ee72c8d 100644 --- a/mesop/bin/BUILD +++ b/mesop/bin/BUILD @@ -15,8 +15,7 @@ py_binary( main = "bin.py", deps = [ "//mesop", # Keep dep to ensure the entire Mesop library is loaded. - "//mesop/cli:execute_module", - "//mesop/exceptions", "//mesop/runtime", + "//mesop/server", ] + THIRD_PARTY_PY_ABSL_PY, ) diff --git a/mesop/bin/bin.py b/mesop/bin/bin.py index c8eeed534..784fd66b2 100644 --- a/mesop/bin/bin.py +++ b/mesop/bin/bin.py @@ -1,23 +1,12 @@ import logging import os import sys -import threading -import time -from typing import Sequence, cast +from typing import Sequence from absl import app, flags -from watchdog.events import FileSystemEvent, FileSystemEventHandler -from watchdog.observers import Observer -import mesop.protos.ui_pb2 as pb -from mesop.cli.execute_module import execute_module, get_module_name_from_path -from mesop.exceptions import format_traceback -from mesop.runtime import ( - enable_debug_mode, - hot_reload_finished, - reset_runtime, - runtime, -) +from mesop.runtime import enable_debug_mode +from mesop.server.hot_reload import execute_main_module, start_file_watcher from mesop.server.wsgi_app import create_app FLAGS = flags.FLAGS @@ -98,7 +87,9 @@ def main(argv: Sequence[str]): app = create_app( prod_mode=FLAGS.prod, - run_block=lambda: execute_main_module(absolute_path=absolute_path), + run_block=lambda: execute_main_module( + absolute_path=absolute_path, prod_mode=FLAGS.prod + ), ) # WARNING: this needs to run *after* the initial `execute_module` @@ -107,171 +98,12 @@ def main(argv: Sequence[str]): # leads to obscure hot reloading bugs. if not FLAGS.prod: print("Running with hot reload:") - stdin_thread = threading.Thread( - target=lambda: fs_watcher(absolute_path), name="mesop_fs_watcher_thread" - ) - stdin_thread.daemon = True - stdin_thread.start() + start_file_watcher(absolute_path, prod_mode=False) logging.getLogger("werkzeug").setLevel(logging.WARN) app.run() -app_modules: set[str] = set() - - -def clear_app_modules() -> None: - # Remove labs modules because they function as application code - # and it needs to be re-executed so that their stateclass is - # re-registered b/c the runtime is reset. - labs_modules: set[str] = set() - for module in sys.modules: - if module.startswith("mesop.labs"): - # Do not delete the module directly, because this causes - # an error where the dictionary size is changed during iteration. - labs_modules.add(module) - for module in labs_modules: - del sys.modules[module] - - for module in app_modules: - # Not every module has been loaded into sys.modules (e.g. the main module) - if module in sys.modules: - del sys.modules[module] - - -def add_app_module(workspace_dir_path: str, app_module_path: str) -> None: - module_name = get_app_module_name( - workspace_dir_path=workspace_dir_path, app_module_path=app_module_path - ) - app_modules.add(module_name) - - -def remove_app_module(workspace_dir_path: str, app_module_path: str) -> None: - module_name = get_app_module_name( - workspace_dir_path=workspace_dir_path, app_module_path=app_module_path - ) - app_modules.remove(module_name) - - -def get_app_module_name(workspace_dir_path: str, app_module_path: str) -> str: - relative_path = os.path.relpath(app_module_path, workspace_dir_path) - - return ( - relative_path.replace(os.sep, ".") - # Special case __init__.py: - # e.g. foo.bar.__init__.py -> foo.bar - # Otherwise, remove the ".py" suffix - .removesuffix(".__init__.py") - .removesuffix(".py") - ) - - -class ReloadEventHandler(FileSystemEventHandler): - def __init__(self, absolute_path: str, workspace_dir_path: str): - self.count = 0 - self.absolute_path = absolute_path - self.workspace_dir_path = workspace_dir_path - - def on_modified(self, event: FileSystemEvent): - src_path = cast(str, event.src_path) - # This could potentially over-trigger if .py files which are - # not application modules are modified (e.g. in venv directories) - # but this should be rare. - if src_path.endswith(".py"): - try: - self.count += 1 - print(f"Hot reload #{self.count}: starting...") - reset_runtime() - execute_main_module(absolute_path=self.absolute_path) - hot_reload_finished() - print(f"Hot reload #{self.count}: finished!") - except Exception as e: - logging.log( - logging.ERROR, "Could not hot reload due to error:", exc_info=e - ) - - def on_created(self, event: FileSystemEvent): - src_path = cast(str, event.src_path) - if src_path.endswith(".py"): - print(f"Watching new Python module: {event.src_path}") - add_app_module( - workspace_dir_path=self.workspace_dir_path, - app_module_path=event.src_path, - ) - - def on_deleted(self, event: FileSystemEvent): - src_path = cast(str, event.src_path) - if src_path.endswith(".py"): - print(f"Stopped watching deleted Python module: {event.src_path}") - remove_app_module( - workspace_dir_path=self.workspace_dir_path, - app_module_path=event.src_path, - ) - - -def fs_watcher(absolute_path: str): - """ - Filesystem watcher using watchdog. Watches for any changes in the specified directory - and triggers hot reload on change. - """ - workspace_dir_path = os.path.dirname(absolute_path) - # Initially track all the files on the file system and then rely on watchdog. - for root, dirnames, files in os.walk(workspace_dir_path): - # Filter out unusual directories, e.g. starting with "." because they - # can be special directories, because venv directories - # can have lots of Python files that are not application Python modules. - new_dirnames: list[str] = [] - for d in dirnames: - if d.startswith("."): - continue - if d == "__pycache__": - continue - if d == "venv": - continue - new_dirnames.append(d) - - dirnames[:] = new_dirnames - - for file in files: - if file.endswith(".py"): - full_path = os.path.join(root, file) - relative_path = os.path.relpath(full_path, workspace_dir_path) - add_app_module( - workspace_dir_path=workspace_dir_path, app_module_path=relative_path - ) - - event_handler = ReloadEventHandler( - absolute_path=absolute_path, - workspace_dir_path=workspace_dir_path, - ) - observer = Observer() - observer.schedule(event_handler, path=workspace_dir_path, recursive=True) # type: ignore - observer.start() - try: - while True: - time.sleep(0.1) - except KeyboardInterrupt: - observer.stop() - observer.join() - - -def execute_main_module(absolute_path: str): - try: - # Clear app modules - clear_app_modules() - execute_module( - module_path=absolute_path, - module_name=get_module_name_from_path(absolute_path), - ) - except Exception as e: - if not FLAGS.prod: - runtime().add_loading_error( - pb.ServerError(exception=str(e), traceback=format_traceback()) - ) - # Always raise an error to make it easy to debug - raise e - - def make_path_absolute(file_path: str): if os.path.isabs(file_path): return file_path diff --git a/mesop/dataclass_utils/dataclass_utils.py b/mesop/dataclass_utils/dataclass_utils.py index f913a4a1d..ef3ac2d14 100644 --- a/mesop/dataclass_utils/dataclass_utils.py +++ b/mesop/dataclass_utils/dataclass_utils.py @@ -1,7 +1,7 @@ # ruff: noqa: E721 import base64 import json -from dataclasses import Field, asdict, dataclass, field, is_dataclass +from dataclasses import Field, asdict, dataclass, field, fields, is_dataclass from datetime import date, datetime from io import StringIO from typing import Any, Type, TypeVar, cast, get_origin, get_type_hints @@ -114,9 +114,51 @@ def has_parent(cls: Type[Any]) -> bool: return len(cls.__bases__) > 0 and cls.__bases__[0] != object +# JSON scalar types that are valid as dict keys without conversion. +_JSON_SCALAR_KEY_TYPES = (str, int, float, bool) + + +def _normalize_state_for_diff(obj: Any) -> Any: + """Normalize a state object so that DeepDiff produces correct paths. + + JSON only supports string (and a few scalar) dict keys. Non-scalar keys + such as tuples are converted to their ``str()`` representation so that: + + * ``diff_state`` generates a correct, single path element (e.g. + ``"(1, 2)"``) instead of incorrectly flattening the tuple into + multiple path elements. + * ``serialize_dataclass`` does not raise ``TypeError`` when encoding a + dict whose keys are tuples or other non-JSON-scalar types. + + Dataclass instances are converted to plain dicts of their fields (without + deep-copying leaf values), so that custom DeepDiff operators that rely on + ``isinstance`` checks (e.g. ``DataFrameOperator``, ``EqualityOperator``) + continue to work correctly. + """ + if is_dataclass(obj) and not isinstance(obj, type): + return { + f.name: _normalize_state_for_diff(getattr(obj, f.name)) + for f in fields(obj) + } + if isinstance(obj, dict): + return { + ( + str(k) + if not isinstance(k, _JSON_SCALAR_KEY_TYPES) and k is not None + else k + ): _normalize_state_for_diff(v) + for k, v in obj.items() + } + if isinstance(obj, list): + return [_normalize_state_for_diff(item) for item in obj] + return obj + + def serialize_dataclass(state: Any): if is_dataclass(state): - json_str = json.dumps(asdict(state), cls=MesopJSONEncoder) + json_str = json.dumps( + _normalize_state_for_diff(asdict(state)), cls=MesopJSONEncoder + ) return json_str else: raise MesopException("Tried to serialize state which was not a dataclass") @@ -354,13 +396,20 @@ def diff_state(state1: Any, state2: Any) -> str: if not is_dataclass(state1) or not is_dataclass(state2): raise MesopException("Tried to diff state which was not a dataclass") + # Normalize both states before diffing so that non-JSON-scalar dict keys + # (e.g. tuples) are converted to their string representation. This + # ensures DeepDiff emits a single, correct path element (e.g. "(1, 2)") + # instead of incorrectly expanding the tuple into multiple path segments. + norm1 = _normalize_state_for_diff(state1) + norm2 = _normalize_state_for_diff(state2) + custom_actions = [] custom_operators = [EqualityOperator()] # Only use the `DataFrameOperator` if pandas exists. if _has_pandas: differences = DeepDiff( - state1, - state2, + norm1, + norm2, custom_operators=[*custom_operators, DataFrameOperator()], threshold_to_diff_deeper=0, ) @@ -377,8 +426,8 @@ def diff_state(state1: Any, state2: Any) -> str: ] else: differences = DeepDiff( - state1, - state2, + norm1, + norm2, custom_operators=custom_operators, threshold_to_diff_deeper=0, ) diff --git a/mesop/dataclass_utils/diff_state_test.py b/mesop/dataclass_utils/diff_state_test.py index cea30d510..a600577a0 100644 --- a/mesop/dataclass_utils/diff_state_test.py +++ b/mesop/dataclass_utils/diff_state_test.py @@ -579,7 +579,6 @@ class C: ] -# This looks like a bug. def test_diff_tuple_dict_keys(): @dataclass class C: @@ -596,14 +595,17 @@ class C: } ) + # Tuple keys are encoded as their str() representation so that the diff + # path is a single, unambiguous element "(1, 2)" rather than the key being + # incorrectly expanded into two separate path segments 1 and 2. assert json.loads(diff_state(s1, s2)) == [ { - "path": ["val1", 1, 2], + "path": ["val1", "(1, 2)"], "action": "values_changed", "value": "V1", - "old_value": "unknown___", + "old_value": "v1", "type": "", - "old_type": "unknown___", + "old_type": "", "new_path": None, "t1_from_index": None, "t1_to_index": None, diff --git a/mesop/server/BUILD b/mesop/server/BUILD index ad1633bce..a4d2e9f9c 100644 --- a/mesop/server/BUILD +++ b/mesop/server/BUILD @@ -9,6 +9,7 @@ load( "THIRD_PARTY_PY_MSGPACK", "THIRD_PARTY_PY_PYTEST", "THIRD_PARTY_PY_SQLALCHEMY", + "THIRD_PARTY_PY_WATCHDOG", "py_library", "py_test", ) @@ -31,15 +32,19 @@ py_library( ] + STATE_SESSIONS_SRCS, ), deps = [ + "//mesop/cli:execute_module", "//mesop/component_helpers", "//mesop/env", "//mesop/events", + "//mesop/exceptions", "//mesop/protos:ui_py_pb2", + "//mesop/runtime", "//mesop/utils", "//mesop/warn", ] + THIRD_PARTY_PY_ABSL_PY + THIRD_PARTY_PY_FLASK + - THIRD_PARTY_PY_FLASK_SOCK, + THIRD_PARTY_PY_FLASK_SOCK + + THIRD_PARTY_PY_WATCHDOG, ) py_library( diff --git a/mesop/server/hot_reload.py b/mesop/server/hot_reload.py new file mode 100644 index 000000000..d984675d2 --- /dev/null +++ b/mesop/server/hot_reload.py @@ -0,0 +1,182 @@ +import logging +import os +import sys +import threading +import time +from typing import cast + +from watchdog.events import FileSystemEvent, FileSystemEventHandler +from watchdog.observers import Observer + +import mesop.protos.ui_pb2 as pb +from mesop.cli.execute_module import execute_module, get_module_name_from_path +from mesop.exceptions import format_traceback +from mesop.runtime import hot_reload_finished, reset_runtime, runtime + +app_modules: set[str] = set() + + +def clear_app_modules() -> None: + # Remove labs modules because they function as application code + # and they need to be re-executed so that their stateclass is + # re-registered because the runtime is reset. + labs_modules: set[str] = set() + for module in sys.modules: + if module.startswith("mesop.labs"): + labs_modules.add(module) + for module in labs_modules: + del sys.modules[module] + + for module in app_modules: + if module in sys.modules: + del sys.modules[module] + + +def add_app_module(workspace_dir_path: str, app_module_path: str) -> None: + module_name = get_app_module_name( + workspace_dir_path=workspace_dir_path, app_module_path=app_module_path + ) + app_modules.add(module_name) + + +def remove_app_module(workspace_dir_path: str, app_module_path: str) -> None: + module_name = get_app_module_name( + workspace_dir_path=workspace_dir_path, app_module_path=app_module_path + ) + app_modules.discard(module_name) + + +def get_app_module_name(workspace_dir_path: str, app_module_path: str) -> str: + relative_path = os.path.relpath(app_module_path, workspace_dir_path) + + return ( + relative_path.replace(os.sep, ".") + # Special case __init__.py: + # e.g. foo.bar.__init__.py -> foo.bar + # Otherwise, remove the ".py" suffix + .removesuffix(".__init__.py") + .removesuffix(".py") + ) + + +class ReloadEventHandler(FileSystemEventHandler): + def __init__( + self, absolute_path: str, workspace_dir_path: str, prod_mode: bool = False + ): + self.count = 0 + self.absolute_path = absolute_path + self.workspace_dir_path = workspace_dir_path + self.prod_mode = prod_mode + + def on_modified(self, event: FileSystemEvent): + src_path = cast(str, event.src_path) + # This could potentially over-trigger if .py files which are + # not application modules are modified (e.g. in venv directories) + # but this should be rare. + if src_path.endswith(".py"): + try: + self.count += 1 + print(f"Hot reload #{self.count}: starting...") + reset_runtime() + execute_main_module( + absolute_path=self.absolute_path, prod_mode=self.prod_mode + ) + hot_reload_finished() + print(f"Hot reload #{self.count}: finished!") + except Exception as e: + logging.log( + logging.ERROR, "Could not hot reload due to error:", exc_info=e + ) + + def on_created(self, event: FileSystemEvent): + src_path = cast(str, event.src_path) + if src_path.endswith(".py"): + print(f"Watching new Python module: {event.src_path}") + add_app_module( + workspace_dir_path=self.workspace_dir_path, + app_module_path=event.src_path, + ) + + def on_deleted(self, event: FileSystemEvent): + src_path = cast(str, event.src_path) + if src_path.endswith(".py"): + print(f"Stopped watching deleted Python module: {event.src_path}") + remove_app_module( + workspace_dir_path=self.workspace_dir_path, + app_module_path=event.src_path, + ) + + +def fs_watcher(absolute_path: str, prod_mode: bool = False): + """ + Filesystem watcher using watchdog. Watches for any changes in the specified directory + and triggers hot reload on change. + """ + workspace_dir_path = os.path.dirname(absolute_path) + # Initially track all the files on the file system and then rely on watchdog. + for root, dirnames, files in os.walk(workspace_dir_path): + # Filter out unusual directories, e.g. starting with "." because they + # can be special directories, because venv directories + # can have lots of Python files that are not application Python modules. + new_dirnames: list[str] = [] + for d in dirnames: + if d.startswith("."): + continue + if d == "__pycache__": + continue + if d == "venv": + continue + new_dirnames.append(d) + + dirnames[:] = new_dirnames + + for file in files: + if file.endswith(".py"): + full_path = os.path.join(root, file) + relative_path = os.path.relpath(full_path, workspace_dir_path) + add_app_module( + workspace_dir_path=workspace_dir_path, app_module_path=relative_path + ) + + event_handler = ReloadEventHandler( + absolute_path=absolute_path, + workspace_dir_path=workspace_dir_path, + prod_mode=prod_mode, + ) + observer = Observer() + observer.schedule(event_handler, path=workspace_dir_path, recursive=True) # type: ignore + observer.start() + try: + while True: + time.sleep(0.1) + except KeyboardInterrupt: + observer.stop() + observer.join() + + +def execute_main_module(absolute_path: str, prod_mode: bool = False): + try: + clear_app_modules() + execute_module( + module_path=absolute_path, + module_name=get_module_name_from_path(absolute_path), + ) + except Exception as e: + if not prod_mode: + runtime().add_loading_error( + pb.ServerError(exception=str(e), traceback=format_traceback()) + ) + raise e + + +def start_file_watcher(absolute_path: str, prod_mode: bool = False) -> None: + """ + Starts a background daemon thread that watches for file changes and + triggers hot reload. Safe to call multiple times — only starts once. + """ + thread = threading.Thread( + target=lambda: fs_watcher(absolute_path, prod_mode=prod_mode), + name="mesop_fs_watcher_thread", + ) + thread.daemon = True + thread.start() diff --git a/mesop/server/wsgi_app.py b/mesop/server/wsgi_app.py index b9c3b56a9..9f0f5622b 100644 --- a/mesop/server/wsgi_app.py +++ b/mesop/server/wsgi_app.py @@ -1,3 +1,4 @@ +import os import sys from typing import Any, Callable @@ -6,6 +7,7 @@ from mesop.runtime import enable_debug_mode from mesop.server.constants import EDITOR_PACKAGE_PATH, PROD_PACKAGE_PATH +from mesop.server.hot_reload import start_file_watcher from mesop.server.flags import port from mesop.server.logging import log_startup from mesop.server.server import configure_flask_app @@ -48,15 +50,42 @@ def create_app( return App(flask_app=flask_app) -def create_wsgi_app(*, debug_mode: bool = False): +def create_wsgi_app( + *, debug_mode: bool = False, watch_path: str | None = None +): """ Creates a WSGI app that can be used to run Mesop in a WSGI server like gunicorn. Args: - debug_mode: If True, enables debug mode for the Mesop app. + debug_mode: If True, enables debug mode and hot reloading for the Mesop app. + watch_path: Path to the main Python file to watch for changes. When provided + alongside ``debug_mode=True``, a background file-watcher thread is started + so that hot reloading works even when Mesop is mounted inside another web + server (e.g. FastAPI). Typically set to ``__file__`` of your entry-point + module. + + Example:: + + import mesop as me + from mesop.server.wsgi_app import create_wsgi_app + + @me.page() + def home(): + me.text("Hello") + + mesop_app = create_wsgi_app(debug_mode=True, watch_path=__file__) """ _app = None + # Start the file watcher immediately so that changes made before the first + # request are still detected. The watcher runs in a daemon thread and does + # not depend on the Flask app being initialised yet. + if debug_mode and watch_path: + absolute_path = ( + watch_path if os.path.isabs(watch_path) else os.path.abspath(watch_path) + ) + start_file_watcher(absolute_path, prod_mode=False) + def wsgi_app(environ: dict[Any, Any], start_response: Callable[..., Any]): # Lazily create and reuse a flask app instance to avoid # the overhead for each WSGI request. diff --git a/mesop/web/src/utils/diff_state_spec.ts b/mesop/web/src/utils/diff_state_spec.ts index 7d3a5727c..17b27f796 100644 --- a/mesop/web/src/utils/diff_state_spec.ts +++ b/mesop/web/src/utils/diff_state_spec.ts @@ -213,6 +213,23 @@ describe('applyStateDiff functionality', () => { ); }); + it('applies updates to tuple dict keys (encoded as strings)', () => { + // Python tuple keys like (1, 2) are serialised as the string "(1, 2)" + // in both the state JSON and the diff path. + const state1 = JSON.stringify({ + val1: {'(1, 2)': 'v1', '(3, 4)': 'v2'}, + }); + const diff = JSON.stringify([ + {path: ['val1', '(1, 2)'], action: 'values_changed', value: 'V1'}, + ]); + + expect(applyStateDiff(state1, diff)).toBe( + JSON.stringify({ + val1: {'(1, 2)': 'V1', '(3, 4)': 'v2'}, + }), + ); + }); + it('applies updates to int dict keys', () => { const state1 = JSON.stringify({ val1: {