Release 1.172.0

See release notes.

Author: cjdsellers

.github/workflows/build.yml

@@ -15,11 +15,12 @@ jobs:
       matrix:
         arch: [x64]
         os: [ubuntu-latest, macos-latest, windows-latest]
-        python-version: ["3.9", "3.10"]
+        python-version: ["3.9", "3.10", "3.11"]
     name: build - Python ${{ matrix.python-version }} (${{ matrix.arch }} ${{ matrix.os }})
     runs-on: ${{ matrix.os }}
     env:
       BUILD_MODE: debug
+      RUST_BACKTRACE: 1
 
     steps:
       - name: Checkout repository
@@ -120,6 +121,6 @@ jobs:
         if: runner.os == 'Windows'
         run: |
           poetry install --with test --all-extras
-          poetry run pytest --ignore=tests/performance_tests -k "not no_ci" --new-first --failed-first
+          poetry run pytest --ignore=tests/performance_tests --new-first --failed-first
         env:
           PARALLEL_BUILD: false

.gitignore

@@ -30,14 +30,17 @@
 __pycache__
 _build/
 build/
+data_catalog/
 dist/
 env/
 log/
 logs/
+*temp/
 *target/
 venv*/
 
 .DS_Store
+null-ls*
 PERF.JSON
 output.json
 *dask-worker-space*

.pre-commit-config.yaml

@@ -55,15 +55,6 @@ repos:
         args: ["--manifest-path", "nautilus_core/Cargo.toml", "--", "-D", "warnings"]
         files: \.rs$
         pass_filenames: false
-      - id: cargo-check
-        name: cargo check
-        description: Check the package for errors.
-        entry: cargo check
-        language: system
-        types: [rust]
-        args: ["--manifest-path", "nautilus_core/Cargo.toml"]
-        files: \.rs$
-        pass_filenames: false
 
   ##############################################################################
   #  Python/Cython formatting and linting
@@ -81,7 +72,7 @@ repos:
       - id: isort
         types_or: [python, cython]
         entry: "isort"
-        args: ["--settings-file", "pyproject.toml", "."]
+        args: ["--settings-file", "pyproject.toml"]
 
   - repo: https://github.com/psf/black
     rev: 23.3.0
@@ -92,13 +83,20 @@ repos:
         args: ["--config", "pyproject.toml"]
         exclude: "docs/_pygments/monokai.py"
 
+  - repo: https://github.com/charliermarsh/ruff-pre-commit
+    rev: v0.0.263
+    hooks:
+      - id: ruff
+        args: ["--fix"]
+
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.1.1
+    rev: v1.2.0
     hooks:
       - id: mypy
         args: [
-          --no-strict-optional,
           --ignore-missing-imports,
+          --allow-incomplete-defs,
+          --no-strict-optional,  # Fixing in progress
           --warn-no-return,
         ]
         additional_dependencies: [
@@ -110,11 +108,6 @@ repos:
           types-toml,
         ]
 
-  - repo: https://github.com/charliermarsh/ruff-pre-commit
-    rev: v0.0.260
-    hooks:
-      - id: ruff
-
   - repo: https://github.com/kynan/nbstripout
     rev: 0.6.1
     hooks:

Makefile

@@ -10,13 +10,22 @@ IMAGE_FULL?=${IMAGE}:${GIT_TAG}
 .PHONY: pytest pytest-coverage
 
 install:
-	poetry install --with dev,test --all-extras
+	BUILD_MODE=release poetry install --with dev,test --all-extras
+
+install-debug:
+	BUILD_MODE=debug poetry install --with dev,test --all-extras
 
 install-just-deps:
 	poetry install --with dev,test --all-extras --no-root
 
+install-just-deps-all:
+	poetry install --with dev,test,docs --all-extras --no-root
+
 build: nautilus_trader
-	poetry run python build.py
+	BUILD_MODE=release poetry run python build.py
+
+build-debug: nautilus_trader
+	BUILD_MODE=debug poetry run python build.py
 
 clean:
 	git clean -fxd

README.md

@@ -13,11 +13,12 @@
 | `master`  | ![version](https://img.shields.io/endpoint?url=https%3A%2F%2Fraw.githubusercontent.com%2Fnautechsystems%2Fnautilus_trader%2Fmaster%2Fversion.json) | [![build](https://github.com/nautechsystems/nautilus_trader/actions/workflows/build.yml/badge.svg?branch=master)](https://github.com/nautechsystems/nautilus_trader/actions/workflows/build.yml) |
 | `develop` | ![version](https://img.shields.io/endpoint?url=https%3A%2F%2Fraw.githubusercontent.com%2Fnautechsystems%2Fnautilus_trader%2Fdevelop%2Fversion.json) | [![build](https://github.com/nautechsystems/nautilus_trader/actions/workflows/build.yml/badge.svg?branch=develop)](https://github.com/nautechsystems/nautilus_trader/actions/workflows/build.yml) |
 
-| Platform          | Rust      | Python |
-|:------------------|:----------|:-------|
-| Linux (x86\_64)   | `1.68.2+` | `3.9+` |
-| macOS (x86\_64)   | `1.68.2+` | `3.9+` |
-| Windows (x86\_64) | `1.68.2+` | `3.9+` |
+| Platform           | Rust    | Python |
+|:-------------------|:--------|:-------|
+| `Linux (x86_64)`   | 1.69.0+ | 3.9+   |
+| `macOS (x86_64)`   | 1.69.0+ | 3.9+   |
+| `macOS (arm64)`    | 1.69.0+ | 3.9+   |
+| `Windows (x86_64)` | 1.69.0+ | 3.9+   |
 
 - **Website:** https://nautilustrader.io
 - **Docs:** https://docs.nautilustrader.io
@@ -114,6 +115,10 @@ Cython, with static libraries linked at compile-time before the wheel binaries a
 does not need to have Rust installed to run NautilusTrader. In the future as more Rust code is introduced,
 [PyO3](https://pyo3.rs/latest) will be leveraged for easier Python bindings.
 
+This project makes the [Soundness Pledge](https://raphlinus.github.io/rust/2020/01/18/soundness-pledge.html):
+> “The intent of this project is to be free of soundness bugs. 
+> The developers will do their best to avoid them, and welcome help in analyzing and fixing them.”
+
 ## Architecture (data flow)
 
 ![Architecture](https://github.com/nautechsystems/nautilus_trader/blob/develop/docs/_images/architecture-overview.png?raw=true "architecture")
@@ -212,13 +217,16 @@ point we will follow a formal process for releases, with deprecation periods for
 ## Makefile
 
 A `Makefile` is provided to automate most installation and build tasks for development. It provides the following targets:
-- `make install` -- Installs the main, dev and test dependencies then installs the package using poetry
-- `make install-just-deps` -- Installs just the main, dev and test dependencies (does not install package)
-- `make build` -- Runs the Cython build script
-- `make clean` -- Cleans all non-source artifacts from the repository
+- `make install` -- Installs in `release` build mode with `main`, `dev` and `test` dependencies then installs the package using poetry (default)
+- `make install-debug` -- Same as `make install` but with `debug` build mode
+- `make install-just-deps` -- Installs just the `main`, `dev` and `test` dependencies (does not install package)
+- `make install-just-deps-all` -- Same as `make install-just-deps` and additionally installs `docs` dependencies
+- `make build` -- Runs the Cython build script in `release` build mode (default)
+- `make build-debug` -- Runs the Cython build script in `debug` build mode
+- `make clean` -- **CAUTION** Cleans all non-source artifacts from the repository
 - `make docs` -- Builds the documentation HTML using Sphinx
 - `make pre-commit` -- Runs the pre-commit checks over all files
-- `make ruff` -- Runs ruff over all files using the pyproject.toml config
+- `make ruff` -- Runs ruff over all files using the `pyproject.toml` config
 - `make pytest` -- Runs all tests with `pytest` (except performance tests)
 - `make pytest-coverage` -- Same as `make pytest` and additionally runs with test coverage and produces a report
 
@@ -395,6 +403,9 @@ Contributions to the project are welcome and require the completion of a standar
 
 NautilusTrader is developed and maintained by Nautech Systems, a technology 
 company specializing in the development of high-performance trading systems. 
+Although the project utilizes the Rust programming language and benefits from its ecosystem, 
+Nautech Systems is not affiliated with the Rust Foundation, and this project is not an official 
+work of the Rust Foundation.
 For more information, visit https://nautilustrader.io.
 
 Copyright (C) 2015-2023 Nautech Systems Pty Ltd. All rights reserved.

RELEASES.md

@@ -1,3 +1,41 @@
+# NautilusTrader 1.172.0 Beta
+
+Released on 30th April 2023 (UTC).
+
+### Breaking Changes
+- Removed legacy Rust parquet data catalog backend (based on arrow2)
+- Removed Binance config for `clock_sync_interval_secs` (redundant/unused and should be handled at system level)
+- Removed redundant rate limiting from Rust logger (and associated `rate_limit` config params)
+- Renamed `Future` instrument to `FuturesContract` (avoids ambiguity)
+- Renamed `Option` instrument to `OptionsContract` (avoids ambiguity and naming conflicts in Rust)
+- Reinstate hours and minutes time component for default order and position identifiers (easier debugging, less collisions)
+- Setting time alerts for in the past or current time will generate an immediate `TimeEvent` (rather than being invalid).
+
+### Enhancements
+- Added new DataFusion Rust parquet data catalog backend (yet to be integrated into Python)
+- Added `external_order_claims` config option for `StrategyConfig` (for claiming external orders per instrument)
+- Added `Order.signed_decimal_qty()`
+- Added `Cache.orders_for_exec_algorithm(...)`
+- Added `Cache.orders_for_exec_spawn(...)`
+- Added `TWAPExecAlgorithm` and `TWAPExecAlgorithmConfig` to examples
+- Build out `ExecAlgorithm` base class for implementing 'first class' executon algorithms
+- Rewired execution for improved flow flexibility between emulated orders, execution algorithms and the `RiskEngine`
+- Improved handling for `OrderEmulator` updating of contingency orders from execution algorithms
+- Define public API for instruments can now import directly from `nautilus_trader.model.instruments` (denest namespace)
+- Define public API for orders, can now import directly from `nautilus_trader.model.orders` (denest namespace)
+- Define public API for order book, can now import directly from `nautilus_trader.model.orderbook` (denest namespace)
+- Now stripping debug symbols after build (reduced binary sizes)
+- Refined build and added additional `debug` Makefile convenience targets
+
+### Fixes
+- Fixed processing of contingency orders when in a pending update state
+- Fixed calculation of PnL for flipped positions (only book realized PnL against open position)
+- Fixed `WebSocketClient` session disconnect, thanks for reporting @miller-moore
+- Added missing `BinanceSymbolFilterType.NOTIONAL`
+- Fixed incorrect `Mul` trait for `Price` and `Quantity` (not being used in Cython/Python layer)
+
+---
+
 # NautilusTrader 1.171.0 Beta
 
 Released on 30th March 2023 (UTC).

build.py

@@ -8,6 +8,7 @@
 import sysconfig
 from datetime import datetime
 from pathlib import Path
+from typing import Optional
 
 import numpy as np
 from Cython.Build import build_ext
@@ -68,6 +69,7 @@
 # Directories with headers to include
 RUST_INCLUDES = ["nautilus_trader/core/includes"]
 RUST_LIB_PATHS: list[Path] = [
+    TARGET_DIR / f"{RUST_LIB_PFX}nautilus_backtest.{RUST_STATIC_LIB_EXT}",
     TARGET_DIR / f"{RUST_LIB_PFX}nautilus_common.{RUST_STATIC_LIB_EXT}",
     TARGET_DIR / f"{RUST_LIB_PFX}nautilus_core.{RUST_STATIC_LIB_EXT}",
     TARGET_DIR / f"{RUST_LIB_PFX}nautilus_model.{RUST_STATIC_LIB_EXT}",
@@ -80,11 +82,21 @@ def _build_rust_libs() -> None:
     try:
         # Build the Rust libraries using Cargo
         build_options = " --release" if BUILD_MODE == "release" else ""
-        extra_flags = ""
         print("Compiling Rust libraries...")
-        build_cmd = f"(cd nautilus_core && cargo build{build_options}{extra_flags} --all-features)"
-        print(build_cmd)
-        os.system(build_cmd)
+
+        cmd_args = [
+            "cargo",
+            "build",
+            *build_options.split(),
+            "--all-features",
+        ]
+        print(" ".join(cmd_args))
+
+        subprocess.run(
+            cmd_args,  # noqa
+            cwd="nautilus_core",
+            check=True,
+        )
     except subprocess.CalledProcessError as e:
         raise RuntimeError(
             f"Error running cargo: {e.stderr.decode()}",
@@ -121,7 +133,9 @@ def _build_extensions() -> list[Extension]:
     # disable it with " "#define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION"
     # https://stackoverflow.com/questions/52749662/using-deprecated-numpy-api
     # From the Cython docs: "For the time being, it is just a warning that you can ignore."
-    define_macros = [("NPY_NO_DEPRECATED_API", "NPY_1_7_API_VERSION")]
+    define_macros: list[tuple[str, Optional[str]]] = [
+        ("NPY_NO_DEPRECATED_API", "NPY_1_7_API_VERSION"),
+    ]
     if PROFILE_MODE or ANNOTATION_MODE:
         # Profiling requires special macro directives
         define_macros.append(("CYTHON_TRACE", "1"))
@@ -131,6 +145,9 @@ def _build_extensions() -> list[Extension]:
 
     if platform.system() == "Darwin":
         extra_compile_args.append("-Wno-unreachable-code-fallthrough")
+        extra_link_args.append("-flat_namespace")
+        extra_link_args.append("-undefined")
+        extra_link_args.append("suppress")
 
     if platform.system() != "Windows":
         # Suppress warnings produced by Cython boilerplate
@@ -216,9 +233,8 @@ def _copy_rust_dylibs_to_project() -> None:
 def _get_clang_version() -> str:
     try:
         result = subprocess.run(
-            "clang --version",
+            ["clang", "--version"],  # noqa
             check=True,
-            shell=True,
             capture_output=True,
         )
         output = (
@@ -239,9 +255,8 @@ def _get_clang_version() -> str:
 def _get_rustc_version() -> str:
     try:
         result = subprocess.run(
-            "rustc --version",
+            ["rustc", "--version"],  # noqa
             check=True,
-            shell=True,
             capture_output=True,
         )
         output = result.stdout.decode().lstrip("rustc ")[:-1]
@@ -254,6 +269,26 @@ def _get_rustc_version() -> str:
         ) from e
 
 
+def _strip_unneeded_symbols() -> None:
+    try:
+        print("Stripping unneeded symbols from binaries...")
+        for so in itertools.chain(Path("nautilus_trader").rglob("*.so")):
+            if platform.system() == "Linux":
+                strip_cmd = f"strip --strip-unneeded {so}"
+            elif platform.system() == "Darwin":
+                strip_cmd = f"strip -x {so}"
+            else:
+                raise RuntimeError(f"Cannot strip symbols for platform {platform.system()}")
+            subprocess.run(
+                strip_cmd,  # noqa
+                check=True,
+                shell=True,  # noqa
+                capture_output=True,
+            )
+    except subprocess.CalledProcessError as e:
+        raise RuntimeError(f"Error when stripping symbols.\n{e.stderr.decode()}") from e
+
+
 def build() -> None:
     """Construct the extensions and distribution."""
     _build_rust_libs()
@@ -276,6 +311,9 @@ def build() -> None:
             # Copy the build back into the source tree for development and wheel packaging
             _copy_build_dir_to_project(cmd)
 
+    if platform.system() in ("Linux", "Darwin"):
+        _strip_unneeded_symbols()
+
 
 if __name__ == "__main__":
     print("\033[36m")

docs/api_reference/execution.md

@@ -4,6 +4,16 @@
 .. automodule:: nautilus_trader.execution
 ```
 
+## Components
+
+```{eval-rst}
+.. automodule:: nautilus_trader.execution.algorithm
+   :show-inheritance:
+   :inherited-members:
+   :members:
+   :member-order: bysource
+```
+
 ```{eval-rst}
 .. automodule:: nautilus_trader.execution.client
    :show-inheritance:
@@ -12,6 +22,14 @@
    :member-order: bysource
 ```
 
+```{eval-rst}
+.. automodule:: nautilus_trader.execution.emulator
+   :show-inheritance:
+   :inherited-members:
+   :members:
+   :member-order: bysource
+```
+
 ```{eval-rst}
 .. automodule:: nautilus_trader.execution.engine
    :show-inheritance:
@@ -20,6 +38,14 @@
    :member-order: bysource
 ```
 
+```{eval-rst}
+.. automodule:: nautilus_trader.execution.matching_core
+   :show-inheritance:
+   :inherited-members:
+   :members:
+   :member-order: bysource
+```
+
 ## Messages
 
 ```{eval-rst}