Release 1.173.0

See release notes.

Author: cjdsellers

.github/workflows/release.yml

@@ -210,7 +210,7 @@ jobs:
       fail-fast: false
       matrix:
         arch: [x64]
-        os: [ubuntu-latest, macos-latest, windows-latest]
+        os: [ubuntu-20.04, ubuntu-latest, macos-latest, macos-13, windows-latest]
         python-version: ["3.9", "3.10", "3.11"]
     name: publish-wheels - Python ${{ matrix.python-version }} (${{ matrix.arch }} ${{ matrix.os }})
     runs-on: ${{ matrix.os }}

.gitignore

@@ -40,7 +40,7 @@ logs/
 venv*/
 
 .DS_Store
-null-ls*
+.null-ls*
 PERF.JSON
 output.json
 *dask-worker-space*

.pre-commit-config.yaml

@@ -84,7 +84,7 @@ repos:
         exclude: "docs/_pygments/monokai.py"
 
   - repo: https://github.com/charliermarsh/ruff-pre-commit
-    rev: v0.0.263
+    rev: v0.0.264
     hooks:
       - id: ruff
         args: ["--fix"]

RELEASES.md

@@ -1,3 +1,22 @@
+# NautilusTrader 1.173.0 Beta
+
+Released on 5th May 2023 (UTC).
+
+### Breaking Changes
+None
+
+### Enhancements
+None
+
+### Fixes
+- Fixed `BacktestEngine` processing of venue(s) message queue based off time event `ts_init`
+- Fixed `Position.signed_decimal_qty` (incorrect format precision in f-string), thanks for reporting @rsmb7z
+- Fixed trailing stop type order updates for `reduce_only` instruction, thanks for reporting @Otlk
+- Fixed updating of active execution algorithm orders (events weren't being cached)
+- Fixed condition check for applying pending events (do not apply to orders at `INITIALIZED` status)
+
+---
+
 # NautilusTrader 1.172.0 Beta
 
 Released on 30th April 2023 (UTC).
@@ -9,7 +28,7 @@ Released on 30th April 2023 (UTC).
 - Renamed `Future` instrument to `FuturesContract` (avoids ambiguity)
 - Renamed `Option` instrument to `OptionsContract` (avoids ambiguity and naming conflicts in Rust)
 - Reinstate hours and minutes time component for default order and position identifiers (easier debugging, less collisions)
-- Setting time alerts for in the past or current time will generate an immediate `TimeEvent` (rather than being invalid).
+- Setting time alerts for in the past or current time will generate an immediate `TimeEvent` (rather than being invalid)
 
 ### Enhancements
 - Added new DataFusion Rust parquet data catalog backend (yet to be integrated into Python)
@@ -21,7 +40,7 @@ Released on 30th April 2023 (UTC).
 - Build out `ExecAlgorithm` base class for implementing 'first class' executon algorithms
 - Rewired execution for improved flow flexibility between emulated orders, execution algorithms and the `RiskEngine`
 - Improved handling for `OrderEmulator` updating of contingency orders from execution algorithms
-- Define public API for instruments can now import directly from `nautilus_trader.model.instruments` (denest namespace)
+- Define public API for instruments, can now import directly from `nautilus_trader.model.instruments` (denest namespace)
 - Define public API for orders, can now import directly from `nautilus_trader.model.orders` (denest namespace)
 - Define public API for order book, can now import directly from `nautilus_trader.model.orderbook` (denest namespace)
 - Now stripping debug symbols after build (reduced binary sizes)

nautilus_core/Cargo.lock

@@ -507,6 +507,10 @@ async def generate_position_status_reports(
     async def _submit_order(self, command: SubmitOrder) -> None:
         order: Order = command.order
 
+        if order.is_closed:
+            self.log.warning(f"Cannot submit already closed order {command.order}.")
+            return
+
         # Check validity
         self._check_order_validity(order)
         self._log.debug(f"Submitting {order}.")
@@ -531,7 +535,7 @@ async def _submit_order(self, command: SubmitOrder) -> None:
         except KeyError:
             raise RuntimeError(f"unsupported order type, was {order.order_type}")
 
-    def _check_order_validity(self, order: Order):
+    def _check_order_validity(self, order: Order) -> None:
         # Implement in child class
         raise NotImplementedError
 

nautilus_trader/adapters/binance/common/execution.py

@@ -207,7 +207,7 @@ async def _get_binance_active_position_symbols(
 
     # -- COMMAND HANDLERS -------------------------------------------------------------------------
 
-    def _check_order_validity(self, order: Order):
+    def _check_order_validity(self, order: Order) -> None:
         # Check order type valid
         if order.order_type not in self._futures_enum_parser.futures_valid_order_types:
             self._log.error(

nautilus_trader/adapters/binance/futures/execution.py

@@ -20,6 +20,7 @@
 from nautilus_trader.adapters.binance.common.enums import BinanceOrderType
 from nautilus_trader.model.enums import OrderType
 from nautilus_trader.model.enums import TimeInForce
+from nautilus_trader.model.enums import order_type_to_str
 from nautilus_trader.model.orders import Order
 
 
@@ -128,5 +129,5 @@ def parse_internal_order_type(self, order: Order) -> BinanceOrderType:
             return BinanceOrderType.TAKE_PROFIT_LIMIT
         else:
             raise RuntimeError(  # pragma: no cover (design-time error)
-                f"invalid or unsupported `OrderType`, was {order.order_type}",  # pragma: no cover
+                f"invalid or unsupported `OrderType`, was {order_type_to_str(order.order_type)}",  # pragma: no cover
             )

nautilus_trader/adapters/binance/spot/enums.py

@@ -172,7 +172,7 @@ async def _get_binance_active_position_symbols(
 
     # -- COMMAND HANDLERS -------------------------------------------------------------------------
 
-    def _check_order_validity(self, order: Order):
+    def _check_order_validity(self, order: Order) -> None:
         # Check order type valid
         if order.order_type not in self._spot_enum_parser.spot_valid_order_types:
             self._log.error(

nautilus_trader/adapters/binance/spot/execution.py

@@ -97,7 +97,7 @@ def generate_positions_report(positions: list[Position]) -> pd.DataFrame:
         if not positions:
             return pd.DataFrame()
 
-        positions = [p.to_dict() for p in positions if p.is_closed]
+        positions = [p.to_dict() for p in positions]
         if not positions:
             return pd.DataFrame()
 

nautilus_trader/analysis/reporter.py

@@ -1077,10 +1077,12 @@ cdef class BacktestEngine:
         cdef:
             uint64_t i
             uint64_t ts_event_init
+            uint64_t ts_last_init = 0
             TimeEventHandler_t raw_handler
             TimeEvent event
             TestClock clock
             object callback
+            SimulatedExchange exchange
         for i in range(raw_handler_vec.len):
             raw_handler = <TimeEventHandler_t>raw_handlers[i]
             ts_event_init = raw_handler.event.ts_init
@@ -1094,6 +1096,12 @@ cdef class BacktestEngine:
             callback = <object>raw_handler.callback_ptr
             callback(event)
 
+            if ts_event_init != ts_last_init:
+                # Process exchange messages
+                ts_last_init = ts_event_init
+                for exchange in self._venues.values():
+                    exchange.process(ts_event_init)
+
     def _log_pre_run(self):
         log_memory(self._log)
 

nautilus_trader/backtest/engine.pyx

@@ -151,8 +151,8 @@ cdef class OrderMatchingEngine:
     cdef void _update_limit_order(self, Order order, Quantity qty, Price price)
     cdef void _update_stop_market_order(self, StopMarketOrder order, Quantity qty, Price trigger_price)
     cdef void _update_stop_limit_order(self, StopLimitOrder order, Quantity qty, Price price, Price trigger_price)
-    cdef void _update_market_if_touched_order(self, MarketIfTouchedOrder order, Quantity qty, Price trigger_price)
-    cdef void _update_limit_if_touched_order(self, LimitIfTouchedOrder order, Quantity qty, Price price, Price trigger_price)
+    cdef void _update_market_if_touched_order(self, Order order, Quantity qty, Price trigger_price)
+    cdef void _update_limit_if_touched_order(self, Order order, Quantity qty, Price price, Price trigger_price)
     cdef void _update_trailing_stop_order(self, Order order)
 
 # -- ORDER PROCESSING -----------------------------------------------------------------------------

nautilus_trader/backtest/matching_engine.pxd

@@ -987,7 +987,7 @@ cdef class OrderMatchingEngine:
 
     cdef void _update_market_if_touched_order(
         self,
-        MarketIfTouchedOrder order,
+        Order order,
         Quantity qty,
         Price trigger_price,
     ):
@@ -1010,7 +1010,7 @@ cdef class OrderMatchingEngine:
 
     cdef void _update_limit_if_touched_order(
         self,
-        LimitIfTouchedOrder order,
+        Order order,
         Quantity qty,
         Price price,
         Price trigger_price,
@@ -1794,6 +1794,16 @@ cdef class OrderMatchingEngine:
             if trigger_price is None:
                 trigger_price = order.trigger_price
             self._update_limit_if_touched_order(order, qty, price, trigger_price)
+        elif order.order_type == OrderType.TRAILING_STOP_MARKET:
+            if trigger_price is None:
+                trigger_price = order.trigger_price
+            self._update_market_if_touched_order(order, qty, trigger_price)
+        elif order.order_type == OrderType.TRAILING_STOP_LIMIT:
+            if price is None:
+                price = order.price
+            if trigger_price is None:
+                trigger_price = order.trigger_price
+            self._update_limit_if_touched_order(order, qty, price, trigger_price)
         else:
             raise ValueError(
                 f"invalid `OrderType` was {order.order_type}")  # pragma: no cover (design-time error)

nautilus_trader/backtest/matching_engine.pyx

@@ -381,8 +381,6 @@ cdef class ExecAlgorithm(Actor):
 
         Raises
         ------
-        ValueError
-            If `primary.status` is not ``INITIALIZED``.
         ValueError
             If `primary.exec_algorithm_id` is not equal to `self.id`.
         ValueError
@@ -393,7 +391,6 @@ cdef class ExecAlgorithm(Actor):
         """
         Condition.not_none(primary, "primary")
         Condition.not_none(quantity, "quantity")
-        Condition.equal(primary.status, OrderStatus.INITIALIZED, "primary.status", "order_status")
         Condition.equal(primary.exec_algorithm_id, self.id, "primary.exec_algorithm_id", "id")
 
         if reduce_primary:
@@ -468,8 +465,6 @@ cdef class ExecAlgorithm(Actor):
 
         Raises
         ------
-        ValueError
-            If `primary.status` is not ``INITIALIZED``.
         ValueError
             If `primary.exec_algorithm_id` is not equal to `self.id`.
         ValueError
@@ -482,7 +477,6 @@ cdef class ExecAlgorithm(Actor):
         """
         Condition.not_none(primary, "primary")
         Condition.not_none(quantity, "quantity")
-        Condition.equal(primary.status, OrderStatus.INITIALIZED, "primary.status", "order_status")
         Condition.equal(primary.exec_algorithm_id, self.id, "primary.exec_algorithm_id", "id")
 
         if reduce_primary:
@@ -556,8 +550,6 @@ cdef class ExecAlgorithm(Actor):
 
         Raises
         ------
-        ValueError
-            If `primary.status` is not ``INITIALIZED``.
         ValueError
             If `primary.exec_algorithm_id` is not equal to `self.id`.
         ValueError
@@ -570,7 +562,6 @@ cdef class ExecAlgorithm(Actor):
         """
         Condition.not_none(primary, "primary")
         Condition.not_none(quantity, "quantity")
-        Condition.equal(primary.status, OrderStatus.INITIALIZED, "primary.status", "order_status")
         Condition.equal(primary.exec_algorithm_id, self.id, "primary.exec_algorithm_id", "id")
 
         if reduce_primary:
@@ -679,6 +670,13 @@ cdef class ExecAlgorithm(Actor):
 
         # Handle primary (original) order
         primary_command = self.cache.load_submit_order_command(order.client_order_id)
+        cdef Order cached_order = self.cache.order(order.client_order_id)
+        if cached_order.order_type != order.order_type:
+            self.cache.add_order(order, primary_command.position_id, override=True)
+
+        # Replace commands order with transformed order
+        primary_command.order = order
+
         Condition.equal(order.strategy_id, primary_command.strategy_id, "order.strategy_id", "primary_command.strategy_id")
         if primary_command is None:
             self._log.error(
@@ -779,7 +777,7 @@ cdef class ExecAlgorithm(Actor):
             return  # Cannot send command
 
         cdef OrderPendingUpdate event
-        if not order.is_emulated_c():
+        if order.status != OrderStatus.INITIALIZED and not order.is_emulated_c():
             # Generate and apply event
             event = self._generate_order_pending_update(order)
             try:
@@ -942,7 +940,7 @@ cdef class ExecAlgorithm(Actor):
             return  # Cannot send command
 
         cdef OrderPendingCancel event
-        if not order.is_emulated_c():
+        if order.status != OrderStatus.INITIALIZED and not order.is_emulated_c():
             # Generate and apply event
             event = self._generate_order_pending_cancel(order)
             try:

nautilus_trader/execution/algorithm.pyx

@@ -466,16 +466,18 @@ cdef class OrderEmulator(Actor):
         PositionId position_id,
         ClientId client_id,
     ):
-        cdef SubmitOrder submit = SubmitOrder(
-            trader_id=order.trader_id,
-            strategy_id=order.strategy_id,
-            order=order,
-            position_id=position_id,
-            client_id=client_id,
-            command_id=UUID4(),
-            ts_init=self.clock.timestamp_ns(),
-        )
-        self.cache.add_submit_order_command(submit)
+        cdef SubmitOrder submit = self.cache.load_submit_order_command(order.client_order_id)
+        if submit is None:
+            submit = SubmitOrder(
+                trader_id=order.trader_id,
+                strategy_id=order.strategy_id,
+                order=order,
+                position_id=position_id,
+                client_id=client_id,
+                command_id=UUID4(),
+                ts_init=self.clock.timestamp_ns(),
+            )
+            self.cache.add_submit_order_command(submit)
 
         if order.emulation_trigger == TriggerType.NO_TRIGGER:
             if order.exec_algorithm_id is not None:

nautilus_trader/execution/emulator.pyx

@@ -780,7 +780,8 @@ cdef class RedisCacheDatabase(CacheDatabase):
 
         # Check data integrity of reply
         if reply > 1:  # Reply = The length of the list after the push operation
-            self._log.warning(
+            # Dropped the log level to debug as this is expected for transformed orders
+            self._log.debug(
                 f"The {repr(order.client_order_id)} already existed and was appended to.",
             )