update: update to latest versions of rippled/witness server (#112)

## High Level Overview of Change

Title says it all.

### Context of Change

rippled was modified to only allow non-batch attestations.

### Type of Change

- [x] Bug fix (non-breaking change which fixes an issue)
- [x] New feature (non-breaking change which adds functionality)

## Test Plan

Tests pass locally. I can't run CI yet because the docker containers haven't been updated.

Author: mvadari

docker/docker-compose.yml docker-setup/docker-compose.ymldocker/docker-compose.yml renamed to docker-setup/docker-compose.yml

@@ -30,7 +30,8 @@ websockets = "^10.3"
 Jinja2 = "^3.1.2"
 psutil = "^5.9.2"
 docker = "^6.0.0"
-xrpl-py = "^1.8.0b0"
+xrpl-py = { git = "https://github.com/XRPLF/xrpl-py.git", rev = "sidechain-2.5" }
+pycryptodome = "^3.17"
 
 [tool.poetry.dev-dependencies]
 flake8 = "^5.0.4"

poetry.lock

@@ -3,7 +3,7 @@
 import json
 import os
 from pprint import pformat
-from typing import Any, Dict, List, Optional, Tuple
+from typing import Any, Dict, List, Literal, Optional, TypedDict, Union, cast
 
 import click
 from xrpl import CryptoAlgorithm
@@ -15,6 +15,7 @@
     XRP,
     AccountSet,
     AccountSetFlag,
+    Amount,
     Currency,
     IssuedCurrency,
     ServerState,
@@ -23,15 +24,11 @@
     Transaction,
     TrustSet,
     XChainAccountCreateCommit,
-    XChainAddAttestation,
+    XChainAddAccountCreateAttestation,
     XChainBridge,
     XChainCreateBridge,
 )
 from xrpl.models.transactions.transaction import transaction_json_to_binary_codec_form
-from xrpl.models.transactions.xchain_add_attestation import (
-    XChainAttestationBatch,
-    XChainCreateAccountAttestationBatchElement,
-)
 from xrpl.wallet import Wallet
 
 from sidechain_cli.exceptions import SidechainCLIException
@@ -43,8 +40,25 @@
     submit_tx,
 )
 
-_ATTESTATION_ENCODE_ORDER: List[Tuple[str, int]] = [
-    ("account", 4),
+
+class _UnsignedAttestation(TypedDict):
+    xchain_bridge: XChainBridge
+    other_chain_source: str
+    amount: Amount
+    attestation_reward_account: str
+    was_locking_chain_send: Union[Literal[0], Literal[1]]
+    xchain_account_create_count: str
+    destination: str
+    signature_reward: Amount
+
+
+class _SignedAttestation(_UnsignedAttestation):
+    public_key: str
+    signature: str
+
+
+_ATTESTATION_ENCODE_ORDER = [
+    ("other_chain_source", 6),
     ("amount", 2),
     ("signature_reward", 4),
     ("attestation_reward_account", 6),
@@ -55,29 +69,30 @@
 
 
 def _sign_attestation(
-    attestation: XChainCreateAccountAttestationBatchElement,
+    attestation: _UnsignedAttestation,
     bridge: XChainBridge,
-    private_key: str,
-) -> XChainCreateAccountAttestationBatchElement:
-    attestation_dict = attestation.to_dict()[
-        "xchain_create_account_attestation_batch_element"
-    ]
+    wallet: Wallet,
+) -> _SignedAttestation:
     # TODO: use this instead once it's been implemented
-    # attestation_xrpl = transaction_json_to_binary_codec_form(attestation_dict)
+    # attestation_xrpl = transaction_json_to_binary_codec_form(attestation)
     # encoded_obj = encode(attestation_xrpl)
     bridge_dict: Dict[str, Any] = {"xchain_bridge": bridge.to_dict()}
     encoded_obj = encode(transaction_json_to_binary_codec_form(bridge_dict))[4:]
     for key, prefix in _ATTESTATION_ENCODE_ORDER:
-        value = attestation_dict[key]
+        value = attestation[key]  # type: ignore  # Hard to type
         if key == "was_locking_chain_send":
             encoded_obj += "0" + str(value)
         else:
             xrpl_attestation = transaction_json_to_binary_codec_form({key: value})
             encoded_obj += encode(xrpl_attestation)[prefix:]
-    signature = sign(bytes.fromhex(encoded_obj), private_key)
-    attestation_dict["signature"] = signature
-    signed_attestation = XChainCreateAccountAttestationBatchElement.from_dict(
-        attestation_dict
+    signature = sign(bytes.fromhex(encoded_obj), wallet.private_key)
+    signed_attestation: _SignedAttestation = cast(
+        _SignedAttestation,
+        {
+            **attestation,
+            "signature": signature,
+            "public_key": wallet.public_key,
+        },
     )
     return signed_attestation
 
@@ -342,16 +357,14 @@ def setup_bridge(
         # we need to create the witness reward + submission accounts via the bridge
 
         # helper function for submitting the attestations
-        def _submit_attestations(
-            attestations: List[XChainCreateAccountAttestationBatchElement],
+        def _submit_attestation(
+            attestation: _SignedAttestation,
         ) -> None:
-            attestation_tx = XChainAddAttestation(
-                account=issuing_door,
-                xchain_attestation_batch=XChainAttestationBatch(
-                    xchain_bridge=bridge_obj,
-                    xchain_claim_attestation_batch=[],
-                    xchain_create_account_attestation_batch=attestations,
-                ),
+            attestation_tx = XChainAddAccountCreateAttestation.from_dict(
+                {
+                    **attestation,
+                    "account": issuing_door,
+                }
             )
             submit_tx(
                 attestation_tx,
@@ -368,7 +381,6 @@ def _submit_attestations(
         funding_wallet = Wallet(funding_seed, 0, algorithm=funding_wallet_algo)
 
         amount = str(min_create2 * 2)  # submit accounts need spare funds
-        attestations = []
         count = 1
 
         # create the accounts
@@ -388,30 +400,22 @@ def _submit_attestations(
 
         # set up the attestations for the commit
         for account in accounts_issuing_check:
-            init_attestation = XChainCreateAccountAttestationBatchElement(
-                account=funding_wallet.classic_address,
+            init_attestation = _UnsignedAttestation(
+                xchain_bridge=bridge_obj,
+                other_chain_source=funding_wallet.classic_address,
                 amount=amount,
                 attestation_reward_account=issuing_door,
                 destination=account,
-                public_key=issuing_door_wallet.public_key,
-                signature="",
                 signature_reward=signature_reward,
                 was_locking_chain_send=1,
                 xchain_account_create_count=str(count),
             )
             signed_attestation = _sign_attestation(
-                init_attestation, bridge_obj, issuing_door_wallet.private_key
+                init_attestation, bridge_obj, issuing_door_wallet
             )
-            attestations.append(signed_attestation)
+            _submit_attestation(signed_attestation)
             count += 1
 
-            # we can only have 8 attestations in an XChainAddAttestation tx
-            if len(attestations) == 8:
-                _submit_attestations(attestations)
-                attestations = []
-
-        _submit_attestations(attestations)
-
     # set up multisign on the door account
     signer_tx2 = SignerListSet(
         account=issuing_door,

pyproject.toml

@@ -194,9 +194,11 @@ def send_transfer(
         node["CreatedNode"] for node in nodes if "CreatedNode" in node.keys()
     ]
     claim_ids_ledger_entries = [
-        node for node in created_nodes if node["LedgerEntryType"] == "XChainClaimID"
+        node
+        for node in created_nodes
+        if node["LedgerEntryType"] == "XChainOwnedClaimID"
     ]
-    assert len(claim_ids_ledger_entries) == 1
+    assert len(claim_ids_ledger_entries) == 1, len(claim_ids_ledger_entries)
     xchain_claim_id = claim_ids_ledger_entries[0]["NewFields"]["XChainClaimID"]
 
     # XChainTransfer

sidechain_cli/bridge/setup.py

@@ -9,9 +9,9 @@
 from typing import List, Tuple
 
 import click
+import docker
 import httpx
 
-import docker
 from sidechain_cli.exceptions import SidechainCLIException
 from sidechain_cli.utils import (
     ChainData,
@@ -29,7 +29,7 @@
         "..",
         "..",
         "..",
-        "docker",
+        "docker-setup",
         "docker-compose.yml",
     )
 )
@@ -103,7 +103,7 @@ def _run_process(
     "--exe",
     required=True,
     prompt=True,
-    type=click.Path(exists=True),
+    type=str,  # TODO: should be Union[Literal["docker"], click.Path(exists=True)]
     help="The filepath to the executable.",
 )
 @click.option(
@@ -227,15 +227,15 @@ def start_server(
     envvar="RIPPLED_EXE",
     required=True,
     prompt=True,
-    type=click.Path(exists=True),
+    type=str,  # TODO: should be Union[Literal["docker"], click.Path(exists=True)]
     help="The filepath to the rippled executable.",
 )
 @click.option(
     "--witnessd_exe",
     envvar="WITNESSD_EXE",
     required=True,
     prompt=True,
-    type=click.Path(exists=True),
+    type=str,  # TODO: should be Union[Literal["docker"], click.Path(exists=True)]
     help="The filepath to the witnessd executable.",
 )
 @click.option("--docker", is_flag=True, help="Use executables from Docker.")

sidechain_cli/bridge/transfer.py

@@ -63,9 +63,9 @@ def wait_for_attestations(
         transfer_amount = amount.to_dict()
 
     if is_transfer:
-        batch_name = "XChainClaimAttestationBatch"
+        tx_type = "XChainAddClaimAttestation"
     else:
-        batch_name = "XChainCreateAccountAttestationBatch"
+        tx_type = "XChainAddAccountCreateAttestation"
 
     if close_ledgers:
         wait_time = _WAIT_STEP_LENGTH
@@ -82,55 +82,48 @@ def wait_for_attestations(
     while True:
         time.sleep(wait_time)
         if close_ledgers:
-            ledger = to_client.request(
-                Ledger(ledger_index="current", transactions=True, expand=True)
-            )
-        else:
-            ledger = to_client.request(
-                Ledger(ledger_index="validated", transactions=True, expand=True)
-            )
+            to_client.request(GenericRequest(method="ledger_accept"))
+        ledger = to_client.request(
+            Ledger(ledger_index="validated", transactions=True, expand=True)
+        )
 
         new_txs = ledger.result["ledger"]["transactions"]
         for tx in new_txs:
-            if tx["TransactionType"] == "XChainAddAttestation":
-                batch = tx["XChainAttestationBatch"]
-                if batch["XChainBridge"] != bridge_config.to_xrpl():
+            if tx["TransactionType"] == tx_type:
+                if tx["XChainBridge"] != bridge_config.to_xrpl():
                     # make sure attestation is for this bridge
                     continue
-                attestations = batch[batch_name]
-                for attestation in attestations:
-                    element = attestation[f"{batch_name}Element"]
-                    # check that the attestation actually matches this transfer
-                    if element["Account"] != from_wallet.classic_address:
-                        continue
-                    if element["Amount"] != transfer_amount:
-                        continue
-                    if element["Destination"] != to_account:
-                        continue
-                    if is_transfer:
-                        if element["XChainClaimID"] != xchain_claim_id:
-                            continue
-                    if element["PublicKey"] in attestations_seen:
-                        # already seen this attestation, skip
+                # check that the attestation actually matches this transfer
+                if tx["OtherChainSource"] != from_wallet.classic_address:
+                    continue
+                if tx["Amount"] != transfer_amount:
+                    continue
+                if tx["Destination"] != to_account:
+                    continue
+                if is_transfer:
+                    if tx["XChainClaimID"] != xchain_claim_id:
                         continue
-                    attestations_seen.add(element["PublicKey"])
-                    if verbose > 1:
-                        click.echo(pformat(element))
-                    if verbose > 0:
-                        click.secho(
-                            f"Received {len(attestations_seen)} attestations",
-                            fg="bright_green",
-                        )
+                if tx["PublicKey"] in attestations_seen:
+                    # already seen this attestation, skip
+                    continue
+                attestations_seen.add(tx["PublicKey"])
+                if verbose > 1:
+                    click.echo(pformat(tx))
+                if verbose > 0:
+                    click.secho(
+                        f"Received {len(attestations_seen)} attestations",
+                        fg="bright_green",
+                    )
         if len(new_txs) > 0:  # TODO: only count attestations
             time_count = 0
         else:
             time_count += wait_time
-        if close_ledgers:
-            to_client.request(GenericRequest(method="ledger_accept"))
+
         if len(attestations_seen) >= bridge_config.quorum:
             # received enough attestations for quorum
             break
 
         if time_count > attestation_time_limit:
-            print(pformat(to_client.request(LedgerData()).result))
+            if verbose >= 2:
+                click.echo(pformat(to_client.request(LedgerData()).result))
             raise AttestationTimeoutException()

sidechain_cli/server/start.py

@@ -32,6 +32,10 @@ def pytest_configure(config):
     global mocked_home_dir, config_dir, mocked_vars
     runner = CliRunner()
     runner.invoke(main, ["server", "stop", "--all"])
+    if os.getenv("RIPPLED_EXE") is None:
+        raise Exception("Environment variable `RIPPLED_EXE` is not defined.")
+    if os.getenv("WITNESSD_EXE") is None:
+        raise Exception("Environment variable `WITNESSD_EXE` is not defined.")
 
     if os.getenv("CI") != "True":
         config_dir = tempfile.TemporaryDirectory()

sidechain_cli/utils/attestations.py

@@ -5,10 +5,10 @@
 import subprocess
 import time
 
+import docker
 import psutil
 import pytest
 
-import docker
 from sidechain_cli.main import main
 from sidechain_cli.server.start import _DOCKER_COMPOSE
 from sidechain_cli.utils import get_config
@@ -98,6 +98,7 @@ def test_list_dead_process(self, runner):
         else:
             witness = get_config().get_witness(process_to_kill)
             os.kill(witness.pid, signal.SIGINT)
+            time.sleep(0.01)
 
             process = psutil.Process(pid=witness.pid)
             assert (