[Morse -> Shannon Migration] account import & claim E2E tests (part 1)

.github/pull_request_template.md

@@ -29,4 +29,5 @@ Select one or more from the following:
 - [ ] For code, I have run `make go_develop_and_test` and `make test_e2e`
 - [ ] For code, I have added the `devnet-test-e2e` label to run E2E tests in CI
 - [ ] For configurations, I have update the documentation
+- [ ] For code covered by @oneshot E2E tests, `make test_e2e_oneshot` passes on localnet
 - [ ] I added TODOs where applicable

.gitignore

@@ -109,3 +109,6 @@ path_config.yaml
 
 # vultr files created during creation and deletion of new instances that may contain sensitive information
 vultr_*.json
+
+# Ignore Morse migration state working files
+tools/scripts/migration/*.json

Makefile

@@ -4,6 +4,7 @@ SHELL = /bin/sh
 
 POCKETD_HOME ?= ./localnet/pocketd
 POCKET_NODE ?= tcp://127.0.0.1:26657 # The pocket node (validator in the localnet context)
+DEFAULT_POCKET_NODE_GRPC_ADDR ?= "localhost:9090"
 TESTNET_RPC ?= https://testnet-validated-validator-rpc.poktroll.com/ # TestNet RPC endpoint for validator maintained by Grove. Needs to be update if there's another "primary" testnet.
 PATH_URL ?= http://localhost:3000
 POCKET_ADDR_PREFIX = pokt
@@ -473,3 +474,4 @@ include ./makefiles/session.mk
 include ./makefiles/claims.mk
 include ./makefiles/relay.mk
 include ./makefiles/ping.mk
+include ./makefiles/migrate.mk

cmd/morse_keys.go

@@ -0,0 +1,13 @@
+package cmd
+
+// TODO_TECHDEBT: This file is not part of pkg/crypto because it is intended to be removed after the migration.
+
+// ArmoredJson is a data structure which is used to (de)serialize the encrypted exported Morse private key file.
+// Copy-pasted from https://github.com/pokt-network/pocket-core/blob/2cd25e82095dc52939fef58e9cc3deb1923c01f7/crypto/keys/mintkey/mintkey.go#L96
+type ArmoredJson struct {
+	Kdf        string `json:"kdf" yaml:"kdf"`
+	Salt       string `json:"salt" yaml:"salt"`
+	SecParam   string `json:"secparam" yaml:"secparam"`
+	Hint       string `json:"hint" yaml:"hint"`
+	Ciphertext string `json:"ciphertext" yaml:"ciphertext"`
+}

e2e/tests/init_test.go

@@ -26,6 +26,8 @@ import (
 	cometcli "github.com/cometbft/cometbft/libs/cli"
 	cometjson "github.com/cometbft/cometbft/libs/json"
 	"github.com/cosmos/cosmos-sdk/codec"
+	codectypes "github.com/cosmos/cosmos-sdk/codec/types"
+	authtypes "github.com/cosmos/cosmos-sdk/x/auth/types"
 	"github.com/gorilla/websocket"
 	"github.com/regen-network/gocuke"
 	"github.com/stretchr/testify/require"
@@ -50,6 +52,8 @@ const (
 	numQueryRetries = uint8(3)
 	unbondingPeriod = "unbonding"
 	transferPeriod  = "transfer"
+	oneshotTag      = "@oneshot"
+	manualTag       = "@manual"
 )
 
 var (
@@ -67,6 +71,10 @@ var (
 	chainIdFlag      = "--chain-id=pocket"
 	// pathUrl points to a local gateway using the PATH framework in centralized mode.
 	pathUrl = "http://localhost:3000/v1" // localhost is kept as the default to streamline local development & testing.
+
+	// allFeaturesTags is a tag expression that filters out features (i.e. E2E tests)
+	// which SHOULD NOT be included in a wildcard/glob of feature files.
+	allFeaturesTags = fmt.Sprintf("not %s and not %s", manualTag, oneshotTag)
 )
 
 func init() {
@@ -182,8 +190,10 @@ func (s *suite) Before() {
 func TestFeatures(t *testing.T) {
 	gocuke.NewRunner(t, &suite{}).Path(flagFeaturesPath).
 		// Ignore test elements (e.g. Features, Scenarios, etc.)
-		// with the @manual tag (e.g. migration.feature).
-		Tags("not @manual").Run()
+		// which should not be included in a wildcard/glob of feature files.
+		// For example, these include the @manual or @oneshot tags used for migration*.feature.
+		Tags(allFeaturesTags).
+		Run()
 }
 
 // TODO_TECHDEBT: rename `pocketd` to `pocketd`.
@@ -936,15 +946,15 @@ func (s *suite) getCurrentBlockHeight() int64 {
 	res.Stdout = strings.Join(stdoutLines[1:], "\n")
 
 	var blockRes struct {
-		LastCommit struct {
+		Header struct {
 			Height int64 `json:"height"`
-		} `json:"last_commit"`
+		} `json:"header"`
 	}
 
 	err = cometjson.Unmarshal([]byte(res.Stdout), &blockRes)
 	require.NoError(s, err)
 
-	return blockRes.LastCommit.Height
+	return blockRes.Header.Height
 }
 
 // readEVMSubscriptionEvents reads the eth_subscription events from the websocket
@@ -980,6 +990,30 @@ func (s *suite) readEVMSubscriptionEvents() context.Context {
 	return ctx
 }
 
+// queryAccount queries the auth module for the account associated with the given address.
+func (s *suite) queryAccount(accAddr string) (account *codectypes.Any, isFound bool) {
+	args := []string{
+		"query",
+		"auth",
+		"account",
+		accAddr,
+		"--output=json",
+	}
+
+	result, err := s.pocketd.RunCommandOnHost("", args...)
+	require.NotNil(s, result)
+	if strings.Contains(result.Stderr, "not found") {
+		return nil, false
+	}
+	require.NoError(s, err, "error getting account for address %s", accAddr)
+
+	var resp authtypes.QueryAccountResponse
+	responseBz := []byte(strings.TrimSpace(result.Stdout))
+	s.cdc.MustUnmarshalJSON(responseBz, &resp)
+
+	return resp.Account, true
+}
+
 // accBalanceKey is a helper function to create a key to store the balance
 // for accName in the context of a scenario state.
 func accBalanceKey(accName string) string {

e2e/tests/migration_fixture.feature

@@ -0,0 +1,60 @@
+# NOTE: The @oneshot tag allows this feature to be
+# excluded from any wildcard feature file execution (e.g. *.feature).
+#
+# The @oneshot tag indicates that a given feature is non-idempotent with respect to its impact on the network state.
+# In such cases, a complete network reset is required before running these features again.
+@oneshot
+Feature: Morse account import and claim all account types (with fixture data)
+
+  Background:
+    Given the user has the pocketd binary installed
+    And a MorseAccountState with "10" accounts in a "round-robin" distribution has successfully been imported
+    And an unclaimed MorseClaimableAccount with a known private key exists
+    And a Shannon destination key exists in the local keyring
+
+  Rule: Non-actor account claims MAY reference existing Shannon accounts
+    Scenario: Morse account-holder claims as a new non-actor account
+      Given the Shannon destination account does not exist onchain
+      # TODO_MAINNET_CRITICAL(@bryanchriswhite): Use a new token denomination.
+      And the Shannon account is funded with "1upokt"
+      When the Morse private key is used to claim a MorseClaimableAccount as a non-actor account
+      Then the Shannon destination account balance is increased by the sum of all MorseClaimableAccount tokens
+      And the Morse claimable account is marked as claimed by the shannon account at a recent block height
+
+    Scenario: Morse account-holder claims as an existing non-actor account
+      Given the Shannon destination account exists onchain
+      And the Shannon destination account upokt balance is non-zero
+      When the Morse private key is used to claim a MorseClaimableAccount as a non-actor account
+      Then the Shannon destination account balance is increased by the sum of all MorseClaimableAccount tokens
+      And the Morse claimable account is marked as claimed by the shannon account at a recent block height
+
+# TODO_MAINNET_CRITICAL(@bryanchriswhite, #1034: Uncomment the following scenarios once application and supplier Morse account claiming is available.
+#
+#  Rule: Actor (re-)stake claims MAY reference existing Shannon actors
+#    Scenario Outline: Morse account-holder claims as a new staked actor
+#      Given the Shannon destination account is not staked as an "<actor>"
+#      When the Morse private key is used to claim a MorseClaimableAccount as an "<actor>"
+#      Then the Shannon destination account balance is increased by the unstaked balance amount of the MorseClaimableAccount
+#      And the Shannon destination account is staked as an "<actor>"
+#      And the Shannon "<actor>" stake increased by the "<stake_amount_field>" of the MorseClaimableAccount
+#      And the Shannon "<actor>" service config is updated, if applicable
+#
+#      Examples:
+#        | actor       | stake_amount_field |
+#        | application | application_stake  |
+#        | supplier    | supplier_stake     |
+#
+#    Scenario Outline: Morse account-holder claims as an existing staked actor
+#      Given the Shannon destination account is staked as an "<actor>"
+#      When the Morse private key is used to claim a MorseClaimableAccount as an "<actor>"
+#      Then the Shannon destination account balance is increased by the unstaked balance amount of the MorseClaimableAccount
+#      And the Shannon destination account is staked as an "<actor>"
+#      And the Shannon "<actor>" stake increased by the "<stake_amount_field>" of the MorseClaimableAccount
+#      And the Shannon "<actor>" service config is updated, if applicable
+#
+#      Examples:
+#        | actor       | stake_amount_field |
+#        | application | application_stake  |
+#        | supplier    | supplier_stake     |
+
+# TODO_MAINNET_CRITICAL(@bryanchriswhite, #1034): Enumerate and implement error scenarios.

e2e/tests/migration_snapshot.feature

@@ -0,0 +1,71 @@
+# NOTE: The @oneshot & @manual tags allows this feature to be
+# excluded from any wildcard feature file execution (e.g. *.feature).
+#
+# The @manual tag indicates that a given feature depends on some non-automated
+# setup which MUST be performed manually, prior to running the feature.
+#
+# The @oneshot tag indicates that a given feature is non-idempotent with respect
+# to its impact on the network state. In such cases, a complete network reset
+# is required before running these features again.
+@oneshot @manual
+Feature: Morse account import and claim all account types (with snapshot data)
+
+  # TODO_MAINNET_CRITICAL(@bryanchriswhite, #1034): The snapshot based Morse account import feature is incomplete.
+
+  Scenario: Authority generates and imports MorseAccountState
+    # TODO_MAINNET_CRITICAL(@bryanchriswhite, #1034): Print a link to the latest liquify snapshot if no local state exists.
+    Given a local Morse node persisted state exists
+    # TODO_POST_MAINNET: Replace current height with the published "canonical" export/migration/cutover height.
+    When the authority exports the Morse Account State at height "130000" to "morse_state_export.json"
+    Then a MorseStateExport is written to "morse_state_export.json"
+
+    When the authority executes "poktrolld tx migration collect-morse-accounts morse_state_export.json morse_account_state.json"
+    Then a MorseAccountState is written to "morse_account_state.json"
+
+    Given no MorseClaimableAccounts exist
+    And the MorseAccountState in "morse_account_state.json" is valid
+    When the authority executes "poktrolld tx migration import-morse-accounts morse_account_state.json"
+    Then the MorseClaimableAccounts are persisted onchain
+
+  Scenario:
+    Given a Morse node snapshot is available
+    And the authority successfully imports MorseAccountState generated from the snapshot state
+    # TODO_MAINNET_CRITICAL(@bryanchriswhite): Ensure the liquify Morse snapshot includes known Morse
+    # private keys such that valid claim signatures can be generated for testing.
+    #
+    # Use a distinct SECRET random number to seed each private key needed.
+    # The implementation could read these out of a single new-line delimited
+    # env var (e.g. MORSE_KEY_SEED_0).
+    And "6" Morse private keys are available in a "round-robin" actor type distribution
+
+    When a Morse account-holder claims as a new non-actor account
+    Then the Shannon destination account balance is increased by the sum of all MorseClaimableAccount tokens
+    And the Morse claimable account is marked as claimed by the shannon account at a recent block height
+
+    When a Morse account-holder claims as an existing non-actor account
+    Then the Shannon destination account balance is increased by the MorseClaimableAccount unstaked balance
+    And the Morse claimable account is marked as claimed by the shannon account at a recent block height
+
+    When a Morse account-holder claims as a new application
+    Then the Shannon destination account balance is increased by the MorseClaimableAccount unstaked balance
+    And the Morse claimable account is marked as claimed by the shannon account at a recent block height
+    And the Shannon destination account is staked as an application with the stake equal to the onchain MorseClaimableAccount
+
+    Given an application is staked
+    When a Morse account-holder claims as an existing application
+    Then the Shannon destination account balance is increased by the MorseClaimableAccount unstaked balance
+    And the Morse claimable account is marked as claimed by the shannon account at a recent block height
+    And the Shannon destination account application stake is increased by the MorseClaimableAccount application stake
+
+    When a Morse account-holder claims as a new supplier
+    Then the Shannon destination account balance is increased by the MorseClaimableAccount unstaked balance
+    And the Morse claimable account is marked as claimed by the shannon account at a recent block height
+    And the Shannon destination account is staked as an supplier with the stake equal to the onchain MorseClaimableAccount
+
+    Given a supplier is staked
+    When a Morse account-holder claims as an existing supplier
+    Then the Shannon destination account balance is increased by the MorseClaimableAccount unstaked balance
+    And the Morse claimable account is marked as claimed by the shannon account at a recent block height
+    And the Shannon destination account supplier stake is increased by the MorseClaimableAccount supplier stake
+
+  # TODO_MAINNET_CRITICAL(@bryanchriswhite, #1034): Enumerate and implement error scenarios.

e2e/tests/migration_snapshot_test.go

@@ -0,0 +1,24 @@
+//go:build e2e && oneshot && manual
+
+package e2e
+
+import (
+	"testing"
+
+	"github.com/regen-network/gocuke"
+)
+
+// TestMigrationWithSnapshotData runs the migration_snapshot.feature file ONLY.
+// NOTE: This test depends on a large Morse node snapshot being available locally.
+// See: https://pocket-snapshot.liquify.com/#/pruned/
+//
+// To run this test use:
+//
+//	$ make test_e2e_migration_snapshot
+//
+// TODO_MAINNET_CRITICAL(@bryanchriswhite): Add an example of how to get the snapshot (e.g. wget ...)
+func TestMigrationWithSnapshotData(t *testing.T) {
+	gocuke.NewRunner(t, &migrationSuite{}).
+		Path("migration_snapshot.feature").
+		Run()
+}