Merge pull request #17 from DataHighway-DHX/luke/migrate-standalone-pr-215-step-by-step

feat: Migrate pallets from standalone chain that was updated to common branch 'polkadot-v0.9.17' progressively

Author: ltfschoen

.env.sample

@@ -18,16 +18,16 @@ jobs:
       - name: Add wasm toolchain
         uses: actions-rs/toolchain@v1
         with:
-          toolchain: nightly-2021-09-29
+          toolchain: nightly-2021-12-15
           target: wasm32-unknown-unknown
           override: true
 
       - name: Initialize environment and build
         run: |
           rustup update stable
-          rustup toolchain install nightly-2021-09-29
-          rustup default nightly-2021-09-29
-          rustup target add wasm32-unknown-unknown --toolchain nightly-2021-09-29
+          rustup toolchain install nightly-2021-12-15
+          rustup default nightly-2021-12-15
+          rustup target add wasm32-unknown-unknown --toolchain nightly-2021-12-15
           cargo build --release
 
       - name: Upload artifact

.github/workflows/release.yml

@@ -26,12 +26,12 @@ jobs:
       - name: Initialize WASM build environment
         run: |
           rustup update stable
-          rustup toolchain install nightly-2021-11-07
-          rustup default nightly-2021-11-07
-          rustup target add wasm32-unknown-unknown --toolchain nightly-2021-11-07
+          rustup toolchain install nightly-2021-12-15
+          rustup default nightly-2021-12-15
+          rustup target add wasm32-unknown-unknown --toolchain nightly-2021-12-15
 
       - name: Run cargo check
-        run: cargo +nightly-2021-11-07 check
+        run: cargo +nightly-2021-12-15 check
 
   test:
     name: Test Suite
@@ -56,14 +56,16 @@ jobs:
       - name: Initialize WASM build environment
         run: |
           rustup update stable
-          rustup toolchain install nightly-2021-11-07
-          rustup default nightly-2021-11-07
-          rustup target add wasm32-unknown-unknown --toolchain nightly-2021-11-07
+          rustup toolchain install nightly-2021-12-15
+          rustup default nightly-2021-12-15
+          rustup target add wasm32-unknown-unknown --toolchain nightly-2021-12-15
 
       # Runs integration tests in the runtime/tests folder
+      # To run tests in a specific package run
+      # `cargo +nightly-2021-12-15 test -p <package name> (i.e. datahighway-runtime)
       - name: Run All Tests
         run: |
-          cargo +nightly-2021-11-07 test
+          cargo +nightly-2021-12-15 test
 
   lints:
     name: Lints
@@ -88,19 +90,19 @@ jobs:
           override: true
           components: rustfmt, clippy
 
-      # nightly-2021-11-07-x86_64-unknown-linux-gnu
+      # nightly-2021-12-15-x86_64-unknown-linux-gnu
       - name: Initialize WASM build environment
         run: |
           rustup update stable
-          rustup toolchain install nightly-2021-11-07
-          rustup default nightly-2021-11-07
-          rustup target add wasm32-unknown-unknown --toolchain nightly-2021-11-07
+          rustup toolchain install nightly-2021-12-15
+          rustup default nightly-2021-12-15
+          rustup target add wasm32-unknown-unknown --toolchain nightly-2021-12-15
 
       # Ignore use of Rust Linting since rearranging makes it challenging to update to latest Substrate
       # - name: Run cargo fmt
       # run: |
-      #   rustup component add rustfmt --toolchain nightly-2021-11-07-x86_64-unknown-linux-gnu
-      #   cargo +nightly-2021-11-07 fmt --all -- --check
+      #   rustup component add rustfmt --toolchain nightly-2021-12-15-x86_64-unknown-linux-gnu
+      #   cargo +nightly-2021-12-15 fmt --all -- --check
 
       # Ignore use of Clippy until a clippy.toml file may be used. See README.md
       # - name: Run cargo clippy
@@ -110,6 +112,6 @@ jobs:
       #   #   args:  --release -- -D warnings
       #   # FIXME - temporary fix below. See https://github.com/rust-lang/rust-clippy/issues/5094#issuecomment-579116431
       #   run: |
-      #     rustup component add clippy --toolchain nightly-2021-11-07-x86_64-unknown-linux-gnu
-      #     rustup component add clippy-preview --toolchain nightly-2021-11-07-x86_64-unknown-linux-gnu
-      #     cargo +nightly-2021-11-07 clippy-preview -Zunstable-options
+      #     rustup component add clippy --toolchain nightly-2021-12-15-x86_64-unknown-linux-gnu
+      #     rustup component add clippy-preview --toolchain nightly-2021-12-15-x86_64-unknown-linux-gnu
+      #     cargo +nightly-2021-12-15 clippy-preview -Zunstable-options

.github/workflows/test.yml

@@ -15,3 +15,10 @@ pallets/.DS_Store
 
 *-babe.json
 *-gran.json
+
+# autogenerated chain_specs
+chain_dumps/
+rococo-local*
+westend-local*
+kusama-local*
+polkadot-local*

.gitignore

@@ -5,6 +5,9 @@
 * [Linting](#chapter-c345d7)
 * [Debugging](#chapter-93c645)
 * [Testing](#chapter-e146ec)
+* [Benchmarking](#chapter-6c1b24)
+* [Try-Runtime](#chapter-397b84)
+* [Memory Profiling](#chapter-585a25)
 * [Code Editor Configuration](#chapter-d5a9de)
 * [Create new runtime modules](#chapter-18873f)
 * [FAQ](#chapter-f078a2)
@@ -23,7 +26,12 @@ To skip running the CI unnecessarily for simple changes such as updating the doc
 
 ### Linting
 
-Please apply Rust Format on your changes prior to creating a PR. See [Linting](#chapter-c345d7).
+Check with Rust Format. Note: If you need a specific version of it replace `+nightly` with say `+nightly-2022-03-16`
+```
+cargo +nightly fmt --all -- --check
+```
+
+If you wish to apply Rust Format on your changes prior to creating a PR. See [Linting](#chapter-c345d7).
 
 ```bash
 cargo +nightly fmt --all
@@ -61,9 +69,10 @@ use log::{error, info, debug, trace};
 ...
 log::debug!("hello {:?}", world); // Only shows in terminal in debug mode
 log::info!("hello {:?}", world); // Shows in terminal in release mode
-debug::native::info!("hello {:?}", world);
 ```
 
+Note: The use of `debug::native::info!("hello {:?}", world);` does not appear to work anymore since Substrate updates in Feb 2021.
+
 ### Detailed Debugging
 
 ```bash
@@ -72,6 +81,8 @@ RUST_LOG=debug RUST_BACKTRACE=1 ./target/release/datahighway ... \
   -lruntime=debug
 ```
 
+Refer to Susbtrate Debugging documentation [here](https://docs.substrate.io/v3/runtime/debugging/)
+
 ## Testing <a id="chapter-e146ec"></a>
 
 ### Run All Tests
@@ -93,16 +104,16 @@ cargo test -p roaming-sessions &&
 cargo test -p roaming-billing-policies &&
 cargo test -p roaming-charging-policies &&
 cargo test -p roaming-packet-bundles &&
-cargo test -p mining-speed-boosts-configuration-token-mining &&
-cargo test -p mining-speed-boosts-configuration-hardware-mining &&
-cargo test -p mining-speed-boosts-rates-token-mining &&
-cargo test -p mining-speed-boosts-rates-hardware-mining &&
-cargo test -p mining-speed-boosts-sampling-token-mining &&
-cargo test -p mining-speed-boosts-sampling-hardware-mining &&
-cargo test -p mining-speed-boosts-eligibility-token-mining &&
-cargo test -p mining-speed-boosts-eligibility-hardware-mining &&
-cargo test -p mining-speed-boosts-lodgements-token-mining &&
-cargo test -p mining-speed-boosts-lodgements-hardware-mining
+cargo test -p mining-setting-token &&
+cargo test -p mining-setting-hardware &&
+cargo test -p mining-rates-token &&
+cargo test -p mining-rates-hardware &&
+cargo test -p mining-sampling-token &&
+cargo test -p mining-sampling-hardware &&
+cargo test -p mining-eligibility-token &&
+cargo test -p mining-eligibility-hardware &&
+cargo test -p mining-claims-token &&
+cargo test -p mining-claims-hardware
 ```
 
 ### Run Integration Tests Only
@@ -118,6 +129,78 @@ Example
 cargo test -p datahighway-parachain-runtime --test cli_integration_tests_mining_tokens
 ```
 
+## Benchmarking <a id="chapter-6c1b24"></a>
+
+Run the following:
+```
+./scripts/benchmark_all_pallets.sh
+```
+
+## Try-Runtime <a id="chapter-397b84"></a>
+
+* Run Collator nodes
+
+* Build whilst specifying the `try-runtime` feature
+```
+cargo build --release features=try-runtime
+```
+
+* Run Try-Runtime so `on-runtime-upgrade` will invoke all  `OnRuntimeUpgrade` hooks in pallets and the runtime
+```
+RUST_LOG=runtime=trace,try-runtime::cli=trace,executor=trace \
+./target/release/datahighway-collator \
+try-runtime \
+--chain <chain-spec> \
+--execution Wasm \
+--wasm-execution Compiled \
+--uri <ws/s port>
+--block-at <block-hash> \
+on-runtime-upgrade \
+live
+```
+
+Notes:
+* Ensure that the Collator node was run with:
+```
+--rpc-max-payload 1000 \
+--rpc-cors=all \
+```
+* The `--chain` argument must be provided
+* Provide a `--uri` and `--block-at` hash from the testnet where the Collator node was launched. The defaults are the wss://127.0.0.1:9944 port and the latest finalized block respectively.
+* `live` means we are going to scrape a live testnet, as opposed to loading a saved file.
+
+References:
+* https://docs.substrate.io/how-to-guides/v3/tools/try-runtime/
+* https://docs.substrate.io/v3/tools/try-runtime/
+
+## Memory Profiling <a id="chapter-585a25"></a>
+
+```
+curl -L https://github.com/koute/memory-profiler/releases/download/0.6.1/memory-profiler-x86_64-unknown-linux-gnu.tgz -o memory-profiler-x86_64-unknown-linux-gnu.tgz
+tar -xf memory-profiler-x86_64-unknown-linux-gnu.tgz
+
+export MEMORY_PROFILER_LOG=info
+export MEMORY_PROFILER_LOGFILE=profiling_%e_%t.log
+export MEMORY_PROFILER_OUTPUT=profiling_%e_%t.dat
+export MEMORY_PROFILER_CULL_TEMPORARY_ALLOCATIONS=1
+```
+
+It should only be run on a testnet. See https://github.com/paritytech/subport/issues/257.
+Purge local chain from previous tests, then:
+```
+LD_PRELOAD=<INSERT_PATH_TO_MEMORY_PROFILER>/libmemory_profiler.so \
+./target/release/datahighway-collator <INSERT_TESTNET_ARGS>
+```
+
+```
+./memory-profiler-cli server *.dat
+```
+
+View output at http://localhost:8080/
+
+Reference:
+* https://docs.substrate.io/v3/tools/memory-profiling/
+
 ## Continuous Integration <a id="chapter-7a8301"></a>
 
 Github Actions are used for Continuous Integration.
@@ -206,68 +289,8 @@ substrate-module-new <module-name> <author>
 
 ## FAQ <a id="chapter-f078a2"></a>
 
-* Question: Why do we need to install Rust Stable and Rust Nightly?
-	* Answer: In .github/workflows/rust.yml, we need to run the following,
-	because Substrate builds two binaries: 1) Wasm binary of your Runtime;
-	and 2) Native executable containing all your other Substrate components
-	including your runtimes too. The Wasm build requires rust nightly and
-	wasm32-unknown-unknown to be installed. Note that we do not use
-	`rustup update nightly` since the latest Rust Nightly may break our build,
-	so we must manually change this to the latest Rust Nightly version only
-	when it is known to work.
-		```bash
-		rustup toolchain install nightly-2020-12-12
-		rustup update stable
-		rustup target add wasm32-unknown-unknown --toolchain nightly
-		```
-
-* Question: Why do we install a specific version of Rust Nightly in the CI?
-	* Answer: Since the latest version of Rust Nightly may break our build,
-	and because developers may forget to update to the latest version of Rust
-	Nightly locally. So the solution is to install a specific version of
-	Rust Nightly in .github/workflows/rust.yml (i.e.
-	`rustup toolchain install nightly-2020-12-12`), since for example
-	the latest Rust Nightly version nightly-2020-02-20 may cause our CI tests
-	to fail (i.e. https://github.com/DataHighway-DHX/node/issues/32)
-
-* Question: Why does the `SessionKeys` struct of our chain only have [babe and grandpa](https://github.com/DataHighway-DHX/node/blob/master/runtime/src/lib.rs#L94), and not [im_online and authority_discovery](https://github.com/paritytech/substrate/blob/master/bin/node/runtime/src/lib.rs#L242).
-	* Answer: Since we'll be a parachain im_online and authority_discovery are not required here.
-
-* Question: How do I install specific dependencies
-	* Answer:
-		```bash
-		cargo install cargo-edit
-		cargo add ...
-		```
-* Question: How do I upgrade the runtime without stopping the blockchain
-	* Answer: https://www.youtube.com/watch?v=0aTnxHrV_j4&list=PLOyWqupZ-WGt3mA_d9wu74vVe0bM37-39&index=9&t=0s
-
-* Question: How may I debug and contribute to fixing UI errors or any errors in the browser console that I encounter when using Polkadot.js Apps https://polkadot.js.org/apps?
-	* Answer: If you run Polkadot.js Apps locally from your machine then the errors are easier to debug. Follow the instructions at https://github.com/polkadot-js/apps, including cloning it, and running it. Try to identify and fix the error, and raise an issue in that repository if necessary.
-
-* Question: How do I stop and remove all the Docker containers and images?
-	* Answer: Run `./scripts/docker-clean.sh`
-	* **WARNING**: This stops and removes **all** your Docker containers and images, not just DataHighway relates ones.
-
-* Question: How to access the Docker container of a running node and run shell commands?
-	* Answer: `docker exec -it node_alice_1 /bin/bash`, where `node_alice_1` is the Container Name that is shown when you run `docker ps -a`.
-
-* Question: How do I restart the testnet Docker containers (including each chain databases)?
-	* Answer: Run the following, where `node_alice_1` is a Container Name that is shown when you run `docker ps -a`.
-		```bash
-		docker stop node_alice_1 node_bob_1 node_charlie_1
-		docker rm node_alice_1 node_bob_1 node_charlie_1
-		docker-compose --verbose up -d
-		docker-compose logs -f
-		```
-
-* Question: Why can't I syncronize my node?
-	* Answer: Run `./scripts/docker-clean.sh` before starting them again with either `docker-compose up` or `docker-compose --verbose up -d; docker-compose logs -f`, incase a cached image is still being used locally
-	* **WARNING**: This stops and removes **all** your Docker containers and images, not just DataHighway relates ones.
-
-* Question: How do I run two nodes on the same host machine?
-	* Answer:
-		* Refer to "Testnet (Alpha) "testnet_latest" PoS testnet (with multiple nodes)" in [EXAMPLES](./EXAMPLES.md).
+The latest FAQ is still recorded on the DataHighway standalone codebase [here](https://github.com/DataHighway-DHX/node/blob/master/CONTRIBUTING.md#faq-), or modified in subsequent PRs.
+It will be migrated into the DataHighway/documentation codebase.
 
 ## Technical Support <a id="chapter-c00ab7"></a>