feat: rate limit and logging improvements (#533)

* feat: rate limit and logging improvements

* Update cmd/loadtest/uniswapv3.go

Co-authored-by: Léo Vincent <[email protected]>

* ci: make gen

* fix: shadow

* fix: protoc

---------

Co-authored-by: Léo Vincent <[email protected]>

Author: praetoriansentry

cmd/loadtest/app.go

@@ -219,7 +219,7 @@ func initFlags() {
 	ltp.ToRandom = LoadtestCmd.PersistentFlags().Bool("to-random", false, "When doing a transfer test, should we send to random addresses rather than DEADBEEFx5")
 	ltp.CallOnly = LoadtestCmd.PersistentFlags().Bool("call-only", false, "When using this mode, rather than sending a transaction, we'll just call. This mode is incompatible with adaptive rate limiting, summarization, and a few other features.")
 	ltp.CallOnlyLatestBlock = LoadtestCmd.PersistentFlags().Bool("call-only-latest", false, "When using call only mode with recall, should we execute on the latest block or on the original block")
-	ltp.EthAmountInWei = LoadtestCmd.PersistentFlags().Float64("eth-amount", 0.001, "The amount of ether to send on every transaction")
+	ltp.EthAmountInWei = LoadtestCmd.PersistentFlags().Float64("eth-amount", 0, "The amount of ether to send on every transaction")
 	ltp.RateLimit = LoadtestCmd.PersistentFlags().Float64("rate-limit", 4, "An overall limit to the number of requests per second. Give a number less than zero to remove this limit all together")
 	ltp.AdaptiveRateLimit = LoadtestCmd.PersistentFlags().Bool("adaptive-rate-limit", false, "Enable AIMD-style congestion control to automatically adjust request rate")
 	ltp.SteadyStateTxPoolSize = LoadtestCmd.PersistentFlags().Uint64("steady-state-tx-pool-size", 1000, "When using adaptive rate limiting, this value sets the target queue size. If the queue is smaller than this value, we'll speed up. If the queue is smaller than this value, we'll back off.")

cmd/loadtest/loadtest.go

@@ -5,6 +5,7 @@ import (
 	_ "embed"
 	"errors"
 	"fmt"
+	ethtypes "github.com/ethereum/go-ethereum/core/types"
 	"math/big"
 	"time"
 
@@ -130,12 +131,15 @@ func initUniswapV3Loadtest(ctx context.Context, c *ethclient.Client, tops *bind.
 }
 
 // Run UniswapV3 loadtest.
-func runUniswapV3Loadtest(ctx context.Context, c *ethclient.Client, nonce uint64, uniswapV3Config uniswapv3loadtest.UniswapV3Config, poolConfig uniswapv3loadtest.PoolConfig, swapAmountIn *big.Int) (t1 time.Time, t2 time.Time, err error) {
+func runUniswapV3Loadtest(ctx context.Context, c *ethclient.Client, nonce uint64, uniswapV3Config uniswapv3loadtest.UniswapV3Config, poolConfig uniswapv3loadtest.PoolConfig, swapAmountIn *big.Int) (t1 time.Time, t2 time.Time, txHash common.Hash, err error) {
+	var tops *bind.TransactOpts
+	var tx *ethtypes.Transaction
+
 	ltp := inputLoadTestParams
 	chainID := new(big.Int).SetUint64(*ltp.ChainID)
 	privateKey := ltp.ECDSAPrivateKey
 
-	tops, err := bind.NewKeyedTransactorWithChainID(privateKey, chainID)
+	tops, err = bind.NewKeyedTransactorWithChainID(privateKey, chainID)
 	if err != nil {
 		log.Error().Err(err).Msg("Unable create transaction signer")
 		return
@@ -145,6 +149,9 @@ func runUniswapV3Loadtest(ctx context.Context, c *ethclient.Client, nonce uint64
 
 	t1 = time.Now()
 	defer func() { t2 = time.Now() }()
-	err = uniswapv3loadtest.ExactInputSingleSwap(tops, uniswapV3Config.SwapRouter02.Contract, poolConfig, swapAmountIn, *ltp.FromETHAddress, nonce)
+	tx, err = uniswapv3loadtest.ExactInputSingleSwap(tops, uniswapV3Config.SwapRouter02.Contract, poolConfig, swapAmountIn, *ltp.FromETHAddress, nonce)
+	if err == nil && tx != nil {
+		txHash = tx.Hash()
+	}
 	return
 }

cmd/loadtest/uniswapv3.go

@@ -1,6 +1,7 @@
 package uniswapv3loadtest
 
 import (
+	"github.com/ethereum/go-ethereum/core/types"
 	"math/big"
 
 	"github.com/0xPolygon/polygon-cli/bindings/uniswapv3"
@@ -15,13 +16,13 @@ var SwapAmountInput = big.NewInt(1_000)
 // ExactInputSingleSwap performs a UniswapV3 swap using the `ExactInputSingle` method which swaps a fixed amount of
 // one token for a maximum possible amount of another token. The direction of the swap is determined
 // by the nonce value.
-func ExactInputSingleSwap(tops *bind.TransactOpts, swapRouter *uniswapv3.SwapRouter02, poolConfig PoolConfig, amountIn *big.Int, recipient common.Address, nonce uint64) error {
+func ExactInputSingleSwap(tops *bind.TransactOpts, swapRouter *uniswapv3.SwapRouter02, poolConfig PoolConfig, amountIn *big.Int, recipient common.Address, nonce uint64) (tx *types.Transaction, err error) {
 	// Determine the direction of the swap.
 	swapDirection := getSwapDirection(nonce, poolConfig)
 
 	// Perform swap.
 	amountOut := new(big.Int).Mul(amountIn, new(big.Int).Div(big.NewInt(98), big.NewInt(100)))
-	_, err := swapRouter.ExactInputSingle(tops, uniswapv3.IV3SwapRouterExactInputSingleParams{
+	tx, err = swapRouter.ExactInputSingle(tops, uniswapv3.IV3SwapRouterExactInputSingleParams{
 		// The contract address of the inbound token.
 		TokenIn: swapDirection.tokenIn,
 		// The contract address of the outbound token.
@@ -41,30 +42,30 @@ func ExactInputSingleSwap(tops *bind.TransactOpts, swapRouter *uniswapv3.SwapRou
 	})
 	if err != nil {
 		log.Error().Err(err).Str("tokenIn", swapDirection.tokenInName).Str("tokenOut", swapDirection.tokenOutName).Interface("amountIn", amountIn).Msg("Unable to swap")
-		return err
+		return
 	}
 	log.Trace().Str("tokenIn", swapDirection.tokenInName).Str("tokenOut", swapDirection.tokenOutName).Interface("amountIn", amountIn).Msg("Successful swap")
-	return nil
+	return
 }
 
 // swapDirection represents a swap direction with the inbound and outbound tokens.
-type swapDirection struct {
+type uniswapDirection struct {
 	tokenIn, tokenOut         common.Address
 	tokenInName, tokenOutName string
 }
 
 // Return the direction of the swap given the nonce value.
-func getSwapDirection(nonce uint64, poolConfig PoolConfig) swapDirection {
+func getSwapDirection(nonce uint64, poolConfig PoolConfig) uniswapDirection {
 	if nonce%2 == 0 {
-		return swapDirection{
+		return uniswapDirection{
 			tokenIn:     poolConfig.Token0.Address,
 			tokenInName: "token0",
 
 			tokenOut:     poolConfig.Token1.Address,
 			tokenOutName: "token1",
 		}
 	}
-	return swapDirection{
+	return uniswapDirection{
 		tokenIn:     poolConfig.Token1.Address,
 		tokenInName: "token1",
 

cmd/loadtest/uniswapv3/swap.go

@@ -9,6 +9,13 @@ To try this out, first checkout https://github.com/ethereum/tests/
 # Move into the filler directory
 cd tests/src
 
+# at this point, it's a good time to check if you have a good version
+# of yq installed. If you run this command or something like it and
+# the values are turned into decimals or floating point numbers, this
+# will be a problem. This particular version of yq handles this well:
+# https://github.com/mikefarah/yq
+cat ./GeneralStateTestsFiller/stPreCompiledContracts2/ecrecoverShortBuffFiller.yml | ~/go/bin/yq -o json
+
 # Convert the yaml based tests to json. There will be some failures depending on the version of yq used
 find . -type f -name '*.yml'  | while read yaml_file ; do
     yq -o json '.' $yaml_file > $yaml_file.json
@@ -18,17 +25,17 @@ find . -type f -name '*.yml'  | while read yaml_file ; do
     fi
 done
 
-
 # Check for duplicates... There are a few so we should be mindful of that
 find . -type f -name '*.json' | xargs cat | jq -r 'to_entries[].key' | uniq -c | sort
 
 # Consolidate everything.. The kzg tests seem to be a different format.. So excluding them with the array check
 find . -type f -name '*.json' | xargs cat | jq 'select((. | type) != "array")' | jq -s 'add' > merged.json
+
 # there are some fields like "//comment" that make parsing very difficult
 jq 'walk(if type == "object" then with_entries(select(.key | startswith("//") | not)) else . end)' merged.json  > merged.nocomment.json
 ```
 
-Now we should have a giant file filled with an array of transactions. We can take that output and process it witht the `retest` command now
+Now we should have a giant file filled with an array of transactions. We can take that output and process it with the `retest` command now
 
 ```bash
 polycli retest -v 500 --file merged.nocomment.json > simple.json
@@ -39,7 +46,9 @@ polycli retest -v 500 --file merged.nocomment.json > simple.json
 This project will depend on an installation of `solc` (specifically
 0.8.20) and `lllc`. Installing solidity is pretty easy, but LLLC can
 be a little tricky.
-If you got multiple solc versions, you can specify full patch to the right binary to use with env var ```SOLC_PATH```.
+
+If you've got multiple solc versions, you can specify full path to the
+right binary to use with env var ```SOLC_PATH```.
 
 Since the version is pretty old, it might not build well on your host
 os. Building within docker might make your life easier:

cmd/retest/usage.md

@@ -3,6 +3,7 @@ package cmd
 import (
 	"fmt"
 	"github.com/0xPolygon/polygon-cli/cmd/foldtrace"
+	"github.com/0xPolygon/polygon-cli/util"
 	"os"
 
 	"github.com/0xPolygon/polygon-cli/cmd/cdk"
@@ -13,8 +14,6 @@ import (
 	"github.com/0xPolygon/polygon-cli/cmd/fork"
 	"github.com/0xPolygon/polygon-cli/cmd/p2p"
 	"github.com/0xPolygon/polygon-cli/cmd/parseethwallet"
-	"github.com/0xPolygon/polygon-cli/util"
-
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 
@@ -49,6 +48,7 @@ var rootCmd *cobra.Command
 // Execute adds all child commands to the root command and sets flags appropriately.
 // This is called by main.main(). It only needs to happen once to the rootCmd.
 func Execute() {
+	cobra.EnableTraverseRunHooks = true
 	err := rootCmd.Execute()
 	if err != nil {
 		os.Exit(1)
@@ -91,13 +91,13 @@ func NewPolycliCommand() *cobra.Command {
 		Use:   "polycli",
 		Short: "A Swiss Army knife of blockchain tools.",
 		Long:  "Polycli is a collection of tools that are meant to be useful while building, testing, and running block chain applications.",
-		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
+		PersistentPreRun: func(cmd *cobra.Command, args []string) {
 			util.SetLogLevel(verbosity)
 			logMode := util.JSON
 			if pretty {
 				logMode = util.Console
 			}
-			return util.SetLogMode(logMode)
+			_ = util.SetLogMode(logMode)
 		},
 	}
 

cmd/root.go

@@ -116,7 +116,7 @@ The codebase has a contract that used for load testing. It's written in Solidity
       --contract-call-payable                  Use this flag if the function is payable, the value amount passed will be from --eth-amount. This must be paired up with --mode contract-call and --contract-address
       --erc20-address string                   The address of a pre-deployed ERC20 contract
       --erc721-address string                  The address of a pre-deployed ERC721 contract
-      --eth-amount float                       The amount of ether to send on every transaction (default 0.001)
+      --eth-amount float                       The amount of ether to send on every transaction
       --force-contract-deploy                  Some load test modes don't require a contract deployment. Set this flag to true to force contract deployments. This will still respect the --lt-address flags.
   -f, --function uint                          A specific function to be called if running with --mode f or a specific precompiled contract when running with --mode a (default 1)
       --function-arg strings                   The arguments that will be passed to a contract function call. This must be paired up with "--mode contract-call" and "--contract-address". Args can be passed multiple times: "--function-arg 'test' --function-arg 999" or comma separated values "--function-arg "test",9". The ordering of the arguments must match the ordering of the function parameters.

doc/polycli_loadtest.md

@@ -86,7 +86,7 @@ The command also inherits flags from parent commands.
       --chain-id uint                          The chain id for the transactions.
   -c, --concurrency int                        Number of requests to perform concurrently. Default is one request at a time. (default 1)
       --config string                          config file (default is $HOME/.polygon-cli.yaml)
-      --eth-amount float                       The amount of ether to send on every transaction (default 0.001)
+      --eth-amount float                       The amount of ether to send on every transaction
       --gas-limit uint                         In environments where the gas limit can't be computed on the fly, we can specify it manually. This can also be used to avoid eth_estimateGas
       --gas-price uint                         In environments where the gas price can't be determined automatically, we can specify it manually
   -i, --iterations uint                        If we're making contract calls, this controls how many times the contract will execute the instruction in a loop. If we are making ERC721 Mints, this indicates the minting batch size (default 1)

doc/polycli_loadtest_uniswapv3.md

@@ -30,6 +30,13 @@ To try this out, first checkout https://github.com/ethereum/tests/
 # Move into the filler directory
 cd tests/src
 
+# at this point, it's a good time to check if you have a good version
+# of yq installed. If you run this command or something like it and
+# the values are turned into decimals or floating point numbers, this
+# will be a problem. This particular version of yq handles this well:
+# https://github.com/mikefarah/yq
+cat ./GeneralStateTestsFiller/stPreCompiledContracts2/ecrecoverShortBuffFiller.yml | ~/go/bin/yq -o json
+
 # Convert the yaml based tests to json. There will be some failures depending on the version of yq used
 find . -type f -name '*.yml'  | while read yaml_file ; do
     yq -o json '.' $yaml_file > $yaml_file.json
@@ -39,17 +46,17 @@ find . -type f -name '*.yml'  | while read yaml_file ; do
     fi
 done
 
-
 # Check for duplicates... There are a few so we should be mindful of that
 find . -type f -name '*.json' | xargs cat | jq -r 'to_entries[].key' | uniq -c | sort
 
 # Consolidate everything.. The kzg tests seem to be a different format.. So excluding them with the array check
 find . -type f -name '*.json' | xargs cat | jq 'select((. | type) != "array")' | jq -s 'add' > merged.json
+
 # there are some fields like "//comment" that make parsing very difficult
 jq 'walk(if type == "object" then with_entries(select(.key | startswith("//") | not)) else . end)' merged.json  > merged.nocomment.json
 ```
 
-Now we should have a giant file filled with an array of transactions. We can take that output and process it witht the `retest` command now
+Now we should have a giant file filled with an array of transactions. We can take that output and process it with the `retest` command now
 
 ```bash
 polycli retest -v 500 --file merged.nocomment.json > simple.json
@@ -60,7 +67,9 @@ polycli retest -v 500 --file merged.nocomment.json > simple.json
 This project will depend on an installation of `solc` (specifically
 0.8.20) and `lllc`. Installing solidity is pretty easy, but LLLC can
 be a little tricky.
-If you got multiple solc versions, you can specify full patch to the right binary to use with env var ```SOLC_PATH```.
+
+If you've got multiple solc versions, you can specify full path to the
+right binary to use with env var ```SOLC_PATH```.
 
 Since the version is pretty old, it might not build well on your host
 os. Building within docker might make your life easier: