loop on op until it succeeds before idempotency check

Signed-off-by: hfuss <hayden.fuss@kaleido.io>

Author: onelapahead

test/e2e/client/restclient.go

@@ -21,6 +21,7 @@ import (
 	"crypto/rand"
 	"crypto/sha256"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"math/big"
@@ -821,13 +822,13 @@ func (client *FireFlyClient) DeleteContractListener(t *testing.T, id *fftypes.UU
 	require.Equal(t, 204, resp.StatusCode(), "DELETE %s [%d]: %s", path, resp.StatusCode(), resp.String())
 }
 
-func (client *FireFlyClient) InvokeContractMethod(t *testing.T, req *core.ContractCallRequest, expectedStatus ...int) (interface{}, error) {
-	var res interface{}
+func (client *FireFlyClient) InvokeContractMethod(t *testing.T, req *core.ContractCallRequest, expectedStatus ...int) (*core.Operation, error) {
+	res := &core.Operation{}
 	path := client.namespaced(urlContractInvoke)
 	var errResult fftypes.RESTError
 	resp, err := client.Client.R().
 		SetBody(req).
-		SetResult(&res).
+		SetResult(res).
 		SetError(&errResult).
 		Post(path)
 	require.NoError(t, err)
@@ -836,7 +837,7 @@ func (client *FireFlyClient) InvokeContractMethod(t *testing.T, req *core.Contra
 	}
 	require.Equal(t, expectedStatus[0], resp.StatusCode(), "POST %s [%d]: %s", path, resp.StatusCode(), resp.String())
 	if err == nil && errResult.Error != "" {
-		return res, fmt.Errorf(errResult.Error)
+		return nil, errors.New(errResult.Error)
 	}
 	return res, err
 }
@@ -1055,6 +1056,16 @@ func (client *FireFlyClient) GetOperations(t *testing.T, startTime time.Time) (o
 	return operations
 }
 
+func (client *FireFlyClient) GetOperation(t *testing.T, operationID string) (operation *core.Operation) {
+	path := client.namespaced(urlOperations + "/" + operationID)
+	resp, err := client.Client.R().
+		SetResult(&operation).
+		Get(path)
+	require.NoError(t, err)
+	require.Equal(t, 200, resp.StatusCode(), "GET %s [%d]: %s", path, resp.StatusCode(), resp.String())
+	return operation
+}
+
 func (client *FireFlyClient) NetworkAction(t *testing.T, action core.NetworkActionType) {
 	path := client.namespaced(urlNetworkAction)
 	input := &core.NetworkAction{Type: action}

test/e2e/multiparty/ethereum_contracts.go

@@ -190,9 +190,9 @@ func (suite *EthereumContractTestSuite) TestDirectInvokeMethod() {
 		Location: fftypes.JSONAnyPtrBytes(locationBytes),
 		Method:   simpleStorageFFIGet(),
 	}
-	res, err = suite.testState.client1.QueryContractMethod(suite.T(), queryContractRequest)
+	queryRes, err := suite.testState.client1.QueryContractMethod(suite.T(), queryContractRequest)
 	assert.NoError(suite.T(), err)
-	resJSON, err := json.Marshal(res)
+	resJSON, err := json.Marshal(queryRes)
 	assert.NoError(suite.T(), err)
 	assert.Equal(suite.T(), `{"output":"2"}`, string(resJSON))
 	suite.testState.client1.DeleteContractListener(suite.T(), listener.ID)
@@ -232,7 +232,18 @@ func (suite *EthereumContractTestSuite) TestFFIInvokeMethod() {
 	assert.NoError(suite.T(), err)
 	assert.NotNil(suite.T(), res)
 
-	time.Sleep(1 * time.Second) // Wait for the operation to be processed
+	// Wait for the operation to succeed before idempotency check
+	// If the operation never succeeds, the larger test timeout will fail
+	for {
+		suite.T().Logf("Waiting for invoke operation to succeed: %s", res.ID.String())
+		op := suite.testState.client1.GetOperation(suite.T(), res.ID.String())
+		if op.Status == core.OpStatusSucceeded {
+			suite.T().Logf("Invoke operation succeeded: %s", res.ID.String())
+			break
+		}
+		time.Sleep(100 * time.Millisecond)
+		suite.T().Logf("Retrying, invoke operation status: %s", op.Status)
+	}
 
 	// Idempotency check
 	_, err = suite.testState.client1.InvokeContractMethod(suite.T(), invokeContractRequest, 409)
@@ -256,9 +267,9 @@ func (suite *EthereumContractTestSuite) TestFFIInvokeMethod() {
 		Interface:  suite.interfaceID,
 		MethodPath: "get",
 	}
-	res, err = suite.testState.client1.QueryContractMethod(suite.T(), queryContractRequest)
+	queryRes, err := suite.testState.client1.QueryContractMethod(suite.T(), queryContractRequest)
 	assert.NoError(suite.T(), err)
-	resJSON, err := json.Marshal(res)
+	resJSON, err := json.Marshal(queryRes)
 	assert.NoError(suite.T(), err)
 	assert.Equal(suite.T(), `{"output":"42"}`, string(resJSON))
 	suite.testState.client1.DeleteContractListener(suite.T(), listener.ID)