fix: corrected the cluster that was being saved to mirror node component in remote config, extended dual-cluster-full-test to test mirror node deploy (#1743)

Signed-off-by: Jeromy Cannon <[email protected]>

Author: jeromy-cannon

src/commands/explorer.ts

@@ -79,6 +79,7 @@ export class ExplorerCommand extends BaseCommand {
   private static readonly DEPLOY_FLAGS_LIST = {
     required: [],
     optional: [
+      flags.cacheDir,
       flags.chartDirectory,
       flags.clusterRef,
       flags.enableIngress,

src/commands/mirror-node.ts

@@ -40,6 +40,7 @@ import {PathEx} from '../business/utils/path-ex.js';
 interface MirrorNodeDeployConfigClass {
   chartDirectory: string;
   clusterContext: string;
+  clusterRef: ClusterRef;
   namespace: NamespaceName;
   enableIngress: boolean;
   mirrorStaticIp: string;
@@ -103,6 +104,7 @@ export class MirrorNodeCommand extends BaseCommand {
   private static readonly DEPLOY_FLAGS_LIST = {
     required: [],
     optional: [
+      flags.cacheDir,
       flags.clusterRef,
       flags.chartDirectory,
       flags.deployment,
@@ -277,9 +279,8 @@ export class MirrorNodeCommand extends BaseCommand {
             // user defined values later to override predefined values
             ctx.config.valuesArg += await self.prepareValuesArg(ctx.config);
 
-            const clusterRef = this.configManager.getFlag<string>(flags.clusterRef) as string;
-            ctx.config.clusterContext = clusterRef
-              ? this.localConfig.clusterRefs[clusterRef]
+            ctx.config.clusterContext = ctx.config.clusterRef
+              ? this.localConfig.clusterRefs[ctx.config.clusterRef]
               : this.k8Factory.default().contexts().readCurrent();
 
             await self.accountManager.loadNodeClient(
@@ -874,11 +875,10 @@ export class MirrorNodeCommand extends BaseCommand {
       task: async (ctx): Promise<void> => {
         await this.remoteConfigManager.modify(async remoteConfig => {
           const {
-            config: {namespace},
+            config: {namespace, clusterRef},
           } = ctx;
-          const cluster = this.remoteConfigManager.currentCluster;
 
-          remoteConfig.components.add(new MirrorNodeComponent('mirrorNode', cluster, namespace.name));
+          remoteConfig.components.add(new MirrorNodeComponent('mirrorNode', clusterRef, namespace.name));
         });
       },
     };

test/e2e/commands/dual-cluster-full.test.ts

@@ -35,23 +35,40 @@ import {PodRef} from '../../../src/integration/kube/resources/pod/pod-ref.js';
 import {type SoloWinstonLogger} from '../../../src/core/logging/solo-winston-logger.js';
 import {type NodeAlias} from '../../../src/types/aliases.js';
 import * as constants from '../../../src/core/constants.js';
+import {type ExtendedNetServer} from '../../../src/types/index.js';
+import http from 'http';
+import {sleep} from '../../../src/core/helpers.js';
+import {type AccountManager} from '../../../src/core/account-manager.js';
+import {
+  AccountCreateTransaction,
+  Hbar,
+  HbarUnit,
+  PrivateKey,
+  type TransactionReceipt,
+  type TransactionResponse,
+} from '@hashgraph/sdk';
+import {type PackageDownloader} from '../../../src/core/package-downloader.js';
 
 const testName: string = 'dual-cluster-full';
 
 describe('Dual Cluster Full E2E Test', async function dualClusterFullE2eTest(): Promise<void> {
   this.bail(true);
   const namespace: NamespaceName = NamespaceName.of(testName);
-  const deployment: string = `${testName}-deployment`;
-  const testClusterRefs: ClusterRef[] = ['e2e-cluster-alpha', 'e2e-cluster-beta'];
+  const deployment: DeploymentName = `${testName}-deployment`;
+  const testClusterArray: ClusterRef[] = ['e2e-cluster-alpha', 'e2e-cluster-beta'];
   const soloTestCluster: string = getTestCluster();
   const testCluster: string =
     soloTestCluster.includes('c1') || soloTestCluster.includes('c2') ? soloTestCluster : `${soloTestCluster}-c1`;
   const contexts: string[] = [
     `${testCluster}`,
     `${testCluster.replace(soloTestCluster.includes('-c1') ? '-c1' : '-c2', soloTestCluster.includes('-c1') ? '-c2' : '-c1')}`,
   ];
+  const testClusterRefs: ClusterRefs = {};
+  testClusterRefs[testClusterArray[0]] = contexts[0];
+  testClusterRefs[testClusterArray[1]] = contexts[1];
   const testCacheDir: string = getTestCacheDir(testName);
   let testLogger: SoloWinstonLogger;
+  const createdAccountIds: string[] = [];
 
   // TODO the kube config context causes issues if it isn't one of the selected clusters we are deploying to
   before(async (): Promise<void> => {
@@ -86,13 +103,13 @@ describe('Dual Cluster Full E2E Test', async function dualClusterFullE2eTest():
 
   it(`${testName}: solo cluster-ref connect`, async (): Promise<void> => {
     testLogger.info(`${testName}: beginning solo cluster-ref connect`);
-    for (let index: number = 0; index < testClusterRefs.length; index++) {
-      await main(soloClusterRefConnectArgv(testClusterRefs[index], contexts[index]));
+    for (let index: number = 0; index < testClusterArray.length; index++) {
+      await main(soloClusterRefConnectArgv(testClusterArray[index], contexts[index]));
     }
     const localConfig: LocalConfig = container.resolve<LocalConfig>(InjectTokens.LocalConfig);
     const clusterRefs: ClusterRefs = localConfig.clusterRefs;
-    expect(clusterRefs[testClusterRefs[0]]).to.equal(contexts[0]);
-    expect(clusterRefs[testClusterRefs[1]]).to.equal(contexts[1]);
+    expect(clusterRefs[testClusterArray[0]]).to.equal(contexts[0]);
+    expect(clusterRefs[testClusterArray[1]]).to.equal(contexts[1]);
     testLogger.info(`${testName}: finished solo cluster-ref connect`);
   });
 
@@ -104,22 +121,22 @@ describe('Dual Cluster Full E2E Test', async function dualClusterFullE2eTest():
 
   it(`${testName}: solo deployment add-cluster`, async (): Promise<void> => {
     testLogger.info(`${testName}: beginning solo deployment add-cluster`);
-    for (let index: number = 0; index < testClusterRefs.length; index++) {
-      await main(soloDeploymentAddClusterArgv(deployment, testClusterRefs[index], 1));
+    for (let index: number = 0; index < testClusterArray.length; index++) {
+      await main(soloDeploymentAddClusterArgv(deployment, testClusterArray[index], 1));
     }
     const remoteConfigManager: RemoteConfigManager = container.resolve(InjectTokens.RemoteConfigManager);
     expect(remoteConfigManager.isLoaded(), 'remote config manager should be loaded').to.be.true;
     const consensusNodes: Record<string, ConsensusNodeComponent> = remoteConfigManager.components.consensusNodes;
     expect(Object.entries(consensusNodes).length, 'consensus node count should be 2').to.equal(2);
-    expect(consensusNodes['node1'].cluster).to.equal(testClusterRefs[0]);
-    expect(consensusNodes['node2'].cluster).to.equal(testClusterRefs[1]);
+    expect(consensusNodes['node1'].cluster).to.equal(testClusterArray[0]);
+    expect(consensusNodes['node2'].cluster).to.equal(testClusterArray[1]);
     testLogger.info(`${testName}: finished solo deployment add-cluster`);
   });
 
   it(`${testName}: solo cluster-ref setup`, async (): Promise<void> => {
     testLogger.info(`${testName}: beginning solo cluster-ref setup`);
-    for (let index: number = 0; index < testClusterRefs.length; index++) {
-      await main(soloClusterRefSetup(testClusterRefs[index]));
+    for (let index: number = 0; index < testClusterArray.length; index++) {
+      await main(soloClusterRefSetup(testClusterArray[index]));
     }
     testLogger.info(`${testName}: finishing solo cluster-ref setup`);
   });
@@ -193,11 +210,45 @@ describe('Dual Cluster Full E2E Test', async function dualClusterFullE2eTest():
           constants.NETWORK_PROXY_DELAY,
         );
       expect(haProxyPod).to.have.lengthOf(1);
+      createdAccountIds.push(await verifyAccountCreateWasSuccessful(namespace, testClusterRefs));
+      createdAccountIds.push(await verifyAccountCreateWasSuccessful(namespace, testClusterRefs));
     }
   }).timeout(Duration.ofMinutes(5).toMillis());
 
-  // TODO mirror node deploy
+  it(`${testName}: mirror node deploy`, async (): Promise<void> => {
+    await main(soloMirrorNodeDeployArgv(deployment, testClusterArray[1]));
+    await verifyMirrorNodeDeployWasSuccessful(contexts, namespace, testLogger);
+    // TODO validate the new accounts are showing up with the mirror node rest url
+  }).timeout(Duration.ofMinutes(10).toMillis());
+
   // TODO explorer deploy
+  xit(`${testName}: explorer deploy`, async (): Promise<void> => {
+    await main(soloExplorerDeployArgv(deployment, testClusterArray[1]));
+    const k8Factory: K8Factory = container.resolve<K8Factory>(InjectTokens.K8Factory);
+    const k8: K8 = k8Factory.getK8(contexts[1]);
+    const hederaExplorerPods: Pod[] = await k8
+      .pods()
+      .list(namespace, [
+        'app.kubernetes.io/instance=hedera-explorer',
+        'app.kubernetes.io/name=hedera-explorer-chart',
+        'app.kubernetes.io/component=hedera-explorer',
+      ]);
+    expect(hederaExplorerPods).to.have.lengthOf(1);
+    let portForwarder: ExtendedNetServer = null;
+    try {
+      portForwarder = await k8.pods().readByRef(hederaExplorerPods[0].podRef).portForward(8_080, 8_080);
+      await sleep(Duration.ofSeconds(2));
+      const guiUrl: string = 'http://127.0.0.1:8080/localnet/dashboard';
+      const packageDownloader: PackageDownloader = container.resolve<PackageDownloader>(InjectTokens.PackageDownloader);
+      expect(await packageDownloader.urlExists(guiUrl), 'the hedera explorer GUI URL should exist').to.be.true;
+      // TODO validate the new accounts are showing up with the hedera explorer url
+    } finally {
+      if (portForwarder) {
+        await k8.pods().readByRef(null).stopPortForward(portForwarder);
+      }
+    }
+  });
+
   // TODO json rpc relay deploy
   // TODO json rpc relay destroy
   // TODO explorer destroy
@@ -207,6 +258,7 @@ describe('Dual Cluster Full E2E Test', async function dualClusterFullE2eTest():
     await main(soloNetworkDestroyArgv(deployment));
   });
 });
+
 function newArgv(): string[] {
   return ['${PATH}/node', '${SOLO_ROOT}/solo.ts'];
 }
@@ -236,7 +288,7 @@ function soloClusterRefConnectArgv(clusterRef: ClusterRef, context: string): str
   return argv;
 }
 
-function soloDeploymentCreateArgv(deployment: string, namespace: NamespaceName): string[] {
+function soloDeploymentCreateArgv(deployment: DeploymentName, namespace: NamespaceName): string[] {
   const argv: string[] = newArgv();
   argv.push('deployment');
   argv.push('create');
@@ -248,7 +300,11 @@ function soloDeploymentCreateArgv(deployment: string, namespace: NamespaceName):
   return argv;
 }
 
-function soloDeploymentAddClusterArgv(deployment: string, clusterRef: ClusterRef, numberOfNodes: number): string[] {
+function soloDeploymentAddClusterArgv(
+  deployment: DeploymentName,
+  clusterRef: ClusterRef,
+  numberOfNodes: number,
+): string[] {
   const argv: string[] = newArgv();
   argv.push('deployment');
   argv.push('add-cluster');
@@ -285,7 +341,7 @@ function soloNodeKeysArgv(deployment: DeploymentName): string[] {
   return argv;
 }
 
-function soloNetworkDeployArgv(deployment: string): string[] {
+function soloNetworkDeployArgv(deployment: DeploymentName): string[] {
   const argv: string[] = newArgv();
   argv.push('network');
   argv.push('deploy');
@@ -296,7 +352,7 @@ function soloNetworkDeployArgv(deployment: string): string[] {
   return argv;
 }
 
-function soloNodeSetupArgv(deployment: string): string[] {
+function soloNodeSetupArgv(deployment: DeploymentName): string[] {
   const argv: string[] = newArgv();
   argv.push('node');
   argv.push('setup');
@@ -306,7 +362,7 @@ function soloNodeSetupArgv(deployment: string): string[] {
   return argv;
 }
 
-function soloNodeStartArgv(deployment: string): string[] {
+function soloNodeStartArgv(deployment: DeploymentName): string[] {
   const argv: string[] = newArgv();
   argv.push('node');
   argv.push('start');
@@ -316,7 +372,125 @@ function soloNodeStartArgv(deployment: string): string[] {
   return argv;
 }
 
-function soloNetworkDestroyArgv(deployment: string): string[] {
+async function verifyAccountCreateWasSuccessful(namespace: NamespaceName, clusterRefs: ClusterRefs): Promise<string> {
+  const accountManager: AccountManager = container.resolve<AccountManager>(InjectTokens.AccountManager);
+  try {
+    await accountManager.refreshNodeClient(namespace, clusterRefs);
+    expect(accountManager._nodeClient).not.to.be.null;
+    const privateKey: PrivateKey = PrivateKey.generate();
+    const amount: number = 100;
+
+    const newAccount: TransactionResponse = await new AccountCreateTransaction()
+      .setKeyWithoutAlias(privateKey)
+      .setInitialBalance(Hbar.from(amount, HbarUnit.Hbar))
+      .execute(accountManager._nodeClient);
+
+    // Get the new account ID
+    const getReceipt: TransactionReceipt = await newAccount.getReceipt(accountManager._nodeClient);
+    const accountInfo: {accountId: string; privateKey: string; balance: number; publicKey: string} = {
+      accountId: getReceipt.accountId.toString(),
+      privateKey: privateKey.toString(),
+      publicKey: privateKey.publicKey.toString(),
+      balance: amount,
+    };
+
+    expect(accountInfo.accountId).not.to.be.null;
+    expect(accountInfo.balance).to.equal(amount);
+
+    return accountInfo.accountId;
+  } finally {
+    await accountManager.close();
+    // @ts-expect-error - TS2341: Property _portForwards is private and only accessible within class AccountManager
+    expect(accountManager._portForwards, 'port forwards should be empty after accountManager.close()').to.have.lengthOf(
+      0,
+    );
+  }
+}
+
+function soloMirrorNodeDeployArgv(deployment: DeploymentName, clusterRef: ClusterRef): string[] {
+  const argv: string[] = newArgv();
+  argv.push('mirror-node');
+  argv.push('deploy');
+  argv.push(optionFromFlag(Flags.deployment));
+  argv.push(deployment);
+  argv.push(optionFromFlag(Flags.clusterRef));
+  argv.push(clusterRef);
+  argv.push(optionFromFlag(Flags.pinger));
+  argvPushGlobalFlags(argv, true, true);
+  return argv;
+}
+
+async function verifyMirrorNodeDeployWasSuccessful(
+  contexts: string[],
+  namespace: NamespaceName,
+  testLogger: SoloWinstonLogger,
+): Promise<void> {
+  const k8Factory: K8Factory = container.resolve<K8Factory>(InjectTokens.K8Factory);
+  const k8: K8 = k8Factory.getK8(contexts[1]);
+  const mirrorNodeRestPods: Pod[] = await k8
+    .pods()
+    .list(namespace, [
+      'app.kubernetes.io/instance=mirror',
+      'app.kubernetes.io/name=rest',
+      'app.kubernetes.io/component=rest',
+    ]);
+  expect(mirrorNodeRestPods).to.have.lengthOf(1);
+  let portForwarder: ExtendedNetServer = null;
+  try {
+    portForwarder = await k8.pods().readByRef(mirrorNodeRestPods[0].podRef).portForward(5_551, 5_551);
+    await sleep(Duration.ofSeconds(2));
+    const queryUrl: string = 'http://localhost:5551/api/v1/network/nodes';
+    let received: boolean = false;
+    // wait until the transaction reached consensus and retrievable from the mirror node API
+    while (!received) {
+      const req: http.ClientRequest = http.request(
+        queryUrl,
+        {method: 'GET', timeout: 100, headers: {Connection: 'close'}},
+        (res: http.IncomingMessage): void => {
+          res.setEncoding('utf8');
+          res.on('data', (chunk): void => {
+            // convert chunk to json object
+            const obj: {nodes: unknown[]} = JSON.parse(chunk);
+            expect(
+              obj.nodes?.length,
+              "expect there to be two nodes in the mirror node's copy of the address book",
+            ).to.equal(2);
+            // TODO need to enable this, but looks like mirror node currently is getting no service endpoints
+            // expect(
+            //   obj.nodes[0].service_endpoints?.length,
+            //   'expect there to be at least one service endpoint',
+            // ).to.be.greaterThan(0);
+            received = true;
+          });
+        },
+      );
+      req.on('error', (e: Error): void => {
+        testLogger.debug(`problem with request: ${e.message}`, e);
+      });
+      req.end(); // make the request
+      await sleep(Duration.ofSeconds(2));
+    }
+    await sleep(Duration.ofSeconds(1));
+  } finally {
+    if (portForwarder) {
+      await k8.pods().readByRef(null).stopPortForward(portForwarder);
+    }
+  }
+}
+
+function soloExplorerDeployArgv(deployment: DeploymentName, clusterRef: ClusterRef): string[] {
+  const argv: string[] = newArgv();
+  argv.push('explorer');
+  argv.push('deploy');
+  argv.push(optionFromFlag(Flags.deployment));
+  argv.push(deployment);
+  argv.push(optionFromFlag(Flags.clusterRef));
+  argv.push(clusterRef);
+  argvPushGlobalFlags(argv, true, true);
+  return argv;
+}
+
+function soloNetworkDestroyArgv(deployment: DeploymentName): string[] {
   const argv: string[] = newArgv();
   argv.push('network');
   argv.push('destroy');