diff --git a/Makefile b/Makefile index 7258a08c..48820ac2 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: clean test bins lint tools +.PHONY: clean test bins lint tools tcld PROJECT_ROOT = github.com/temporalio/tcld # default target diff --git a/README.md b/README.md index faf4e2ea..c3eb995f 100644 --- a/README.md +++ b/README.md @@ -234,6 +234,53 @@ The `--namespace-role` flag can be repeated for each namespace role the group sh The account and namespace roles replace the definition, so any namespace roles omitted will be removed from the group level access. +# Migration Management (Preview) + +*The Migration feature is currently in "Preview Release". Customers must be invited to use this feature. Please reach out to Temporal Cloud support for more information.* + +Migrations provide a way to migrate a namespace and its workflow between a self-hosted Temporal server and Temporal Cloud. Migrations rely on active/passive replication built-in to Temporal. Before starting a migration, deploy the [s2s-proxy](https://github.com/temporalio/s2s-proxy/) alongside your self-hosted cluster and obtain a migration endpoint id from Temporal Cloud support. Please reach out to Temporal Cloud support for more information. + +### Start a migration + +To start a migration, provide the migration endpoint id and the source and target namespace names. +Starting the migration enables active/passive namespace replication. + +``` +tcld migration start --endpoint-id --source-namespace --target-namespace +``` + +### Get a migration + +``` +tcld migration get --id +``` + +### Perform handover during a migration + +To handover, provide the migration id and the replica id. +Handover changes the active replica to the given replica. +The active replica is the replica currently accepting write operations. + +``` +tcld migration handover --id --to-replica-id +``` + +### Confirm a migration + +Confirming the migration completes the migration and disables replication. + +``` +tcld migration confirm --id +``` + +### Abort a migration + +Aborting the migration cancels the migration and disables replication. + +``` +tcld migration abort --id +``` + # Asynchronous Operations Any update operations making changes to the namespaces or user groups hosted on Temporal Cloud are asynchronous. Such operations are tracked using a `request-id` that can be passed in when invoking the update operation or will be auto-generated by the server if one is not specified. Once an asynchronous request is initiated, a `request-id` is returned. Use the `request get` command to query the status of an asynchronous request. ``` diff --git a/app/migration.go b/app/migration.go new file mode 100644 index 00000000..0b41e6fb --- /dev/null +++ b/app/migration.go @@ -0,0 +1,255 @@ +package app + +import ( + "context" + + "github.com/urfave/cli/v2" + "google.golang.org/grpc" + + "github.com/temporalio/tcld/protogen/api/cloud/cloudservice/v1" + "github.com/temporalio/tcld/protogen/api/cloud/namespace/v1" +) + +type ( + MigrationClient struct { + client cloudservice.CloudServiceClient + ctx context.Context + } + GetMigrationClientFn func(ctx *cli.Context) (*MigrationClient, error) +) + +func NewMigrationClient(ctx context.Context, conn *grpc.ClientConn) *MigrationClient { + return &MigrationClient{ + client: cloudservice.NewCloudServiceClient(conn), + ctx: ctx, + } +} + +func GetMigrationClient(ctx *cli.Context) (*MigrationClient, error) { + ct, conn, err := GetServerConnection(ctx) + if err != nil { + return nil, err + } + return NewMigrationClient(ct, conn), nil +} + +func (c *MigrationClient) getMigration(migrationId string) (*namespace.Migration, error) { + resp, err := c.client.GetMigration(c.ctx, &cloudservice.GetMigrationRequest{ + MigrationId: migrationId, + }) + if err != nil { + return nil, err + } + return resp.Migration, nil +} + +func (c *MigrationClient) listMigrations() error { + totalRes := &cloudservice.GetMigrationsResponse{} + pageToken := "" + for { + resp, err := c.client.GetMigrations(c.ctx, &cloudservice.GetMigrationsRequest{ + PageToken: pageToken, + }) + if err != nil { + return err + } + totalRes.Migrations = append(totalRes.Migrations, resp.Migrations...) + pageToken = resp.NextPageToken + if len(pageToken) == 0 { + return PrintProto(totalRes) + } + } +} + +func (c *MigrationClient) startMigration(requestId, migrationEndpointId, sourceNamespace, targetNamespace string) error { + resp, err := c.client.StartMigration(c.ctx, &cloudservice.StartMigrationRequest{ + Spec: &namespace.MigrationSpec{ + MigrationEndpointId: migrationEndpointId, + Spec: &namespace.MigrationSpec_ToCloudSpec{ + ToCloudSpec: &namespace.MigrationToCloudSpec{ + SourceNamespace: sourceNamespace, + TargetNamespace: targetNamespace, + }, + }, + }, + AsyncOperationId: requestId, + }) + if err != nil { + return err + } + return PrintProto(resp) +} + +func (c *MigrationClient) migrationHandover(requestId, migrationId, replicaId string) error { + resp, err := c.client.HandoverNamespace(c.ctx, &cloudservice.HandoverNamespaceRequest{ + MigrationId: migrationId, + ToReplicaId: replicaId, + AsyncOperationId: requestId, + }) + if err != nil { + return err + } + return PrintProto(resp) +} + +func (c *MigrationClient) confirmMigration(requestId, migrationId string) error { + resp, err := c.client.ConfirmMigration(c.ctx, &cloudservice.ConfirmMigrationRequest{ + MigrationId: migrationId, + AsyncOperationId: requestId, + }) + if err != nil { + return err + } + return PrintProto(resp) +} + +func (c *MigrationClient) abortMigration(requestId, migrationId string) error { + resp, err := c.client.AbortMigration(c.ctx, &cloudservice.AbortMigrationRequest{ + MigrationId: migrationId, + AsyncOperationId: requestId, + }) + if err != nil { + return err + } + return PrintProto(resp) +} + +func NewMigrationCommand(getMigrationClient GetMigrationClientFn) (CommandOut, error) { + var c *MigrationClient + migrationIdFlag := &cli.StringFlag{ + Name: "id", + Aliases: []string{"i"}, + Usage: "Migration id", + Required: true, + } + migrationEndpointIdFlag := &cli.StringFlag{ + Name: "endpoint-id", + Aliases: []string{"e"}, + Usage: "Migration endpoint id", + Required: true, + } + sourceNamespaceFlag := &cli.StringFlag{ + Name: "source-namespace", + Aliases: []string{"s"}, + Usage: "Source namespace name", + Required: true, + } + targetNamespaceFlag := &cli.StringFlag{ + Name: "target-namespace", + Aliases: []string{"t"}, + Usage: "Target namespace name", + Required: true, + } + toReplicaIdFlag := &cli.StringFlag{ + Name: "to-replica-id", + Aliases: []string{"rp"}, + Usage: "The id of the replica to make active", + Required: true, + } + + return CommandOut{ + Command: &cli.Command{ + Name: "migration", + Aliases: []string{"m"}, + Before: func(ctx *cli.Context) error { + var err error + c, err = getMigrationClient(ctx) + return err + }, + Usage: "(private preview) Manage migrations between self-hosted Temporal and Temporal cloud", + Subcommands: []*cli.Command{ + { + Name: "get", + Aliases: []string{"g"}, + Usage: "Get a migration", + Flags: []cli.Flag{ + migrationIdFlag, + }, + Action: func(ctx *cli.Context) error { + id := ctx.String(migrationIdFlag.Name) + m, err := c.getMigration(id) + if err != nil { + return err + } + return PrintProto(m) + }, + }, + { + Name: "list", + Aliases: []string{"l"}, + Usage: "List migrations", + Flags: []cli.Flag{}, + Action: func(ctx *cli.Context) error { + return c.listMigrations() + }, + }, + { + Name: "start", + Aliases: []string{"s"}, + Usage: "Start a new migration", + Flags: []cli.Flag{ + RequestIDFlag, + migrationEndpointIdFlag, + sourceNamespaceFlag, + targetNamespaceFlag, + }, + Action: func(ctx *cli.Context) error { + return c.startMigration( + ctx.String(RequestIDFlag.Name), + ctx.String(migrationEndpointIdFlag.Name), + ctx.String(sourceNamespaceFlag.Name), + ctx.String(targetNamespaceFlag.Name), + ) + }, + }, + { + Name: "handover", + Aliases: []string{"s"}, + Usage: "Handover the namespace from on-prem to cloud, or from cloud back to on-prem", + Flags: []cli.Flag{ + RequestIDFlag, + migrationIdFlag, + toReplicaIdFlag, + }, + Action: func(ctx *cli.Context) error { + return c.migrationHandover( + ctx.String(RequestIDFlag.Name), + ctx.String(migrationIdFlag.Name), + ctx.String(toReplicaIdFlag.Name), + ) + }, + }, + { + Name: "confirm", + Aliases: []string{"c"}, + Usage: "Confirm the migration", + Flags: []cli.Flag{ + RequestIDFlag, + migrationIdFlag, + }, + Action: func(ctx *cli.Context) error { + return c.confirmMigration( + ctx.String(RequestIDFlag.Name), + ctx.String(migrationIdFlag.Name), + ) + }, + }, + { + Name: "abort", + Aliases: []string{"a"}, + Usage: "Abort the migration", + Flags: []cli.Flag{ + RequestIDFlag, + migrationIdFlag, + }, + Action: func(ctx *cli.Context) error { + return c.abortMigration( + ctx.String(RequestIDFlag.Name), + ctx.String(migrationIdFlag.Name), + ) + }, + }, + }, + }, + }, nil +} diff --git a/app/migration_test.go b/app/migration_test.go new file mode 100644 index 00000000..2b77b31b --- /dev/null +++ b/app/migration_test.go @@ -0,0 +1,536 @@ +package app + +import ( + "context" + "errors" + "testing" + + "github.com/golang/mock/gomock" + "github.com/stretchr/testify/suite" + "github.com/temporalio/tcld/protogen/api/cloud/cloudservice/v1" + cloudNamespace "github.com/temporalio/tcld/protogen/api/cloud/namespace/v1" + "github.com/temporalio/tcld/protogen/api/cloud/operation/v1" + apimock "github.com/temporalio/tcld/protogen/apimock/cloudservice/v1" + "github.com/urfave/cli/v2" +) + +func TestMigration(t *testing.T) { + suite.Run(t, new(MigrationTestSuite)) +} + +type MigrationTestSuite struct { + suite.Suite + cliApp *cli.App + mockCtrl *gomock.Controller + mockCloudApiClient *apimock.MockCloudServiceClient +} + +func (s *MigrationTestSuite) SetupTest() { + s.mockCtrl = gomock.NewController(s.T()) + s.mockCloudApiClient = apimock.NewMockCloudServiceClient(s.mockCtrl) + + out, err := NewMigrationCommand(func(ctx *cli.Context) (*MigrationClient, error) { + return &MigrationClient{ + ctx: context.TODO(), + client: s.mockCloudApiClient, + }, nil + }) + s.Require().NoError(err) + + cmds := []*cli.Command{ + out.Command, + } + flags := []cli.Flag{ + AutoConfirmFlag, + } + + s.cliApp, _ = NewTestApp(s.T(), cmds, flags) +} + +func (s *MigrationTestSuite) SetupSubTest() { + s.SetupTest() +} + +func (s *MigrationTestSuite) RunCmd(args []string) error { + return s.cliApp.Run(append([]string{"tcld"}, args...)) +} + +func (s *MigrationTestSuite) AfterTest() { + s.mockCtrl.Finish() +} + +func (s *MigrationTestSuite) TearDownSubTest(suiteName, testName string) { + s.mockCtrl.Finish() +} + +func (s *MigrationTestSuite) TestGet() { + testcases := []struct { + name string + cmd []string + expReq *cloudservice.GetMigrationRequest + mockResp *cloudservice.GetMigrationResponse + mockErr error + + expErr string + }{ + { + name: "missing id", + cmd: []string{"migration", "get"}, + expErr: `Required flag "id" not set`, + }, + { + name: "api error", + cmd: []string{"migration", "get", "--id", "abc"}, + expReq: &cloudservice.GetMigrationRequest{ + MigrationId: "abc", + }, + mockErr: errors.New("some err"), + expErr: "some err", + }, + { + name: "success", + cmd: []string{"migration", "get", "--id", "abc"}, + expReq: &cloudservice.GetMigrationRequest{ + MigrationId: "abc", + }, + mockResp: &cloudservice.GetMigrationResponse{ + Migration: s.makeMigration("abc"), + }, + }, + } + + for _, tc := range testcases { + s.Run(tc.name, func() { + if tc.expReq != nil { + s.mockCloudApiClient.EXPECT().GetMigration(gomock.Any(), tc.expReq). + Return(tc.mockResp, tc.mockErr).Times(1) + } + + err := s.RunCmd(tc.cmd) + if len(tc.expErr) != 0 { + s.ErrorContains(err, tc.expErr) + } else { + s.NoError(err) + } + }) + } +} + +func (s *MigrationTestSuite) TestList() { + testcases := []struct { + name string + cmd []string + expReq []*cloudservice.GetMigrationsRequest + mockResp []*cloudservice.GetMigrationsResponse + mockErr []error + + expErr string + }{ + { + name: "api error", + cmd: []string{"migration", "list"}, + expReq: []*cloudservice.GetMigrationsRequest{ + {}, + }, + mockResp: []*cloudservice.GetMigrationsResponse{ + nil, + }, + mockErr: []error{ + errors.New("some err"), + }, + expErr: "some err", + }, + { + name: "api error on page 2", + cmd: []string{"migration", "list"}, + expReq: []*cloudservice.GetMigrationsRequest{ + {}, + {PageToken: "page2"}, + }, + mockResp: []*cloudservice.GetMigrationsResponse{ + { + Migrations: []*cloudNamespace.Migration{ + s.makeMigration("abc"), + }, + NextPageToken: "page2", + }, + nil, + }, + mockErr: []error{ + nil, + errors.New("page 2 err"), + }, + expErr: "page 2 err", + }, + { + name: "1 page - success", + cmd: []string{"migration", "list"}, + expReq: []*cloudservice.GetMigrationsRequest{ + {}, + }, + mockResp: []*cloudservice.GetMigrationsResponse{ + { + Migrations: []*cloudNamespace.Migration{ + s.makeMigration("abc"), + }, + }, + }, + mockErr: []error{ + nil, + }, + }, + { + name: "2 pages - success", + cmd: []string{"migration", "list"}, + expReq: []*cloudservice.GetMigrationsRequest{ + {}, + {PageToken: "page2"}, + }, + mockResp: []*cloudservice.GetMigrationsResponse{ + { + Migrations: []*cloudNamespace.Migration{ + s.makeMigration("abc"), + }, + NextPageToken: "page2", + }, + { + Migrations: []*cloudNamespace.Migration{ + s.makeMigration("def"), + }, + NextPageToken: "", + }, + }, + mockErr: []error{ + nil, + nil, + }, + }, + } + + for _, tc := range testcases { + s.Run(tc.name, func() { + s.Require().Equal(len(tc.expReq), len(tc.mockResp)) + s.Require().Equal(len(tc.expReq), len(tc.mockErr)) + + for i := range tc.expReq { + s.mockCloudApiClient.EXPECT().GetMigrations(gomock.Any(), tc.expReq[i]). + Return(tc.mockResp[i], tc.mockErr[i]).Times(1) + } + + err := s.RunCmd(tc.cmd) + if len(tc.expErr) != 0 { + s.ErrorContains(err, tc.expErr) + } else { + s.NoError(err) + } + }) + } +} + +func (s *MigrationTestSuite) TestStart() { + fullTestCmd := []string{ + "migration", "start", + "--endpoint-id", "ep-123", + "--source-namespace", "src-ns", + "--target-namespace", "tgt-ns", + "--request-id", "req-xyz", + } + fullTestSpec := &cloudNamespace.MigrationSpec{ + MigrationEndpointId: "ep-123", + Spec: &cloudNamespace.MigrationSpec_ToCloudSpec{ + ToCloudSpec: &cloudNamespace.MigrationToCloudSpec{ + SourceNamespace: "src-ns", + TargetNamespace: "tgt-ns", + }, + }, + } + + testcases := []struct { + name string + cmd []string + expReq *cloudservice.StartMigrationRequest + mockResp *cloudservice.StartMigrationResponse + mockErr error + + expErr string + }{ + { + name: "missing endpoint-id", + cmd: []string{ + "migration", "start", + "--source-namespace", "src-ns", + "--target-namespace", "tgt-ns", + }, + expErr: `Required flag "endpoint-id" not set`, + }, + { + name: "missing source-namespace", + cmd: []string{ + "migration", "start", + "--endpoint-id", "ep-123", + "--target-namespace", "tgt-ns", + }, + expErr: `Required flag "source-namespace" not set`, + }, + { + name: "mising target-namespace", + cmd: []string{ + "migration", "start", + "--endpoint-id", "ep-123", + "--source-namespace", "src-ns", + }, + expErr: `Required flag "target-namespace" not set`, + }, + { + name: "api error", + cmd: fullTestCmd, + expReq: &cloudservice.StartMigrationRequest{ + Spec: fullTestSpec, + AsyncOperationId: "req-xyz", + }, + mockResp: nil, + mockErr: errors.New("some start err"), + expErr: "some start err", + }, + { + name: "success", + cmd: fullTestCmd, + expReq: &cloudservice.StartMigrationRequest{ + Spec: fullTestSpec, + AsyncOperationId: "req-xyz", + }, + mockResp: &cloudservice.StartMigrationResponse{ + MigrationId: "abc", + AsyncOperation: &operation.AsyncOperation{Id: "req-xyz"}, + }, + }, + } + + for _, tc := range testcases { + s.Run(tc.name, func() { + if tc.expReq != nil { + s.mockCloudApiClient.EXPECT().StartMigration(gomock.Any(), tc.expReq). + Return(tc.mockResp, tc.mockErr).Times(1) + } + + err := s.RunCmd(tc.cmd) + if len(tc.expErr) != 0 { + s.ErrorContains(err, tc.expErr) + } else { + s.NoError(err) + } + }) + } +} + +func (s *MigrationTestSuite) TestHandover() { + testcases := []struct { + name string + cmd []string + expReq *cloudservice.HandoverNamespaceRequest + mockResp *cloudservice.HandoverNamespaceResponse + mockErr error + + expErr string + }{ + { + name: "missing id", + cmd: []string{ + "migration", "handover", + "--to-replica-id", "cloud", + }, + expErr: `Required flag "id" not set`, + }, + { + name: "missing to-replica-id", + cmd: []string{ + "migration", "handover", + "--id", "abc", + }, + expErr: `Required flag "to-replica-id" not set`, + }, + { + name: "api error", + cmd: []string{ + "migration", "handover", + "--id", "abc", + "--to-replica-id", "cloud", + }, + expReq: &cloudservice.HandoverNamespaceRequest{ + MigrationId: "abc", + ToReplicaId: "cloud", + }, + mockErr: errors.New("some err"), + expErr: "some err", + }, + { + name: "success", + cmd: []string{ + "migration", "handover", + "--id", "abc", + "--to-replica-id", "cloud", + "--request-id", "req-xyz", + }, + expReq: &cloudservice.HandoverNamespaceRequest{ + MigrationId: "abc", + ToReplicaId: "cloud", + AsyncOperationId: "req-xyz", + }, + mockResp: &cloudservice.HandoverNamespaceResponse{ + AsyncOperation: &operation.AsyncOperation{Id: "req-xyz"}, + }, + }, + } + + for _, tc := range testcases { + s.Run(tc.name, func() { + if tc.expReq != nil { + s.mockCloudApiClient.EXPECT().HandoverNamespace(gomock.Any(), tc.expReq). + Return(tc.mockResp, tc.mockErr).Times(1) + } + + err := s.RunCmd(tc.cmd) + if len(tc.expErr) != 0 { + s.ErrorContains(err, tc.expErr) + } else { + s.NoError(err) + } + }) + } +} + +func (s *MigrationTestSuite) TestConfirm() { + testcases := []struct { + name string + cmd []string + expReq *cloudservice.ConfirmMigrationRequest + mockResp *cloudservice.ConfirmMigrationResponse + mockErr error + + expErr string + }{ + { + name: "missing id", + cmd: []string{"migration", "confirm"}, + expErr: `Required flag "id" not set`, + }, + { + name: "api error", + cmd: []string{"migration", "confirm", "--id", "abc"}, + expReq: &cloudservice.ConfirmMigrationRequest{ + MigrationId: "abc", + }, + mockErr: errors.New("some err"), + expErr: "some err", + }, + { + name: "success", + cmd: []string{ + "migration", "confirm", + "--id", "abc", + "--request-id", "req-xyz", + }, + expReq: &cloudservice.ConfirmMigrationRequest{ + MigrationId: "abc", + AsyncOperationId: "req-xyz", + }, + mockResp: &cloudservice.ConfirmMigrationResponse{ + AsyncOperation: &operation.AsyncOperation{Id: "req-xyz"}, + }, + }, + } + + for _, tc := range testcases { + s.Run(tc.name, func() { + if tc.expReq != nil { + s.mockCloudApiClient.EXPECT().ConfirmMigration(gomock.Any(), tc.expReq). + Return(tc.mockResp, tc.mockErr).Times(1) + } + + err := s.RunCmd(tc.cmd) + if len(tc.expErr) != 0 { + s.ErrorContains(err, tc.expErr) + } else { + s.NoError(err) + } + }) + } +} + +func (s *MigrationTestSuite) TestAbort() { + testcases := []struct { + name string + cmd []string + expReq *cloudservice.AbortMigrationRequest + mockResp *cloudservice.AbortMigrationResponse + mockErr error + + expErr string + }{ + { + name: "missing id", + cmd: []string{"migration", "abort"}, + expErr: `Required flag "id" not set`, + }, + { + name: "api error", + cmd: []string{"migration", "abort", "--id", "abc"}, + expReq: &cloudservice.AbortMigrationRequest{ + MigrationId: "abc", + }, + mockErr: errors.New("some err"), + expErr: "some err", + }, + { + name: "success", + cmd: []string{ + "migration", "abort", + "--id", "abc", + "--request-id", "req-xyz", + }, + expReq: &cloudservice.AbortMigrationRequest{ + MigrationId: "abc", + AsyncOperationId: "req-xyz", + }, + mockResp: &cloudservice.AbortMigrationResponse{ + AsyncOperation: &operation.AsyncOperation{Id: "req-xyz"}, + }, + }, + } + + for _, tc := range testcases { + s.Run(tc.name, func() { + if tc.expReq != nil { + s.mockCloudApiClient.EXPECT().AbortMigration(gomock.Any(), tc.expReq). + Return(tc.mockResp, tc.mockErr).Times(1) + } + + err := s.RunCmd(tc.cmd) + if len(tc.expErr) != 0 { + s.ErrorContains(err, tc.expErr) + } else { + s.NoError(err) + } + }) + } +} + +func (s *MigrationTestSuite) makeMigration(id string) *cloudNamespace.Migration { + return &cloudNamespace.Migration{ + MigrationId: id, + Spec: &cloudNamespace.MigrationSpec{ + MigrationEndpointId: "ep-123", + Spec: &cloudNamespace.MigrationSpec_ToCloudSpec{ + ToCloudSpec: &cloudNamespace.MigrationToCloudSpec{ + SourceNamespace: "src-ns", + TargetNamespace: "tgt-ns", + }, + }, + }, + State: 2, + Replicas: []*cloudNamespace.MigrationReplica{ + {Id: "on-prem", State: 1}, + {Id: "cloud", State: 2}, + }, + } +} diff --git a/cmd/tcld/fx.go b/cmd/tcld/fx.go index 28926672..555e2417 100644 --- a/cmd/tcld/fx.go +++ b/cmd/tcld/fx.go @@ -26,6 +26,7 @@ func fxOptions() fx.Option { app.NewFeatureCommand, app.NewServiceAccountCommand, app.NewNexusCommand, + app.NewMigrationCommand, func() app.GetNamespaceClientFn { return app.GetNamespaceClient }, @@ -50,6 +51,9 @@ func fxOptions() fx.Option { func() app.GetNexusClientFn { return app.GetNexusClient }, + func() app.GetMigrationClientFn { + return app.GetMigrationClient + }, ), fx.Invoke(func(app *cli.App, shutdowner fx.Shutdowner) error { err := app.Run(os.Args)