diff --git a/aws/examples/migration/auto-migrate.py b/aws/examples/migration/auto-migrate.py new file mode 100644 index 00000000..bd98c700 --- /dev/null +++ b/aws/examples/migration/auto-migrate.py @@ -0,0 +1,558 @@ +#!/usr/bin/env python3 +""" +Automated Terraform State Migration Tool for AWS + +Migrates resources from the old monolithic AWS module to the new +modular structure. Uses the shared BaseStateMigrator from scripts/. + +AWS-specific extras: + - Normalizes instance_namespaces for_each keys after migration + - Prepares and runs imports for SG rules and DB SG rules + - Strips IPv6 ranges from EKS node SG rules before import + +Usage: + ./auto-migrate.py /path/to/old/terraform /path/to/new/terraform [--dry-run] +""" + +import json +import subprocess +import sys +from pathlib import Path +from typing import List + +# Add scripts/ to path for shared base module +sys.path.insert(0, str(Path(__file__).resolve().parents[3] / 'scripts')) + +from migrate_base import ( + BaseStateMigrator, + MigrationRule, + parse_old_tfvars, + pull_state_json, + run_main, +) + + +class AWSStateMigrator(BaseStateMigrator): + provider_name = "AWS" + module_detection_names = ("networking", "eks", "database") + expected_resource_count_hint = "100+" + state_backend_hint = "S3, Terraform Cloud, etc." + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._pending_imports = [] + + def _build_rules(self) -> List[MigrationRule]: + return [ + # Remove [0] index from various modules + MigrationRule( + pattern=r'^(module\.operator)\[0\]\.(.+)$', + transform=lambda m: f"{m.group(1)}.{m.group(2)}", + description="Remove [0] index from operator module" + ), + + MigrationRule( + pattern=r'^(module\.aws_lbc)\[0\]\.(.+)$', + transform=lambda m: f"{m.group(1)}.{m.group(2)}", + description="Remove [0] index from aws_lbc module" + ), + + # Rename certificates module to cert_manager + MigrationRule( + pattern=r'^module\.certificates\.(.+)$', + transform=lambda m: f'module.cert_manager.{m.group(1)}', + description="Rename certificates module to cert_manager" + ), + + # Skip self-signed cert-manager resources (kubernetes_manifest → kubectl_manifest type change) + MigrationRule( + pattern=r'^module\.cert_manager\.kubernetes_manifest\.(self_signed_cluster_issuer|self_signed_root_ca_certificate|root_ca_cluster_issuer)\[0\]$', + transform=lambda m: None, + description="Skip self-signed cert resources (type change kubernetes_manifest→kubectl_manifest)" + ), + + # Move root-level IAM roles to storage module + MigrationRule( + pattern=r'^aws_iam_role\.materialize_s3$', + transform=lambda m: 'module.storage.aws_iam_role.materialize_s3', + description="Move IAM role to storage module" + ), + + MigrationRule( + pattern=r'^aws_iam_role_policy\.materialize_s3$', + transform=lambda m: 'module.storage.aws_iam_role_policy.materialize_s3', + description="Move IAM role policy to storage module" + ), + + # Rename node group modules + MigrationRule( + pattern=r'^module\.materialize_node_group\.(.+)$', + transform=lambda m: f'module.mz_node_group.{m.group(1)}', + description="Rename materialize_node_group to mz_node_group" + ), + + # Migrate EKS internal managed node group to base_node_group + MigrationRule( + pattern=r'^module\.eks\.module\.eks\.module\.eks_managed_node_group\[.+?\]\.(.+)$', + transform=lambda m: f'module.base_node_group.module.node_group.{m.group(1)}', + description="Migrate EKS internal managed node group to base_node_group" + ), + + # Move Materialize instance resources from operator module to root level + MigrationRule( + pattern=r'^module\.operator\.kubernetes_manifest\.materialize_instances(\[.+?\])$', + transform=lambda m: f'kubernetes_manifest.materialize_instances{m.group(1)}', + description="Move Materialize instance manifests from operator to root" + ), + + # Move Materialize instance data sources from operator module to root level + MigrationRule( + pattern=r'^module\.operator\.data\.kubernetes_resource\.materialize_instances(\[.+?\])$', + transform=lambda m: f'data.kubernetes_resource.materialize_instances{m.group(1)}', + description="Move Materialize instance data sources from operator to root" + ), + + # Move instance namespaces from operator module to root level + MigrationRule( + pattern=r'^module\.operator\.kubernetes_namespace\.instance_namespaces(\[.+?\])$', + transform=lambda m: f'kubernetes_namespace.instance_namespaces{m.group(1)}', + description="Move instance namespaces from operator to root" + ), + + # Move backend secrets from operator module to root level + MigrationRule( + pattern=r'^module\.operator\.kubernetes_secret\.materialize_backends(\[.+?\])$', + transform=lambda m: f'kubernetes_secret.materialize_backends{m.group(1)}', + description="Move backend secrets from operator to root" + ), + + # Move db init jobs from operator module to root level + MigrationRule( + pattern=r'^module\.operator\.kubernetes_job\.db_init_job(\[.+?\])$', + transform=lambda m: f'kubernetes_job.db_init_job{m.group(1)}', + description="Move db init jobs from operator to root" + ), + + # Skip NLB TargetGroupBinding resources (kubernetes_manifest → kubectl_manifest type change) + MigrationRule( + pattern=r'^module\.nlb\[.+?\]\.module\.target_.+\.kubernetes_manifest\.target_group_binding$', + transform=lambda m: None, + description="Skip NLB target group binding (type change kubernetes_manifest→kubectl_manifest)" + ), + + # Keep NLB resources (handles both indexed and nested submodules) + MigrationRule( + pattern=r'^(module\.nlb\[.+)$', + transform=lambda m: m.group(1), + description="Keep NLB module path unchanged" + ), + + # Keep operator module helm releases unchanged + MigrationRule( + pattern=r'^(module\.operator\.helm_release\..+)$', + transform=lambda m: m.group(1), + description="Keep operator helm releases unchanged" + ), + + # Keep aws_lbc module helm release unchanged + MigrationRule( + pattern=r'^(module\.aws_lbc\.helm_release\..+)$', + transform=lambda m: m.group(1), + description="Keep AWS LBC helm release unchanged" + ), + + # Keep cert_manager module helm release unchanged + MigrationRule( + pattern=r'^(module\.cert_manager\.helm_release\..+)$', + transform=lambda m: m.group(1), + description="Keep cert-manager helm release unchanged" + ), + + # Default: keep same path (for networking, eks, database, storage, operator, aws_lbc, cert_manager) + MigrationRule( + pattern=r'^(module\.(networking|eks|database|storage|operator|aws_lbc|cert_manager)\..+)$', + transform=lambda m: m.group(1), + description="Keep module path unchanged" + ), + + # Skip data sources + MigrationRule( + pattern=r'^(.*\.)?data\..*$', + transform=lambda m: None, + description="Skip data source (will be recreated)" + ), + + # Skip old cloudwatch log groups + MigrationRule( + pattern=r'^aws_cloudwatch_log_group\.materialize\[.*\]$', + transform=lambda m: None, + description="Skip old cloudwatch log group" + ), + ] + + # ================================================================= + # AWS-specific hook overrides + # ================================================================= + + def _post_transform(self): + """Normalize instance_namespaces for_each keys after resource moves.""" + self.log_section("Normalizing for_each Keys") + self.log("Checking for instance_namespaces with namespace-name keys...") + self._normalize_instance_namespace_keys() + + def _pre_push(self): + """Prepare resource imports (discover SG IDs, strip IPv6).""" + self.log_section("Preparing Resource Imports") + self.log("Discovering AWS resources that need importing after state push...") + self._pending_imports = self._prepare_imports() + + def _post_push(self): + """Run terraform import for SG rules after state push.""" + if self._pending_imports: + self.log_section("Importing Resources into Remote State") + self._run_imports(self._pending_imports) + + def _completion_hints(self) -> List[str]: + return [ + "- Expect 6 additions (cert-manager + NLB target bindings: kubectl_manifest adopts existing K8s resources)", + ] + + # ================================================================= + # AWS-specific methods + # ================================================================= + + def _normalize_instance_namespace_keys(self): + """ + Normalize instance_namespaces keys from namespace names to instance names. + + The old operator module used namespace names as keys for instance_namespaces, + but instance names for other resources. This normalizes to use instance names consistently. + """ + try: + state = json.loads((self.work_dir / 'new.tfstate').read_text()) + + # Build mapping: namespace_name -> instance_name + namespace_to_instance = {} + + for resource in state.get('resources', []): + if (resource.get('module') == 'module.operator' and + resource['type'] == 'kubernetes_manifest' and + resource['name'] == 'materialize_instances'): + + for instance in resource.get('instances', []): + if 'index_key' in instance and 'attributes' in instance: + instance_name = instance['index_key'] + manifest_attr = instance['attributes'].get('manifest', {}) + + if 'value' in manifest_attr: + manifest = manifest_attr['value'] + else: + manifest = manifest_attr + + namespace_name = manifest.get('metadata', {}).get('namespace') + + if namespace_name: + namespace_to_instance[namespace_name] = instance_name + self.log(f"Detected mapping: namespace '{namespace_name}' -> instance '{instance_name}'") + + if not namespace_to_instance: + self.log("No namespace-to-instance mappings found (may already be normalized)") + return + + state_modified = False + + for resource in state.get('resources', []): + if (resource.get('module') == 'module.operator' and + resource['type'] == 'kubernetes_namespace' and + resource['name'] == 'instance_namespaces'): + + existing_keys = {inst.get('index_key') for inst in resource.get('instances', [])} + instances_to_keep = [] + + for instance in resource.get('instances', []): + if 'index_key' in instance: + current_key = instance['index_key'] + + if current_key in namespace_to_instance: + instance_name = namespace_to_instance[current_key] + + if current_key != instance_name: + old_addr = f'module.operator.kubernetes_namespace.instance_namespaces[{json.dumps(current_key)}]' + new_addr = f'module.operator.kubernetes_namespace.instance_namespaces[{json.dumps(instance_name)}]' + + self.log(f"→ Normalizing namespace key:") + self.log(f" {old_addr}") + self.log(f" ↳ {new_addr}") + + if not self.dry_run: + if instance_name in existing_keys and instance_name != current_key: + self.log(f" ⊘ Skipped (target key already exists)") + continue + else: + instance['index_key'] = instance_name + self.log(f" ✓ Normalized") + state_modified = True + + instances_to_keep.append(instance) + + if not self.dry_run and len(instances_to_keep) != len(resource.get('instances', [])): + resource['instances'] = instances_to_keep + state_modified = True + + if state_modified and not self.dry_run: + self.log("Saving normalized state...") + state['serial'] += 1 + (self.work_dir / 'new.tfstate').write_text(json.dumps(state, indent=2)) + self.log("✓ State saved") + + except Exception as e: + self.log(f"Error during key normalization: {e}", 'WARN') + self.log("This is non-fatal - you may need to manually normalize keys") + + def _prepare_imports(self) -> list: + """ + Discover SG IDs, strip IPv6 from AWS, and return imports to run after push. + """ + imports_to_run = [] + + try: + new_state = json.loads((self.work_dir / 'new.tfstate').read_text()) + except Exception as e: + self.log(f"Error reading state files: {e}", 'WARN') + return imports_to_run + + # Discover Security Group IDs from state + db_sg_id = None + cluster_sg_id = None + node_sg_id = None + + for resource in new_state.get('resources', []): + module = resource.get('module', '') + + if ('database' in module and resource['type'] == 'aws_db_instance' + and resource['name'] == 'this'): + for inst in resource.get('instances', []): + vpc_sgs = inst.get('attributes', {}).get('vpc_security_group_ids', []) + if vpc_sgs: + db_sg_id = vpc_sgs[0] + + if (module == 'module.eks.module.eks' and resource['type'] == 'aws_security_group' + and resource['name'] == 'cluster'): + for inst in resource.get('instances', []): + cluster_sg_id = inst.get('attributes', {}).get('id') + + if (module == 'module.eks.module.eks' and resource['type'] == 'aws_security_group' + and resource['name'] == 'node'): + for inst in resource.get('instances', []): + node_sg_id = inst.get('attributes', {}).get('id') + + # Database Security Group Rules + if db_sg_id: + self.log(f"Found DB security group: {db_sg_id}") + + imports_to_run.append({ + 'addr': 'module.database.aws_security_group_rule.allow_all_egress', + 'import_id': f'{db_sg_id}_egress_-1_0_0_0.0.0.0/0', + 'desc': 'DB SG egress all', + }) + + if cluster_sg_id: + imports_to_run.append({ + 'addr': 'module.database.aws_security_group_rule.eks_cluster_postgres_ingress', + 'import_id': f'{db_sg_id}_ingress_tcp_5432_5432_{cluster_sg_id}', + 'desc': f'DB SG ingress TCP 5432 from cluster SG', + }) + + if node_sg_id: + imports_to_run.append({ + 'addr': 'module.database.aws_security_group_rule.eks_nodes_postgres_ingress', + 'import_id': f'{db_sg_id}_ingress_tcp_5432_5432_{node_sg_id}', + 'desc': f'DB SG ingress TCP 5432 from node SG', + }) + else: + self.log("Could not find DB security group ID - skipping DB SG rule import", 'WARN') + + # EKS Node Security Group Rules — strip IPv6 and re-import + mz_sg_rules = { + 'mz_ingress_pgwire': 6875, + 'mz_ingress_http': 6876, + 'mz_ingress_nlb_health_checks': 8080, + } + + if node_sg_id: + self.log(f"Found node security group: {node_sg_id}") + self.log("Stripping IPv6 ranges from materialize SG rules (IPv4 rules stay intact)") + + for rule_key, port in mz_sg_rules.items(): + ipv6_perm = {'IpProtocol': 'tcp', 'FromPort': port, 'ToPort': port, + 'Ipv6Ranges': [{'CidrIpv6': '::/0'}]} + self.log(f" Revoking: TCP {port} IPv6 ::/0 on {node_sg_id}") + if not self.dry_run: + result = subprocess.run( + ['aws', 'ec2', 'revoke-security-group-ingress', + '--group-id', node_sg_id, + '--ip-permissions', json.dumps([ipv6_perm])], + capture_output=True, text=True, check=False + ) + if result.returncode == 0: + self.log(f" ✓ Revoked") + else: + stderr = result.stderr.strip() + if 'InvalidPermission.NotFound' in stderr: + self.log(f" ⊘ Already gone") + else: + self.log(f" ✗ Failed: {stderr}", 'WARN') + + # Remove stale mz SG rule instances from local state + self.log("Removing stale mz SG rule instances from local state (will re-import after push)") + if not self.dry_run: + try: + state = json.loads((self.work_dir / 'new.tfstate').read_text()) + for resource in state.get('resources', []): + if (resource.get('module', '') == 'module.eks.module.eks' + and resource['type'] == 'aws_security_group_rule' + and resource['name'] == 'node'): + original_count = len(resource.get('instances', [])) + resource['instances'] = [ + inst for inst in resource.get('instances', []) + if inst.get('index_key') not in mz_sg_rules + ] + removed = original_count - len(resource['instances']) + if removed > 0: + self.log(f" Removed {removed} stale mz SG rule instance(s) from state") + state['serial'] += 1 + (self.work_dir / 'new.tfstate').write_text(json.dumps(state, indent=2)) + break + except Exception as e: + self.log(f" Warning: could not remove stale instances: {e}", 'WARN') + + # Queue EKS node SG rules for fresh import after push + for rule_key, port in mz_sg_rules.items(): + resource_addr = f'module.eks.module.eks.aws_security_group_rule.node["{rule_key}"]' + import_id = f"{node_sg_id}_ingress_tcp_{port}_{port}_0.0.0.0/0" + imports_to_run.append({ + 'addr': resource_addr, + 'import_id': import_id, + 'desc': f'EKS node SG {rule_key}', + }) + else: + self.log("Could not find node security group ID - skipping EKS SG rule cleanup", 'WARN') + + self.log(f"Queued {len(imports_to_run)} resource(s) for import after state push") + return imports_to_run + + def _run_imports(self, imports: list): + """Run terraform import for resources that exist in AWS but aren't in state.""" + if not imports: + self.log("No imports to run") + return + + self.log(f"Importing {len(imports)} resource(s) into remote state") + + for item in imports: + self.log(f" Importing: {item['desc']} ({item['import_id']})") + if not self.dry_run: + result = self.run_terraform( + ['import', item['addr'], item['import_id']], + cwd=self.new_dir, + capture=True + ) + if result.returncode == 0: + self.log(f" ✓ Imported") + else: + stderr = result.stderr.strip() + if 'Resource already managed' in stderr: + self.log(f" ⊘ Already in state") + else: + self.log(f" ✗ Failed: {stderr}", 'WARN') + + self.log("Import phase complete") + + @staticmethod + def generate_tfvars(old_dir: Path, new_dir: Path): + """Generate terraform.tfvars from old AWS configuration""" + print("Generating terraform.tfvars from old configuration...") + + old_values = parse_old_tfvars(old_dir) + state = pull_state_json(old_dir) + + if state: + # Extract region from resources + for resource in state.get('resources', []): + if 'instances' in resource and len(resource['instances']) > 0: + attrs = resource['instances'][0].get('attributes', {}) + if 'region' in attrs and 'aws_region' not in old_values: + old_values['aws_region'] = attrs['region'] + break + + # Extract name prefix from VPC name + for resource in state.get('resources', []): + if resource.get('type') == 'aws_vpc': + instances = resource.get('instances', []) + if instances: + tags = instances[0].get('attributes', {}).get('tags', {}) + name = tags.get('Name', '') + if name and '-vpc' in name: + prefix = name.replace('-vpc', '') + old_values['name_prefix'] = prefix + break + + # Build tfvars content + tfvars_content = '''# ============================================================================= +# Terraform Variables +# ============================================================================= +# Auto-generated from old configuration +# Review and update as needed, especially the license_key +# ============================================================================= + +''' + + if 'aws_region' in old_values: + tfvars_content += f'aws_region = "{old_values["aws_region"]}"\n' + else: + tfvars_content += 'aws_region = "us-east-1" # TODO: Update this\n' + + if 'aws_profile' in old_values: + tfvars_content += f'aws_profile = "{old_values["aws_profile"]}"\n' + else: + tfvars_content += 'aws_profile = "default" # TODO: Update this\n' + + if 'name_prefix' in old_values: + tfvars_content += f'name_prefix = "{old_values["name_prefix"]}"\n' + else: + tfvars_content += 'name_prefix = "materialize" # TODO: Update this\n' + + tfvars_content += ''' +# TODO: Add your Materialize license key from https://materialize.com/register +license_key = "your-license-key-here" + +# CIDR blocks for access control +ingress_cidr_blocks = ["0.0.0.0/0"] +k8s_apiserver_authorized_networks = ["0.0.0.0/0"] + +# Load balancer configuration +internal_load_balancer = true + +# Tags +tags = { + Environment = "production" + ManagedBy = "terraform" + Project = "materialize" +} +''' + + output_path = new_dir / 'terraform.tfvars' + output_path.write_text(tfvars_content) + + print(f" ✅ Generated: {output_path}") + print(f"\nDetected values:") + for key, value in old_values.items(): + print(f" - {key}: {value}") + print(f"\n⚠️ Please review and update the generated file, especially:") + print(f" - license_key (required)") + print(f" - name_prefix (must match your existing resources)") + print(f" - CIDR blocks (restrict for production)") + + +if __name__ == '__main__': + run_main(AWSStateMigrator) diff --git a/azure/examples/migration/auto-migrate.py b/azure/examples/migration/auto-migrate.py new file mode 100644 index 00000000..a6fe537b --- /dev/null +++ b/azure/examples/migration/auto-migrate.py @@ -0,0 +1,300 @@ +#!/usr/bin/env python3 +""" +Automated Terraform State Migration Tool for Azure + +Migrates resources from the old monolithic Azure module to the new +modular structure. Uses the shared BaseStateMigrator from scripts/. + +Usage: + ./auto-migrate.py /path/to/old/terraform /path/to/new/terraform [--dry-run] +""" + +import sys +from pathlib import Path +from typing import List + +# Add scripts/ to path for shared base module +sys.path.insert(0, str(Path(__file__).resolve().parents[3] / 'scripts')) + +from migrate_base import ( + BaseStateMigrator, + MigrationRule, + parse_old_tfvars, + pull_state_json, + run_main, +) + + +class AzureStateMigrator(BaseStateMigrator): + provider_name = "Azure" + module_detection_names = ("networking", "aks", "database") + expected_resource_count_hint = "30+" + state_backend_hint = "Azure Storage, Terraform Cloud, etc." + + def _build_rules(self) -> List[MigrationRule]: + return [ + # ================================================================= + # Networking — move from module to root (inline in migration config) + # ================================================================= + MigrationRule( + pattern=r'^module\.networking\.(.+)$', + transform=lambda m: m.group(1), + description="Move networking resources from module to root" + ), + + # ================================================================= + # AKS — move from module to root (inline in migration config) + # ================================================================= + # Rename the old system nodepool: module.aks.*.materialize → *.system + MigrationRule( + pattern=r'^module\.aks\.azurerm_kubernetes_cluster_node_pool\.materialize$', + transform=lambda m: 'azurerm_kubernetes_cluster_node_pool.system', + description="Move and rename AKS system nodepool to root" + ), + + # Move other AKS resources to root + MigrationRule( + pattern=r'^module\.aks\.(.+)$', + transform=lambda m: m.group(1), + description="Move AKS resources from module to root" + ), + + # ================================================================= + # Database — move from module to root (inline in migration config) + # ================================================================= + MigrationRule( + pattern=r'^module\.database\.(.+)$', + transform=lambda m: m.group(1), + description="Move database resources from module to root" + ), + + # ================================================================= + # Materialize Nodepool — keep same paths + # ================================================================= + MigrationRule( + pattern=r'^(module\.materialize_nodepool\..+)$', + transform=lambda m: m.group(1), + description="Keep materialize nodepool path unchanged" + ), + + # ================================================================= + # Storage — keep Azure resources, skip Key Vault and SAS token + # ================================================================= + # Skip Key Vault (not in new module — user deletes manually) + MigrationRule( + pattern=r'^module\.storage\.azurerm_key_vault\..*$', + transform=lambda m: None, + description="Skip Key Vault (not in new module)" + ), + + # Keep other storage resources + MigrationRule( + pattern=r'^(module\.storage\..+)$', + transform=lambda m: m.group(1), + description="Keep storage module path unchanged" + ), + + # ================================================================= + # Certificates → cert_manager + self_signed_cluster_issuer + # ================================================================= + # Skip self-signed cert resources (kubernetes_manifest → kubectl_manifest type change) + MigrationRule( + pattern=r'^module\.certificates\.kubernetes_manifest\.(self_signed_cluster_issuer|self_signed_root_ca_certificate|root_ca_cluster_issuer)\[0\]$', + transform=lambda m: None, + description="Skip self-signed cert resources (type change kubernetes_manifest→kubectl_manifest)" + ), + + # Move cert-manager namespace (remove [0] count index) + MigrationRule( + pattern=r'^module\.certificates\.kubernetes_namespace\.cert_manager\[0\]$', + transform=lambda m: 'module.cert_manager.kubernetes_namespace.cert_manager', + description="Move cert-manager namespace to cert_manager module" + ), + + # Move cert-manager helm release (remove [0] count index) + MigrationRule( + pattern=r'^module\.certificates\.helm_release\.cert_manager\[0\]$', + transform=lambda m: 'module.cert_manager.helm_release.cert_manager', + description="Move cert-manager helm release to cert_manager module" + ), + + # ================================================================= + # Operator — remove [0] index (old used count, new is direct) + # ================================================================= + # Skip metrics server (AKS has built-in) + MigrationRule( + pattern=r'^module\.operator\[0\]\.helm_release\.metrics_server.*$', + transform=lambda m: None, + description="Skip metrics server helm release (AKS has built-in)" + ), + + # Move instance namespace to materialize_instance module + MigrationRule( + pattern=r'^module\.operator\[0\]\.kubernetes_namespace\.instance_namespaces\[.+?\]$', + transform=lambda m: 'module.materialize_instance.kubernetes_namespace.instance[0]', + description="Move instance namespace to materialize_instance module" + ), + + # Move instance backend secret to materialize_instance module + MigrationRule( + pattern=r'^module\.operator\[0\]\.kubernetes_secret\.materialize_backends\[.+?\]$', + transform=lambda m: 'module.materialize_instance.kubernetes_secret.materialize_backend', + description="Move backend secret to materialize_instance module" + ), + + # Skip instance manifests (type change kubernetes_manifest → kubectl_manifest) + MigrationRule( + pattern=r'^module\.operator\[0\]\.kubernetes_manifest\.materialize_instances\[.+?\]$', + transform=lambda m: None, + description="Skip Materialize instance manifest (type change kubernetes_manifest→kubectl_manifest)" + ), + + # Skip db init jobs (not in new module) + MigrationRule( + pattern=r'^module\.operator\[0\]\.kubernetes_job\.db_init_job\[.+?\]$', + transform=lambda m: None, + description="Skip db init job (not in new module)" + ), + + # Move operator namespaces (remove [0]) + MigrationRule( + pattern=r'^module\.operator\[0\]\.(.+)$', + transform=lambda m: f'module.operator.{m.group(1)}', + description="Remove [0] index from operator module" + ), + + # ================================================================= + # Load Balancers — remove for_each key + # ================================================================= + MigrationRule( + pattern=r'^module\.load_balancers\[.+?\]\.(.+)$', + transform=lambda m: f'module.load_balancers.{m.group(1)}', + description="Remove for_each key from load_balancers module" + ), + + # ================================================================= + # Data sources — skip (will be recreated automatically) + # ================================================================= + MigrationRule( + pattern=r'^(.*\.)?data\..*$', + transform=lambda m: None, + description="Skip data source (will be recreated)" + ), + ] + + def _completion_hints(self) -> List[str]: + return [ + "- Expect additions: self-signed cert resources + materialize instance (kubectl_manifest adopts existing K8s resources)", + "- Expect additions: federated identity credential (new workload identity)", + ] + + @staticmethod + def generate_tfvars(old_dir: Path, new_dir: Path): + """Generate terraform.tfvars from old Azure configuration""" + print("Generating terraform.tfvars from old configuration...") + + old_values = parse_old_tfvars(old_dir) + state = pull_state_json(old_dir) + + if state: + # Extract location from AKS cluster + for resource in state.get('resources', []): + if resource.get('type') == 'azurerm_kubernetes_cluster': + instances = resource.get('instances', []) + if instances: + attrs = instances[0].get('attributes', {}) + if 'location' in attrs and 'location' not in old_values: + old_values['location'] = attrs['location'] + break + + # Extract prefix from VNet name + for resource in state.get('resources', []): + if resource.get('type') == 'azurerm_virtual_network': + instances = resource.get('instances', []) + if instances: + name = instances[0].get('attributes', {}).get('name', '') + if name and '-vnet' in name: + prefix = name.replace('-vnet', '') + old_values['name_prefix'] = prefix + break + + # Extract resource group + for resource in state.get('resources', []): + if resource.get('type') == 'azurerm_kubernetes_cluster': + instances = resource.get('instances', []) + if instances: + rg = instances[0].get('attributes', {}).get('resource_group_name', '') + if rg: + old_values['resource_group_name'] = rg + break + + # Build tfvars content + tfvars_content = '''# ============================================================================= +# Terraform Variables +# ============================================================================= +# Auto-generated from old configuration +# Review and update as needed, especially the license_key and passwords +# ============================================================================= + +''' + + if 'location' in old_values: + tfvars_content += f'location = "{old_values["location"]}"\n' + else: + tfvars_content += 'location = "eastus2" # TODO: Update this\n' + + if 'resource_group_name' in old_values: + tfvars_content += f'resource_group_name = "{old_values["resource_group_name"]}"\n' + else: + tfvars_content += 'resource_group_name = "your-rg" # TODO: Update this\n' + + if 'name_prefix' in old_values: + tfvars_content += f'name_prefix = "{old_values["name_prefix"]}"\n' + else: + tfvars_content += 'name_prefix = "materialize" # TODO: Update this\n' + + tfvars_content += ''' +# TODO: Set your Azure subscription ID +subscription_id = "your-subscription-id" + +# TODO: Add your Materialize license key from https://materialize.com/register +license_key = "your-license-key-here" + +# TODO: Set your Materialize instance name +# Run: kubectl get materialize -A -o jsonpath='{.items[0].metadata.name}' +materialize_instance_name = "your-instance-name" + +# TODO: Set your existing database password +old_db_password = "your-db-password" + +# TODO: Set your existing mz_system user password +external_login_password_mz_system = "your-mz-system-password" + +# Load balancer configuration +internal_load_balancer = true + +# Tags +tags = { + managed_by = "terraform" + module = "materialize" +} +''' + + output_path = new_dir / 'terraform.tfvars' + output_path.write_text(tfvars_content) + + print(f" ✅ Generated: {output_path}") + print(f"\nDetected values:") + for key, value in old_values.items(): + print(f" - {key}: {value}") + print(f"\n⚠️ Please review and update the generated file, especially:") + print(f" - subscription_id (required)") + print(f" - license_key (required)") + print(f" - name_prefix (must match your existing resources)") + print(f" - old_db_password (required)") + print(f" - external_login_password_mz_system (required)") + print(f" - materialize_instance_name (required)") + + +if __name__ == '__main__': + run_main(AzureStateMigrator) diff --git a/gcp/examples/migration/auto-migrate.py b/gcp/examples/migration/auto-migrate.py new file mode 100644 index 00000000..b77555d4 --- /dev/null +++ b/gcp/examples/migration/auto-migrate.py @@ -0,0 +1,304 @@ +#!/usr/bin/env python3 +""" +Automated Terraform State Migration Tool for GCP + +Migrates resources from the old monolithic GCP module to the new +modular structure. Uses the shared BaseStateMigrator from scripts/. + +Usage: + ./auto-migrate.py /path/to/old/terraform /path/to/new/terraform [--dry-run] +""" + +import sys +from pathlib import Path +from typing import List + +# Add scripts/ to path for shared base module +sys.path.insert(0, str(Path(__file__).resolve().parents[3] / 'scripts')) + +from migrate_base import ( + BaseStateMigrator, + MigrationRule, + parse_old_tfvars, + pull_state_json, + run_main, +) + + +class GCPStateMigrator(BaseStateMigrator): + provider_name = "GCP" + module_detection_names = ("networking", "gke", "database") + expected_resource_count_hint = "25+" + state_backend_hint = "GCS bucket, Terraform Cloud, etc." + + def _build_rules(self) -> List[MigrationRule]: + return [ + # ================================================================= + # Networking — move from module to root (inline in migration config) + # ================================================================= + MigrationRule( + pattern=r'^module\.networking\.(.+)$', + transform=lambda m: m.group(1), + description="Move networking resources from module to root" + ), + + # ================================================================= + # GKE — move from module to root (inline in migration config) + # ================================================================= + # Rename the system node pool to match migration config + MigrationRule( + pattern=r'^module\.gke\.google_container_node_pool\.primary_nodes$', + transform=lambda m: 'google_container_node_pool.system', + description="Move and rename GKE system node pool to root" + ), + + # Move other GKE resources to root + MigrationRule( + pattern=r'^module\.gke\.(.+)$', + transform=lambda m: m.group(1), + description="Move GKE resources from module to root" + ), + + # ================================================================= + # Database — move from module to root (inline in migration config) + # ================================================================= + MigrationRule( + pattern=r'^module\.database\.time_sleep\.wait_for_vpc$', + transform=lambda m: 'time_sleep.wait_for_vpc', + description="Move database VPC wait to root" + ), + + MigrationRule( + pattern=r'^module\.database\.(.+)$', + transform=lambda m: m.group(1), + description="Move database resources from module to root" + ), + + # ================================================================= + # Materialize Nodepool — keep same module paths + # ================================================================= + MigrationRule( + pattern=r'^(module\.materialize_nodepool\..+)$', + transform=lambda m: m.group(1), + description="Keep materialize nodepool path unchanged" + ), + + # Handle legacy swap_nodepool[0] name (before old module's moved block) + MigrationRule( + pattern=r'^module\.swap_nodepool\[0\]\.(.+)$', + transform=lambda m: f'module.materialize_nodepool.{m.group(1)}', + description="Rename legacy swap_nodepool[0] to materialize_nodepool" + ), + + # ================================================================= + # Storage — keep same module paths (old and new are identical) + # ================================================================= + MigrationRule( + pattern=r'^(module\.storage\..+)$', + transform=lambda m: m.group(1), + description="Keep storage module path unchanged" + ), + + # ================================================================= + # Certificates → cert_manager + self_signed_cluster_issuer + # ================================================================= + # Skip self-signed cert resources (kubernetes_manifest → kubectl_manifest type change) + # kubectl_manifest will adopt the existing K8s resources with zero disruption. + MigrationRule( + pattern=r'^module\.certificates\.kubernetes_manifest\.(self_signed_cluster_issuer|self_signed_root_ca_certificate|root_ca_cluster_issuer)\[0\]$', + transform=lambda m: None, + description="Skip self-signed cert resources (type change kubernetes_manifest→kubectl_manifest)" + ), + + # Move cert-manager namespace (remove [0] count index) + MigrationRule( + pattern=r'^module\.certificates\.kubernetes_namespace\.cert_manager\[0\]$', + transform=lambda m: 'module.cert_manager.kubernetes_namespace.cert_manager', + description="Move cert-manager namespace to cert_manager module" + ), + + # Move cert-manager helm release (remove [0] count index) + MigrationRule( + pattern=r'^module\.certificates\.helm_release\.cert_manager\[0\]$', + transform=lambda m: 'module.cert_manager.helm_release.cert_manager', + description="Move cert-manager helm release to cert_manager module" + ), + + # ================================================================= + # Operator — remove [0] index (old used count, new is direct) + # ================================================================= + # Skip metrics server (GKE has built-in) + MigrationRule( + pattern=r'^module\.operator\[0\]\.helm_release\.metrics_server.*$', + transform=lambda m: None, + description="Skip metrics server helm release (GKE has built-in)" + ), + + # Move instance namespace to materialize_instance module + MigrationRule( + pattern=r'^module\.operator\[0\]\.kubernetes_namespace\.instance_namespaces\[.+?\]$', + transform=lambda m: 'module.materialize_instance.kubernetes_namespace.instance[0]', + description="Move instance namespace to materialize_instance module" + ), + + # Move instance backend secret to materialize_instance module + MigrationRule( + pattern=r'^module\.operator\[0\]\.kubernetes_secret\.materialize_backends\[.+?\]$', + transform=lambda m: 'module.materialize_instance.kubernetes_secret.materialize_backend', + description="Move backend secret to materialize_instance module" + ), + + # Skip instance manifests (type change kubernetes_manifest → kubectl_manifest) + MigrationRule( + pattern=r'^module\.operator\[0\]\.kubernetes_manifest\.materialize_instances\[.+?\]$', + transform=lambda m: None, + description="Skip Materialize instance manifest (type change kubernetes_manifest→kubectl_manifest)" + ), + + # Skip db init jobs (not in new module) + MigrationRule( + pattern=r'^module\.operator\[0\]\.kubernetes_job\.db_init_job\[.+?\]$', + transform=lambda m: None, + description="Skip db init job (not in new module)" + ), + + # Move operator namespaces and helm release (remove [0]) + MigrationRule( + pattern=r'^module\.operator\[0\]\.(.+)$', + transform=lambda m: f'module.operator.{m.group(1)}', + description="Remove [0] index from operator module" + ), + + # ================================================================= + # Load Balancers — remove for_each key + # ================================================================= + MigrationRule( + pattern=r'^module\.load_balancers\[.+?\]\.(.+)$', + transform=lambda m: f'module.load_balancers.{m.group(1)}', + description="Remove for_each key from load_balancers module" + ), + + # ================================================================= + # Data sources — skip (will be recreated automatically) + # ================================================================= + MigrationRule( + pattern=r'^(.*\.)?data\..*$', + transform=lambda m: None, + description="Skip data source (will be recreated)" + ), + ] + + def _completion_hints(self) -> List[str]: + return [ + "- Expect additions: self-signed cert resources + materialize instance (kubectl_manifest adopts existing K8s resources)", + "- Expect additions: firewall rules for load balancers (new in load_balancers module)", + ] + + @staticmethod + def generate_tfvars(old_dir: Path, new_dir: Path): + """Generate terraform.tfvars from old GCP configuration""" + print("Generating terraform.tfvars from old configuration...") + + old_values = parse_old_tfvars(old_dir) + state = pull_state_json(old_dir) + + if state: + # Extract region from GKE cluster + for resource in state.get('resources', []): + if resource.get('type') == 'google_container_cluster': + instances = resource.get('instances', []) + if instances: + attrs = instances[0].get('attributes', {}) + if 'location' in attrs and 'region' not in old_values: + old_values['region'] = attrs['location'] + break + + # Extract prefix from VPC name + for resource in state.get('resources', []): + if resource.get('type') == 'google_compute_network': + instances = resource.get('instances', []) + if instances: + name = instances[0].get('attributes', {}).get('name', '') + if name and '-network' in name: + prefix = name.replace('-network', '') + old_values['prefix'] = prefix + break + + # Extract project_id from GKE cluster + for resource in state.get('resources', []): + if resource.get('type') == 'google_container_cluster': + instances = resource.get('instances', []) + if instances: + project = instances[0].get('attributes', {}).get('project', '') + if project: + old_values['project_id'] = project + break + + # Build tfvars content + tfvars_content = '''# ============================================================================= +# Terraform Variables +# ============================================================================= +# Auto-generated from old configuration +# Review and update as needed, especially the license_key and passwords +# ============================================================================= + +''' + + if 'project_id' in old_values: + tfvars_content += f'project_id = "{old_values["project_id"]}"\n' + else: + tfvars_content += 'project_id = "your-project-id" # TODO: Update this\n' + + if 'region' in old_values: + tfvars_content += f'region = "{old_values["region"]}"\n' + else: + tfvars_content += 'region = "us-central1" # TODO: Update this\n' + + if 'prefix' in old_values: + tfvars_content += f'prefix = "{old_values["prefix"]}"\n' + else: + tfvars_content += 'prefix = "materialize" # TODO: Update this\n' + + tfvars_content += ''' +# TODO: Add your Materialize license key from https://materialize.com/register +license_key = "your-license-key-here" + +# TODO: Set your Materialize instance name +# Run: kubectl get materialize -A -o jsonpath='{.items[0].metadata.name}' +materialize_instance_name = "your-instance-name" + +# TODO: Set your Materialize instance namespace +# Run: kubectl get materialize -A -o jsonpath='{.items[0].metadata.namespace}' +materialize_instance_namespace = "your-instance-namespace" + +# TODO: Set your existing database password +database_password = "your-db-password" + +# TODO: Set your existing mz_system user password (if using Password auth) +# external_login_password_mz_system = "your-mz-system-password" + +# Load balancer configuration +internal_load_balancer = true + +# Labels +labels = {} +''' + + output_path = new_dir / 'terraform.tfvars' + output_path.write_text(tfvars_content) + + print(f" ✅ Generated: {output_path}") + print(f"\nDetected values:") + for key, value in old_values.items(): + print(f" - {key}: {value}") + print(f"\n⚠️ Please review and update the generated file, especially:") + print(f" - project_id (required)") + print(f" - license_key (required)") + print(f" - prefix (must match your existing resources)") + print(f" - database_password (required)") + print(f" - materialize_instance_name (required)") + print(f" - materialize_instance_namespace (required)") + + +if __name__ == '__main__': + run_main(GCPStateMigrator) diff --git a/scripts/migrate_base.py b/scripts/migrate_base.py new file mode 100644 index 00000000..46b6080e --- /dev/null +++ b/scripts/migrate_base.py @@ -0,0 +1,672 @@ +#!/usr/bin/env python3 +""" +Base module for Terraform State Migration Tools. + +Provides the shared BaseStateMigrator class and MigrationRule dataclass +used by cloud-specific migration scripts (AWS, Azure, GCP). + +Each cloud provider's auto-migrate.py subclasses BaseStateMigrator and +only defines the provider-specific parts: + - Migration rules (_build_rules) + - Module detection names (module_detection_names) + - Tfvars generation (generate_tfvars) + - Completion hints (_completion_hints) +""" + +import abc +import argparse +import json +import os +import re +import shutil +import subprocess +import sys +from dataclasses import dataclass +from datetime import datetime +from pathlib import Path +from typing import Optional, List, Tuple, Type + + +@dataclass +class MigrationRule: + """A rule for transforming old resource paths to new paths""" + pattern: str + transform: callable + description: str + + +class BaseStateMigrator(abc.ABC): + """ + Base class for Terraform state migration across cloud providers. + + Subclasses must implement: + - provider_name (class attribute): e.g., "GCP", "Azure", "AWS" + - module_detection_names (class attribute): tuple of module names for prefix detection + - expected_resource_count_hint (class attribute): e.g., "25+" + - state_backend_hint (class attribute): e.g., "GCS bucket, Terraform Cloud, etc." + - _build_rules(): returns list of MigrationRule + - generate_tfvars(old_dir, new_dir): classmethod for tfvars generation + - _completion_hints(): returns list of hint strings for post-migration output + + Subclasses may override for extra migration steps: + - _post_transform(): called after resource moves (before validation) + - _pre_push(): called after validation/cleanup, before state push + - _post_push(): called after state push + """ + + # -- Class attributes to override -- + provider_name: str = "Unknown" + module_detection_names: tuple = ("networking", "database") + expected_resource_count_hint: str = "25+" + state_backend_hint: str = "remote backend, Terraform Cloud, etc." + + def __init__(self, old_dir: Path, new_dir: Path, dry_run: bool = False): + self.old_dir = old_dir + self.new_dir = new_dir + self.dry_run = dry_run + self.work_dir = None + self.module_prefix = None + self.stats = { + 'total': 0, + 'moved': 0, + 'skipped': 0, + 'failed': 0, + } + + # Define transformation rules + self.rules = self._build_rules() + + # ===================================================================== + # Abstract methods — must be implemented by subclasses + # ===================================================================== + + @abc.abstractmethod + def _build_rules(self) -> List[MigrationRule]: + """ + Build transformation rules for resource path mapping. + + Rules are evaluated in order. The first matching rule wins. + Return None from a rule's transform to skip the resource. + """ + ... + + @staticmethod + @abc.abstractmethod + def generate_tfvars(old_dir: Path, new_dir: Path): + """Generate terraform.tfvars from old configuration.""" + ... + + @abc.abstractmethod + def _completion_hints(self) -> List[str]: + """ + Return cloud-specific hint lines for the post-migration summary. + + Example: + return [ + "- Expect additions: self-signed cert resources + materialize instance", + "- Expect additions: firewall rules for load balancers", + ] + """ + ... + + # ===================================================================== + # Hook methods — override in subclasses for extra steps + # ===================================================================== + + def _post_transform(self): + """Called after all resources are transformed and moved. + + Override for extra steps like normalizing for_each keys (AWS). + """ + pass + + def _pre_push(self): + """Called after validation and cleanup, before state push. + + Override for extra pre-push steps like preparing imports (AWS). + """ + pass + + def _post_push(self): + """Called after state push completes. + + Override for post-push steps like running imports (AWS). + """ + pass + + # ===================================================================== + # Shared methods — identical across all providers + # ===================================================================== + + def log(self, message: str, level: str = 'INFO'): + """Log a message with timestamp""" + timestamp = datetime.now().strftime('%H:%M:%S') + prefix = {'INFO': ' ', 'WARN': '⚠️ ', 'ERROR': '❌', 'SUCCESS': '✅'} + print(f"[{timestamp}] {prefix.get(level, ' ')}{message}") + + def log_section(self, title: str): + """Log a section header""" + print(f"\n{'='*60}") + print(f" {title}") + print('='*60) + + def run_terraform(self, args: List[str], cwd: Path, capture: bool = True) -> subprocess.CompletedProcess: + """Run terraform command""" + cmd = ['terraform'] + args + return subprocess.run( + cmd, + cwd=cwd, + capture_output=capture, + text=True, + check=False + ) + + def detect_module_prefix(self, resources: List[str]) -> Optional[str]: + """Auto-detect if resources are wrapped in a parent module""" + names = '|'.join(self.module_detection_names) + pattern = re.compile(rf'^module\.([^.]+)\.module\.({names})') + + for resource in resources: + match = pattern.match(resource) + if match: + prefix_name = match.group(1) + module_name = match.group(2) + + # Skip if prefix matches module name (internal nesting) + if prefix_name == module_name: + continue + + return f"module.{prefix_name}" + + return None + + def strip_prefix(self, path: str) -> str: + """Remove module prefix if present""" + if self.module_prefix and path.startswith(f"{self.module_prefix}."): + return path[len(self.module_prefix) + 1:] + return path + + def transform_path(self, old_path: str) -> Tuple[Optional[str], Optional[str]]: + """ + Transform old path to new path using rules. + Returns (new_path, rule_description) or (None, None) if no rule matches. + """ + path_to_transform = self.strip_prefix(old_path) + + for rule in self.rules: + match = re.match(rule.pattern, path_to_transform) + if match: + new_path = rule.transform(match) + return new_path, rule.description + + return None, None + + def get_resources(self, state_file: Path) -> List[str]: + """Get list of resources from state file by parsing JSON directly. + + Raises on parse errors — callers that need tolerance should catch exceptions. + """ + if not state_file.exists() or state_file.stat().st_size == 0: + return [] + + with open(state_file, 'r') as f: + state = json.load(f) + + resources = [] + for resource in state.get('resources', []): + resource_type = resource.get('type', '') + resource_name = resource.get('name', '') + resource_mode = resource.get('mode', 'managed') + + module_path = resource.get('module', '') + if module_path: + if resource_mode == 'data': + full_path = f"{module_path}.data.{resource_type}.{resource_name}" + else: + full_path = f"{module_path}.{resource_type}.{resource_name}" + else: + if resource_mode == 'data': + full_path = f"data.{resource_type}.{resource_name}" + else: + full_path = f"{resource_type}.{resource_name}" + + instances = resource.get('instances', []) + if not instances: + resources.append(full_path) + else: + for instance in instances: + index_key = instance.get('index_key') + if index_key is not None: + if isinstance(index_key, int): + resources.append(f"{full_path}[{index_key}]") + else: + resources.append(f'{full_path}["{index_key}"]') + else: + resources.append(full_path) + + return resources + + def resource_exists_in_new(self, resource: str) -> bool: + """Check if resource already exists in new state""" + new_state = self.work_dir / 'new.tfstate' + try: + resources = self.get_resources(new_state) + return resource in resources + except Exception: + return False + + def move_resource(self, old_path: str, new_path: str) -> bool: + """Move resource from old state to new state""" + if self.dry_run: + return True + + old_state_rel = os.path.relpath(self.work_dir / "old.tfstate", self.new_dir) + new_state_rel = os.path.relpath(self.work_dir / "new.tfstate", self.new_dir) + + result = self.run_terraform( + [ + 'state', 'mv', + f'-state={old_state_rel}', + f'-state-out={new_state_rel}', + old_path, + new_path + ], + cwd=self.new_dir, + capture=True + ) + + return result.returncode == 0 + + def cleanup_old_state(self): + """ + Remove skipped managed resources from old state. + + After state moves, skipped resources (e.g., kubernetes_manifest instances + that can't be state-moved due to type changes) remain in old state. + If someone runs 'terraform destroy' on the old config, these resources + would be destroyed — including your running Materialize instance. + + This method strips all remaining managed resources from the old state, + keeping only data sources (which are harmless). The actual Kubernetes/cloud + resources continue running — only Terraform's ownership is removed. + """ + if self.dry_run: + return + + old_state_path = self.work_dir / 'old.tfstate' + try: + state = json.loads(old_state_path.read_text()) + + original_resources = state.get('resources', []) + managed_remaining = [ + r for r in original_resources + if r.get('mode') != 'data' + ] + + if not managed_remaining: + self.log("No orphaned managed resources in old state") + return + + # Keep only data sources + state['resources'] = [ + r for r in original_resources + if r.get('mode') == 'data' + ] + + old_state_path.write_text(json.dumps(state, indent=2)) + + for r in managed_remaining: + module_path = r.get('module', '') + resource_id = f"{r['type']}.{r['name']}" + if module_path: + resource_id = f"{module_path}.{resource_id}" + self.log(f" Removed from old state: {resource_id}") + + self.log(f"Cleaned {len(managed_remaining)} orphaned resource(s) from old state") + self.log(f"This prevents accidental destruction via the old config") + + except Exception as e: + self.log(f"Warning: Could not clean up old state: {e}", 'WARN') + self.log(f"Consider manually running 'terraform state rm' on skipped resources in old config", 'WARN') + + def validate_migrated_state(self): + """ + Validate migrated resources in new state. + + Only inspects resources that we moved — never touches anything else. + Reports issues but does NOT delete anything from state. + """ + try: + state = json.loads((self.work_dir / 'new.tfstate').read_text()) + issues = [] + + for resource in state.get('resources', []): + resource_path = f"{resource.get('module', '')}.{resource['type']}.{resource['name']}" if resource.get('module') else f"{resource['type']}.{resource['name']}" + + for instance in resource.get('instances', []): + attrs = instance.get('attributes', {}) + index_key = instance.get('index_key') + full_path = f"{resource_path}[{json.dumps(index_key)}]" if index_key else resource_path + + if not attrs or attrs.get('id') is None: + issues.append(full_path) + self.log(f"⚠ Resource may have null attributes after state mv: {full_path}") + self.log(f" This can happen when moving resources across module boundaries.") + self.log(f" If terraform apply fails for this resource, import it manually:") + self.log(f" terraform import '{full_path}' ''") + + if issues: + self.log(f"\nFound {len(issues)} resource(s) that may need attention after apply.") + self.log(f"These were NOT removed from state — review terraform apply output.") + else: + self.log("✓ All migrated resources look valid") + + except Exception as e: + self.log(f"Error during state validation: {e}", 'WARN') + + def pull_state(self, tf_dir: Path, output_file: Path): + """Pull Terraform state to local file""" + result = self.run_terraform(['state', 'pull'], cwd=tf_dir) + + if result.returncode != 0: + self.log(f"Failed to pull state from {tf_dir}: {result.stderr}", 'ERROR') + self.log(f"Ensure the directory is initialized (terraform init) and the backend is accessible.", 'ERROR') + sys.exit(1) + + output_file.write_text(result.stdout) + + def push_state(self, state_file: Path, tf_dir: Path): + """Push local state file to Terraform backend""" + if self.dry_run: + return + + abs_state_file = state_file.resolve() + + result = self.run_terraform( + ['state', 'push', str(abs_state_file)], + cwd=tf_dir + ) + + if result.returncode != 0: + self.log(f"Failed to push state: {result.stderr}", 'ERROR') + sys.exit(1) + + # ===================================================================== + # Main migration flow — template method + # ===================================================================== + + def migrate(self): + """Run the full migration workflow.""" + title = f"Automated State Migration ({self.provider_name})" + self.log_section(title) + + self.log(f"Old config: {self.old_dir}") + self.log(f"New config: {self.new_dir}") + if self.dry_run: + self.log("DRY RUN MODE — No changes will be made", 'WARN') + + # Create working directory + self.work_dir = self.new_dir / '.migration-work' + self.work_dir.mkdir(exist_ok=True) + self.log(f"Working directory: {self.work_dir}") + + try: + # Step 1: Pull states + self.log_section("Step 1: Pulling States") + + self.log("Pulling old state...") + self.pull_state(self.old_dir, self.work_dir / 'old.tfstate') + + self.log("Backing up old state...") + backup_file = self.work_dir / f"old-state-backup-{datetime.now().strftime('%Y%m%d-%H%M%S')}.tfstate" + shutil.copy2(self.work_dir / 'old.tfstate', backup_file) + self.log(f"Backup saved to: {backup_file}") + + self.log("Pulling new state...") + self.run_terraform(['init', '-input=false'], cwd=self.new_dir, capture=True) + self.pull_state(self.new_dir, self.work_dir / 'new.tfstate') + + # Step 2: Analyze + self.log_section("Step 2: Analyzing Old State") + + resources = self.get_resources(self.work_dir / 'old.tfstate') + self.stats['total'] = len(resources) + + # Validate that old state has actual infrastructure + if len(resources) < 10: + self.log(f"", 'ERROR') + self.log(f"⚠️ VALIDATION FAILED: Old state only has {len(resources)} resources", 'ERROR') + self.log(f"", 'ERROR') + self.log(f"A typical Materialize deployment has {self.expected_resource_count_hint} resources.", 'ERROR') + self.log(f"This suggests the old state wasn't properly pulled.", 'ERROR') + self.log(f"", 'ERROR') + self.log(f"Common causes:", 'ERROR') + self.log(f" 1. Old directory doesn't contain actual Materialize infrastructure", 'ERROR') + self.log(f" 2. Remote backend not configured in old directory", 'ERROR') + self.log(f" 3. State stored elsewhere ({self.state_backend_hint})", 'ERROR') + self.log(f"", 'ERROR') + self.log(f"Solutions:", 'ERROR') + self.log(f" 1. Ensure old directory has terraform.tfstate or backend config", 'ERROR') + self.log(f" 2. Run 'terraform init' in old directory first", 'ERROR') + self.log(f" 3. Run 'terraform state pull' in old directory to verify state access", 'ERROR') + self.log(f"", 'ERROR') + self.log(f"Cannot proceed with migration.", 'ERROR') + sys.exit(1) + + self.log(f"Found {len(resources)} resources") + + self.module_prefix = self.detect_module_prefix(resources) + if self.module_prefix: + self.log(f"Detected module prefix: {self.module_prefix}") + else: + self.log("No module prefix detected (using root module)") + + # Step 3: Transform and move + self.log_section("Step 3: Processing Resources") + + for resource in resources: + new_path, rule_desc = self.transform_path(resource) + + if new_path is None: + if rule_desc: + self.log(f"⊘ {resource}") + self.log(f" {rule_desc}") + else: + self.log(f"⊘ {resource}", 'WARN') + self.log(f" No transformation rule", 'WARN') + self.stats['skipped'] += 1 + continue + + # Check if already exists in new state + if self.resource_exists_in_new(new_path): + self.log(f"⊘ {resource}") + self.log(f" Already exists: {new_path}") + self.stats['skipped'] += 1 + continue + + # Show transformation + if new_path == self.strip_prefix(resource): + self.log(f"→ {resource}") + else: + self.log(f"→ {resource}") + self.log(f" ↳ {new_path}") + + # Move resource + if self.move_resource(resource, new_path): + self.stats['moved'] += 1 + else: + self.log(f" Failed to move", 'ERROR') + self.stats['failed'] += 1 + + # Post-transform hook (e.g., AWS normalizes for_each keys) + self._post_transform() + + # Validate migrated state + self.log_section("Validating Migrated State") + self.log("Checking migrated resources for potential issues...") + self.validate_migrated_state() + + # Clean up old state + self.log_section("Cleaning Up Old State") + self.log("Removing skipped resources from old state to prevent accidental destruction...") + self.cleanup_old_state() + + # Pre-push hook (e.g., AWS prepares imports) + self._pre_push() + + # Step 4: Push states + if not self.dry_run: + self.log_section("Updating States") + + old_state_text = (self.work_dir / 'old.tfstate').read_text().strip() + if old_state_text and self.stats['total'] > 0: + old_state = json.loads(old_state_text) + old_state['serial'] += 1 + (self.work_dir / 'old-updated.tfstate').write_text( + json.dumps(old_state, indent=2) + ) + + self.log("Pushing old state...") + self.push_state(self.work_dir / 'old-updated.tfstate', self.old_dir) + else: + self.log("Skipping old state push (no resources migrated)") + + self.log("Pushing new state...") + self.push_state(self.work_dir / 'new.tfstate', self.new_dir) + + # Post-push hook (e.g., AWS runs imports) + self._post_push() + + # Summary + self.log_section("Summary") + + print(f" Total resources: {self.stats['total']}") + print(f" ✅ Moved: {self.stats['moved']}") + print(f" ⊘ Skipped: {self.stats['skipped']}") + print(f" ❌ Failed: {self.stats['failed']}") + + if self.dry_run: + print(f"\n This was a DRY RUN. Re-run without --dry-run to apply changes.") + else: + print(f"\n ✅ Migration complete!") + print(f" ") + print(f" Next steps:") + print(f" 1. cd {self.new_dir}") + print(f" 2. terraform plan") + print(f" 3. Review the plan carefully") + for hint in self._completion_hints(): + print(f" {hint}") + print(f" 4. terraform apply") + print(f" ") + print(f" ℹ️ Some resources may already exist in {self.provider_name} but weren't in your old state.") + print(f" If terraform apply fails with 'already exists' errors, import them:") + print(f" terraform import '' ''") + print(f" The error message shows the exact resource path and ID to use.") + print(f" See README.md troubleshooting section for examples.") + + finally: + if self.work_dir: + self.log(f"\nWork files kept in: {self.work_dir}") + self.log(f"You can safely delete this directory after verifying the migration") + + +# ========================================================================= +# Shared tfvars helper +# ========================================================================= + +def parse_old_tfvars(old_dir: Path) -> dict: + """Parse simple key = "value" patterns from old terraform.tfvars.""" + old_values = {} + old_tfvars_path = old_dir / 'terraform.tfvars' + + if old_tfvars_path.exists(): + print(f" Found old terraform.tfvars, extracting values...") + content = old_tfvars_path.read_text() + + for line in content.split('\n'): + line = line.strip() + if line and not line.startswith('#'): + match = re.match(r'(\w+)\s*=\s*"([^"]+)"', line) + if match: + old_values[match.group(1)] = match.group(2) + + return old_values + + +def pull_state_json(old_dir: Path) -> Optional[dict]: + """Pull terraform state from old directory and return parsed JSON.""" + result = subprocess.run( + ['terraform', 'state', 'pull'], + cwd=old_dir, + capture_output=True, + text=True + ) + + if result.returncode == 0: + try: + return json.loads(result.stdout) + except: + pass + return None + + +# ========================================================================= +# Shared CLI entry point +# ========================================================================= + +def run_main(migrator_class: Type[BaseStateMigrator]): + """Shared CLI entry point for all cloud-specific migration scripts.""" + parser = argparse.ArgumentParser( + description=f'Automated Terraform state migration tool for {migrator_class.provider_name}' + ) + parser.add_argument( + 'old_dir', + type=Path, + help='Path to old Terraform configuration' + ) + parser.add_argument( + 'new_dir', + type=Path, + nargs='?', + help='Path to new Terraform configuration' + ) + parser.add_argument( + '--dry-run', + action='store_true', + help='Show what would be done without making changes' + ) + parser.add_argument( + '--generate-tfvars', + action='store_true', + help='Generate terraform.tfvars from old configuration (run before migration)' + ) + + args = parser.parse_args() + + # Handle generate-tfvars mode + if args.generate_tfvars: + if not args.new_dir: + print("Error: new_dir is required with --generate-tfvars") + print("Usage: ./auto-migrate.py /old/dir /new/dir --generate-tfvars") + sys.exit(1) + migrator_class.generate_tfvars(args.old_dir, args.new_dir) + return + + # Validate directories + if not args.old_dir.exists(): + print(f"Error: Old directory not found: {args.old_dir}") + sys.exit(1) + + if not args.new_dir: + print("Error: new_dir is required for migration") + print("Usage: ./auto-migrate.py /old/dir /new/dir [--dry-run]") + print(" or: ./auto-migrate.py /old/dir /new/dir --generate-tfvars") + sys.exit(1) + + if not args.new_dir.exists(): + print(f"Error: New directory not found: {args.new_dir}") + sys.exit(1) + + # Run migration + migrator = migrator_class(args.old_dir, args.new_dir, args.dry_run) + migrator.migrate()