Merge pull request #37 from bikashrc25/portworx

Add support for Portworx and future storage modules

Author: displague

README.md

@@ -227,3 +227,24 @@ metal_project_id                  = "YOUR-PROJECT-ID"
 ## Google Anthos Documentation
 Once Anthos is deployed on Equinix Metal, all of the documentation for using Google Anthos is located on the [Anthos Documentation Page](https://cloud.google.com/anthos/docs).
 
+## Storage Providers
+
+Storage providers are made available through optional storage modules. These storage providers include CSI (Container Native Storage) `StorageClasses`.
+
+Changing or disabling a storage provider is not currently supported.
+
+To enable a storage module, set the `storage_module` variable to the name of the name of the included module.
+
+* `portworx`: To enable the Pure Storage Portworx installation, use the following settings in `terraform.tfvars`:
+
+  ```hcl
+  storage_module = "portworx"
+  storage_options = {
+    # portworx_version = "2.6"
+    # portworx_license = "c0ffe-fefe-activation-123"
+  }
+  ```
+
+  When enabled, Portworx will manage the local disks attached to each worker node, providing a fault tolerant distributed storage solution.
+
+  [Read more about the Portworx module](modules/portworx/README).

main.tf

@@ -145,6 +145,13 @@ data "template_file" "deploy_anthos_cluster" {
   }
 }
 
+data "template_file" "pre_reqs_worker" {
+  template = file("templates/pre_reqs_worker.sh")
+  vars = {
+    operating_system = var.operating_system
+  }
+}
+
 resource "null_resource" "prep_anthos_cluster" {
   depends_on = [
     google_project_service.enabled-apis
@@ -206,6 +213,9 @@ data "template_file" "create_cluster" {
   }
 }
 
+// Initialize Anthos on the first control plane node.
+// This will also trigger installs (including apt)
+// on the worker nodes.
 resource "null_resource" "deploy_anthos_cluster" {
   depends_on = [
     null_resource.prep_anthos_cluster,
@@ -420,3 +430,52 @@ resource "null_resource" "install_kube_vip_daemonset" {
   }
 }
 
+resource "null_resource" "worker_pre_reqs" {
+  count = var.worker_count
+  depends_on = [
+    null_resource.deploy_anthos_cluster,
+  ]
+
+  connection {
+    type        = "ssh"
+    user        = "root"
+    private_key = chomp(tls_private_key.ssh_key_pair.private_key_pem)
+    host        = element(metal_device.worker_nodes.*.access_public_ipv4, count.index)
+  }
+
+  provisioner "remote-exec" {
+    inline = ["mkdir -p /root/bootstrap/"]
+  }
+
+  # Unless /root/bootstrap/ is created in advance, this will be
+  # copied to /root/bootstrap (file)
+  # https://github.com/hashicorp/terraform/issues/16330
+  provisioner "file" {
+    content     = data.template_file.pre_reqs_worker.rendered
+    destination = "/root/bootstrap/pre_reqs_worker.sh"
+  }
+
+  provisioner "remote-exec" {
+    inline = ["bash /root/bootstrap/pre_reqs_worker.sh"]
+  }
+}
+
+module "storage" {
+  source = "./modules/storage"
+
+  depends_on = [
+    null_resource.add_kubelet_flags_to_workers,
+  ]
+
+  ssh = {
+    host             = metal_device.control_plane.0.access_public_ipv4
+    private_key      = chomp(tls_private_key.ssh_key_pair.private_key_pem)
+    user             = "root"
+    kubeconfig       = "/root/baremetal/bmctl-workspace/${local.cluster_name}/${local.cluster_name}-kubeconfig"
+    worker_addresses = metal_device.worker_nodes.*.access_public_ipv4
+  }
+
+  cluster_name    = local.cluster_name
+  storage_module  = var.storage_module
+  storage_options = var.storage_options
+}

modules/portworx/README.md

@@ -0,0 +1,36 @@
+# Pure Storage Portworx installation
+
+Portworx by Pure Storage is a distributed and high available data storage that takes advantage of the local and attached storage provided on each Equinix Metal device.  Portworx includes a [Container Storage Interface (CSI)](https://kubernetes-csi.github.io/docs/) driver.
+
+Portworx differentiates between device disks using priority labels that can be applied to create distinct `StorageClasses`. See [Portworx: Dynamic Provisioning](https://docs.portworx.com/portworx-install-with-kubernetes/storage-operations/create-pvcs/dynamic-provisioning/) for more details.
+
+Login to any one of the Anthos cluster nodes and run `pxctl status` to check the state of the Portworx services.
+
+You can also use the Kubernetes API to check the status:
+
+```sh
+kubectl get pods -lapp=portworx -n kube-system
+```
+
+Portworx logs can be viewed by running:
+
+```sh
+kubectl logs -lapp=portworx -n kube-system --all-containers
+```
+
+By default, Portworx 2.6 is installed in the Anthos Cluster. The version of Portworx can be changed using the `portworx_version` variable.
+
+## Licensing
+
+Portworx is installed with a Trial license. To continue use at the end of the trial period, you will need a Portworx Enterprise Metal license.
+
+More information about these licenses, their restrictions and enablement can be found at <https://docs.portworx.com/reference/knowledge-base/px-licensing/>.
+
+To active the Portworx license through this module:
+
+* Set the `portworx_license` variable to your license key
+* Run `terraform apply`
+
+**Note**: The `portworx_license` variable can not be set and defined before the Portworx installation is ready. This takes about 15 minutes today.  If you attempt to provide the license too early the `terraform apply` will fail, affecting only this licensing task. A subsequent and successful `terraform apply` will be needed to correct the licensing.
+
+Alternatively, `ssh` into any worker node and run `/opt/pwx/bin/pxctl license activate _key_`.

modules/portworx/assets/portworx_disk_setup.sh

@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+createlvm=true
+deletelvm=false
+dsksize="0"
+
+function largest_free_disk {
+  lsblk -f -d -b -n -oNAME,SIZE | while read disk size; do
+    # ignore disks with filesystems
+    if ! lsblk -f -b -n  -oNAME,SIZE,FSTYPE -i /dev/$disk | egrep "xfs|ext3|ext4|btrfs|sr0" >/dev/null; then
+      echo -en "$disk $size"
+    fi
+  done | sort -n -k2 | head -n1 | cut -f1 -d" "
+}
+
+
+dskname=$(largest_free_disk)
+echo "Will use $dskname for Portworx KVDB LVM by running the following commands(will only run if createlvm=true)"
+dev="/dev/$dskname"
+echo "pvcreate $dev"
+echo "vgcreate pwx_vg $dev"
+echo "lvcreate -l 100%FREE -n pwxkvdb pwx_vg"
+if $createlvm; then
+    pvcreate $dev
+    vgcreate pwx_vg $dev
+    lvcreate -l 100%FREE -n pwxkvdb pwx_vg
+fi
+if $deletelvm; then
+   lvremove /dev/pwx_vg/pwxkvdb
+   vgremove pwx_vg
+   pvremove $dev
+   wipefs -a $dev
+fi

modules/portworx/main.tf

@@ -0,0 +1,70 @@
+resource "null_resource" "worker_disks" {
+  count = length(var.ssh.worker_addresses)
+
+  connection {
+    type        = "ssh"
+    user        = var.ssh.user
+    private_key = var.ssh.private_key
+    host        = var.ssh.worker_addresses[count.index]
+  }
+
+  provisioner "remote-exec" {
+    inline = ["mkdir -p /root/bootstrap/"]
+  }
+
+  provisioner "file" {
+    source      = "${path.module}/assets/portworx_disk_setup.sh"
+    destination = "/root/bootstrap/portworx_disk_setup.sh"
+
+  }
+  provisioner "remote-exec" {
+    inline = [
+      "bash /root/bootstrap/portworx_disk_setup.sh"
+    ]
+  }
+}
+
+locals {
+  portworx_version = try(length(var.portworx_version) ? var.portworx_version
+  : var.latest_portworx_version, var.latest_portworx_version)
+}
+
+resource "null_resource" "install_portworx" {
+  depends_on = [
+    null_resource.worker_disks
+  ]
+  connection {
+    type        = "ssh"
+    user        = var.ssh.user
+    private_key = var.ssh.private_key
+    host        = var.ssh.host
+  }
+
+  provisioner "remote-exec" {
+    inline = [
+      "VER=$(kubectl version --short | awk -Fv '/Server Version: / {print $3}')",
+      "URL='https://install.portworx.com/${local.portworx_version}?mc=false&kbver='$VER'&b=true&j=auto&kd=${urlencode("/dev/pwx_vg/pwxkvdb")}&c=${var.cluster_name}&stork=true&st=k8s&pp=IfNotPresent'",
+      "kubectl --kubeconfig ${var.ssh.kubeconfig} apply -f $URL"
+    ]
+  }
+}
+
+resource "null_resource" "license_portworx" {
+  count = length(var.portworx_license) > 0 ? 1 : 0
+
+  depends_on = [
+    null_resource.install_portworx
+  ]
+  connection {
+    type        = "ssh"
+    user        = var.ssh.user
+    private_key = var.ssh.private_key
+    host        = var.ssh.worker_addresses[0]
+  }
+
+  provisioner "remote-exec" {
+    inline = [
+      "/opt/pwx/bin/pxctl license activate ${var.portworx_license}"
+    ]
+  }
+}

modules/portworx/variables.tf

@@ -0,0 +1,34 @@
+variable "portworx_version" {
+  type        = string
+  description = "The version of Portworx to install (latest_portworx_version will be used if not set)"
+  default     = ""
+}
+
+variable "portworx_license" {
+  type        = string
+  description = "License key for Portworx. A Trial license is used by default. Setting this value before Portworx is installed and ready will result in a failed `apply` that can be corrected by applying again after the Portworx install has completed."
+  default     = ""
+}
+
+variable "latest_portworx_version" {
+  type        = string
+  description = "The version of Portworx to install"
+  default     = "2.6"
+}
+
+variable "cluster_name" {
+  type        = string
+  description = "Name of the cluster"
+}
+
+variable "ssh" {
+  description = "SSH options for the storage provider including SSH details to access the control plane including the remote path to the kubeconfig file and a list of worker addresses"
+
+  type = object({
+    host             = string
+    private_key      = string
+    user             = string
+    kubeconfig       = string
+    worker_addresses = list(string)
+  })
+}

modules/storage/main.tf

@@ -0,0 +1,8 @@
+module "portworx" {
+  count            = var.storage_module == "portworx" ? 1 : 0
+  source           = "../portworx"
+  portworx_version = try(var.storage_options.portworx_version, "")
+  portworx_license = try(var.storage_options.portworx_license, "")
+  ssh              = var.ssh
+  cluster_name     = var.cluster_name
+}

modules/storage/variables.tf

@@ -0,0 +1,27 @@
+variable "storage_module" {
+  description = "The name of the Storage provider module (ex. \"portworx\")"
+  default     = ""
+}
+
+variable "storage_options" {
+  type        = any
+  description = "Options for the Storage provider module. Option names can be found in the documentation for each module and are prefixed with the vendor name (\"portworx_version\")"
+  default     = {}
+}
+
+variable "cluster_name" {
+  type        = string
+  description = "Name of the cluster"
+}
+
+variable "ssh" {
+  description = "SSH options for the storage provider including SSH details to access the control plane including the remote path to the kubeconfig file and a list of worker addresses."
+
+  type = object({
+    host             = string
+    private_key      = string
+    user             = string
+    kubeconfig       = string
+    worker_addresses = list(string)
+  })
+}

templates/pre_reqs.sh

@@ -54,7 +54,7 @@ EOM
 
 
 function unknown_os {
-    echo "I don't konw who I am" > /root/who_am_i.txt
+    echo "I don't know who I am" > /root/who_am_i.txt
 }
 
 if [ "$${OS:0:6}" = "centos" ] || [ "$${OS:0:4}" = "rhel" ]; then

templates/pre_reqs_worker.sh

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+OS='${operating_system}'
+
+function ubuntu_pre_reqs {
+    # Install Docker
+    export DEBIAN_FRONTEND=noninteractive
+    sudo apt-get update -qy
+    sudo apt-get install -qy lvm2
+}
+
+
+function rhel_pre_reqs {
+    sudo dnf install lvm2 -y
+}
+
+
+function unknown_os {
+    echo "I don't know who I am" > /root/who_am_i.txt
+}
+
+if [ "$${OS:0:6}" = "centos" ] || [ "$${OS:0:4}" = "rhel" ]; then
+    rhel_pre_reqs
+elif [ "$${OS:0:6}" = "ubuntu" ]; then
+    ubuntu_pre_reqs
+else
+    unknown_os
+fi