updates for cluster autoscaler functionality. fixed issues with index…

autoscaling.tf

@@ -24,14 +24,17 @@ module "icpautoscaling" {
     #icpuser         = "aws_lb_target_group_attachment.master-8001.arn" // attempt at workaround for missing depends on
 
     kube_api_url    = "https://${aws_lb.icp-console.dns_name}:8001"
+    docker_registry = "${var.user_provided_cert_dns != "" ? var.user_provided_cert_dns : aws_lb.icp-console.dns_name}:8500"
 
     aws_region            = "${var.aws_region}"
     azs                   = ["${var.azs}"]
-    ami                   = "${var.worker["ami"] != "" ? var.worker["ami"] : lookup(local.default_searched_ami, var.ami, var.ami)}"
+    ami                   = "${var.worker["ami"] != "" ? var.worker["ami"] : local.default_ami }"
     worker_root_disk_size = "${var.worker["disk"]}"
     worker_docker_vol_size = "${var.worker["docker_vol"]}"
     key_name              = "${var.key_name}"
     instance_type         = "${var.worker["type"]}"
+    ebs_optimized         = "${var.worker["ebs_optimized"]}"
+    instance_name         = "${var.instance_name}"
     security_groups = [
       "${aws_security_group.default.id}"
     ]
@@ -44,4 +47,90 @@ module "icpautoscaling" {
     image_location            = "${local.image_package_uri}"
     icp_inception_image       = "${var.icp_inception_image}"
     lambda_s3_bucket          = "${local.lambda_s3_bucket}"
+    icp_config_s3_bucket      = "${aws_s3_bucket.icp_config_backup.id}"
+    asg_tags                  = ["${data.null_data_source.asg-tags.*.outputs}"]
 }
+
+data "null_data_source" "asg-tags" {
+  count = "${length(keys(var.default_tags))}"
+  inputs = {
+    key                 = "${element(keys(var.default_tags), count.index)}"
+    value               = "${element(values(var.default_tags), count.index)}"
+    propagate_at_launch = "true"
+  }
+}
+
+resource "aws_s3_bucket_object" "icp_cluster_autoscaler_yaml" {
+  bucket = "${aws_s3_bucket.icp_config_backup.id}"
+  key    = "scripts/cluster-autoscaler-deployment.yaml"
+  content = <<EOF
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: cluster-autoscaler
+  namespace: kube-system
+  labels:
+    app: cluster-autoscaler
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: cluster-autoscaler
+  template:
+    metadata:
+      labels:
+        app: cluster-autoscaler
+    spec:
+      serviceAccountName: cluster-autoscaler
+      containers:
+        - image: k8s.gcr.io/cluster-autoscaler:v1.2.2
+          name: cluster-autoscaler
+          resources:
+            limits:
+              cpu: 100m
+              memory: 300Mi
+            requests:
+              cpu: 100m
+              memory: 300Mi
+          command:
+            - ./cluster-autoscaler
+            - --v=4
+            - --stderrthreshold=info
+            - --cloud-provider=aws
+            - --skip-nodes-with-local-storage=false
+            - --skip-nodes-with-system-pods=false
+            - --expander=least-waste
+            - --node-group-auto-discovery=asg:tag=k8s.io/cluster-autoscaler/enabled,kubernetes.io/cluster/${random_id.clusterid.hex}
+            - --balance-similar-node-groups=true
+          volumeMounts:
+            - name: ssl-certs
+              mountPath: /etc/ssl/certs/ca-certificates.crt
+              readOnly: true
+          imagePullPolicy: "Always"
+      nodeSelector:
+        master: "true"
+      tolerations:
+      - effect: NoSchedule
+        key: dedicated
+        operator: Exists
+      - key: CriticalAddonsOnly
+        operator: Exists
+      volumes:
+        - name: ssl-certs
+          hostPath:
+            path: "/etc/ssl/certs/ca-bundle.crt"
+EOF
+}
+
+resource "aws_s3_bucket_object" "asg_configmap" {
+  bucket = "${aws_s3_bucket.icp_config_backup.id}"
+  key    = "scripts/asg-configmap.yaml"
+  source = "${path.module}/scripts/asg-configmap.yaml"
+}
+
+resource "aws_s3_bucket_object" "cluster_autoscaler_rbac_yaml" {
+  bucket = "${aws_s3_bucket.icp_config_backup.id}"
+  key    = "scripts/cluster-autoscaler-rbac.yaml"
+  source = "${path.module}/scripts/cluster-autoscaler-rbac.yaml"
+}
+

autoscaling/autoscalinggroup.tf

@@ -10,7 +10,7 @@ resource "aws_launch_configuration" "icp_worker_lc" {
 
   security_groups = ["${var.security_groups}"]
 
-  ebs_optimized = true
+  ebs_optimized = "${var.ebs_optimized}"
   root_block_device {
     volume_size = "${var.worker_root_disk_size}"
   }
@@ -27,15 +27,17 @@ resource "aws_launch_configuration" "icp_worker_lc" {
 packages:
   - unzip
   - python
+  - bind-utils
 rh_subscription:
   enable-repo: rhui-REGION-rhel-server-optional
 write_files:
-  - path: /tmp/bootstrap.sh
+  - path: /tmp/bootstrap-node.sh
     permissions: '0755'
     encoding: b64
-    content: ${base64encode(file("${path.module}/../scripts/bootstrap.sh"))}
+    content: ${base64encode(file("${path.module}/../scripts/bootstrap-node.sh"))}
 runcmd:
-  - /tmp/bootstrap.sh ${var.docker_package_location != "" ? "-p ${var.docker_package_location}" : "" } -d /dev/xvdx ${var.image_location != "" ? "-i ${var.image_location}" : "" } -s ${var.icp_inception_image}
+  - /tmp/bootstrap-node.sh -c ${var.icp_config_s3_bucket} -s "bootstrap.sh"
+  - /tmp/icp_scripts/bootstrap.sh ${var.docker_package_location != "" ? "-p ${var.docker_package_location}" : "" } -d /dev/xvdx ${var.image_location != "" ? "-i ${var.image_location}" : "" } -s ${var.icp_inception_image}
 users:
   - default
   - name: icpdeploy
@@ -54,57 +56,58 @@ EOF
 }
 
 resource "aws_autoscaling_group" "icp_worker_asg" {
-  count = "${var.enabled ? 1 : 0}"
-  name                 = "icp-worker-asg-${var.cluster_id}"
+  count = "${length(var.azs)}"
+  name                 = "icp-worker-asg-${var.aws_region}${element(var.azs, count.index)}-${var.cluster_id}"
   launch_configuration = "${aws_launch_configuration.icp_worker_lc.name}"
   min_size             = 0
   max_size             = 20
   force_delete         = true
 
-  availability_zones   = "${formatlist("%v%v", var.aws_region, var.azs)}"
-  vpc_zone_identifier  = ["${var.private_subnet_ids}"]
+  vpc_zone_identifier  = ["${element(var.private_subnet_ids, count.index)}"]
 
-  tags = [
-    {
-      key                 = "kubernetes.io/cluster/${var.cluster_id}",
-      value               = "${var.cluster_id}",
-      propagate_at_launch = true
-    }
-  ]
+  tags = ["${concat(
+    var.asg_tags,
+    list(map("key", "k8s.io/cluster-autoscaler/enabled", "value", "${var.enabled}", "propagate_at_launch", "false")),
+    list(map("key", "kubernetes.io/cluster/${var.cluster_id}", "value", "${var.cluster_id}", "propagate_at_launch", "true"))
+  )}"]
 }
 
 resource "aws_autoscaling_lifecycle_hook" "icp_add_worker_hook" {
-  count = "${var.enabled ? 1 : 0}"
-  name                   = "icp-workernode-added-${var.cluster_id}"
-  autoscaling_group_name = "${aws_autoscaling_group.icp_worker_asg.name}"
+  count = "${length(var.azs)}"
+  name                   = "icp-workernode-added-${var.aws_region}${element(var.azs, count.index)}-${var.cluster_id}"
+  autoscaling_group_name = "${element(aws_autoscaling_group.icp_worker_asg.*.name, count.index)}"
   default_result         = "ABANDON"
   heartbeat_timeout      = 3600
   lifecycle_transition   = "autoscaling:EC2_INSTANCE_LAUNCHING"
 
   notification_metadata = <<EOF
 {
-  "icp_inception_image": "${var.icp_inception_image}",
+  "icp_inception_image": "${var.docker_registry}/${var.icp_inception_image}",
   "docker_package_location": "${var.docker_package_location}",
   "image_location": "${var.image_location}",
-  "cluster_backup": "icpbackup-${var.cluster_id}"
+  "cluster_backup": "icpbackup-${var.cluster_id}",
+  "cluster_id": "${var.cluster_id}",
+  "instance_name": "${var.instance_name}"
 }
 EOF
 }
 
 resource "aws_autoscaling_lifecycle_hook" "icp_del_worker_hook" {
-  count = "${var.enabled ? 1 : 0}"
-  name                   = "icp-workernode-removed-${var.cluster_id}"
-  autoscaling_group_name = "${aws_autoscaling_group.icp_worker_asg.name}"
+  count = "${length(var.azs)}"
+  name                   = "icp-workernode-removed-${var.aws_region}${element(var.azs, count.index)}-${var.cluster_id}"
+  autoscaling_group_name = "${element(aws_autoscaling_group.icp_worker_asg.*.name, count.index)}"
   default_result         = "ABANDON"
   heartbeat_timeout      = 3600
   lifecycle_transition   = "autoscaling:EC2_INSTANCE_TERMINATING"
 
   notification_metadata = <<EOF
 {
-  "icp_inception_image": "${var.icp_inception_image}",
+  "icp_inception_image": "${var.docker_registry}/${var.icp_inception_image}",
   "docker_package_location": "${var.docker_package_location}",
   "image_location": "${var.image_location}",
-  "cluster_backup": "icpbackup-${var.cluster_id}"
+  "cluster_backup": "icpbackup-${var.cluster_id}",
+  "cluster_id": "${var.cluster_id}",
+  "instance_name": "${var.instance_name}"
 }
 EOF
 }

autoscaling/cloudwatch.tf

@@ -13,7 +13,7 @@ resource "aws_cloudwatch_event_rule" "icp_worker_node_added_event" {
   ],
   "detail": {
     "AutoScalingGroupName": [
-      "${aws_autoscaling_group.icp_worker_asg.name}"
+      ${join(",", formatlist("\"%v\"", aws_autoscaling_group.icp_worker_asg.*.name))}
     ]
   }
 }
@@ -35,7 +35,7 @@ resource "aws_cloudwatch_event_rule" "icp_worker_node_remove_event" {
   ],
   "detail": {
     "AutoScalingGroupName": [
-      "${aws_autoscaling_group.icp_worker_asg.name}"
+      ${join(",", formatlist("\"%v\"", aws_autoscaling_group.icp_worker_asg.*.name))}
     ]
   }
 }

autoscaling/lambda.tf

@@ -47,7 +47,7 @@ resource "aws_lambda_function" "icp_autoscale" {
   function_name    = "icp-worker-autoscale-${var.cluster_id}"
   role             = "${local.iam_lambda_role_arn}"
   handler          = "index.handler"
-  runtime          = "nodejs6.10"
+  runtime          = "nodejs8.10"
   timeout          = 10
 
   vpc_config {

autoscaling/lambda/common.js

@@ -1,29 +1,42 @@
 var aws = require('aws-sdk');
 var kubeapi = require('kubernetes-client');
 var fs = require('fs');
-
+var region; 
 aws.config.update({region: region});
 aws.config.setPromisesDependency(Promise);
 
 function get_instance_ip(region, instance_id) {
   var ec2 = new aws.EC2({apiVersion: '2016-11-15'});
 
-  console.log("InstanceID: " + instance_id);
+  console.log("get_instance_ip: Region: " + region);
+  console.log("get_instance_ip: InstanceID: " + instance_id);
 
   var params = {
-    DryRun: false,
-    InstanceIds: [ instance_id ],
+    //DryRun: false,
+    InstanceIds: [ instance_id ]
   };
 
-  return ec2.describeInstances(params, function(err, result) {
-    if (err) {
-      console.log(err, err.stack);
+  var request = ec2.describeInstances(params);
+
+  var promise = request.promise();
+
+  // handle promise's fulfilled/rejected states
+  promise.then(
+    function(data) {
+      console.log("get_instance_ip: Instance IP address is: " + data.Reservations[0].Instances[0].PrivateIpAddress);
+      console.log("get_instance_ip: " + JSON.stringify(data, null, 2));
+
+      //return data.Reservations[0].Instances[0].PrivateIpAddress;
+      /* process the data */
+    },
+    function(err) {
+      /* handle the error */
+      console.log("get_instance_ip: " + err, err.stack);
       throw err;
-    } else {
-      console.log("Instance IP address is: " + result.Reservations[0].Instances[0].PrivateIpAddress);
-      return result.Reservations[0].Instances[0].PrivateIpAddress;
     }
-  });
+  );
+
+  return promise;
 }
 
 function get_bucket_object(bucketName, key) {
@@ -34,16 +47,26 @@ function get_bucket_object(bucketName, key) {
     Key: key
   };
 
-  s3.getObject(params, function(err, data) {
-    if (err) {
-      console.log(err, err.stack);
+  var request = s3.getObject(params);
+
+  var promise = request.promise();
+
+  // handle promise's fulfilled/rejected states
+  promise.then(
+    function(data) {
+      console.log("get_bucket_object: body is: " + data.Body);
+
+      //return data.Reservations[0].Instances[0].PrivateIpAddress;
+      /* process the data */
+    },
+    function(err) {
+      /* handle the error */
+      console.log("get_bucket_object: " + err, err.stack);
       throw err;
-    } else {
-      console.log(data);           // successful response
-
-      return data.Body;
     }
-  });
+  );
+
+  return promise;
 }
 
 function create_job(ca_crt, client_cert, client_key, job) {
@@ -77,8 +100,10 @@ function fail_autoscaling(params) {
     .then(function(result) {
         console.log("competed lifecycle action");
     });
-};
+}
 
 module.exports.get_instance_ip = get_instance_ip;
 module.exports.create_job = create_job;
 module.exports.fail_autoscaling = fail_autoscaling;
+module.exports.get_bucket_object = get_bucket_object;
+