Send Journald logs to AWS CloudWatch (#707)

jollinshead · mumoshu · commit c67b4699d296 · 2017-06-27T08:56:13.000+09:00
A service has been introduced which runs a dockerised image of journald-cloudwatch-logs. This service forwards journald logs to AWS CloudWatch to a LogGroup with the name of .ClusterName and is run on all nodes (Etcds, Controllers and Workers). journald-cloudwatch-logs is a goLang project https://github.com/saymedia/journald-cloudwatch-logs. This feature is disabled by default and configurable in cluster.yaml: ``` cloudWatchLogging: enabled: false retentionInDays: 7 journaldCloudWatchLogsImage: jollinshead/journald-cloudwatch-logs:0.1 ```
diff --git a/Documentation/kubernetes-on-aws-journald-cloudwatch-logs.md b/Documentation/kubernetes-on-aws-journald-cloudwatch-logs.md
@@ -0,0 +1,16 @@
+# Journald logging to AWS CloudWatch
+
+A service has been introduced which runs a dockerised image of *journald-cloudwatch-logs*. This service forwards journald logs to AWS CloudWatch to a LogGroup with the name of .ClusterName and is run on all nodes (Etcds, Controllers and Workers).
+
+*journald-cloudwatch-logs* is a goLang project https://github.com/saymedia/journald-cloudwatch-logs.
+
+The default docker image *[jollinshead/journald-cloudwatch-logs](https://hub.docker.com/r/jollinshead/journald-cloudwatch-logs/)* is a wrapper around the go binary of *journald-cloudwatch-logs*.
+
+This feature is disabled by default and configurable in cluster.yaml:
+
+```
+cloudWatchLogging:
+ enabled: false
+ imageWithTag: jollinshead/journald-cloudwatch-logs:0.1
+ retentionInDays: 7
+```
diff --git a/README.md b/README.md
@@ -47,6 +47,7 @@ Check out our getting started tutorial on launching your first Kubernetes cluste
   * [Backup and restore for etcd](/Documentation/kubernetes-on-aws-backup-and-restore-for-etcd.md)
   * [Backup Kubernetes resources](/Documentation/kubernetes-on-aws-backup.md)
   * [Restore Kubernetes resources](/contrib/cluster-backup/README.md)
+  * [Journald logging to AWS CloudWatch](/Documentation/kubernetes-on-aws-journald-cloudwatch-logs.md)
 
 ## Examples
 
diff --git a/core/controlplane/config/config.go b/core/controlplane/config/config.go
@@ -103,16 +103,20 @@ func NewDefaultCluster() *Cluster {
 
 	return &Cluster{
 		DeploymentSettings: DeploymentSettings{
-			ClusterName:                        "kubernetes",
-			VPCCIDR:                            "10.0.0.0/16",
-			ReleaseChannel:                     "stable",
-			K8sVer:                             k8sVer,
-			ContainerRuntime:                   "docker",
-			Subnets:                            []model.Subnet{},
-			EIPAllocationIDs:                   []string{},
-			MapPublicIPs:                       true,
-			Experimental:                       experimental,
-			ManageCertificates:                 true,
+			ClusterName:        "kubernetes",
+			VPCCIDR:            "10.0.0.0/16",
+			ReleaseChannel:     "stable",
+			K8sVer:             k8sVer,
+			ContainerRuntime:   "docker",
+			Subnets:            []model.Subnet{},
+			EIPAllocationIDs:   []string{},
+			MapPublicIPs:       true,
+			Experimental:       experimental,
+			ManageCertificates: true,
+			CloudWatchLogging: CloudWatchLogging{
+				Enabled:         false,
+				RetentionInDays: 7,
+			},
 			HyperkubeImage:                     model.Image{Repo: "quay.io/coreos/hyperkube", Tag: k8sVer, RktPullDocker: false},
 			AWSCliImage:                        model.Image{Repo: "quay.io/coreos/awscli", Tag: "master", RktPullDocker: false},
 			CalicoNodeImage:                    model.Image{Repo: "quay.io/calico/node", Tag: "v1.2.1", RktPullDocker: false},
@@ -132,6 +136,7 @@ func NewDefaultCluster() *Cluster {
 			PauseImage:                         model.Image{Repo: "gcr.io/google_containers/pause-amd64", Tag: "3.0", RktPullDocker: false},
 			FlannelImage:                       model.Image{Repo: "quay.io/coreos/flannel", Tag: "v0.7.1", RktPullDocker: false},
 			DexImage:                           model.Image{Repo: "quay.io/coreos/dex", Tag: "v2.4.1", RktPullDocker: false},
+			JournaldCloudWatchLogsImage:        model.Image{Repo: "jollinshead/journald-cloudwatch-logs", Tag: "0.1", RktPullDocker: true},
 		},
 		KubeClusterSettings: KubeClusterSettings{
 			DNSServiceIP: "10.3.0.10",
@@ -485,6 +490,7 @@ type DeploymentSettings struct {
 	Experimental           Experimental      `yaml:"experimental"`
 	ManageCertificates     bool              `yaml:"manageCertificates,omitempty"`
 	WaitSignal             WaitSignal        `yaml:"waitSignal"`
+	CloudWatchLogging      `yaml:"cloudWatchLogging,omitempty"`
 
 	// Images repository
 	HyperkubeImage                     model.Image `yaml:"hyperkubeImage,omitempty"`
@@ -506,6 +512,7 @@ type DeploymentSettings struct {
 	PauseImage                         model.Image `yaml:"pauseImage,omitempty"`
 	FlannelImage                       model.Image `yaml:"flannelImage,omitempty"`
 	DexImage                           model.Image `yaml:"dexImage,omitempty"`
+	JournaldCloudWatchLogsImage        model.Image `yaml:"journaldCloudWatchLogsImage,omitempty"`
 }
 
 // Part of configuration which is specific to worker nodes
@@ -742,6 +749,11 @@ type KubeResourcesAutosave struct {
 	S3Path  string
 }
 
+type CloudWatchLogging struct {
+	Enabled         bool `yaml:"enabled"`
+	RetentionInDays int  `yaml:"retentionInDays"`
+}
+
 type LoadBalancer struct {
 	Enabled          bool     `yaml:"enabled"`
 	Names            []string `yaml:"names"`
diff --git a/core/controlplane/config/templates/cloud-config-controller b/core/controlplane/config/templates/cloud-config-controller
@@ -41,6 +41,34 @@ coreos:
         {{ $l }}
         {{- end }}
 {{- end}}
+{{if .CloudWatchLogging.Enabled}}
+    - name: journald-cloudwatch-logs.service
+      command: start
+      content: |
+        [Unit]
+        Description=Docker run journald-cloudwatch-logs to send journald logs to CloudWatch
+        Requires=network-online.target
+        After=network-online.target
+
+        [Service]
+        ExecStartPre=-/usr/bin/mkdir -p /var/journald-cloudwatch-logs
+        ExecStart=/usr/bin/rkt run \
+                  --insecure-options=image \
+                  --volume resolv,kind=host,source=/etc/resolv.conf,readOnly=true \
+                  --mount volume=resolv,target=/etc/resolv.conf \
+                  --volume journald-cloudwatch-logs,kind=host,source=/var/journald-cloudwatch-logs \
+                  --mount volume=journald-cloudwatch-logs,target=/var/journald-cloudwatch-logs \
+                  --volume journal,kind=host,source=/var/log/journal,readOnly=true \
+                  --mount volume=journal,target=/var/log/journal \
+                  --volume machine-id,kind=host,source=/etc/machine-id,readOnly=true \
+                  --mount volume=machine-id,target=/etc/machine-id \
+                  {{ .JournaldCloudWatchLogsImage.RktRepo }} -- {{.ClusterName}}
+        Restart=always
+        RestartSec=60s
+
+        [Install]
+        WantedBy=multi-user.target
+{{end}}
     - name: cfn-etcd-environment.service
       enable: true
       command: start
@@ -2309,4 +2337,4 @@ write_files:
          {{ end -}}
   {{ end }}
 
-{{ end }}
+{{ end }}
diff --git a/core/controlplane/config/templates/cloud-config-etcd b/core/controlplane/config/templates/cloud-config-etcd
@@ -73,7 +73,34 @@ coreos:
 
         [Install]
         RequiredBy=format-etcd2-volume.service
+{{if .CloudWatchLogging.Enabled}}
+    - name: journald-cloudwatch-logs.service
+      command: start
+      content: |
+        [Unit]
+        Description=Docker run journald-cloudwatch-logs to send journald logs to CloudWatch
+        Requires=network-online.target
+        After=network-online.target
 
+        [Service]
+        ExecStartPre=-/usr/bin/mkdir -p /var/journald-cloudwatch-logs
+        ExecStart=/usr/bin/rkt run \
+                  --insecure-options=image \
+                  --volume resolv,kind=host,source=/etc/resolv.conf,readOnly=true \
+                  --mount volume=resolv,target=/etc/resolv.conf \
+                  --volume journald-cloudwatch-logs,kind=host,source=/var/journald-cloudwatch-logs \
+                  --mount volume=journald-cloudwatch-logs,target=/var/journald-cloudwatch-logs \
+                  --volume journal,kind=host,source=/var/log/journal,readOnly=true \
+                  --mount volume=journal,target=/var/log/journal \
+                  --volume machine-id,kind=host,source=/etc/machine-id,readOnly=true \
+                  --mount volume=machine-id,target=/etc/machine-id \
+                  {{ .JournaldCloudWatchLogsImage.RktRepo }} -- {{.ClusterName}}
+        Restart=always
+        RestartSec=60s
+
+        [Install]
+        WantedBy=multi-user.target
+{{end}}
     {{if .Etcd.DisasterRecovery.SupportsEtcdVersion .Etcd.Version -}}
     - name: etcdadm-reconfigure.service
       enable: true
diff --git a/core/controlplane/config/templates/cloud-config-worker b/core/controlplane/config/templates/cloud-config-worker
@@ -80,6 +80,34 @@ coreos:
         [Mount]
         What={{$volumeMountSpec.Device}}
         Where={{$volumeMountSpec.Path}}
+{{end}}
+{{if .CloudWatchLogging.Enabled}}
+    - name: journald-cloudwatch-logs.service
+      command: start
+      content: |
+        [Unit]
+        Description=Docker run journald-cloudwatch-logs to send journald logs to CloudWatch
+        Requires=network-online.target
+        After=network-online.target
+
+        [Service]
+        ExecStartPre=-/usr/bin/mkdir -p /var/journald-cloudwatch-logs
+        ExecStart=/usr/bin/rkt run \
+                  --insecure-options=image \
+                  --volume resolv,kind=host,source=/etc/resolv.conf,readOnly=true \
+                  --mount volume=resolv,target=/etc/resolv.conf \
+                  --volume journald-cloudwatch-logs,kind=host,source=/var/journald-cloudwatch-logs \
+                  --mount volume=journald-cloudwatch-logs,target=/var/journald-cloudwatch-logs \
+                  --volume journal,kind=host,source=/var/log/journal,readOnly=true \
+                  --mount volume=journal,target=/var/log/journal \
+                  --volume machine-id,kind=host,source=/etc/machine-id,readOnly=true \
+                  --mount volume=machine-id,target=/etc/machine-id \
+                  {{ .JournaldCloudWatchLogsImage.RktRepo }} -- {{.ClusterName}}
+        Restart=always
+        RestartSec=60s
+
+        [Install]
+        WantedBy=multi-user.target
 {{end}}
     - name: cfn-etcd-environment.service
       enable: true
diff --git a/core/controlplane/config/templates/cluster.yaml b/core/controlplane/config/templates/cluster.yaml
@@ -1077,6 +1077,12 @@ worker:
 #  tag: v0.7.1
 #  rktPullDocker: false
 
+# JournaldCloudWatchLogsImage image repository to use. This sends journald logs to CloudWatch.
+#journaldCloudWatchLogsImage:
+#  repo: "jollinshead/journald-cloudwatch-logs"
+#  tag: "0.1"
+#  rktPullDocker: true
+
 # Use Calico for network policy.
 # useCalico: false
 
@@ -1124,6 +1130,12 @@ worker:
 #kubeResourcesAutosave:
 #  enabled: false
 
+# When enabled, all nodes will forward journald logs to AWS CloudWatch.
+# It is disabled by default.
+#cloudWatchLogging:
+# enabled: false
+# retentionInDays: 7
+
 # Addon features
 addons:
   # Will provision controller nodes with IAM permissions to run cluster-autoscaler and
diff --git a/core/controlplane/config/templates/stack-template.json b/core/controlplane/config/templates/stack-template.json
@@ -10,6 +10,13 @@
       "Type": "String",
       "Description": "Role to be used by ASG LifecycleHook to publish notification"
     }
+    {{if .CloudWatchLogging.Enabled}}
+    ,
+    "CloudWatchLogGroupARN": {
+      "Type": "String",
+      "Description": "CloudWatch LogGroup to send journald logs to"
+    }
+    {{end}}
   },
   "Resources": {
     "{{.Controller.LogicalName}}": {
@@ -183,6 +190,19 @@
                   "Effect": "Allow",
                   "Resource": "*"
                 },
+                {{if .CloudWatchLogging.Enabled}}
+                {
+                  "Effect": "Allow",
+                  "Action": [
+                    "logs:CreateLogStream",
+                    "logs:PutLogEvents",
+                    "logs:DescribeLogStreams"
+                  ],
+                  "Resource": [
+                    { "Ref": "CloudWatchLogGroupARN" },
+                    { "Fn::Join" : [ "", [{ "Ref": "CloudWatchLogGroupARN" }, ":log-stream:*"]] }
+                  ]
+                },{{ end }}
                 {{ if .UserDataController.Parts.s3 }}
 		            {
                   "Effect": "Allow",
@@ -192,6 +212,19 @@
                   "Resource": "arn:{{.Region.Partition}}:s3:::{{ .UserDataController.Parts.s3.Asset.S3Prefix }}*"
 		            },
                 {{ end }}
+                {{if .CloudWatchLogging.Enabled}}
+                {
+                  "Effect": "Allow",
+                  "Action": [
+                    "logs:CreateLogStream",
+                    "logs:PutLogEvents",
+                    "logs:DescribeLogStreams"
+                  ],
+                  "Resource": [
+                    { "Ref": "CloudWatchLogGroupARN" },
+                    { "Fn::Join" : [ "", [{ "Ref": "CloudWatchLogGroupARN" }, ":log-stream:*"]] }
+                  ]
+                },{{ end }}
                 {{if .WaitSignal.Enabled}}
                 {
                   "Action": "cloudformation:SignalResource",
@@ -301,7 +334,7 @@
         {{if .Controller.IAMConfig.Role.Name }}
         "RoleName":  {"Fn::Join": ["",[{"Ref": "AWS::Region"},"-","{{.Controller.IAMConfig.Role.Name}}"]]},
         {{end}}
-        "ManagedPolicyArns": [ 
+        "ManagedPolicyArns": [
           {{range $policyIndex, $policyArn := .Controller.IAMConfig.Role.ManagedPolicies }}
             "{{$policyArn.Arn}}",
           {{end}}
@@ -420,6 +453,19 @@
                   ],
                   "Resource": "arn:{{.Region.Partition}}:s3:::{{$.EtcdSnapshotsS3Bucket}}"
                 },
+                {{if .CloudWatchLogging.Enabled}}
+                {
+                  "Effect": "Allow",
+                  "Action": [
+                    "logs:CreateLogStream",
+                    "logs:PutLogEvents",
+                    "logs:DescribeLogStreams"
+                  ],
+                  "Resource": [
+                    { "Ref": "CloudWatchLogGroupARN" },
+                    { "Fn::Join" : [ "", [{ "Ref": "CloudWatchLogGroupARN" }, ":log-stream:*"]] }
+                  ]
+                },{{ end }}
                 {
                   "Effect": "Allow",
                   "Action": [
@@ -981,7 +1027,7 @@
     {{/* 443 ingress from controller to controller is required by:
       - API server endpoint with HA controllers
       - calico-policy-controller when calico is enabled
-      See https://github.com/kubernetes-incubator/kube-aws/issues/494#issuecomment-291687137 
+      See https://github.com/kubernetes-incubator/kube-aws/issues/494#issuecomment-291687137
       and https://github.com/kubernetes-incubator/kube-aws/issues/512 */}}
     "SecurityGroupControllerIngressFromControllerToController": {
       "Properties": {
diff --git a/core/nodepool/config/templates/stack-template.json b/core/nodepool/config/templates/stack-template.json
@@ -314,6 +314,20 @@
                   "Resource": "arn:{{.Region.Partition}}:s3:::{{ $.UserDataWorker.Parts.s3.Asset.S3Prefix }}*"
                 },
                 {{- end }}
+                {{if .CloudWatchLogging.Enabled}}
+                {
+                  "Effect": "Allow",
+                  "Action": [
+                    "logs:CreateLogStream",
+                    "logs:PutLogEvents",
+                    "logs:DescribeLogStreams"
+                  ],
+                  "Resource": [
+                    { "Ref": "CloudWatchLogGroupARN" },
+                    { "Fn::Join" : [ "", [{ "Ref": "CloudWatchLogGroupARN" }, ":log-stream:*"]] }
+                  ]
+                },
+                {{ end }}
                 {{ if .KubeResourcesAutosave.Enabled }}
                 {
                   "Effect": "Allow",
@@ -481,6 +495,13 @@
       "Type": "String",
       "Description": "Role to be used by ASG LifecycleHook to publish notification"
     }
+    {{if .CloudWatchLogging.Enabled}}
+    ,
+    "CloudWatchLogGroupARN": {
+      "Type": "String",
+      "Description": "CloudWatch LogGroup to send journald logs to"
+    }
+    {{ end }}
   },
   "Resources": {
     {{if .SpotFleet.Enabled}}
diff --git a/core/root/config/templates/stack-template.json b/core/root/config/templates/stack-template.json
@@ -35,12 +35,24 @@
       },
       "Type": "AWS::IAM::Role"
     },
+      {{if .CloudWatchLogging.Enabled}}
+      "CloudWatchLogGroup": {
+      "Type" : "AWS::Logs::LogGroup",
+      "Properties" : {
+        "LogGroupName" : "{{.ClusterName}}",
+        "RetentionInDays" : {{.CloudWatchLogging.RetentionInDays}}
+      }
+    },{{ end }}
     "{{.ControlPlane.Name}}": {
       "Type" : "AWS::CloudFormation::Stack",
       "Properties" : {
         "Parameters": {
           "NotificationTargetARN": { "Fn::GetAtt": [ "ASGNotificationTarget", "Arn" ] },
           "NotificationRoleARN": { "Fn::GetAtt": [ "ASGNotificationRole", "Arn" ] }
+          {{if .CloudWatchLogging.Enabled}}
+          ,
+          "CloudWatchLogGroupARN": { "Fn::GetAtt": [ "CloudWatchLogGroup", "Arn" ] }
+          {{ end }}
         },
         "Tags" : [
           {
@@ -62,6 +74,10 @@
           "ControlPlaneStackName": {"Fn::GetAtt" : [ "{{$.ControlPlane.Name}}" , "Outputs.StackName" ]},
           "NotificationTargetARN": { "Fn::GetAtt": [ "ASGNotificationTarget", "Arn" ] },
           "NotificationRoleARN": { "Fn::GetAtt": [ "ASGNotificationRole", "Arn" ] }
+          {{if .CloudWatchLogging.Enabled}}
+          ,
+          "CloudWatchLogGroupARN": { "Fn::GetAtt": [ "CloudWatchLogGroup", "Arn" ] }
+          {{ end }}
         },
         "Tags" : [
           {
diff --git a/core/root/template_params.go b/core/root/template_params.go
diff --git a/e2e/run b/e2e/run
diff --git a/test/integration/maincluster_test.go b/test/integration/maincluster_test.go
