Add s3 reliability test

ansible/roles/vector/tasks/configure.yml

@@ -1,6 +1,6 @@
 ---
 - name: Install Vector configuration
-  template:
+  copy:
     src: "{{ configuration_file }}"
     dest: /etc/vector/vector.toml
     mode: 0644

ansible/roles/verifiable-logger/tasks/install.yml

@@ -0,0 +1,21 @@
+---
+- name: Create verifiable-logger directory
+  file:
+    path: /var/lib/verifiable-logger
+    state: directory
+
+- name: Copy verifiable-logger binary
+  get_url:
+    url: "https://verifiable-logger-builds.s3.us-east-2.amazonaws.com/verifiable-logger"
+    dest: /usr/local/bin/verifiable-logger
+    mode: 755
+
+- name: Copy generate-logs unit file
+  template:
+    src: generate-logs.service
+    dest: /etc/systemd/system/generate-logs.service
+
+- name: Copy verify-logs unit file
+  template:
+    src: verify-logs.service
+    dest: /etc/systemd/system/verify-logs.service

ansible/roles/verifiable-logger/tasks/main.yml

@@ -0,0 +1,2 @@
+---
+- include: "{{ action }}.yml"

ansible/roles/verifiable-logger/tasks/start-generating.yml

@@ -0,0 +1,6 @@
+---
+- name: Start generating logs
+  systemd:
+    name: generate-logs
+    state: started
+    daemon-reload: yes

ansible/roles/verifiable-logger/tasks/start-verifying.yml

@@ -0,0 +1,6 @@
+---
+- name: Start verifying logs
+  systemd:
+    name: verify-logs
+    state: started
+    daemon-reload: yes

ansible/roles/verifiable-logger/templates/generate-logs.service

@@ -0,0 +1,6 @@
+[Unit]
+Description=generate-logs
+
+[Service]
+ExecStart=/usr/local/bin/verifiable-logger generate --rate 10 --output output.log
+WorkingDirectory=/var/lib/verifiable-logger/

ansible/roles/verifiable-logger/templates/verify-logs.service

@@ -0,0 +1,6 @@
+[Unit]
+Description=verify-logs
+
+[Service]
+ExecStart=/usr/local/bin/verifiable-logger verify file-to-s3-reliability-test-data us-east-1 --prefix "host=%H" --tail
+WorkingDirectory=/var/lib/verifiable-logger/

bin/test

@@ -218,6 +218,7 @@ if [ "$BOOTSTRAP" == 'true' ] && [ "$SKIP_TERRAFORM" != 'true' ]; then
     -var test_configuration="$TEST_CONFIGURATION" \
     -var user_id="$TEST_USER_ID" \
     -var results_s3_bucket_name="$TEST_RESULTS_S3_BUCKET_NAME" \
+    -var test_harness_aws_account_ids="[\"$VECTOR_TEST_HARNESS_AWS_ACCOUNT_ID\"]" \
     -var-file="../configurations/$TEST_CONFIGURATION/terraform.tfvars"
 
   cd ..

cases/file_to_s3_reliability/README.md

@@ -0,0 +1,25 @@
+# File to S3 Reliability Test
+
+This is a long-running test of Vector tailing a file and sending the data to S3.
+It uses the [`verifiable-logger`][0] project both to generate the log data and
+verify that it all reaches S3.
+
+## Try it
+
+You can run this test via:
+
+```
+test -t file_to_s3_reliability
+```
+
+## Resources
+
+* [Setup][setup]
+* [Development][development]
+* [How it works][how_it_works]
+* [Vector docs][docs]
+* [Vector repo][repo]
+* [Vector website][website]
+
+
+[0]: https://github.com/timberio/verifiable-logger

cases/file_to_s3_reliability/ansible/bootstrap.yml

@@ -0,0 +1,21 @@
+---
+- hosts: '{{ test_namespace }}'
+  tasks:
+    - meta: refresh_inventory
+    - name: Wait 600 seconds for target connection to become reachable/usable
+      wait_for_connection:
+
+    - debug:
+        var: playbook_dir
+
+- hosts: '{{ test_namespace }}:&tag_TestRole_subject'
+  become: true
+  roles:
+    - role: vector
+      action: install
+
+    - role: vector
+      action: configure
+
+    - role: verifiable-logger
+      action: install

cases/file_to_s3_reliability/ansible/config_files/vector.toml

@@ -0,0 +1,16 @@
+data_dir = "/var/lib/vector"
+
+[sources.file]
+type = "file"
+include = ["/var/lib/verifiable-logger/output.log"]
+start_at_beginning = true
+
+[sinks.s3]
+inputs = ["file"]
+type = "aws_s3"
+region = "us-east-1"
+bucket = "file-to-s3-reliability-test-data"
+key_prefix = "host={{ host }}/date=%F/"
+encoding = "text"
+compression = "none"
+batch.timeout_secs = 30

cases/file_to_s3_reliability/ansible/run.yml

@@ -0,0 +1,12 @@
+---
+- hosts: '{{ test_namespace }}:&tag_TestRole_subject'
+  become: true
+  roles:
+    - role: verifiable-logger
+      action: start-generating
+
+    - role: verifiable-logger
+      action: start-verifying
+
+    - role: vector
+      action: start

cases/file_to_s3_reliability/configurations/default/ansible.yml

@@ -0,0 +1,2 @@
+---
+foo: "bar"

cases/file_to_s3_reliability/configurations/default/terraform.tfvars

@@ -0,0 +1,6 @@
+# WARNING!
+#
+# Do not modify the parameters of this file since historical test results are
+# based on these parameters. Please create a new configuration and specify taht
+# configuration instead.
+subject_instance_type = "c5.large"

cases/file_to_s3_reliability/terraform/main.tf

@@ -0,0 +1,70 @@
+provider "aws" {
+  region  = "us-east-1"
+  version = "~> 2.53"
+}
+
+terraform {
+  required_version = ">= 0.12"
+  backend "s3" {}
+}
+
+module "topology" {
+  source = "../../../terraform/aws_uni_topology"
+
+  providers = {
+    aws = aws
+  }
+
+  pub_key                 = var.pub_key
+  subject_instance_type   = var.subject_instance_type
+  test_configuration      = var.test_configuration
+  test_name               = var.test_name
+  user_id                 = var.user_id
+  results_s3_bucket_name  = var.results_s3_bucket_name
+}
+
+resource "aws_s3_bucket" "logs-bucket" {
+  # data is namespaced by host within the bucket
+  bucket = "file-to-s3-reliability-test-data"
+
+  lifecycle_rule {
+    enabled = true
+
+    expiration {
+      days = 14
+    }
+  }
+}
+
+data "aws_iam_policy_document" "logs-bucket-policy" {
+  statement {
+    sid = "AllowTestHarnessListBucket"
+
+    actions = [
+      "s3:ListBucket",
+    ]
+
+    resources = [
+      aws_s3_bucket.logs-bucket.arn,
+    ]
+  }
+
+  statement {
+    sid = "AllowTestHarnessEverythingElse"
+
+    actions = [
+      "s3:GetObject",
+      "s3:PutObject",
+      "s3:DeleteObject",
+    ]
+
+    resources = [
+      "${aws_s3_bucket.logs-bucket.arn}/*",
+    ]
+  }
+}
+
+resource "aws_iam_role_policy" "default" {
+  role   = module.topology.instance_profile_name
+  policy = data.aws_iam_policy_document.logs-bucket-policy.json
+}

cases/file_to_s3_reliability/terraform/variables.tf

@@ -0,0 +1,28 @@
+variable "pub_key" {
+  type = string
+}
+
+variable "subject_instance_type" {
+  type = string
+}
+
+variable "test_configuration" {
+  type = string
+}
+
+variable "test_name" {
+  type = string
+}
+
+variable "user_id" {
+  type = string
+}
+
+variable "test_harness_aws_account_ids" {
+  type = list(string)
+}
+
+// don't actually need this, but need to accept it
+variable "results_s3_bucket_name" {
+  type = string
+}

cases/file_to_s3_reliability/terraform/versions.tf

@@ -0,0 +1,3 @@
+terraform {
+  required_version = ">= 0.12"
+}

terraform/aws_uni_topology/main.tf

@@ -11,7 +11,7 @@ locals {
 }
 
 module "vpc" {
-  source = "../../../terraform/aws_vpc"
+  source = "../aws_vpc"
 
   providers = {
     aws = aws
@@ -24,7 +24,7 @@ module "vpc" {
 }
 
 module "aws_instance_profile" {
-  source = "../../../terraform/aws_instance_profile"
+  source = "../aws_instance_profile"
 
   providers = {
     aws = aws
@@ -36,7 +36,7 @@ module "aws_instance_profile" {
 }
 
 module "aws_instance_subject" {
-  source = "../../../terraform/aws_instance"
+  source = "../aws_instance"
 
   providers = {
     aws = aws

terraform/aws_uni_topology/outputs.tf

@@ -0,0 +1,3 @@
+output "instance_profile_name" {
+  value = module.aws_instance_profile.name
+}