Hashicorp vault implementation and puppetserver integration

files/vault/vault-process-init-output

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+KEY_OWNERS=${1}
+INPUT=$(/usr/bin/cat)
+
+while IFS=',' read -ra OWNER; do
+    for counter in `/usr/bin/seq 0 $(( ${#OWNER[@]} - 1 ))`; do
+        owner="${OWNER[${counter}]}"
+	key=$(/usr/bin/echo ${INPUT} | /usr/bin/jq -r ".unseal_keys_b64[${counter}]")
+
+        /usr/bin/echo $(/usr/bin/jq -n --arg key "$owner" --arg value "$key" '{ ($key): ($value) }')
+    done | /usr/bin/jq -s add
+done <<< "${KEY_OWNERS}" | /usr/bin/jq '{ "vault_encrypted_unseal_keys": . }'

files/vault/vault-unseal

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+encrypted_unseal_key=$(/usr/bin/cat ${1})
+
+/usr/bin/vault operator unseal -tls-skip-verify $(/usr/bin/echo ${encrypted_unseal_key} | /usr/bin/base64 --decode | /usr/bin/gpg --decrypt --quiet)

files/vault/vaultbackup

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+mv /data/backup/vault/current/*.tar.gz /data/backup/vault/archive
+
+tar cvzf /data/backup/vault/current/vault_$(date +%Y.%m.%d.%H%M%S).tar.gz \
+        /opt/vault \
+        /home/vault/.vault-token \
+        /home/vault/encrypted_unseal_key \

lib/puppet/functions/hiera_vault.rb

@@ -0,0 +1,156 @@
+# Originally copied from Peter Souter hiera_vault repository, licensed under the Apache License, Version 2.0
+#
+# https://github.com/petems/petems-hiera_vault/blob/master/lib/puppet/functions/hiera_vault.rb
+
+require_relative 'hiera_vault/authentication.rb'
+
+Puppet::Functions.create_function(:hiera_vault) do
+  begin
+    require 'vault'
+  rescue LoadError => _e
+    raise Puppet::DataBinding::LookupError, "[hiera-vault] Must install vault gem to use hiera-vault backend"
+  end
+
+  dispatch :lookup_key do
+    param 'Variant[String, Numeric]', :key
+    param 'Hash', :options
+    param 'Puppet::LookupContext', :context
+  end
+
+  def lookup_key(key, options, context)
+    if confine_keys = options['confine_to_keys']
+      raise ArgumentError, '[hiera-vault] confine_to_keys must be an array' unless confine_keys.is_a?(Array)
+
+      begin
+        confine_keys = confine_keys.map { |r| Regexp.new(r) }
+      rescue StandardError => e
+        raise Puppet::DataBinding::LookupError, "[hiera-vault] creating regexp failed with: #{e}"
+      end
+
+      regex_key_match = Regexp.union(confine_keys)
+
+      unless key[regex_key_match] == key
+        context.explain { "[hiera-vault] Skipping hiera_vault backend because key '#{key}' does not match confine_to_keys" }
+        context.not_found
+      end
+    end
+
+    if strip_from_keys = options['strip_from_keys']
+      raise ArgumentError, '[hiera-vault] strip_from_keys must be an array' unless strip_from_keys.is_a?(Array)
+
+      strip_from_keys.each do |prefix|
+        key = key.gsub(Regexp.new(prefix), '')
+      end
+    end
+
+    vault_get(key, options, context)
+  end
+
+  def vault_get(key, options, context)
+    hiera_vault_client = Vault::Client.new
+
+    begin
+      hiera_vault_client.configure do |config|
+        config.address = options['address'] unless options['address'].nil?
+        config.ssl_verify = options['ssl_verify'] unless options['ssl_verify'].nil?
+        config.ssl_ca_cert = options['ssl_ca_cert'] if config.respond_to? :ssl_ca_cert
+      end
+
+      context.explain { "[hiera-vault] Using #{options['authentication']['method']} authentication" }
+      authenticate(options['authentication'], hiera_vault_client, context)
+
+      if hiera_vault_client.sys.seal_status.sealed?
+        raise Puppet::DataBinding::LookupError, "[hiera-vault] vault is sealed"
+      end
+
+      context.explain { "[hiera-vault] Client configured to connect to #{hiera_vault_client.address}" }
+    rescue StandardError => e
+      hiera_vault_client.shutdown
+      raise Puppet::DataBinding::LookupError, "[hiera-vault] Skipping backend. Configuration error: #{e}"
+    end
+
+    answer = nil
+
+    # Only kv v2 mounts supported
+    options['mounts'].each_pair do |mount, paths|
+      interpolate(context, paths).each do |path|
+        context.explain { "[hiera-vault] Looking on mount #{mount} in path #{path} for #{key}" }
+
+        secret = nil
+
+        begin
+          context.explain { "[hiera-vault] Checking path: #{path} on mount: #{mount}" }
+          secret = hiera_vault_client.kv(mount).read(File.join(path, key).chomp('/'))
+        rescue Vault::HTTPConnectionError
+          msg = "[hiera-vault] Could not connect to read secret: #{path} on mount: #{mount}"
+          context.explain { msg }
+          raise Puppet::DataBinding::LookupError, msg
+        rescue Vault::HTTPError => e
+          msg = "[hiera-vault] Could not read secret #{path} on mount #{mount}: #{e.errors.join("\n").rstrip}"
+          context.explain { msg }
+          raise Puppet::DataBinding::LookupError, msg
+        end
+
+        next if secret.nil?
+
+        context.explain { "[hiera-vault] Read secret: #{key}" }
+        # Turn secret's hash keys into strings allow for nested arrays and hashes
+        # this enables support for create resources etc
+        answer = secret.data.inject({}) { |h, (k, v)| h[k.to_s] = stringify_keys v; h }
+        break
+      end
+
+      break unless answer.nil?
+    end
+
+    raise Puppet::DataBinding::LookupError, "[hiera-vault] Could not find secret #{key}" if answer.nil?
+
+    answer = context.not_found if answer.nil?
+    hiera_vault_client.shutdown
+
+    return answer
+  end
+
+  # Stringify key:values so user sees expected results and nested objects
+  def stringify_keys(value)
+    case value
+    when String
+      value
+    when Hash
+      result = {}
+      value.each_pair { |k, v| result[k.to_s] = stringify_keys v }
+      result
+    when Array
+      value.map { |v| stringify_keys v }
+    else
+      value
+    end
+  end
+
+  def interpolate(context, paths)
+    allowed_paths = []
+    paths.each do |path|
+      path = context.interpolate(path)
+      # TODO: Unify usage of '/' - File.join seems to be a mistake, since it won't work on Windows
+      # secret/puppet/scope1,scope2 => [[secret], [puppet], [scope1, scope2]]
+      segments = path.split('/').map { |segment| segment.split(',') }
+      allowed_paths += build_paths(segments) unless segments.empty?
+    end
+    allowed_paths
+  end
+
+  # [[secret], [puppet], [scope1, scope2]] => ['secret/puppet/scope1', 'secret/puppet/scope2']
+  def build_paths(segments)
+    paths = [[]]
+    segments.each do |segment|
+      p = paths.dup
+      paths.clear
+      segment.each do |option|
+        p.each do |path|
+          paths << path + [option]
+        end
+      end
+    end
+    paths.map { |p| File.join(*p) }
+  end
+end

lib/puppet/functions/hiera_vault/authentication.rb

@@ -0,0 +1,25 @@
+def authenticate(options, client, context)
+  auth_methods = {
+    'token'           => method(:token),
+    'tls_certificate' => method(:tls)
+  }
+
+  auth_methods[options['method']].(options['config'], client, context)
+end
+
+def token(config, client, context)
+  context.explain { "[hiera-vault] Starting token authentication with config: #{config}" }
+  client.auth.token(config['token'])
+end
+
+def tls(config, client, context)
+  privatekeydir = Puppet.settings['privatekeydir']
+  certdir       = Puppet.settings['certdir']
+  certname      = config['certname']
+
+  privatekey    = File.read("#{privatekeydir}/#{certname}.pem")
+  certificate   = File.read("#{certdir}/#{certname}.pem")
+
+  context.explain { "[hiera-vault] Starting tls_certificate authentication with config: #{config}" }
+  client.auth.tls(privatekey + certificate)
+end

manifests/apt/repositories.pp

@@ -128,6 +128,13 @@
     repos        => 'puppet'
   }
 
+  @apt::source { 'hashicorp':
+    location     => "https://apt-mirror.publiq.be/hashicorp-${codename}-${environment}",
+    release      => $codename,
+    architecture => $arch,
+    repos        => 'main'
+  }
+
   # Project repositories
   @apt::source { 'uit-mail-subscriptions':
     location => "https://apt.publiq.be/uit-mail-subscriptions-${environment}",

manifests/files.pp

@@ -17,16 +17,39 @@
 
   @file { '/data/backup':
     ensure  => 'directory',
-    owner  => 'root',
-    group  => 'root',
-    mode   => '0755',
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0755',
     require => File['/data']
   }
 
   @file { '/etc/gcloud':
-    ensure  => 'directory',
+    ensure => 'directory',
+    owner  => 'root',
+    group  => 'root',
+    mode   => '0755'
+  }
+
+  @file { '/etc/puppetlabs':
+    ensure => 'directory',
     owner  => 'root',
     group  => 'root',
     mode   => '0755'
   }
+
+  @file { '/etc/puppetlabs/facter':
+    ensure => 'directory',
+    owner  => 'root',
+    group  => 'root',
+    mode   => '0755',
+    require => File['/etc/puppetlabs']
+  }
+
+  @file { '/etc/puppetlabs/facter/facts.d':
+    ensure  => 'directory',
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0755',
+    require => File['/etc/puppetlabs/facter']
+  }
 }

manifests/firewall/rules.pp

@@ -66,6 +66,12 @@
     action => 'accept'
   }
 
+  @firewall { '400 accept vault traffic':
+    proto  => 'tcp',
+    dport  => '8200',
+    action => 'accept'
+  }
+
   @firewall { '500 accept carbon traffic':
     proto  => 'tcp',
     dport  => '2003',

manifests/groups.pp

@@ -45,6 +45,11 @@
     gid    => '457'
   }
 
+  @group { 'vault':
+    ensure => 'present',
+    gid    => '458'
+  }
+
   @group { 'ubuntu':
     ensure => 'present',
     gid    => '1000'

manifests/puppet/agent.pp

@@ -5,15 +5,19 @@
 ) inherits ::profiles {
 
   $default_ini_setting_attributes = {
-                                      path    => '/etc/puppetlabs/puppet/puppet.conf',
-                                      notify  => Service['puppet']
+                                      path   => '/etc/puppetlabs/puppet/puppet.conf',
+                                      notify => Service['puppet']
                                     }
 
   realize Apt::Source['puppet']
 
+  realize File['/etc/puppetlabs']
+  realize File['/etc/puppetlabs/facter']
+  realize File['/etc/puppetlabs/facter/facts.d']
+
   package { 'puppet-agent':
     ensure  => $version,
-    require => Apt::Source['puppet'],
+    require => [Apt::Source['puppet'], File['/etc/puppetlabs/facter/facts.d']],
     notify  => Service['puppet']
   }
 
@@ -30,18 +34,6 @@
     require => Package['puppet-agent']
   }
 
-  file { 'puppet agent facter datadir':
-    ensure  => 'directory',
-    path    => '/etc/puppetlabs/facter',
-    require => Package['puppet-agent']
-  }
-
-  file { 'puppet agent facts.d datadir':
-    ensure  => 'directory',
-    path    => '/etc/puppetlabs/facter/facts.d',
-    require => File['puppet agent facter datadir']
-  }
-
   if $puppetserver {
     ini_setting { 'puppetserver':
       ensure  => 'present',

manifests/puppet/puppetserver.pp

@@ -10,6 +10,9 @@
                                                                        { 'name' => 'Per-node data', 'path' => 'nodes/%{::trusted.certname}.yaml' },
                                                                        { 'name' => 'Common data', 'path' => 'common.yaml' }
                                                                      ],
+  Boolean                                  $vault_integration      = false,
+  Optional[String]                         $vault_address          = undef,
+  Hash                                     $vault_mounts           = {},
   Boolean                                  $terraform_integration  = lookup('data::puppet::terraform_integration', Boolean, 'first', false),
   Optional[String]                         $terraform_bucket       = undef,
   Boolean                                  $terraform_use_iam_role = true,
@@ -101,10 +104,18 @@
     gpg_key               => $eyaml_gpg_key,
     lookup_hierarchy      => $lookup_hierarchy,
     terraform_integration => $terraform_integration,
+    vault_integration     => $vault_integration,
+    vault_address         => $vault_address,
+    vault_mounts          => $vault_mounts,
     require               => Class['profiles::puppet::puppetserver::install'],
     notify                => Class['profiles::puppet::puppetserver::service']
   }
 
+  class { 'profiles::puppet::puppetserver::vault':
+    before => Class['profiles::puppet::puppetserver::hiera'],
+    notify => Class['profiles::puppet::puppetserver::service']
+  }
+
   if $terraform_integration {
     class { 'profiles::puppet::puppetserver::terraform':
       bucket       => $terraform_bucket,