Skip to content

Commit d536f76

Browse files
authored
Merge pull request #491 from cultuurnet/feature/OPS-1248
Hashicorp vault implementation and puppetserver integration
2 parents 6e99305 + a242771 commit d536f76

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

55 files changed

+2717
-241
lines changed
Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
#!/bin/bash
2+
3+
KEY_OWNERS=${1}
4+
INPUT=$(/usr/bin/cat)
5+
6+
while IFS=',' read -ra OWNER; do
7+
for counter in `/usr/bin/seq 0 $(( ${#OWNER[@]} - 1 ))`; do
8+
owner="${OWNER[${counter}]}"
9+
key=$(/usr/bin/echo ${INPUT} | /usr/bin/jq -r ".unseal_keys_b64[${counter}]")
10+
11+
/usr/bin/echo $(/usr/bin/jq -n --arg key "$owner" --arg value "$key" '{ ($key): ($value) }')
12+
done | /usr/bin/jq -s add
13+
done <<< "${KEY_OWNERS}" | /usr/bin/jq '{ "vault_encrypted_unseal_keys": . }'

files/vault/vault-unseal

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
#!/bin/bash
2+
3+
encrypted_unseal_key=$(/usr/bin/cat ${1})
4+
5+
/usr/bin/vault operator unseal -tls-skip-verify $(/usr/bin/echo ${encrypted_unseal_key} | /usr/bin/base64 --decode | /usr/bin/gpg --decrypt --quiet)

files/vault/vaultbackup

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
#!/bin/bash
2+
3+
mv /data/backup/vault/current/*.tar.gz /data/backup/vault/archive
4+
5+
tar cvzf /data/backup/vault/current/vault_$(date +%Y.%m.%d.%H%M%S).tar.gz \
6+
/opt/vault \
7+
/home/vault/.vault-token \
8+
/home/vault/encrypted_unseal_key \
Lines changed: 156 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,156 @@
1+
# Originally copied from Peter Souter hiera_vault repository, licensed under the Apache License, Version 2.0
2+
#
3+
# https://github.com/petems/petems-hiera_vault/blob/master/lib/puppet/functions/hiera_vault.rb
4+
5+
require_relative 'hiera_vault/authentication.rb'
6+
7+
Puppet::Functions.create_function(:hiera_vault) do
8+
begin
9+
require 'vault'
10+
rescue LoadError => _e
11+
raise Puppet::DataBinding::LookupError, "[hiera-vault] Must install vault gem to use hiera-vault backend"
12+
end
13+
14+
dispatch :lookup_key do
15+
param 'Variant[String, Numeric]', :key
16+
param 'Hash', :options
17+
param 'Puppet::LookupContext', :context
18+
end
19+
20+
def lookup_key(key, options, context)
21+
if confine_keys = options['confine_to_keys']
22+
raise ArgumentError, '[hiera-vault] confine_to_keys must be an array' unless confine_keys.is_a?(Array)
23+
24+
begin
25+
confine_keys = confine_keys.map { |r| Regexp.new(r) }
26+
rescue StandardError => e
27+
raise Puppet::DataBinding::LookupError, "[hiera-vault] creating regexp failed with: #{e}"
28+
end
29+
30+
regex_key_match = Regexp.union(confine_keys)
31+
32+
unless key[regex_key_match] == key
33+
context.explain { "[hiera-vault] Skipping hiera_vault backend because key '#{key}' does not match confine_to_keys" }
34+
context.not_found
35+
end
36+
end
37+
38+
if strip_from_keys = options['strip_from_keys']
39+
raise ArgumentError, '[hiera-vault] strip_from_keys must be an array' unless strip_from_keys.is_a?(Array)
40+
41+
strip_from_keys.each do |prefix|
42+
key = key.gsub(Regexp.new(prefix), '')
43+
end
44+
end
45+
46+
vault_get(key, options, context)
47+
end
48+
49+
def vault_get(key, options, context)
50+
hiera_vault_client = Vault::Client.new
51+
52+
begin
53+
hiera_vault_client.configure do |config|
54+
config.address = options['address'] unless options['address'].nil?
55+
config.ssl_verify = options['ssl_verify'] unless options['ssl_verify'].nil?
56+
config.ssl_ca_cert = options['ssl_ca_cert'] if config.respond_to? :ssl_ca_cert
57+
end
58+
59+
context.explain { "[hiera-vault] Using #{options['authentication']['method']} authentication" }
60+
authenticate(options['authentication'], hiera_vault_client, context)
61+
62+
if hiera_vault_client.sys.seal_status.sealed?
63+
raise Puppet::DataBinding::LookupError, "[hiera-vault] vault is sealed"
64+
end
65+
66+
context.explain { "[hiera-vault] Client configured to connect to #{hiera_vault_client.address}" }
67+
rescue StandardError => e
68+
hiera_vault_client.shutdown
69+
raise Puppet::DataBinding::LookupError, "[hiera-vault] Skipping backend. Configuration error: #{e}"
70+
end
71+
72+
answer = nil
73+
74+
# Only kv v2 mounts supported
75+
options['mounts'].each_pair do |mount, paths|
76+
interpolate(context, paths).each do |path|
77+
context.explain { "[hiera-vault] Looking on mount #{mount} in path #{path} for #{key}" }
78+
79+
secret = nil
80+
81+
begin
82+
context.explain { "[hiera-vault] Checking path: #{path} on mount: #{mount}" }
83+
secret = hiera_vault_client.kv(mount).read(File.join(path, key).chomp('/'))
84+
rescue Vault::HTTPConnectionError
85+
msg = "[hiera-vault] Could not connect to read secret: #{path} on mount: #{mount}"
86+
context.explain { msg }
87+
raise Puppet::DataBinding::LookupError, msg
88+
rescue Vault::HTTPError => e
89+
msg = "[hiera-vault] Could not read secret #{path} on mount #{mount}: #{e.errors.join("\n").rstrip}"
90+
context.explain { msg }
91+
raise Puppet::DataBinding::LookupError, msg
92+
end
93+
94+
next if secret.nil?
95+
96+
context.explain { "[hiera-vault] Read secret: #{key}" }
97+
# Turn secret's hash keys into strings allow for nested arrays and hashes
98+
# this enables support for create resources etc
99+
answer = secret.data.inject({}) { |h, (k, v)| h[k.to_s] = stringify_keys v; h }
100+
break
101+
end
102+
103+
break unless answer.nil?
104+
end
105+
106+
raise Puppet::DataBinding::LookupError, "[hiera-vault] Could not find secret #{key}" if answer.nil?
107+
108+
answer = context.not_found if answer.nil?
109+
hiera_vault_client.shutdown
110+
111+
return answer
112+
end
113+
114+
# Stringify key:values so user sees expected results and nested objects
115+
def stringify_keys(value)
116+
case value
117+
when String
118+
value
119+
when Hash
120+
result = {}
121+
value.each_pair { |k, v| result[k.to_s] = stringify_keys v }
122+
result
123+
when Array
124+
value.map { |v| stringify_keys v }
125+
else
126+
value
127+
end
128+
end
129+
130+
def interpolate(context, paths)
131+
allowed_paths = []
132+
paths.each do |path|
133+
path = context.interpolate(path)
134+
# TODO: Unify usage of '/' - File.join seems to be a mistake, since it won't work on Windows
135+
# secret/puppet/scope1,scope2 => [[secret], [puppet], [scope1, scope2]]
136+
segments = path.split('/').map { |segment| segment.split(',') }
137+
allowed_paths += build_paths(segments) unless segments.empty?
138+
end
139+
allowed_paths
140+
end
141+
142+
# [[secret], [puppet], [scope1, scope2]] => ['secret/puppet/scope1', 'secret/puppet/scope2']
143+
def build_paths(segments)
144+
paths = [[]]
145+
segments.each do |segment|
146+
p = paths.dup
147+
paths.clear
148+
segment.each do |option|
149+
p.each do |path|
150+
paths << path + [option]
151+
end
152+
end
153+
end
154+
paths.map { |p| File.join(*p) }
155+
end
156+
end
Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
def authenticate(options, client, context)
2+
auth_methods = {
3+
'token' => method(:token),
4+
'tls_certificate' => method(:tls)
5+
}
6+
7+
auth_methods[options['method']].(options['config'], client, context)
8+
end
9+
10+
def token(config, client, context)
11+
context.explain { "[hiera-vault] Starting token authentication with config: #{config}" }
12+
client.auth.token(config['token'])
13+
end
14+
15+
def tls(config, client, context)
16+
privatekeydir = Puppet.settings['privatekeydir']
17+
certdir = Puppet.settings['certdir']
18+
certname = config['certname']
19+
20+
privatekey = File.read("#{privatekeydir}/#{certname}.pem")
21+
certificate = File.read("#{certdir}/#{certname}.pem")
22+
23+
context.explain { "[hiera-vault] Starting tls_certificate authentication with config: #{config}" }
24+
client.auth.tls(privatekey + certificate)
25+
end

manifests/apt/repositories.pp

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -128,6 +128,13 @@
128128
repos => 'puppet'
129129
}
130130

131+
@apt::source { 'hashicorp':
132+
location => "https://apt-mirror.publiq.be/hashicorp-${codename}-${environment}",
133+
release => $codename,
134+
architecture => $arch,
135+
repos => 'main'
136+
}
137+
131138
# Project repositories
132139
@apt::source { 'uit-mail-subscriptions':
133140
location => "https://apt.publiq.be/uit-mail-subscriptions-${environment}",

manifests/files.pp

Lines changed: 27 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -17,16 +17,39 @@
1717

1818
@file { '/data/backup':
1919
ensure => 'directory',
20-
owner => 'root',
21-
group => 'root',
22-
mode => '0755',
20+
owner => 'root',
21+
group => 'root',
22+
mode => '0755',
2323
require => File['/data']
2424
}
2525

2626
@file { '/etc/gcloud':
27-
ensure => 'directory',
27+
ensure => 'directory',
28+
owner => 'root',
29+
group => 'root',
30+
mode => '0755'
31+
}
32+
33+
@file { '/etc/puppetlabs':
34+
ensure => 'directory',
2835
owner => 'root',
2936
group => 'root',
3037
mode => '0755'
3138
}
39+
40+
@file { '/etc/puppetlabs/facter':
41+
ensure => 'directory',
42+
owner => 'root',
43+
group => 'root',
44+
mode => '0755',
45+
require => File['/etc/puppetlabs']
46+
}
47+
48+
@file { '/etc/puppetlabs/facter/facts.d':
49+
ensure => 'directory',
50+
owner => 'root',
51+
group => 'root',
52+
mode => '0755',
53+
require => File['/etc/puppetlabs/facter']
54+
}
3255
}

manifests/firewall/rules.pp

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,12 @@
6666
action => 'accept'
6767
}
6868

69+
@firewall { '400 accept vault traffic':
70+
proto => 'tcp',
71+
dport => '8200',
72+
action => 'accept'
73+
}
74+
6975
@firewall { '500 accept carbon traffic':
7076
proto => 'tcp',
7177
dport => '2003',

manifests/groups.pp

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,11 @@
4545
gid => '457'
4646
}
4747

48+
@group { 'vault':
49+
ensure => 'present',
50+
gid => '458'
51+
}
52+
4853
@group { 'ubuntu':
4954
ensure => 'present',
5055
gid => '1000'

manifests/puppet/agent.pp

Lines changed: 7 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -5,15 +5,19 @@
55
) inherits ::profiles {
66

77
$default_ini_setting_attributes = {
8-
path => '/etc/puppetlabs/puppet/puppet.conf',
9-
notify => Service['puppet']
8+
path => '/etc/puppetlabs/puppet/puppet.conf',
9+
notify => Service['puppet']
1010
}
1111

1212
realize Apt::Source['puppet']
1313

14+
realize File['/etc/puppetlabs']
15+
realize File['/etc/puppetlabs/facter']
16+
realize File['/etc/puppetlabs/facter/facts.d']
17+
1418
package { 'puppet-agent':
1519
ensure => $version,
16-
require => Apt::Source['puppet'],
20+
require => [Apt::Source['puppet'], File['/etc/puppetlabs/facter/facts.d']],
1721
notify => Service['puppet']
1822
}
1923

@@ -30,18 +34,6 @@
3034
require => Package['puppet-agent']
3135
}
3236

33-
file { 'puppet agent facter datadir':
34-
ensure => 'directory',
35-
path => '/etc/puppetlabs/facter',
36-
require => Package['puppet-agent']
37-
}
38-
39-
file { 'puppet agent facts.d datadir':
40-
ensure => 'directory',
41-
path => '/etc/puppetlabs/facter/facts.d',
42-
require => File['puppet agent facter datadir']
43-
}
44-
4537
if $puppetserver {
4638
ini_setting { 'puppetserver':
4739
ensure => 'present',

0 commit comments

Comments
 (0)