Skip to content

Commit d9fdfaa

Browse files
mmatthiesencscmmatthiesencsc
authored
Merge pull request #15 from CSCfi/feature/k8s_plan
Implementing K8s plan for HPCS Server side
2 parents 53183aa + d39d585 commit d9fdfaa

18 files changed

+729
-10
lines changed

‎k8s/deploy-all.yaml

+246
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,246 @@
1+
- hosts: localhost
2+
vars:
3+
hpcs_server_policy: |
4+
path "auth/jwt/role/*" {
5+
capabilities = ["sudo","read","create","delete","update"]
6+
}
7+
path "sys/policies/acl/*" {
8+
capabilities = ["sudo","read","create","delete","update"]
9+
}
10+
11+
tasks:
12+
- name: create hpcs namespace
13+
k8s:
14+
state: present
15+
src: hpcs-namespace.yaml
16+
17+
- name: create spire-server account
18+
k8s:
19+
state: present
20+
src: spire-server-account.yaml
21+
22+
- name: create spire-server clusterrole
23+
k8s:
24+
state: present
25+
src: spire-server-cluster-role.yaml
26+
27+
- name: create spire-server configmap
28+
k8s:
29+
state: present
30+
src: spire-server-configmap.yaml
31+
32+
- name: create spire-oidc configmap
33+
k8s:
34+
state: present
35+
src: spire-oidc-configmap.yaml
36+
37+
- name: create spire nginx proxy configmap
38+
k8s:
39+
state: present
40+
src: spire-server-nginx-configmap.yaml
41+
42+
- name: Create spire-oidc private key
43+
openssl_privatekey:
44+
path: /etc/certs/hpcs-spire-oidc/selfsigned.key
45+
size: 4096
46+
47+
- name: Create spire-oidc csr
48+
openssl_csr:
49+
path: /etc/certs/hpcs-spire-oidc/selfsigned.csr
50+
privatekey_path: /etc/certs/hpcs-spire-oidc/selfsigned.key
51+
52+
- name: Create spire-oidc certificate
53+
openssl_certificate:
54+
provider: selfsigned
55+
path: /etc/certs/hpcs-spire-oidc/selfsigned.crt
56+
privatekey_path: /etc/certs/hpcs-spire-oidc/selfsigned.key
57+
csr_path: /etc/certs/hpcs-spire-oidc/selfsigned.csr
58+
59+
- name: create spire-server pod (spire-server, spire-oidc, hpcs-nginx)
60+
k8s:
61+
state: present
62+
src: spire-server-statefulset.yaml
63+
64+
- name: create spire-server service (expose spire server port)
65+
k8s:
66+
state: present
67+
src: spire-server-service.yaml
68+
69+
- name: create spire-server service (expose spire oidc port)
70+
k8s:
71+
state: present
72+
src: spire-oidc-service.yaml
73+
74+
- name: Add hashicorp to helm repositories
75+
kubernetes.core.helm_repository:
76+
name: stable
77+
repo_url: "https://helm.releases.hashicorp.com"
78+
79+
- name: Deploy hashicorp vault
80+
kubernetes.core.helm:
81+
release_name: vault
82+
chart_ref: hashicorp/vault
83+
release_namespace: hpcs
84+
chart_version: 0.27.0
85+
86+
- name: Wait for vault to be created
87+
shell: "kubectl get po -n hpcs vault-0 --output=jsonpath='{.status}'"
88+
register: pod_ready_for_init
89+
until: (pod_ready_for_init.stdout | from_json)['containerStatuses'] is defined
90+
retries: 10
91+
delay: 2
92+
93+
- name: Initialize vault
94+
kubernetes.core.k8s_exec:
95+
namespace: hpcs
96+
pod: vault-0
97+
command: vault operator init -n 1 -t 1 -format json
98+
register: vault_init
99+
ignore_errors: True
100+
101+
- name: Showing tokens
102+
ansible.builtin.debug:
103+
msg:
104+
- "Please note the unseal token : {{ (vault_init.stdout | from_json)['unseal_keys_b64'][0] }}"
105+
- "Please note the root-token : '{{ (vault_init.stdout | from_json)['root_token' ] }}'"
106+
when: vault_init.rc == 0
107+
108+
- name: Unseal vault
109+
kubernetes.core.k8s_exec:
110+
namespace: hpcs
111+
pod: vault-0
112+
command: vault operator unseal {{ (vault_init.stdout | from_json)['unseal_keys_b64'][0] }}
113+
when: vault_init.rc == 0
114+
ignore_errors: True
115+
116+
- name: Enable jwt authentication in vault
117+
kubernetes.core.k8s_exec:
118+
namespace: hpcs
119+
pod: vault-0
120+
command: sh -c "export VAULT_TOKEN={{ (vault_init.stdout | from_json)['root_token' ] }} ; vault auth enable jwt"
121+
when: vault_init.rc == 0
122+
123+
- name: Enable kv secrets in vault
124+
kubernetes.core.k8s_exec:
125+
namespace: hpcs
126+
pod: vault-0
127+
command: sh -c "export VAULT_TOKEN={{ (vault_init.stdout | from_json)['root_token' ] }} ; vault secrets enable -version=2 kv"
128+
when: vault_init.rc == 0
129+
130+
- name: Create hpcs-server vault policy file
131+
copy:
132+
content: "{{ hpcs_server_policy }}"
133+
dest: /tmp/policy
134+
when: vault_init.rc == 0
135+
136+
- name: Copy oidc cert to vault's pod
137+
kubernetes.core.k8s_cp:
138+
namespace: hpcs
139+
pod: vault-0
140+
remote_path: /tmp/cert
141+
local_path: /etc/certs/hpcs-spire-oidc/selfsigned.crt
142+
when: vault_init.rc == 0
143+
144+
- name: Write oidc config to vault
145+
kubernetes.core.k8s_exec:
146+
namespace: hpcs
147+
pod: vault-0
148+
command: sh -c "export VAULT_TOKEN={{ (vault_init.stdout | from_json)['root_token'] }} ; vault write auth/jwt/config oidc_discovery_url=https://spire-oidc oidc_discovery_ca_pem=\"$(cat /tmp/cert)\""
149+
when: vault_init.rc == 0
150+
151+
- name: Copy policy file to vault's pod
152+
kubernetes.core.k8s_cp:
153+
namespace: hpcs
154+
pod: vault-0
155+
remote_path: /tmp/policy
156+
local_path: /tmp/policy
157+
when: vault_init.rc == 0
158+
159+
- name: Write hpcs-server vault policy
160+
kubernetes.core.k8s_exec:
161+
namespace: hpcs
162+
pod: vault-0
163+
command: sh -c "export VAULT_TOKEN={{ (vault_init.stdout | from_json)['root_token'] }} ; vault policy write hpcs-server /tmp/policy"
164+
when: vault_init.rc == 0
165+
166+
- name: Write hpcs-server vault role
167+
kubernetes.core.k8s_exec:
168+
namespace: hpcs
169+
pod: vault-0
170+
command: sh -c "export VAULT_TOKEN={{ (vault_init.stdout | from_json)['root_token'] }} ; vault write auth/jwt/role/hpcs-server role_type=jwt user_claim=sub bound_audiences=TESTING bound_subject=spiffe://hpcs/hpcs-server/workload token_ttl=24h token_policies=hpcs-server"
171+
when: vault_init.rc == 0
172+
173+
- name: Check cgroups version
174+
kubernetes.core.k8s_exec:
175+
namespace: hpcs
176+
pod: vault-0
177+
command: sh -c "cat /proc/filesystems | grep cgroup2"
178+
register: cgroups_check
179+
180+
- name: Register node uid and nodename
181+
shell: "kubectl get nodes -o json"
182+
register: kubectl_node_info
183+
184+
- name: Register hpcs-server identity
185+
kubernetes.core.k8s_exec:
186+
namespace: hpcs
187+
pod: spire-server-0
188+
container: spire-server
189+
command: ./bin/spire-server entry create -parentID spiffe://hpcs/spire/agent/k8s_psat/{{ (kubectl_node_info.stdout | from_json)['items'][0]['metadata']['name'] }}/{{ (kubectl_node_info.stdout | from_json)['items'][0]['metadata']['uid'] }} -spiffeID spiffe://hpcs/hpcs-server/workload -selector unix:uid:0
190+
register: cgroups_check
191+
when: cgroups_check.rc == 0
192+
ignore_errors: True
193+
194+
- name: Register hpcs-server identity
195+
kubernetes.core.k8s_exec:
196+
namespace: hpcs
197+
pod: spire-server-0
198+
container: spire-server
199+
command: ./bin/spire-server entry create -parentID spiffe://hpcs/spire/agent/k8s_psat/{{ (kubectl_node_info.stdout | from_json)['items'][0]['metadata']['name'] }}/{{ (kubectl_node_info.stdout | from_json)['items'][0]['metadata']['uid'] }} -spiffeID spiffe://hpcs/hpcs-server/workload -selector k8s:pod-name:hpcs-server
200+
register: cgroups_check
201+
when: cgroups_check.rc == 1
202+
ignore_errors: True
203+
204+
- name: Expose vault's web port
205+
kubernetes.core.k8s_service:
206+
state: present
207+
name: vault-external
208+
type: NodePort
209+
namespace: hpcs
210+
ports:
211+
- port: 8200
212+
protocol: TCP
213+
selector:
214+
service: vault
215+
216+
- name: Create hpcs-server account
217+
k8s:
218+
state: present
219+
src: hpcs-server-account.yaml
220+
221+
- name: Create hpcs-spire account
222+
k8s:
223+
state: present
224+
src: hpcs-spire-account.yaml
225+
226+
- name: Create hpcs-server configmap
227+
k8s:
228+
state: present
229+
src: hpcs-server-configmap.yaml
230+
231+
- name: Create hpcs-server statefulset and pod
232+
k8s:
233+
state: present
234+
src: hpcs-server-statefulset.yaml
235+
236+
- name: Expose hpcs-server's web port
237+
kubernetes.core.k8s_service:
238+
state: present
239+
name: hpcs-external
240+
type: NodePort
241+
namespace: hpcs
242+
ports:
243+
- port: 10080
244+
protocol: TCP
245+
selector:
246+
service: hpcs-server

‎k8s/hpcs-namespace.yaml

+4
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
apiVersion: v1
2+
kind: Namespace
3+
metadata:
4+
name: hpcs

‎k8s/hpcs-server-account.yaml

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
apiVersion: v1
2+
kind: ServiceAccount
3+
metadata:
4+
name: hpcs-server
5+
namespace: hpcs

‎k8s/hpcs-server-configmap.yaml

+61
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
apiVersion: v1
2+
kind: ConfigMap
3+
metadata:
4+
name: hpcs-server
5+
namespace: hpcs
6+
data:
7+
hpcs-server.conf: |
8+
[spire-server]
9+
address = localhost
10+
port = 8081
11+
trust-domain = hpcs
12+
pre-command = ""
13+
spire-server-bin = spire-server
14+
socket-path = /var/run/sockets/server/api.sock
15+
16+
[spire-agent]
17+
spire-agent-socket = /run/sockets/agent/agent.sock
18+
19+
[vault]
20+
url = http://vault:8200
21+
server-role = hpcs-server
22+
23+
agent.conf: |
24+
agent {
25+
data_dir = "./data/agent"
26+
log_level = "DEBUG"
27+
trust_domain = "hpcs"
28+
server_address = "spire-server"
29+
server_port = 8081
30+
socket_path = "/var/run/sockets/agent/agent.sock"
31+
admin_socket_path = "/var/run/sockets/admin/admin.sock"
32+
33+
# Insecure bootstrap is NOT appropriate for production use but is ok for
34+
# simple testing/evaluation purposes.
35+
insecure_bootstrap = true
36+
}
37+
38+
plugins {
39+
KeyManager "disk" {
40+
plugin_data {
41+
directory = "./data/agent"
42+
}
43+
}
44+
45+
NodeAttestor "k8s_psat" {
46+
plugin_data {
47+
cluster = "docker-desktop"
48+
}
49+
}
50+
51+
WorkloadAttestor "k8s" {
52+
plugin_data {
53+
}
54+
}
55+
56+
WorkloadAttestor "unix" {
57+
plugin_data {
58+
discover_workload_path = true
59+
}
60+
}
61+
}

‎k8s/hpcs-server-service.yaml

+14
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
# Service definition for spire-oidc (expose the OIDC socket)
2+
apiVersion: v1
3+
kind: Service
4+
metadata:
5+
name: hpcs-server
6+
namespace: hpcs
7+
spec:
8+
clusterIP: None
9+
selector:
10+
app: hpcs-server
11+
ports:
12+
- name: https
13+
port: 10080
14+
targetPort: hpcs-server

‎k8s/hpcs-server-statefulset.yaml

+62
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
apiVersion: apps/v1
2+
kind: StatefulSet
3+
metadata:
4+
name: hpcs-server
5+
namespace: hpcs
6+
labels:
7+
app: hpcs-server
8+
spec:
9+
replicas: 1
10+
selector:
11+
matchLabels:
12+
app: hpcs-server
13+
serviceName: hpcs-server
14+
template:
15+
metadata:
16+
namespace: hpcs
17+
labels:
18+
app: hpcs-server
19+
spec:
20+
serviceAccountName: hpcs-server
21+
shareProcessNamespace: true
22+
containers:
23+
- name: hpcs-server
24+
image: ghcr.io/cscfi/hpcs/server:0.1.1
25+
ports:
26+
- containerPort: 10080
27+
name: hpcs-server
28+
volumeMounts:
29+
- name: hpcs-server-configs
30+
mountPath: /tmp/
31+
readOnly: false
32+
- name: hpcs-spire-sockets
33+
mountPath: /var/run/sockets
34+
readOnly: false
35+
- name: hpcs-spire-agent-token
36+
mountPath: /var/run/secrets/tokens
37+
readOnly: true
38+
volumes:
39+
- name: hpcs-server-configs
40+
configMap:
41+
name: hpcs-server
42+
- name: hpcs-spire-sockets
43+
hostPath:
44+
path: /run/spire/sockets
45+
type: DirectoryOrCreate
46+
- name: hpcs-spire-agent-token
47+
projected:
48+
sources:
49+
- serviceAccountToken:
50+
path: spire-agent
51+
expirationSeconds: 7200
52+
audience: spire-server
53+
volumeClaimTemplates:
54+
- metadata:
55+
name: spire-agent-data
56+
namespace: hpcs
57+
spec:
58+
accessModes:
59+
- ReadWriteOnce
60+
resources:
61+
requests:
62+
storage: 1Gi

0 commit comments

Comments
 (0)