Skip to content

Commit 2dd3408

Browse files
sbauzasbauza
committed
[Draft] Add nvidia-mdev VA
1 parent 932b0d9 commit 2dd3408

20 files changed

+740
-0
lines changed

‎automation/vars/nvidia-mdev.yaml

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
---
2+
vas:
3+
nvidia-mdev:
4+
stages:
5+
- path: examples/va/nvidia-mdev/nncp
6+
wait_conditions:
7+
- >-
8+
oc -n openstack wait nncp
9+
-l osp/nncm-config-type=standard
10+
--for jsonpath='{.status.conditions[0].reason}'=SuccessfullyConfigured
11+
--timeout=60s
12+
values:
13+
- name: network-values
14+
src_file: values.yaml
15+
build_output: nncp.yaml
16+
17+
- path: examples/va/nvidia-mdev
18+
wait_conditions:
19+
- >-
20+
oc -n openstack wait osctlplane controlplane --for condition=Ready
21+
--timeout=600s
22+
values:
23+
- name: network-values
24+
src_file: nncp/values.yaml
25+
- name: service-values
26+
src_file: service-values.yaml
27+
build_output: control-plane.yaml
28+
29+
- path: examples/va/nvidia-mdev/edpm/nodeset
30+
wait_conditions:
31+
- >-
32+
oc -n openstack wait
33+
osdpns openstack-edpm --for condition=SetupReady
34+
--timeout=60m
35+
values:
36+
- name: edpm-nodeset-values
37+
src_file: values.yaml
38+
build_output: nodeset.yaml
39+
post_stage_run:
40+
- name: Install nvidia driver
41+
type: playbook
42+
# TODO: I haven't passed the secret URL but I don't know how to do it.
43+
source: "../../playbooks/nvidia-mdev.yml"
44+
inventory: "${HOME}/ci-framework-data/artifacts/zuul_inventory.yml"
45+
46+
- path: examples/va/nvidia-mdev/edpm/deployment
47+
wait_conditions:
48+
- >-
49+
oc -n openstack wait
50+
osdpns openstack-edpm --for condition=Ready
51+
--timeout=60m
52+
values:
53+
- name: edpm-deployment-values
54+
src_file: values.yaml
55+
build_output: deployment.yaml

‎examples/va/nvidia-mdev/.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
control-plane.yaml

‎examples/va/nvidia-mdev/edpm/.gitignore

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
dataplane-deployment.yaml
2+
dataplane-nodeset.yaml

‎examples/va/nvidia-mdev/edpm/deployment/.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
dataplane-deployment.yaml

‎examples/va/nvidia-mdev/edpm/deployment/kustomization.yaml

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
---
2+
apiVersion: kustomize.config.k8s.io/v1beta1
3+
kind: Kustomization
4+
5+
components:
6+
- ../../../../../va/nvidia-mdev/edpm/deployment
7+
# - https://github.com/openstack-k8s-operators/architecture/va/nvidia-mdev/edpm/deployment?ref=main
8+
## It's possible to replace ../../../../../va/nvidia-mdev/edpm/deployment/ with a git checkout URL as per:
9+
## https://github.com/kubernetes-sigs/kustomize/blob/master/examples/remoteBuild.md
10+
11+
resources:
12+
- values.yaml

‎examples/va/nvidia-mdev/edpm/deployment/values.yaml

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
# yamllint disable rule:line-length
2+
# local-config: referenced, but not emitted by kustomize
3+
---
4+
apiVersion: v1
5+
kind: ConfigMap
6+
metadata:
7+
name: edpm-deployment-values
8+
annotations:
9+
config.kubernetes.io/local-config: "true"
10+
data: {}

‎examples/va/nvidia-mdev/edpm/nodeset/.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
dataplane-nodeset.yaml

‎examples/va/nvidia-mdev/edpm/nodeset/kustomization.yaml

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
---
2+
apiVersion: kustomize.config.k8s.io/v1beta1
3+
kind: Kustomization
4+
5+
components:
6+
- ../../../../../va/nvidia-mdev/edpm/nodeset
7+
# - https://github.com/openstack-k8s-operators/architecture/va/nvidia-mdev/edpm/nodeset?ref=main
8+
## It's possible to replace ../../../../../va/nvidia-mdev/edpm/nodeset/ with a git checkout URL as per:
9+
## https://github.com/kubernetes-sigs/kustomize/blob/master/examples/remoteBuild.md
10+
11+
resources:
12+
- values.yaml

‎examples/va/nvidia-mdev/edpm/nodeset/values.yaml

Lines changed: 160 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,160 @@
1+
# yamllint disable rule:line-length
2+
# local-config: referenced, but not emitted by kustomize
3+
---
4+
apiVersion: v1
5+
kind: ConfigMap
6+
metadata:
7+
name: edpm-nodeset-values
8+
annotations:
9+
config.kubernetes.io/local-config: "true"
10+
data:
11+
root_password: cmVkaGF0Cg==
12+
preProvisioned: false
13+
baremetalSetTemplate:
14+
ctlplaneInterface: eno2 # CHANGEME
15+
cloudUserName: cloud-admin
16+
provisioningInterface: enp1s0 # CHANGEME
17+
bmhLabelSelector:
18+
app: openstack # CHANGEME
19+
passwordSecret:
20+
name: baremetalset-password-secret
21+
namespace: openstack
22+
ssh_keys:
23+
# Authorized keys that will have access to the dataplane computes via SSH
24+
authorized: CHANGEME
25+
# The private key that will have access to the dataplane computes via SSH
26+
private: CHANGEME2
27+
# The public key that will have access to the dataplane computes via SSH
28+
public: CHANGEME3
29+
nodeset:
30+
ansible:
31+
ansibleUser: cloud-admin
32+
ansiblePort: 22
33+
ansibleVars:
34+
# CHANGEME -- see https://access.redhat.com/solutions/253273
35+
# edpm_bootstrap_command: |
36+
# subscription-manager register --username <subscription_manager_username> --password <subscription_manager_password>
37+
# podman login -u <registry_username> -p <registry_password> registry.redhat.io
38+
timesync_ntp_servers:
39+
- hostname: pool.ntp.org
40+
# CPU pinning settings
41+
edpm_kernel_args: "default_hugepagesz=1GB hugepagesz=1G hugepages=16 intel_iommu=on iommu=pt isolcpus=4-23,28-47"
42+
edpm_tuned_profile: "cpu-partitioning-powersave"
43+
edpm_tuned_isolated_cores: "4-23,28-47"
44+
# edpm_network_config
45+
# These vars are edpm_network_config role vars
46+
edpm_network_config_hide_sensitive_logs: false
47+
edpm_network_config_os_net_config_mappings:
48+
edpm-compute-0:
49+
nic2: 6c:fe:54:3f:8a:02 # CHANGEME
50+
nic3: 6c:fe:54:3f:8a:03 # CHANGEME
51+
edpm-compute-1:
52+
nic2: 6b:fe:54:3f:8a:02 # CHANGEME
53+
nic3: 6b:fe:54:3f:8a:03 # CHANGEME
54+
edpm_network_config_template: |
55+
---
56+
{% set mtu_list = [ctlplane_mtu] %}
57+
{% for network in nodeset_networks %}
58+
{{ mtu_list.append(lookup('vars', networks_lower[network] ~ '_mtu')) }}
59+
{%- endfor %}
60+
{% set min_viable_mtu = mtu_list | max %}
61+
network_config:
62+
- type: ovs_bridge
63+
name: {{ neutron_physical_bridge_name }}
64+
mtu: {{ min_viable_mtu }}
65+
use_dhcp: false
66+
dns_servers: {{ ctlplane_dns_nameservers }}
67+
domain: {{ dns_search_domains }}
68+
addresses:
69+
- ip_netmask: {{ ctlplane_ip }}/{{ ctlplane_cidr }}
70+
routes: {{ ctlplane_host_routes }}
71+
members:
72+
- type: interface
73+
name: nic2
74+
mtu: {{ min_viable_mtu }}
75+
# force the MAC address of the bridge to this interface
76+
primary: true
77+
{% for network in nodeset_networks %}
78+
- type: vlan
79+
mtu: {{ lookup('vars', networks_lower[network] ~ '_mtu') }}
80+
vlan_id: {{ lookup('vars', networks_lower[network] ~ '_vlan_id') }}
81+
addresses:
82+
- ip_netmask:
83+
{{ lookup('vars', networks_lower[network] ~ '_ip') }}/{{ lookup('vars', networks_lower[network] ~ '_cidr') }}
84+
routes: {{ lookup('vars', networks_lower[network] ~ '_host_routes') }}
85+
{% endfor %}
86+
- type: sriov_pf
87+
name: nic3
88+
numvfs: 10
89+
use_dhcp: false
90+
promisc: true
91+
92+
# These vars are for the network config templates themselves and are
93+
# considered EDPM network defaults.
94+
neutron_physical_bridge_name: br-ex
95+
neutron_public_interface_name: eth0
96+
# edpm_nodes_validation
97+
edpm_nodes_validation_validate_controllers_icmp: false
98+
edpm_nodes_validation_validate_gateway_icmp: false
99+
dns_search_domains: []
100+
gather_facts: false
101+
# edpm firewall, change the allowed CIDR if needed
102+
edpm_sshd_configure_firewall: true
103+
edpm_sshd_allowed_ranges:
104+
- 192.168.122.0/24
105+
# SRIOV settings
106+
edpm_neutron_sriov_agent_SRIOV_NIC_physical_device_mappings: 'sriov-phy4:eno4'
107+
networks:
108+
- defaultRoute: true
109+
name: ctlplane
110+
subnetName: subnet1
111+
- name: internalapi
112+
subnetName: subnet1
113+
- name: storage
114+
subnetName: subnet1
115+
- name: tenant
116+
subnetName: subnet1
117+
nodes:
118+
edpm-compute-0:
119+
hostName: edpm-compute-0
120+
edpm-compute-1:
121+
hostName: edpm-compute-1
122+
services:
123+
- bootstrap
124+
- download-cache
125+
- configure-network
126+
- validate-network
127+
- install-os
128+
- configure-os
129+
- ssh-known-hosts
130+
- run-os
131+
- reboot-os
132+
- install-certs
133+
- libvirt
134+
- ovn
135+
- neutron-ovn
136+
- nova-custom-sriov
137+
- neutron-sriov
138+
- neutron-metadata
139+
nova:
140+
compute:
141+
conf: |
142+
# CHANGEME
143+
[DEFAULT]
144+
reserved_host_memory_mb = 4096
145+
reserved_huge_pages = node:0,size:4,count:524160
146+
reserved_huge_pages = node:1,size:4,count:524160
147+
[compute]
148+
cpu_shared_set = 0-3,24-27
149+
cpu_dedicated_set = 8-23,32-47
150+
[devices]
151+
mdev_enabled_types = nvidia-268
152+
migration:
153+
ssh_keys:
154+
private: CHANGEME4
155+
public: CHANGEME5
156+
pci:
157+
conf: |
158+
# CHANGEME
159+
[pci]
160+
device_spec = {"vendor_id":"8086", "product_id":"1572", "address": "0000:19:00.3", "physical_network":"sriov-phy4", "trusted":"true"}

‎examples/va/nvidia-mdev/kustomization.yaml

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
---
2+
apiVersion: kustomize.config.k8s.io/v1beta1
3+
kind: Kustomization
4+
5+
components:
6+
- ../../../va/nvidia-mdev/
7+
# - https://github.com/openstack-k8s-operators/architecture/va/nvidia-mdev?ref=main
8+
## It's possible to replace ../../../va/nvidia-mdev/ with a git checkout URL as per:
9+
## https://github.com/kubernetes-sigs/kustomize/blob/master/examples/remoteBuild.md
10+
11+
resources:
12+
- nncp/values.yaml
13+
- service-values.yaml

‎examples/va/nvidia-mdev/nncp/.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
nncp.yaml

‎examples/va/nvidia-mdev/nncp/kustomization.yaml

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
---
2+
apiVersion: kustomize.config.k8s.io/v1beta1
3+
kind: Kustomization
4+
5+
transformers:
6+
# Set namespace to OpenStack on all namespaced objects without a namespace
7+
- |-
8+
apiVersion: builtin
9+
kind: NamespaceTransformer
10+
metadata:
11+
name: _ignored_
12+
namespace: openstack
13+
setRoleBindingSubjects: none
14+
unsetOnly: true
15+
fieldSpecs:
16+
- path: metadata/name
17+
kind: Namespace
18+
create: true
19+
20+
components:
21+
- ../../../../lib/nncp
22+
23+
resources:
24+
- values.yaml

0 commit comments

Comments
 (0)