Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion docs/canonicalk8s/assets/configuration.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,6 @@ cluster-config:
local-storage:
enabled: true
extra-node-kubelet-args:
--reserved-cpus: "0-31"
--reserved-cpus: <reserved CPUs (physical core IDs e.g. 0-2)>
--cpu-manager-policy: "static"
--topology-manager-policy: "best-effort"
83 changes: 15 additions & 68 deletions docs/canonicalk8s/assets/how-to-epa-maas-cloud-init
Original file line number Diff line number Diff line change
@@ -1,10 +1,11 @@
#cloud-config

apt:
sources:
rtk.list:
source: "deb https://<launchpad_id>:<ppa_subscription_password>@private-ppa.launchpadcontent.net/canonical-kernel-rt/ppa/ubuntu jammy main"

package_update: true
package_upgrade: false
packages:
- ubuntu-advantage-tools
- git
- pciutils
write_files:
# set kernel option with hugepages and cpu isolation
- path: /etc/default/grub.d/100-telco_kernel_options.cfg
Expand All @@ -16,9 +17,9 @@ write_files:
- path: /etc/netplan/99-sriov_vfs.yaml
content: |
network:
ethernets:
enp152s0f1:
virtual-function-count: 128
ethernets:
enp152s0f1:
virtual-function-count: 128
permissions: "0600"

# ensure VFs are bound to vfio-pci driver (so they can be consumed by pods)
Expand All @@ -32,68 +33,14 @@ write_files:
fi
permissions: "0755"

# set proxy variables
- path: /etc/environment
content: |
HTTPS_PROXY=http://10.18.2.1:3128
HTTP_PROXY=http://10.18.2.1:3128
NO_PROXY=10.0.0.0/8,192.168.0.0/16,127.0.0.1,172.16.0.0/16,.svc,localhost
https_proxy=http://10.18.2.1:3128
http_proxy=http://10.18.2.1:3128
no_proxy=10.0.0.0/8,192.168.0.0/16,127.0.0.1,172.16.0.0/16,.svc,localhost
append: true

# add rtk ppa key
- path: /etc/apt/trusted.gpg.d/rtk.asc
content: |
-----BEGIN PGP PUBLIC KEY BLOCK-----
Comment: Hostname:
Version: Hockeypuck 2.2

xsFNBGAervwBEADHCeEuR7WKRiEII+uFOu8J+W47MZOcVhfNpu4rdcveL4qe4gj4
nsROMHaINeUPCmv7/4EXdXtTm1VksXeh4xTeqH6ZaQre8YZ9Hf4OYNRcnFOn0KR+
aCk0OWe9xkoDbrSYd3wmx8NG/Eau2C7URzYzYWwdHgZv6elUKk6RDbDh6XzIaChm
kLsErCP1SiYhKQvD3Q0qfXdRG908lycCxgejcJIdYxgxOYFFPcyC+kJy2OynnvQr
4Yw6LJ2LhwsA7bJ5hhQDCYZ4foKCXX9I59G71dO1fFit5O/0/oq0xe7yUYCejf7Z
OqD+TzEK4lxLr1u8j8lXoQyUXzkKIL0SWEFT4tzOFpWQ2IBs/sT4X2oVA18dPDoZ
H2SGxCUcABfne5zrEDgkUkbnQRihBtTyR7QRiE3GpU19RNVs6yAu+wA/hti8Pg9O
U/5hqifQrhJXiuEoSmmgNb9QfbR3tc0ZhKevz4y+J3vcnka6qlrP1lAirOVm2HA7
STGRnaEJcTama85MSIzJ6aCx4omCgUIfDmsi9nAZRkmeomERVlIAvcUYxtqprLfu
6plDs+aeff/MAmHbak7yF+Txj8+8F4k6FcfNBT51oVSZuqFwyLswjGVzWol6aEY7
akVIrn3OdN2u6VWlU4ZO5+sjP4QYsf5K2oVnzFVIpYvqtO2fGbxq/8dRJQARAQAB
zSVMYXVuY2hwYWQgUFBBIGZvciBDYW5vbmljYWwgS2VybmVsIFJUwsGOBBMBCgA4
FiEEc4Tsv+pcopCX6lNfLz1Vl/FsjCEFAmAervwCGwMFCwkIBwIGFQoJCAsCBBYC
AwECHgECF4AACgkQLz1Vl/FsjCF9WhAAnwfx9njs1M3rfsMMuhvPxx0WS65HDlq8
SRgl9K2EHtZIcS7lHmcjiTR5RD1w+4rlKZuE5J3EuMnNX1PdCYLSyMQed+7UAtX6
TNyuiuVZVxuzJ5iS7L2ZoX05ASgyoh/Loipc+an6HzHqQnNC16ZdrBL4AkkGhDgP
ZbYjM3FbBQkL2T/08NcwTrKuVz8DIxgH7yPAOpBzm91n/pV248eK0a46sKauR2DB
zPKjcc180qmaVWyv9C60roSslvnkZsqe/jYyDFuSsRWqGgE5jNyIb8EY7K7KraPv
3AkusgCh4fqlBxOvF6FJkiYeZZs5YXvGQ296HTfVhPLOqctSFX2kuUKGIq2Z+H/9
qfJFGS1iaUsoDEUOaU27lQg5wsYa8EsCm9otroH2P3g7435JYRbeiwlwfHMS9EfK
dwD38d8UzZj7TnxGG4T1aLb3Lj5tNG6DSko69+zqHhuknjkRuAxRAZfHeuRbACgE
nIa7Chit8EGhC2GB12pr5XFWzTvNFdxFhbG+ed7EiGn/v0pVQc0ZfE73FXltg7et
bkoC26o5Ksk1wK2SEs/f8aDZFtG01Ys0ASFICDGW2tusFvDs6LpPUUggMjf41s7j
4tKotEE1Hzr38EdY+8faRaAS9teQdH5yob5a5Bp5F5wgmpqZom/gjle4JBVaV5dI
N5rcnHzcvXw=
=asqr
-----END PGP PUBLIC KEY BLOCK-----
permissions: "0644"

# install the snap
snap:
commands:
00: 'snap install k8s --classic --channel=1.32/stable'

runcmd:
# fetch dpdk driver binding script
- su ubuntu -c "git config --global http.proxy http://10.18.2.1:3128"
- su ubuntu -c "git clone https://github.com/DPDK/dpdk.git /home/ubuntu/dpdk"
- apt update
- DEBIAN_FRONTEND=noninteractive apt install -y linux-headers-6.8.1-1004-realtime linux-image-6.8.1-1004-realtime linux-modules-6.8.1-1004-realtime linux-modules-extra-6.8.1-1004-realtime
- pro attach <UBUNTU PRO TOKEN> --no-auto-enable
- pro enable realtime-kernel --variant generic --assume-yes
# fetch dpdk driver binding script
- su ubuntu -c "git clone https://github.com/DPDK/dpdk.git /home/ubuntu/dpdk"

# enable kernel options
- update-grub
# update grub with new kernel options
- update-grub

# reboot to activate realtime-kernel and kernel options
power_state:
mode: reboot
30 changes: 8 additions & 22 deletions docs/canonicalk8s/snap/howto/epa.md
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ membind: 0 1
The real-time kernel enablement requires an ubuntu pro subscription and some additional tools to be available.
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
The real-time kernel enablement requires an ubuntu pro subscription and some additional tools to be available.
The real-time kernel enablement requires an Ubuntu pro subscription and some additional tools to be available.


```
sudo pro attach
sudo pro attach <UBUNTU PRO TOKEN> --no-auto-enable
sudo apt update && sudo apt install ubuntu-advantage-tools
sudo pro enable realtime-kernel
```
Expand Down Expand Up @@ -287,21 +287,12 @@ With these preparation steps we have enabled the features of EPA:
To prepare a machine for CPU isolation, HugePages, real-time kernel,
SR-IOV and DPDK we leverage cloud-init through MAAS available to download {download}`here </assets/how-to-epa-maas-cloud-init>`.

```{literalinclude} /assets/how-to-epa-maas-cloud-init
```

```{note}

In the above file, the `realtime kernel` 6.8 is installed from a private PPA.
It was recently backported from 24.04 to 22.04 and is still going through
some validation stages. Once it is officially released, it will be
installable via the Ubuntu Pro CLI.
Make sure to replace `<UBUNTU PRO TOKEN>` with the actual token.
```

<!--VERSION:
The MAAS setup is inextricably linked to particular versions and will need to be
updated for these -->

```{literalinclude} /assets/how-to-epa-maas-cloud-init
```
````
`````

Expand All @@ -316,10 +307,6 @@ EPA capabilities.

1. [Install the snap][install-link] from the relevant [channel][channel].

```{note}
A pre-release channel is required currently until there is a stable release of {{product}}.
```

For example:

<!-- This uses a generic include for this branch to insert the standard
Expand All @@ -331,8 +318,7 @@ EPA capabilities.

2. Create a file called *configuration.yaml* or download it
{download}`here </assets/configuration.yaml>`. In this configuration file
we let the snap start with its default CNI (cilium), with CoreDNS deployed and
we also point k8s to the external etcd.
we let the snap start with its default CNI (cilium) and CoreDNS enabled.
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
we let the snap start with its default CNI (cilium) and CoreDNS enabled.
we let the snap start with its default CNI (Cilium) and CoreDNS enabled.


```{literalinclude} /assets/configuration.yaml
:language: yaml
Expand Down Expand Up @@ -372,7 +358,7 @@ sudo k8s kubectl get all -A

```
extra-node-kubelet-args:
--reserved-cpus: "0-31"
--reserved-cpus: <reserved CPUs (physical core IDs e.g. 0-2)>
--cpu-manager-policy: "static"
--topology-manager-policy: "best-effort"
```
Expand Down Expand Up @@ -704,13 +690,13 @@ First check if CPU Manager and NUMA Topology Manager is set up in the worker
node:

```
ps -ef | grep /snap/k8s/678/bin/kubelet
ps -ef | grep bin/kubelet
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is it /bin/kubelet?

```

The process output will indicate the arguments used when running the kubelet:

```
root 9139 1 1 Jul17 ? 00:20:03 /snap/k8s/678/bin/kubelet --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --client-ca-file=/etc/kubernetes/pki/client-ca.crt --cluster-dns=10.152.183.97 --cluster-domain=cluster.local --container-runtime-endpoint=/var/snap/k8s/common/run/containerd.sock --containerd=/var/snap/k8s/common/run/containerd.sock --cpu-manager-policy=static --eviction-hard=memory.available<100Mi,nodefs.available<1Gi,imagefs.available<1Gi --fail-swap-on=false --kubeconfig=/etc/kubernetes/kubelet.conf --node-ip=10.18.2.153 --node-labels=node-role.kubernetes.io/worker=,k8sd.io/role=worker --read-only-port=0 --register-with-taints= --reserved-cpus=0-31 --root-dir=/var/lib/kubelet --serialize-image-pulls=false --tls-cert-file=/etc/kubernetes/pki/kubelet.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_GCM_SHA384 --tls-private-key-file=/etc/kubernetes/pki/kubelet.key --topology-manager-policy=best-effort
root 9139 1 1 Jul17 ? 00:20:03 /snap/k8s/*/bin/kubelet --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --client-ca-file=/etc/kubernetes/pki/client-ca.crt --cluster-dns=10.152.183.97 --cluster-domain=cluster.local --container-runtime-endpoint=/var/snap/k8s/common/run/containerd.sock --containerd=/var/snap/k8s/common/run/containerd.sock --cpu-manager-policy=static --eviction-hard=memory.available<100Mi,nodefs.available<1Gi,imagefs.available<1Gi --fail-swap-on=false --kubeconfig=/etc/kubernetes/kubelet.conf --node-ip=10.18.2.153 --node-labels=node-role.kubernetes.io/worker=,k8sd.io/role=worker --read-only-port=0 --register-with-taints= --reserved-cpus=0-31 --root-dir=/var/lib/kubelet --serialize-image-pulls=false --tls-cert-file=/etc/kubernetes/pki/kubelet.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_GCM_SHA384 --tls-private-key-file=/etc/kubernetes/pki/kubelet.key --topology-manager-policy=best-effort
```

```{dropdown} Explanation of output
Expand Down
Loading