Skip to content

Commit 503c6d6

Browse files
Anirudh2112Anirudh2112
authored
Merge branch 'main' into add-kernelversion-to-traces
2 parents d03cdc6 + 499e3cf commit 503c6d6

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

54 files changed

+1192
-405
lines changed

‎.goreleaser.yaml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
# yaml-language-server: $schema=https://goreleaser.com/static/schema.json
22
# vim: set ts=2 sw=2 tw=0 fo=cnqoj
33

4-
version: 1
4+
version: 2
55

66
before:
77
hooks:
@@ -22,7 +22,7 @@ builds:
2222
- windows
2323
- darwin
2424
ldflags:
25-
- -X github.com/microsoft/retina/cli/cmd.Version=v{{.Version}}
25+
- -X github.com/microsoft/retina/internal/buildinfo.Version=v{{.Version}}
2626
main: cli/main.go
2727

2828
archives:

‎cli/cmd/shell.go

Lines changed: 173 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,173 @@
1+
package cmd
2+
3+
import (
4+
"errors"
5+
"fmt"
6+
"os"
7+
"time"
8+
9+
"github.com/microsoft/retina/internal/buildinfo"
10+
"github.com/microsoft/retina/shell"
11+
"github.com/spf13/cobra"
12+
v1 "k8s.io/api/core/v1"
13+
"k8s.io/cli-runtime/pkg/genericclioptions"
14+
"k8s.io/cli-runtime/pkg/resource"
15+
cmdutil "k8s.io/kubectl/pkg/cmd/util"
16+
"k8s.io/kubectl/pkg/scheme"
17+
"k8s.io/kubectl/pkg/util/templates"
18+
)
19+
20+
var (
21+
configFlags *genericclioptions.ConfigFlags
22+
matchVersionFlags *cmdutil.MatchVersionFlags
23+
retinaShellImageRepo string
24+
retinaShellImageVersion string
25+
mountHostFilesystem bool
26+
allowHostFilesystemWrite bool
27+
hostPID bool
28+
capabilities []string
29+
timeout time.Duration
30+
)
31+
32+
var (
33+
// AKS requires clusters to allow access to MCR, so use this repository by default.
34+
defaultRetinaShellImageRepo = "mcr.microsoft.com/containernetworking/retina-shell"
35+
36+
// Default version is the same as CLI version, set at link time.
37+
defaultRetinaShellImageVersion = buildinfo.Version
38+
39+
defaultTimeout = 30 * time.Second
40+
41+
errMissingRequiredRetinaShellImageVersionArg = errors.New("missing required --retina-shell-image-version")
42+
errUnsupportedResourceType = errors.New("unsupported resource type")
43+
)
44+
45+
var shellCmd = &cobra.Command{
46+
Use: "shell (NODE | TYPE[[.VERSION].GROUP]/NAME)",
47+
Short: "[EXPERIMENTAL] Interactively debug a node or pod",
48+
Long: templates.LongDesc(`
49+
[EXPERIMENTAL] This is an experimental command. The flags and behavior may change in the future.
50+
51+
Start a shell with networking tools in a node or pod for adhoc debugging.
52+
53+
* For nodes, this creates a pod on the node in the root network namespace.
54+
* For pods, this creates an ephemeral container inside the pod's network namespace.
55+
56+
You can override the default image used for the shell container with either
57+
CLI flags (--retina-shell-image-repo and --retina-shell-image-version) or
58+
environment variables (RETINA_SHELL_IMAGE_REPO and RETINA_SHELL_IMAGE_VERSION).
59+
CLI flags take precedence over env vars.
60+
`),
61+
62+
Example: templates.Examples(`
63+
# start a shell in a node
64+
kubectl retina shell node0001
65+
66+
# start a shell in a node, with debug pod in kube-system namespace
67+
kubectl retina shell -n kube-system node0001
68+
69+
# start a shell as an ephemeral container inside an existing pod
70+
kubectl retina shell -n kube-system pod/coredns-d459997b4-7cpzx
71+
72+
# start a shell in a node, mounting the host filesystem to /host with ability to chroot
73+
kubectl retina shell node001 --mount-host-filesystem --capabilities SYS_CHROOT
74+
75+
# start a shell in a node, with NET_RAW and NET_ADMIN capabilities
76+
# (required for iptables and tcpdump)
77+
kubectl retina shell node001 --capabilities NET_RAW,NET_ADMIN
78+
`),
79+
Args: cobra.ExactArgs(1),
80+
RunE: func(_ *cobra.Command, args []string) error {
81+
// retinaShellImageVersion defaults to the CLI version, but that might not be set if the CLI is built without -ldflags.
82+
if retinaShellImageVersion == "" {
83+
return errMissingRequiredRetinaShellImageVersionArg
84+
}
85+
86+
namespace, explicitNamespace, err := matchVersionFlags.ToRawKubeConfigLoader().Namespace()
87+
if err != nil {
88+
return fmt.Errorf("error retrieving namespace arg: %w", err)
89+
}
90+
91+
// This interprets the first arg as either a node or pod (same as kubectl):
92+
// "node001" -> node
93+
// "node/node001" -> node
94+
// "pod/example-7cpzx" -> pod
95+
r := resource.NewBuilder(configFlags).
96+
WithScheme(scheme.Scheme, scheme.Scheme.PrioritizedVersionsAllGroups()...).
97+
FilenameParam(explicitNamespace, &resource.FilenameOptions{}).
98+
NamespaceParam(namespace).DefaultNamespace().ResourceNames("nodes", args[0]).
99+
Do()
100+
if rerr := r.Err(); rerr != nil {
101+
return fmt.Errorf("error constructing resource builder: %w", rerr)
102+
}
103+
104+
restConfig, err := matchVersionFlags.ToRESTConfig()
105+
if err != nil {
106+
return fmt.Errorf("error constructing REST config: %w", err)
107+
}
108+
109+
config := shell.Config{
110+
RestConfig: restConfig,
111+
RetinaShellImage: fmt.Sprintf("%s:%s", retinaShellImageRepo, retinaShellImageVersion),
112+
MountHostFilesystem: mountHostFilesystem,
113+
AllowHostFilesystemWrite: allowHostFilesystemWrite,
114+
HostPID: hostPID,
115+
Capabilities: capabilities,
116+
Timeout: timeout,
117+
}
118+
119+
return r.Visit(func(info *resource.Info, err error) error {
120+
if err != nil {
121+
return err
122+
}
123+
124+
switch obj := info.Object.(type) {
125+
case *v1.Node:
126+
podDebugNamespace := namespace
127+
nodeName := obj.Name
128+
return shell.RunInNode(config, nodeName, podDebugNamespace)
129+
case *v1.Pod:
130+
return shell.RunInPod(config, obj.Namespace, obj.Name)
131+
default:
132+
gvk := obj.GetObjectKind().GroupVersionKind()
133+
return fmt.Errorf("unsupported resource %s/%s: %w", gvk.GroupVersion(), gvk.Kind, errUnsupportedResourceType)
134+
}
135+
})
136+
},
137+
}
138+
139+
func init() {
140+
Retina.AddCommand(shellCmd)
141+
shellCmd.PersistentPreRun = func(cmd *cobra.Command, _ []string) {
142+
// Avoid printing full usage message if the command exits with an error.
143+
cmd.SilenceUsage = true
144+
cmd.SilenceErrors = true
145+
146+
// Allow setting image repo and version via environment variables (CLI flags still take precedence).
147+
if !cmd.Flags().Changed("retina-shell-image-repo") {
148+
if envRepo := os.Getenv("RETINA_SHELL_IMAGE_REPO"); envRepo != "" {
149+
retinaShellImageRepo = envRepo
150+
}
151+
}
152+
if !cmd.Flags().Changed("retina-shell-image-version") {
153+
if envVersion := os.Getenv("RETINA_SHELL_IMAGE_VERSION"); envVersion != "" {
154+
retinaShellImageVersion = envVersion
155+
}
156+
}
157+
}
158+
shellCmd.Flags().StringVar(&retinaShellImageRepo, "retina-shell-image-repo", defaultRetinaShellImageRepo, "The container registry repository for the image to use for the shell container")
159+
shellCmd.Flags().StringVar(&retinaShellImageVersion, "retina-shell-image-version", defaultRetinaShellImageVersion, "The version (tag) of the image to use for the shell container")
160+
shellCmd.Flags().BoolVarP(&mountHostFilesystem, "mount-host-filesystem", "m", false, "Mount the host filesystem to /host. Applies only to nodes, not pods.")
161+
shellCmd.Flags().BoolVarP(&allowHostFilesystemWrite, "allow-host-filesystem-write", "w", false,
162+
"Allow write access to the host filesystem. Implies --mount-host-filesystem. Applies only to nodes, not pods.")
163+
shellCmd.Flags().BoolVar(&hostPID, "host-pid", false, "Set HostPID on the shell container. Applies only to nodes, not pods.")
164+
shellCmd.Flags().StringSliceVarP(&capabilities, "capabilities", "c", []string{}, "Add capabilities to the shell container")
165+
shellCmd.Flags().DurationVar(&timeout, "timeout", defaultTimeout, "The maximum time to wait for the shell container to start")
166+
167+
// configFlags and matchVersion flags are used to load kubeconfig.
168+
// This uses the same mechanism as `kubectl debug` to connect to apiserver and attach to containers.
169+
configFlags = genericclioptions.NewConfigFlags(true)
170+
configFlags.AddFlags(shellCmd.PersistentFlags())
171+
matchVersionFlags = cmdutil.NewMatchVersionFlags(configFlags)
172+
matchVersionFlags.AddFlags(shellCmd.PersistentFlags())
173+
}

‎docs/06-Troubleshooting/shell.md

Lines changed: 185 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,185 @@
1+
# Shell TSG
2+
3+
**EXPERIMENTAL: `retina shell` is an experimental feature, so the flags and behavior may change in future versions.**
4+
5+
The `retina shell` command allows you to start an interactive shell on a Kubernetes node or pod. This runs a container image with many common networking tools installed (`ping`, `curl`, etc.).
6+
7+
## Testing connectivity
8+
9+
Start a shell on a node or inside a pod
10+
11+
```bash
12+
# To start a shell in a node (root network namespace):
13+
kubectl retina shell aks-nodepool1-15232018-vmss000001
14+
15+
# To start a shell inside a pod (pod network namespace):
16+
kubectl retina shell -n kube-system pods/coredns-d459997b4-7cpzx
17+
```
18+
19+
Check connectivity using `ping`:
20+
21+
```text
22+
root [ / ]# ping 10.224.0.4
23+
PING 10.224.0.4 (10.224.0.4) 56(84) bytes of data.
24+
64 bytes from 10.224.0.4: icmp_seq=1 ttl=64 time=0.964 ms
25+
64 bytes from 10.224.0.4: icmp_seq=2 ttl=64 time=1.13 ms
26+
64 bytes from 10.224.0.4: icmp_seq=3 ttl=64 time=0.908 ms
27+
64 bytes from 10.224.0.4: icmp_seq=4 ttl=64 time=1.07 ms
28+
64 bytes from 10.224.0.4: icmp_seq=5 ttl=64 time=1.01 ms
29+
30+
--- 10.224.0.4 ping statistics ---
31+
5 packets transmitted, 5 received, 0% packet loss, time 4022ms
32+
rtt min/avg/max/mdev = 0.908/1.015/1.128/0.077 ms
33+
```
34+
35+
Check DNS resolution using `dig`:
36+
37+
```text
38+
root [ / ]# dig example.com +short
39+
93.184.215.14
40+
```
41+
42+
The tools `nslookup` and `drill` are also available if you prefer those.
43+
44+
Check connectivity to apiserver using `nc` and `curl`:
45+
46+
```text
47+
root [ / ]# nc -zv 10.0.0.1 443
48+
Ncat: Version 7.95 ( https://nmap.org/ncat )
49+
Ncat: Connected to 10.0.0.1:443.
50+
Ncat: 0 bytes sent, 0 bytes received in 0.06 seconds.
51+
52+
root [ / ]# curl -k https://10.0.0.1
53+
{
54+
"kind": "Status",
55+
"apiVersion": "v1",
56+
"metadata": {},
57+
"status": "Failure",
58+
"message": "Unauthorized",
59+
"reason": "Unauthorized",
60+
"code": 401
61+
}
62+
```
63+
64+
### nftables and iptables
65+
66+
Accessing nftables and iptables rules requires `NET_RAW` and `NET_ADMIN` capabilities.
67+
68+
```bash
69+
kubectl retina shell aks-nodepool1-15232018-vmss000002 --capabilities NET_ADMIN,NET_RAW
70+
```
71+
72+
Then you can run `iptables` and `nft`:
73+
74+
```text
75+
root [ / ]# iptables -nvL | head -n 2
76+
Chain INPUT (policy ACCEPT 1191K packets, 346M bytes)
77+
pkts bytes target prot opt in out source destination
78+
root [ / ]# nft list ruleset | head -n 2
79+
# Warning: table ip filter is managed by iptables-nft, do not touch!
80+
table ip filter {
81+
```
82+
83+
**If you see the error "Operation not permitted (you must be root)", check that your `kubectl retina shell` command sets `--capabilities NET_RAW,NET_ADMIN`.**
84+
85+
`iptables` in the shell image uses `iptables-legacy`, which may or may not match the configuration on the node. For example, Ubuntu maps `iptables` to `iptables-nft`. To use the exact same `iptables` binary as installed on the node, you will need to `chroot` into the host filesystem (see below).
86+
87+
## Accessing the host filesystem
88+
89+
On nodes, you can mount the host filesystem to `/host`:
90+
91+
```bash
92+
kubectl retina shell aks-nodepool1-15232018-vmss000002 --mount-host-filesystem
93+
```
94+
95+
This mounts the host filesystem (`/`) to `/host` in the debug pod:
96+
97+
```text
98+
root [ / ]# ls /host
99+
NOTICE.txt bin boot dev etc home lib lib64 libx32 lost+found media mnt opt proc root run sbin srv sys tmp usr var
100+
```
101+
102+
The host filesystem is mounted read-only by default. If you need write access, use the `--allow-host-filesystem-write` flag.
103+
104+
Symlinks between files on the host filesystem may not resolve correctly. If you see "No such file or directory" errors for symlinks, try following the instructions below to `chroot` to the host filesystem.
105+
106+
## Chroot to the host filesystem
107+
108+
`chroot` requires the `SYS_CHROOT` capability:
109+
110+
```bash
111+
kubectl retina shell aks-nodepool1-15232018-vmss000002 --mount-host-filesystem --capabilities SYS_CHROOT
112+
```
113+
114+
Then you can use `chroot` to switch to start a shell inside the host filesystem:
115+
116+
```text
117+
root [ / ]# chroot /host bash
118+
root@aks-nodepool1-15232018-vmss000002:/# cat /etc/resolv.conf | tail -n 2
119+
nameserver 168.63.129.16
120+
search shncgv2kgepuhm1ls1dwgholsd.cx.internal.cloudapp.net
121+
```
122+
123+
`chroot` allows you to:
124+
125+
* Execute binaries installed on the node.
126+
* Resolve symlinks that point to files in the host filesystem (such as /etc/resolv.conf -> /run/systemd/resolve/resolv.conf)
127+
* Use `sysctl` to view or modify kernel parameters.
128+
* Use `journalctl` to view systemd unit and kernel logs.
129+
* Use `ip netns` to view network namespaces. (However, `ip netns exec` does not work.)
130+
131+
## Systemctl
132+
133+
`systemctl` commands require both `chroot` to the host filesystem and host PID:
134+
135+
```bash
136+
kubectl retina shell aks-nodepool1-15232018-vmss000002 --mount-host-filesystem --capabilities SYS_CHROOT --host-pid
137+
```
138+
139+
Then `chroot` to the host filesystem and run `systemctl status`:
140+
141+
```text
142+
root [ / ]# chroot /host systemctl status | head -n 2
143+
● aks-nodepool1-15232018-vmss000002
144+
State: running
145+
```
146+
147+
**If `systemctl` shows an error "Failed to connect to bus: No data available", check that the `retina shell` command has `--host-pid` set and that you have chroot'd to /host.**
148+
149+
## Troubleshooting
150+
151+
### Timeouts
152+
153+
If `kubectl retina shell` fails with a timeout error, then:
154+
155+
1. Increase the timeout by setting `--timeout` flag.
156+
2. Check the pod using `kubectl describe pod` to determine why retina shell is failing to start.
157+
158+
Example:
159+
160+
```bash
161+
kubectl retina shell --timeout 10m node001 # increase timeout to 10 minutes
162+
```
163+
164+
### Firewalls and ImagePullBackoff
165+
166+
Some clusters are behind a firewall that blocks pulling the retina-shell image. To workaround this:
167+
168+
1. Replicate the retina-shell images to a container registry accessible from within the cluster.
169+
2. Override the image used by Retina CLI with the environment variable `RETINA_SHELL_IMAGE_REPO`.
170+
171+
Example:
172+
173+
```bash
174+
export RETINA_SHELL_IMAGE_REPO="example.azurecr.io/retina/retina-shell"
175+
export RETINA_SHELL_IMAGE_VERSION=v0.0.1 # optional, if not set defaults to the Retina CLI version.
176+
kubectl retina shell node0001 # this will use the image "example.azurecr.io/retina/retina-shell:v0.0.1"
177+
```
178+
179+
## Limitations
180+
181+
* Windows nodes and pods are not yet supported.
182+
* `bpftool` and `bpftrace` are not supported.
183+
* The shell image link `iptables` commands to `iptables-legacy`, even if the node itself links to `iptables-nft`.
184+
* `nsenter` is not supported.
185+
* `ip netns` will not work without `chroot` to the host filesystem.

0 commit comments

Comments
 (0)