|
| 1 | +package main |
| 2 | + |
| 3 | +import ( |
| 4 | + "flag" |
| 5 | + "fmt" |
| 6 | + "os" |
| 7 | + "regexp" |
| 8 | + "strconv" |
| 9 | + "strings" |
| 10 | + "time" |
| 11 | + |
| 12 | + k8sinformers "k8s.io/client-go/informers" |
| 13 | + "k8s.io/client-go/kubernetes" |
| 14 | + "k8s.io/client-go/tools/clientcmd" |
| 15 | + |
| 16 | + networkv1alpha1 "github.com/yunify/hostnic-cni/pkg/apis/network/v1alpha1" |
| 17 | + clientset "github.com/yunify/hostnic-cni/pkg/client/clientset/versioned" |
| 18 | + informers "github.com/yunify/hostnic-cni/pkg/client/informers/externalversions" |
| 19 | + "github.com/yunify/hostnic-cni/pkg/constants" |
| 20 | + "github.com/yunify/hostnic-cni/pkg/networkutils" |
| 21 | + "github.com/yunify/hostnic-cni/pkg/rpc" |
| 22 | + "github.com/yunify/hostnic-cni/pkg/signals" |
| 23 | + "github.com/yunify/hostnic-cni/pkg/simple/client/network/ippool/ipam" |
| 24 | +) |
| 25 | + |
| 26 | +var kubeconfig string |
| 27 | +var delete, debug bool |
| 28 | + |
| 29 | +func usage() { |
| 30 | + fmt.Println("This tool is used to check leak ip arp rules and release ip, it will list all leak ip arp rules on this instance by default, if you want to delete them, please use '-delete' flag") |
| 31 | + flag.PrintDefaults() |
| 32 | +} |
| 33 | + |
| 34 | +func main() { |
| 35 | + flag.StringVar(&kubeconfig, "kubeconfig", "/root/.kube/config", "Path to a kubeconfig. Only required if out-of-cluster.If not set, use the default path") |
| 36 | + flag.BoolVar(&delete, "delete", false, "delete leak ip arp rules and release ip") |
| 37 | + flag.BoolVar(&debug, "debug", false, "show debug info") |
| 38 | + flag.Usage = usage |
| 39 | + flag.Parse() |
| 40 | + |
| 41 | + //get instance-id |
| 42 | + instanceIDByte, err := os.ReadFile(constants.InstanceIDFile) |
| 43 | + if err != nil { |
| 44 | + fmt.Printf("failed to load instance-id: %v", err) |
| 45 | + return |
| 46 | + } |
| 47 | + instanceID := strings.TrimSpace(string(instanceIDByte)) |
| 48 | + fmt.Println("get instanceID: ", instanceID) |
| 49 | + |
| 50 | + // set up signals so we handle the first shutdown signals gracefully |
| 51 | + stopCh := signals.SetupSignalHandler() |
| 52 | + |
| 53 | + //client |
| 54 | + cfg, err := clientcmd.BuildConfigFromFlags("", kubeconfig) |
| 55 | + if err != nil { |
| 56 | + fmt.Printf("Error building kubeconfig: %v", err) |
| 57 | + return |
| 58 | + } |
| 59 | + |
| 60 | + k8sClient, err := kubernetes.NewForConfig(cfg) |
| 61 | + if err != nil { |
| 62 | + fmt.Printf("Error building kubernetes clientset: %v", err) |
| 63 | + return |
| 64 | + } |
| 65 | + |
| 66 | + client, err := clientset.NewForConfig(cfg) |
| 67 | + if err != nil { |
| 68 | + fmt.Printf("Error building example clientset: %v", err) |
| 69 | + return |
| 70 | + } |
| 71 | + |
| 72 | + k8sInformerFactory := k8sinformers.NewSharedInformerFactory(k8sClient, time.Second*10) |
| 73 | + informerFactory := informers.NewSharedInformerFactory(client, time.Second*30) |
| 74 | + |
| 75 | + ipamClient := ipam.NewIPAMClient(client, networkv1alpha1.IPPoolTypeLocal, informerFactory, k8sInformerFactory) |
| 76 | + |
| 77 | + k8sInformerFactory.Start(stopCh) |
| 78 | + informerFactory.Start(stopCh) |
| 79 | + |
| 80 | + if err = ipamClient.Sync(stopCh); err != nil { |
| 81 | + fmt.Printf("ipamclient sync error: %v", err) |
| 82 | + return |
| 83 | + } |
| 84 | + |
| 85 | + //network tools for clean pod network |
| 86 | + networkutils.SetupNetworkHelper() |
| 87 | + |
| 88 | + //get all arp rules on the instance |
| 89 | + rules, err := getArpRuleList() |
| 90 | + if err != nil { |
| 91 | + fmt.Printf("getArpRuleList failed: %v\n", err) |
| 92 | + return |
| 93 | + } |
| 94 | + if debug { |
| 95 | + fmt.Printf("all arp rules on instance %s: \n", instanceID) |
| 96 | + for _, rule := range rules { |
| 97 | + fmt.Printf("\t%s\n", rule.rule) |
| 98 | + } |
| 99 | + } |
| 100 | + fmt.Printf("\n\n") |
| 101 | + |
| 102 | + //get pods on the instance |
| 103 | + podsMap, err := ipamClient.ListInstancePods(instanceID) |
| 104 | + if err != nil { |
| 105 | + fmt.Printf("ListInstancePods failed: %v\n", err) |
| 106 | + return |
| 107 | + } |
| 108 | + |
| 109 | + //check and gather leak ip arp rules |
| 110 | + rulesToClear := []arpReplyInfo{} |
| 111 | + for _, rule := range rules { |
| 112 | + if _, ok := podsMap[rule.ip]; !ok { |
| 113 | + rulesToClear = append(rulesToClear, rule) |
| 114 | + } |
| 115 | + } |
| 116 | + |
| 117 | + if len(rulesToClear) == 0 { |
| 118 | + fmt.Printf("no leak ip arp rules found on instance %s\n\n", instanceID) |
| 119 | + return |
| 120 | + } |
| 121 | + |
| 122 | + fmt.Printf("leak ip arp rules on instance %s: \n", instanceID) |
| 123 | + for _, rule := range rulesToClear { |
| 124 | + fmt.Printf("\t%s\n", rule.rule) |
| 125 | + } |
| 126 | + fmt.Printf("\n\n") |
| 127 | + |
| 128 | + if !delete { |
| 129 | + return |
| 130 | + } |
| 131 | + |
| 132 | + // delete leak ip arp rules and release ip |
| 133 | + // should delete arp rules first, then release ip, or if release ip error,the rule will left but ip was released and will be allocated to other pod |
| 134 | + fmt.Printf("going to delete leak ip arp rules and release ip\n\n") |
| 135 | + for _, rule := range rulesToClear { |
| 136 | + if err := clearArpReplyRule(rule); err != nil { |
| 137 | + fmt.Printf("delete arp rule for leak ip %s error: %v, skip\n\n", rule.ip, err) |
| 138 | + continue |
| 139 | + } |
| 140 | + fmt.Printf("delete arp rule for leak ip %s success\n", rule.ip) |
| 141 | + if err := ipamClient.ReleaseByIP(rule.ip); err != nil { |
| 142 | + fmt.Printf("release leak ip %s error: %v, skip\n\n", rule.ip, err) |
| 143 | + } |
| 144 | + fmt.Printf("release leak ip %s success\n\n", rule.ip) |
| 145 | + } |
| 146 | +} |
| 147 | + |
| 148 | +// list arp rules |
| 149 | +type arpReplyInfo struct { |
| 150 | + routeTableNum int32 |
| 151 | + ip string |
| 152 | + macAddr string |
| 153 | + rule string |
| 154 | +} |
| 155 | + |
| 156 | +func getArpRuleList() (ruleList []arpReplyInfo, err error) { |
| 157 | + //get all arp rules |
| 158 | + ruleStr, err := networkutils.ListArpReply() |
| 159 | + if err != nil { |
| 160 | + return nil, fmt.Errorf("list arp rules error: %v", err) |
| 161 | + } |
| 162 | + |
| 163 | + //parse arp rules |
| 164 | + return parseARPRules(ruleStr) |
| 165 | +} |
| 166 | + |
| 167 | +func parseARPRules(output string) (ruleList []arpReplyInfo, err error) { |
| 168 | + lines := strings.Split(output, "\n") |
| 169 | + |
| 170 | + ruleRegex := regexp.MustCompile(`-p ARP --logical-in (br_\d+) --arp-op Request --arp-ip-dst (\d+\.\d+\.\d+\.\d+) -j arpreply --arpreply-mac ([\da-fA-F:]+)`) |
| 171 | + |
| 172 | + for _, line := range lines { |
| 173 | + line = strings.TrimSpace(line) |
| 174 | + if line == "" || strings.HasPrefix(line, "Bridge chain:") || strings.HasPrefix(line, "policy:") { |
| 175 | + // skip empty line and header |
| 176 | + continue |
| 177 | + } |
| 178 | + |
| 179 | + match := ruleRegex.FindStringSubmatch(line) |
| 180 | + if len(match) > 3 { |
| 181 | + logicalIn := match[1] |
| 182 | + ip := match[2] |
| 183 | + mac := match[3] |
| 184 | + |
| 185 | + tableNumStr := strings.TrimPrefix(logicalIn, "br_") |
| 186 | + tableNum, err := strconv.Atoi(tableNumStr) |
| 187 | + if err != nil { |
| 188 | + return nil, fmt.Errorf("parse tableNumStr %s error: %v", tableNumStr, err) |
| 189 | + } |
| 190 | + |
| 191 | + ruleList = append(ruleList, arpReplyInfo{ |
| 192 | + routeTableNum: int32(tableNum), |
| 193 | + ip: ip, |
| 194 | + macAddr: mac, |
| 195 | + rule: line, |
| 196 | + }) |
| 197 | + } |
| 198 | + } |
| 199 | + return |
| 200 | +} |
| 201 | + |
| 202 | +// check and delete arp rules |
| 203 | +func clearArpReplyRule(item arpReplyInfo) error { |
| 204 | + //delete arp rule |
| 205 | + err := networkutils.NetworkHelper.CleanupPodNetwork(&rpc.HostNic{ |
| 206 | + RouteTableNum: item.routeTableNum, |
| 207 | + HardwareAddr: item.macAddr, |
| 208 | + }, item.ip) |
| 209 | + if err != nil && !strings.Contains(err.Error(), "rule does not exist") { |
| 210 | + return fmt.Errorf("failed to delete ebtables rule %s: %s", item.rule, err) |
| 211 | + } |
| 212 | + |
| 213 | + return nil |
| 214 | +} |
0 commit comments