From e74d90aae7ef7a76b5f175aa8b6aa0feb431b252 Mon Sep 17 00:00:00 2001 From: Jakub Sliacan Date: Wed, 19 Oct 2022 13:25:46 +0200 Subject: [PATCH] Add the option to record start durations for crc cluster --- cmd/monictl.go | 106 +++++++++++++++++++++++++++++------------------- tools/tools.go | 107 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 172 insertions(+), 41 deletions(-) diff --git a/cmd/monictl.go b/cmd/monictl.go index 2813647..f375a32 100644 --- a/cmd/monictl.go +++ b/cmd/monictl.go @@ -37,6 +37,12 @@ func main() { flag.IntVar(&numRepeats, "n", 5, "number of checks of CPU load") var sleepLength int flag.IntVar(&sleepLength, "s", 1, "sleep between repeats [in seconds]") + var repeatStarts bool + flag.BoolVar(&repeatStarts, "r", false, "repeatedly start and delete the cluster") + var pullSecretPath string + flag.StringVar(&pullSecretPath, "p", "", "path to pull secret file [needed if -r flag is set to true") + var bundlePath string + flag.StringVar(&bundlePath, "b", "", "path to CRC bundle [needed if -r flag is set to true") flag.Parse() @@ -46,7 +52,8 @@ func main() { log.Fatalf("Unable to create directory: %s", dirPath) } - if !tools.IsCRCRunning() { + // Require running cluster if not doing start/delete testing + if !repeatStarts && !tools.IsCRCRunning() { fmt.Println("CRC VM is not running") os.Exit(1) } @@ -56,7 +63,9 @@ func main() { fmt.Println("Running monitoring tools with the following settings:") fmt.Printf("Data directory: %s\n", dirPath) fmt.Printf("Number of repeats: %d\n", numRepeats) - fmt.Printf("Pauses between repeats: %ds\n", sleepLength) + if !repeatStarts { + fmt.Printf("Pauses between repeats: %ds\n", sleepLength) + } fmt.Printf("Logging into: %s\n", logFilePath) fmt.Println("-------------") @@ -64,54 +73,69 @@ func main() { trafficChan := make(chan error) crioChan := make(chan error) nodeDesChan := make(chan error) + startChan := make(chan error) // ================ // start collecting // ================ - // transmitted/received MiB on crc interface - trafficFile := filepath.Join(dirPath, "traffic.json") - go tools.RecordTraffic(trafficFile, numRepeats, sleepLength, trafficChan) - log.Println("going to record traffic going in/out of the VM") - - // CPU usage by 'qemu' process - cpuFile := filepath.Join(dirPath, "cpu.json") - go tools.RecordHostCPUUsage(cpuFile, numRepeats, sleepLength, cpuChan) - log.Println("going to record CPU usage percentage attributed to qemu") - - // CRI-O stats as reported by 'crictl' - go tools.GetCRIStatsFromVM(dirPath, crioChan) - log.Println("going to retrieve crictl stats from the CRC VM") - - // Node Description - nodeDescription := filepath.Join(dirPath, "node.json") - go tools.GetNodeResource(nodeDescription, nodeDesChan) - + if repeatStarts { + // start times for CRC cluster + startTimesFile := filepath.Join(dirPath, "startTimes.json") + go tools.RecordStartTimes(startTimesFile, numRepeats, startChan, pullSecretPath, bundlePath) + log.Println("going to record start times for CRC cluster") + } else { + // transmitted/received MiB on crc interface + trafficFile := filepath.Join(dirPath, "traffic.json") + go tools.RecordTraffic(trafficFile, numRepeats, sleepLength, trafficChan) + log.Println("going to record traffic going in/out of the VM") + + // CPU usage by 'qemu' process + cpuFile := filepath.Join(dirPath, "cpu.json") + go tools.RecordHostCPUUsage(cpuFile, numRepeats, sleepLength, cpuChan) + log.Println("going to record CPU usage percentage attributed to qemu") + + // CRI-O stats as reported by 'crictl' + go tools.GetCRIStatsFromVM(dirPath, crioChan) + log.Println("going to retrieve crictl stats from the CRC VM") + + // Node Description + nodeDescription := filepath.Join(dirPath, "node.json") + go tools.GetNodeResource(nodeDescription, nodeDesChan) + } // ================ // done collecting // ================ - if err := <-trafficChan; err != nil { - log.Fatalf("failed to record traffic flow %s", err) - } else { - log.Printf("recorded traffic (RX/TX) %d times at %d sec intervals", numRepeats, sleepLength) - } - - if err := <-cpuChan; err != nil { - log.Fatalf("failed to record CPU percentage %s", err) - } else { - log.Printf("recorded CPU usage percentage %d times at %d sec intervals", numRepeats, sleepLength) - } - - if err := <-crioChan; err != nil { - log.Fatalf("could not retrieve crictl stats: %s", err) - } else { - log.Println("crictl stats successfully retrieved") - } - - if err := <-nodeDesChan; err != nil { - log.Fatalf("could not retrieve node description stats: %s", err) + if repeatStarts { + if err := <-startChan; err != nil { + log.Fatalf("failed to record start times: %s", err) + } else { + log.Printf("recorded start duration %d times", numRepeats) + } } else { - log.Println("node description successfully retrieved") + if err := <-trafficChan; err != nil { + log.Fatalf("failed to record traffic flow %s", err) + } else { + log.Printf("recorded traffic (RX/TX) %d times at %d sec intervals", numRepeats, sleepLength) + } + + if err := <-cpuChan; err != nil { + log.Fatalf("failed to record CPU percentage %s", err) + } else { + log.Printf("recorded CPU usage percentage %d times at %d sec intervals", numRepeats, sleepLength) + } + + if err := <-crioChan; err != nil { + log.Fatalf("could not retrieve crictl stats: %s", err) + } else { + log.Println("crictl stats successfully retrieved") + } + + if err := <-nodeDesChan; err != nil { + log.Fatalf("could not retrieve node description stats: %s", err) + } else { + log.Println("node description successfully retrieved") + } } } diff --git a/tools/tools.go b/tools/tools.go index 8fd2d6b..0851fe9 100644 --- a/tools/tools.go +++ b/tools/tools.go @@ -258,3 +258,110 @@ func parseNodeDescribeToJSON(path string) error { return nil } + +func timedClusterStart(pullSecretPath string, bundlePath string) (string, error) { + + cmd := exec.Command("crc", "start", "-b", bundlePath, "-p", pullSecretPath) + s := time.Now() + out, err := cmd.Output() + log.Printf("%s", string(out)) + startDuration := time.Since(s) + if err != nil { + return "0h0m0.0s", err + } + + return startDuration.String(), nil +} + +func clusterDelete() error { + + cmd := exec.Command("crc", "delete", "-f") + err := cmd.Run() + + return err +} + +func clusterCleanup() error { + cmd := exec.Command("crc", "cleanup") + out, err := cmd.Output() + if err != nil { + log.Printf("%s", string(out)) + } + + return err +} + +func clusterSetup(bundlePath string) error { + cmd := exec.Command("crc", "setup", "-b", bundlePath) + out, err := cmd.Output() + if err != nil { + log.Printf("%s", string(out)) + } + + return err +} + +func recordStartTimes(filename string, reps int, pullSecretPath string, bundlePath string) error { + + // crc cleanup + err := clusterCleanup() + if err != nil { + log.Printf("Could not clean up the host: %s", err) + return err + } + homeDir, err := os.UserHomeDir() + if err != nil { + log.Printf("Could not retrieve user's home directory: %s", err) + return err + } + // remove config file + err = os.Remove(filepath.Join(homeDir, ".crc", "crc.json")) + if err != nil { + log.Printf("Could not remove crc config: %s", err) + return err + } + // crc setup + err = clusterSetup(bundlePath) + if err != nil { + log.Printf("Could not set up the host: %s", err) + return err + } + + // collect data + var startTimes []string + for i := 0; i < reps; i++ { + + startTime, _ := timedClusterStart(pullSecretPath, bundlePath) + startTimes = append(startTimes, startTime) + + err1 := clusterDelete() + if err1 != nil { + log.Printf("Failed to delete cluster: %s", err1) + os.Exit(1) + } + } + + // create CSV file and write data to it + f, err := os.Create(filename) + if err != nil { + log.Printf("could not create %s err: %s", filename, err) + } + defer f.Close() + + jsonStartTimes, _ := json.MarshalIndent(startTimes, "", " ") + err = ioutil.WriteFile(filename, jsonStartTimes, 0644) + if err != nil { + log.Printf("Could not write data to %s", filename) + } + + return nil +} + +// RecordStartTimes returns a list of n cpu usage stats +// (in %) taken with nap breaks in between each poll +// filename : relative location of file to write into +// reps : number of times to record CPU usage +// c : channel used to report back to the main process +func RecordStartTimes(filename string, reps int, c chan error, pullSecretPath string, bundlePath string) { + c <- recordStartTimes(filename, reps, pullSecretPath, bundlePath) +}