Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add the option to record start durations for crc cluster #10

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Add the option to record start durations for crc cluster #10

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
106 changes: 65 additions & 41 deletions cmd/monictl.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,12 @@ func main() {
flag.IntVar(&numRepeats, "n", 5, "number of checks of CPU load")
var sleepLength int
flag.IntVar(&sleepLength, "s", 1, "sleep between repeats [in seconds]")
var repeatStarts bool
flag.BoolVar(&repeatStarts, "r", false, "repeatedly start and delete the cluster")
var pullSecretPath string
flag.StringVar(&pullSecretPath, "p", "", "path to pull secret file [needed if -r flag is set to true")
var bundlePath string
flag.StringVar(&bundlePath, "b", "", "path to CRC bundle [needed if -r flag is set to true")

flag.Parse()

Expand All @@ -46,7 +52,8 @@ func main() {
log.Fatalf("Unable to create directory: %s", dirPath)
}

if !tools.IsCRCRunning() {
// Require running cluster if not doing start/delete testing
if !repeatStarts && !tools.IsCRCRunning() {
fmt.Println("CRC VM is not running")
os.Exit(1)
}
Expand All @@ -56,62 +63,79 @@ func main() {
fmt.Println("Running monitoring tools with the following settings:")
fmt.Printf("Data directory: %s\n", dirPath)
fmt.Printf("Number of repeats: %d\n", numRepeats)
fmt.Printf("Pauses between repeats: %ds\n", sleepLength)
if !repeatStarts {
fmt.Printf("Pauses between repeats: %ds\n", sleepLength)
}
fmt.Printf("Logging into: %s\n", logFilePath)
fmt.Println("-------------")

cpuChan := make(chan error)
trafficChan := make(chan error)
crioChan := make(chan error)
nodeDesChan := make(chan error)
startChan := make(chan error)

// ================
// start collecting
// ================

// transmitted/received MiB on crc interface
trafficFile := filepath.Join(dirPath, "traffic.json")
go tools.RecordTraffic(trafficFile, numRepeats, sleepLength, trafficChan)
log.Println("going to record traffic going in/out of the VM")

// CPU usage by 'qemu' process
cpuFile := filepath.Join(dirPath, "cpu.json")
go tools.RecordHostCPUUsage(cpuFile, numRepeats, sleepLength, cpuChan)
log.Println("going to record CPU usage percentage attributed to qemu")

// CRI-O stats as reported by 'crictl'
go tools.GetCRIStatsFromVM(dirPath, crioChan)
log.Println("going to retrieve crictl stats from the CRC VM")

// Node Description
nodeDescription := filepath.Join(dirPath, "node.json")
go tools.GetNodeResource(nodeDescription, nodeDesChan)

if repeatStarts {
// start times for CRC cluster
startTimesFile := filepath.Join(dirPath, "startTimes.json")
go tools.RecordStartTimes(startTimesFile, numRepeats, startChan, pullSecretPath, bundlePath)
log.Println("going to record start times for CRC cluster")
} else {
// transmitted/received MiB on crc interface
trafficFile := filepath.Join(dirPath, "traffic.json")
go tools.RecordTraffic(trafficFile, numRepeats, sleepLength, trafficChan)
log.Println("going to record traffic going in/out of the VM")

// CPU usage by 'qemu' process
cpuFile := filepath.Join(dirPath, "cpu.json")
go tools.RecordHostCPUUsage(cpuFile, numRepeats, sleepLength, cpuChan)
log.Println("going to record CPU usage percentage attributed to qemu")

// CRI-O stats as reported by 'crictl'
go tools.GetCRIStatsFromVM(dirPath, crioChan)
log.Println("going to retrieve crictl stats from the CRC VM")

// Node Description
nodeDescription := filepath.Join(dirPath, "node.json")
go tools.GetNodeResource(nodeDescription, nodeDesChan)
}
// ================
// done collecting
// ================

if err := <-trafficChan; err != nil {
log.Fatalf("failed to record traffic flow %s", err)
} else {
log.Printf("recorded traffic (RX/TX) %d times at %d sec intervals", numRepeats, sleepLength)
}

if err := <-cpuChan; err != nil {
log.Fatalf("failed to record CPU percentage %s", err)
} else {
log.Printf("recorded CPU usage percentage %d times at %d sec intervals", numRepeats, sleepLength)
}

if err := <-crioChan; err != nil {
log.Fatalf("could not retrieve crictl stats: %s", err)
} else {
log.Println("crictl stats successfully retrieved")
}

if err := <-nodeDesChan; err != nil {
log.Fatalf("could not retrieve node description stats: %s", err)
if repeatStarts {
if err := <-startChan; err != nil {
log.Fatalf("failed to record start times: %s", err)
} else {
log.Printf("recorded start duration %d times", numRepeats)
}
} else {
log.Println("node description successfully retrieved")
if err := <-trafficChan; err != nil {
log.Fatalf("failed to record traffic flow %s", err)
} else {
log.Printf("recorded traffic (RX/TX) %d times at %d sec intervals", numRepeats, sleepLength)
}

if err := <-cpuChan; err != nil {
log.Fatalf("failed to record CPU percentage %s", err)
} else {
log.Printf("recorded CPU usage percentage %d times at %d sec intervals", numRepeats, sleepLength)
}

if err := <-crioChan; err != nil {
log.Fatalf("could not retrieve crictl stats: %s", err)
} else {
log.Println("crictl stats successfully retrieved")
}

if err := <-nodeDesChan; err != nil {
log.Fatalf("could not retrieve node description stats: %s", err)
} else {
log.Println("node description successfully retrieved")
}
}
}
107 changes: 107 additions & 0 deletions tools/tools.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -258,3 +258,110 @@ func parseNodeDescribeToJSON(path string) error {

return nil
}

func timedClusterStart(pullSecretPath string, bundlePath string) (string, error) {

cmd := exec.Command("crc", "start", "-b", bundlePath, "-p", pullSecretPath)
s := time.Now()
out, err := cmd.Output()
log.Printf("%s", string(out))
startDuration := time.Since(s)
if err != nil {
return "0h0m0.0s", err
}

return startDuration.String(), nil
}

func clusterDelete() error {

cmd := exec.Command("crc", "delete", "-f")
err := cmd.Run()

return err
}

func clusterCleanup() error {
cmd := exec.Command("crc", "cleanup")
out, err := cmd.Output()
if err != nil {
log.Printf("%s", string(out))
}

return err
}

func clusterSetup(bundlePath string) error {
cmd := exec.Command("crc", "setup", "-b", bundlePath)
out, err := cmd.Output()
if err != nil {
log.Printf("%s", string(out))
}

return err
}

func recordStartTimes(filename string, reps int, pullSecretPath string, bundlePath string) error {

// crc cleanup
err := clusterCleanup()
if err != nil {
log.Printf("Could not clean up the host: %s", err)
return err
}
homeDir, err := os.UserHomeDir()
if err != nil {
log.Printf("Could not retrieve user's home directory: %s", err)
return err
}
// remove config file
err = os.Remove(filepath.Join(homeDir, ".crc", "crc.json"))
if err != nil {
log.Printf("Could not remove crc config: %s", err)
return err
}
// crc setup
err = clusterSetup(bundlePath)
if err != nil {
log.Printf("Could not set up the host: %s", err)
return err
}

// collect data
var startTimes []string
for i := 0; i < reps; i++ {

startTime, _ := timedClusterStart(pullSecretPath, bundlePath)
startTimes = append(startTimes, startTime)

err1 := clusterDelete()
if err1 != nil {
log.Printf("Failed to delete cluster: %s", err1)
os.Exit(1)
}
}

// create CSV file and write data to it
f, err := os.Create(filename)
if err != nil {
log.Printf("could not create %s err: %s", filename, err)
}
defer f.Close()

jsonStartTimes, _ := json.MarshalIndent(startTimes, "", " ")
err = ioutil.WriteFile(filename, jsonStartTimes, 0644)
if err != nil {
log.Printf("Could not write data to %s", filename)
}

return nil
}

// RecordStartTimes returns a list of n cpu usage stats
// (in %) taken with nap breaks in between each poll
// filename : relative location of file to write into
// reps : number of times to record CPU usage
// c : channel used to report back to the main process
func RecordStartTimes(filename string, reps int, c chan error, pullSecretPath string, bundlePath string) {
c <- recordStartTimes(filename, reps, pullSecretPath, bundlePath)
}