diff --git a/integration_test/gce-testing-internal/gce/gce_testing.go b/integration_test/gce-testing-internal/gce/gce_testing.go index 0e8281680..620ea8750 100644 --- a/integration_test/gce-testing-internal/gce/gce_testing.go +++ b/integration_test/gce-testing-internal/gce/gce_testing.go @@ -604,11 +604,10 @@ func AssertMetricMissing(ctx context.Context, logger *log.Logger, vm *VM, metric return nil } -// hasMatchingLog looks in the logging backend for a log matching the given query, +// findMatchingLogs looks in the logging backend for logs matching the given query, // over the trailing time interval specified by the given window. -// Returns a boolean indicating whether the log was present in the backend, -// plus the first log entry found, or an error if the lookup failed. -func hasMatchingLog(ctx context.Context, logger *log.Logger, vm *VM, logNameRegex string, window time.Duration, query string) (bool, *cloudlogging.Entry, error) { +// Returns all the matching log entries found, or an error if the lookup failed. +func findMatchingLogs(ctx context.Context, logger *log.Logger, vm *VM, logNameRegex string, window time.Duration, query string) ([]*cloudlogging.Entry, error) { start := time.Now().Add(-window) t := start.Format(time.RFC3339) @@ -620,29 +619,25 @@ func hasMatchingLog(ctx context.Context, logger *log.Logger, vm *VM, logNameRege logClient, err := logClients.new(vm.Project) if err != nil { - return false, nil, fmt.Errorf("hasMatchingLog() failed to obtain logClient for project %v: %v", vm.Project, err) + return nil, fmt.Errorf("findMatchingLogs() failed to obtain logClient for project %v: %v", vm.Project, err) } it := logClient.Entries(ctx, logadmin.Filter(filter)) - found := false - var first *cloudlogging.Entry - // Loop through the iterator printing out each matching log entry. We could return true on the - // first match, but it's nice for debugging to print out all matches into the logs. + var matchingLogs []*cloudlogging.Entry + // Loop through the iterator printing out each matching log entry. + // It is helpful for debugging to print out all matches into the logs. for { entry, err := it.Next() if err == iterator.Done { break } if err != nil { - return false, nil, err + return nil, err } logger.Printf("Found matching log entry: %v", entry) - found = true - if first == nil { - first = entry - } + matchingLogs = append(matchingLogs, entry) } - return found, first, nil + return matchingLogs, nil } // WaitForLog looks in the logging backend for a log matching the given query, @@ -666,12 +661,13 @@ func shouldRetryHasMatchingLog(err error) bool { // found after some retries. func QueryLog(ctx context.Context, logger *log.Logger, vm *VM, logNameRegex string, window time.Duration, query string, maxAttempts int) (*cloudlogging.Entry, error) { for attempt := 1; attempt <= maxAttempts; attempt++ { - found, first, err := hasMatchingLog(ctx, logger, vm, logNameRegex, window, query) - if found { + matchingLogs, err := findMatchingLogs(ctx, logger, vm, logNameRegex, window, query) + found := len(matchingLogs) > 0 + if err == nil && found { // Success. - return first, nil + return matchingLogs[0], nil } - logger.Printf("Query returned found=%v, err=%v, attempt=%d", found, err, attempt) + logger.Printf("Query returned found=%t, matchingLogs=%v, err=%v, attempt=%d", found, matchingLogs, err, attempt) if err != nil && !shouldRetryHasMatchingLog(err) { // A non-retryable error. return nil, fmt.Errorf("QueryLog() failed: %v", err) @@ -682,27 +678,44 @@ func QueryLog(ctx context.Context, logger *log.Logger, vm *VM, logNameRegex stri return nil, fmt.Errorf("QueryLog() failed: %s not found, exhausted retries", logNameRegex) } -// AssertLogMissing looks in the logging backend for a log matching the given query -// and returns success if no data is found. To consider possible transient errors -// while querying the backend we make queryMaxAttemptsLogMissing query attempts. -func AssertLogMissing(ctx context.Context, logger *log.Logger, vm *VM, logNameRegex string, window time.Duration, query string) error { - for attempt := 1; attempt <= queryMaxAttemptsLogMissing; attempt++ { - found, _, err := hasMatchingLog(ctx, logger, vm, logNameRegex, window, query) +// QueryAllLogs looks in the logging backend for logs matching the given query, +// over the trailing time interval specified by the given window. +// Returns all the log entries found, or an error if no successful queries to the +// backend. To consider possible transient errors while querying the backend we +// make LogQueryMaxAttempts query attempts. +func QueryAllLogs(ctx context.Context, logger *log.Logger, vm *VM, logNameRegex string, window time.Duration, query string, maxAttempts int) ([]*cloudlogging.Entry, error) { + for attempt := 1; attempt <= LogQueryMaxAttempts; attempt++ { + matchingLogs, err := findMatchingLogs(ctx, logger, vm, logNameRegex, window, query) if err == nil { - if found { - return fmt.Errorf("AssertLogMissing(log=%q): %v failed: unexpectedly found data for log", query, err) - } - // Success - return nil + // Success. + return matchingLogs, nil } - logger.Printf("Query returned found=%v, err=%v, attempt=%d", found, err, attempt) + logger.Printf("Query returned matchingLogs=%v, err=%v, attempt=%d", matchingLogs, err, attempt) if err != nil && !shouldRetryHasMatchingLog(err) { // A non-retryable error. - return fmt.Errorf("AssertLogMissing() failed: %v", err) + return nil, fmt.Errorf("QueryAllLogs() failed: %v", err) } // found was false, or we hit a retryable error. time.Sleep(logQueryBackoffDuration) } + return nil, fmt.Errorf("QueryAllLogs() failed: %s no succesful queries to the backend, exhausted retries", logNameRegex) +} + +// AssertLogMissing looks in the logging backend for a log matching the given query +// and returns success if no data is found. To consider possible transient errors +// while querying the backend we make queryMaxAttemptsLogMissing query attempts. +func AssertLogMissing(ctx context.Context, logger *log.Logger, vm *VM, logNameRegex string, window time.Duration, query string) error { + matchingLogs, err := QueryAllLogs(ctx, logger, vm, logNameRegex, window, query, queryMaxAttemptsLogMissing) + if err == nil { + if len(matchingLogs) > 0 { + return fmt.Errorf("AssertLogMissing(log=%q): %v failed: unexpectedly found data for log", query, err) + } + // Success. + return nil + } + if err != nil { + return fmt.Errorf("AssertLogMissing() failed: %v", err) + } return fmt.Errorf("AssertLogMissing() failed: no successful queries to the backend for log %s, exhausted retries", logNameRegex) }