examples: Poll for expected server logs in test (#8871)

Fixes: #8046

## Problem

The test is flaky due to a race condition in the driver script.
Currently, the script waits for the client to exit and immediately
validates the server output:
https://github.com/grpc/grpc-go/blob/49e224f832c21df2cd3fe833189c8a50244f0eee/examples/examples_test.sh#L195-L219

In the graceful shutdown example, the expected log is written only after
the shutdown is complete:
https://github.com/grpc/grpc-go/blob/49e224f832c21df2cd3fe833189c8a50244f0eee/examples/features/gracefulstop/server/main.go#L99

If the test driver validates the server output after the client stream
closes but before the server logs the expected message, the test fails.

## Fix

This PR updates the test driver to poll for the expected log in the
server output, instead of checking it only once. Additionally, the
shutdown of the test server is blocked until either the server logs a
failure or a success message.
## Validation

To reproduce the failure, I added a 1s sleep just before the line
`log.Println("Server stopped gracefully.")` in the server code. I
verified that the test no longer fails with this fix, even with the
artificial delay.

RELEASE NOTES: N/A

Author: arjan-bal

examples/examples_test.sh

@@ -206,7 +206,18 @@ for example in ${EXAMPLES[@]}; do
     # Check server log for expected output if expecting an
     # output
     if [ -n "${EXPECTED_SERVER_OUTPUT[$example]}" ]; then
-        if ! grep -q "${EXPECTED_SERVER_OUTPUT[$example]}" $SERVER_LOG; then
+        found=false
+
+        # Poll for up to 10 seconds.
+        for i in {1..10}; do
+            if grep -q "${EXPECTED_SERVER_OUTPUT[$example]}" "$SERVER_LOG"; then
+                found=true
+                break
+            fi
+            sleep 1
+        done
+
+        if [ "$found" = "false" ]; then
             fail "server log missing output: ${EXPECTED_SERVER_OUTPUT[$example]}
             got server log:
             $(cat $SERVER_LOG)

examples/features/gracefulstop/server/main.go

@@ -85,6 +85,7 @@ func main() {
 	s := grpc.NewServer()
 	ss := &server{streamStart: make(chan struct{})}
 	pb.RegisterEchoServer(s, ss)
+	serverStopped := make(chan struct{}, 1)
 
 	go func() {
 		<-ss.streamStart // wait until server streaming starts
@@ -93,13 +94,24 @@ func main() {
 		timer := time.AfterFunc(10*time.Second, func() {
 			log.Println("Server couldn't stop gracefully in time. Doing force stop.")
 			s.Stop()
+			// Unblock the main function.
+			select {
+			case serverStopped <- struct{}{}:
+			default:
+			}
 		})
 		defer timer.Stop()
 		s.GracefulStop() // gracefully stop server after in-flight server streaming rpc finishes
 		log.Println("Server stopped gracefully.")
+		// Unblock the main function.
+		select {
+		case serverStopped <- struct{}{}:
+		default:
+		}
 	}()
 
 	if err := s.Serve(lis); err != nil {
 		log.Fatalf("failed to serve: %v", err)
 	}
+	<-serverStopped
 }