test(storage): add support for Storage TestBench (#210)

The Google Cloud Storage team provides a project called "Storage
Testbench" that is used for failure testing in Google Cloud Storage
clients. Storage Testbench can be run using docker and provides a
partial GCS implementation that can be scripted to produce failures.

This PR adds the ability to create tests that target Storage Testbench
and a trivial test to prove that it works.

- Creates a new test tag `storage-testbench`
- Adds a new GitHub workflow that starts the docker image and runs tests
with that tag
- Adds test utilities to work with Storage Testbench
- Documents how to run Storage Testbench locally

---------

Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
Co-authored-by: Nate Bosch <nbosch1@gmail.com>

Author: 3 people

.github/workflows/dart_checks.yaml

@@ -43,8 +43,8 @@ jobs:
       - name: Format
         run: dart format --output=none --set-exit-if-changed .
 
-  tests:
-    name: Tests
+  unit_tests:
+    name: Unit Tests
     runs-on: ubuntu-latest
     strategy:
       matrix:
@@ -57,6 +57,18 @@ jobs:
       - run: dart pub get
       - run: dart test -p ${{ matrix.platform }} .
 
+  storage_testbench_tests:
+    name: Storage Testbench Tests
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
+      - uses: dart-lang/setup-dart@e51d8e571e22473a2ddebf0ef8a2123f0ab2c02c
+        with:
+          sdk: ${{ env.DART_VERSION }}
+      - run: dart pub get
+      - run: docker run -d --rm -p 9000:9000 -p 8888:8888 gcr.io/cloud-devrel-public-resources/storage-testbench:latest
+      - run: dart test -P storage-testbench .
+
   generator:
     name: Generators
     runs-on: ubuntu-latest

DEVELOPER_GUIDE.md

@@ -59,6 +59,27 @@ To run these tests locally (they are automatically run for PRs using
 *   **Missing Scopes:** If a test fails with `insufficient_scope`, check the error message for the required scope URL and re-authenticate with that scope.
 *   **Disabled APIs:** If a test fails with a `ForbiddenException` stating an API has not been used, follow the URL in the error message to enable the API in the Google Cloud Console.
 
+### Running against Storage Testbench
+
+Some integration tests in `package:google_cloud_storage` use the
+[Storage Testbench][]. These tests are tagged with
+`@Tags(['storage-testbench'])` and are not run by default, i.e., `dart test`
+will not run them.
+
+To run these tests locally (they are automatically run for PRs using a
+[GitHub workflow](.github/workflows/dart_checks.yaml)):
+
+1.  **Start Storage Testbench:**
+    ```bash
+    $ docker run -d --rm -p 9000:9000 -p 8888:8888 \
+        gcr.io/cloud-devrel-public-resources/storage-testbench:latest
+    ```
+
+2.  **Run the tests:**
+    ```bash
+    $ dart test . -P storage-testbench
+    ```
+
 ## Pull Requests
 
 * PRs should follow [Conventional Commits][]
@@ -131,3 +152,4 @@ go run github.com/googleapis/librarian/cmd/librarian@main update conformance goo
 
 [Google Cloud Console]: https://console.cloud.google.com/
 [Conventional Commits]: https://www.conventionalcommits.org/
+[Storage Testbench]: https://github.com/googleapis/storage-testbench

dart_test.yaml

@@ -21,7 +21,8 @@ concurrency: 1
 tags:
   google-cloud:
   firebase-emulator:
-exclude_tags: google-cloud || firebase-emulator
+  storage-testbench:
+exclude_tags: google-cloud || firebase-emulator || storage-testbench
 
 presets:
   google-cloud:
@@ -32,3 +33,6 @@ presets:
     platforms: [vm]
   firebase-emulator:
     include_tags: firebase-emulator
+  storage-testbench:
+    include_tags: storage-testbench
+    platforms: [vm]

generated/google_cloud_rpc/lib/src/exceptions.dart

@@ -201,8 +201,14 @@ final class ServiceException implements Exception {
     }
 
     final Status status;
-    if (json is Map<String, dynamic> && json['error'] is Map<String, dynamic>) {
-      status = Status.fromJson(json['error'] as Map<String, dynamic>);
+    // The Storage Testbench sometimes returns non-conformant `Status` responses
+    // with an object for the `'message'` field.
+    // See https://github.com/googleapis/storage-testbench
+    if (json case {'error': final Map<String, Object?> error}
+        when error['code'] is int? &&
+            error['message'] is String? &&
+            error['details'] is List<Object?>?) {
+      status = Status.fromJson(error);
     } else {
       return ServiceException._fromDecodedResponse(
         responseBody,

pkgs/google_cloud_storage/dart_test.yaml

@@ -21,7 +21,8 @@ concurrency: 1
 tags:
   google-cloud:
   firebase-emulator:
-exclude_tags: google-cloud || firebase-emulator
+  storage-testbench:
+exclude_tags: google-cloud || firebase-emulator || storage-testbench
 
 presets:
   google-cloud:
@@ -32,3 +33,6 @@ presets:
     platforms: [vm]
   firebase-emulator:
     include_tags: firebase-emulator
+  storage-testbench:
+    include_tags: storage-testbench
+    platforms: [vm]

pkgs/google_cloud_storage/test/download_object_test.dart

@@ -12,9 +12,6 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-@TestOn('vm')
-library;
-
 import 'dart:convert';
 import 'dart:io';
 import 'dart:typed_data';
@@ -28,8 +25,54 @@ import 'test_utils.dart';
 
 void main() async {
   late Storage storage;
+  late RetryTestHttpClient client;
+  late RetryTestCreator retryCreator;
 
   group('download object', () {
+    group('storage-testbench', tags: ['storage-testbench'], () {
+      setUp(() async {
+        client = RetryTestHttpClient(http.Client());
+        storage = Storage(
+          projectId: 'test-project',
+          apiEndpoint: 'localhost:9000',
+          useAuthWithCustomEndpoint: false,
+          client: client,
+        );
+        retryCreator = RetryTestCreator(http.Client());
+      });
+
+      tearDown(() async {
+        await retryCreator.close();
+        storage.close();
+      });
+
+      test('two 503s and then success', () async {
+        final id = await retryCreator.createRetryTest({
+          'instructions': {
+            'storage.objects.get': ['return-503', 'return-503'],
+          },
+          'transport': 'HTTP',
+        });
+
+        final bucketName = await createBucketWithTearDown(
+          storage,
+          'dl_obj_empty',
+        );
+
+        await storage.uploadObject(
+          bucketName,
+          'object1',
+          [],
+          ifGenerationMatch: BigInt.zero,
+        );
+
+        client.retryTestId = id;
+        final data = await storage.downloadObject(bucketName, 'object1');
+        client.retryTestId = null;
+        expect(data, isEmpty);
+      });
+    });
+
     group('google-cloud', tags: ['google-cloud'], () {
       setUp(() async {
         storage = Storage();
@@ -223,27 +266,26 @@ void main() async {
         );
       });
     });
+  });
 
-    test('hash failure', () async {
-      var count = 0;
-      final mockClient = MockClient((request) async {
-        count++;
-        final headers = {'content-type': 'text/plain; charset=UTF-8'};
-        if (count == 1) {
-          headers['x-goog-hash'] = 'crc32c=/BAD';
-        } else if (count == 2) {
-          headers['x-goog-hash'] = 'md5=/BAD';
-        } else {
-          headers['x-goog-hash'] =
-              'crc32c=/mzx3A==,md5=7Qdih1MuhjZehB6Sv8UNjA==';
-        }
-        return http.Response('Hello World!', 200, headers: headers);
-      });
+  test('hash failure', () async {
+    var count = 0;
+    final mockClient = MockClient((request) async {
+      count++;
+      final headers = {'content-type': 'text/plain; charset=UTF-8'};
+      if (count == 1) {
+        headers['x-goog-hash'] = 'crc32c=/BAD';
+      } else if (count == 2) {
+        headers['x-goog-hash'] = 'md5=/BAD';
+      } else {
+        headers['x-goog-hash'] = 'crc32c=/mzx3A==,md5=7Qdih1MuhjZehB6Sv8UNjA==';
+      }
+      return http.Response('Hello World!', 200, headers: headers);
+    });
 
-      final storage = Storage(client: mockClient, projectId: 'fake project');
+    final storage = Storage(client: mockClient, projectId: 'fake project');
 
-      final actualData = await storage.downloadObject('bucket', 'object');
-      expect(actualData, utf8.encode('Hello World!'));
-    });
+    final actualData = await storage.downloadObject('bucket', 'object');
+    expect(actualData, utf8.encode('Hello World!'));
   });
 }

pkgs/google_cloud_storage/test/test_utils.dart

@@ -12,9 +12,11 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+import 'dart:convert';
 import 'dart:math';
 
 import 'package:google_cloud_storage/google_cloud_storage.dart';
+import 'package:http/http.dart' as http;
 import 'package:test/test.dart';
 
 const _bucketChars = 'abcdefghijklmnopqrstuvwxyz0123456789';
@@ -79,3 +81,60 @@ Future<String> createBucketWithTearDown(
   );
   return bucketName;
 }
+
+/// An HTTP client that can add a `x-retry-test-id` header to requests for
+/// testing with Storage Testbench.
+///
+/// See https://github.com/googleapis/storage-testbench.
+final class RetryTestHttpClient extends http.BaseClient {
+  final http.Client _client;
+  String? retryTestId;
+
+  RetryTestHttpClient(this._client);
+
+  @override
+  Future<http.StreamedResponse> send(http.BaseRequest originalRequest) {
+    if (retryTestId case final id?) {
+      originalRequest.headers['x-retry-test-id'] = id;
+    }
+    return _client.send(originalRequest);
+  }
+
+  @override
+  void close() => _client.close();
+}
+
+/// A client that can create Storage Testbench Retry Tests.
+///
+/// See https://github.com/googleapis/storage-testbench?tab=readme-ov-file#creating-a-new-retry-test
+final class RetryTestCreator {
+  final http.Client _client;
+  final List<String> _retryTests = [];
+
+  RetryTestCreator(this._client);
+
+  /// Creates a new retry test and returns the test id that can be used in the
+  /// `x-retry-test-id` header.
+  ///
+  /// The [test] object is a JSON-serializable object that describes the retry
+  /// test. See
+  /// https://github.com/googleapis/storage-testbench?tab=readme-ov-file#creating-a-new-retry-test
+  Future<String> createRetryTest(Object test) async {
+    final responseBody = (await _client.post(
+      Uri.http('localhost:9000', '/retry_test'),
+      headers: {'Content-Type': 'application/json'},
+      body: jsonEncode(test),
+    )).body;
+    final id =
+        (jsonDecode(responseBody) as Map<String, dynamic>)['id'] as String;
+    _retryTests.add(id);
+    return id;
+  }
+
+  Future<void> close() async {
+    for (var id in _retryTests) {
+      await _client.delete(Uri.http('localhost:9000', '/retry_test/$id'));
+    }
+    _client.close();
+  }
+}