Test sharding

.github/workflows/test.yml

@@ -55,34 +55,62 @@ jobs:
         submodules: true
         persist-credentials: false
     - name: Checkout p/examples
-      if: matrix.testTarget == 'pulumiExamples'
       uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
       with:
         repository: pulumi/examples
         path: p-examples
     - name: Setup tools
       uses: ./.github/actions/setup-tools
-      with:
-        tools: pulumictl, pulumicli, ${{ matrix.language }}
     - name: Prepare local workspace
       run: make prepare_local_workspace
     - name: Download bin
       uses: ./.github/actions/download-bin
-    - name: Download SDK
+    - name: Download nodejs SDK
+      uses: ./.github/actions/download-sdk
+      with:
+        language: nodejs
+    - name: Restore makefile progress
+      run: make --touch provider schema build_nodejs
+    - name: Install dependencies
+      run: make install_nodejs_sdk
+    - name: Download python SDK
       uses: ./.github/actions/download-sdk
       with:
-        language: ${{ matrix.language }}
+        language: python
     - name: Restore makefile progress
-      run: make --touch provider schema build_${{ matrix.language }}
+      run: make --touch provider schema build_python
+    - name: Install dependencies
+      run: make install_python_sdk
+    - name: Download dotnet SDK
+      uses: ./.github/actions/download-sdk
+      with:
+        language: dotnet
+    - name: Restore makefile progress
+      run: make --touch provider schema build_dotnet
+    - name: Install dependencies
+      run: make install_dotnet_sdk
+    - name: Download go SDK
+      uses: ./.github/actions/download-sdk
+      with:
+        language: go
+    - name: Restore makefile progress
+      run: make --touch provider schema build_go
+    - name: Install dependencies
+      run: make install_go_sdk
+    - name: Download java SDK
+      uses: ./.github/actions/download-sdk
+      with:
+        language: java
+    - name: Restore makefile progress
+      run: make --touch provider schema build_java
+    - name: Install dependencies
+      run: make install_java_sdk
     - name: Update path
       run: echo "${{ github.workspace }}/bin" >> "$GITHUB_PATH"
     - name: Install Python deps
-      if: matrix.language == 'python'
       run: |-
         pip3 install virtualenv==20.0.23
         pip3 install pipenv
-    - name: Install dependencies
-      run: make install_${{ matrix.language}}_sdk
     - name: Install gotestfmt
       uses: GoTestTools/gotestfmt-action@v2
       with:
@@ -97,19 +125,26 @@ jobs:
         role-duration-seconds: 7200
         role-session-name: aws@githubActions
         role-to-assume: ${{ secrets.AWS_CI_ROLE_ARN }}
-    - name: Run tests
-      if: matrix.testTarget == 'local'
-      run: cd examples && go test -v -count=1 -cover -timeout 2h -tags=${{ matrix.language }} -skip TestPulumiExamples -parallel 4 .
-    - name: Run pulumi/examples tests
-      if: matrix.testTarget == 'pulumiExamples'
-      run: cd examples && go test -v -count=1 -cover -timeout 2h -tags=${{ matrix.language }} -run TestPulumiExamples -parallel 4 .
+    - name: Generate shard
+      run: make shard
+      env:
+        SHARD_TOTAL: ${{ matrix.total }}
+        SHARD_INDEX: ${{ matrix.index }}
+    - name: Run integration tests
+      run: make test_shard
     strategy:
-      fail-fast: false
-      matrix:
-        language:
-        - nodejs
-        - python
-        - dotnet
-        - go
-        - java
-        testTarget: [local]
+       fail-fast: false
+       matrix:
+         total:
+         - 10
+         index:
+         - 0
+         - 1
+         - 2
+         - 3
+         - 4
+         - 5
+         - 6
+         - 7
+         - 8
+         - 9

Makefile

@@ -393,6 +393,13 @@ provider_dist-darwin-arm64: bin/$(PROVIDER)-v$(VERSION_GENERIC)-darwin-arm64.tar
 provider_dist-windows-amd64: bin/$(PROVIDER)-v$(VERSION_GENERIC)-windows-amd64.tar.gz
 provider_dist: provider_dist-linux-amd64 provider_dist-linux-arm64 provider_dist-darwin-amd64 provider_dist-darwin-arm64 provider_dist-windows-amd64
 .PHONY: provider_dist-linux-amd64 provider_dist-linux-arm64 provider_dist-darwin-amd64 provider_dist-darwin-arm64 provider_dist-windows-amd64 provider_dist
-
+# shard computes tests to run and modifies the CI runner's environment.
+shard:
+	@(cd examples && go run github.com/blampe/shard@latest --total $(SHARD_TOTAL) --index $(SHARD_INDEX) --output env) >> "$(GITHUB_ENV)"
+
+# test_shard runs the tests specified by a regex contained in $SHARD_TESTS for paths $SHARD_PATHS.
+test_shard:
+	cd examples && \
+		go test -tags=all -v -count=1 -coverprofile="coverage.txt" -coverpkg=./... -timeout 3h -parallel ${TESTPARALLELISM} -run "$(SHARD_TESTS)" $(SHARD_PATHS)
 # Permit providers to extend the Makefile with provider-specific Make includes.
 include $(wildcard .mk/*.mk)