AI-118: use GitHub hosted runners; consolidate main and presubmit (#77)

Linux CI jobs are currently not picked up because we disabled legacy
CI Runners for open source repos. This repo has light usage, so just use
GitHub hosted runners instead.

Since we no longer need separate configuration for Linux and other OSs,
fold Linux jobs into the matrix strategy where it makes sense.

This PR also squashes the main and presubmit workflows. There's no
good reason for them to be separate.

Signed-off-by: Jay Conrod <jay@engflow.com>

---------

Signed-off-by: Jay Conrod <jay@engflow.com>

Author: jayconrod

.bazelrc

@@ -58,10 +58,13 @@ build:engflow_bes --bes_results_url=https://opal.cluster.engflow.com/invocations
 build:engflow_bes --bes_instance_name=auth
 build:engflow_bes --bes_lifecycle_events
 
-build:engflow --config=engflow_common
-build:engflow --config=engflow_bes
+build:engflow_cache --config=engflow_common
+build:engflow_cache --config=engflow_bes
+build:engflow_cache --remote_cache=grpcs://opal.cluster.engflow.com
+build:engflow_cache --remote_instance_name=auth
+
+build:engflow --config=engflow_cache
 build:engflow --remote_executor=grpcs://opal.cluster.engflow.com
-build:engflow --remote_instance_name=auth
 
 # To authenticate with the clusters above, either add flags to this
 # .bazelrc.user file or to your $HOME/.bazelrc. These files may contain

.github/workflows/main.yml

@@ -12,17 +12,18 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# Postsubmit checks that run on the `main` branch after merge.
 name: "main"
 
 on:
   workflow_dispatch:
+  pull_request:
   push:
     branches:
       - main
 
 concurrency:
   group: ${{ github.workflow }}.${{ github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 env:
   # Recommended here: https://github.com/bazelbuild/bazelisk/issues/88#issuecomment-625178467
@@ -33,62 +34,30 @@ env:
 
 jobs:
   # TODO(OND-616): use remote execution and caching for all CI Runner jobs.
-  bazel-builder:
-    runs-on:
-      - self-hosted
-      - os=linux
-      - arch=x64
-      - "engflow-cluster=glass"
-      - "engflow-container-image=docker://645088952840.dkr.ecr.eu-west-1.amazonaws.com/engflow-ci/debian12-dind-x64@sha256:0a8ee2164b24a0bcfffa95a071967e1495cd524a8eb52a1bcf168e52199e86b0"
-      - "engflow-pool=ci_sysbox_x64"
-      - "engflow-runtime=sysbox-runc"
-      - "engflow-runner-id=${{ github.repository_id }}_bazel-builder_${{ github.run_id }}_${{ github.run_number }}_${{ github.run_attempt }}"
-      - "engflow-run-id=${{ github.run_id }}"
+  build-and-test:
+    runs-on: ${{ matrix.runner }}
     timeout-minutes: 30
-
-    env:
-      ARCH: "x64"
-      OS: "linux"
-
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: Log in
-        run: infra/login.sh
-
-      - name: Run all tests
-        if: success()
-        run: |
-          bazel test --config=noninteractive --config=engflow --config=remote_linux_x64 //...
-
-      - name: Log out
-        run: infra/logout.sh
-
-  golang-builder:
-    runs-on:
-      - self-hosted
-      - "purpose=github-actions"
-      - "os=${{ matrix.os }}"
-      - "arch=${{ matrix.arch }}"
-      - "os_distribution=${{ matrix.os_distribution }}"
-      - "os_version=${{ matrix.os_version }}"
-      - "revision=${{ matrix.revision }}"
-    timeout-minutes: 10
     strategy:
       fail-fast: false
       matrix:
+        # Use remote execution on Linux, remote cache for others. macOS and Windows
+        # are sensitive to the C++ toolchain installed on the runner, and it may not
+        # match what's installed on the remote worker.
         include:
+          - os: "linux"
+            arch: "x64"
+            runner: "ubuntu-latest"
+            ef_config: "engflow"
+
           - os: "macos"
             arch: "arm64"
-            os_distribution: "sonoma"
-            os_version: "14"
-            revision: "c12cc4655257fcf7da86fe06e87542b7e6814a1f"
+            runner: "macos-14"
+            ef_config: "engflow_cache"
 
           - os: "windows"
             arch: "x64"
-            os_distribution: "server"
-            os_version: "2022"
-            revision: "c12cc4655257fcf7da86fe06e87542b7e6814a1f"
+            runner: "windows-2022"
+            ef_config: "engflow_cache"
 
     env:
       ARCH: "${{ matrix.arch }}"
@@ -97,66 +66,29 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
-      - name: Log in
+      - name: "Log in"
         shell: bash
         run: infra/login.sh
 
-      - name: Run all tests
+      - name: "Bazel tests"
         if: success()
         shell: bash
         run: |
-          # TODO(CUS-345): Enable remote execution
-          bazel run --config=noninteractive --config=engflow_bes @rules_go//go -- test ./...
-          bazel run --config=noninteractive --config=engflow_bes @rules_go//go -- clean -cache -modcache
-
-      - name: Log out
-        shell: bash
-        run: infra/logout.sh
-
-  golang-builder-ci-runners:
-    runs-on:
-      - self-hosted
-      - os=linux
-      - arch=x64
-      - "engflow-cluster=glass"
-      - "engflow-container-image=docker://645088952840.dkr.ecr.eu-west-1.amazonaws.com/engflow-ci/debian12-dind-x64@sha256:0a8ee2164b24a0bcfffa95a071967e1495cd524a8eb52a1bcf168e52199e86b0"
-      - "engflow-pool=ci_sysbox_x64"
-      - "engflow-runtime=sysbox-runc"
-      - "engflow-runner-id=${{ github.repository_id }}_golang-builder-ci-runners_${{ github.run_id }}_${{ github.run_number }}_${{ github.run_attempt }}"
-      - "engflow-run-id=${{ github.run_id }}"
-    timeout-minutes: 10
-
-    env:
-      ARCH: "x64"
-      OS: "linux"
-
-    steps:
-      - uses: actions/checkout@v4
+          bazel test --config=noninteractive --config=${{ matrix.ef_config }} --config=remote_${{ matrix.os }}_${{ matrix.arch }} //...
 
-      - name: Log in
-        run: infra/login.sh
-
-      - name: Run all tests
+      - name: "Go tests"
         if: success()
+        shell: bash
         run: |
-          # TODO(CUS-345): Enable remote execution
           bazel run --config=noninteractive --config=engflow_bes @rules_go//go -- test ./...
           bazel run --config=noninteractive --config=engflow_bes @rules_go//go -- clean -cache -modcache
 
       - name: Log out
+        shell: bash
         run: infra/logout.sh
 
   copyright-headers-check:
-    runs-on:
-      - self-hosted
-      - os=linux
-      - arch=x64
-      - "engflow-cluster=glass"
-      - "engflow-container-image=docker://645088952840.dkr.ecr.eu-west-1.amazonaws.com/engflow-ci/debian12-dind-x64@sha256:0a8ee2164b24a0bcfffa95a071967e1495cd524a8eb52a1bcf168e52199e86b0"
-      - "engflow-pool=ci_sysbox_x64"
-      - "engflow-runtime=sysbox-runc"
-      - "engflow-runner-id=${{ github.repository_id }}_copyright-headers-check_${{ github.run_id }}_${{ github.run_number }}_${{ github.run_attempt }}"
-      - "engflow-run-id=${{ github.run_id }}"
+    runs-on: ubuntu-latest
     timeout-minutes: 10
 
     env: