Skip redundant Vitess rebuilds and record per-task timings

ansible/ansible.cfg

@@ -0,0 +1,16 @@
+# Copyright 2021 The Vitess Authors.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#    http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+[defaults]
+# Emit a per-task timing summary at the end of every play. The output is
+# captured in each execution's exec-stdout.log and lets us see where the wall
+# clock actually goes (build vs cluster start vs sysbench prepare vs run).
+callbacks_enabled = ansible.builtin.profile_tasks

ansible/roles/vitess_build/tasks/main.yml

@@ -16,33 +16,86 @@
   become: true
   become_user: root
   block:
-    - name: Fetch Updated Vitess
-      ansible.builtin.include_tasks: install_vitess.yml
-    - name: Tmp directory gopath
-      become: true
-      become_user: root
-      ansible.builtin.file:
-        state: directory
-        path: "{{ item }}"
-        mode: "0755"
-      with_items:
-        - /root/tmp
-
-    - name: Remove Vitess binaries
-      ansible.builtin.shell: |
-        rm -f /usr/local/bin/vt*
-    - name: Install Vitess Binaries
-      ansible.builtin.shell: |
-        export TMPDIR=/root/tmp
-        cd /go/src/vitess.io/vitess
-        NOVTADMINBUILD=1 make install PREFIX=/usr/local VTROOT=/go/src/vitess.io/vitess
+    # Building Vitess from source is the single most expensive step of a
+    # benchmark run (several minutes). When the same commit is benchmarked
+    # across several workloads, the cluster is torn down and recreated between
+    # each workload even though the exact same binaries are needed. We record
+    # the ref+Go version that is currently installed in a marker file and skip
+    # the fetch/build when it already matches (and the binaries are still
+    # present). The teardown no longer removes the vt* binaries, so they survive
+    # between workloads of the same commit.
+    - name: Read the installed Vitess build marker
+      ansible.builtin.slurp:
+        src: /usr/local/vitess-build.ref
+      register: vitess_build_marker
+      failed_when: false
       changed_when: false
 
-    - name: Install Vitess Other Binaries
-      ansible.builtin.shell: |
-        cd /go/src/vitess.io/vitess
-        cp bin/vtctl /usr/local/bin/
-      changed_when: false
+    - name: Check whether the vtgate binary is present
+      ansible.builtin.stat:
+        path: /usr/local/bin/vtgate
+      register: vitess_vtgate_binary
+
+    - name: Determine whether the installed Vitess build can be reused
+      ansible.builtin.set_fact:
+        vitess_build_current: >-
+          {{
+            vitess_vtgate_binary.stat.exists
+            and vitess_build_marker.content is defined
+            and (vitess_build_marker.content | b64decode | trim)
+                == (vitess_git_version | default('') ~ '-' ~ golang_gover | default(''))
+          }}
+
+    - name: Report whether the Vitess build is reused
+      ansible.builtin.debug:
+        msg: >-
+          Vitess build {{ vitess_git_version | default('') }}-{{ golang_gover | default('') }}
+          reuse={{ vitess_build_current }}
+
+    - name: Build and install Vitess
+      when: not (vitess_build_current | bool)
+      block:
+        # Remove the marker first so an interrupted build cannot leave a stale
+        # marker that would make the next run wrongly skip the build.
+        - name: Invalidate the Vitess build marker before building
+          ansible.builtin.file:
+            path: /usr/local/vitess-build.ref
+            state: absent
+
+        - name: Fetch Updated Vitess
+          ansible.builtin.include_tasks: install_vitess.yml
+        - name: Tmp directory gopath
+          become: true
+          become_user: root
+          ansible.builtin.file:
+            state: directory
+            path: "{{ item }}"
+            mode: "0755"
+          with_items:
+            - /root/tmp
+
+        - name: Remove Vitess binaries
+          ansible.builtin.shell: |
+            rm -f /usr/local/bin/vt*
+        - name: Install Vitess Binaries
+          ansible.builtin.shell: |
+            export TMPDIR=/root/tmp
+            cd /go/src/vitess.io/vitess
+            NOVTADMINBUILD=1 make install PREFIX=/usr/local VTROOT=/go/src/vitess.io/vitess
+          changed_when: false
+
+        - name: Install Vitess Other Binaries
+          ansible.builtin.shell: |
+            cd /go/src/vitess.io/vitess
+            cp bin/vtctl /usr/local/bin/
+          changed_when: false
+
+        # Record the ref+Go version only after a fully successful install.
+        - name: Record the installed Vitess build marker
+          ansible.builtin.copy:
+            dest: /usr/local/vitess-build.ref
+            content: "{{ vitess_git_version | default('') }}-{{ golang_gover | default('') }}"
+            mode: "0644"
 
 - name: Disbale AppArmor
   block:

ansible/roles/vtctld/tasks/clean.yml

@@ -76,5 +76,7 @@
     daemon_reload: true
   failed_when: false
 
-- name: Remove binary
-  ansible.builtin.shell: rm -f $(which vtctld vtctl vtctlclient vtctldclient)
+# NOTE: the vt* binaries are intentionally left in place so they can be reused
+# across workloads of the same commit (see roles/vitess_build/tasks/main.yml,
+# which rebuilds only when the installed ref+Go version changes). A rebuild
+# overwrites them, so nothing accumulates.

ansible/roles/vtgate/tasks/clean.yml

@@ -33,5 +33,7 @@
   ansible.builtin.systemd:
     daemon_reload: true
 
-- name: Remove binary
-  ansible.builtin.shell: rm -f $(which vtgate)
+# NOTE: the vtgate binary is intentionally left in place so it can be reused
+# across workloads of the same commit (see roles/vitess_build/tasks/main.yml,
+# which rebuilds only when the installed ref+Go version changes). A rebuild
+# overwrites it, so nothing accumulates.

ansible/roles/vttablet/tasks/clean.yml

@@ -35,5 +35,7 @@
   ansible.builtin.systemd:
     daemon_reload: true
 
-- name: Remove binary
-  ansible.builtin.shell: rm -f $(which vttablet mysqlctld mysqlctl)
+# NOTE: the vttablet/mysqlctl binaries are intentionally left in place so they
+# can be reused across workloads of the same commit (see
+# roles/vitess_build/tasks/main.yml, which rebuilds only when the installed
+# ref+Go version changes). A rebuild overwrites them, so nothing accumulates.

go/infra/ansible/ansible.go

@@ -116,6 +116,12 @@ func Run(c *Config) error {
 			execute.WithShowDuration(),
 			execute.WithWrite(c.stdout),
 			execute.WithWriteError(c.stderr),
+			// ansible-playbook only discovers an ansible.cfg from the current
+			// working directory, which is the api process' cwd rather than the
+			// per-execution playbook directory. Point it explicitly at the
+			// ansible.cfg copied alongside the playbook so its callbacks (e.g.
+			// profile_tasks) are enabled.
+			execute.WithEnvVar("ANSIBLE_CONFIG", path.Join(c.RootDir, "ansible.cfg")),
 		),
 	}