Fix CI pre-commit check failure. (#1195)

* Fix CI and add memory monitoring

* minor scripts improvements

* Update .github/workflows/pre-commit.yml

resolve boyue's comment.

Co-authored-by: Boyue Li <[email protected]>

* make monitor process non-critical upon failure

---------

Co-authored-by: Boyue Li <[email protected]>

Author: raytomato

.github/scripts/monitor_memory.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+# Simple memory monitor that prints to console periodically.
+# Designed for GitHub Actions monitoring.
+
+INFO="\033[90m"
+RESET="\033[0m"
+
+echo -e "${INFO}====== Starting memory monitor at $(date) ======${RESET}"
+
+while true; do
+  # Get current timestamp
+  timestamp=$(date '+%Y-%m-%d %H:%M:%S')
+
+  echo ""
+  echo -e "${INFO}=== Memory Check: $timestamp ===${RESET}"
+
+  # Print simplified memory stats.
+  free -h | grep "Mem:" | awk "{printf \"${INFO}Memory: %s used, %s free, %s total\n${RESET}\", \$3, \$4, \$2}"
+
+  # Print memory usage percentage.
+  mem_total=$(free | grep Mem | awk '{print $2}')
+  mem_used=$(free | grep Mem | awk '{print $3}')
+  mem_percent=$(awk "BEGIN {printf \"%.1f\", $mem_used/$mem_total*100}")
+  echo -e "${INFO}Memory usage: $mem_percent%${RESET}"
+
+  # Sleep for 30 seconds
+  sleep 30
+done

.github/workflows/pre-commit.yml

@@ -16,5 +16,21 @@ jobs:
       # TODO(markblee): Remove gcp,vertexai_tensorboard from CI. (needed by pytype)
       - run: pip install '.[core,dev,grain,gcp,vertexai_tensorboard,open_api]'
       # pylint uses approx 12GB of memory during this run, look into split to decrease?
-      - run: pre-commit run --all-files
+      - run: |
+          # Start memory monitor as a background process.
+          { .github/scripts/monitor_memory.sh & } || true
+          MONITOR_PID=$!
+          # A short sleep to wait for monitor process to start.
+          sleep 1
+
+          # Start pre-commit check.
+          echo "====== Starting pre-commit... ======"
+          pre-commit run --all-files
+          echo "====== pre-commit completed. ======"
+
+          # Clean up memory monitor process.
+          if kill -0 $MONITOR_PID 2>/dev/null; then
+            echo "Manually stopping monitor process..."
+            kill $MONITOR_PID || true
+          fi
       - run: pytype -j auto .

.pylintrc

@@ -28,7 +28,7 @@ persistent=no
 # load-plugins=pylint.extensions.docparams
 
 # Use multiple processes to speed up Pylint.
-jobs=4
+jobs=1
 
 # Allow loading of arbitrary C extensions. Extensions are imported into the
 # active Python interpreter and may run arbitrary code.