[server][da-vinci]: Raise disk health check thread priority (linkedin#2911)

Run the disk health check background thread at priority 8 so it stays
responsive when other server components are under load without using max
priority.

Add test coverage to verify the created health check thread priority.

Testing: ./gradlew :clients:da-vinci-client:test --tests
com.linkedin.davinci.storage.DiskHealthCheckServiceTest

Co-authored-by: Ali Poursamadi <apoursamadi@linkedin.com>

Author: eldernewborn

clients/da-vinci-client/src/main/java/com/linkedin/davinci/storage/DiskHealthCheckService.java

@@ -46,6 +46,8 @@ public class DiskHealthCheckService extends AbstractVeniceService {
   // Visible for testing
   static final String TMP_FILE_NAME = ".health_check_file";
   private static final int TMP_FILE_SIZE_IN_BYTES = 64 * 1024; // 64KB
+  // Keep the disk health check responsive even when other components are under heavy load, without using max priority.
+  static final int HEALTH_CHECK_THREAD_PRIORITY = 8;
 
   // lock object protects diskHealthy and lastStatusUpdateTimeInNS.
   private final Lock lock = new ReentrantLock();
@@ -118,7 +120,8 @@ public boolean startInner() {
     setLastStatusUpdateTimeInNS(System.nanoTime());
     healthCheckTask = new DiskHealthCheckTask();
     runner =
-        new DaemonThreadFactory("Storage Disk Health Check Background Thread", logContext).newThread(healthCheckTask);
+        new DaemonThreadFactory("Storage Disk Health Check Background Thread", HEALTH_CHECK_THREAD_PRIORITY, logContext)
+            .newThread(healthCheckTask);
     runner.start();
 
     return true;
@@ -161,6 +164,11 @@ DiskHealthCheckTask getHealthCheckTask() {
     return healthCheckTask;
   }
 
+  // Only for testing
+  Thread getHealthCheckThread() {
+    return runner;
+  }
+
   // Visible for testing
   class DiskHealthCheckTask implements Runnable {
     private volatile boolean stop = false;

clients/da-vinci-client/src/test/java/com/linkedin/davinci/storage/DiskHealthCheckServiceTest.java

@@ -29,6 +29,9 @@ public void testBasicFunctionality() {
         LogContext.forTests(VeniceComponent.SERVER.name()))) {
       diskHealthCheckService.start();
 
+      assertEquals(
+          diskHealthCheckService.getHealthCheckThread().getPriority(),
+          DiskHealthCheckService.HEALTH_CHECK_THREAD_PRIORITY);
       assertTrue(diskHealthCheckService.isDiskHealthy());
       assertNull(diskHealthCheckService.getErrorMessage());
       assertTrue(diskHealthCheckService.getDiskHealthy());