log small and large digests separately

Summary:
Small and large digests have very different perf characteristics (e.g. they use different storages like zippy and manifold),
so it would be great to tell them apart in our logging to better understand what storage is used.

This is what this diff does.

Reviewed By: stepancheg

fbshipit-source-id: ecfc75774ae0d027d2a5be7865296a8fa58ae0c8

Author: StanislavGlebik

src/com/facebook/buck/remoteexecution/event/CasBlobBatchInfo.java

@@ -0,0 +1,78 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.facebook.buck.remoteexecution.event;
+
+/**
+ * Class that's used to store info about a batch of blobs upload or downloaded from CAS. This class
+ * is primarily used for logging
+ */
+public class CasBlobBatchInfo {
+  // "Large" and "small" digests go to different backends (e.g. zippy and manifold),
+  // so it makes sense to split them in two. See https://fburl.com/code/saw12ezj.
+  private static final int LARGE_DIGEST_THRESHOLD = 2621440; // 2.5mb
+
+  private final int smallBlobCount;
+  private final long smallSizeBytes;
+  private final int largeBlobCount;
+  private final long largeSizeBytes;
+
+  public CasBlobBatchInfo(long[] digestSizes) {
+    int smallBlobCount = 0;
+    long smallSizeBytes = 0;
+    int largeBlobCount = 0;
+    long largeSizeBytes = 0;
+
+    for (long digestSize : digestSizes) {
+      if (digestSize <= LARGE_DIGEST_THRESHOLD) {
+        smallBlobCount += 1;
+        smallSizeBytes += digestSize;
+      } else {
+        largeBlobCount += 1;
+        largeSizeBytes += digestSize;
+      }
+    }
+
+    this.smallBlobCount = smallBlobCount;
+    this.smallSizeBytes = smallSizeBytes;
+    this.largeBlobCount = largeBlobCount;
+    this.largeSizeBytes = largeSizeBytes;
+  }
+
+  public int getSmallBlobCount() {
+    return smallBlobCount;
+  }
+
+  public long getSmallBlobSize() {
+    return smallSizeBytes;
+  }
+
+  public int getLargeBlobCount() {
+    return largeBlobCount;
+  }
+
+  public long getLargeBlobSize() {
+    return largeSizeBytes;
+  }
+
+  public int getBlobCount() {
+    return getSmallBlobCount() + getLargeBlobCount();
+  }
+
+  public long getBlobSize() {
+    return getSmallBlobSize() + getLargeBlobSize();
+  }
+}

src/com/facebook/buck/remoteexecution/event/CasBlobDownloadEvent.java

@@ -34,29 +34,43 @@ public String getEventName() {
   }
 
   /** Send the Started and returns a Scoped object that sends the Finished event. */
-  public static Scope sendEvent(final BuckEventBus eventBus, int blobCount, long sizeBytes) {
-    final Started startedEvent = new Started(blobCount, sizeBytes);
+  public static Scope sendEvent(final BuckEventBus eventBus, CasBlobBatchInfo batchInfo) {
+    final Started startedEvent = new Started(batchInfo);
     eventBus.post(startedEvent);
     return () -> eventBus.post(new Finished(startedEvent));
   }
 
   /** Download to the CAS has started. */
   public static final class Started extends CasBlobDownloadEvent {
-    private final int blobCount;
-    private final long sizeBytes;
+    private final CasBlobBatchInfo batchInfo;
 
-    public Started(int blobCount, long sizeBytes) {
+    public Started(CasBlobBatchInfo batchInfo) {
       super(EventKey.unique());
-      this.blobCount = blobCount;
-      this.sizeBytes = sizeBytes;
+      this.batchInfo = batchInfo;
     }
 
     public int getBlobCount() {
-      return blobCount;
+      return batchInfo.getBlobCount();
+    }
+
+    public int getSmallBlobCount() {
+      return batchInfo.getSmallBlobCount();
+    }
+
+    public int getLargeBlobCount() {
+      return batchInfo.getLargeBlobCount();
     }
 
     public long getSizeBytes() {
-      return sizeBytes;
+      return batchInfo.getBlobSize();
+    }
+
+    public long getSmallSizeBytes() {
+      return batchInfo.getSmallBlobSize();
+    }
+
+    public long getLargeSizeBytes() {
+      return batchInfo.getLargeBlobSize();
     }
 
     @Override

src/com/facebook/buck/remoteexecution/event/CasBlobUploadEvent.java

@@ -30,22 +30,20 @@ protected CasBlobUploadEvent(EventKey eventKey) {
   }
 
   /** Send the Started and returns a Scoped object that sends the Finished event. */
-  public static Scope sendEvent(final BuckEventBus eventBus, int blobCount, long sizeBytes) {
-    final Started startedEvent = new Started(blobCount, sizeBytes);
+  public static Scope sendEvent(final BuckEventBus eventBus, CasBlobBatchInfo info) {
+    final Started startedEvent = new Started(info);
     eventBus.post(startedEvent);
     return () -> eventBus.post(new Finished(startedEvent));
   }
 
   /** Upload to the CAS has started. */
   public static class Started extends CasBlobUploadEvent {
-    private final int blobCount;
-    private final long sizeBytes;
+    private final CasBlobBatchInfo batchInfo;
 
     @VisibleForTesting
-    Started(int blobCount, long sizeBytes) {
+    Started(CasBlobBatchInfo info) {
       super(EventKey.unique());
-      this.blobCount = blobCount;
-      this.sizeBytes = sizeBytes;
+      this.batchInfo = info;
     }
 
     @Override
@@ -54,11 +52,27 @@ protected String getValueString() {
     }
 
     public int getBlobCount() {
-      return blobCount;
+      return batchInfo.getBlobCount();
+    }
+
+    public int getSmallBlobCount() {
+      return batchInfo.getSmallBlobCount();
+    }
+
+    public int getLargeBlobCount() {
+      return batchInfo.getLargeBlobCount();
     }
 
     public long getSizeBytes() {
-      return sizeBytes;
+      return batchInfo.getBlobSize();
+    }
+
+    public long getSmallSizeBytes() {
+      return batchInfo.getSmallBlobSize();
+    }
+
+    public long getLargeSizeBytes() {
+      return batchInfo.getLargeBlobSize();
     }
   }
 

src/com/facebook/buck/remoteexecution/event/CasFindMissingEvent.java

@@ -30,23 +30,31 @@ protected CasFindMissingEvent(EventKey eventKey) {
   }
 
   /** Send the Started and returns a Scoped object that sends the Finished event. */
-  public static Scope sendEvent(final BuckEventBus eventBus, int blobCount) {
-    final CasFindMissingEvent.Started startedEvent = new CasFindMissingEvent.Started(blobCount);
+  public static Scope sendEvent(final BuckEventBus eventBus, CasBlobBatchInfo info) {
+    final CasFindMissingEvent.Started startedEvent = new CasFindMissingEvent.Started(info);
     eventBus.post(startedEvent);
     return () -> eventBus.post(new CasFindMissingEvent.Finished(startedEvent));
   }
 
   /** FindMissing call has started. */
   public static final class Started extends CasFindMissingEvent {
-    private final int blobCount;
+    private final CasBlobBatchInfo batchInfo;
 
-    public Started(int blobCount) {
+    public Started(CasBlobBatchInfo batchInfo) {
       super(EventKey.unique());
-      this.blobCount = blobCount;
+      this.batchInfo = batchInfo;
     }
 
     public int getBlobCount() {
-      return blobCount;
+      return batchInfo.getBlobCount();
+    }
+
+    public int getSmallBlobCount() {
+      return batchInfo.getSmallBlobCount();
+    }
+
+    public int getLargeBlobCount() {
+      return batchInfo.getLargeBlobCount();
     }
 
     @Override

src/com/facebook/buck/remoteexecution/event/RemoteExecutionStatsProvider.java

@@ -28,18 +28,48 @@ public interface RemoteExecutionStatsProvider {
   /** Total number of downloads. */
   int getCasDownloads();
 
+  /** Number of "small" downloads (usually go to zippy). */
+  int getCasSmallDownloads();
+
+  /** Number of "large" downloads (usually go to manifold). */
+  int getCasLargeDownloads();
+
+  /** Number of "small" downloaded bytes from CAS. */
+  long getCasSmallDownloadSizeBytes();
+
+  /** Number of "large" downloaded bytes from CAS. */
+  long getCasLargeDownloadSizeBytes();
+
   /** Total number of downloaded bytes from CAS. */
   long getCasDownloadSizeBytes();
 
   /** Total number of downloads. */
   int getCasUploads();
 
+  /** Number of "small" uploads (usually go to zippy). */
+  int getCasSmallUploads();
+
+  /** Number of "large" uploads (usually go to manifold). */
+  int getCasLargeUploads();
+
+  /** Number of "small" uploaded bytes from CAS. */
+  long getCasSmallUploadSizeBytes();
+
+  /** Number of "large" uploaded bytes from CAS. */
+  long getCasLargeUploadSizeBytes();
+
   /** Total of uploaded bytes to CAS. */
   long getCasUploadSizeBytes();
 
   /** Total number of digests that were sent in findMissing calls. */
   long getCasFindMissingCount();
 
+  /** Number of small digests that were sent in findMissing calls. */
+  long getCasFindMissingSmallCount();
+
+  /** Number of large digests that were sent in findMissing calls. */
+  long getCasFindMissingLargeCount();
+
   /** Get the total number of BuildRules that are finished. (both local and remote) */
   int getTotalRulesBuilt();