ENHANCE_YOUR_CALM even under light load

[rejuvenile]

In this case, there are only ten workers and a single builder against a 64 CPU scheduler and CAS instance.

  2025-03-20T21:03:22.510784Z ERROR nativelink::services: Failed running service, err: hyper::Error(Http2, Error { kind: GoAway(b"", ENHANCE_YOUR_CALM, Library) })
    at src/bin/nativelink.rs:892
    in nativelink::services::http_connection with remote_addr: 192.168.191.244:56728, socket_addr: 0.0.0.0:50051

[MarcusSorealheis]

Could you share your json config?

[rejuvenile]

{
  "stores": [
    {
      "name": "AC_FAST_SLOW_STORE",
      "fast_slow": {
        "fast": {
          "size_partitioning": {
            "size": 1000,
            "lower_store": {
              "memory": {
                "eviction_policy": {
                  "max_bytes": 2000000000,
                  "max_count": 150000000
                }
              }
            },
            "upper_store": {
              "noop": {}
            }
          }
        },
        "slow": {
          "filesystem": {
            "content_path": "/work/data-worker-test/content_path-ac",
            "temp_path": "/work/data-worker-test/tmp_path-ac",
            "eviction_policy": {
              // 10gb.
              "max_bytes": 10000000000
            }
          }
        }
      }
    },
    {
      "name": "AC_STORE",
      "completeness_checking": {
        "backend": {
          "ref_store": {
            "name": "AC_FAST_SLOW_STORE"
          }
        },
        "cas_store": {
          "ref_store": {
            "name": "cas_STORE"
          }
        }
      }
    },
    {
      "name": "cas_FAST_SLOW_STORE",
      "fast_slow": {
        "fast": {
          "size_partitioning": {
            "size": 4194304,
            "upper_store": {
              "noop": {}
            },
            "lower_store": {
              "memory": {
                "eviction_policy": {
                  "max_bytes": 2000000000,
                  "max_count": 150000000
                }
              }
            }
          }
        },
        "slow": {
          "filesystem": {
            "content_path": "/work/data-worker-test/content_path-cas",
            "temp_path": "/work/data-worker-test/tmp_path-cas",
            "eviction_policy": {
              // 100gb.
              "max_bytes": 100000000000,
            }
          }
        }
      }
    },
    {
      "name": "cas_STORE",
      "existence_cache": {
        "backend": {
          "ref_store": {
            "name": "cas_FAST_SLOW_STORE"
          }
        },
        "eviction_policy": {
          "max_count": 10000000,
        }
      }
    }
  ],
  "schedulers": [
    {
      "name": "MAIN_SCHEDULER",
      "simple": {
        "supported_platform_properties": {
          "cpu_count": "minimum",
          "memory_kb": "minimum",
          "network_kbps": "minimum",
          "disk_read_iops": "minimum",
          "disk_read_bps": "minimum",
          "disk_write_iops": "minimum",
          "disk_write_bps": "minimum",
          "shm_size": "minimum",
          "gpu_count": "minimum",
          "gpu_model": "exact",
          "cpu_vendor": "exact",
          "cpu_arch": "exact",
          "cpu_model": "exact",
          "kernel_version": "exact",
          "OSFamily": "exact",
          "container-image": "priority",
          "lre-rs": "priority"
        }
      }
    }
  ],
  "workers": [],
  "servers": [
    {
      "name": "public",
      "listener": {
        "http": {
          "socket_address": "0.0.0.0:50051",
        }
      },
      "services": {
        "cas": {
          "main": {
            "cas_store": "cas_STORE"
          }
        },
        "ac": {
          "main": {
            "ac_store": "AC_STORE"
          }
        },
        "execution": {
          "main": {
            "cas_store": "cas_STORE",
            "scheduler": "MAIN_SCHEDULER"
          }
        },
        "capabilities": {
          "main": {
            "remote_execution": {
              "scheduler": "MAIN_SCHEDULER"
            }
          }
        },
        "bytestream": {
          "cas_stores": {
            "main": "cas_STORE"
          }
        }
      }
    },
    {
      "name": "private_workers_servers",
      "listener": {
        "http": {
          "socket_address": "0.0.0.0:50061"
        }
      },
      "services": {
        "experimental_prometheus": {
          "path": "/metrics"
        },
        // Note: This should be served on a different port, because it has
        // a different permission set than the other services.
        // In other words, this service is a backend api. The ones above
        // are a frontend api.
        "worker_api": {
          "scheduler": "MAIN_SCHEDULER"
        },
        "admin": {},
        "health": {}
      }
    }
  ],
  "global": {
    "max_open_files": 49152
  }
}

v0.6.0