Merge pull request #31 from gitpod-io/nv/docker-resource-limits

fix: add memory and CPU limits to all docker containers (A19)

Author: nandajavarma

files/proxy-cloud-init.tftpl

@@ -277,6 +277,8 @@ write_files:
       ExecStart=/usr/bin/docker%{ if DOCKER_CONFIG_ENABLED } --config /var/lib/gitpod/docker-config%{ endif } run --rm --name prometheus \
         --network host \
         --hostname %H \
+        --memory=512m \
+        --cpus=0.25 \
         --volume /var/lib/prometheus:/etc/prometheus:ro \
         --volume /var/lib/prometheus/data:/prometheus \
 %{ if CA_ENABLED ~}
@@ -336,6 +338,8 @@ write_files:
       ExecStart=/usr/bin/docker%{ if DOCKER_CONFIG_ENABLED } --config /var/lib/gitpod/docker-config%{ endif } run --rm --name node-exporter \
         --network host \
         --pid host \
+        --memory=256m \
+        --cpus=0.25 \
         --volume /:/host:ro,rslave \
 %{ if CA_ENABLED ~}
         --volume /var/lib/gitpod/certs/gitpod-custom-ca.crt:/etc/ssl/certs/gitpod-trust-bundle.crt:ro \
@@ -665,6 +669,8 @@ write_files:
       ExecStartPre=-/usr/bin/docker rm gitpod-proxy
       ExecStart=/usr/bin/docker%{ if DOCKER_CONFIG_ENABLED } --config /var/lib/gitpod/docker-config%{ endif } run --rm --name gitpod-proxy \
         --network host \
+        --memory=${PROXY_CONTAINER_MEMORY} \
+        --cpus=${PROXY_CONTAINER_CPUS} \
         --volume /var/lib/gitpod/tls:/tmp/certs:rw \
 %{ if CA_ENABLED ~}
         --volume /var/lib/gitpod/certs/gitpod-custom-ca.crt:/etc/ssl/certs/gitpod-trust-bundle.crt:ro \

files/runner-cloud-init.tftpl

@@ -296,6 +296,8 @@ write_files:
       ExecStart=/usr/bin/docker%{ if DOCKER_CONFIG_ENABLED } --config /var/lib/gitpod/docker-config%{ endif } run --rm --name prometheus \
         --network host \
         --hostname %H \
+        --memory=1g \
+        --cpus=0.5 \
         --volume /var/lib/prometheus:/etc/prometheus:ro \
         --volume /var/lib/prometheus/data:/prometheus \
         --volume /var/lib/gitpod/certs/gitpod-custom-ca.crt:/etc/ssl/certs/gitpod-trust-bundle.crt:ro \
@@ -344,6 +346,8 @@ write_files:
       ExecStart=/usr/bin/docker%{ if DOCKER_CONFIG_ENABLED } --config /var/lib/gitpod/docker-config%{ endif } run --rm --name node-exporter \
         --network host \
         --pid host \
+        --memory=256m \
+        --cpus=0.25 \
         --volume /:/host:ro,rslave \
         --volume /var/lib/gitpod/certs/gitpod-custom-ca.crt:/etc/ssl/certs/gitpod-trust-bundle.crt:ro \
         --env http_proxy=${HTTP_PROXY} \
@@ -652,6 +656,8 @@ write_files:
       ExecStart=/usr/bin/docker%{ if DOCKER_CONFIG_ENABLED } --config /var/lib/gitpod/docker-config%{ endif } run --rm --name gitpod-auth-proxy \
         --network host \
         --hostname %H \
+        --memory=512m \
+        --cpus=0.5 \
         --volume /var/lib/gitpod/certs:/var/lib/gitpod/certs:ro \
 %{ if HAS_TRUST_BUNDLE ~}
         --volume /var/lib/gitpod/certs/gitpod-custom-ca.crt:/etc/ssl/certs/gitpod-trust-bundle.crt:ro \
@@ -700,6 +706,8 @@ write_files:
       ExecStart=/usr/bin/docker%{ if DOCKER_CONFIG_ENABLED } --config /var/lib/gitpod/docker-config%{ endif } run --rm --name gitpod-runner \
         --network host \
         --hostname %H \
+        --memory=${RUNNER_CONTAINER_MEMORY} \
+        --cpus=${RUNNER_CONTAINER_CPUS} \
         --volume /var/lib/prometheus:/var/lib/prometheus \
         --volume /tmp:/tmp \
         --env http_proxy=${HTTP_PROXY} \

locals.tf

@@ -32,6 +32,39 @@ locals {
 
   runner_dev_image = var.development_version != "" ? "us-docker.pkg.dev/gitpod-next-production/gitpod-next/gitpod-gcp-runner:${var.development_version}" : local.runner_image
 
+  # Container resource limits derived from VM machine type.
+  # GCP standard machine types follow the pattern {family}-standard-{vcpus}
+  # with memory = vcpus * 4 GB. We reserve ~25% for the host OS and Docker
+  # daemon, then allocate the rest across containers.
+  #
+  # Aligned with EC2 Fargate runner limits:
+  #   EC2 small:  1 vCPU /  3 GB task  → runner gets ~1 GB
+  #   EC2 large:  8 vCPU / 16 GB task  → runner gets ~14 GB
+  #   GCP small:  2 vCPU /  8 GB VM    → runner gets  5 GB / 1.25 CPU
+  #   GCP regular: 4 vCPU / 16 GB VM   → runner gets 12 GB / 3 CPU
+
+  runner_vcpus     = tonumber(regex("-(\\d+)$", var.runner_vm_config.machine_type)[0])
+  runner_memory_gb = local.runner_vcpus * 4
+
+  # Sidecar limits are fixed (small footprint). The main runner container
+  # gets whatever remains after sidecars and OS overhead.
+  runner_sidecar_memory_mb = 1792 # prometheus 1024 + auth-proxy 512 + node-exporter 256
+  runner_sidecar_cpus      = 1.25 # prometheus 0.5 + auth-proxy 0.5 + node-exporter 0.25
+  runner_os_reserve_mb     = 512
+
+  runner_container_memory_mb = (local.runner_memory_gb * 1024) - local.runner_sidecar_memory_mb - local.runner_os_reserve_mb
+  runner_container_cpus      = local.runner_vcpus - local.runner_sidecar_cpus
+
+  proxy_vcpus     = tonumber(regex("-(\\d+)$", var.proxy_vm_config.machine_type)[0])
+  proxy_memory_gb = local.proxy_vcpus * 4
+
+  proxy_sidecar_memory_mb = 768 # prometheus 512 + node-exporter 256
+  proxy_sidecar_cpus      = 0.5 # prometheus 0.25 + node-exporter 0.25
+  proxy_os_reserve_mb     = 512
+
+  proxy_container_memory_mb = (local.proxy_memory_gb * 1024) - local.proxy_sidecar_memory_mb - local.proxy_os_reserve_mb
+  proxy_container_cpus      = local.proxy_vcpus - local.proxy_sidecar_cpus
+
   # Docker config handling
   docker_config_enabled     = var.custom_images.docker_config_json != ""
   docker_config_bucket_name = local.docker_config_enabled ? google_storage_bucket.runner_assets.name : ""

proxy-vm.tf

@@ -37,6 +37,9 @@ data "cloudinit_config" "proxy" {
       # Insecure registries configuration
       INSECURE_REGISTRIES_ENABLED = local.insecure_registries_enabled
       INSECURE_REGISTRIES_JSON    = local.insecure_registries_json
+      # Container resource limits (computed from machine type)
+      PROXY_CONTAINER_MEMORY = "${local.proxy_container_memory_mb}m"
+      PROXY_CONTAINER_CPUS   = tostring(local.proxy_container_cpus)
     })
   }
 }

runner-vm.tf

@@ -167,6 +167,9 @@ data "cloudinit_config" "runner" {
       ENVIRONMENT_VM_LABELS = join(",", [for k, v in var.labels : "${k}=${v}"])
       # Module version reported to the management plane
       TERRAFORM_MODULE_VERSION = local.module_version
+      # Container resource limits (computed from machine type)
+      RUNNER_CONTAINER_MEMORY = "${local.runner_container_memory_mb}m"
+      RUNNER_CONTAINER_CPUS   = tostring(local.runner_container_cpus)
     })
   }
 }