feat: Use S3 node store with garage

docker-compose.yml

@@ -34,6 +34,8 @@ x-sentry-defaults: &sentry_defaults
       <<: *depends_on-default
     smtp:
       <<: *depends_on-default
+    garage:
+      <<: *depends_on-default
     snuba-api:
       <<: *depends_on-default
     symbolicator:
@@ -139,6 +141,7 @@ services:
   kafka:
     <<: *restart_policy
     image: "confluentinc/cp-kafka:7.6.1"
+    user: root
     environment:
       # https://docs.confluent.io/platform/current/installation/docker/config-reference.html#cp-kakfa-example
       KAFKA_PROCESS_ROLES: "broker,controller"
@@ -207,6 +210,19 @@ services:
       interval: 10s
       timeout: 10s
       retries: 30
+  garage:
+    image: dxflrs/garage:v1.0.1
+    volumes:
+      - type: bind
+        read_only: true
+        source: ./garage.toml
+        target: /etc/garage.toml
+      - "sentry-garage:/var/lib/garage/"
+    healthcheck:
+      test: "wget -no-verbose --tries=1 --spider http://localhost:3903/health || exit 1"
+      interval: 20s
+      timeout: 20s
+      retries: 10      
   snuba-api:
     <<: *snuba_defaults
   # Kafka consumer responsible for feeding events into Clickhouse
@@ -540,3 +556,4 @@ volumes:
   sentry-kafka-log:
   sentry-smtp-log:
   sentry-clickhouse-log:
+  sentry-garage:

garage.toml

@@ -0,0 +1,29 @@
+metadata_dir = "/var/lib/garage/meta"
+data_dir = "/var/lib/garage/data"
+db_engine = "sqlite"
+
+replication_factor = 1
+# idk why rcp_ is needed without replication but ok
+rpc_bind_addr = "[::]:3901"
+rpc_public_addr = "127.0.0.1:3901"
+# This secret will not ever be used as we are in a single node setup
+# so this static value is OK. Don't forget to change it if you ever
+# decide to use garage in a multi-node setup.
+rpc_secret = "28f2fc651d4f280cea95e166f9113fe589039f9711d6c4f461872c1c5aaf7dec"
+
+compression_level = 2
+
+[s3_api]
+s3_region = "garage"
+api_bind_addr = "[::]:3900"
+root_domain = ".s3.garage.localhost"
+
+[s3_web]
+bind_addr = "[::]:3902"
+root_domain = ".web.garage.localhost"
+index = "index.html"
+
+[admin]
+api_bind_addr = "0.0.0.0:3903"
+metrics_token = "c6k30kKqovA5NoTfKxMVRmnKKVwd7jaF4OB2xshQVEA="
+admin_token = "c6k30kKqovA5NoTfKxMVRmnKKVwd7jaF4OB2xshQVEA="

install.sh

@@ -32,6 +32,7 @@ source install/ensure-relay-credentials.sh
 source install/generate-secret-key.sh
 source install/update-docker-images.sh
 source install/build-docker-images.sh
+source install/bootstrap-garage.sh
 source install/bootstrap-snuba.sh
 source install/upgrade-postgres.sh
 source install/set-up-and-migrate-database.sh

install/bootstrap-garage.sh

@@ -0,0 +1,25 @@
+echo "${_group}Bootstrapping garage (node store)..."
+
+$dc up --wait garage
+garage="$dc exec garage /garage"
+
+if [[ $($garage bucket list | tail -1 | awk '{print $1}') != 'nodestore' ]]; then
+  node_id=$($garage status | tail -1 | awk '{print $1}')
+  $garage layout assign -z dc1 -c 100G "$node_id"
+  $garage layout apply --version 1
+
+  $garage bucket create nodestore
+  key_info=$($garage key create nodestore-key | head -3 | tail -2)
+  key_id=$(echo "$key_info" | head -1 | awk '{print $3}')
+  key_secret=$(echo "$key_info" | tail -1 | awk '{print $3}')
+
+  $garage bucket allow --read --write --owner nodestore --key nodestore-key
+
+  sed -i -e "s/<GARAGE_KEY_ID>/$key_id/" $SENTRY_CONFIG_PY
+  sed -i -e "s/<GARAGE_SECRET_KEY>/$key_secret/" $SENTRY_CONFIG_PY
+  echo "Set Garage keys for SENTRY_NODESTORE_OPTIONS in $SENTRY_CONFIG_PY"
+else
+  echo "Node store already exists, skipping..."
+fi
+
+echo "${_endgroup}"

sentry/Dockerfile

@@ -1,13 +1,15 @@
 ARG SENTRY_IMAGE
 FROM ${SENTRY_IMAGE}
 
+RUN pip install https://github.com/stayallive/sentry-nodestore-s3/archive/main.zip
+
 COPY . /usr/src/sentry
 
 RUN if [ -s /usr/src/sentry/enhance-image.sh ]; then \
     /usr/src/sentry/enhance-image.sh; \
-fi
+    fi
 
 RUN if [ -s /usr/src/sentry/requirements.txt ]; then \
     echo "sentry/requirements.txt is deprecated, use sentry/enhance-image.sh - see https://develop.sentry.dev/self-hosted/#enhance-sentry-image"; \
     pip install -r /usr/src/sentry/requirements.txt; \
-fi
+    fi

sentry/sentry.conf.example.py

@@ -84,6 +84,25 @@ def get_internal_network():
 # See https://develop.sentry.dev/self-hosted/experimental/errors-only/
 SENTRY_SELF_HOSTED_ERRORS_ONLY = env("COMPOSE_PROFILES") != "feature-complete"
 
+##############
+# Node Store #
+##############
+
+SENTRY_NODESTORE = "sentry_nodestore_s3.S3PassthroughDjangoNodeStorage"
+SENTRY_NODESTORE_OPTIONS = {
+    "delete_through": True,
+    "write_through": False,
+    "read_through": True,
+    "compression": False,  # we have compression enabled in Garage itself
+    "endpoint_url": "http://garage:3900",
+    "bucket_path": "nodestore",
+    "bucket_name": "nodestore",
+    "retry_attempts": 3,
+    "region_name": "garage",
+    "aws_access_key_id": "<GARAGE_KEY_ID>",
+    "aws_secret_access_key": "<GARAGE_SECRET_KEY>",
+}
+
 #########
 # Redis #
 #########