release 0.53

.env.example

@@ -3,17 +3,13 @@
 # Docker Compose files. Only specific ones are (see docker-compose.yaml).
 # If you need to add new environment variables, refer to them in the Compose file in the appropriate
 # container for your configuration to be correctly applied.
+# See https://docs.checkmarble.com/docs/technical-configuration#/ for a full guide to Marble environment variables, agnostic
+# of the docker-compose context.
 
 #
 # REQUIRED SETTINGS
 #
 
-# If equal to "development", applies configurations on the backend and frontend containers that
-# are not suitable for production, related to CORS, secure cookies and https.
-# Any other value works for a production environment, and will be used by sentry to mark the environment
-# an error occurred in.
-ENV=production
-
 # Ports respectively used by the backend API and the frontends containers
 # Maps to the "PORT" environment variable in the backend and frontend containers
 # respectively.
@@ -29,9 +25,9 @@ HOST_APP_PORT=3000
 # Must include the scheme (e.g http://), and the port if non-standard.
 MARBLE_APP_URL="http://localhost:3000"
 
-# Set the value to the URL the Marble API can be reached at. Must include the scheme (e.g http://)
-# "MARBLE_API_URL_SERVER" must be reachable from the frontend host. With docker-compose, use the api container name as host name.
-MARBLE_API_URL_SERVER="http://api:8080"
+# Set the value to the URL the Marble API can be reached at, from the frontend host. Must include the scheme (e.g http://)
+# With docker-compose, use the api container name as host name.
+MARBLE_API_URL="http://api:8080"
 
 # RSA private key, in PEM format, used for for signing authentication tokens. MUST be changed for production.
 # We recommend using AUTHENTICATION_JWT_SIGNING_KEY_FILE to point to a private key on disk because multi-line
@@ -45,7 +41,7 @@ MARBLE_API_URL_SERVER="http://api:8080"
 AUTHENTICATION_JWT_SIGNING_KEY=
 # AUTHENTICATION_JWT_SIGNING_KEY_FILE=/shared/jwt_key.pem
 
-# Configure your PostgreSQL database connection information, either by providing a DSN using this form:
+# Configure your PostgreSQL database connection information, either by providing a DSN using this form (no other format is accepted):
 # PG_CONNECTION_STRING='postgres://postgres:marble@localhost:5432/marble?sslmode=prefer'
 # Or by setting each piece of information in those variables:
 PG_HOSTNAME=db
@@ -91,12 +87,19 @@ FIREBASE_API_KEY=
 
 # Generate a random string, for instance using `openssl rand -base64 128 | tr -d "\n"`
 SESSION_SECRET=changeme
-SESSION_MAX_AGE=43200
 
 #
 # OPTIONAL SETTINGS
 #
 
+# Used for minor logging and telemetry settings. In particular, it will be used by sentry to mark the environment
+# an error occurred in. Do not touch it in the general case and it should work fine.
+# *DO NOT* set it to "development" for production purposes, as this specifically enables some unsafe settings.
+# ENV=production
+
+# Configure the frontend cookie session max age, in seconds. Defaults to 43200
+# SESSION_MAX_AGE=43200
+
 # Set your license key here if you have one in order to access premium features.
 LICENSE_KEY=
 

docker-compose-dev.yaml

@@ -1,17 +1,16 @@
 name: marble
 
 x-backend-image-version: &backend-image-version
-  image: europe-west1-docker.pkg.dev/marble-infra/marble/marble-backend:v0.52.1
+  image: europe-west1-docker.pkg.dev/marble-infra/marble/marble-backend:v0.53.3
 x-frontend-image-version: &frontend-image-version
-  image: europe-west1-docker.pkg.dev/marble-infra/marble/marble-frontend:v0.52.0
+  image: europe-west1-docker.pkg.dev/marble-infra/marble/marble-frontend:v0.53.1
 
 x-shared-environment: &shared-env
-  ENV: development
-  MARBLE_APP_URL: http://localhost:${HOST_APP_PORT:-3000}
   DISABLE_SEGMENT: ${DISABLE_SEGMENT:-false}
 x-backend-environment: &backend-env
   LICENSE_KEY: ${LICENSE_KEY:-LICENSE_KEY}
   PORT: 8080
+  MARBLE_APP_URL: http://localhost:${HOST_APP_PORT:-3000}
 
   PG_HOSTNAME: db
   PG_PORT: ${PG_PORT:-5432}
@@ -23,6 +22,7 @@ x-backend-environment: &backend-env
   CASE_MANAGER_BUCKET_URL: ${CASE_MANAGER_BUCKET_URL:-}
 
   FIREBASE_AUTH_EMULATOR_HOST: firebase-auth:9099
+  FIREBASE_API_KEY: placeholder
 
   # default value of GOOGLE_CLOUD_PROJECT must be kept if working with the emulator (and the emulator is running in the docker image below)
   GOOGLE_CLOUD_PROJECT: test-project
@@ -38,16 +38,12 @@ x-backend-environment: &backend-env
   CONVOY_API_URL: ${CONVOY_API_URL:-}
   CONVOY_PROJECT_ID: ${CONVOY_PROJECT_ID:-}
 x-frontend-environment: &frontend-env
-  NODE_ENV: production
   SESSION_SECRET: ${SESSION_SECRET:-}
   SESSION_MAX_AGE: 43200
 
-  MARBLE_API_URL_SERVER: http://api:8080
+  MARBLE_API_URL: http://api:8080
 
-  FIREBASE_AUTH_EMULATOR_HOST: localhost:9099
-  FIREBASE_API_KEY: placeholder
-  # default value of FIREBASE_PROJECT_ID must be kept if working with the emulator (and the emulator is running in the docker image below)
-  FIREBASE_PROJECT_ID: test-project
+  TEST_FIREBASE_AUTH_EMULATOR_HOST: localhost:9099
 
 services:
   db:

docker-compose.yaml

@@ -1,19 +1,19 @@
 name: marble
 
 x-backend-image-version: &backend-image-version
-  image: europe-west1-docker.pkg.dev/marble-infra/marble/marble-backend:v0.52.1
+  image: europe-west1-docker.pkg.dev/marble-infra/marble/marble-backend:v0.53.3
 x-frontend-image-version: &frontend-image-version
-  image: europe-west1-docker.pkg.dev/marble-infra/marble/marble-frontend:v0.52.0
+  image: europe-west1-docker.pkg.dev/marble-infra/marble/marble-frontend:v0.53.1
 
 x-shared-environment: &shared-env
   ENV: ${ENV}
-  MARBLE_APP_URL: ${MARBLE_APP_URL}
   SENTRY_DSN: ${SENTRY_DSN}
   DISABLE_SEGMENT: ${DISABLE_SEGMENT:-false}
 x-backend-environment: &backend-env
   LICENSE_KEY: ${LICENSE_KEY}
 
   PORT: ${API_PORT:-8080}
+  MARBLE_APP_URL: ${MARBLE_APP_URL}
 
   PG_HOSTNAME: ${PG_HOSTNAME}
   PG_PORT: ${PG_PORT}
@@ -30,6 +30,9 @@ x-backend-environment: &backend-env
   GOOGLE_CLOUD_PROJECT: ${GOOGLE_CLOUD_PROJECT}
   GOOGLE_APPLICATION_CREDENTIALS: ${GOOGLE_APPLICATION_CREDENTIALS}
 
+  FIREBASE_API_KEY: ${FIREBASE_API_KEY}
+  FIREBASE_PROJECT_ID: ${FIREBASE_PROJECT_ID}
+
   REQUEST_LOGGING_LEVEL: ${REQUEST_LOGGING_LEVEL}
   LOGGING_FORMAT: ${LOGGING_FORMAT}
 
@@ -56,18 +59,12 @@ x-backend-environment: &backend-env
   OFFLOADING_SAVE_POINTS: ${OFFLOADING_SAVE_POINTS}
   OFFLOADING_WRITES_PER_SEC: ${OFFLOADING_WRITES_PER_SEC}
 x-frontend-environment: &frontend-env
-  NODE_ENV: production
   PORT: ${APP_PORT:-8080}
 
   SESSION_SECRET: ${SESSION_SECRET}
   SESSION_MAX_AGE: ${SESSION_MAX_AGE}
 
-  MARBLE_API_URL_SERVER: ${MARBLE_API_URL_SERVER}
-
-  FIREBASE_PROJECT_ID: ${GOOGLE_CLOUD_PROJECT}
-  FIREBASE_API_KEY: ${FIREBASE_API_KEY}
-  FIREBASE_AUTH_DOMAIN: ${FIREBASE_AUTH_DOMAIN}
-  FIREBASE_APP_ID: ${FIREBASE_APP_ID}
+  MARBLE_API_URL: ${MARBLE_API_URL}
 
 services:
   # NB: the docker compose file includes a postgres image for the sake of completeness, but we strongly

installation/baremetal.md

@@ -70,20 +70,21 @@ Copy the [example configuration](https://github.com/checkmarble/marble-backend/b
 
 - Set the `PG_*` variables to point to your PostgresSQL instance.
 - Set the `GOOGLE_CLOUD_PROJECT` to the ID of your Firebase project.
-- The `GOOGLE_APPLICATION_CREDENTIALS` should contain the path to the JSON private key file downloaded from Firebase.
+- The `GOOGLE_APPLICATION_CREDENTIALS` should contain the path to the JSON private key file downloaded from Firebase/GCP.
+- Set `FIREBASE_API_KEY` with the information retrieved from your Firebase account.
 - Set `AUTHENTICATION_JWT_SIGNING_KEY_FILE` to point to the generated RSA private key at `/etc/marble/jwtsigningkey.pem`.
 - `*_BUCKET_URL`should point to buckets in your blob storage platform (S3, Azure Blob or GCS).
-If you are using Minio, use a URL such as `s3://<bucket>?awssdk=v1&endpoint=minio.domain.com&region=us-east-1&s3ForcePathStyle=true`. In this case, you may also add `disableSSL=true` if your MinIO instance is in cleartext.
-You might have to add provider-specific configuration, for example for authentication (for S3 and MinIO, for example, set `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY`.
-- Set the `CREATE_*` variables to specify your initial organization and admin user.
+  If you are using Minio, use a URL such as `s3://<bucket>?awssdk=v1&endpoint=minio.domain.com&region=us-east-1&s3ForcePathStyle=true`. In this case, you may also add `disableSSL=true` if your MinIO instance is in cleartext.
+  You might have to add provider-specific configuration, for example for authentication (for S3 and MinIO, for example, set `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY`).
+- Set the `CREATE_*` variables to specify your initial organization and admin user - this needs to be run only once and can be omitted after the first run.
 - Set your `LICENCE_KEY`.
-- Set `MARBLE_APP_URL` to the URL used to access the frontend.
+- Set `MARBLE_APP_URL` to the external HTTP base your users’ browsers can use to reach the frontend.
 
 ### Service configuration
 
 Marble’s backend is composed of five discrete services, three of which are scheduled tasks. Those can be configured with the following systemd units:
 
-#### API *(persistent service)*
+#### API _(persistent service)_
 
 ```bash
 # /etc/systemd/system/marble-api.service
@@ -103,7 +104,7 @@ ExecStart=/opt/marble/api/marble-backend -server
 WantedBy=multi-user.target
 ```
 
-#### Worker *(persistent service)*
+#### Worker _(persistent service)_
 
 ```bash
 # /etc/systemd/system/marble-worker.service
@@ -123,7 +124,7 @@ ExecStart=/opt/marble/api/marble-backend -worker
 WantedBy=multi-user.target
 ```
 
-#### Database migrator *(manual run only)*
+#### Database migrator _(manual run only)_
 
 ```bash
 # /etc/systemd/system/marble-migrate.service
@@ -170,16 +171,12 @@ $  curl http://127.0.0.1:8080/liveness
 Copy the [example configuration](https://github.com/checkmarble/marble-frontend/blob/main/packages/app-builder/.env.example) file into `/etc/marble/front.conf`and edit the relevant settings. Among others:
 
 - `NODE_ENV` must be set to `production`.
-- `ENV` should be set to `production`.
-Note that this will require setting up TLS certificates to access the frontend. If prototyping without TLS, set this to `development`.
 - Set a random, high-entropy `SESSION_SECRET`.
-- `MARBLE_API_URL_SERVER` should be the external HTTP base your users’ browsers can use to reach the API.
-- `MARBLE_APP_URL` should be set to the external HTTP base your users’ browsers can use to reach the frontend.
-- Set the different Firebase configuration settings with the information retrieved from your Firebase account.
+- `MARBLE_API_URL` should be the internal HTTP base your frontend service can use to reach the API.
 
 ### Service configuration
 
-Drop this `systemd`unit file in ``/etc/systemd/system/marble-front.service``:
+Drop this `systemd`unit file in `/etc/systemd/system/marble-front.service`:
 
 #### Frontend
 
@@ -201,7 +198,6 @@ ExecStart=/opt/marble/front/node_modules/@remix-run/serve/dist/cli.js ./build/se
 WantedBy=multi-user.target
 ```
 
-
 And start the frontend service by running:
 
 ```bash

installation/configuration.md

@@ -0,0 +1,5 @@
+## Configuring Marble
+
+See our [product documentation](https://docs.checkmarble.com/docs/technical-configuration#/) for a full guide on which environment variables are available, which ones are required, and their expected format, for every part of the Marble app.
+
+Configuration via config files is currently not available.

installation/data_offloading.md

@@ -15,7 +15,8 @@ The `OFFLOADING_JOB_INTERVAL`, `OFFLOADING_BEFORE`, `OFFLOADING_BATCH_SIZE`, `OF
 
 ### Storage classes
 
-Old decisions are likely to be rarely consulted. For further cost savings, we recommend you configure your blob storage bucket to automatically move the data to long-term storage classes using lifecycle rules. Objects are stored by decision status prefix (`offloading/decision_rules/{status}` where `status` is one of `error`, `hit`, `no_hit`), so it is possible in most blob storage solutions to configure different lifecycle rules for rules that did or did not result in a hit.
+The objects thus stored have a small size, even if they are cumbersome to store in a relational database. We strongly suggest that you write the objects to the "standard" object class, and do not use any lifecycle rules other than (optionally) deleting objects after a given time. Deleting the objects under `{bucket}/offloading/decision_rules` will not cause any errors in the application.
 
-Data offloaded to Google Cloud Storage may further use the custom time metadata attribute instead of the object's creation time for lifecycle rules.
-As an example, Marble's managed environment moves objects to "Nearline" storage class after a month, "Coldline" after 3 months and "Archive" after a year.
+Specifically, moving objects to "archive" type storage classes is dangerous because the per-operation cost greatly exceeds the storage cost.
+
+Objects are stored by decision status prefix (`offloading/decision_rules/{status}` where `status` is one of `error`, `hit`, `no_hit`), so it is possible in most blob storage solutions to configure different lifecycle rules for rules that did or did not result in a hit.