Split up organizations across metrics exporters (#5127)

# What this PR does
Limits organizations that a metrics exporter is responsible for. As more
organizations are added it becomes more difficult for the exporter to
deliver metrics within the scrape timeout. This would let us use the
settings to divide up the organizations between multiple exporters.

## Which issue(s) this PR closes

Related to [issue link here]

<!--
*Note*: If you want the issue to be auto-closed once the PR is merged,
change "Related to" to "Closes" in the line above.
If you have more than one GitHub issue that this PR closes, be sure to
preface
each issue link with a [closing
keyword](https://docs.github.com/en/get-started/writing-on-github/working-with-advanced-formatting/using-keywords-in-issues-and-pull-requests#linking-a-pull-request-to-an-issue).
This ensures that the issue(s) are auto-closed once the PR has been
merged.
-->

## Checklist

- [x] Unit, integration, and e2e (if applicable) tests updated
- [x] Documentation added (or `pr:no public docs` PR label added if not
required)
- [x] Added the relevant release notes label (see labels prefixed w/
`release:`). These labels dictate how your PR will
    show up in the autogenerated release notes.

Author: mderynck

engine/apps/metrics_exporter/helpers.py

@@ -2,6 +2,7 @@
 import random
 import typing
 
+from django.conf import settings
 from django.core.cache import cache
 from django.utils import timezone
 
@@ -50,7 +51,10 @@ def get_organization_ids():
     if not organizations_ids:
         organizations_ids = get_organization_ids_from_db()
         cache.set(organizations_ids, METRICS_ORGANIZATIONS_IDS, METRICS_ORGANIZATIONS_IDS_CACHE_TIMEOUT)
-    return organizations_ids
+
+    group_id = settings.METRICS_EXPORTER_ORGANIZATION_GROUP_ID
+    group_count = settings.METRICS_EXPORTER_TOTAL_ORGANIZATION_GROUPS
+    return [i for i in organizations_ids if i % group_count == group_id]
 
 
 def is_allowed_to_start_metrics_calculation(organization_id, force=False) -> bool:

engine/settings/base.py

@@ -121,6 +121,11 @@
 # List of metrics to collect. Collect all available application metrics by default
 METRICS_TO_COLLECT = getenv_list("METRICS_TO_COLLECT", METRICS_ALL)
 
+# Total number of exporters collecting the same set of metrics
+METRICS_EXPORTER_TOTAL_ORGANIZATION_GROUPS = getenv_integer("METRICS_EXPORTER_TOTAL_ORGANIZATION_GROUPS", 1)
+# ID of this exporter, used to filter which orgs to collect for
+METRICS_EXPORTER_ORGANIZATION_GROUP_ID = getenv_integer("METRICS_EXPORTER_ORGANIZATION_GROUP_ID", 0)
+
 
 # Database
 class DatabaseTypes: