Skip to content

Commit 5f7539c

Browse files
authored
Calculate compute usage per user (#9)
- new API endpoint at `/total-usage` - allow filters by combination of hub, component and user with URL parameters - e.g. `total-usage?from=2025-08-12T00:00Z&to=2025-08-12T23:59Z&hub=prod&component=compute&user=***` - returns a JSON list of arrays formatted as: ```json [ { "component": "compute", "date": "2025-08-12", "hub": "prod", "user": "***", "value": 2929250197800 }, ] ``` - uses resource memory requests/guarantees _only_ to account for usage in the "compute" component - memory requested is a more accurate indicator of incurred cloud costs, not actual user CPU and RAM utilisation - we use just memory and not cpu to approximate usage, since memory is the limiting factor for user workloads. See background in https://infrastructure.2i2c.org/topic/resource-allocation/#factors-to-balance
1 parent 82e3cea commit 5f7539c

9 files changed

Lines changed: 560 additions & 78 deletions

File tree

.github/workflows/publish-helm-chart.yaml

Lines changed: 36 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -5,14 +5,14 @@ on:
55
workflow_dispatch:
66
push:
77
paths-ignore:
8-
- "docs/**"
9-
- "**.md"
10-
- ".github/workflows/*"
11-
- "!.github/workflows/publish-helm-chart.yaml"
8+
- docs/**
9+
- '**.md'
10+
- .github/workflows/*
11+
- '!.github/workflows/publish-helm-chart.yaml'
1212
branches:
13-
- "main"
13+
- main
1414
tags:
15-
- "**"
15+
- '**'
1616

1717
jobs:
1818
# Packages the Helm chart, and pushes it to 2i2c-org/jupyterhub-cost-monitoring@gh-pages.
@@ -26,34 +26,34 @@ jobs:
2626
contents: write
2727

2828
steps:
29-
- uses: actions/checkout@v4
30-
with:
29+
- uses: actions/checkout@v4
30+
with:
3131
# chartpress needs git history
32-
fetch-depth: 0
33-
34-
- uses: actions/setup-python@v5
35-
with:
36-
python-version: "3.x"
37-
38-
- name: Login to Quay.io
39-
uses: docker/login-action@v3
40-
with:
41-
registry: quay.io
42-
username: ${{ secrets.QUAY_USERNAME }}
43-
password: ${{ secrets.QUAY_PASSWORD }}
44-
45-
- name: Configure a git user
46-
run: |
47-
git config --global user.email "github-actions@github.com"
48-
git config --global user.name "github-actions"
49-
50-
- name: Install dependencies
51-
run: |
52-
pip install -r dev-requirements.txt
53-
pip list
54-
helm version
55-
56-
- name: Run chartpress
57-
run: chartpress --push --publish-chart
58-
env:
59-
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
32+
fetch-depth: 0
33+
34+
- uses: actions/setup-python@v5
35+
with:
36+
python-version: 3.x
37+
38+
- name: Login to Quay.io
39+
uses: docker/login-action@v3
40+
with:
41+
registry: quay.io
42+
username: ${{ secrets.QUAY_USERNAME }}
43+
password: ${{ secrets.QUAY_PASSWORD }}
44+
45+
- name: Configure a git user
46+
run: |
47+
git config --global user.email "github-actions@github.com"
48+
git config --global user.name "github-actions"
49+
50+
- name: Install dependencies
51+
run: |
52+
pip install -r dev-requirements.txt
53+
pip list
54+
helm version
55+
56+
- name: Run chartpress
57+
run: chartpress --push --publish-chart
58+
env:
59+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

chartpress.yaml

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -10,18 +10,18 @@
1010
#
1111

1212
charts:
13-
- name: jupyterhub-cost-monitoring
14-
chartPath: helm/jupyterhub-cost-monitoring
15-
imagePrefix: quay.io/2i2c/
13+
- name: jupyterhub-cost-monitoring
14+
chartPath: helm/jupyterhub-cost-monitoring
15+
imagePrefix: quay.io/2i2c/
1616
# Set dev version by taking latest tag and incrementing patch
17-
baseVersion: patch
17+
baseVersion: patch
1818

19-
repo:
20-
git: 2i2c-org/jupyterhub-cost-monitoring
21-
published: https://2i2c.org/jupyterhub-cost-monitoring/
19+
repo:
20+
git: 2i2c-org/jupyterhub-cost-monitoring
21+
published: https://2i2c.org/jupyterhub-cost-monitoring/
2222

23-
images:
24-
jupyterhub-cost-monitoring:
25-
dockerfilePath: Dockerfile
26-
contextPath: .
27-
valuesPath: image
23+
images:
24+
jupyterhub-cost-monitoring:
25+
dockerfilePath: Dockerfile
26+
contextPath: .
27+
valuesPath: image

pyproject.toml

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,10 +7,15 @@ requires-python = ">=3.12"
77
dependencies = [
88
"boto3>=1.39.12",
99
"flask>=3.1.1",
10+
"requests>=2.32.4",
11+
"yarl>=1.20.1",
1012
]
1113

1214
[dependency-groups]
1315
dev = [
1416
"mystmd>=1.6.0",
1517
"ruff>=0.12.5",
1618
]
19+
20+
[tool.isort]
21+
profile = "black"
Lines changed: 71 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -1,108 +1,155 @@
11
import logging
22
from datetime import datetime, timedelta, timezone
33

4-
from flask import Flask, request, url_for, render_template_string
4+
from flask import Flask, render_template_string, request, url_for
55

66
from .query_cost_aws import (
77
query_hub_names,
88
query_total_costs,
99
query_total_costs_per_component,
1010
query_total_costs_per_hub,
1111
)
12+
from .query_usage import query_usage
1213

1314
app = Flask(__name__)
1415
logging.basicConfig(level=logging.INFO)
1516

1617

17-
def _parse_from_to_in_query_params():
18+
def _parse_from_to_in_query_params(api_provider: str = "prometheus" or "aws"):
1819
"""
1920
Parse "from" and "to" query parameters, expected to be passed as YYYY-MM-DD
20-
formatted strings or including time as well.
21+
api_providerted strings or including time as well.
22+
23+
Args:
24+
api_provider (str): The api_provider, such as "prometheus" or "aws". Formats dates accordingly.
2125
2226
- "to" defaults to current date (UTC)
23-
- "from" defaults to 30 days before what to is set to
27+
- "from" defaults to 30 days prior to the "to" date
28+
29+
Returns:
30+
from_date and to_date as datetime.date objects, or their string api_providers, according to the `api_provider` argument.
2431
2532
Note that Python 3.11 is required to parse a datetime like
2633
2024-07-27T15:50:18.231Z with a Z in the end, and that Grafana's
2734
`${__from:date}` variable is UTC based, but as soon as its adjusted with a
28-
custom format, it no longer is UTC based. Due to that, we need to be able to
35+
custom api_provider, it no longer is UTC based. Due to that, we need to be able to
2936
parse the full datetime string.
3037
"""
31-
now_date = datetime.now(timezone.utc).date()
38+
now_date = datetime.now(timezone.utc).replace(
39+
hour=0, minute=0, second=0, microsecond=0
40+
)
3241
if request.args.get("to"):
33-
to_date = datetime.fromisoformat(request.args["to"]).date()
42+
to_date = datetime.fromisoformat(request.args["to"])
3443
else:
3544
to_date = now_date
3645
if request.args.get("from"):
37-
from_date = datetime.fromisoformat(request.args["from"]).date()
46+
from_date = datetime.fromisoformat(request.args["from"])
3847
else:
3948
from_date = to_date - timedelta(days=30)
4049

41-
# the to_date isn't included when declaring start/end dates against the AWS
42-
# CE API, so we try to add one day to it to make it inclusive
43-
to_date = to_date + timedelta(days=1)
4450
# prevent "end date past the beginning of next month" errors
4551
if to_date > now_date:
4652
to_date = now_date
4753
# prevent "Start date (and hour) should be before end date (and hour)"
4854
if from_date >= now_date:
4955
from_date = to_date - timedelta(days=1)
5056

51-
# format back to YYYY-MM-DD strings
52-
from_date = from_date.strftime("%Y-%m-%d")
53-
to_date = to_date.strftime("%Y-%m-%d")
57+
if api_provider == "prometheus":
58+
return from_date.isoformat(), to_date.isoformat()
59+
else:
60+
# "aws" to_dates are exclusive, so we add one day to include it
61+
to_date = to_date + timedelta(days=1)
62+
from_date = from_date.strftime("%Y-%m-%d")
63+
to_date = to_date.strftime("%Y-%m-%d")
64+
return from_date, to_date
5465

55-
return from_date, to_date
5666

5767
@app.route("/")
5868
def index():
69+
"""
70+
Index page that lists all available endpoints in the application.
71+
"""
5972
links = []
6073
for rule in app.url_map.iter_rules():
6174
# Skip static routes and those requiring parameters
62-
if rule.endpoint != 'static' and len(rule.arguments) == 0:
75+
if rule.endpoint != "static" and len(rule.arguments) == 0:
6376
url = url_for(rule.endpoint)
6477
links.append((rule.endpoint, url))
65-
78+
6679
# Render links using a simple HTML template
67-
return render_template_string('''
80+
return render_template_string(
81+
"""
6882
<h1>Available Endpoints</h1>
6983
<ul>
7084
{% for endpoint, url in links %}
7185
<li><a href="{{ url }}">{{ endpoint }}</a></li>
7286
{% endfor %}
7387
</ul>
74-
''', links=links)
88+
""",
89+
links=links,
90+
)
7591

7692

7793
@app.route("/health/ready")
7894
def ready():
95+
"""
96+
Readiness probe endpoint.
97+
"""
7998
return ("200: OK", 200)
8099

81100

82101
@app.route("/hub-names")
83102
def hub_names():
84-
from_date, to_date = _parse_from_to_in_query_params()
103+
"""
104+
Endpoint to query hub names.
105+
"""
106+
from_date, to_date = _parse_from_to_in_query_params(api_provider="aws")
85107

86108
return query_hub_names(from_date, to_date)
87109

88110

89111
@app.route("/total-costs")
90112
def total_costs():
91-
from_date, to_date = _parse_from_to_in_query_params()
113+
"""
114+
Endpoint to query total costs.
115+
"""
116+
from_date, to_date = _parse_from_to_in_query_params(api_provider="aws")
92117

93118
return query_total_costs(from_date, to_date)
94119

95120

96121
@app.route("/total-costs-per-hub")
97122
def total_costs_per_hub():
98-
from_date, to_date = _parse_from_to_in_query_params()
123+
"""
124+
Endpoint to query total costs per hub.
125+
"""
126+
from_date, to_date = _parse_from_to_in_query_params(api_provider="aws")
99127

100128
return query_total_costs_per_hub(from_date, to_date)
101129

102130

103131
@app.route("/total-costs-per-component")
104132
def total_costs_per_component():
105-
from_date, to_date = _parse_from_to_in_query_params()
133+
"""
134+
Endpoint to query total costs per component.
135+
"""
136+
from_date, to_date = _parse_from_to_in_query_params(api_provider="aws")
137+
hub_name = request.args.get("hub")
138+
component = request.args.get("component")
139+
140+
return query_total_costs_per_component(from_date, to_date, hub_name, component)
141+
142+
143+
@app.route("/total-usage")
144+
def total_usage():
145+
"""
146+
Endpoint to query total usage.
147+
Expects 'from' and 'to' query parameters in the api_provider YYYY-MM-DD.
148+
Optionally accepts 'hub', 'component' and 'user', query parameters.
149+
"""
150+
from_date, to_date = _parse_from_to_in_query_params(api_provider="prometheus")
106151
hub_name = request.args.get("hub")
152+
component_name = request.args.get("component")
153+
user_name = request.args.get("user")
107154

108-
return query_total_costs_per_component(from_date, to_date, hub_name)
155+
return query_usage(from_date, to_date, hub_name, component_name, user_name)
File renamed without changes.
Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
"""
2+
Constants used to query Prometheus for JupyterHub usage data.
3+
"""
4+
5+
TIME_RESOLUTION = "5m"
6+
7+
MEMORY_REQUESTS_PER_USER = """
8+
sum(
9+
kube_pod_container_resource_requests{resource=\"memory\", namespace=~\".*\", pod=~\"jupyter-.*\"} * on (namespace, pod)
10+
group_left(annotation_hub_jupyter_org_username) group(
11+
kube_pod_annotations{namespace=~\".*\", annotation_hub_jupyter_org_username=~\".*\"}
12+
) by (pod, namespace, annotation_hub_jupyter_org_username)
13+
) by (annotation_hub_jupyter_org_username, namespace)
14+
"""
15+
16+
USAGE_MAP = {
17+
"compute": MEMORY_REQUESTS_PER_USER,
18+
}

src/jupyterhub_cost_monitoring/query_cost_aws.py

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88
import boto3
99

1010
from .cache import ttl_lru_cache
11-
from .const import (
11+
from .const_cost_aws import (
1212
FILTER_ATTRIBUTABLE_COSTS,
1313
FILTER_USAGE_COSTS,
1414
GRANULARITY_DAILY,
@@ -270,13 +270,16 @@ def query_total_costs_per_hub(from_date, to_date):
270270

271271

272272
@ttl_lru_cache(seconds_to_live=3600)
273-
def query_total_costs_per_component(from_date, to_date, hub_name=None):
273+
def query_total_costs_per_component(from_date, to_date, hub_name=None, component=None):
274274
"""
275275
A query with processing of the response tailored to report total costs per
276276
component - a grouping of services.
277277
278278
If a hub_name is specified, component costs are filtered to only consider
279279
costs directly attributable to the hub name.
280+
281+
If a component is specified, the response is filtered to only include that
282+
component only.
280283
"""
281284
filter = {
282285
"And": [
@@ -387,18 +390,19 @@ def query_total_costs_per_component(from_date, to_date, hub_name=None):
387390
component_costs = {}
388391
for g in e["Groups"]:
389392
service_name = g["Keys"][0]
390-
name = _get_component_name(service_name)
393+
if not component:
394+
component = _get_component_name(service_name)
391395
cost = float(g["Metrics"]["UnblendedCost"]["Amount"])
392-
component_costs[name] = component_costs.get(name, 0.0) + cost
396+
component_costs[component] = component_costs.get(component, 0.0) + cost
393397

394398
processed_response.extend(
395399
[
396400
{
397401
"date": e["TimePeriod"]["Start"],
398402
"cost": f"{cost:.2f}",
399-
"name": name,
403+
"component": component,
400404
}
401-
for name, cost in component_costs.items()
405+
for component, cost in component_costs.items()
402406
]
403407
)
404408

0 commit comments

Comments
 (0)