Skip to content

Commit dfe17c9

Browse files
isegall-dacocreature
isegall-da
and
cocreature
authored
Use Postgres for Grafana backend (#2494)
--------- Signed-off-by: Itai Segall <itai.segall@digitalasset.com> Signed-off-by: Moritz Kiefer <moritz.kiefer@purelyfunctional.org> Co-authored-by: Moritz Kiefer <moritz.kiefer@purelyfunctional.org>
1 parent 44772b0 commit dfe17c9

File tree

14 files changed

+149
-74
lines changed

14 files changed

+149
-74
lines changed

‎build-tools/cncluster‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1346,15 +1346,14 @@ function _unlock() {
13461346
}
13471347

13481348
function _check_stateful_sets() {
1349-
# Checks if any StatefulSets EXCEPT those with a prometheus image (which are part of the infra stack)
1349+
# Checks if any StatefulSets EXCEPT those in the observability namespace (which are part of the infra stack)
13501350
# are currently deployed
13511351
# grep exits with return code 1 if it finds no matches so we account for that
1352-
query_result=$(kubectl get StatefulSets -A -o json 2> /dev/null | jq '.items[].spec.template.spec.containers[].image' | { grep -v prometheus || test $? = 1; })
1353-
echo "$query_result"
1352+
kubectl get statefulset -A -o json | jq '.items[] | select(.metadata.namespace != "observability" and .metadata.namespace != "gmp-system")'
13541353
}
13551354

13561355
function _check_reset_and_unlock() {
1357-
s=$(_check_stateful_sets)
1356+
s="$(_check_stateful_sets)"
13581357

13591358
if [ -n "$s" ]; then
13601359
echo -e "Cluster not reset, it is recommended to reset it before unlocking. Reset now?"

‎cluster/expected/canton-network/expected.json‎

Lines changed: 0 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -110,14 +110,6 @@
110110
"provider": "",
111111
"type": "kubernetes:core/v1:Secret"
112112
},
113-
{
114-
"custom": true,
115-
"id": "projects/test-project/global/networks/default",
116-
"inputs": {},
117-
"name": "default",
118-
"provider": "",
119-
"type": "gcp:compute/network:Network"
120-
},
121113
{
122114
"custom": true,
123115
"id": "",

‎cluster/expected/deployment/expected.json‎

Lines changed: 0 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -283,14 +283,6 @@
283283
"provider": "",
284284
"type": "kubernetes:pulumi.com/v1:Stack"
285285
},
286-
{
287-
"custom": true,
288-
"id": "projects/test-project/global/networks/default",
289-
"inputs": {},
290-
"name": "default",
291-
"provider": "",
292-
"type": "gcp:compute/network:Network"
293-
},
294286
{
295287
"custom": true,
296288
"id": "",

‎cluster/expected/infra/expected.json‎

Lines changed: 106 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -593,6 +593,28 @@
593593
"provider": "",
594594
"type": "kubernetes:core/v1:Namespace"
595595
},
596+
{
597+
"custom": true,
598+
"id": "",
599+
"inputs": {
600+
"apiVersion": "v1",
601+
"data": {
602+
"4dabf18193072939515e22adb298388d": "1b47061264138c4ac30d75fd1eb44270",
603+
"value": {
604+
"postgresPassword": ""
605+
}
606+
},
607+
"kind": "Secret",
608+
"metadata": {
609+
"name": "grafana-pg-secret",
610+
"namespace": "observability"
611+
},
612+
"type": "Opaque"
613+
},
614+
"name": "cn-app-observability-grafana-pg-secret",
615+
"provider": "",
616+
"type": "kubernetes:core/v1:Secret"
617+
},
596618
{
597619
"custom": true,
598620
"id": "",
@@ -2405,6 +2427,80 @@
24052427
"provider": "",
24062428
"type": "gcp:compute/routerNat:RouterNat"
24072429
},
2430+
{
2431+
"custom": true,
2432+
"id": "",
2433+
"inputs": {
2434+
"length": 16,
2435+
"overrideSpecial": "_%@",
2436+
"special": true
2437+
},
2438+
"name": "observability-grafana-pg-passwd",
2439+
"provider": "",
2440+
"type": "random:index/randomPassword:RandomPassword"
2441+
},
2442+
{
2443+
"custom": false,
2444+
"id": "",
2445+
"inputs": {},
2446+
"name": "observability-grafana-pg",
2447+
"provider": "",
2448+
"type": "canton:network:postgres"
2449+
},
2450+
{
2451+
"custom": true,
2452+
"id": "",
2453+
"inputs": {
2454+
"chart": "oci://ghcr.io/digital-asset/decentralized-canton-sync-dev/helm/splice-postgres",
2455+
"compat": "true",
2456+
"maxHistory": 10,
2457+
"name": "grafana-pg",
2458+
"namespace": "observability",
2459+
"timeout": 600,
2460+
"values": {
2461+
"affinity": {
2462+
"nodeAffinity": {
2463+
"requiredDuringSchedulingIgnoredDuringExecution": {
2464+
"nodeSelectorTerms": [
2465+
{
2466+
"matchExpressions": [
2467+
{
2468+
"key": "cn_apps",
2469+
"operator": "Exists"
2470+
}
2471+
]
2472+
}
2473+
]
2474+
}
2475+
}
2476+
},
2477+
"cluster": {
2478+
"dnsName": "mock.global.canton.network.digitalasset.com",
2479+
"fixedTokens": false,
2480+
"hostname": "mock.global.canton.network.digitalasset.com",
2481+
"name": "cn-mocknet"
2482+
},
2483+
"db": {
2484+
"volumeSize": "20Gi"
2485+
},
2486+
"imageRepo": "us-central1-docker.pkg.dev/da-cn-shared/ghcr/digital-asset/decentralized-canton-sync-dev/docker",
2487+
"persistence": {
2488+
"secretName": "grafana-pg-secret"
2489+
},
2490+
"tolerations": [
2491+
{
2492+
"effect": "NoSchedule",
2493+
"key": "cn_apps",
2494+
"operator": "Exists"
2495+
}
2496+
]
2497+
},
2498+
"version": "0.3.20"
2499+
},
2500+
"name": "observability-grafana-pg",
2501+
"provider": "",
2502+
"type": "kubernetes:helm.sh/v3:Release"
2503+
},
24082504
{
24092505
"custom": true,
24102506
"id": "",
@@ -2513,6 +2609,7 @@
25132609
"coreDns": {
25142610
"enabled": false
25152611
},
2612+
"database": {},
25162613
"defaultRules": {
25172614
"create": true
25182615
},
@@ -2609,8 +2706,16 @@
26092706
"deploymentStrategy": {
26102707
"type": "Recreate"
26112708
},
2709+
"envFromSecret": "grafana-pg-secret",
26122710
"fullnameOverride": "grafana",
26132711
"grafana.ini": {
2712+
"database": {
2713+
"host": "grafana-pg.observability.svc.cluster.local:5432",
2714+
"name": "cantonnet",
2715+
"password": "${postgresPassword}",
2716+
"type": "postgres",
2717+
"user": "cnadmin"
2718+
},
26142719
"date_formats": {
26152720
"default_timezone": "UTC"
26162721
},
@@ -2982,7 +3087,7 @@
29823087
"name": "observability"
29833088
}
29843089
},
2985-
"name": "observabilty",
3090+
"name": "observability",
29863091
"provider": "",
29873092
"type": "kubernetes:core/v1:Namespace"
29883093
},

‎cluster/expected/splitwell/expected.json‎

Lines changed: 0 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -87,14 +87,6 @@
8787
"provider": "",
8888
"type": "kubernetes:core/v1:Secret"
8989
},
90-
{
91-
"custom": true,
92-
"id": "projects/test-project/global/networks/default",
93-
"inputs": {},
94-
"name": "default",
95-
"provider": "",
96-
"type": "gcp:compute/network:Network"
97-
},
9890
{
9991
"custom": true,
10092
"id": "",

‎cluster/expected/sv-canton/expected.json‎

Lines changed: 0 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -837,14 +837,6 @@
837837
"provider": "",
838838
"type": "kubernetes:core/v1:Secret"
839839
},
840-
{
841-
"custom": true,
842-
"id": "projects/test-project/global/networks/default",
843-
"inputs": {},
844-
"name": "default",
845-
"provider": "",
846-
"type": "gcp:compute/network:Network"
847-
},
848840
{
849841
"custom": true,
850842
"id": "",

‎cluster/expected/sv-runbook/expected.json‎

Lines changed: 0 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -170,14 +170,6 @@
170170
"provider": "",
171171
"type": "kubernetes:core/v1:Secret"
172172
},
173-
{
174-
"custom": true,
175-
"id": "projects/test-project/global/networks/default",
176-
"inputs": {},
177-
"name": "default",
178-
"provider": "",
179-
"type": "gcp:compute/network:Network"
180-
},
181173
{
182174
"custom": true,
183175
"id": "",

‎cluster/expected/validator-runbook/expected.json‎

Lines changed: 0 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -135,14 +135,6 @@
135135
"provider": "",
136136
"type": "kubernetes:core/v1:Secret"
137137
},
138-
{
139-
"custom": true,
140-
"id": "projects/test-project/global/networks/default",
141-
"inputs": {},
142-
"name": "default",
143-
"provider": "",
144-
"type": "gcp:compute/network:Network"
145-
},
146138
{
147139
"custom": true,
148140
"id": "",

‎cluster/expected/validator1/expected.json‎

Lines changed: 0 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -65,14 +65,6 @@
6565
"provider": "",
6666
"type": "kubernetes:core/v1:Secret"
6767
},
68-
{
69-
"custom": true,
70-
"id": "projects/test-project/global/networks/default",
71-
"inputs": {},
72-
"name": "default",
73-
"provider": "",
74-
"type": "gcp:compute/network:Network"
75-
},
7668
{
7769
"custom": true,
7870
"id": "",

‎cluster/pulumi/canton-network/src/bigQuery.ts‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ import {
1414
Postgres,
1515
CloudPostgres,
1616
generatePassword,
17-
privateNetwork,
17+
privateNetworkId,
1818
protectCloudSql,
1919
} from '@lfdecentralizedtrust/splice-pulumi-common/src/postgres';
2020
import {
@@ -367,7 +367,7 @@ function installPrivateConnectivityConfiguration(
367367
privateConnectionId: privateConnectionName,
368368
displayName: privateConnectionName,
369369
location: cloudsdkComputeRegion(),
370-
vpcPeeringConfig: { subnet: pickDatastreamPeeringCidr(), vpc: privateNetwork.id },
370+
vpcPeeringConfig: { subnet: pickDatastreamPeeringCidr(), vpc: privateNetworkId },
371371
labels: {
372372
cluster: CLUSTER_BASENAME,
373373
},
@@ -450,7 +450,7 @@ function createPublicationAndReplicationSlots(
450450
const schemaName = dbName;
451451
const path = commandScriptPath('cluster/pulumi/canton-network/bigquery-cloudsql.sh');
452452
const scriptArgs = pulumi.interpolate`\\
453-
--private-network-project="${privateNetwork.project}" \\
453+
--private-network-project="${gcp.organizations.getProjectOutput({})}" \\
454454
--compute-region="${cloudsdkComputeRegion()}" \\
455455
--service-account-email="${postgres.databaseInstance.serviceAccountEmailAddress}" \\
456456
--tables-to-replicate-length="${tablesToReplicate.length}" \\

0 commit comments

Comments
 (0)