Skip to content

Commit 4a7b2df

Browse files
shnizzedyshnizzedy
authored
🚚 Split Curious applets and REDCap projects (#25)
2 parents 9ea56ed + 693cc62 commit 4a7b2df

40 files changed

Lines changed: 3108 additions & 3274 deletions

‎CHANGELOG.md‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,60 @@ All notable changes to this project will be documented in this file.
55
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
66
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
77

8+
## 1.12.3
9+
10+
### Fixed
11+
12+
- Bug fetching REDCap data introduced in v1.12.1.
13+
14+
## 1.12.2
15+
16+
### Fixed
17+
18+
- Automatic deployment of services to `systemd`.
19+
20+
## 1.12.1
21+
22+
### Added
23+
24+
- `redcap-track-curious`: REDCap PID 891 `{instrument}_complete` → `"Curious Track.{instrument}_received".
25+
26+
### Deprecated
27+
28+
- `hbnmigration.from_redcap.from_redcap` `export_fields` parameter.
29+
30+
## 1.12.0
31+
32+
### Changed
33+
34+
- Copies participant information to both operations and Curious data REDCap projects from intake.
35+
- Moved logic for looking up REDCap indicies from this repository to `mindlogger-data-export`.
36+
37+
### Fixes
38+
39+
- Fixes bug where Curious-to-REDCap data were falling back on potentially incorrect values.
40+
41+
## 1.11.1
42+
43+
### Fixed
44+
45+
- Only send one email per participant to Curious when creating an account.
46+
47+
## 1.11.0
48+
49+
### Changed
50+
51+
- Split Curious into parent and child applets.
52+
- Split REDCap into Operations and Curious projects.
53+
54+
### Removed
55+
56+
- `._config_variables.curious_variables.curious_variables.activity_ids`
57+
- `._config_variables.curious_variables.curious_variables.applet_ids`
58+
- `._config_variables.curious_variables.curious_variables.Credentials`
59+
- `._config_variables.curious_variables.curious_variables.AppletCredentials.hbn_mindlogger`
60+
- `.from_redcap.config.Values.PID247.enrollment_complete`
61+
862
## 1.10.11
963

1064
### Fixed
@@ -153,6 +207,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
153207
- `._config_variables.curious_variables.curious_variables.applet_ids`
154208
- `._config_variables.curious_variables.curious_variables.Credentials`
155209
- `._config_variables.curious_variables.curious_variables.AppletCredentials.hbn_mindlogger`
210+
- `.from_redcap.config.Values.PID247.enrollment_complete`
156211

157212
## 1.8.0
158213

‎VERSION‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
1.10.11
1+
1.12.3

‎infrastructure/Makefile‎

Lines changed: 48 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -1,65 +1,73 @@
11
.PHONY: help test init plan apply destroy fmt validate lint security clean install-tools
22

33
help:
4-
@echo "Available targets:"
5-
@echo " test - Run all tests"
6-
@echo " install-tools - Install testing dependencies"
7-
@echo " init - Initialize Terraform"
8-
@echo " plan - Show execution plan"
9-
@echo " apply - Apply infrastructure changes"
10-
@echo " destroy - Destroy infrastructure"
11-
@echo " fmt - Format Terraform files"
12-
@echo " validate - Validate configuration"
13-
@echo " lint - Run tflint"
14-
@echo " security - Run security scan"
15-
@echo " clean - Clean up generated files"
4+
@echo "Available targets:"
5+
@echo " test - Run all tests"
6+
@echo " install-tools - Install testing dependencies"
7+
@echo " init - Initialize Terraform"
8+
@echo " install - Install services to systemd"
9+
@echo " plan - Show execution plan"
10+
@echo " apply - Apply infrastructure changes"
11+
@echo " destroy - Destroy infrastructure"
12+
@echo " fmt - Format Terraform files"
13+
@echo " validate - Validate configuration"
14+
@echo " lint - Run tflint"
15+
@echo " security - Run security scan"
16+
@echo " clean - Clean up generated files"
1617

1718
test:
18-
@./tests/test-terraform.sh
19+
@./tests/test-terraform.sh
1920

2021
install-tools:
21-
@chmod +x tests/install-tools.sh
22-
@./tests/install-tools.sh
22+
@chmod +x tests/install-tools.sh
23+
@./tests/install-tools.sh
2324

2425
init:
25-
terraform init
26+
terraform init
27+
28+
install:
29+
@echo "Installing services to /etc/systemd/system/..."
30+
sudo ln -sf $(PWD)/generated/*.service /etc/systemd/system/
31+
sudo ln -sf $(PWD)/generated/*.timer /etc/systemd/system/
32+
sudo systemctl daemon-reload
33+
@echo "Services installed. Use 'sudo systemctl start <name>' to run them."
2634

2735
plan: init
28-
terraform plan
36+
terraform plan
2937

3038
apply: test
31-
./safe-apply.sh
39+
./safe-apply.sh
3240

3341
destroy:
34-
terraform destroy
42+
terraform destroy
3543

3644
fmt:
37-
terraform fmt -recursive
45+
terraform fmt -recursive
3846

3947
validate: init
40-
terraform validate
48+
terraform validate
4149

4250
lint:
43-
@if command -v tflint >/dev/null 2>&1; then \
44-
tflint --config tests/.tflint.hcl --init; \
45-
tflint --config tests/.tflint.hcl; \
46-
else \
47-
echo "tflint not installed. Run: make install-tools"; \
48-
exit 1; \
49-
fi
51+
@if command -v tflint >/dev/null 2>&1; then \
52+
tflint --config tests/.tflint.hcl --init; \
53+
tflint --config tests/.tflint.hcl; \
54+
else \
55+
echo "tflint not installed. Run: make install-tools"; \
56+
exit 1; \
57+
fi
5058

5159
security:
52-
@if command -v checkov >/dev/null 2>&1; then \
53-
checkov -d . --config-file tests/.checkov.yaml; \
54-
else \
55-
echo "checkov not installed. Run: make install-tools"; \
56-
exit 1; \
57-
fi
60+
@if command -v checkov >/dev/null 2>&1; then \
61+
checkov -d . --config-file tests/.checkov.yaml; \
62+
else \
63+
echo "checkov not installed. Run: make install-tools"; \
64+
exit 1; \
65+
fi
5866

5967
clean:
60-
rm -rf .terraform
61-
rm -f .terraform.lock.hcl
62-
rm -f terraform.tfstate
63-
rm -f terraform.tfstate.backup
64-
rm -f plan.tfplan
65-
rm -rf generated/
68+
rm -rf .terraform
69+
rm -f .terraform.lock.hcl
70+
rm -f terraform.tfstate
71+
rm -f terraform.tfstate.backup
72+
rm -f plan.tfplan
73+
rm -rf generated/

‎infrastructure/main.tf‎

Lines changed: 18 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,17 +11,28 @@ terraform {
1111

1212
locals {
1313
workspace = terraform.workspace
14-
template_vars = {
14+
common_vars = {
1515
workspace = terraform.workspace
1616
user_group = var.user_group
1717
project_root = var.project_root
18-
venv_path = var.venv_path
18+
venv_path = "${var.project_root}/${var.venv_path}"
1919
log_directory = var.log_directory
2020
project_status = var.project_status
2121
recovery_mode = var.recovery_mode
2222
}
23+
24+
common_service_config = templatefile("${path.module}/services/_common_metadata.tpl", local.common_vars)
25+
oneshot_timeout_config = templatefile("${path.module}/services/_oneshot_timeouts.tpl", local.common_vars)
26+
async_timeout_config = templatefile("${path.module}/services/_async_timeouts.tpl", local.common_vars)
27+
28+
template_vars = merge(local.common_vars, {
29+
common_config = local.common_service_config
30+
oneshot_timeouts = local.oneshot_timeout_config,
31+
async_timeouts = local.async_timeout_config
32+
})
2333
}
2434

35+
2536
# Webhook services (long-running uvicorn servers)
2637
resource "local_file" "redcap_to_redcap_service" {
2738
content = templatefile("${path.module}/services/redcap-to-redcap.service.tpl", local.template_vars)
@@ -80,3 +91,8 @@ resource "local_file" "hbn_sync_timer" {
8091
})
8192
filename = "${path.module}/generated/hbn-sync.timer"
8293
}
94+
95+
resource "local_file" "redcap_track_curious_service" {
96+
content = templatefile("${path.module}/services/redcap-track-curious.service.tpl", local.template_vars)
97+
filename = "${path.module}/generated/redcap-track-curious.service"
98+
}

‎infrastructure/services/_async_timeouts.tpl‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
# Timeouts
2+
TimeoutStartSec=60
3+
TimeoutStopSec=30
4+
Restart=always
5+
RestartSec=10

‎infrastructure/services/_common_metadata.tpl‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
User=${user_group}
2+
Group=${user_group}
3+
WorkingDirectory=${project_root}
4+
Environment="WORKSPACE=${workspace}"
5+
Environment="HBNMIGRATION_PROJECT_ROOT=${project_root}"
6+
Environment="HBNMIGRATION_LOG_ROOT=${log_directory}"
7+
Environment="HBNMIGRATION_PROJECT_STATUS=${project_status}"
8+
Environment="HBNMIGRATION_RECOVERY_MODE=${recovery_mode ? "1" : "0"}"
9+
Environment="PATH=${venv_path}/bin:/usr/local/bin:/usr/bin:/bin"
10+
Environment="PYTHONPATH=${project_root}/python_jobs/src"
11+
12+
# Security hardening
13+
NoNewPrivileges=true
14+
PrivateTmp=true
15+
ProtectSystem=strict
16+
ReadWritePaths=${project_root}
17+
ReadWritePaths=${log_directory}
18+
BindPaths=/data/logs/hbnmigration:/home/hbnmigration/hbnmigration/.hbnmigration_logs

‎infrastructure/services/_oneshot_timeouts.tpl‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
# Timeouts
2+
TimeoutStartSec=300
3+
TimeoutStopSec=30

‎infrastructure/services/curious-accounts-to-redcap.service.tpl‎

Lines changed: 3 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -5,33 +5,17 @@ PartOf=hbn-sync.service
55

66
[Service]
77
Type=oneshot
8-
User=${user_group}
9-
Group=${user_group}
10-
WorkingDirectory=${project_root}
118
ExecStart=${venv_path}/bin/curious-invitations-to-redcap
12-
Environment="WORKSPACE=${workspace}"
13-
Environment="HBNMIGRATION_PROJECT_ROOT=${project_root}"
14-
Environment="HBNMIGRATION_LOG_ROOT=${log_directory}"
15-
Environment="HBNMIGRATION_PROJECT_STATUS=${project_status}"
16-
Environment="HBNMIGRATION_RECOVERY_MODE=${recovery_mode ? "1" : "0"}"
179

18-
# Timeouts
19-
TimeoutStartSec=300
20-
TimeoutStopSec=30
10+
# Inject SSOT common config
11+
${common_config}
12+
${oneshot_timeouts}
2113

2214
# Logging
23-
BindPaths=/data/logs/hbnmigration:/home/hbnmigration/hbnmigration/.hbnmigration_logs
2415
StandardOutput=append:${log_directory}/curious-invitations-to-redcap.log
2516
StandardError=append:${log_directory}/curious-invitations-to-redcap.log
2617
SyslogIdentifier=curious-invitations-to-redcap-${workspace}
2718

28-
# Security hardening
29-
NoNewPrivileges=true
30-
PrivateTmp=true
31-
ProtectSystem=strict
32-
ReadWritePaths=${project_root}
33-
ReadWritePaths=${log_directory}
34-
ReadWritePaths=${project_root}/.hbnmigration_cache
3519

3620
[Install]
3721
WantedBy=hbn-sync.timer

‎infrastructure/services/curious-alerts-websocket.service.tpl‎

Lines changed: 3 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -4,35 +4,19 @@ After=network.target
44

55
[Service]
66
Type=simple
7-
User=${user_group}
8-
Group=${user_group}
9-
WorkingDirectory=${project_root}
107
ExecStart=${venv_path}/bin/curious-alerts-to-redcap --asynchronous
11-
Environment="WORKSPACE=${workspace}"
12-
Environment="HBNMIGRATION_PROJECT_ROOT=${project_root}"
13-
Environment="HBNMIGRATION_LOG_ROOT=${log_directory}"
14-
Environment="HBNMIGRATION_PROJECT_STATUS=${project_status}"
15-
Environment="HBNMIGRATION_RECOVERY_MODE=${recovery_mode ? "1" : "0"}"
8+
9+
# Inject SSOT common config
10+
${common_config}
1611

1712
# Timeouts
1813
TimeoutStartSec=60
1914
TimeoutStopSec=30
2015

21-
Restart=always
22-
RestartSec=10
23-
2416
# Logging
25-
BindPaths=/data/logs/hbnmigration:/home/hbnmigration/hbnmigration/.hbnmigration_logs
2617
StandardOutput=append:${log_directory}/curious-alerts-websocket.log
2718
StandardError=append:${log_directory}/curious-alerts-websocket-error.log
2819
SyslogIdentifier=curious-alerts-websocket-${workspace}
2920

30-
# Security hardening
31-
NoNewPrivileges=true
32-
PrivateTmp=true
33-
ProtectSystem=strict
34-
ReadWritePaths=${project_root}
35-
ReadWritePaths=${log_directory}
36-
3721
[Install]
3822
WantedBy=multi-user.target

‎infrastructure/services/curious-data-to-redcap.service.tpl‎

Lines changed: 4 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -5,33 +5,17 @@ PartOf=hbn-sync.service
55

66
[Service]
77
Type=oneshot
8-
User=${user_group}
9-
Group=${user_group}
10-
WorkingDirectory=${project_root}
118
ExecStart=${venv_path}/bin/curious-data-to-redcap
12-
Environment="WORKSPACE=${workspace}"
13-
Environment="HBNMIGRATION_PROJECT_ROOT=${project_root}"
14-
Environment="HBNMIGRATION_LOG_ROOT=${log_directory}"
15-
Environment="HBNMIGRATION_PROJECT_STATUS=${project_status}"
16-
Environment="HBNMIGRATION_RECOVERY_MODE=${recovery_mode ? "1" : "0"}"
179

18-
# Timeouts
19-
TimeoutStartSec=300
20-
TimeoutStopSec=30
10+
# Inject SSOT common config
11+
${common_config}
12+
${oneshot_timeouts}
2113

2214
# Logging
23-
BindPaths=/data/logs/hbnmigration:/home/hbnmigration/hbnmigration/.hbnmigration_logs
15+
2416
StandardOutput=append:${log_directory}/curious-data-to-redcap.log
2517
StandardError=append:${log_directory}/curious-data-to-redcap.log
2618
SyslogIdentifier=curious-data-to-redcap-${workspace}
2719

28-
# Security hardening
29-
NoNewPrivileges=true
30-
PrivateTmp=true
31-
ProtectSystem=strict
32-
ReadWritePaths=${project_root}
33-
ReadWritePaths=${log_directory}
34-
ReadWritePaths=${project_root}/.hbnmigration_cache
35-
3620
[Install]
3721
WantedBy=hbn-sync.timer

0 commit comments

Comments
 (0)