Skip to content

Commit 15a96da

Browse files
shnizzedyshnizzedy
committed
🧱 Restore batch jobs until updated permissions
1 parent bf610e2 commit 15a96da

13 files changed

Lines changed: 142 additions & 240 deletions

‎CHANGELOG.md‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file.
55
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
66
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
77

8+
## 1.10.2
9+
10+
### Changed
11+
12+
- Restored minute-by-minute jobs pending AWS permission update.
13+
814
## 1.10.1
915

1016
### Added

‎VERSION‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
1.10.1
1+
1.10.2

‎infrastructure/main.tf‎

Lines changed: 39 additions & 102 deletions
Original file line numberDiff line numberDiff line change
@@ -2,134 +2,71 @@ terraform {
22
required_version = ">= 1.0"
33

44
required_providers {
5-
aws = {
6-
source = "hashicorp/aws"
7-
version = "~> 5.0"
8-
}
95
local = {
106
source = "hashicorp/local"
117
version = "~> 2.4"
128
}
139
}
1410
}
1511

16-
provider "aws" {
17-
region = var.aws_region
18-
19-
default_tags {
20-
tags = {
21-
Project = "HBN Migration"
22-
Environment = var.environment
23-
ManagedBy = "Terraform"
24-
}
25-
}
12+
locals {
13+
workspace = terraform.workspace
2614
}
2715

28-
# EC2 Instance
29-
resource "aws_instance" "hbn_migration" {
30-
ami = var.ami_id
31-
instance_type = var.instance_type
32-
key_name = var.key_name
33-
ebs_optimized = true
34-
monitoring = true
35-
36-
vpc_security_group_ids = [aws_security_group.hbn_migration.id]
37-
subnet_id = var.subnet_id
38-
39-
metadata_options {
40-
http_endpoint = "enabled"
41-
http_tokens = "required"
42-
}
43-
44-
root_block_device {
45-
encrypted = true
46-
}
47-
48-
user_data = templatefile("${path.module}/user_data.sh", {
49-
github_repo = var.github_repo
50-
github_branch = var.github_branch
51-
working_directory = var.working_directory
52-
python_venv = var.python_venv
16+
# Webhook services (long-running uvicorn servers)
17+
resource "local_file" "redcap_to_redcap_service" {
18+
content = templatefile("${path.module}/services/redcap-to-redcap.service.tpl", {
5319
service_user = var.service_user
5420
service_group = var.service_group
21+
working_directory = var.working_directory
22+
python_venv = var.python_venv
5523
})
56-
57-
tags = {
58-
Name = "hbn-migration-${var.environment}"
59-
}
60-
61-
lifecycle {
62-
create_before_destroy = true
63-
}
24+
filename = "${path.module}/generated/redcap-to-redcap.service"
6425
}
6526

66-
# Security Group
67-
resource "aws_security_group" "hbn_migration" {
68-
name = "hbn-migration-${var.environment}"
69-
description = "Security group for HBN migration webhook services"
70-
vpc_id = var.vpc_id
71-
72-
# SSH access (conditional - only if CIDRs specified)
73-
dynamic "ingress" {
74-
for_each = length(var.ssh_allowed_cidrs) > 0 ? [1] : []
75-
content {
76-
from_port = 22
77-
to_port = 22
78-
protocol = "tcp"
79-
cidr_blocks = var.ssh_allowed_cidrs
80-
description = "SSH access"
81-
}
82-
}
83-
84-
# REDCap to REDCap webhook
85-
ingress {
86-
from_port = 8001
87-
to_port = 8001
88-
protocol = "tcp"
89-
cidr_blocks = var.webhook_allowed_cidrs
90-
description = "REDCap to Intake webhook"
91-
}
92-
93-
# REDCap to Curious webhook
94-
ingress {
95-
from_port = 8002
96-
to_port = 8002
97-
protocol = "tcp"
98-
cidr_blocks = var.webhook_allowed_cidrs
99-
description = "REDCap to Curious webhook"
100-
}
101-
102-
# Outbound traffic (required for apt, pip, API calls to REDCap/Curious)
103-
egress {
104-
from_port = 0
105-
to_port = 0
106-
protocol = "-1"
107-
cidr_blocks = ["0.0.0.0/0"]
108-
description = "All outbound traffic (required for updates and API calls)"
109-
}
110-
111-
tags = {
112-
Name = "hbn-migration-${var.environment}"
113-
}
27+
resource "local_file" "redcap_to_curious_service" {
28+
content = templatefile("${path.module}/services/redcap-to-curious.service.tpl", {
29+
service_user = var.service_user
30+
service_group = var.service_group
31+
working_directory = var.working_directory
32+
python_venv = var.python_venv
33+
})
34+
filename = "${path.module}/generated/redcap-to-curious.service"
11435
}
11536

116-
# Generate systemd service files
117-
resource "local_file" "redcap_to_redcap_service" {
118-
content = templatefile("${path.module}/services/redcap-to-redcap.service.tpl", {
37+
# Batch services (oneshot, triggered by timer)
38+
resource "local_file" "redcap_to_redcap_batch_service" {
39+
content = templatefile("${path.module}/services/redcap-to-redcap-batch.service.tpl", {
11940
service_user = var.service_user
12041
service_group = var.service_group
12142
working_directory = var.working_directory
12243
python_venv = var.python_venv
12344
})
124-
filename = "${path.module}/generated/redcap-to-redcap.service"
45+
filename = "${path.module}/generated/redcap-to-redcap-batch.service"
12546
}
12647

127-
resource "local_file" "redcap_to_curious_service" {
128-
content = templatefile("${path.module}/services/redcap-to-curious.service.tpl", {
48+
resource "local_file" "redcap_to_curious_batch_service" {
49+
content = templatefile("${path.module}/services/redcap-to-curious-batch.service.tpl", {
12950
service_user = var.service_user
13051
service_group = var.service_group
13152
working_directory = var.working_directory
13253
python_venv = var.python_venv
13354
})
134-
filename = "${path.module}/generated/redcap-to-curious.service"
55+
filename = "${path.module}/generated/redcap-to-curious-batch.service"
56+
}
57+
58+
# Timer and sync target
59+
resource "local_file" "hbn_sync_service" {
60+
content = templatefile("${path.module}/services/hbn-sync.service.tpl", {
61+
workspace = local.workspace
62+
})
63+
filename = "${path.module}/generated/hbn-sync.service"
64+
}
65+
66+
resource "local_file" "hbn_sync_timer" {
67+
content = templatefile("${path.module}/services/hbn-sync.timer.tpl", {
68+
workspace = local.workspace
69+
sync_interval_minutes = var.sync_interval_minutes
70+
})
71+
filename = "${path.module}/generated/hbn-sync.timer"
13572
}

‎infrastructure/outputs.tf‎

Lines changed: 35 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -1,42 +1,46 @@
1-
output "instance_id" {
2-
description = "EC2 instance ID"
3-
value = aws_instance.hbn_migration.id
1+
output "redcap_to_intake_webhook_path" {
2+
description = "Webhook path for REDCap to Intake (configure in REDCap Data Entry Trigger)"
3+
value = "/webhook/redcap-to-intake (port 8001)"
44
}
55

6-
output "instance_public_ip" {
7-
description = "Public IP of EC2 instance"
8-
value = aws_instance.hbn_migration.public_ip
6+
output "redcap_to_curious_webhook_path" {
7+
description = "Webhook path for REDCap to Curious (configure in REDCap Data Entry Trigger)"
8+
value = "/webhook/redcap-to-curious (port 8002)"
99
}
1010

11-
output "instance_private_ip" {
12-
description = "Private IP of EC2 instance"
13-
value = aws_instance.hbn_migration.private_ip
14-
}
15-
16-
output "redcap_to_intake_webhook_url" {
17-
description = "Webhook URL for REDCap to Intake (configure in REDCap Data Entry Trigger)"
18-
value = "http://${aws_instance.hbn_migration.public_ip}:8001/webhook/redcap-to-intake"
19-
}
20-
21-
output "redcap_to_curious_webhook_url" {
22-
description = "Webhook URL for REDCap to Curious (configure in REDCap Data Entry Trigger)"
23-
value = "http://${aws_instance.hbn_migration.public_ip}:8002/webhook/redcap-to-curious"
11+
output "service_commands" {
12+
description = "Commands to manage services (run on the VM)"
13+
value = {
14+
install_webhooks = "sudo cp generated/redcap-to-redcap.service generated/redcap-to-curious.service /etc/systemd/system/ && sudo systemctl daemon-reload"
15+
install_timer = "sudo cp generated/hbn-sync.service generated/hbn-sync.timer /etc/systemd/system/ && sudo systemctl daemon-reload"
16+
enable_webhooks = "sudo systemctl enable --now redcap-to-redcap.service redcap-to-curious.service"
17+
enable_timer = "sudo systemctl enable --now hbn-sync.timer"
18+
disable_timer = "sudo systemctl disable --now hbn-sync.timer"
19+
status = "sudo systemctl status redcap-to-redcap.service redcap-to-curious.service hbn-sync.timer"
20+
logs = "sudo journalctl -u redcap-to-redcap.service -u redcap-to-curious.service -f"
21+
}
2422
}
2523

26-
output "ssh_command" {
27-
description = "SSH command to connect"
28-
value = "ssh -i /path/to/${var.key_name}.pem ubuntu@${aws_instance.hbn_migration.public_ip}"
24+
output "health_check_commands" {
25+
description = "Commands to check service health (run on the VM)"
26+
value = {
27+
redcap_to_intake = "curl http://localhost:8001/health"
28+
redcap_to_curious = "curl http://localhost:8002/health"
29+
}
2930
}
3031

31-
output "security_group_id" {
32-
description = "Security group ID"
33-
value = aws_security_group.hbn_migration.id
32+
output "migration_note" {
33+
description = "Steps to migrate from timer to webhooks"
34+
value = <<-EOT
35+
Once security group rules are in place for ports 8001/8002:
36+
1. Enable webhooks: sudo systemctl enable --now redcap-to-redcap.service redcap-to-curious.service
37+
2. Configure REDCap Data Entry Triggers
38+
3. Test webhooks: curl http://localhost:8001/health
39+
4. Disable timer: sudo systemctl disable --now hbn-sync.timer
40+
EOT
3441
}
3542

36-
output "health_check_commands" {
37-
description = "Commands to check service health"
38-
value = {
39-
redcap_to_intake = "curl http://${aws_instance.hbn_migration.public_ip}:8001/health"
40-
redcap_to_curious = "curl http://${aws_instance.hbn_migration.public_ip}:8002/health"
41-
}
43+
output "security_group_note" {
44+
description = "Manual step required for webhooks"
45+
value = "Ask AWS admin to add inbound rules for ports 8001 and 8002 (TCP) from your REDCap server IP(s)"
4246
}

‎infrastructure/services/hbn-sync.service.tpl‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
[Unit]
22
Description=HBN Sync Service - Runs All Sync Services [${workspace}]
33
After=network.target
4-
Wants=ripple-to-redcap.service redcap-to-redcap.service redcap-to-curious.service curious-accounts-to-redcap.service curious-data-to-redcap.service
4+
Wants=ripple-to-redcap.service redcap-to-redcap-batch.service redcap-to-curious-batch.service curious-accounts-to-redcap.service curious-data-to-redcap.service
55

66
[Service]
77
Type=oneshot

‎infrastructure/services/redcap-to-curious-batch.service.tpl‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
[Unit]
2+
Description=REDCap to Curious Batch Sync
3+
After=network.target
4+
5+
[Service]
6+
Type=oneshot
7+
User=${service_user}
8+
Group=${service_group}
9+
WorkingDirectory=${working_directory}
10+
Environment="PATH=${python_venv}/bin:/usr/local/bin:/usr/bin:/bin"
11+
Environment="PYTHONPATH=${working_directory}/python_jobs/src"
12+
ExecStart=${python_venv}/bin/redcap-to-curious
13+
StandardOutput=journal
14+
StandardError=journal
15+
SyslogIdentifier=redcap-to-curious-batch
16+
17+
[Install]
18+
WantedBy=multi-user.target

‎infrastructure/services/redcap-to-redcap-batch.service.tpl‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
[Unit]
2+
Description=REDCap to REDCap Batch Sync
3+
After=network.target
4+
5+
[Service]
6+
Type=oneshot
7+
User=${service_user}
8+
Group=${service_group}
9+
WorkingDirectory=${working_directory}
10+
Environment="PATH=${python_venv}/bin:/usr/local/bin:/usr/bin:/bin"
11+
Environment="PYTHONPATH=${working_directory}/python_jobs/src"
12+
ExecStart=${python_venv}/bin/redcap-to-redcap
13+
StandardOutput=journal
14+
StandardError=journal
15+
SyslogIdentifier=redcap-to-redcap-batch
16+
17+
[Install]
18+
WantedBy=multi-user.target

‎infrastructure/tests/.checkov.yaml‎

Lines changed: 1 addition & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,8 @@
11
branch: main
22
compact: true
33
quiet: false
4-
skip-check:
5-
# SSH ingress - controlled by ssh_allowed_cidrs variable, conditional rule only
6-
# created when CIDRs are specified. Checkov can't evaluate dynamic blocks.
7-
- CKV_AWS_24
8-
9-
# Egress to 0.0.0.0/0 - required for outbound API calls to REDCap, Curious,
10-
# and MindLogger, plus apt and pip for system/package updates.
11-
- CKV_AWS_382
124

5+
# No AWS resources to scan - only local file generation
136
framework:
147
- terraform
158

‎infrastructure/tests/.tflint.hcl‎

Lines changed: 0 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -3,12 +3,6 @@ plugin "terraform" {
33
preset = "recommended"
44
}
55

6-
plugin "aws" {
7-
enabled = true
8-
version = "0.29.0"
9-
source = "github.com/terraform-linters/tflint-ruleset-aws"
10-
}
11-
126
rule "terraform_naming_convention" {
137
enabled = true
148
format = "snake_case"
@@ -37,7 +31,3 @@ rule "terraform_required_version" {
3731
rule "terraform_required_providers" {
3832
enabled = true
3933
}
40-
41-
rule "aws_instance_invalid_type" {
42-
enabled = true
43-
}

‎infrastructure/tests/test-terraform.sh‎

Lines changed: 11 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -46,16 +46,7 @@ run_test "Terraform init" "terraform init -backend=false"
4646
# Test 3: Validation
4747
run_test "Terraform validate" "terraform validate"
4848

49-
# Test 4: Security scan with Checkov (if installed)
50-
if command -v checkov &> /dev/null; then
51-
run_test "Security scan (Checkov)" "checkov -d . --quiet --compact --config-file tests/.checkov.yaml"
52-
else
53-
echo -e "${YELLOW}⊘ SKIPPED: Security scan (Checkov not installed)${NC}"
54-
echo " Install with: pip install checkov"
55-
echo ""
56-
fi
57-
58-
# Test 5: Lint with tflint (if installed)
49+
# Test 4: Lint with tflint (if installed)
5950
if command -v tflint &> /dev/null; then
6051
run_test "Terraform lint (tflint)" "tflint --config tests/.tflint.hcl --init && tflint --config tests/.tflint.hcl"
6152
else
@@ -64,16 +55,19 @@ else
6455
echo ""
6556
fi
6657

67-
# Test 6: Check required files exist
58+
# Test 5: Check required files exist
6859
run_test "Required files exist" "test -f main.tf && test -f variables.tf && test -f outputs.tf"
6960

70-
# Test 7: Check service templates exist
71-
run_test "Service templates exist" "test -f services/redcap-to-redcap.service.tpl && test -f services/redcap-to-curious.service.tpl"
72-
73-
# Test 8: Check user_data script exists
74-
run_test "User data script exists" "test -f user_data.sh"
61+
# Test 6: Check service templates exist
62+
run_test "Service templates exist" \
63+
"test -f services/redcap-to-redcap.service.tpl && \
64+
test -f services/redcap-to-curious.service.tpl && \
65+
test -f services/redcap-to-redcap-batch.service.tpl && \
66+
test -f services/redcap-to-curious-batch.service.tpl && \
67+
test -f services/hbn-sync.service.tpl && \
68+
test -f services/hbn-sync.timer.tpl"
7569

76-
# Test 9: Check for common issues
70+
# Test 7: Check for common issues
7771
echo -e "${YELLOW}Running: Common issues check${NC}"
7872
issues_found=0
7973

0 commit comments

Comments
 (0)