Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,7 @@ jobs:
permissions_monitoring_config: ${{ vars.ACTIONS_PERMISSIONS_CONFIG }}
- id: setup-env
uses: cisagov/setup-env-github-action@v1
- uses: actions/checkout@v5
- uses: actions/checkout@v6
- id: setup-python
uses: actions/setup-python@v6
with:
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/codeql-analysis.yml
Original file line number Diff line number Diff line change
Expand Up @@ -113,7 +113,7 @@ jobs:
permissions_monitoring_config: ${{ vars.ACTIONS_PERMISSIONS_CONFIG }}

- name: Checkout repository
uses: actions/checkout@v5
uses: actions/checkout@v6

# Initializes the CodeQL tools for scanning.
- name: Initialize CodeQL
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/dependency-review.yml
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,7 @@ jobs:
permissions_monitoring_config: ${{ vars.ACTIONS_PERMISSIONS_CONFIG }}
- id: checkout-repo
name: Checkout the repository
uses: actions/checkout@v5
uses: actions/checkout@v6
- id: dependency-review
name: Review dependency changes for vulnerabilities and license changes
uses: actions/dependency-review-action@v4
2 changes: 1 addition & 1 deletion .github/workflows/sync-labels.yml
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,7 @@ jobs:
# monitoring configuration *does not* require you to modify
# this workflow.
permissions_monitoring_config: ${{ vars.ACTIONS_PERMISSIONS_CONFIG }}
- uses: actions/checkout@v5
- uses: actions/checkout@v6
- name: Sync repository labels
if: success()
uses: crazy-max/ghaction-github-labeler@v5
Expand Down
30 changes: 9 additions & 21 deletions .pre-commit-config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -173,10 +173,16 @@ repos:
rev: v3.21.1
hooks:
- id: pyupgrade
args:
# Python 3.10 is currently the oldest non-EOL version of
# Python, so we want to apply all rules that apply to this
# version or later. See here for more details:
# https://www.gyford.com/phil/writing/2025/08/26/how-to-use-pyupgrade/
- --py310-plus

# Ansible hooks
- repo: https://github.com/ansible/ansible-lint
rev: v25.11.0
rev: v25.11.1
hooks:
- id: ansible-lint
additional_dependencies:
Expand All @@ -192,31 +198,13 @@ repos:
# hook identifies a vulnerability in ansible-core 2.16.13,
# but all versions of ansible 9 have a dependency on
# ~=2.16.X.
#
# It is also a good idea to go ahead and upgrade to version
# 10 since version 9 is going EOL at the end of November:
# https://endoflife.date/ansible
# - ansible>=10,<11
# ansible-core 2.16.3 through 2.16.6 suffer from the bug
# discussed in ansible/ansible#82702, which breaks any
# symlinked files in vars, tasks, etc. for any Ansible role
# installed via ansible-galaxy. Hence we never want to
# install those versions.
#
# Note that the pip-audit pre-commit hook identifies a
# vulnerability in ansible-core 2.16.13. The pin of
# ansible-core to >=2.17 effectively also pins ansible to
# >=10.
#
# It is also a good idea to go ahead and upgrade to
# ansible-core 2.17 since security support for ansible-core
# 2.16 ends this month:
# https://docs.ansible.com/ansible/devel/reference_appendices/release_and_maintenance.html#ansible-core-support-matrix
# ansible-core<2.17.7 suffers from GHSA-99w6-3xph-cx78.
#
# Note that any changes made to this dependency must also be
# made in requirements.txt in cisagov/skeleton-packer and
# requirements-test.txt in cisagov/skeleton-ansible-role.
- ansible-core>=2.17
- ansible-core>=2.17.7

# Terraform hooks
- repo: https://github.com/antonbabenko/pre-commit-terraform
Expand Down
88 changes: 44 additions & 44 deletions project_setup/scripts/terraform-to-secrets
Original file line number Diff line number Diff line change
Expand Up @@ -52,12 +52,13 @@ Options:

# Standard Python Libraries
from base64 import b64encode
from collections.abc import Generator
import json
import logging
import re
import subprocess # nosec : security implications have been considered
import sys
from typing import Any, Dict, Generator, Optional, Tuple, Union
from typing import Any

# Third-Party Libraries
import docopt
Expand All @@ -74,12 +75,12 @@ KEYRING_SERVICE = "terraform-to-secrets"
KEYRING_USERNAME = "GitHub PAT"


def get_terraform_state(filename: str = "") -> Dict:
def get_terraform_state(filename: str = "") -> dict:
"""Retrieve IAM credentials from Terraform state.

Returns the Terraform state as a dict.
"""
data: Union[str, bytes, bytearray]
data: str | bytes | bytearray
if filename:
logging.info(f"Reading state from json file {filename}")
with open(filename) as f:
Expand All @@ -92,7 +93,7 @@ def get_terraform_state(filename: str = "") -> Dict:
data = process.stdout
# Normally we'd check the process return code here. But Terraform is perfectly
# happy to return zero even if there were no state files.
json_state: Dict = json.loads(data)
json_state: dict = json.loads(data)

if not json_state.get("values"):
logging.critical("Is there a .terraform state directory here?")
Expand All @@ -101,20 +102,20 @@ def get_terraform_state(filename: str = "") -> Dict:


def find_tagged_secret(
resource_name: str, resource_data: Dict
) -> Generator[Tuple[str, str], None, None]:
resource_name: str, resource_data: dict
) -> Generator[tuple[str, str], None, None]:
"""Extract a tagged secret from a resource."""
# Ensure "tags" key exists in resource_data and if it does, make sure
# its value is not None. Both of these cases can occur.
tags: Dict[str, str]
tags: dict[str, str]
if "tags" not in resource_data or resource_data.get("tags") is None:
tags = dict()
else:
tags = resource_data["tags"]

secret_name: Optional[str] = tags.get(GITHUB_SECRET_NAME_TAG)
lookup_tag: Optional[str] = tags.get(GITHUB_SECRET_TERRAFORM_LOOKUP_TAG)
secret_value: Optional[str]
secret_name: str | None = tags.get(GITHUB_SECRET_NAME_TAG)
lookup_tag: str | None = tags.get(GITHUB_SECRET_TERRAFORM_LOOKUP_TAG)
secret_value: str | None
if secret_name:
logging.debug(
f"Found {GITHUB_SECRET_NAME_TAG} on {resource_name} "
Expand All @@ -139,8 +140,8 @@ def find_tagged_secret(


def find_outputs(
terraform_state: Dict, include_remote_state: bool
) -> Generator[Dict, None, None]:
terraform_state: dict, include_remote_state: bool
) -> Generator[dict, None, None]:
"""Search for resources with outputs in the Terraform state."""
for resource in terraform_state["values"]["root_module"].get("resources", []):
# Exclude remote state resources unless requested
Expand All @@ -154,9 +155,9 @@ def find_outputs(


def parse_tagged_outputs(
terraform_state: Dict,
terraform_state: dict,
include_remote_state: bool,
) -> Generator[Tuple[str, str], None, None]:
) -> Generator[tuple[str, str], None, None]:
"""Search all outputs for tags requesting the creation of a secret."""
for outputs in find_outputs(terraform_state, include_remote_state):
for output_name, output_data in outputs.items():
Expand All @@ -165,8 +166,8 @@ def parse_tagged_outputs(


def find_resources_in_child_modules(
child_modules: list, resource_type: Optional[str]
) -> Generator[Dict, None, None]:
child_modules: list, resource_type: str | None
) -> Generator[dict, None, None]:
"""
Search for resources of a certain type in a Terraform child_modules list.

Expand All @@ -185,8 +186,8 @@ def find_resources_in_child_modules(


def find_resources(
terraform_state: Dict, resource_type: Optional[str]
) -> Generator[Dict, None, None]:
terraform_state: dict, resource_type: str | None
) -> Generator[dict, None, None]:
"""Search for resources of a certain type in the Terraform state.

resource_type None yields all resources.
Expand All @@ -202,7 +203,7 @@ def find_resources(
yield resource


def parse_creds(terraform_state: Dict) -> Generator[Tuple[str, str, str], None, None]:
def parse_creds(terraform_state: dict) -> Generator[tuple[str, str, str], None, None]:
"""Search for IAM access keys in resources.

Yields (user, key_id, secret) when found.
Expand All @@ -216,9 +217,9 @@ def parse_creds(terraform_state: Dict) -> Generator[Tuple[str, str, str], None,


def parse_tagged_resources(
terraform_state: Dict,
terraform_state: dict,
include_remote_state: bool,
) -> Generator[Tuple[str, str], None, None]:
) -> Generator[tuple[str, str], None, None]:
"""Search all resources for tags requesting the creation of a secret."""
for resource in find_resources(terraform_state, None):
# Exclude remote state resources unless requested
Expand All @@ -239,7 +240,7 @@ def encrypt(public_key: str, secret_value: str) -> str:
return b64encode(encrypted).decode("utf-8")


def get_public_key(session: requests.Session, repo_name, github_env) -> Dict[str, str]:
def get_public_key(session: requests.Session, repo_name, github_env) -> dict[str, str]:
"""Fetch the public key for a repository or environment."""
if github_env:
logging.info(
Expand All @@ -263,7 +264,7 @@ def set_secret(
github_env: str,
secret_name: str,
secret_value: str,
public_key: Dict[str, str],
public_key: dict[str, str],
) -> None:
"""Create a secret in a repository or environment."""
if github_env:
Expand All @@ -290,8 +291,7 @@ def get_repo_name() -> str:
logging.debug("Trying to determine GitHub repository name using git.")
c = subprocess.run( # nosec
["git", "remote", "get-url", "origin"],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
capture_output=True,
)
if c.returncode != 0:
logging.critical("Could not determine GitHub repository name.")
Expand All @@ -306,15 +306,15 @@ def get_repo_name() -> str:
return repo_name


def get_users(terraform_state: Dict) -> Dict[str, Tuple[str, str]]:
def get_users(terraform_state: dict) -> dict[str, tuple[str, str]]:
"""Return a dictionary of users.

Returns: a dictionary mapping usernames to (key_id, key_secret)
"""
aws_user: Optional[str] = None
aws_key_id: Optional[str] = None
aws_secret: Optional[str] = None
user_creds: Dict[str, Tuple[str, str]] = dict()
aws_user: str | None = None
aws_key_id: str | None = None
aws_secret: str | None = None
user_creds: dict[str, tuple[str, str]] = dict()

logging.info("Searching Terraform state for IAM credentials.")
for aws_user, aws_key_id, aws_secret in parse_creds(terraform_state):
Expand All @@ -327,10 +327,10 @@ def get_users(terraform_state: Dict) -> Dict[str, Tuple[str, str]]:


def get_resource_secrets(
terraform_state: Dict, include_remote_state: bool
) -> Dict[str, str]:
terraform_state: dict, include_remote_state: bool
) -> dict[str, str]:
"""Collect secrets from tagged Terraform resources."""
secrets: Dict[str, str] = dict()
secrets: dict[str, str] = dict()
logging.info("Searching Terraform state for tagged resources.")
for secret_name, secret_value in parse_tagged_resources(
terraform_state, include_remote_state
Expand All @@ -345,9 +345,9 @@ def get_resource_secrets(
return secrets


def create_user_secrets(user_creds: Dict[str, Tuple[str, str]]) -> Dict[str, str]:
def create_user_secrets(user_creds: dict[str, tuple[str, str]]) -> dict[str, str]:
"""Create secrets for user key IDs and key values."""
secrets: Dict[str, str] = dict()
secrets: dict[str, str] = dict()
for user_name, creds in user_creds.items():
# If there is more than one user add the name as a suffix
if len(user_creds) > 1:
Expand All @@ -361,7 +361,7 @@ def create_user_secrets(user_creds: Dict[str, Tuple[str, str]]) -> Dict[str, str


def create_all_secrets(
secrets: Dict[str, str],
secrets: dict[str, str],
github_env: str,
github_token: str,
repo_name: str,
Expand All @@ -383,7 +383,7 @@ def create_all_secrets(
raise Exception(f"Environment {github_env} not found in {repo_name}.")

# Get the repo or environment public key to be used to encrypt secrets
public_key: Dict[str, str] = get_public_key(session, repo_name, github_env)
public_key: dict[str, str] = get_public_key(session, repo_name, github_env)

for secret_name, secret_value in secrets.items():
if dry_run:
Expand All @@ -396,7 +396,7 @@ def create_all_secrets(

def main() -> int:
"""Set up logging and call the requested commands."""
args: Dict[str, Any] = docopt.docopt(__doc__, version="1.1.0")
args: dict[str, Any] = docopt.docopt(__doc__, version="1.1.0")

# Validate and convert arguments as needed
schema: Schema = Schema(
Expand Down Expand Up @@ -437,7 +437,7 @@ def main() -> int:
)

try:
validated_args: Dict[str, Any] = schema.validate(args)
validated_args: dict[str, Any] = schema.validate(args)
except SchemaError as err:
# Exit because one or more of the arguments were invalid
print(err, file=sys.stderr)
Expand Down Expand Up @@ -482,16 +482,16 @@ def main() -> int:
logging.info("GitHub token retrieved from keyring.")

# Get the state from Terraform or a json file
terraform_state: Dict = get_terraform_state(state_filename)
terraform_state: dict = get_terraform_state(state_filename)

# Users mapped to their (key, secret)
user_creds: Dict[str, Tuple[str, str]] = get_users(terraform_state)
user_creds: dict[str, tuple[str, str]] = get_users(terraform_state)

# User secrets created from credentials. Names mapped to value.
user_secrets: Dict[str, str] = create_user_secrets(user_creds)
user_secrets: dict[str, str] = create_user_secrets(user_creds)

# Secrets created from tagged resources. Names mapped to value.
resource_secrets: Dict[str, str] = get_resource_secrets(
resource_secrets: dict[str, str] = get_resource_secrets(
terraform_state, include_remote_state
)

Expand All @@ -500,7 +500,7 @@ def main() -> int:
logging.warning("User secret names overlap with resource secret names.")

# Merge the two dictionaries together
all_secrets: Dict[str, str] = resource_secrets.copy()
all_secrets: dict[str, str] = resource_secrets.copy()
all_secrets.update(user_secrets)

# All the ducks are in a row, let's do this thang!
Expand Down