Merge branch 'main' into analyze/eks-versions

weklund · web-flow · commit 3568c8b39e79 · 2026-01-27T08:13:25.000-05:00
diff --git a/.github/workflows/check-integration-tests.yml b/.github/workflows/check-integration-tests.yml
@@ -9,6 +9,8 @@ jobs:
   get-modules:
     name: Get Modules
     runs-on: ubuntu-latest
+    permissions:
+      contents: read
     outputs:
       matrix: ${{ steps.get-modules.outputs.matrix }}
     steps:
diff --git a/.github/workflows/module-checks-dev-requirements-change.yml b/.github/workflows/module-checks-dev-requirements-change.yml
@@ -19,6 +19,8 @@ jobs:
   get-modules:
     name: Get Modules
     runs-on: ubuntu-latest
+    permissions:
+      contents: read
     outputs:
       matrix: ${{ steps.get-modules.outputs.matrix }}
     steps:
@@ -42,6 +44,8 @@ jobs:
   test:
     name: Run unit tests for module ${{ matrix.modules.module_name }}
     needs: get-modules
+    permissions:
+      contents: read
     strategy:
       fail-fast: false
       matrix:
diff --git a/.github/workflows/module-checks.yml b/.github/workflows/module-checks.yml
@@ -21,6 +21,8 @@ jobs:
   get-modules:
     name: Get Modules
     runs-on: ubuntu-latest
+    permissions:
+      contents: read
     outputs:
       matrix: ${{ steps.get-modules.outputs.matrix }}
     steps:
@@ -30,7 +32,7 @@ jobs:
           fetch-depth: 0
       - name: Get changed files
         id: changed-files
-        uses: tj-actions/changed-files@v44
+        uses: tj-actions/changed-files@v46.0.1
         with:
           files: modules/**
       - name: List all changed files
@@ -58,6 +60,8 @@ jobs:
   test:
     name: Run unit tests for module ${{ matrix.modules.module_name }}
     needs: get-modules
+    permissions:
+      contents: read
     strategy:
       fail-fast: false
       matrix:
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -23,6 +23,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - updated all pyproject.toml files to target Python 3.11
 - regenerated all requirements.txt files with Python 3.11 compatibility
 - updated default EKS version in manifests and documentation from 1.25/1.29 to 1.30
+- upgraded `tj-actions/changed-files` from v44 to v46.0.1 in GitHub Actions workflows
+- upgraded Apache Airflow from 2.5.1 to 2.11.1 in `mwaa` module
+- upgraded `apache-airflow-providers-cncf-kubernetes` from ~3.0.0 to ~8.5.0
+- upgraded `sagemaker` from 2.218.0 to 2.237.3 in MWAA requirements files
+- upgraded `wheel` from ~0.43.0 to ~0.46.2 in dev dependencies
+- added explicit `permissions: contents: read` to GitHub Actions workflow jobs
+- fixed clear-text logging in `dockerimage-replication` and `integration-tests` modules
 
 ### **Removed**
 - removed deprecated EKS version files: 1.23, 1.24, 1.25, 1.26, 1.29 (end of life or extended support ending soon)
diff --git a/modules/compute/eks/requirements.in b/modules/compute/eks/requirements.in
@@ -6,4 +6,4 @@ constructs==10.3.0
 deepmerge==1.1.0
 moto>=5.0.0,<6.0.0
 packaging==23.0
-PyYAML==6.0
+PyYAML>=6.0.1
diff --git a/modules/compute/eks/requirements.txt b/modules/compute/eks/requirements.txt
@@ -94,7 +94,7 @@ python-dateutil==2.9.0.post0
     #   botocore
     #   jsii
     #   moto
-pyyaml==6.0
+pyyaml==6.0.3
     # via
     #   -r requirements.in
     #   responses
diff --git a/modules/database/rds/pyproject.toml b/modules/database/rds/pyproject.toml
@@ -27,6 +27,7 @@ ignore_missing_imports = true
 disallow_untyped_decorators = false
 exclude = "codeseeder.out/|example/|tests/|integ/"
 warn_unused_ignores = false
+plugins = ["pydantic.mypy"]
 
 [tool.pytest.ini_options]
 addopts = "-v --cov=. --cov-report term"
diff --git a/modules/database/rds/settings.py b/modules/database/rds/settings.py
@@ -103,6 +103,6 @@ class CDKSettings(EnvBaseSettings):
 class ApplicationSettings(EnvBaseSettings):
     """Application settings."""
 
-    seedfarmer_settings: SeedFarmerSettings = Field(default_factory=SeedFarmerSettings)
-    module_settings: ModuleSettings = Field(default_factory=ModuleSettings)
-    cdk_settings: CDKSettings = Field(default_factory=CDKSettings)
+    seedfarmer_settings: SeedFarmerSettings = Field(default_factory=lambda: SeedFarmerSettings())
+    module_settings: ModuleSettings = Field(default_factory=lambda: ModuleSettings())
+    cdk_settings: CDKSettings = Field(default_factory=lambda: CDKSettings())
diff --git a/modules/orchestration/mwaa/README.md b/modules/orchestration/mwaa/README.md
@@ -54,7 +54,7 @@ parameters:
 - `dag-path`: path in the S3 Bucket to configure the MWAA Environment to monitor for DAG artifacts. Defaults to `dags` if none is provided
 - `environment-class`: the MWAA Environement Instance Class. Defaults to `mw1.small` if none is provided
 - `max-workers`: the Maximum number of workers to configure the MWAA Environment to allow. Defaults to `25` if none is provided
-- `airflow-version`: The Airflow version you would want to set in the module. It is defaulted to `2.5.1`
+- `airflow-version`: The Airflow version you would want to set in the module. It is defaulted to `2.11.1`
 - `mwaa-requirements-file` - Support for customized requiremements file installed on MWAA
   - ANY file referenced MUST be located `modules/core/mwaa/requirements/*.txt` and be python requirements compliant
   - in if not provided, default is `requirements.txt`
diff --git a/modules/orchestration/mwaa/app.py b/modules/orchestration/mwaa/app.py
@@ -63,7 +63,7 @@ def generate_description() -> str:
     optional_args["environment_class"] = environment_class
 if max_workers and max_workers.isnumeric():
     optional_args["max_workers"] = int(max_workers)  # type: ignore
-optional_args["airflow_version"] = airflow_version if airflow_version else "2.5.1"
+optional_args["airflow_version"] = airflow_version if airflow_version else "2.11.1"
 
 
 stack = MWAAStack(
diff --git a/modules/orchestration/mwaa/requirements/requirements-emr.txt b/modules/orchestration/mwaa/requirements/requirements-emr.txt
@@ -1,9 +1,9 @@
 Pillow~=10.3.0
 awscli~=1.22.80
 airflow-kubernetes-job-operator~=2.0.4
-apache-airflow[package-extra]==2.5.1
+apache-airflow[package-extra]==2.11.1
 apache-airflow[amazon]
-apache-airflow-providers-cncf-kubernetes~=3.0.0
+apache-airflow-providers-cncf-kubernetes~=8.5.0
 boto3-stubs[essential]~=1.42.0
 mypy-boto3-batch~=1.42.0
-sagemaker==2.218.0
+sagemaker==2.237.3
diff --git a/modules/orchestration/mwaa/requirements/requirements-rosbag.txt b/modules/orchestration/mwaa/requirements/requirements-rosbag.txt
@@ -4,4 +4,4 @@ airflow-kubernetes-job-operator~=2.0.4
 apache-airflow-providers-cncf-kubernetes~=2.1.0
 boto3-stubs[essential]~=1.42.0
 mypy-boto3-batch~=1.42.0
-sagemaker==2.218.0
+sagemaker==2.237.3
diff --git a/modules/orchestration/mwaa/requirements/requirements.txt b/modules/orchestration/mwaa/requirements/requirements.txt
@@ -4,4 +4,4 @@ airflow-kubernetes-job-operator~=2.0.4
 apache-airflow-providers-cncf-kubernetes~=2.1.0
 boto3-stubs[essential]~=1.21.37
 mypy-boto3-batch~=1.22.8
-sagemaker==2.218.0
+sagemaker==2.237.3
diff --git a/modules/orchestration/mwaa/tests/test_app.py b/modules/orchestration/mwaa/tests/test_app.py
@@ -19,7 +19,7 @@ def stack_defaults():
     os.environ["SEEDFARMER_PARAMETER_PRIVATE_SUBNET_IDS"] = '["subnet-12345", "subnet-54321"]'
     os.environ["SEEDFARMER_PARAMETER_DAG_BUCKET_NAME"] = "test-idf-dag-bucket"
     os.environ["SEEDFARMER_PARAMETER_DAG_PATH"] = "test-dags-path"
-    os.environ["SEEDFARMER_PARAMETER_AIRFLOW_VERSION"] = "2.5.1"
+    os.environ["SEEDFARMER_PARAMETER_AIRFLOW_VERSION"] = "2.11.1"
 
     # Unload the app import so that subsequent tests don't reuse
     if "app" in sys.modules:
diff --git a/modules/orchestration/mwaa/tests/test_stack.py b/modules/orchestration/mwaa/tests/test_stack.py
@@ -38,7 +38,7 @@ def test_synthesize_stack(stack_defaults):
         private_subnet_ids=["subnet-12345", "subnet-54321"],
         dag_bucket_name="test-idf-dag-bucket",
         dag_path="test-dags-path",
-        airflow_version="2.5.1",
+        airflow_version="2.11.1",
         environment_class="mw1.small",
         max_workers=25,
         unique_requirements_file="test.txt",
diff --git a/modules/replication/dockerimage-replication/install_helm.py b/modules/replication/dockerimage-replication/install_helm.py
@@ -17,19 +17,20 @@
 
 def get_distro() -> requests.models.Response:
     if distro_secret is None:
-        logger.info(f"Using with URL: {distro_url} without authentication")
+        # Security: This log message only indicates authentication mode, no credentials are logged.
+        logger.info("Downloading Helm distribution without authentication")  # nosec
         response = requests.get(distro_url, stream=True)
         return response
     else:
         user, pwd = get_credentials(distro_secret, distro_key)
         if user is None or pwd is None:
-            logger.info(
-                f"Using with URL: {distro_url} without authentication as no user/pwd were found with {distro_secret}"
-            )
+            # Security: This log message only indicates authentication mode, no credentials are logged.
+            logger.info("Downloading Helm distribution without authentication (credentials not found in secret)")  # nosec
             response = requests.get(distro_url, stream=True)
             return response
         else:
-            logger.info(f"Using with URL: {distro_url} with authentication from {distro_secret}")
+            # Security: This log message only indicates authentication mode, no credentials are logged.
+            logger.info("Downloading Helm distribution with authentication")  # nosec
             response = requests.get(distro_url, stream=True, auth=(user, pwd))
             return response
 
diff --git a/modules/replication/dockerimage-replication/replication/utils.py b/modules/replication/dockerimage-replication/replication/utils.py
@@ -77,14 +77,18 @@ def run_command(
     try:
         result = subprocess.run(command, input=input, shell=shell, check=True, text=True, capture_output=capture_output)  # type: ignore
         if error_indicator in result.stderr:
-            logger.info(f"Error Occurred execution command {mask_sensitive_data(command)}")  # type: ignore
+            # Security: mask_sensitive_data() sanitizes passwords by replacing them with '******'
+            # before logging, so no sensitive data is exposed. See mask_sensitive_data() above.
+            logger.info(f"Error occurred executing command {mask_sensitive_data(command)}")  # type: ignore  # nosec
             return False
         else:
             return True
-    except subprocess.CalledProcessError as e:
-        # Sanitize the command to hide passwords
+    except subprocess.CalledProcessError:
+        # Security: mask_sensitive_data() sanitizes passwords by replacing them with '******'
+        # before logging, so no sensitive data is exposed. See mask_sensitive_data() above.
         sanitized_command = mask_sensitive_data(command)  # type: ignore
-        logger.info(f"Error running command: {sanitized_command}\n{e.stderr if capture_output else ''}")
+        # Do not log stderr as it may contain sensitive information such as credentials
+        logger.info(f"Error running command: {sanitized_command}")  # nosec
         return False
 
 
diff --git a/modules/testing/integration-tests/artifacts/bin/role_assume_update.py b/modules/testing/integration-tests/artifacts/bin/role_assume_update.py
@@ -14,19 +14,22 @@
 ]["Statement"][0]["Principal"]["AWS"]
 if not isinstance(trusted_principals, list):
     trusted_principals = [trusted_principals]
-    print(
-        f"Current principals in assume role policy for {ROLE_NAME}: {trusted_principals}"
+    # Security: Only logging a count (integer) and role name - no secrets or credentials exposed.
+    print(  # nosec
+        f"Current principals count in assume role policy for {ROLE_NAME}: {len(trusted_principals)}"
     )
 
 principals = []
 for principal in trusted_principals:
     if "arn:" not in principal:
-        print(f"Removing non-existent principal: {principal}")
+        # Security: Status message only, no sensitive data logged.
+        print("Removing non-existent principal (non-ARN format)")  # nosec
     else:
         principals.append(principal)
 
 if ROLE_ARN not in principals:
-    print(f"{ROLE_ARN} not in trusted principals list and will be added.")
+    # Security: Role ARNs are resource identifiers, not secrets or credentials.
+    print(f"{ROLE_ARN} not in trusted principals list and will be added.")  # nosec
     principals.append(ROLE_ARN)
 
 policy_document = json.dumps(
diff --git a/requirements-dev.txt b/requirements-dev.txt
@@ -13,6 +13,6 @@ pytest-ordering~=0.6
 twine~=5.1.1
 types-PyYAML~=6.0.12.20240311
 types-setuptools~=70.3.0.20240710
-wheel~=0.43.0
+wheel~=0.46.2
 cdk-nag==2.28.161
 click==8.1.7
