Skip to content

Commit cfa8c6d

Browse files
authored
Merge pull request #31 from awslabs/fix/container-security-and-linting
fix: improve container build reliability and security workflow
2 parents 9d3b1db + 96f8317 commit cfa8c6d

449 files changed

Lines changed: 2826 additions & 2197 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

.github/ISSUE_TEMPLATE/bug_report.yml

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ body:
1616
attributes:
1717
label: Version
1818
description: What version of the plugin are you running?
19-
placeholder: e.g., 1.0.0
19+
placeholder: e.g., 0.1.0
2020
validations:
2121
required: true
2222

@@ -26,10 +26,11 @@ body:
2626
label: Python Version
2727
description: What version of Python are you using?
2828
options:
29-
- Python 3.8
3029
- Python 3.9
3130
- Python 3.10
3231
- Python 3.11
32+
- Python 3.12
33+
- Python 3.13
3334
- Other (specify in environment details)
3435
validations:
3536
required: true

.github/workflows/container.yml

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -220,6 +220,10 @@ jobs:
220220
username: ${{ github.actor }}
221221
password: ${{ secrets.GITHUB_TOKEN }}
222222

223+
- name: Build package wheel
224+
run: |
225+
make build
226+
223227
- name: Build container using dev-tools script
224228
env:
225229
REGISTRY: ${{ needs.get-config.outputs.container-registry }}
@@ -229,6 +233,7 @@ jobs:
229233
PLATFORMS: linux/amd64,linux/arm64
230234
PUSH: ${{ github.event_name != 'pull_request' && (github.ref == 'refs/heads/main' || github.event_name == 'release' || github.event.inputs.push_images == 'true') }}
231235
CACHE: true
236+
SKIP_BUILD: true
232237
run: |
233238
make container-build-single PYTHON_VERSION=${{ matrix.python-version }}
234239
@@ -257,7 +262,6 @@ jobs:
257262
258263
# Stop test container
259264
docker stop test-container
260-
continue-on-error: true
261265
262266
container-manifest:
263267
name: Create Multi-Architecture Manifests

.github/workflows/security.yml

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -149,7 +149,6 @@ jobs:
149149
150150
- name: Run Bandit security scan
151151
run: make ci-security-bandit
152-
continue-on-error: true
153152

154153
security-safety:
155154
name: Safety (Dependency Scan)
@@ -183,7 +182,6 @@ jobs:
183182
184183
- name: Run Safety dependency scan
185184
run: make ci-security-safety
186-
continue-on-error: true
187185
security-hadolint:
188186
name: Hadolint (Dockerfile Scan)
189187
runs-on: ubuntu-latest

.gitignore

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,12 +12,22 @@ requirements-dev.txt
1212
work/
1313

1414
# Security and analysis reports
15+
*.sarif
16+
*-results.sarif
17+
*-report.json
18+
*-report.sarif
1519
bandit-report.json
1620
bandit-report.sarif
1721
bandit-results.sarif
1822
hadolint-results.sarif
1923
trivy-results.json
2024
trivy-results.sarif
25+
semgrep.sarif
26+
semgrep-results.sarif
27+
safety-report.*
28+
codeql-results/
29+
security-reports/
30+
vulnerability-reports/
2131
safety-report.json
2232
security-report.json
2333
security-report.md

.pre-commit-config.yaml

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,6 @@ repos:
5959
files: \.(py)$
6060
pass_filenames: false
6161
description: "Run bandit security analysis"
62-
warning_only: true # Allow this to fail without blocking commit
6362

6463
- id: safety
6564
name: safety (Dependency Security)
@@ -68,7 +67,6 @@ repos:
6867
files: requirements.*\.txt$|pyproject\.toml$
6968
pass_filenames: false
7069
description: "Check dependencies for known security vulnerabilities"
71-
warning_only: true # Allow this to fail without blocking commit
7270

7371
- id: hadolint
7472
name: hadolint (Dockerfile Linting)

Makefile

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -60,9 +60,9 @@ DOCS_DIR := docs
6060
DOCS_BUILD_DIR := $(DOCS_DIR)/site
6161

6262
# Centralized tool execution function
63-
# Usage: $(call run-tool,tool-name,arguments)
63+
# Usage: $(call run-tool,tool-name,arguments[,working-dir])
6464
define run-tool
65-
@dev-tools/scripts/run_tool.sh $(1) $(2)
65+
$(if $(3),cd $(3) && ../dev-tools/scripts/run_tool.sh $(1) $(2),@dev-tools/scripts/run_tool.sh $(1) $(2))
6666
endef
6767

6868
help: ## Show this help message
@@ -476,7 +476,7 @@ ci-quality-flake8: ## Run flake8 style guide check
476476

477477
ci-quality-mypy: ## Run mypy type checking
478478
@echo "Running mypy type check..."
479-
$(call run-tool,mypy,$(PACKAGE) $(TESTS))
479+
$(call run-tool,mypy,.,$(PACKAGE))
480480

481481
ci-quality-pylint: ## Run pylint code analysis
482482
@echo "Running pylint analysis..."

dev-tools/scripts/container_build.sh

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ VCS_REF=$(git rev-parse --short HEAD 2>/dev/null || echo 'unknown')
3333
# Python version support (from Makefile environment variables)
3434
PYTHON_VERSION="${PYTHON_VERSION:-$(make -s print-DEFAULT_PYTHON_VERSION 2>/dev/null || echo '3.13')}" # Dynamic from Makefile with fallback
3535
MULTI_PYTHON="${MULTI_PYTHON:-false}" # Flag for multi-Python builds
36+
SKIP_BUILD="${SKIP_BUILD:-false}" # Flag to skip package building
3637

3738
# Build arguments
3839
PLATFORMS="${PLATFORMS:-linux/amd64,linux/arm64}"
@@ -100,8 +101,14 @@ setup_builder() {
100101
build_image() {
101102
log_info "Building Docker image..."
102103

103-
# Build wheel first if it doesn't exist
104-
if ! ls dist/*.whl 1> /dev/null 2>&1; then
104+
# Build wheel first if it doesn't exist and not skipped
105+
if [[ "${SKIP_BUILD}" == "true" ]]; then
106+
log_info "Skipping package build (SKIP_BUILD=true)"
107+
if ! ls dist/*.whl 1> /dev/null 2>&1; then
108+
log_error "No wheel package found and SKIP_BUILD=true. Build package first."
109+
exit 1
110+
fi
111+
elif ! ls dist/*.whl 1> /dev/null 2>&1; then
105112
log_info "Building wheel package..."
106113
make build || {
107114
log_error "Failed to build wheel package"
@@ -269,6 +276,7 @@ usage() {
269276
echo " PLATFORMS Target platforms"
270277
echo " PUSH Push to registry (true/false)"
271278
echo " CACHE Use build cache (true/false)"
279+
echo " SKIP_BUILD Skip package building (true/false)"
272280
echo ""
273281
echo "Examples:"
274282
echo " $0 # Build local image"

dev-tools/scripts/run_tool.sh

Lines changed: 82 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -11,33 +11,28 @@ shift # Remove tool name from arguments
1111
setup_environment() {
1212
# UV-first approach (fastest and most reliable)
1313
if command -v uv >/dev/null 2>&1; then
14-
# Check if we're in a UV project
15-
if [ -f "pyproject.toml" ] || [ -f "uv.lock" ]; then
16-
echo "Using UV-managed environment..."
17-
return 0
18-
fi
19-
fi
20-
21-
# Fallback to .venv if it exists
22-
if [ -f ".venv/bin/activate" ]; then
23-
echo "Using existing .venv environment..."
24-
# shellcheck disable=SC1091
25-
source .venv/bin/activate
26-
return 0
14+
# Check if we're in a UV project (current dir or parent dirs)
15+
current_dir="$PWD"
16+
while [ "$current_dir" != "/" ]; do
17+
if [ -f "$current_dir/pyproject.toml" ] || [ -f "$current_dir/uv.lock" ]; then
18+
echo "Using UV-managed environment..."
19+
return 0
20+
fi
21+
current_dir=$(dirname "$current_dir")
22+
done
2723
fi
2824

29-
# Create venv if none exists and we're in a project directory
30-
if [ -f "pyproject.toml" ] && [ ! -d ".venv" ]; then
31-
echo "Creating virtual environment..."
32-
if command -v uv >/dev/null 2>&1; then
33-
uv venv
34-
else
35-
python3 -m venv .venv
25+
# Fallback to .venv if it exists (check parent dirs too)
26+
current_dir="$PWD"
27+
while [ "$current_dir" != "/" ]; do
28+
if [ -f "$current_dir/.venv/bin/activate" ]; then
29+
echo "Using existing .venv environment..."
3630
# shellcheck disable=SC1091
37-
source .venv/bin/activate
31+
source "$current_dir/.venv/bin/activate"
32+
return 0
3833
fi
39-
return 0
40-
fi
34+
current_dir=$(dirname "$current_dir")
35+
done
4136

4237
# Use system environment as last resort
4338
echo "Using system environment..."
@@ -48,10 +43,45 @@ run_tool() {
4843

4944
echo "Running ${TOOL_NAME}..."
5045

46+
# Find project root for UV
47+
project_root="$PWD"
48+
while [ "$project_root" != "/" ]; do
49+
if [ -f "$project_root/pyproject.toml" ] || [ -f "$project_root/uv.lock" ]; then
50+
break
51+
fi
52+
project_root=$(dirname "$project_root")
53+
done
54+
5155
# Try different execution methods in order of preference
52-
if command -v uv >/dev/null 2>&1 && { [ -f "pyproject.toml" ] || [ -f "uv.lock" ]; }; then
56+
if command -v uv >/dev/null 2>&1 && [ -f "$project_root/pyproject.toml" ]; then
5357
echo "Executing with UV..."
54-
uv run "${TOOL_NAME}" "$@"
58+
# If we're in a subdirectory of the project, adjust paths for UV
59+
if [ "$PWD" != "$project_root" ]; then
60+
# We're in a subdirectory (like src/), need to adjust paths
61+
adjusted_args=""
62+
for arg in "$@"; do
63+
case "$arg" in
64+
.*)
65+
# Relative path - adjust it relative to project root
66+
rel_to_root=$(realpath --relative-to="$project_root" "$PWD/$arg" 2>/dev/null || python3 -c "import os; print(os.path.relpath(os.path.join('$PWD', '$arg'), '$project_root'))")
67+
adjusted_args="$adjusted_args $rel_to_root"
68+
;;
69+
*)
70+
# Other arguments - keep as is
71+
adjusted_args="$adjusted_args $arg"
72+
;;
73+
esac
74+
done
75+
if [ -n "$adjusted_args" ]; then
76+
# shellcheck disable=SC2086
77+
(cd "$project_root" && uv run "${TOOL_NAME}" $adjusted_args)
78+
else
79+
(cd "$project_root" && uv run "${TOOL_NAME}")
80+
fi
81+
else
82+
# We're in project root
83+
uv run "${TOOL_NAME}" "$@"
84+
fi
5585
elif [ -f ".venv/bin/${TOOL_NAME}" ]; then
5686
echo "Executing with venv..."
5787
.venv/bin/"${TOOL_NAME}" "$@"
@@ -60,7 +90,33 @@ run_tool() {
6090
"${TOOL_NAME}" "$@"
6191
else
6292
echo "Executing as Python module..."
63-
python3 -m "${TOOL_NAME}" "$@"
93+
if command -v uv >/dev/null 2>&1 && [ -f "$project_root/pyproject.toml" ]; then
94+
# Same path adjustment for python -m
95+
if [ "$PWD" != "$project_root" ]; then
96+
adjusted_args=""
97+
for arg in "$@"; do
98+
case "$arg" in
99+
.*)
100+
rel_to_root=$(realpath --relative-to="$project_root" "$PWD/$arg" 2>/dev/null || python3 -c "import os; print(os.path.relpath(os.path.join('$PWD', '$arg'), '$project_root'))")
101+
adjusted_args="$adjusted_args $rel_to_root"
102+
;;
103+
*)
104+
adjusted_args="$adjusted_args $arg"
105+
;;
106+
esac
107+
done
108+
if [ -n "$adjusted_args" ]; then
109+
# shellcheck disable=SC2086
110+
(cd "$project_root" && uv run python -m "${TOOL_NAME}" $adjusted_args)
111+
else
112+
(cd "$project_root" && uv run python -m "${TOOL_NAME}")
113+
fi
114+
else
115+
uv run python -m "${TOOL_NAME}" "$@"
116+
fi
117+
else
118+
python3 -m "${TOOL_NAME}" "$@"
119+
fi
64120
fi
65121
}
66122

dev-tools/security/detect_secrets.py

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,9 @@ def detect_secrets(source_dir: str = "src") -> bool:
5757
logger.error("Potential hardcoded secrets found:")
5858
for secret in found_secrets:
5959
# Security: Don't log the actual secret content, only the location
60-
logger.error(f" Secret detected at: {secret.split(':')[0] if ':' in secret else 'unknown location'}")
60+
logger.error(
61+
f" Secret detected at: {secret.split(':')[0] if ':' in secret else 'unknown location'}"
62+
)
6163
return False
6264
else:
6365
logger.info("No hardcoded secrets detected")

mypy.ini

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,12 @@ python_version = 3.11
44
warn_return_any = True
55
warn_unused_configs = True
66

7+
# Pydantic plugin
8+
plugins = pydantic.mypy
9+
10+
# Exclude installed packages to avoid module conflicts
11+
exclude = ^(\.venv/|build/|dist/|\.eggs/|.*\.egg-info/|docs/|memory-bank/)
12+
713
# Type checking strictness
814
disallow_untyped_defs = True
915
disallow_incomplete_defs = True
@@ -33,6 +39,7 @@ error_summary = True
3339
ignore_missing_imports = False
3440
follow_imports = normal
3541
follow_imports_for_stubs = True
42+
explicit_package_bases = True
3643

3744
# Disallow dynamic typing
3845
disallow_any_unimported = True

0 commit comments

Comments
 (0)