Skip to content

Commit e4dc506

Browse files
awsntheuleawsntheule
authored
Merge pull request #237 from awslabs/fix/detect-secrets
perf(logging): reduce log file size and prevent accumulation
2 parents c0ef685 + 5f2ecc3 commit e4dc506

18 files changed

Lines changed: 82 additions & 63 deletions

File tree

‎.vscode/launch.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -76,7 +76,7 @@
7676
"args": [
7777
"--verbose",
7878
"--scanners",
79-
"snyk-code,bandit",
79+
"detect-secrets",
8080
"--config-overrides",
8181
"ash_plugin_modules+=[\"automated_security_helper.plugin_modules.ash_snyk_plugins\"]"
8282
],

‎README.md‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -91,15 +91,15 @@ ASH v3 integrates multiple open-source security tools as scanners. Tools like Ba
9191
curl -sSfL https://astral.sh/uv/install.sh | sh
9292

9393
# Create an alias for ASH
94-
alias ash="uvx git+https://github.com/awslabs/automated-security-helper.git@v3.1.12"
94+
alias ash="uvx git+https://github.com/awslabs/automated-security-helper.git@v3.1.13"
9595
```
9696

9797
```powershell
9898
# Install uv on Windows with PowerShell if it isn't installed already
9999
irm https://astral.sh/uv/install.ps1 | iex
100100
101101
# Create a function for ASH
102-
function ash { uvx git+https://github.com/awslabs/automated-security-helper.git@v3.1.12 $args }
102+
function ash { uvx git+https://github.com/awslabs/automated-security-helper.git@v3.1.13 $args }
103103
```
104104

105105
### Other Installation Methods
@@ -120,13 +120,13 @@ ash --help
120120
#### Using `pip`
121121

122122
```bash
123-
pip install git+https://github.com/awslabs/automated-security-helper.git@v3.1.12
123+
pip install git+https://github.com/awslabs/automated-security-helper.git@v3.1.13
124124
```
125125

126126
#### Clone the Repository
127127

128128
```bash
129-
git clone https://github.com/awslabs/automated-security-helper.git --branch v3.1.12
129+
git clone https://github.com/awslabs/automated-security-helper.git --branch v3.1.13
130130
cd automated-security-helper
131131
pip install .
132132
```

‎automated_security_helper/core/orchestrator.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -186,7 +186,10 @@ def model_post_init(self, context):
186186
"ash_aggregated_results.json",
187187
"ash-ignore-report.txt",
188188
"ash-scan-set-files-list.txt",
189+
# Don't delete log files here - they're managed by the logger
190+
# which truncates them on initialization
189191
# "ash.log",
192+
# "ash.log.jsonl",
190193
]
191194
]:
192195
ASH_LOGGER.debug(f"Removing old file: {old_file}")

‎automated_security_helper/interactions/run_ash_scan.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -144,6 +144,8 @@ def run_ash_scan(
144144
not in ["YES", "1", "TRUE"],
145145
use_color=color,
146146
simple_format=simple_logging, # Pass the simple flag to the logger
147+
truncate_log=existing_results
148+
is None, # Don't truncate when using existing results
147149
)
148150
# Initialize results as None at the start to avoid UnboundLocalError
149151
results = None
@@ -478,7 +480,6 @@ def run_ash_scan(
478480
)
479481
)
480482

481-
482483
# Get the count of actionable findings from unified metrics
483484
scanner_metrics = get_unified_scanner_metrics(asharp_model=results)
484485
actionable_findings = 0

‎automated_security_helper/plugin_modules/ash_builtin/scanners/detect_secrets_scanner.py‎

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -278,14 +278,10 @@ def scan(
278278
else scan_set(
279279
source=self.context.source_dir,
280280
output=self.context.output_dir,
281-
# filter_pattern=r"\.(yaml|yml|json)$",
282281
)
283282
)
284-
if Path(item).name
285-
not in [
286-
"ash_aggregated_results.json",
287-
*KNOWN_LOCKFILE_NAMES,
288-
]
283+
if Path(item).name not in [*KNOWN_LOCKFILE_NAMES]
284+
and "/.ash/" not in str(item)
289285
]
290286
if len(scannable) == 0:
291287
message = f"There were no scannable files found in target '{target}'"

‎automated_security_helper/schemas/AshAggregatedResults.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21199,7 +21199,7 @@
2119921199
"supportedTaxonomies": [],
2120021200
"taxa": [],
2120121201
"translationMetadata": null,
21202-
"version": "3.1.12"
21202+
"version": "3.1.13"
2120321203
},
2120421204
"extensions": [],
2120521205
"properties": null

‎automated_security_helper/utils/log.py‎

Lines changed: 25 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -442,6 +442,8 @@ def get_logger(
442442
show_progress: bool = True,
443443
use_color: bool = True,
444444
simple_format: bool = False,
445+
file_log_level: str | int | None = None,
446+
truncate_log: bool = True,
445447
) -> logging.Logger:
446448
# Configure Windows-safe logging early
447449
configure_windows_safe_logging()
@@ -586,15 +588,26 @@ def _safe_log(level, msg, args, **kwargs):
586588
# # Determine log path/file name; create output_dir if necessary
587589
# now = datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%S")
588590

589-
# Create JSONLines/JSONL file handler for logging to a file (log all five levels)
591+
# Determine file log level (default to INFO to avoid huge log files)
592+
final_file_log_level = (
593+
file_log_level if file_log_level is not None else logging.INFO
594+
)
595+
596+
# Create JSONLines/JSONL file handler for logging to a file
590597
if log_format in ["JSONL", "BOTH"]:
591598
jsonl_log_file = Path(output_dir).joinpath(f"{name}.log.jsonl")
592599
jsonl_log_file.parent.mkdir(parents=True, exist_ok=True)
593-
jsonl_log_file.touch(exist_ok=True)
600+
# Truncate the file if requested (default), otherwise append
601+
if truncate_log:
602+
jsonl_log_file.write_text("") # Clear the file
603+
else:
604+
jsonl_log_file.touch(
605+
exist_ok=True
606+
) # Create if doesn't exist, but don't truncate
594607

595608
jsonl_file_handler = logging.FileHandler(jsonl_log_file.as_posix())
596609
logger.verbose("Logging to JSONL file: %s", jsonl_log_file.as_posix())
597-
jsonl_file_handler.setLevel(logging.DEBUG)
610+
jsonl_file_handler.setLevel(final_file_log_level)
598611
jsonl_file_handler.setFormatter(
599612
JsonFormatter(
600613
{
@@ -612,15 +625,21 @@ def _safe_log(level, msg, args, **kwargs):
612625
)
613626
)
614627
logger.addHandler(jsonl_file_handler)
615-
# Create tabular file handler for logging to a file (log all five levels)
628+
# Create tabular file handler for logging to a file
616629
if log_format in ["TABULAR", "BOTH"]:
617630
tab_log_file = Path(output_dir).joinpath(f"{name}.log")
618631
tab_log_file.parent.mkdir(parents=True, exist_ok=True)
619-
tab_log_file.touch(exist_ok=True)
632+
# Truncate the file if requested (default), otherwise append
633+
if truncate_log:
634+
tab_log_file.write_text("") # Clear the file
635+
else:
636+
tab_log_file.touch(
637+
exist_ok=True
638+
) # Create if doesn't exist, but don't truncate
620639

621640
tab_file_handler = logging.FileHandler(tab_log_file.as_posix())
622641
logger.verbose("Logging to tabular file: %s", tab_log_file.as_posix())
623-
tab_file_handler.setLevel(logging.DEBUG)
642+
tab_file_handler.setLevel(final_file_log_level)
624643
tab_file_handler.setFormatter(
625644
logging.Formatter(
626645
fmt="%(asctime)s\t%(levelname)s\t%(filename)s:%(lineno)d\t%(message)s "

‎docs/content/docs/advanced-usage.md‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -237,7 +237,7 @@ jobs:
237237
with:
238238
python-version: '3.10'
239239
- name: Install ASH
240-
run: pip install git+https://github.com/awslabs/automated-security-helper.git@v3.1.12
240+
run: pip install git+https://github.com/awslabs/automated-security-helper.git@v3.1.13
241241
- name: Run ASH scan
242242
run: ash --mode local
243243
- name: Upload scan results
@@ -253,7 +253,7 @@ jobs:
253253
ash-scan:
254254
image: python:3.10
255255
script:
256-
- pip install git+https://github.com/awslabs/automated-security-helper.git@v3.1.12
256+
- pip install git+https://github.com/awslabs/automated-security-helper.git@v3.1.13
257257
- ash --mode local
258258
artifacts:
259259
paths:

‎docs/content/docs/installation-guide.md‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ ASH v3 uses UV's tool isolation system to automatically manage most scanner depe
3333
curl -sSf https://astral.sh/uv/install.sh | sh
3434

3535
# Create an alias for ASH
36-
alias ash="uvx git+https://github.com/awslabs/automated-security-helper.git@v3.1.12"
36+
alias ash="uvx git+https://github.com/awslabs/automated-security-helper.git@v3.1.13"
3737

3838
# Use as normal
3939
ash --help
@@ -45,7 +45,7 @@ ash --help
4545
irm https://astral.sh/uv/install.ps1 | iex
4646
4747
# Create a function for ASH
48-
function ash { uvx git+https://github.com/awslabs/automated-security-helper.git@v3.1.12 $args }
48+
function ash { uvx git+https://github.com/awslabs/automated-security-helper.git@v3.1.13 $args }
4949
5050
# Use as normal
5151
ash --help
@@ -57,7 +57,7 @@ ash --help
5757

5858
```bash
5959
# Works on Windows, macOS, and Linux
60-
pipx install git+https://github.com/awslabs/automated-security-helper.git@v3.1.12
60+
pipx install git+https://github.com/awslabs/automated-security-helper.git@v3.1.13
6161

6262
# Use as normal
6363
ash --help
@@ -69,7 +69,7 @@ Standard Python package installation:
6969

7070
```bash
7171
# Works on Windows, macOS, and Linux
72-
pip install git+https://github.com/awslabs/automated-security-helper.git@v3.1.12
72+
pip install git+https://github.com/awslabs/automated-security-helper.git@v3.1.13
7373

7474
# Use as normal
7575
ash --help
@@ -81,7 +81,7 @@ For development or if you want to modify ASH:
8181

8282
```bash
8383
# Works on Windows, macOS, and Linux
84-
git clone https://github.com/awslabs/automated-security-helper.git --branch v3.1.12
84+
git clone https://github.com/awslabs/automated-security-helper.git --branch v3.1.13
8585
cd automated-security-helper
8686
pip install .
8787

@@ -131,7 +131,7 @@ To upgrade ASH to the latest version:
131131
### If installed with `uvx`
132132
```bash
133133
# Your alias will use the latest version when specified
134-
alias ash="uvx git+https://github.com/awslabs/automated-security-helper.git@v3.1.12"
134+
alias ash="uvx git+https://github.com/awslabs/automated-security-helper.git@v3.1.13"
135135
```
136136

137137
### If installed with `pipx`
@@ -141,7 +141,7 @@ pipx upgrade automated-security-helper
141141

142142
### If installed with `pip`
143143
```bash
144-
pip install --upgrade git+https://github.com/awslabs/automated-security-helper.git@v3.1.12
144+
pip install --upgrade git+https://github.com/awslabs/automated-security-helper.git@v3.1.13
145145
```
146146

147147
### If installed from repository

‎docs/content/docs/migration-guide.md‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -48,13 +48,13 @@ export PATH="${PATH}:/path/to/automated-security-helper"
4848

4949
```bash
5050
# Option 1: Using uvx (recommended) -- add to shell profile
51-
alias ash="uvx git+https://github.com/awslabs/automated-security-helper.git@v3.1.12"
51+
alias ash="uvx git+https://github.com/awslabs/automated-security-helper.git@v3.1.13"
5252

5353
# Option 2: Using pipx
54-
pipx install git+https://github.com/awslabs/automated-security-helper.git@v3.1.12
54+
pipx install git+https://github.com/awslabs/automated-security-helper.git@v3.1.13
5555

5656
# Option 3: Using pip
57-
pip install git+https://github.com/awslabs/automated-security-helper.git@v3.1.12
57+
pip install git+https://github.com/awslabs/automated-security-helper.git@v3.1.13
5858
```
5959

6060
## Tool Management Changes

0 commit comments

Comments
 (0)