Skip to content

Commit e4dd489

Browse files
Bornunique911Bornunique911
committed
Add refresh scripts for OWASP resources
1 parent 8df31e9 commit e4dd489

3 files changed

Lines changed: 222 additions & 0 deletions

File tree

‎scripts/update-cheatsheets.sh‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
#!/usr/bin/env bash
2+
set -euo pipefail
3+
4+
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
5+
DB_PATH="${1:-$ROOT_DIR/standards_cache.sqlite}"
6+
VENV_DIR="$ROOT_DIR/venv"
7+
8+
if [[ ! -d "$VENV_DIR" ]]; then
9+
python3 -m venv "$VENV_DIR"
10+
fi
11+
12+
source "$VENV_DIR/bin/activate"
13+
14+
if ! python -c "import flask" >/dev/null 2>&1; then
15+
pip install -r "$ROOT_DIR/requirements.txt"
16+
fi
17+
18+
CRE_NO_CALCULATE_GAP_ANALYSIS=1 \
19+
CRE_NO_GEN_EMBEDDINGS=1 \
20+
python "$ROOT_DIR/cre.py" --cheatsheets_in --cache_file "$DB_PATH"
21+
22+
python - "$DB_PATH" <<'PY'
23+
import os
24+
import sqlite3
25+
import sys
26+
27+
db_path = sys.argv[1]
28+
conn = sqlite3.connect(db_path)
29+
cur = conn.cursor()
30+
31+
github_prefix = "https://github.com/OWASP/CheatSheetSeries/tree/master/cheatsheets/"
32+
official_prefix = "https://cheatsheetseries.owasp.org/cheatsheets/"
33+
34+
rows = cur.execute(
35+
"""
36+
select id, link
37+
from node
38+
where name = 'OWASP Cheat Sheets'
39+
and link like ?
40+
""",
41+
(f"{github_prefix}%",),
42+
).fetchall()
43+
44+
for node_id, link in rows:
45+
filename = os.path.basename(link)
46+
html_name = os.path.splitext(filename)[0] + ".html"
47+
cur.execute(
48+
"update node set link = ? where id = ?",
49+
(f"{official_prefix}{html_name}", node_id),
50+
)
51+
52+
conn.commit()
53+
conn.close()
54+
print(f"Normalized {len(rows)} OWASP Cheat Sheet links")
55+
PY

‎scripts/update-owasp-top10-2025-mappings.sh‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
#!/usr/bin/env bash
2+
3+
set -euo pipefail
4+
5+
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
6+
VENV_DIR="$ROOT_DIR/venv"
7+
CACHE_FILE="${1:-$ROOT_DIR/standards_cache.sqlite}"
8+
TIMESTAMP="$(date +%Y%m%d-%H%M%S)"
9+
BACKUP_FILE="${CACHE_FILE}.bak.${TIMESTAMP}"
10+
11+
if [[ ! -d "$VENV_DIR" ]]; then
12+
echo "Creating virtual environment in $VENV_DIR"
13+
python3 -m venv "$VENV_DIR"
14+
fi
15+
16+
source "$VENV_DIR/bin/activate"
17+
18+
if ! python -c "import requests" >/dev/null 2>&1; then
19+
echo "Installing Python dependencies"
20+
pip install -r "$ROOT_DIR/requirements.txt"
21+
fi
22+
23+
if [[ -f "$CACHE_FILE" ]]; then
24+
cp "$CACHE_FILE" "$BACKUP_FILE"
25+
echo "Backed up database to $BACKUP_FILE"
26+
fi
27+
28+
export CRE_NO_NEO4J="${CRE_NO_NEO4J:-1}"
29+
export CRE_NO_GEN_EMBEDDINGS="${CRE_NO_GEN_EMBEDDINGS:-1}"
30+
export CRE_UPSTREAM_MAX_ATTEMPTS="${CRE_UPSTREAM_MAX_ATTEMPTS:-6}"
31+
export CRE_UPSTREAM_RETRY_BACKOFF_SECONDS="${CRE_UPSTREAM_RETRY_BACKOFF_SECONDS:-2}"
32+
export CRE_UPSTREAM_TIMEOUT_SECONDS="${CRE_UPSTREAM_TIMEOUT_SECONDS:-30}"
33+
34+
echo "Refreshing official OpenCRE upstream data in $CACHE_FILE"
35+
python "$ROOT_DIR/cre.py" --upstream_sync --cache_file "$CACHE_FILE"
36+
37+
echo "Reapplying OWASP Top 10 2025 CRE mappings"
38+
exec python "$ROOT_DIR/cre.py" --owasp_top10_2025_in --cache_file "$CACHE_FILE"

‎scripts/update-owasp-top10-standards.sh‎

Lines changed: 129 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,129 @@
1+
#!/usr/bin/env bash
2+
3+
set -euo pipefail
4+
5+
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
6+
VENV_DIR="$ROOT_DIR/venv"
7+
CACHE_FILE="${1:-$ROOT_DIR/standards_cache.sqlite}"
8+
TIMESTAMP="$(date +%Y%m%d-%H%M%S)"
9+
BACKUP_FILE="${CACHE_FILE}.bak.${TIMESTAMP}"
10+
11+
if [[ ! -d "$VENV_DIR" ]]; then
12+
echo "Creating virtual environment in $VENV_DIR"
13+
python3 -m venv "$VENV_DIR"
14+
fi
15+
16+
source "$VENV_DIR/bin/activate"
17+
18+
if ! python -c "import requests" >/dev/null 2>&1; then
19+
echo "Installing Python dependencies"
20+
pip install -r "$ROOT_DIR/requirements.txt"
21+
fi
22+
23+
if [[ -f "$CACHE_FILE" ]]; then
24+
cp "$CACHE_FILE" "$BACKUP_FILE"
25+
echo "Backed up database to $BACKUP_FILE"
26+
fi
27+
28+
export CRE_NO_NEO4J="${CRE_NO_NEO4J:-1}"
29+
export CRE_NO_GEN_EMBEDDINGS="${CRE_NO_GEN_EMBEDDINGS:-1}"
30+
export CRE_UPSTREAM_MAX_ATTEMPTS="${CRE_UPSTREAM_MAX_ATTEMPTS:-6}"
31+
export CRE_UPSTREAM_RETRY_BACKOFF_SECONDS="${CRE_UPSTREAM_RETRY_BACKOFF_SECONDS:-2}"
32+
export CRE_UPSTREAM_TIMEOUT_SECONDS="${CRE_UPSTREAM_TIMEOUT_SECONDS:-30}"
33+
34+
echo "Refreshing official OpenCRE upstream data in $CACHE_FILE"
35+
python "$ROOT_DIR/cre.py" --upstream_sync --cache_file "$CACHE_FILE"
36+
37+
echo "Reapplying OWASP Top 10 standards and CRE mappings"
38+
python "$ROOT_DIR/cre.py" \
39+
--owasp_top10_2025_in \
40+
--owasp_api_top10_2023_in \
41+
--owasp_kubernetes_top10_2025_in \
42+
--owasp_llm_top10_2025_in \
43+
--owasp_aisvs_in \
44+
--cache_file "$CACHE_FILE"
45+
46+
echo "Selecting preferred Kubernetes Top Ten version"
47+
if python - <<'PY' "$CACHE_FILE"
48+
import sqlite3
49+
import sys
50+
51+
cache_file = sys.argv[1]
52+
name_2025 = "OWASP Kubernetes Top Ten 2025 (Draft)"
53+
name_2022 = "OWASP Kubernetes Top Ten 2022"
54+
55+
conn = sqlite3.connect(cache_file)
56+
cur = conn.cursor()
57+
58+
linked_2025 = cur.execute(
59+
"""
60+
select count(*)
61+
from node n
62+
join cre_node_links l on l.node = n.id
63+
where n.name = ?
64+
""",
65+
(name_2025,),
66+
).fetchone()[0]
67+
68+
if linked_2025 > 0:
69+
cur.execute("delete from node where name = ?", (name_2022,))
70+
print(f"Using {name_2025}; removed {name_2022}")
71+
else:
72+
raise SystemExit(f"{name_2025} not linked")
73+
74+
conn.commit()
75+
conn.close()
76+
PY
77+
then
78+
:
79+
else
80+
echo "OWASP Kubernetes Top Ten 2025 (Draft) is unavailable or unmapped, importing 2022"
81+
python "$ROOT_DIR/cre.py" \
82+
--owasp_kubernetes_top10_2022_in \
83+
--cache_file "$CACHE_FILE"
84+
fi
85+
86+
echo "Pruning OWASP Top 10 entries that still have no CRE links"
87+
python - <<'PY' "$CACHE_FILE"
88+
import sqlite3
89+
import sys
90+
91+
cache_file = sys.argv[1]
92+
standard_names = (
93+
"OWASP Top 10 2025",
94+
"OWASP API Security Top 10 2023",
95+
"OWASP Kubernetes Top Ten 2025 (Draft)",
96+
"OWASP Top 10 for LLM and Gen AI Apps 2025",
97+
"OWASP AI Security Verification Standard (AISVS)",
98+
)
99+
100+
conn = sqlite3.connect(cache_file)
101+
cur = conn.cursor()
102+
103+
has_2022 = cur.execute(
104+
"select 1 from node where name = 'OWASP Kubernetes Top Ten 2022' limit 1"
105+
).fetchone()
106+
if has_2022:
107+
standard_names = standard_names + ("OWASP Kubernetes Top Ten 2022",)
108+
109+
rows = list(
110+
cur.execute(
111+
f"""
112+
select n.id, n.name, coalesce(n.section_id, ''), coalesce(n.section, '')
113+
from node n
114+
left join cre_node_links l on l.node = n.id
115+
where n.name in ({','.join('?' for _ in standard_names)})
116+
group by n.id
117+
having count(l.cre) = 0
118+
""",
119+
standard_names,
120+
)
121+
)
122+
123+
for node_id, name, section_id, section in rows:
124+
cur.execute("delete from node where id = ?", (node_id,))
125+
print(f"Removed unmapped entry: {name} {section_id} {section}".strip())
126+
127+
conn.commit()
128+
conn.close()
129+
PY

0 commit comments

Comments
 (0)