|
| 1 | +# Copyright 2026 The LoongForge Authors. |
| 2 | +# SPDX-License-Identifier: Apache-2.0 |
| 3 | + |
| 4 | +"""Rule definitions for the sensitive-information scanner. |
| 5 | +
|
| 6 | +Kept as a Python module (rather than JSON/TOML) so that patterns can use raw |
| 7 | +strings without double-escaping, and so that each rule can carry an inline |
| 8 | +rationale. Consumed by ``ci/sensitive_scan.py``. |
| 9 | +
|
| 10 | +Severity contract: |
| 11 | + error - blocks CI. Directly identifies internal infrastructure, a person, |
| 12 | + or credential material. |
| 13 | + warn - reported but does not block. Usually a naming or path-hygiene |
| 14 | + issue: real, but high-volume and not individually disclosive. |
| 15 | +
|
| 16 | +To silence a single line, append a trailing comment: |
| 17 | + sensitive-scan: allow (all rules on that line) |
| 18 | + sensitive-scan: allow[rule-id,rule-id] (specific rules) |
| 19 | +
|
| 20 | +To silence a whole path or pattern, add an entry to ALLOWLIST below with a |
| 21 | +``reason``. Entries without a reason are rejected by the scanner. |
| 22 | +""" |
| 23 | + |
| 24 | +CONFIG = { |
| 25 | + # Paths never scanned. Matched with fnmatch against the repo-relative path. |
| 26 | + "exclude_paths": [ |
| 27 | + ".git/*", |
| 28 | + "third_party/*", |
| 29 | + "output/*", |
| 30 | + "outputs/*", |
| 31 | + # Model/training data: Chinese SFT corpora legitimately contain words |
| 32 | + # such as 机密 and long digit strings that would swamp the report. |
| 33 | + "tests/llm_vlm/datasets/*", |
| 34 | + "*.png", |
| 35 | + "*.jpg", |
| 36 | + "*.jpeg", |
| 37 | + "*.gif", |
| 38 | + "*.svg", |
| 39 | + "*.ico", |
| 40 | + "*.mp4", |
| 41 | + "*.pdf", |
| 42 | + "*.whl", |
| 43 | + "*.tar.gz", |
| 44 | + "*.pt", |
| 45 | + "*.bin", |
| 46 | + "*.safetensors", |
| 47 | + "*.DS_Store", |
| 48 | + ], |
| 49 | + # Cap findings printed per rule so a high-volume warn rule cannot drown the |
| 50 | + # report. Full counts are always shown in the summary. |
| 51 | + "max_findings_per_rule": 20, |
| 52 | +} |
| 53 | + |
| 54 | +RULES = [ |
| 55 | + # ---------------------------------------------------------------- error -- |
| 56 | + { |
| 57 | + "id": "internal-domain", |
| 58 | + "severity": "error", |
| 59 | + "title": "Internal-only hostname", |
| 60 | + "why": "Corporate intranet hosts are unreachable externally and expose " |
| 61 | + "internal tooling topology.", |
| 62 | + "pattern": r"(?i)\b[\w.-]*\.baidu-int\.com\b" |
| 63 | + r"|\biregistry\.[\w.-]+\b" |
| 64 | + r"|\bicode\.baidu\.com\b" |
| 65 | + r"|\b(?:ku|wiki|note|agile|icafe|newicafe|noah)\.baidu(?:-int)?\.com\b", |
| 66 | + "hint": "Point at public documentation, or drop the reference entirely.", |
| 67 | + }, |
| 68 | + { |
| 69 | + "id": "internal-package-mirror", |
| 70 | + "severity": "error", |
| 71 | + "title": "Internal package registry or mirror", |
| 72 | + "why": "Hardcoded internal mirrors make builds fail for external users " |
| 73 | + "and leak the internal build chain.", |
| 74 | + "pattern": r"(?i)\b(?:registry|mirrors)\.baidubce\.com\b" |
| 75 | + r"|\bpip\.baidu(?:-int)?\.com\b" |
| 76 | + r"|--index-url\s+\S*baidu\S*", |
| 77 | + "hint": "Use the public index (pypi.org / nvcr.io / docker.io), or make " |
| 78 | + "it an overridable build arg.", |
| 79 | + }, |
| 80 | + { |
| 81 | + "id": "private-object-storage", |
| 82 | + "severity": "error", |
| 83 | + "title": "Private object-storage location", |
| 84 | + "why": "Private bucket prefixes are not publicly readable and disclose " |
| 85 | + "the internal artifact-distribution layout.", |
| 86 | + "pattern": r"(?i)\bbos:/\S+" |
| 87 | + r"|\baihc-private-[\w-]+" |
| 88 | + r"|\baiak[_-]share\b", |
| 89 | + "hint": "Replace with a public URL or a documented placeholder such as " |
| 90 | + "bos:/path/to/<artifact>/.", |
| 91 | + }, |
| 92 | + { |
| 93 | + "id": "corp-email", |
| 94 | + "severity": "error", |
| 95 | + "title": "Corporate email address", |
| 96 | + "why": "Directly identifies an employee.", |
| 97 | + "pattern": r"(?i)\b[a-z0-9._%+-]+@(?:baidu|baidu-int|baidubce)\.com\b", |
| 98 | + "hint": "Remove, or replace with a role address / 'The LoongForge Authors'.", |
| 99 | + "redact": True, |
| 100 | + }, |
| 101 | + { |
| 102 | + "id": "developer-home-path", |
| 103 | + "severity": "error", |
| 104 | + "title": "Developer home directory", |
| 105 | + "why": "A /home/<login>/ path names an individual and never resolves on " |
| 106 | + "another machine.", |
| 107 | + "pattern": r"/home/(?!opt/|user/|users/|ubuntu/|runner/|admin/|work/)" |
| 108 | + r"[a-z][a-z0-9_.-]{2,}/", |
| 109 | + "hint": "Use a generic root such as /workspace/ or an env-var override.", |
| 110 | + }, |
| 111 | + { |
| 112 | + "id": "secret-material", |
| 113 | + "severity": "error", |
| 114 | + "title": "Key material or provider token", |
| 115 | + "why": "Live credential material.", |
| 116 | + "pattern": r"-----BEGIN (?:RSA |EC |DSA |OPENSSH |PGP )?PRIVATE KEY-----" |
| 117 | + r"|\bhf_[A-Za-z0-9]{20,}\b" |
| 118 | + r"|\bgh[pousr]_[A-Za-z0-9]{20,}\b" |
| 119 | + r"|\bglpat-[A-Za-z0-9_-]{20,}\b" |
| 120 | + r"|\bxox[baprs]-[A-Za-z0-9-]{10,}\b" |
| 121 | + r"|\bsk-[A-Za-z0-9]{20,}\b" |
| 122 | + r"|\bAKIA[0-9A-Z]{16}\b", |
| 123 | + "hint": "Revoke the credential, then read it from the environment or a " |
| 124 | + "secret store.", |
| 125 | + "redact": True, |
| 126 | + }, |
| 127 | + { |
| 128 | + "id": "credential-assignment", |
| 129 | + "severity": "error", |
| 130 | + "title": "Credential assigned to a literal value", |
| 131 | + "why": "AK/SK/token/password bound to an inline literal.", |
| 132 | + # Requires the value to be a real literal: either quoted, or a |
| 133 | + # space-free shell assignment. The lookaheads demand at least one digit |
| 134 | + # and one letter, which excludes code expressions (`sk = index_k.size(0)`, |
| 135 | + # `token = os.environ.get(...)`) and obvious placeholders (`xxx`, |
| 136 | + # `your-token`). |
| 137 | + "pattern": r"(?i)\b(?:ak|sk|access_?key(?:_id)?|secret_?(?:access_)?key" |
| 138 | + r"|api_?key|passwo?rd|passwd|auth_?token|token)\b" |
| 139 | + r"(?:" |
| 140 | + r"\s*[:=]\s*\"(?=[^\"]*\d)(?=[^\"]*[A-Za-z])[A-Za-z0-9/+=_.-]{12,}\"" |
| 141 | + r"|\s*[:=]\s*'(?=[^']*\d)(?=[^']*[A-Za-z])[A-Za-z0-9/+=_.-]{12,}'" |
| 142 | + r"|=(?=[^\s\"']*\d)(?=[^\s\"']*[A-Za-z])[A-Za-z0-9/+=_-]{12,}\b" |
| 143 | + r")", |
| 144 | + "hint": "Read from the environment instead of committing the value.", |
| 145 | + "redact": True, |
| 146 | + }, |
| 147 | + { |
| 148 | + "id": "internal-network-address", |
| 149 | + "severity": "error", |
| 150 | + "title": "RFC1918 address", |
| 151 | + "why": "A private-range address identifies an internal host and is " |
| 152 | + "useless (or actively misleading) outside the corporate network.", |
| 153 | + "pattern": r"\b(?:10|192\.168|172\.(?:1[6-9]|2\d|3[01]))" |
| 154 | + r"\.\d{1,3}\.\d{1,3}\.\d{1,3}\b(?::\d{1,5})?" |
| 155 | + r"|\b(?:10|192\.168|172\.(?:1[6-9]|2\d|3[01]))" |
| 156 | + r"\.\d{1,3}\.\d{1,3}\b:\d{1,5}", |
| 157 | + "hint": "Use localhost, a DNS name, or an env-var override.", |
| 158 | + }, |
| 159 | + { |
| 160 | + "id": "hardcoded-proxy", |
| 161 | + "severity": "error", |
| 162 | + "title": "Hardcoded proxy endpoint", |
| 163 | + "why": "Proxy addresses reveal internal egress infrastructure.", |
| 164 | + "pattern": r"(?i)\b(?:https?_proxy|all_proxy)\s*=\s*[\"']?" |
| 165 | + r"(?:https?://)?(?!\$|\{|<|your-|127\.0\.0\.1|localhost)" |
| 166 | + r"[a-z0-9][\w.-]*(?::\d+)?", |
| 167 | + "hint": "Expose it as ${http_proxy} and let the caller supply the value.", |
| 168 | + }, |
| 169 | + { |
| 170 | + "id": "confidentiality-marker", |
| 171 | + "severity": "error", |
| 172 | + "title": "Confidentiality / do-not-publish marker", |
| 173 | + "why": "Explicit internal-only markings must not survive into a public " |
| 174 | + "repository.", |
| 175 | + "pattern": r"(?i)Private\s*::\s*Do Not Upload" |
| 176 | + r"|\bConfidential\b" |
| 177 | + r"|\bInternal Use Only\b" |
| 178 | + r"|\bProprietary and Confidential\b" |
| 179 | + r"|机密|绝密|内部资料|不打算开源", |
| 180 | + "hint": "Remove the marker and confirm the file is cleared for release.", |
| 181 | + }, |
| 182 | + # ----------------------------------------------------------------- warn -- |
| 183 | + { |
| 184 | + "id": "user-scoped-path", |
| 185 | + "severity": "warn", |
| 186 | + "title": "Per-user directory in a path", |
| 187 | + "why": "A /users/<name>/ segment names a person or team account.", |
| 188 | + "pattern": r"(?i)/users?/[a-z][a-z0-9_.-]{2,}/", |
| 189 | + "hint": "Drop the user segment or move it behind an env-var override.", |
| 190 | + }, |
| 191 | + { |
| 192 | + "id": "internal-cluster-path", |
| 193 | + "severity": "warn", |
| 194 | + "title": "Internal cluster filesystem path", |
| 195 | + "why": "Collectively these describe the internal cluster layout. Usually " |
| 196 | + "harmless ${VAR:-default} values, but they should not be the " |
| 197 | + "shipped defaults.", |
| 198 | + "pattern": r"(?i)(?:^|[\s\"'=:(\[])/(?:mnt/(?:cluster|rapidfs|cfs[\w-]*|data)" |
| 199 | + r"|ssd\d)(?:/|\b)", |
| 200 | + "hint": "Prefer /workspace/... or a documented placeholder as the default.", |
| 201 | + }, |
| 202 | + { |
| 203 | + "id": "legacy-internal-name", |
| 204 | + "severity": "warn", |
| 205 | + "title": "Pre-rename internal project name", |
| 206 | + "why": "Leftover internal names leak the rename history and, when used " |
| 207 | + "as a path or symbol default, break external users.", |
| 208 | + "pattern": r"(?i)\bAIAK[-_](?:Training[-_](?:Omni|LLM)|Megatron)\b" |
| 209 | + r"|\bBaigeOmni\b" |
| 210 | + r"|\bLoongForge-VLA\b" |
| 211 | + r"|\baiak_training_omni\b" |
| 212 | + r"|\bset_aiak_\w+|\binitialize_baige_\w+|\bUSE_AIAK_\w+" |
| 213 | + r"|\baiak-ckpt\b", |
| 214 | + "hint": "Rename to the public equivalent (LoongForge / Loong-Megatron).", |
| 215 | + }, |
| 216 | + { |
| 217 | + "id": "internal-doc-reference", |
| 218 | + "severity": "warn", |
| 219 | + "title": "Reference to an internal-only document or library", |
| 220 | + "why": "Dangling pointers to internal docs are unactionable externally.", |
| 221 | + "pattern": r"(?i)<\s*Baige[^>]*>" |
| 222 | + r"|internal Baige\b" |
| 223 | + r"|\[internal use only\]" |
| 224 | + r"|内部文档|内网文档|详见\s*wiki", |
| 225 | + "hint": "Inline the needed information or remove the pointer.", |
| 226 | + }, |
| 227 | + { |
| 228 | + "id": "internal-codename", |
| 229 | + "severity": "warn", |
| 230 | + "title": "Internal hardware or product codename", |
| 231 | + "why": "Non-public chip/cluster/product codenames disclose the internal " |
| 232 | + "hardware fleet and roadmap.", |
| 233 | + "pattern": r"(?i)\bBZZ\d?\b|\bP6K\b|\bDECK_STD\w*\b|\bqianfan\b|\bwenxin\b", |
| 234 | + "hint": "Use a public model designation, or parameterize it.", |
| 235 | + }, |
| 236 | + { |
| 237 | + "id": "attributed-todo", |
| 238 | + "severity": "warn", |
| 239 | + "title": "TODO/FIXME attributed to a named handle", |
| 240 | + "why": "Handles may be corporate logins. Upstream handles inherited from " |
| 241 | + "vendored code are fine and belong in the allowlist.", |
| 242 | + "pattern": r"\b(?:TODO|FIXME|XXX|HACK)\(\s*([A-Za-z][\w.-]{2,})\s*\)", |
| 243 | + "hint": "Drop the handle, or allowlist it if it came from upstream code.", |
| 244 | + }, |
| 245 | + { |
| 246 | + "id": "internal-ticket-id", |
| 247 | + "severity": "warn", |
| 248 | + "title": "Internal issue-tracker identifier", |
| 249 | + "why": "Ticket IDs disclose the internal tracker and numbering scheme.", |
| 250 | + "pattern": r"(?i)\baiak-train-\d+\b|\bicafe/\S+|\bhac-aiacc\b", |
| 251 | + "hint": "Remove, or restate the change without the ticket reference.", |
| 252 | + }, |
| 253 | +] |
| 254 | + |
| 255 | +# Each entry needs: rule, path (fnmatch glob), reason. |
| 256 | +# Optional: match (regex applied to the matched text; when absent, every match |
| 257 | +# under `path` is allowed). |
| 258 | +ALLOWLIST = [ |
| 259 | + { |
| 260 | + "rule": "legacy-internal-name", |
| 261 | + "path": "README*.md", |
| 262 | + "match": r"AIAK-Training-LLM", |
| 263 | + "reason": "Deliberate public attribution of the former product name.", |
| 264 | + }, |
| 265 | + { |
| 266 | + "rule": "legacy-internal-name", |
| 267 | + "path": "docs/source*/get_started/README.md", |
| 268 | + "match": r"AIAK-Training-LLM", |
| 269 | + "reason": "Same public attribution, mirrored into the docs site.", |
| 270 | + }, |
| 271 | + { |
| 272 | + "rule": "internal-codename", |
| 273 | + "path": "*", |
| 274 | + "match": r"(?i)Qianfan-VL", |
| 275 | + "reason": "Link to the public Qianfan-VL model release on GitHub.", |
| 276 | + }, |
| 277 | + { |
| 278 | + "rule": "private-object-storage", |
| 279 | + "path": "*", |
| 280 | + "match": r"bos:/path/to/", |
| 281 | + "reason": "Documented placeholder, not a real bucket.", |
| 282 | + }, |
| 283 | + { |
| 284 | + "rule": "private-object-storage", |
| 285 | + "path": ".github/workflows/*.yml", |
| 286 | + "match": r"bos:/\$\{?BOS_BUCKET", |
| 287 | + "reason": "Bucket name comes from a repository variable, not from the source.", |
| 288 | + }, |
| 289 | + { |
| 290 | + "rule": "*", |
| 291 | + "path": "ci/sensitive_rules.py", |
| 292 | + "reason": "Rule patterns necessarily contain the strings they match.", |
| 293 | + }, |
| 294 | + { |
| 295 | + "rule": "*", |
| 296 | + "path": "skills/sensitive-scan/SKILL.md", |
| 297 | + "reason": "Skill documentation quotes example findings.", |
| 298 | + }, |
| 299 | +] |
0 commit comments