fix(security): make audit-secrets.sh actually scan the files that leaked
The audit passed for months while two live Anthropic keys and the fleet SSH password sat in tracked files. Three independent reasons: - ALLOW_PATTERNS was matched against the whole "file:line:content" string, not the path, so bare words like "test", "demo" and "example" dropped any hit whose *content* merely mentioned them. - `\.md$` was in that same allowlist and `--include` never listed *.md or *.yml, so docs and CI workflows — where every real leak has lived — were never scanned at all. - The false-positive filter spelled the single-quote class `\x27\x27`, which GNU grep does not expand in an ERE, so the empty-string rule never fired. Now: scans tracked files via `git ls-files` (exactly the set that would be published), covers md/yml/mjs/kt/toml, allowlists by path only, and adds patterns for credentialed URLs and inline `sshpass -p`. Test fixtures under testdata/ are exempted narrowly rather than by substring. Verified by planting canary secrets in docs/api-reference.md and .gitea/workflows/build-iso.yml — both file types the old version ignored — and confirming the audit fails on them. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
archipelago
co-authored by
Claude Opus 5
Claude Opus 5
parent
bd98ec6e3d
commit
e3b98ed18f
+34
-10
@@ -18,14 +18,26 @@ PATTERNS=(
|
||||
"api_key\s*=\s*['\"][^'\"]*['\"]"
|
||||
"secret\s*=\s*['\"][^'\"]*['\"]"
|
||||
"private_key\s*=\s*['\"][^'\"]*['\"]"
|
||||
"sk-ant-"
|
||||
"sk-ant-[A-Za-z0-9_-]{20,}"
|
||||
"AKIA[A-Z0-9]{16}"
|
||||
"ghp_[a-zA-Z0-9]{36}"
|
||||
"glpat-[a-zA-Z0-9_-]{20}"
|
||||
# Credentialed URLs: scheme://user:pass@host
|
||||
"://[A-Za-z0-9_.-]+:[A-Za-z0-9_.@!%-]{8,}@"
|
||||
# sshpass with an inline literal
|
||||
"sshpass\s+-p\s*['\"][^'\"]+['\"]"
|
||||
)
|
||||
|
||||
# Allowed files (config templates, docs, test fixtures)
|
||||
ALLOW_PATTERNS="test|e2e|mock|demo|example|Example|template|CLAUDE.md|deploy-config|\.md$|node_modules|dist|target|default\)|grep.*rpc|audit-secrets|startsWith|should start with"
|
||||
# Path allowlist — anchored to the PATH only, never to line content.
|
||||
# The old version allow-matched the whole "file:line:content" string against
|
||||
# bare words like "test" and "\.md$", so any hit whose path or text contained
|
||||
# "test"/"demo"/"example" was silently dropped, and *.md was never scanned at
|
||||
# all. That is why live API keys and node passwords survived this audit.
|
||||
ALLOW_PATHS="(^|/)node_modules/|(^|/)(dist|target|\.git)/|\.example($|\.)|(^|/)package-lock\.json$|(^|/)Cargo\.lock$|(^|/)scripts/audit-secrets\.sh$"
|
||||
|
||||
# File types to scan. Markdown and YAML are in scope: docs and CI workflows are
|
||||
# where the real leaks have historically lived.
|
||||
SCAN_EXTS='\.(rs|ts|vue|js|mjs|cjs|json|sh|py|md|ya?ml|toml|kt|java|gradle|env)$'
|
||||
|
||||
main() {
|
||||
log "=== Secrets Audit ==="
|
||||
@@ -60,16 +72,26 @@ main() {
|
||||
# 3. Scan source for hardcoded credentials
|
||||
log "3. Scanning source for hardcoded secrets..."
|
||||
local found_secrets=0
|
||||
# Scan TRACKED files only — that is exactly the set that would be published.
|
||||
local scan_files
|
||||
scan_files=$(cd "$REPO_ROOT" && git ls-files | grep -E "$SCAN_EXTS" | grep -vE "$ALLOW_PATHS" || echo "")
|
||||
if [ -z "$scan_files" ]; then
|
||||
fail "No tracked files matched the scan set (is this a git repo?)"
|
||||
return 1
|
||||
fi
|
||||
for pattern in "${PATTERNS[@]}"; do
|
||||
local matches
|
||||
matches=$(cd "$REPO_ROOT" && grep -rniE "$pattern" \
|
||||
--include='*.rs' --include='*.ts' --include='*.vue' --include='*.js' \
|
||||
--include='*.json' --include='*.sh' --include='*.py' \
|
||||
2>/dev/null | grep -vE "$ALLOW_PATTERNS" || echo "")
|
||||
matches=$(cd "$REPO_ROOT" && echo "$scan_files" | tr '\n' '\0' \
|
||||
| xargs -0 grep -niE "$pattern" 2>/dev/null || echo "")
|
||||
if [ -n "$matches" ]; then
|
||||
# Filter out false positives (empty strings, variable declarations, etc.)
|
||||
# Filter out false positives: empty strings, variable indirection, and
|
||||
# scrubbed <PLACEHOLDER> tokens. NOTE: the previous version wrote the
|
||||
# single-quote class as \x27\x27, which GNU grep does not expand in an
|
||||
# ERE — so the empty-string rule silently never matched. Use a literal
|
||||
# quote via a shell variable instead.
|
||||
local q="'"
|
||||
local real_matches
|
||||
real_matches=$(echo "$matches" | grep -vE '""|\x27\x27|None|null|undefined|TODO|placeholder|example|Option<|\$\{[A-Z0-9_]+:-\}|\$[A-Z0-9_]+|TestPassword|password123|entertoexit' || echo "")
|
||||
real_matches=$(echo "$matches" | grep -vE "\"\"|${q}${q}|<[A-Z_]+>|None|null|undefined|TODO|placeholder|Option<|\\\$\{[A-Za-z0-9_]+(:-[^}]*)?\}|\\\$[A-Za-z0-9_]+|TestPassword|password123|entertoexit|…|\\.\\.\\." || echo "")
|
||||
if [ -n "$real_matches" ]; then
|
||||
echo " WARNING: Pattern '$pattern' found:"
|
||||
echo "$real_matches" | head -5 | sed 's/^/ /'
|
||||
@@ -96,7 +118,9 @@ main() {
|
||||
# 5. Check for credential files in repo
|
||||
log "5. Checking for credential files..."
|
||||
local cred_files
|
||||
cred_files=$(cd "$REPO_ROOT" && git ls-files | grep -Ei '(\.pem$|\.key$|\.p12$|\.pfx$|\.jks$|\.keystore$|id_rsa|id_ed25519|macaroon)' | grep -vE '\.(rs|ts)$' || echo "")
|
||||
# `testdata/` holds throwaway keypairs generated for unit tests (appgate TLS);
|
||||
# they are not credentials for anything real. Narrow, path-anchored exemption.
|
||||
cred_files=$(cd "$REPO_ROOT" && git ls-files | grep -Ei '(\.pem$|\.key$|\.p12$|\.pfx$|\.jks$|\.keystore$|id_rsa|id_ed25519|macaroon)' | grep -vE '\.(rs|ts|sh)$|(^|/)testdata/' || echo "")
|
||||
if [ -z "$cred_files" ]; then
|
||||
pass "No credential files tracked in git"
|
||||
else
|
||||
|
||||
Reference in New Issue
Block a user