From e3b98ed18f7bdd243ed27ca61e7400e3a12a059c Mon Sep 17 00:00:00 2001 From: archipelago Date: Fri, 7 Aug 2026 09:58:51 -0400 Subject: [PATCH] fix(security): make audit-secrets.sh actually scan the files that leaked MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The audit passed for months while two live Anthropic keys and the fleet SSH password sat in tracked files. Three independent reasons: - ALLOW_PATTERNS was matched against the whole "file:line:content" string, not the path, so bare words like "test", "demo" and "example" dropped any hit whose *content* merely mentioned them. - `\.md$` was in that same allowlist and `--include` never listed *.md or *.yml, so docs and CI workflows — where every real leak has lived — were never scanned at all. - The false-positive filter spelled the single-quote class `\x27\x27`, which GNU grep does not expand in an ERE, so the empty-string rule never fired. Now: scans tracked files via `git ls-files` (exactly the set that would be published), covers md/yml/mjs/kt/toml, allowlists by path only, and adds patterns for credentialed URLs and inline `sshpass -p`. Test fixtures under testdata/ are exempted narrowly rather than by substring. Verified by planting canary secrets in docs/api-reference.md and .gitea/workflows/build-iso.yml — both file types the old version ignored — and confirming the audit fails on them. Co-Authored-By: Claude Opus 5 (1M context) --- scripts/audit-secrets.sh | 44 +++++++++++++++++++++++++++++++--------- 1 file changed, 34 insertions(+), 10 deletions(-) diff --git a/scripts/audit-secrets.sh b/scripts/audit-secrets.sh index cb1c4823..f9b5456c 100755 --- a/scripts/audit-secrets.sh +++ b/scripts/audit-secrets.sh @@ -18,14 +18,26 @@ PATTERNS=( "api_key\s*=\s*['\"][^'\"]*['\"]" "secret\s*=\s*['\"][^'\"]*['\"]" "private_key\s*=\s*['\"][^'\"]*['\"]" - "sk-ant-" + "sk-ant-[A-Za-z0-9_-]{20,}" "AKIA[A-Z0-9]{16}" "ghp_[a-zA-Z0-9]{36}" "glpat-[a-zA-Z0-9_-]{20}" + # Credentialed URLs: scheme://user:pass@host + "://[A-Za-z0-9_.-]+:[A-Za-z0-9_.@!%-]{8,}@" + # sshpass with an inline literal + "sshpass\s+-p\s*['\"][^'\"]+['\"]" ) -# Allowed files (config templates, docs, test fixtures) -ALLOW_PATTERNS="test|e2e|mock|demo|example|Example|template|CLAUDE.md|deploy-config|\.md$|node_modules|dist|target|default\)|grep.*rpc|audit-secrets|startsWith|should start with" +# Path allowlist — anchored to the PATH only, never to line content. +# The old version allow-matched the whole "file:line:content" string against +# bare words like "test" and "\.md$", so any hit whose path or text contained +# "test"/"demo"/"example" was silently dropped, and *.md was never scanned at +# all. That is why live API keys and node passwords survived this audit. +ALLOW_PATHS="(^|/)node_modules/|(^|/)(dist|target|\.git)/|\.example($|\.)|(^|/)package-lock\.json$|(^|/)Cargo\.lock$|(^|/)scripts/audit-secrets\.sh$" + +# File types to scan. Markdown and YAML are in scope: docs and CI workflows are +# where the real leaks have historically lived. +SCAN_EXTS='\.(rs|ts|vue|js|mjs|cjs|json|sh|py|md|ya?ml|toml|kt|java|gradle|env)$' main() { log "=== Secrets Audit ===" @@ -60,16 +72,26 @@ main() { # 3. Scan source for hardcoded credentials log "3. Scanning source for hardcoded secrets..." local found_secrets=0 + # Scan TRACKED files only — that is exactly the set that would be published. + local scan_files + scan_files=$(cd "$REPO_ROOT" && git ls-files | grep -E "$SCAN_EXTS" | grep -vE "$ALLOW_PATHS" || echo "") + if [ -z "$scan_files" ]; then + fail "No tracked files matched the scan set (is this a git repo?)" + return 1 + fi for pattern in "${PATTERNS[@]}"; do local matches - matches=$(cd "$REPO_ROOT" && grep -rniE "$pattern" \ - --include='*.rs' --include='*.ts' --include='*.vue' --include='*.js' \ - --include='*.json' --include='*.sh' --include='*.py' \ - 2>/dev/null | grep -vE "$ALLOW_PATTERNS" || echo "") + matches=$(cd "$REPO_ROOT" && echo "$scan_files" | tr '\n' '\0' \ + | xargs -0 grep -niE "$pattern" 2>/dev/null || echo "") if [ -n "$matches" ]; then - # Filter out false positives (empty strings, variable declarations, etc.) + # Filter out false positives: empty strings, variable indirection, and + # scrubbed tokens. NOTE: the previous version wrote the + # single-quote class as \x27\x27, which GNU grep does not expand in an + # ERE — so the empty-string rule silently never matched. Use a literal + # quote via a shell variable instead. + local q="'" local real_matches - real_matches=$(echo "$matches" | grep -vE '""|\x27\x27|None|null|undefined|TODO|placeholder|example|Option<|\$\{[A-Z0-9_]+:-\}|\$[A-Z0-9_]+|TestPassword|password123|entertoexit' || echo "") + real_matches=$(echo "$matches" | grep -vE "\"\"|${q}${q}|<[A-Z_]+>|None|null|undefined|TODO|placeholder|Option<|\\\$\{[A-Za-z0-9_]+(:-[^}]*)?\}|\\\$[A-Za-z0-9_]+|TestPassword|password123|entertoexit|…|\\.\\.\\." || echo "") if [ -n "$real_matches" ]; then echo " WARNING: Pattern '$pattern' found:" echo "$real_matches" | head -5 | sed 's/^/ /' @@ -96,7 +118,9 @@ main() { # 5. Check for credential files in repo log "5. Checking for credential files..." local cred_files - cred_files=$(cd "$REPO_ROOT" && git ls-files | grep -Ei '(\.pem$|\.key$|\.p12$|\.pfx$|\.jks$|\.keystore$|id_rsa|id_ed25519|macaroon)' | grep -vE '\.(rs|ts)$' || echo "") + # `testdata/` holds throwaway keypairs generated for unit tests (appgate TLS); + # they are not credentials for anything real. Narrow, path-anchored exemption. + cred_files=$(cd "$REPO_ROOT" && git ls-files | grep -Ei '(\.pem$|\.key$|\.p12$|\.pfx$|\.jks$|\.keystore$|id_rsa|id_ed25519|macaroon)' | grep -vE '\.(rs|ts|sh)$|(^|/)testdata/' || echo "") if [ -z "$cred_files" ]; then pass "No credential files tracked in git" else