139 lines · 6 KBtext/plain; charset=utf-8
Onset: version control where nobody waits and nothing gets lost. The on command, its core and onset-server.
| 6 |
| 7 | use regex_lite::Regex; |
| 8 | use serde::Serialize; |
| 9 |
| 10 | use crate::{Id, Result, Source, chunk, diff, tree}; |
| 11 |
| 12 | #[derive(Debug, Clone, Serialize, PartialEq)] |
| 13 | pub struct Hit { |
| 14 | pub path: String, |
| 15 | pub line: usize, |
| 16 | pub kind: &'static str, |
| 17 | } |
| 18 |
| 19 | /// High-confidence formats only: a false alarm on every landing would teach |
| 20 | /// people to pass --allow-secrets. Each has text that every match contains, |
| 21 | /// so most lines skip the regex. |
| 22 | fn patterns() -> &'static [(&'static str, &'static [&'static str], Regex)] { |
| 23 | static P: OnceLock<Vec<(&'static str, &'static [&'static str], Regex)>> = OnceLock::new(); |
| 24 | P.get_or_init(|| { |
| 25 | let list: [(&'static str, &'static [&'static str], &str); 14] = [ |
| 26 | ("AWS access key", &["AKIA", "ASIA"], r"\b(AKIA|ASIA)[0-9A-Z]{16}\b"), |
| 27 | ("GitHub token", &["ghp_", "gho_", "ghu_", "ghs_", "ghr_"], r"\b(ghp|gho|ghu|ghs|ghr)_[A-Za-z0-9]{36}\b"), |
| 28 | ("GitHub token", &["github_pat_"], r"\bgithub_pat_[A-Za-z0-9_]{60,}\b"), |
| 29 | ("Stripe live key", &["_live_"], r"\b(sk|rk)_live_[A-Za-z0-9]{20,}\b"), |
| 30 | ("Slack token", &["xox"], r"\bxox[abprs]-[A-Za-z0-9-]{20,}\b"), |
| 31 | ("Google API key", &["AIza"], r"\bAIza[0-9A-Za-z_-]{35}\b"), |
| 32 | ("private key", &["PRIVATE KEY"], r"-----BEGIN (RSA |EC |DSA |OPENSSH |PGP |ENCRYPTED )?PRIVATE KEY( BLOCK)?-----"), |
| 33 | ("OpenAI key", &["T3BlbkFJ"], r"\bsk-(proj-)?[A-Za-z0-9_-]{20,}T3BlbkFJ[A-Za-z0-9_-]{20,}\b"), |
| 34 | ("OpenAI key", &["sk-proj-"], r"\bsk-proj-[A-Za-z0-9_-]{60,}\b"), |
| 35 | ("Anthropic key", &["sk-ant-"], r"\bsk-ant-(api|admin)[0-9]{2}-[A-Za-z0-9_-]{60,}\b"), |
| 36 | ("Upstream token", &["upt_", "ucs_"], r"\b(upt|ucs)_[A-Za-z0-9]{24,}\b"), |
| 37 | ("npm token", &["npm_"], r"\bnpm_[A-Za-z0-9]{36}\b"), |
| 38 | ("SendGrid key", &["SG."], r"\bSG\.[A-Za-z0-9_-]{22}\.[A-Za-z0-9_-]{43}\b"), |
| 39 | ("Twilio key", &["SK"], r"\bSK[0-9a-f]{32}\b"), |
| 40 | ]; |
| 41 | list.into_iter().map(|(k, needles, p)| (k, needles, Regex::new(p).expect("secret pattern"))).collect() |
| 42 | }) |
| 43 | } |
| 44 |
| 45 | /// The kind of secret `line` looks like, if any. |
| 46 | fn find(line: &str) -> Option<&'static str> { |
| 47 | patterns().iter().find(|(_, needles, re)| needles.iter().any(|n| line.contains(n)) && re.is_match(line)).map(|(kind, _, _)| *kind) |
| 48 | } |
| 49 |
| 50 | /// `text` with everything that looks like a secret replaced, and how many |
| 51 | /// were (an agent session is kept this way). |
| 52 | pub fn redact(text: &str) -> (String, usize) { |
| 53 | let mut out = text.to_string(); |
| 54 | let mut count = 0; |
| 55 | for (_, needles, re) in patterns() { |
| 56 | if !needles.iter().any(|n| out.contains(n)) { |
| 57 | continue; |
| 58 | } |
| 59 | let found = re.find_iter(&out).count(); |
| 60 | if found > 0 { |
| 61 | count += found; |
| 62 | out = re.replace_all(&out, "[secret removed by vcs]").into_owned(); |
| 63 | } |
| 64 | } |
| 65 | (out, count) |
| 66 | } |
| 67 |
| 68 | /// Secrets in lines `new` adds over `old`. |
| 69 | pub fn scan(src: &dyn Source, old: &Id, new: &Id) -> Result<Vec<Hit>> { |
| 70 | let mut hits = Vec::new(); |
| 71 | for (path, change) in tree::diff_trees(src, old, new)? { |
| 72 | let (o, n) = match change { |
| 73 | tree::Change::Added(e) => (None, e), |
| 74 | tree::Change::Modified(o, n) => (Some(o), n), |
| 75 | tree::Change::Removed(_) => continue, |
| 76 | }; |
| 77 | let new_bytes = chunk::read_file(src, &n.id)?; |
| 78 | if diff::is_binary(&new_bytes) { |
| 79 | continue; |
| 80 | } |
| 81 | let old_bytes = match o { |
| 82 | Some(o) => chunk::read_file(src, &o.id)?, |
| 83 | None => Vec::new(), |
| 84 | }; |
| 85 | let lines = diff::lines(&new_bytes); |
| 86 | for h in diff::hunks(&old_bytes, &new_bytes) { |
| 87 | for i in h.after.clone() { |
| 88 | if let Some(kind) = find(&String::from_utf8_lossy(lines[i as usize])) { |
| 89 | hits.push(Hit { path: path.clone(), line: i as usize + 1, kind }); |
| 90 | } |
| 91 | } |
| 92 | } |
| 93 | } |
| 94 | Ok(hits) |
| 95 | } |
| 96 |
| 97 | #[cfg(test)] |
| 98 | mod tests { |
| 99 | use super::*; |
| 100 |
| 101 | #[test] |
| 102 | fn finds_live_keys_only() { |
| 103 | let hit = |s: &str| find(s).is_some(); |
| 104 | assert!(hit(&format!("const k = \"sk_live_{}\";", "a".repeat(24)))); |
| 105 | assert!(hit("AKIAIOSFODNN7EXAMPLE")); |
| 106 | assert!(hit("-----BEGIN OPENSSH PRIVATE KEY-----")); |
| 107 | assert!(!hit("sk_test_abcdefabcdefabcdefabcdef")); |
| 108 | assert!(!hit("const apiKey = process.env.STRIPE_KEY;")); |
| 109 | } |
| 110 |
| 111 | /// Every pattern's needles are in everything it matches. |
| 112 | #[test] |
| 113 | fn needles_cover_their_patterns() { |
| 114 | let samples = [ |
| 115 | "AKIAIOSFODNN7EXAMPLE".to_string(), |
| 116 | format!("ghp_{}", "a".repeat(36)), |
| 117 | format!("github_pat_{}", "a".repeat(60)), |
| 118 | format!("rk_live_{}", "a".repeat(20)), |
| 119 | format!("xoxb-{}", "a".repeat(20)), |
| 120 | format!("AIza{}", "a".repeat(35)), |
| 121 | "-----BEGIN PRIVATE KEY-----".to_string(), |
| 122 | format!("sk-{}T3BlbkFJ{}", "a".repeat(20), "a".repeat(20)), |
| 123 | format!("sk-proj-{}", "a".repeat(60)), |
| 124 | format!("sk-ant-api03-{}", "a".repeat(60)), |
| 125 | format!("upt_{}", "a".repeat(24)), |
| 126 | format!("npm_{}", "a".repeat(36)), |
| 127 | format!("SG.{}.{}", "a".repeat(22), "a".repeat(43)), |
| 128 | format!("SK{}", "0".repeat(32)), |
| 129 | ]; |
| 130 | for (kind, needles, re) in patterns() { |
| 131 | let matched: Vec<&String> = samples.iter().filter(|s| re.is_match(s)).collect(); |
| 132 | assert!(!matched.is_empty(), "no sample for {kind}"); |
| 133 | for s in matched { |
| 134 | assert!(needles.iter().any(|n| s.contains(n)), "{kind}: {s} has none of {needles:?}"); |
| 135 | } |
| 136 | } |
| 137 | } |
| 138 | } |
| 139 |