scriptoriumadmin/fabro
1
0
Fork 0
mirror of https://github.com/fabro-sh/fabro.git synced 2026-09-12 23:02:41 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 5

Add redact crate for secret detection and redaction in NDJSON logs

Browse source 
Two-layer detection: Shannon entropy on high-entropy alphanumeric tokens
(threshold 4.5) and gitleaks v8.22.1 pattern matching (202 rules) with
Aho-Corasick keyword pre-filtering. JSONL-aware redaction skips exempt
fields (IDs, paths) and image objects. 43 tests.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
Entire-Checkpoint: 9e8476d16412
This commit is contained in:
Bryan Helmkamp 2026-02-25 15:25:52 -05:00
parent 1388f6539a
commit 90e7bf229a
9 changed files with 4115 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

70
Cargo.lock generated
View file

@ -2096,6 +2096,17 @@ dependencies = [
"getrandom 0.3.4",
]
[[package]]
name = "redact"
version = "0.1.0"
dependencies = [
"aho-corasick",
"regex",
"serde",
"serde_json",
"toml",
]
[[package]]
name = "redox_syscall"
version = "0.5.18"
@ -2542,6 +2553,15 @@ dependencies = [
"syn",
]
[[package]]
name = "serde_spanned"
version = "0.6.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3"
dependencies = [
"serde",
]
[[package]]
name = "serde_urlencoded"
version = "0.7.1"
@ -2890,6 +2910,47 @@ dependencies = [
"tokio",
]
[[package]]
name = "toml"
version = "0.8.23"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362"
dependencies = [
"serde",
"serde_spanned",
"toml_datetime",
"toml_edit",
]
[[package]]
name = "toml_datetime"
version = "0.6.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c"
dependencies = [
"serde",
]
[[package]]
name = "toml_edit"
version = "0.22.27"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a"
dependencies = [
"indexmap 2.13.0",
"serde",
"serde_spanned",
"toml_datetime",
"toml_write",
"winnow",
]
[[package]]
name = "toml_write"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801"
[[package]]
name = "tower"
version = "0.5.3"
@ -3581,6 +3642,15 @@ version = "0.53.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650"
[[package]]
name = "winnow"
version = "0.7.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5a5364e9d77fcdeeaa6062ced926ee3381faa2ee02d3eb83a5c27a8825540829"
dependencies = [
"memchr",
]
[[package]]
name = "wit-bindgen"
version = "0.51.0"

2
Cargo.toml
View file

@ -31,3 +31,5 @@ tar = "0.4"
dialoguer = "0.12"
git2 = "0.19"
walkdir = "2"
regex = "1"
aho-corasick = "1"

20
crates/redact/Cargo.toml Normal file
View file

@ -0,0 +1,20 @@
[package]
name = "redact"
edition.workspace = true
version.workspace = true
license.workspace = true
description = "Secret detection and redaction for log output"
[lib]
doctest = false
[dependencies]
regex.workspace = true
aho-corasick.workspace = true
serde_json.workspace = true
[build-dependencies]
toml = "0.8"
serde = { workspace = true }
[dev-dependencies]

169
crates/redact/build.rs Normal file
View file

@ -0,0 +1,169 @@
use serde::Deserialize;
use std::env;
use std::fs;
use std::path::Path;
#[derive(Deserialize)]
struct Config {
allowlist: Option<GlobalAllowlist>,
rules: Vec<Rule>,
}
#[derive(Deserialize)]
struct GlobalAllowlist {
regexes: Option<Vec<String>>,
stopwords: Option<Vec<String>>,
}
#[derive(Deserialize)]
#[serde(rename_all = "camelCase")]
struct Rule {
id: String,
regex: String,
#[serde(default)]
keywords: Vec<String>,
entropy: Option<f64>,
#[serde(default)]
allowlist: Option<RuleAllowlist>,
#[allow(dead_code)]
description: Option<String>,
}
#[derive(Deserialize)]
#[serde(rename_all = "camelCase")]
struct RuleAllowlist {
regexes: Option<Vec<String>>,
stopwords: Option<Vec<String>>,
regex_target: Option<String>,
}
fn escape_rust_string(s: &str) -> String {
let mut out = String::with_capacity(s.len() + 10);
for ch in s.chars() {
match ch {
'\\' => out.push_str("\\\\"),
'"' => out.push_str("\\\""),
'\n' => out.push_str("\\n"),
'\r' => out.push_str("\\r"),
'\t' => out.push_str("\\t"),
c => out.push(c),
}
}
out
}
fn main() {
println!("cargo:rerun-if-changed=data/gitleaks.toml");
let toml_path = Path::new("data/gitleaks.toml");
let toml_content = fs::read_to_string(toml_path).expect("failed to read gitleaks.toml");
let config: Config = toml::from_str(&toml_content).expect("failed to parse gitleaks.toml");
let out_dir = env::var("OUT_DIR").unwrap();
let out_path = Path::new(&out_dir).join("rules_generated.rs");
let mut code = String::new();
// Global allowlist regexes
code.push_str("pub const GLOBAL_ALLOWLIST_REGEXES: &[&str] = &[\n");
if let Some(ref al) = config.allowlist {
if let Some(ref regexes) = al.regexes {
for r in regexes {
code.push_str(&format!(" \"{}\",\n", escape_rust_string(r)));
}
}
}
code.push_str("];\n\n");
// Global allowlist stopwords
code.push_str("pub const GLOBAL_ALLOWLIST_STOPWORDS: &[&str] = &[\n");
if let Some(ref al) = config.allowlist {
if let Some(ref stopwords) = al.stopwords {
for sw in stopwords {
code.push_str(&format!(" \"{}\",\n", escape_rust_string(sw)));
}
}
}
code.push_str("];\n\n");
// Rule definitions
code.push_str("#[allow(dead_code)]\n");
code.push_str("pub struct RuleDef {\n");
code.push_str(" pub id: &'static str,\n");
code.push_str(" pub regex_pattern: &'static str,\n");
code.push_str(" pub keywords: &'static [&'static str],\n");
code.push_str(" pub entropy: Option<f64>,\n");
code.push_str(" pub allowlist_regexes: &'static [&'static str],\n");
code.push_str(" pub allowlist_stopwords: &'static [&'static str],\n");
code.push_str(" pub allowlist_regex_target: Option<&'static str>,\n");
code.push_str("}\n\n");
code.push_str("pub const RULES: &[RuleDef] = &[\n");
for rule in &config.rules {
code.push_str(" RuleDef {\n");
code.push_str(&format!(
" id: \"{}\",\n",
escape_rust_string(&rule.id)
));
code.push_str(&format!(
" regex_pattern: \"{}\",\n",
escape_rust_string(&rule.regex)
));
// Keywords
code.push_str(" keywords: &[");
for kw in &rule.keywords {
code.push_str(&format!("\"{}\", ", escape_rust_string(kw)));
}
code.push_str("],\n");
// Entropy
match rule.entropy {
Some(e) => code.push_str(&format!(" entropy: Some({:.1}),\n", e)),
None => code.push_str(" entropy: None,\n"),
}
// Allowlist regexes
code.push_str(" allowlist_regexes: &[");
if let Some(ref al) = rule.allowlist {
if let Some(ref regexes) = al.regexes {
for r in regexes {
code.push_str(&format!("\"{}\", ", escape_rust_string(r)));
}
}
}
code.push_str("],\n");
// Allowlist stopwords
code.push_str(" allowlist_stopwords: &[");
if let Some(ref al) = rule.allowlist {
if let Some(ref stopwords) = al.stopwords {
for sw in stopwords {
code.push_str(&format!("\"{}\", ", escape_rust_string(sw)));
}
}
}
code.push_str("],\n");
// Allowlist regex target
if let Some(ref al) = rule.allowlist {
if let Some(ref target) = al.regex_target {
code.push_str(&format!(
" allowlist_regex_target: Some(\"{}\"),\n",
escape_rust_string(target)
));
} else {
code.push_str(" allowlist_regex_target: None,\n");
}
} else {
code.push_str(" allowlist_regex_target: None,\n");
}
code.push_str(" },\n");
}
code.push_str("];\n");
fs::write(&out_path, code).expect("failed to write generated rules");
}

3050
crates/redact/data/gitleaks.toml Normal file
View file

File diff suppressed because it is too large Load diff

120
crates/redact/src/entropy.rs Normal file
View file

@ -0,0 +1,120 @@
use crate::Region;
use regex::Regex;
use std::sync::LazyLock;
/// Matches high-entropy alphanumeric strings (10+ chars).
/// Excludes `/` to avoid matching file paths as single tokens.
static SECRET_PATTERN: LazyLock<Regex> =
LazyLock::new(|| Regex::new(r"[A-Za-z0-9+_=-]{10,}").unwrap());
const ENTROPY_THRESHOLD: f64 = 4.5;
/// Compute Shannon entropy (bits per byte) of a string.
pub fn shannon_entropy(s: &str) -> f64 {
if s.is_empty() {
return 0.0;
}
let mut freq = [0u32; 256];
for &b in s.as_bytes() {
freq[b as usize] += 1;
}
let len = s.len() as f64;
let mut entropy = 0.0;
for &count in &freq {
if count > 0 {
let p = count as f64 / len;
entropy -= p * p.log2();
}
}
entropy
}
/// Find high-entropy alphanumeric tokens in `s`.
///
/// Returns regions where tokens match `[A-Za-z0-9+_=-]{10,}` and have
/// Shannon entropy above the threshold (4.5 bits). Protects against
/// consuming characters from JSON escape sequences.
pub fn find_entropy_regions(s: &str) -> Vec<Region> {
let mut regions = Vec::new();
for m in SECRET_PATTERN.find_iter(s) {
let mut start = m.start();
let end = m.end();
// Protect against consuming characters from JSON escape sequences.
// E.g. in "controller.go\nmodel.go", regex could match "nmodel"
// (consuming 'n' from '\n'). Skip the escape character to avoid
// creating invalid escape sequences after replacement.
if start > 0 && s.as_bytes()[start - 1] == b'\\' {
match s.as_bytes()[start] {
b'n' | b't' | b'r' | b'b' | b'f' | b'u' | b'"' | b'\\' | b'/' => {
start += 1;
if end - start < 10 {
continue;
}
}
_ => {}
}
}
if shannon_entropy(&s[start..end]) > ENTROPY_THRESHOLD {
regions.push(Region { start, end });
}
}
regions
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn entropy_empty_string() {
assert_eq!(shannon_entropy(""), 0.0);
}
#[test]
fn entropy_single_char_repeated() {
assert_eq!(shannon_entropy("aaaa"), 0.0);
}
#[test]
fn entropy_two_equal_chars() {
let e = shannon_entropy("ab");
assert!((e - 1.0).abs() < 0.001, "expected ~1.0, got {e}");
}
#[test]
fn entropy_aws_key_above_3() {
let e = shannon_entropy("AKIAIOSFODNN7EXAMPLE");
assert!(e > 3.0, "expected > 3.0, got {e}");
}
#[test]
fn regions_empty_for_normal_text() {
assert!(find_entropy_regions("hello world").is_empty());
}
#[test]
fn regions_finds_high_entropy_token() {
// `=` is in the regex pattern, so "key=xK9..." matches as one token
let input = "key=xK9mZ2vL8nQ5rT1wY4bC7dF0gH3jE6p";
let regions = find_entropy_regions(input);
assert_eq!(regions.len(), 1);
assert_eq!(regions[0].start, 0);
assert_eq!(regions[0].end, input.len());
}
#[test]
fn regions_empty_for_json_escape_sequence() {
// "controller.go\nmodel.go" — the regex could match across the \n boundary
let regions = find_entropy_regions(r"controller.go\nmodel.go");
assert!(regions.is_empty(), "got regions: {regions:?}");
}
#[test]
fn regions_empty_for_file_path() {
// / is excluded from the pattern, so path segments are short
let regions = find_entropy_regions("/tmp/test/controller.go");
assert!(regions.is_empty(), "got regions: {regions:?}");
}
}

267
crates/redact/src/gitleaks.rs Normal file
View file

@ -0,0 +1,267 @@
use crate::Region;
use aho_corasick::AhoCorasick;
use regex::Regex;
use std::sync::LazyLock;
mod generated {
include!(concat!(env!("OUT_DIR"), "/rules_generated.rs"));
}
struct CompiledRule {
#[allow(dead_code)]
id: &'static str,
regex: Regex,
#[allow(dead_code)]
keywords: &'static [&'static str],
secret_group: usize,
allowlist_regexes: Vec<Regex>,
allowlist_stopwords: &'static [&'static str],
allowlist_regex_target: Option<&'static str>,
}
struct GitleaksEngine {
keyword_filter: AhoCorasick,
/// For each keyword, which rule indices use it.
keyword_to_rules: Vec<Vec<usize>>,
/// Rules that have no keywords (must always be checked).
no_keyword_rules: Vec<usize>,
rules: Vec<CompiledRule>,
global_allowlist_regexes: Vec<Regex>,
global_allowlist_stopwords: &'static [&'static str],
}
impl GitleaksEngine {
fn build() -> Option<GitleaksEngine> {
let mut rules = Vec::new();
let mut all_keywords: Vec<String> = Vec::new();
let mut keyword_to_rules: Vec<Vec<usize>> = Vec::new();
let mut no_keyword_rules = Vec::new();
for def in generated::RULES {
let regex = match Regex::new(def.regex_pattern) {
Ok(r) => r,
Err(_) => continue, // skip rules with invalid regex
};
// Determine which capture group holds the secret.
// If the regex has capture groups, group 1 is the secret.
// Otherwise, group 0 (full match) is the secret.
let secret_group = if regex.captures_len() > 1 { 1 } else { 0 };
let allowlist_regexes: Vec<Regex> = def
.allowlist_regexes
.iter()
.filter_map(|p| Regex::new(p).ok())
.collect();
let rule_idx = rules.len();
if def.keywords.is_empty() {
no_keyword_rules.push(rule_idx);
} else {
for kw in def.keywords {
let kw_lower = kw.to_lowercase();
// Check if this keyword already exists
if let Some(pos) = all_keywords.iter().position(|k| k == &kw_lower) {
keyword_to_rules[pos].push(rule_idx);
} else {
all_keywords.push(kw_lower);
keyword_to_rules.push(vec![rule_idx]);
}
}
}
rules.push(CompiledRule {
id: def.id,
regex,
keywords: def.keywords,
secret_group,
allowlist_regexes,
allowlist_stopwords: def.allowlist_stopwords,
allowlist_regex_target: def.allowlist_regex_target,
});
}
let keyword_filter = AhoCorasick::builder()
.ascii_case_insensitive(true)
.build(&all_keywords)
.ok()?;
let global_allowlist_regexes: Vec<Regex> = generated::GLOBAL_ALLOWLIST_REGEXES
.iter()
.filter_map(|p| Regex::new(p).ok())
.collect();
Some(GitleaksEngine {
keyword_filter,
keyword_to_rules,
no_keyword_rules,
rules,
global_allowlist_regexes,
global_allowlist_stopwords: generated::GLOBAL_ALLOWLIST_STOPWORDS,
})
}
fn find_regions(&self, s: &str) -> Vec<Region> {
// Determine which rules to check based on keyword matches.
let s_lower = s.to_lowercase();
let mut rule_indices: Vec<bool> = vec![false; self.rules.len()];
// Always include rules with no keywords.
for &idx in &self.no_keyword_rules {
rule_indices[idx] = true;
}
// Use overlapping search to ensure short keywords (e.g. "sk") don't
// prevent longer overlapping keywords (e.g. "sk_test") from matching.
for mat in self.keyword_filter.find_overlapping_iter(&s_lower) {
for &rule_idx in &self.keyword_to_rules[mat.pattern().as_usize()] {
rule_indices[rule_idx] = true;
}
}
let mut regions = Vec::new();
for (idx, should_check) in rule_indices.iter().enumerate() {
if !should_check {
continue;
}
let rule = &self.rules[idx];
let captures_iter = rule.regex.captures_iter(s);
for caps in captures_iter {
let full_match = caps.get(0).unwrap();
// Get the secret: group 1 if it exists, otherwise full match
let secret_match = if rule.secret_group > 0 {
match caps.get(rule.secret_group) {
Some(m) => m,
None => continue,
}
} else {
full_match
};
let secret = secret_match.as_str();
if secret.is_empty() {
continue;
}
// Check global allowlist regexes against the secret
if self
.global_allowlist_regexes
.iter()
.any(|r| r.is_match(secret))
{
continue;
}
// Check global allowlist stopwords
if self
.global_allowlist_stopwords.contains(&secret)
{
continue;
}
// Check rule-level allowlist
let allowlist_target = if rule.allowlist_regex_target == Some("match") {
full_match.as_str()
} else {
secret
};
if rule
.allowlist_regexes
.iter()
.any(|r| r.is_match(allowlist_target))
{
continue;
}
// Check rule-level stopwords
let secret_lower = secret.to_lowercase();
if rule
.allowlist_stopwords
.iter()
.any(|sw| secret_lower == *sw)
{
continue;
}
regions.push(Region {
start: secret_match.start(),
end: secret_match.end(),
});
}
}
regions
}
}
static ENGINE: LazyLock<Option<GitleaksEngine>> = LazyLock::new(GitleaksEngine::build);
/// Find regions matching gitleaks rules.
pub fn find_gitleaks_regions(s: &str) -> Vec<Region> {
match ENGINE.as_ref() {
Some(engine) => engine.find_regions(s),
None => Vec::new(),
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn detects_aws_access_key() {
let regions = find_gitleaks_regions("key=AKIAYRWQG5EJLPZLBYNP");
assert_eq!(regions.len(), 1, "expected 1 region, got {regions:?}");
assert_eq!(&"key=AKIAYRWQG5EJLPZLBYNP"[regions[0].start..regions[0].end], "AKIAYRWQG5EJLPZLBYNP");
}
#[test]
fn detects_github_pat() {
// ghp_ + exactly 36 alphanumeric chars
let input = "ghp_ABCDEFGHIJKLMNOPQRSTUVWXYZabcdef0123";
assert_eq!(input.len(), 40); // ghp_(4) + 36 = 40
let regions = find_gitleaks_regions(input);
assert_eq!(regions.len(), 1, "expected 1 region, got {regions:?}");
assert_eq!(&input[regions[0].start..regions[0].end], input);
}
#[test]
fn detects_stripe_key() {
// Stripe regex requires trailing whitespace/punctuation or end of string
let input = "sk_test_4eC39HqLyjWDarjtT1zdp7dc ";
let regions = find_gitleaks_regions(input);
assert_eq!(regions.len(), 1, "expected 1 region, got {regions:?}");
}
#[test]
fn detects_private_key_block() {
let input = "-----BEGIN RSA PRIVATE KEY-----\nMIIEpAIBAAKCAQEA\n-----END RSA PRIVATE KEY-----";
let regions = find_gitleaks_regions(input);
assert_eq!(regions.len(), 1, "expected 1 region, got {regions:?}");
}
#[test]
fn normal_text_not_flagged() {
let regions = find_gitleaks_regions("Hello, this is a normal English sentence with no secrets.");
assert!(regions.is_empty(), "expected no regions, got {regions:?}");
}
#[test]
fn global_allowlist_stopwords_respected() {
// The global allowlist includes a UUID that should not be flagged
let regions = find_gitleaks_regions("014df517-39d1-4453-b7b3-9930c563627c");
assert!(regions.is_empty(), "expected no regions for allowlisted UUID, got {regions:?}");
}
#[test]
fn aws_example_key_not_flagged() {
// AKIAIOSFODNN7EXAMPLE ends with EXAMPLE — per-rule allowlist
let regions = find_gitleaks_regions("key=AKIAIOSFODNN7EXAMPLE");
assert!(regions.is_empty(), "expected no regions for example key, got {regions:?}");
}
}

304
crates/redact/src/jsonl.rs Normal file
View file

@ -0,0 +1,304 @@
use serde_json::Value;
/// Returns true if a JSON key should be excluded from scanning/redaction.
///
/// Skips "signature" (exact), ID fields (ending in "id"/"ids"), and common
/// path/directory fields from agent transcripts.
fn should_skip_field(key: &str) -> bool {
if key == "signature" {
return true;
}
let lower = key.to_lowercase();
// Skip ID fields
if lower.ends_with("id") || lower.ends_with("ids") {
return true;
}
// Skip common path and directory fields
matches!(
lower.as_str(),
"filepath" | "file_path" | "cwd" | "root" | "directory" | "dir" | "path"
)
}
/// Returns true if the object has "type":"image", "type":"image_url", or "type":"base64".
fn should_skip_object(obj: &serde_json::Map<String, Value>) -> bool {
match obj.get("type").and_then(Value::as_str) {
Some(t) => t.starts_with("image") || t == "base64",
None => false,
}
}
/// Walk a parsed JSON value and collect (original, redacted) string pairs.
fn collect_replacements(v: &Value) -> Vec<(String, String)> {
let mut seen = std::collections::HashSet::new();
let mut repls = Vec::new();
fn walk(
v: &Value,
seen: &mut std::collections::HashSet<String>,
repls: &mut Vec<(String, String)>,
) {
match v {
Value::Object(obj) => {
if should_skip_object(obj) {
return;
}
for (k, child) in obj {
if should_skip_field(k) {
continue;
}
walk(child, seen, repls);
}
}
Value::Array(arr) => {
for child in arr {
walk(child, seen, repls);
}
}
Value::String(s) => {
let redacted = crate::redact_string(s);
if redacted != *s && seen.insert(s.clone()) {
repls.push((s.clone(), redacted));
}
}
_ => {}
}
}
walk(v, &mut seen, &mut repls);
repls
}
/// JSON-encode a string value (with quotes), without HTML escaping.
fn json_encode_string(s: &str) -> String {
serde_json::to_string(s).unwrap_or_else(|_| format!("\"{}\"", s))
}
/// Redact secrets in a single JSONL line.
///
/// Parses the line as JSON to determine which string values need redaction,
/// then performs targeted replacements on the raw JSON bytes. Lines with no
/// secrets are returned unchanged, preserving original formatting.
///
/// Falls back to `redact_string` on invalid JSON.
pub fn redact_jsonl_line(line: &str) -> String {
let trimmed = line.trim();
if trimmed.is_empty() {
return line.to_string();
}
let parsed: Value = match serde_json::from_str(trimmed) {
Ok(v) => v,
Err(_) => return crate::redact_string(line),
};
let repls = collect_replacements(&parsed);
if repls.is_empty() {
return line.to_string();
}
let mut result = line.to_string();
for (orig, redacted) in &repls {
let orig_json = json_encode_string(orig);
let redacted_json = json_encode_string(redacted);
result = result.replace(&orig_json, &redacted_json);
}
result
}
#[cfg(test)]
mod tests {
use super::*;
const HIGH_ENTROPY_SECRET: &str = "sk-ant-api03-xK9mZ2vL8nQ5rT1wY4bC7dF0gH3jE6pA";
#[test]
fn skip_field_session_id() {
assert!(should_skip_field("session_id"));
}
#[test]
fn skip_field_content_not_skipped() {
assert!(!should_skip_field("content"));
}
#[test]
fn skip_field_file_path() {
assert!(should_skip_field("file_path"));
}
#[test]
fn skip_field_filepath() {
assert!(should_skip_field("filePath"));
}
#[test]
fn skip_field_id_variants() {
assert!(should_skip_field("id"));
assert!(should_skip_field("sessionId"));
assert!(should_skip_field("checkpoint_id"));
assert!(should_skip_field("checkpointID"));
assert!(should_skip_field("userIds"));
}
#[test]
fn skip_field_path_variants() {
assert!(should_skip_field("cwd"));
assert!(should_skip_field("root"));
assert!(should_skip_field("directory"));
assert!(should_skip_field("dir"));
assert!(should_skip_field("path"));
}
#[test]
fn skip_field_false_positives() {
assert!(!should_skip_field("content"));
assert!(!should_skip_field("type"));
assert!(!should_skip_field("name"));
assert!(!should_skip_field("text"));
assert!(!should_skip_field("output"));
assert!(!should_skip_field("video"));
assert!(!should_skip_field("identify"));
assert!(!should_skip_field("signatures"));
assert!(!should_skip_field("consideration"));
}
#[test]
fn skip_object_image_type() {
let obj: serde_json::Map<String, Value> =
serde_json::from_str(r#"{"type": "image", "data": "base64data"}"#).unwrap();
assert!(should_skip_object(&obj));
}
#[test]
fn skip_object_text_type_not_skipped() {
let obj: serde_json::Map<String, Value> =
serde_json::from_str(r#"{"type": "text", "content": "hello"}"#).unwrap();
assert!(!should_skip_object(&obj));
}
#[test]
fn skip_object_no_type() {
let obj: serde_json::Map<String, Value> =
serde_json::from_str(r#"{"content": "hello"}"#).unwrap();
assert!(!should_skip_object(&obj));
}
#[test]
fn skip_object_image_url() {
let obj: serde_json::Map<String, Value> =
serde_json::from_str(r#"{"type": "image_url"}"#).unwrap();
assert!(should_skip_object(&obj));
}
#[test]
fn skip_object_base64() {
let obj: serde_json::Map<String, Value> =
serde_json::from_str(r#"{"type": "base64"}"#).unwrap();
assert!(should_skip_object(&obj));
}
#[test]
fn redact_jsonl_line_no_secrets() {
let input = r#"{"type":"text","content":"hello"}"#;
assert_eq!(redact_jsonl_line(input), input);
}
#[test]
fn redact_jsonl_line_with_secret_in_content() {
let input = format!(
r#"{{"type":"text","content":"key={HIGH_ENTROPY_SECRET}"}}"#
);
let result = redact_jsonl_line(&input);
assert!(
result.contains("REDACTED"),
"expected REDACTED in: {result}"
);
assert!(
!result.contains(HIGH_ENTROPY_SECRET),
"secret should be redacted: {result}"
);
}
#[test]
fn redact_jsonl_line_preserves_session_id() {
let input = format!(
r#"{{"session_id":"{HIGH_ENTROPY_SECRET}","content":"normal text"}}"#
);
let result = redact_jsonl_line(&input);
assert!(
result.contains(HIGH_ENTROPY_SECRET),
"session_id should be preserved: {result}"
);
assert!(
!result.contains("REDACTED"),
"should have no redactions: {result}"
);
}
#[test]
fn redact_jsonl_line_fallback_on_invalid_json() {
let input = format!(
r#"{{"type":"text", "invalid {HIGH_ENTROPY_SECRET} json"#
);
let result = redact_jsonl_line(&input);
assert!(
result.contains("REDACTED"),
"expected REDACTED in: {result}"
);
}
#[test]
fn redact_jsonl_line_preserves_file_path_field() {
let input = r#"{"file_path":"/private/var/folders/v4/31cd3cg52_sfrpb1mbtr7q7r0000gn/T/test/controller.go","content":"normal text"}"#;
let result = redact_jsonl_line(input);
assert!(
result.contains("/private/var/folders"),
"file_path should be preserved: {result}"
);
assert!(
!result.contains("REDACTED"),
"should have no redactions: {result}"
);
}
#[test]
fn redact_jsonl_line_secrets_in_content_not_in_paths() {
let input = format!(
r#"{{"file_path":"/tmp/test.go","content":"api_key={HIGH_ENTROPY_SECRET}"}}"#
);
let result = redact_jsonl_line(&input);
assert!(
result.contains("/tmp/test.go"),
"file_path should be preserved: {result}"
);
assert!(
!result.contains(HIGH_ENTROPY_SECRET),
"secret in content should be redacted: {result}"
);
assert!(
result.contains("REDACTED"),
"expected REDACTED in: {result}"
);
}
#[test]
fn redact_jsonl_line_image_object_skipped() {
let input = format!(
r#"{{"type":"image","data":"{HIGH_ENTROPY_SECRET}"}}"#
);
let result = redact_jsonl_line(&input);
assert!(
!result.contains("REDACTED"),
"image data should not be redacted: {result}"
);
}
#[test]
fn redact_jsonl_line_empty_line() {
assert_eq!(redact_jsonl_line(""), "");
assert_eq!(redact_jsonl_line(" "), " ");
}
}

113
crates/redact/src/lib.rs Normal file
View file

@ -0,0 +1,113 @@
mod entropy;
mod gitleaks;
mod jsonl;
pub use jsonl::redact_jsonl_line;
/// A byte range within a string that should be redacted.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Region {
pub start: usize,
pub end: usize,
}
/// Replace all detected secrets in `s` with "REDACTED".
///
/// Uses two layers of detection:
/// 1. Shannon entropy on high-entropy alphanumeric tokens
/// 2. Gitleaks regex pattern matching (200+ known secret formats)
pub fn redact_string(s: &str) -> String {
let mut regions = entropy::find_entropy_regions(s);
regions.extend(gitleaks::find_gitleaks_regions(s));
if regions.is_empty() {
return s.to_string();
}
regions.sort_by_key(|r| r.start);
// Merge overlapping regions
let mut merged = vec![regions[0].clone()];
for r in &regions[1..] {
let last = merged.last_mut().unwrap();
if r.start <= last.end {
last.end = last.end.max(r.end);
} else {
merged.push(r.clone());
}
}
let mut result = String::with_capacity(s.len());
let mut prev = 0;
for r in &merged {
result.push_str(&s[prev..r.start]);
result.push_str("REDACTED");
prev = r.end;
}
result.push_str(&s[prev..]);
result
}
#[cfg(test)]
mod tests {
use super::*;
const HIGH_ENTROPY_SECRET: &str = "sk-ant-api03-xK9mZ2vL8nQ5rT1wY4bC7dF0gH3jE6pA";
#[test]
fn redact_string_no_secrets() {
assert_eq!(redact_string("hello world"), "hello world");
}
#[test]
fn redact_string_with_aws_key() {
let result = redact_string("key=AKIAYRWQG5EJLPZLBYNP");
assert_eq!(result, "key=REDACTED");
}
#[test]
fn redact_string_overlapping_detections_produce_single_redacted() {
// A high-entropy string that also matches a gitleaks pattern
// should produce one REDACTED, not two
let input = format!("key={HIGH_ENTROPY_SECRET}");
let result = redact_string(&input);
assert_eq!(
result.matches("REDACTED").count(),
1,
"expected single REDACTED, got: {result}"
);
}
#[test]
fn redact_string_two_secrets_separated_by_space() {
let input = "key=AKIAYRWQG5EJLPZLBYNP AKIAYRWQG5EJLPZLBYNP";
let result = redact_string(input);
assert_eq!(result, "key=REDACTED REDACTED");
}
#[test]
fn redact_string_file_path_preserved() {
let input = "/tmp/test/controller.go";
assert_eq!(redact_string(input), input);
}
#[test]
fn redact_string_json_escape_preserved() {
let input = r"controller.go\nmodel.go";
assert_eq!(redact_string(input), input);
}
#[test]
fn redact_string_github_pat() {
let input = "token=ghp_ABCDEFGHIJKLMNOPQRSTUVWXYZabcdef0123";
let result = redact_string(input);
assert!(result.contains("REDACTED"), "expected REDACTED in: {result}");
}
#[test]
fn redact_string_private_key() {
let input = "-----BEGIN RSA PRIVATE KEY-----\nMIIEpAIBAAKCAQEA\n-----END RSA PRIVATE KEY-----";
let result = redact_string(input);
assert!(result.contains("REDACTED"), "expected REDACTED in: {result}");
}
}