Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 2 additions & 0 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -168,6 +168,8 @@ arc-swap = "1"
blake3 = "1"
# PKCE 的 S256。版本跟着 reqwest 的传递依赖走,不引新的树
sha2 = "0.11"
bech32 = "0.9.1"
curve25519-dalek = { version = "4.1.3", default-features = false, features = ["alloc", "precomputed-tables"] }
uuid = { version = "1", features = ["v4", "serde"] }
rand = "0.10"
# 发给 Codex 后端的请求体用 zstd 压缩(Codex 直连它时就这么发)。只要流式编码,
Expand Down
2 changes: 2 additions & 0 deletions crates/tw-guard/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,8 @@ regex-syntax = { workspace = true }
memchr = { workspace = true }
aho-corasick = { workspace = true }
sha2 = { workspace = true }
bech32 = { workspace = true }
curve25519-dalek = { workspace = true }
thiserror = { workspace = true }
serde_yaml_ng = { workspace = true }
tw-dialect = { workspace = true }
Expand Down
87 changes: 83 additions & 4 deletions crates/tw-guard/data/redact.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -44,8 +44,8 @@
# **前缀类的规则不写 `about`**:说明由 build.rs 按前缀和长度生成,和判据不会说两样。
# 加一条前缀类的规则只改这个文件。
#
# 每一条内置规则都要求「前缀明确」或者「结构上无法误认」。代码里的 hash、base64 的图片、
# UUID 全都长得像「一长串随机字符」,按那个判据匹配会疯狂误报。
# 凭据规则按前缀、结构或字段位置识别。UUID、裸十六进制和链上地址另作隐私规则,
# 不把这些无从区分用途的格式单独当成密钥外传证据。

kinds:
- slug: saved
Expand Down Expand Up @@ -869,8 +869,87 @@ rules:
on_by_default: true
name: Extended private key
name_zh: 扩展私钥
about: 'Starts with `xprv`, `tprv`, `yprv` or `zprv` and passes Base58Check'
about_zh: '`xprv`、`tprv`、`yprv`、`zprv` 开头,Base58Check 校验通过'
about: 'BIP32 / SLIP-0132 extended private keys: `xprv`, `tprv`, `yprv`, `Yprv`, `zprv`, `Zprv`, `uprv`, `Uprv`, `vprv` or `Vprv`; valid Base58Check, private marker and secp256k1 scalar'
about_zh: 'BIP32 / SLIP-0132 扩展私钥:`xprv`、`tprv`、`yprv`、`Yprv`、`zprv`、`Zprv`、`uprv`、`Uprv`、`vprv`、`Vprv`;Base58Check、私钥标记及 secp256k1 标量均有效'


- id: uuid
kind: device
on_by_default: true
name: UUID / compact identifier
name_zh: UUID/紧凑标识符
about: 'Hyphenated UUIDs and 32 hexadecimal characters, ignoring case and without version restrictions, including order and trace IDs; compact hashes of the same shape are also replaced'
about_zh: '带连字符的 UUID 和 32 位十六进制串,不区分大小写、不限制版本,包括订单号和追踪 ID;同形状的紧凑哈希也替换'
- id: crypto-hex-key
kind: crypto
on_by_default: true
name: Raw wallet key / 32-byte hash
name_zh: 裸钱包私钥/32 字节哈希
about: '64 hexadecimal characters with an optional `0x` prefix; covers EVM and TRON private keys as well as indistinguishable transaction hashes and digests'
about_zh: '64 位十六进制串,可带 `0x` 前缀;覆盖 EVM、TRON 裸私钥以及格式无法区分的交易哈希和摘要'
- id: crypto-solana-key
kind: crypto
on_by_default: true
name: Solana keypair
name_zh: Solana 密钥对
about: '64-byte Ed25519 seed and public key encoded as Base58 or a pasted JSON byte array; the public half must match the seed, excluding signatures of the same length'
about_zh: 'Base58 编码或作为文本粘贴的 JSON 字节数组,含 64 字节 Ed25519 种子与公钥;由种子派生的公钥须一致,排除同长度签名'
- id: crypto-sui-key
kind: crypto
on_by_default: true
name: Sui private key
name_zh: Sui 私钥
about: '`suiprivkey1` Bech32 encoding with a valid checksum, a supported signature scheme and 32 key bytes'
about_zh: '`suiprivkey1` Bech32 编码,校验和有效,含支持的签名方案和 32 字节密钥'
- id: crypto-stellar-key
kind: crypto
on_by_default: true
name: Stellar secret seed
name_zh: Stellar 私密种子
about: 'Stellar `S` StrKey seed with the correct version, 32-byte payload and CRC16-XModem checksum'
about_zh: 'Stellar `S` StrKey 种子,版本、32 字节载荷和 CRC16-XModem 校验均有效'
- id: crypto-xrp-seed
kind: crypto
on_by_default: true
name: XRP family seed
name_zh: XRP 账户种子
about: 'XRP-alphabet Base58Check family seeds, including secp256k1 and Ed25519 version encodings'
about_zh: 'XRP 字母表的 Base58Check 账户种子,包括 secp256k1 和 Ed25519 版本'
- id: crypto-address
kind: financial
on_by_default: true
name: Blockchain account / extended public key
name_zh: 区块链账户/扩展公钥
about: 'EVM hex addresses, checksummed Bitcoin/TRON/XRP/Stellar addresses, Bitcoin witness addresses and BIP32 / SLIP-0132 extended public keys; also 32-byte Base58 account shapes such as Solana, which lack a checksum'
about_zh: 'EVM 十六进制地址,经校验的 Bitcoin/TRON/XRP/Stellar 地址、Bitcoin 隔离见证地址和 BIP32/SLIP-0132 扩展公钥;也覆盖 Solana 等无校验和的 32 字节 Base58 账户格式'
- id: crypto-secret-field
kind: in-context
on_by_default: true
name: Wallet secret in a field
name_zh: 字段中的钱包机密
about: 'Literal private keys, seeds, mnemonic/recovery phrases, passphrases, recovery codes and MPC/backup shares in named fields, including `pk`, `privateKey`, `PRIVY_WALLET_PRIVATE_KEY` and Chinese labels; only the value is replaced, with no guessed key length'
about_zh: '按字段识别私钥、种子、助记词、恢复短语、口令、恢复码和 MPC/备份份额,包括 `pk`、`privateKey`、`PRIVY_WALLET_PRIVATE_KEY` 和中文标签;只换值,不猜测密钥长度'
- id: exchange-credential
kind: in-context
on_by_default: true
name: Trading API / signed authentication credential
name_zh: 交易 API/签名鉴权凭证
about: 'Literal API keys, secrets, passphrases, signing memos, OTPs and authentication signatures in SDK/config fields or trading headers; Binance, OKX, Bybit, Coinbase, Kraken, KuCoin, Bitget, Gate, MEXC, Gemini, Bitfinex, HTX, BitMart, Phemex, Bitstamp, Polymarket and predict.fun; only the value is replaced'
about_zh: 'SDK/配置字段或交易请求头中的 API key、secret、passphrase、签名 Memo、OTP 和鉴权签名;覆盖 Binance、OKX、Bybit、Coinbase、Kraken、KuCoin、Bitget、Gate、MEXC、Gemini、Bitfinex、HTX、BitMart、Phemex、Bitstamp、Polymarket 和 predict.fun;只换值'
- id: crypto-keystore
kind: in-context
on_by_default: true
name: Wallet keystore / ciphertext
name_zh: 钱包密钥库/密文
about: 'Literal `keystore` or `ciphertext` fields, including Ethereum V3 encrypted key material; pasted JSON objects are protected as text and only the value is replaced'
about_zh: '字面量 `keystore` 或 `ciphertext` 字段,包括 Ethereum V3 加密密钥材料;作为文本粘贴的 JSON 对象也保护,只换值'
- id: crypto-rpc-key
kind: in-context
on_by_default: true
name: Blockchain RPC URL credential
name_zh: 区块链 RPC 地址凭证
about: 'API credentials in Infura `/v3/`, Alchemy `/v2/`, QuickNode token paths and Helius `api-key` URLs; only the credential part is replaced'
about_zh: 'Infura `/v3/`、Alchemy `/v2/`、QuickNode 令牌路径及 Helius `api-key` URL 中的 API 凭据;只换凭据部分'

# ---- 扩充的个人信息规则(判据在 src/redact/personal/)
- id: cn-resident-id-15
Expand Down
4 changes: 3 additions & 1 deletion crates/tw-guard/src/redact/engine.rs
Original file line number Diff line number Diff line change
Expand Up @@ -748,14 +748,15 @@ pub(super) fn scan_tokens(
let mut personal = super::personal::Want::new(set);
let on = enabled(set, &secrets, &personal);
if on == 0 {
super::secrets::blockchain::scan(text, set, json, None, out);
personal.text(text, json, None, out);
return None;
}
let pre = prescan(
text,
json,
on,
json && (personal.labeled_on() || set.has_custom()),
json && (personal.labeled_on() || set.has_custom() || super::secrets::blockchain::on(set)),
);
let spans = (json && !pre.quotes.is_empty()).then(|| pre.string_spans(text.len()));
// 整段文本过一遍的(中文助记词;护照机读区、住址……)在按 token 认的前面,和老写法一样
Expand Down Expand Up @@ -837,6 +838,7 @@ pub(super) fn scan_tokens(
secrets.word_run(text, r.clone(), out);
}
spaced.finish(text, out);
super::secrets::blockchain::scan(text, set, json, spans.as_deref(), out);
secrets.finish(text, set, out);
personal.finish(text, out);
spans
Expand Down
33 changes: 33 additions & 0 deletions crates/tw-guard/src/redact/flow.rs
Original file line number Diff line number Diff line change
Expand Up @@ -621,6 +621,39 @@ mod tests {
assert!(ledger.is_empty());
}

#[test]
fn uuid_redaction_observes_or_replaces_written_ids_and_restores_them() {
let value = "7b1e2d60-6f9a-4dd7-8f3e-21b8f94c77a2";
let input = serde_json::json!({
"id": value,
"model": value,
"messages": [{"role": "user", "content": format!("order {value}; key {value}")}]
})
.to_string();
let rules = RuleSet::defaults();
for mode in [Mode::Observe, Mode::Enforce] {
let (found, ledger) = look(mode, &rules, input.as_bytes(), S);
assert_eq!(found.len(), 1);
assert_eq!(found[0].rule.id(), "uuid");
let (out, ledger) = replace(mode, &rules, input.clone().into(), &ledger, S);
let out = String::from_utf8(out.to_vec()).unwrap();
if mode == Mode::Observe {
assert_eq!(out, input);
assert!(ledger.is_empty());
} else {
let parsed: serde_json::Value = serde_json::from_str(&out).unwrap();
assert_eq!(parsed["id"], value);
assert_eq!(parsed["model"], value);
assert_eq!(
parsed["messages"][0]["content"],
"order <<TW_SECRET_1>>; key <<TW_SECRET_1>>"
);
assert_eq!(ledger.len(), 1);
assert_eq!(crate::redact::replace::restore(&out, &ledger), input);
}
}
}

#[test]
fn off_does_not_even_look() {
assert!(find(Mode::Off, &RuleSet::defaults(), &body(), S).is_empty());
Expand Down
32 changes: 27 additions & 5 deletions crates/tw-guard/src/redact/rules.rs
Original file line number Diff line number Diff line change
Expand Up @@ -335,8 +335,7 @@ pub const CARD_NETWORKS: &[CardNetwork] = &[
/// 全部内置规则,从 `data/redact.yaml` 编译期生成(见 build.rs)。界面上先按 [`Kind`] 的
/// 顺序分组,组内按这里的顺序(见 `view::redact`);前缀表也按这里的顺序比。
///
/// API 密钥那一段**只收前缀明确、长度有下限的**。代码里的 hash、base64 的
/// 图片、UUID 全都长得像「一长串随机字符」,按那个判据匹配会疯狂误报。
/// API 密钥按前缀、结构或上下文识别;UUID、链上地址和裸十六进制另作隐私规则。
pub const BUILTINS: &[Builtin] = include!(concat!(env!("OUT_DIR"), "/redact_builtins.rs"));

/// 一个分组叫什么(`data/redact.yaml` 的 `kinds`)。
Expand Down Expand Up @@ -410,6 +409,17 @@ const DETECTORS: &[(&str, Matcher)] = &[
("crypto-mnemonic", Matcher::Code),
("crypto-wif-key", Matcher::Code),
("crypto-extended-key", Matcher::Code),
("uuid", Matcher::Code),
("crypto-hex-key", Matcher::Code),
("crypto-solana-key", Matcher::Code),
("crypto-sui-key", Matcher::Code),
("crypto-stellar-key", Matcher::Code),
("crypto-xrp-seed", Matcher::Code),
("crypto-address", Matcher::Code),
("crypto-secret-field", Matcher::Code),
("exchange-credential", Matcher::Code),
("crypto-keystore", Matcher::Code),
("crypto-rpc-key", Matcher::Code),
// 扩充的个人信息规则(personal/)
("cn-resident-id-15", Matcher::Code),
("passport-mrz", Matcher::Code),
Expand Down Expand Up @@ -2398,6 +2408,16 @@ pub fn masked(rule: &Rule, value: &str) -> String {
{
return mask_email(value);
}
if matches!(
rule.id(),
"uuid"
| "crypto-secret-field"
| "exchange-credential"
| "crypto-keystore"
| "crypto-rpc-key"
) {
return "…".to_string();
}
match rule.kind() {
Kind::Internal => value.to_string(),
k if k.is_personal() => last_four(value),
Expand All @@ -2406,7 +2426,7 @@ pub fn masked(rule: &Rule, value: &str) -> String {
// 按位置认的多半是口令,没有前缀:留头五个就是留了大半个口令。只留最后四位,短的全打掉
Kind::InContext => masked_unknown(value),
// 助记词的头几个字母就是头一个词
Kind::Crypto if rule.id() == "crypto-mnemonic" => "…".to_string(),
Kind::Crypto => "…".to_string(),
_ => mask(value),
}
}
Expand Down Expand Up @@ -2658,8 +2678,6 @@ mod tests {
// **宁可漏,不可吵。**一个天天误报的安全功能,用户第二天就关了。
for t in [
"把 sk-test 换成你自己的 key",
"const hash = 'a3f5c9e1b7d2f4a6c8e0b2d4f6a8c0e2';",
"UUID 是 550e8400-e29b-41d4-a716-446655440000",
"版本 1.2.3,路径 a.b.c",
"data:image/png;base64,iVBORw0KGgoAAAANSUhEUg==",
// 全字母或全数字的一长串是 hash / id,不是 key
Expand Down Expand Up @@ -3154,6 +3172,10 @@ mod tests {
}
Matcher::Code if b.kind.is_personal() => {
let text = super::super::personal::tests::sample(b.id)
.or_else(|| {
super::super::secrets::tests::samples::sample(b.id)
.map(|s| s.hits[0].0.clone())
})
.unwrap_or_else(|| panic!("{} 没有样本", b.id));
let hits = scan_plain(&text, &only);
assert!(!hits.is_empty(), "{} 认不出自己的样本:{text}", b.id);
Expand Down
1 change: 1 addition & 0 deletions crates/tw-guard/src/redact/rules/oracle.rs
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,7 @@ pub(crate) fn scan_unresolved(text: &str, set: &RuleSet) -> Vec<Hit> {
}
});
spaced.finish(text, &mut out);
super::super::secrets::blockchain::scan(text, set, json, None, &mut out);
secrets.finish(text, set, &mut out);
personal.finish(text, &mut out);
custom_hits(text, set, json, &mut out);
Expand Down
Loading
Loading