| @@ -24,10 +24,12 @@ runs: | ||
| steps: | ||
| - name: Generate prompt tree | ||
| shell: bash | ||
| + | env: | |
| + | TT_SOURCE: ${{ inputs.source }} | |
| run: | | ||
| set -euo pipefail | ||
| - | if [ -n "${{ inputs.source }}" ]; then | |
| - | npx --yes treetrace --file "${{ inputs.source }}" --redact-auto --quiet | |
| + | if [ -n "$TT_SOURCE" ]; then | |
| + | npx --yes treetrace --file "$TT_SOURCE" --redact-auto --quiet | |
| elif [ -f .treetrace/tree.json ]; then | ||
| echo "::notice::Using committed .treetrace/tree.json" | ||
| node -e " | ||
| @@ -43,6 +45,7 @@ runs: | ||
| shell: bash | ||
| env: | ||
| GH_TOKEN: ${{ github.token }} | ||
| + | PR_NUMBER: ${{ github.event.pull_request.number }} | |
| run: | | ||
| set -euo pipefail | ||
| if [ -f TREETRACE_REPORT.md ]; then | ||
| @@ -53,7 +56,7 @@ runs: | ||
| echo "" | ||
| echo "_Full report: TREETRACE_REPORT.md_" | ||
| } > /tmp/tt-comment.md | ||
| - | gh pr comment "${{ github.event.pull_request.number }}" --body-file /tmp/tt-comment.md | |
| + | gh pr comment "$PR_NUMBER" --body-file /tmp/tt-comment.md | |
| elif [ -f PROMPT_TREE.md ]; then | ||
| { | ||
| echo "### Prompt tree" | ||
| @@ -62,5 +65,5 @@ runs: | ||
| echo "" | ||
| echo "_Full lineage: PROMPT_TREE.md_" | ||
| } > /tmp/tt-comment.md | ||
| - | gh pr comment "${{ github.event.pull_request.number }}" --body-file /tmp/tt-comment.md | |
| + | gh pr comment "$PR_NUMBER" --body-file /tmp/tt-comment.md | |
| fi |
| @@ -106,9 +106,14 @@ export async function main(argv) { | ||
| const ttDir = join(projectDir, '.treetrace'); | ||
| const decisionsPath = join(ttDir, 'redactions.json'); | ||
| - | const priorDecisions = existsSync(decisionsPath) | |
| - | ? JSON.parse(readFileSync(decisionsPath, 'utf8')) | |
| - | : {}; | |
| + | let priorDecisions = {}; | |
| + | if (existsSync(decisionsPath)) { | |
| + | try { | |
| + | priorDecisions = JSON.parse(readFileSync(decisionsPath, 'utf8')); | |
| + | } catch { | |
| + | priorDecisions = {}; | |
| + | } | |
| + | } | |
| const findings = []; | ||
| for (const node of tree.nodes) findings.push(...scanText(node.text)); |
| @@ -22,7 +22,7 @@ export const RULES = [ | ||
| { id: 'jwt', severity: 'high', re: /\beyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{5,}\b/g }, | ||
| { id: 'wireguard-key', severity: 'medium', re: /\b(PrivateKey|PresharedKey)\s*=\s*[A-Za-z0-9+/]{42,44}=?/g }, | ||
| - | { id: 'url-basic-auth', severity: 'medium', re: /[a-z][a-z0-9+.-]*:\/\/[^/\s:@'"`]{2,}:[^/\s@'"`]{2,}@[^\s'"`]+/gi }, | |
| + | { id: 'url-basic-auth', severity: 'medium', re: /\b[a-z][a-z0-9+.-]{0,30}:\/\/[^/\s:@'"`]{2,256}:[^/\s@'"`]{2,256}@[^\s'"`]{1,512}/gi }, | |
| { id: 'bearer-header', severity: 'medium', re: /\bBearer\s+[A-Za-z0-9._+/=-]{20,}\b/g }, | ||
| { id: 'secret-assignment', severity: 'medium', re: /\b(password|passwd|pwd|secret|api[_-]?key|access[_-]?token|auth[_-]?token|client[_-]?secret)\b\s*[:=]\s*(?!(?:['"]?\s*)?(?:\$\{|<|%|\*{3}|\.{3}|REDACTED|xxx+|placeholder|changeme|example|your[-_]))(?:"[^"\r\n]{8,}"|'[^'\r\n]{8,}'|[^\s'"`,;]{8,})/gi }, | ||
| @@ -53,6 +53,15 @@ const JOINED_SCAN_RULE_IDS = new Set([ | ||
| 'jwt', | ||
| ]); | ||
| + | const LOOSE_RULES = RULES.filter((r) => JOINED_SCAN_RULE_IDS.has(r.id)).map((r) => ({ | |
| + | id: r.id, | |
| + | severity: r.severity, | |
| + | re: new RegExp( | |
| + | r.re.source.replace(/^\\b/, '').replace(/\\b$/, '').replace(/\{(\d+),\}/g, '{$1,128}'), | |
| + | 'g' | |
| + | ), | |
| + | })); | |
| + | ||
| export function scanText(text) { | ||
| const findings = []; | ||
| for (const rule of RULES) { | ||
| @@ -99,18 +108,18 @@ function scanJoinedProviderTokens(text, existing) { | ||
| const joined = chars.join(''); | ||
| const existingSpans = existing.map((f) => [f.index, f.index + f.match.length]); | ||
| const findings = []; | ||
| - | for (const rule of RULES) { | |
| - | if (!JOINED_SCAN_RULE_IDS.has(rule.id)) continue; | |
| + | for (const rule of LOOSE_RULES) { | |
| rule.re.lastIndex = 0; | ||
| let m; | ||
| while ((m = rule.re.exec(joined)) !== null) { | ||
| - | const start = indexMap[m.index]; | |
| - | const end = indexMap[m.index + m[0].length - 1] + 1; | |
| - | const original = text.slice(start, end); | |
| - | if (!JOIN_SEPARATOR_RE.test(original)) continue; | |
| - | if (original.length - m[0].length > 20) continue; | |
| - | if (existingSpans.some(([s, e]) => start >= s && start < e)) continue; | |
| - | findings.push({ ruleId: rule.id, severity: rule.severity, match: original, index: start }); | |
| + | if (m[0].length <= 256) { | |
| + | const start = indexMap[m.index]; | |
| + | const end = indexMap[m.index + m[0].length - 1] + 1; | |
| + | const original = text.slice(start, end); | |
| + | if (JOIN_SEPARATOR_RE.test(original) && !existingSpans.some(([s, e]) => start >= s && start < e)) { | |
| + | findings.push({ ruleId: rule.id, severity: rule.severity, match: original, index: start }); | |
| + | } | |
| + | } | |
| if (m.index === rule.re.lastIndex) rule.re.lastIndex++; | ||
| } | ||
| } |
| @@ -132,6 +132,25 @@ test('redaction: split provider tokens are caught before shadow scan', () => { | ||
| assert.ok(!masked.includes('sk-proj-')); | ||
| }); | ||
| + | test('redaction: whitespace-split secret below the length floor is caught', () => { | |
| + | const dirty = 'store key sk-ant-api03-AAAA BBBBCCCCDDDDEEEEFFFFGGGG into the vault'; | |
| + | const findings = scanText(dirty); | |
| + | const hit = findings.find((f) => f.ruleId === 'anthropic-key'); | |
| + | assert.ok(hit, `split anthropic-key missed: ${JSON.stringify(findings)}`); | |
| + | const masked = applyDecisions(dirty, findings, { | |
| + | [sha256(hit.match)]: { action: 'redact', replacement: '[REDACTED:anthropic-key]', ruleId: 'anthropic-key' }, | |
| + | }); | |
| + | assert.ok(!/sk-ant-api03-AAAA/.test(masked), `secret not redacted: ${masked}`); | |
| + | assert.equal(shadowScan(masked, {}).length, 0); | |
| + | }); | |
| + | ||
| + | test('redaction: scan stays fast on long benign input (ReDoS guard)', () => { | |
| + | const big = 'http://' + 'a'.repeat(60000); | |
| + | const start = Date.now(); | |
| + | scanText(big); | |
| + | assert.ok(Date.now() - start < 2000, 'scan should stay linear on long input'); | |
| + | }); | |
| + | ||
| test('redaction: benign text produces no high/medium findings', () => { | ||
| const benign = | ||
| 'Refactor the parser in src/parse.js to handle commit 3f2a1b9c8d7e6f5a4b3c2d1e0f9a8b7c6d5e4f3a and bump to v2.1.0-beta.3. The README.md needs a section on CONTRIBUTING.'; |