● Zion Boggan
repos/TreeTrace/commits/d3fd45d
zionboggan.com ↗
Code Commits Tags

Expand test suite to 15 cases and update examples index for analysis artifacts

d3fd45d   Zion Boggan committed on Jun 12, 2026 (1 week ago)
examples/README.md +18 -5
@@ -1,17 +1,30 @@
# Examples
-Real `treetrace` output, so you can see the artifact before you run it.
+Generated TreeTrace outputs from the synthetic weather-dashboard fixture.
-## [weather-dashboard/](weather-dashboard/)
+## Weather Dashboard
-A short session that exercises every feature: a **root** goal, a **direction**, a **correction** ("scrap the radar map"), and a **scope change** ("also add a settings panel") - plus the redaction gate masking a planted Anthropic key and a basic-auth URL, and a **reusable prompt pack** that folds the correction in as a learned constraint.
+- [weather-dashboard/PROMPT_TREE.md](weather-dashboard/PROMPT_TREE.md) - human-readable lineage
+- [weather-dashboard/TREETRACE_REPORT.md](weather-dashboard/TREETRACE_REPORT.md) - combined human-readable report
+- [weather-dashboard/tree.json](weather-dashboard/tree.json) - canonical v0.2 machine-readable lineage
+- [weather-dashboard/.treetrace/failures.json](weather-dashboard/.treetrace/failures.json) - failure signals and correction chains
+- [weather-dashboard/.treetrace/lessons.md](weather-dashboard/.treetrace/lessons.md) - lessons for future agents
+- [weather-dashboard/.treetrace/evals.jsonl](weather-dashboard/.treetrace/evals.jsonl) - eval candidates
+- [weather-dashboard/.treetrace/agent-memory.md](weather-dashboard/.treetrace/agent-memory.md) - compact memory pack
+
+The root-level example files mirror the same analysis artifacts for quick inspection:
+
+- [failures.json](failures.json)
+- [lessons.md](lessons.md)
+- [evals.jsonl](evals.jsonl)
+- [agent-memory.md](agent-memory.md)
Generated with:
```bash
-treetrace --file session.jsonl --redact-auto
+node bin/treetrace.js --file test/fixtures/synthetic-session.jsonl --dir examples/weather-dashboard --redact-auto --quiet
```
## Dogfooding
-treetrace ships its own [`PROMPT_TREE.md`](../PROMPT_TREE.md) at the repo root - the prompt tree of the tool that makes prompt trees, regenerated from its own build sessions. That's the standing invitation: if you build something with an agent, commit the tree next to the code.
+TreeTrace ships its own [PROMPT_TREE.md](../PROMPT_TREE.md), but the pivot makes that Markdown tree one artifact among several. The structured outputs are the main product: lineage JSON, failure analysis, eval candidates, and agent memory.
test/treetrace.test.js +107 -1
@@ -1,5 +1,7 @@
import { test } from 'node:test';
import assert from 'node:assert/strict';
+import { existsSync, mkdtempSync, readFileSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
import { fileURLToPath } from 'node:url';
import { dirname, join } from 'node:path';
@@ -10,6 +12,15 @@ import { scanText, applyDecisions, shadowScan, maskFor, resolveFindings } from '
import { renderMarkdown, promptPack } from '../src/render-md.js';
import { renderJson } from '../src/render-json.js';
import { renderHandoff } from '../src/handoff.js';
+import { renderReportMarkdown } from '../src/report.js';
+import {
+ analyzeTree,
+ renderFailuresJson,
+ renderLessonsMarkdown,
+ renderEvalsJsonl,
+ renderMemoryMarkdown,
+} from '../src/analyze.js';
+import { main } from '../src/cli.js';
import { mungePath } from '../src/discover.js';
import { sha256 } from '../src/util.js';
@@ -97,6 +108,8 @@ test('redaction: rule coverage on known formats', () => {
['-----BEGIN OPENSSH PRIVATE KEY-----\nb3BlbnNzaA==\n-----END OPENSSH PRIVATE KEY-----', 'private-key-block'],
['eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0NTY3ODkwIn0.dozjgNryP4J3jVmNHl0w5N_XgL0n3I9PlFUP0THsR8U', 'jwt'],
['password = "correct-horse-battery"', 'secret-assignment'],
+ ['SECRET="correct horse battery staple"', 'secret-assignment'],
+ ['https://user:p:a:ss@example.com/path', 'url-basic-auth'],
];
for (const [sample, expected] of cases) {
const hits = scanText(`some text ${sample} more text`).map((f) => f.ruleId);
@@ -104,6 +117,21 @@ test('redaction: rule coverage on known formats', () => {
}
});
+test('redaction: split provider tokens are caught before shadow scan', () => {
+ const dirty = 'token sk-proj-abcdefghijklmnop\nqrstu1234567890ABCDE end';
+ const findings = scanText(dirty);
+ assert.ok(findings.some((f) => f.ruleId === 'openai-key'), `openai-key missed in ${findings}`);
+ const masked = applyDecisions(dirty, findings, {
+ [sha256(findings.find((f) => f.ruleId === 'openai-key').match)]: {
+ action: 'redact',
+ replacement: '[REDACTED:openai-key]',
+ ruleId: 'openai-key',
+ },
+ });
+ assert.equal(shadowScan(masked, {}).length, 0);
+ assert.ok(!masked.includes('sk-proj-'));
+});
+
test('redaction: benign text produces no high/medium findings', () => {
const benign =
'Refactor the parser in src/parse.js to handle commit 3f2a1b9c8d7e6f5a4b3c2d1e0f9a8b7c6d5e4f3a and bump to v2.1.0-beta.3. The README.md needs a section on CONTRIBUTING.';
@@ -113,6 +141,7 @@ test('redaction: benign text produces no high/medium findings', () => {
test('renderers: markdown, json, handoff are consistent and footer-credited', async () => {
const { tree } = await fixtureTree();
+ analyzeTree(tree);
const md = renderMarkdown(tree, { projectName: 'demo' });
assert.ok(md.startsWith('# 🌳 Prompt Tree - demo'));
assert.ok(md.includes('## Goal'));
@@ -120,10 +149,13 @@ test('renderers: markdown, json, handoff are consistent and footer-credited', as
assert.ok(md.includes('generated by [treetrace]') || md.includes('Generated by [treetrace]'));
const json = renderJson(tree, { projectName: 'demo' });
- assert.equal(json.schemaVersion, '0.1');
+ assert.equal(json.schemaVersion, '0.2');
assert.equal(json.nodes.length, tree.nodes.length);
assert.equal(json.edges.length, tree.nodes.filter((n) => n.parent).length);
assert.ok(json.nodes.every((n) => n.id && n.kind && typeof n.text === 'string'));
+ assert.ok(json.analysis.failureSignals >= 1);
+ assert.ok(json.correctionChains.length >= 1);
+ assert.ok(json.nodes.some((n) => Array.isArray(n.failureSignals)));
const pack = promptPack(tree.nodes);
assert.ok(pack.includes('1.'));
@@ -131,6 +163,80 @@ test('renderers: markdown, json, handoff are consistent and footer-credited', as
const handoff = renderHandoff(tree, { projectName: 'demo' });
assert.ok(handoff.includes('## Original goal'));
assert.ok(handoff.includes('Constraints learned the hard way'));
+ assert.ok(handoff.includes('Agent memory lessons'));
+
+ const report = renderReportMarkdown(tree, { projectName: 'demo', generatedAt: '2026-01-01T00:00:00.000Z' });
+ assert.ok(report.startsWith('# TreeTrace Report - demo'));
+ assert.ok(report.includes('## Output map'));
+ assert.ok(report.includes('## Handoff brief'));
+ assert.ok(report.includes('TREETRACE_REPORT.md'));
+});
+
+test('analysis renderers produce failures, lessons, evals, and memory', async () => {
+ const { tree } = await fixtureTree();
+ const failures = renderFailuresJson(tree, { projectName: 'demo', generatedAt: '2026-01-01T00:00:00.000Z' });
+ assert.equal(failures.schemaVersion, '0.2');
+ assert.ok(failures.failures.length >= 1);
+ assert.ok(failures.correctionChains.length >= 1);
+
+ const lessons = renderLessonsMarkdown(tree, { projectName: 'demo' });
+ assert.ok(lessons.includes('# TreeTrace Lessons'));
+ assert.ok(lessons.includes('Source nodes:'));
+
+ const evals = renderEvalsJsonl(tree).trim().split('\n').map((line) => JSON.parse(line));
+ assert.ok(evals.length >= 1);
+ assert.ok(evals.every((e) => e.source === 'treetrace' && e.sourceNodeIds.length >= 1));
+
+ const memory = renderMemoryMarkdown(tree, { projectName: 'demo' });
+ assert.ok(memory.includes('TreeTrace Agent Memory'));
+ assert.ok(memory.includes('Durable project constraints'));
+});
+
+test('analysis: tiny transcript without corrections does not invent failures', () => {
+ const session = parsePlainTranscript('User: build a tiny CLI\nAssistant: done', 'tiny');
+ const nodes = classifyPrompts([session]);
+ const tree = buildTree([session], nodes);
+ const analysis = analyzeTree(tree);
+ assert.equal(analysis.summary.totalFailureSignals, 0);
+ assert.deepEqual(analysis.failures, []);
+});
+
+test('cli: default run writes analysis artifacts with redaction', async () => {
+ const dir = mkdtempSync(join(tmpdir(), 'treetrace-'));
+ try {
+ await main(['--file', FIXTURE, '--dir', dir, '--redact-auto', '--quiet']);
+ for (const file of [
+ 'TREETRACE_REPORT.md',
+ 'PROMPT_TREE.md',
+ '.treetrace/tree.json',
+ '.treetrace/failures.json',
+ '.treetrace/lessons.md',
+ '.treetrace/evals.jsonl',
+ '.treetrace/agent-memory.md',
+ ]) {
+ assert.ok(existsSync(join(dir, file)), `${file} missing`);
+ }
+ const failures = JSON.parse(readFileSync(join(dir, '.treetrace/failures.json'), 'utf8'));
+ assert.equal(failures.schemaVersion, '0.2');
+ assert.ok(failures.failures.length >= 1);
+
+ const evalLine = readFileSync(join(dir, '.treetrace/evals.jsonl'), 'utf8').trim().split('\n')[0];
+ assert.equal(JSON.parse(evalLine).source, 'treetrace');
+
+ const exported = [
+ 'PROMPT_TREE.md',
+ 'TREETRACE_REPORT.md',
+ '.treetrace/tree.json',
+ '.treetrace/failures.json',
+ '.treetrace/lessons.md',
+ '.treetrace/evals.jsonl',
+ '.treetrace/agent-memory.md',
+ ].map((file) => readFileSync(join(dir, file), 'utf8')).join('\n');
+ assert.ok(!exported.includes('sk-ant-'), 'anthropic key leaked');
+ assert.ok(!exported.includes('hunter2pass'), 'basic-auth password leaked');
+ } finally {
+ rmSync(dir, { recursive: true, force: true });
+ }
});
test('plain transcript fallback parses User:/Assistant: markers', () => {