● Zion Boggan
repos/Oversight/commits/f873b9c
zionboggan.com ↗
Code Commits Tags

Fail closed on Rust registry tlog gaps

Co-authored-by: Codex (GPT-5.4) <noreply@openai.com>
f873b9c   Zion Boggan committed on May 23, 2026 (1 month ago)
CHANGELOG.md +4 -1
@@ -29,7 +29,10 @@
identity mismatches, malformed manifest JSON, invalid manifest signatures,
and manifest/file ID divergence before operators declare migration burn-in
complete. It also validates event/corpus JSON sidecars and tlog index
- uniqueness so corrupted migrated evidence cannot look clean.
+ uniqueness so corrupted migrated evidence cannot look clean. Rust registry
+ writes now fail closed if the local transparency log cannot append, and
+ validation checks missing or out-of-range event tlog indexes against the
+ on-disk tlog size.
- **Rust policy test parity.** Fixed the `oversight-policy` crate's manifest
fixture after the v0.4.11 `Recipient.p256_pub` schema addition so the full
Rust workspace test suite compiles again.
README.md +2 -0
@@ -97,6 +97,8 @@ migration tooling (`--migrate-from`, `--migrate-dry-run`) and a native
`--validate-db` integrity report so operators can preflight, copy, and verify
attribution rows, event metadata, corpus metadata, and tlog indexes without
treating the Python reference as a permanent production dependency.
+Rust registry writes now fail closed if the local transparency log cannot
+append, so new evidence rows cannot silently lose their audit trail.
The next Rust-registry gate is operational burn-in: longer-running deployment
tests against real operator databases and a final wire-format stability
docs/REGISTRY_DEPLOYMENT.md +2 -1
@@ -135,7 +135,8 @@ oversight-registry \
The validation command prints JSON counts plus integrity failures for orphaned
beacons, watermarks, events, corpus rows, identity mismatches, malformed
event `extra` JSON, malformed corpus metadata JSON, duplicate or negative
-tlog indexes, malformed manifest JSON, invalid manifest signatures, and
+tlog indexes, missing event tlog indexes, event tlog indexes outside the
+on-disk tlog size, malformed manifest JSON, invalid manifest signatures, and
manifest/file ID divergence. Keep the Python database as a rollback artifact
until validation, live conformance, and evidence-bundle checks pass against
the Rust service.
docs/ROADMAP.md +2 -0
@@ -244,6 +244,8 @@ preflight. As of 2026-05-20, `--validate-db` checks the copied Rust database
for orphan rows, identity mismatches, malformed manifest JSON, invalid
manifest signatures, and manifest/file ID divergence. As of 2026-05-21, that
validation also covers event/corpus JSON sidecars and tlog index uniqueness.
+As of 2026-05-22, registry writes fail closed when tlog append fails and
+`--validate-db` compares event tlog indexes against the on-disk tlog size.
Remaining work: longer-running deployment tests and a wire-format stability
declaration before declaring v1.0 ready.
oversight-rust/oversight-registry/src/db.rs +50 -3
@@ -42,6 +42,9 @@ pub struct RegistryIntegrityReport {
pub malformed_corpus_metadata_json: i64,
pub duplicate_event_tlog_indexes: i64,
pub negative_event_tlog_indexes: i64,
+ pub events_without_tlog_index: i64,
+ pub event_tlog_indexes_out_of_range: i64,
+ pub tlog_size: Option<usize>,
pub malformed_manifest_json: i64,
pub invalid_manifest_signatures: i64,
pub mismatched_manifest_file_ids: i64,
@@ -227,7 +230,10 @@ pub async fn migrate_from_sqlite(
result
}
-pub async fn validate_registry_integrity(pool: &SqlitePool) -> Result<RegistryIntegrityReport> {
+pub async fn validate_registry_integrity(
+ pool: &SqlitePool,
+ tlog_size: Option<usize>,
+) -> Result<RegistryIntegrityReport> {
let counts = registry_counts(pool).await?;
let orphan_beacons = count_query(
pool,
@@ -274,6 +280,20 @@ pub async fn validate_registry_integrity(pool: &SqlitePool) -> Result<RegistryIn
"SELECT COUNT(*) FROM events WHERE tlog_index IS NOT NULL AND tlog_index < 0",
)
.await?;
+ let events_without_tlog_index =
+ count_query(pool, "SELECT COUNT(*) FROM events WHERE tlog_index IS NULL").await?;
+ let event_tlog_indexes_out_of_range = match tlog_size {
+ Some(size) => {
+ let (count,): (i64,) = sqlx::query_as(
+ "SELECT COUNT(*) FROM events WHERE tlog_index IS NOT NULL AND tlog_index >= ?",
+ )
+ .bind(size as i64)
+ .fetch_one(pool)
+ .await?;
+ count
+ }
+ None => 0,
+ };
let event_extra_rows: Vec<String> = sqlx::query_scalar(
"SELECT extra FROM events WHERE extra IS NOT NULL AND TRIM(extra) != ''",
@@ -330,6 +350,8 @@ pub async fn validate_registry_integrity(pool: &SqlitePool) -> Result<RegistryIn
&& malformed_corpus_metadata_json == 0
&& duplicate_event_tlog_indexes == 0
&& negative_event_tlog_indexes == 0
+ && events_without_tlog_index == 0
+ && event_tlog_indexes_out_of_range == 0
&& malformed_manifest_json == 0
&& invalid_manifest_signatures == 0
&& mismatched_manifest_file_ids == 0;
@@ -348,6 +370,9 @@ pub async fn validate_registry_integrity(pool: &SqlitePool) -> Result<RegistryIn
malformed_corpus_metadata_json,
duplicate_event_tlog_indexes,
negative_event_tlog_indexes,
+ events_without_tlog_index,
+ event_tlog_indexes_out_of_range,
+ tlog_size,
malformed_manifest_json,
invalid_manifest_signatures,
mismatched_manifest_file_ids,
@@ -941,7 +966,7 @@ mod tests {
run_migrations(&pool).await.unwrap();
seed_source(&pool).await;
- let report = validate_registry_integrity(&pool).await.unwrap();
+ let report = validate_registry_integrity(&pool, None).await.unwrap();
assert!(report.ok);
assert_eq!(report.counts.manifests, 1);
assert_eq!(report.counts.beacons, 1);
@@ -951,6 +976,9 @@ mod tests {
assert_eq!(report.malformed_corpus_metadata_json, 0);
assert_eq!(report.duplicate_event_tlog_indexes, 0);
assert_eq!(report.negative_event_tlog_indexes, 0);
+ assert_eq!(report.events_without_tlog_index, 0);
+ assert_eq!(report.event_tlog_indexes_out_of_range, 0);
+ assert_eq!(report.tlog_size, None);
pool.close().await;
let _ = std::fs::remove_dir_all(dir);
@@ -1015,6 +1043,22 @@ mod tests {
)
.await
.unwrap();
+ insert_event(
+ &pool,
+ "token-no-tlog",
+ Some("file-1"),
+ Some("recipient-1"),
+ Some("issuer-1"),
+ "dns",
+ None,
+ None,
+ Some(r#"{"ok":true}"#),
+ 23,
+ None,
+ None,
+ )
+ .await
+ .unwrap();
sqlx::query(
"INSERT INTO corpus (file_id, hash_kind, hash_value, metadata, registered_at) VALUES (?, ?, ?, ?, ?)",
)
@@ -1027,7 +1071,7 @@ mod tests {
.await
.unwrap();
- let report = validate_registry_integrity(&pool).await.unwrap();
+ let report = validate_registry_integrity(&pool, Some(1)).await.unwrap();
assert!(!report.ok);
assert_eq!(report.orphan_beacons, 1);
assert_eq!(report.orphan_watermarks, 1);
@@ -1038,6 +1082,9 @@ mod tests {
assert_eq!(report.malformed_corpus_metadata_json, 1);
assert_eq!(report.duplicate_event_tlog_indexes, 1);
assert_eq!(report.negative_event_tlog_indexes, 1);
+ assert_eq!(report.events_without_tlog_index, 1);
+ assert_eq!(report.event_tlog_indexes_out_of_range, 2);
+ assert_eq!(report.tlog_size, Some(1));
pool.close().await;
let _ = std::fs::remove_dir_all(dir);
oversight-rust/oversight-registry/src/main.rs +3 -1
@@ -385,7 +385,9 @@ async fn main() -> anyhow::Result<()> {
}
if args.validate_db {
- let report = db::validate_registry_integrity(&pool)
+ let tlog = TransparencyLog::open(data_dir.join("tlog"))
+ .map_err(|e| anyhow::anyhow!("tlog validation init: {e}"))?;
+ let report = db::validate_registry_integrity(&pool, Some(tlog.size()))
.await
.map_err(|e| anyhow::anyhow!("registry integrity validation failed: {e}"))?;
println!("{}", serde_json::to_string_pretty(&report)?);
oversight-rust/oversight-registry/src/routes/beacon.rs +1 -1
@@ -89,7 +89,7 @@ async fn record_event(
.tlog
.append_event(&tlog_event)
.map(|idx| idx as i64)
- .unwrap_or(-1);
+ .map_err(|e| RegistryError::Internal(format!("tlog append failed: {e}")))?;
let now = SystemTime::now()
.duration_since(UNIX_EPOCH)
oversight-rust/oversight-registry/src/routes/dns_event.rs +1 -1
@@ -53,7 +53,7 @@ pub async fn dns_event(
.tlog
.append_event(&tlog_event)
.map(|idx| idx as i64)
- .unwrap_or(-1);
+ .map_err(|e| RegistryError::Internal(format!("tlog append failed: {e}")))?;
let now = SystemTime::now()
.duration_since(UNIX_EPOCH)
oversight-rust/oversight-registry/src/routes/register.rs +52 -39
@@ -98,17 +98,7 @@ pub async fn register(
.unwrap_or_default()
.as_secs() as i64;
- db::upsert_manifest(
- &state.db,
- file_id,
- recipient_id,
- issuer_id,
- &issuer_pub,
- &manifest_json,
- now,
- )
- .await?;
-
+ let mut beacon_rows = Vec::with_capacity(signed_beacons.len());
for beacon in &signed_beacons {
let token_id = beacon
.get("token_id")
@@ -128,18 +118,10 @@ pub async fn register(
"signed beacon has invalid kind".into(),
));
}
- db::upsert_beacon(
- &state.db,
- token_id,
- file_id,
- recipient_id,
- issuer_id,
- kind,
- now,
- )
- .await?;
+ beacon_rows.push((token_id, kind));
}
+ let mut watermark_rows = Vec::with_capacity(signed_watermarks.len());
for watermark in &signed_watermarks {
let mark_id = watermark
.get("mark_id")
@@ -159,16 +141,7 @@ pub async fn register(
"signed watermark has invalid layer".into(),
));
}
- db::upsert_watermark(
- &state.db,
- mark_id,
- layer,
- file_id,
- recipient_id,
- issuer_id,
- now,
- )
- .await?;
+ watermark_rows.push((mark_id, layer));
}
if let Some(ref corpus) = req.corpus {
@@ -178,13 +151,6 @@ pub async fn register(
MAX_CORPUS_ENTRIES
)));
}
- for (hash_kind, hash_value) in corpus {
- if let Some(hv) = hash_value.as_str() {
- if !hv.is_empty() && hash_kind.len() <= MAX_ID_LEN && hv.len() <= MAX_ID_LEN {
- db::upsert_corpus(&state.db, file_id, hash_kind, hv, now).await?;
- }
- }
- }
}
let timestamp_str = crate::timestamp_stub();
@@ -202,7 +168,54 @@ pub async fn register(
.tlog
.append_event(&tlog_event)
.map(|idx| idx as i64)
- .unwrap_or(-1);
+ .map_err(|e| RegistryError::Internal(format!("tlog append failed: {e}")))?;
+
+ db::upsert_manifest(
+ &state.db,
+ file_id,
+ recipient_id,
+ issuer_id,
+ &issuer_pub,
+ &manifest_json,
+ now,
+ )
+ .await?;
+
+ for (token_id, kind) in beacon_rows {
+ db::upsert_beacon(
+ &state.db,
+ token_id,
+ file_id,
+ recipient_id,
+ issuer_id,
+ kind,
+ now,
+ )
+ .await?;
+ }
+
+ for (mark_id, layer) in watermark_rows {
+ db::upsert_watermark(
+ &state.db,
+ mark_id,
+ layer,
+ file_id,
+ recipient_id,
+ issuer_id,
+ now,
+ )
+ .await?;
+ }
+
+ if let Some(ref corpus) = req.corpus {
+ for (hash_kind, hash_value) in corpus {
+ if let Some(hv) = hash_value.as_str() {
+ if !hv.is_empty() && hash_kind.len() <= MAX_ID_LEN && hv.len() <= MAX_ID_LEN {
+ db::upsert_corpus(&state.db, file_id, hash_kind, hv, now).await?;
+ }
+ }
+ }
+ }
let rekor_result = if state.rekor_enabled {
attest_to_rekor(