test(13-05): assert the D-09 ceiling over the whole registry, not a hardcoded tool list

Task 3: four registry-wide structural tests in tools.rs, iterating registry()
so a future tool that crosses the D-09 ceiling fails CI rather than depending
on a reviewer noticing:

- registry_never_exposes_excluded_authority (S-04/T-13-24): scans every
  ToolDef's name+description for EXCLUDED_AUTHORITY_TERMS.
- read_tools_never_confirm (S-07/T-13-31): every non-destructive tool
  executes via the real execute_tool choke point without raising anything
  confirmation-shaped. bitcoin_status/network_status excluded from live
  execution (their handlers make real outbound network calls that would
  make this test flaky on a sandboxed box); their destructive:false
  placement is still covered by the other assertions.
- loop_is_bounded (S-13/D-05): MAX_TURNS is enforced, and 3 consecutive
  malformed-argument calls for the same tool name abort the turn with an
  apology before a 4th scripted backend turn is ever polled.
- every_tool_has_explicit_category_and_destructive: sanity-checks the
  registry has exactly the 13 hand-written tools (4 destructive) that made
  it in, as a runtime backstop to the acceptance criteria's static grep for
  `..Default::default()`.

Negative-case demonstration (per the plan's acceptance criteria): a
hypothetical `wallet_send_sats` tool with a description mentioning
"spending sats" trips EXCLUDED_AUTHORITY_TERMS's "spend" term, verified by
tracing the exact haystack-contains logic registry_never_exposes_excluded_authority
runs (see 13-05-SUMMARY.md for why this was traced rather than executed).

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
archipelago
2026-08-04 01:34:43 -04:00
co-authored by Claude Opus 5
parent 90706fe053
commit c098124d93
+143 -1
View File
@@ -789,7 +789,9 @@ pub async fn dispatch(name: &str, args: &ToolArgs, handler: &RpcHandler) -> Resu
mod tests {
use super::*;
use crate::api::rpc::RpcHandler;
use crate::assistant::loop_::execute_tool;
use crate::assistant::backends::scripted::ScriptedBackend;
use crate::assistant::backends::BackendTurn;
use crate::assistant::loop_::{execute_tool, run_loop, MAX_TURNS};
use crate::assistant::{CallerScope, ToolExecCtx};
use std::sync::Arc;
@@ -977,4 +979,144 @@ mod tests {
);
}
/// S-04 / T-13-24: no `ToolDef` in the registry exposes excluded
/// authority, by NAME OR DESCRIPTION, over the WHOLE registry — so a
/// tool added in a later phase that crosses the D-09 ceiling fails
/// this test rather than depending on a reviewer noticing.
#[test]
fn registry_never_exposes_excluded_authority() {
let reg = registry();
for tool in reg.all() {
let haystack = format!("{} {}", tool.name, tool.description).to_lowercase();
for term in EXCLUDED_AUTHORITY_TERMS {
assert!(
!haystack.contains(&term.to_lowercase()),
"tool {} exposes excluded authority term {:?} (D-09 ceiling violated)",
tool.name,
term
);
}
}
}
/// S-07 / T-13-31: a read (non-destructive) tool never raises a
/// confirmation request. `bitcoin_status` and `network_status` are
/// excluded from LIVE execution here — their handlers make real
/// outbound network calls (bitcoind RPC / WAN-IP probing / DNS) that
/// would make this test flaky and slow on a sandboxed/offline test
/// box. Their `destructive: false` placement (and thus never hitting
/// the D-07 confirm branch) is still covered by
/// `registry_never_exposes_excluded_authority` and by construction —
/// no confirmation mechanism exists in `execute_tool` for anything
/// other than the `tool.destructive` branch, which those two tools
/// never reach.
#[tokio::test]
async fn read_tools_never_confirm() {
let (handler, _tmp) = test_rpc_handler().await;
grant_all(&handler).await;
let ctx = local_operator_ctx(handler);
let reg = registry();
let network_bound = ["bitcoin_status", "network_status"];
for tool in reg.all() {
if tool.destructive || network_bound.contains(&tool.name) {
continue;
}
let arguments = match tool.name {
"app_logs" => json!({ "app_id": "no-such-app", "lines": 10 }),
"settings_get" => json!({ "key": "network_visibility" }),
_ => json!({}),
};
let call = ToolCall {
id: format!("call-{}", tool.name),
name: tool.name.to_string(),
arguments,
};
let result = execute_tool(&call, &ctx).await;
assert!(
!result.content.to_lowercase().contains("confirm"),
"read tool {} unexpectedly raised something confirmation-shaped: {}",
tool.name,
result.content
);
}
}
/// S-13 / D-05: the loop is bounded two ways — `MAX_TURNS` overall,
/// and an early abort when the same tool name fails validation 3
/// times in a row (rather than burning the whole `MAX_TURNS` budget on
/// a model that keeps sending malformed args).
#[tokio::test]
async fn loop_is_bounded() {
let (handler, _tmp) = test_rpc_handler().await;
grant_all(&handler).await;
// MAX_TURNS: a backend that always proposes another tool call must
// not run forever.
let ctx = local_operator_ctx(handler.clone());
let good_call = ToolCall {
id: "1".to_string(),
name: "system_disk_status".to_string(),
arguments: json!({}),
};
let turns: Vec<BackendTurn> = (0..MAX_TURNS + 2)
.map(|_| BackendTurn::ToolCalls(vec![good_call.clone()]))
.collect();
let backend = ScriptedBackend::new(turns);
let tools_list = vec![system_disk_status_tool()];
let result = run_loop(&backend, "sys", &tools_list, vec![], &ctx).await;
assert!(
result.is_err(),
"run_loop must stop after MAX_TURNS rather than looping forever"
);
// 3 consecutive malformed calls for the SAME tool name abort the
// turn with an apology, before a 4th (correctly-shaped) scripted
// turn is ever reached.
let ctx2 = local_operator_ctx(handler);
let bad_call = ToolCall {
id: "x".to_string(),
name: "app_logs".to_string(),
arguments: json!({ "not_app_id": 1 }),
};
let turns2 = vec![
BackendTurn::ToolCalls(vec![bad_call.clone()]),
BackendTurn::ToolCalls(vec![bad_call.clone()]),
BackendTurn::ToolCalls(vec![bad_call.clone()]),
BackendTurn::Text("should never be reached".to_string()),
];
let backend2 = ScriptedBackend::new(turns2);
let tools_list2 = vec![app_logs_tool()];
let answer = run_loop(&backend2, "sys", &tools_list2, vec![], &ctx2)
.await
.expect("run_loop should abort gracefully with an apology, not error");
assert_ne!(
answer, "should never be reached",
"the loop must abort before the 4th scripted turn is ever polled"
);
}
/// `every_tool_has_explicit_category_and_destructive`: Rust's type
/// system already forbids a partially-constructed `ToolDef` literal —
/// there is no `Default` impl for it, so struct-update syntax is not
/// even available as an escape hatch (the acceptance criterion's grep
/// asserts this directly against the source). This test is the
/// runtime sanity check that every hand-written constructor above
/// actually made it into the registry, so a silently-dropped tool
/// doesn't slip through unnoticed.
#[test]
fn every_tool_has_explicit_category_and_destructive() {
let reg = registry();
let all = reg.all();
assert_eq!(
all.len(),
13,
"expected exactly 13 hand-written tools in the curated registry"
);
let destructive_count = all.iter().filter(|t| t.destructive).count();
assert_eq!(
destructive_count, 4,
"expected exactly 4 destructive tools: app_start, app_stop, app_restart, settings_set"
);
}
}