test(13-05): assert the D-09 ceiling over the whole registry, not a hardcoded tool list
Task 3: four registry-wide structural tests in tools.rs, iterating registry() so a future tool that crosses the D-09 ceiling fails CI rather than depending on a reviewer noticing: - registry_never_exposes_excluded_authority (S-04/T-13-24): scans every ToolDef's name+description for EXCLUDED_AUTHORITY_TERMS. - read_tools_never_confirm (S-07/T-13-31): every non-destructive tool executes via the real execute_tool choke point without raising anything confirmation-shaped. bitcoin_status/network_status excluded from live execution (their handlers make real outbound network calls that would make this test flaky on a sandboxed box); their destructive:false placement is still covered by the other assertions. - loop_is_bounded (S-13/D-05): MAX_TURNS is enforced, and 3 consecutive malformed-argument calls for the same tool name abort the turn with an apology before a 4th scripted backend turn is ever polled. - every_tool_has_explicit_category_and_destructive: sanity-checks the registry has exactly the 13 hand-written tools (4 destructive) that made it in, as a runtime backstop to the acceptance criteria's static grep for `..Default::default()`. Negative-case demonstration (per the plan's acceptance criteria): a hypothetical `wallet_send_sats` tool with a description mentioning "spending sats" trips EXCLUDED_AUTHORITY_TERMS's "spend" term, verified by tracing the exact haystack-contains logic registry_never_exposes_excluded_authority runs (see 13-05-SUMMARY.md for why this was traced rather than executed). Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
90706fe053
commit
c098124d93
@@ -789,7 +789,9 @@ pub async fn dispatch(name: &str, args: &ToolArgs, handler: &RpcHandler) -> Resu
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::api::rpc::RpcHandler;
|
||||
use crate::assistant::loop_::execute_tool;
|
||||
use crate::assistant::backends::scripted::ScriptedBackend;
|
||||
use crate::assistant::backends::BackendTurn;
|
||||
use crate::assistant::loop_::{execute_tool, run_loop, MAX_TURNS};
|
||||
use crate::assistant::{CallerScope, ToolExecCtx};
|
||||
use std::sync::Arc;
|
||||
|
||||
@@ -977,4 +979,144 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
/// S-04 / T-13-24: no `ToolDef` in the registry exposes excluded
|
||||
/// authority, by NAME OR DESCRIPTION, over the WHOLE registry — so a
|
||||
/// tool added in a later phase that crosses the D-09 ceiling fails
|
||||
/// this test rather than depending on a reviewer noticing.
|
||||
#[test]
|
||||
fn registry_never_exposes_excluded_authority() {
|
||||
let reg = registry();
|
||||
for tool in reg.all() {
|
||||
let haystack = format!("{} {}", tool.name, tool.description).to_lowercase();
|
||||
for term in EXCLUDED_AUTHORITY_TERMS {
|
||||
assert!(
|
||||
!haystack.contains(&term.to_lowercase()),
|
||||
"tool {} exposes excluded authority term {:?} (D-09 ceiling violated)",
|
||||
tool.name,
|
||||
term
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// S-07 / T-13-31: a read (non-destructive) tool never raises a
|
||||
/// confirmation request. `bitcoin_status` and `network_status` are
|
||||
/// excluded from LIVE execution here — their handlers make real
|
||||
/// outbound network calls (bitcoind RPC / WAN-IP probing / DNS) that
|
||||
/// would make this test flaky and slow on a sandboxed/offline test
|
||||
/// box. Their `destructive: false` placement (and thus never hitting
|
||||
/// the D-07 confirm branch) is still covered by
|
||||
/// `registry_never_exposes_excluded_authority` and by construction —
|
||||
/// no confirmation mechanism exists in `execute_tool` for anything
|
||||
/// other than the `tool.destructive` branch, which those two tools
|
||||
/// never reach.
|
||||
#[tokio::test]
|
||||
async fn read_tools_never_confirm() {
|
||||
let (handler, _tmp) = test_rpc_handler().await;
|
||||
grant_all(&handler).await;
|
||||
let ctx = local_operator_ctx(handler);
|
||||
let reg = registry();
|
||||
let network_bound = ["bitcoin_status", "network_status"];
|
||||
|
||||
for tool in reg.all() {
|
||||
if tool.destructive || network_bound.contains(&tool.name) {
|
||||
continue;
|
||||
}
|
||||
let arguments = match tool.name {
|
||||
"app_logs" => json!({ "app_id": "no-such-app", "lines": 10 }),
|
||||
"settings_get" => json!({ "key": "network_visibility" }),
|
||||
_ => json!({}),
|
||||
};
|
||||
let call = ToolCall {
|
||||
id: format!("call-{}", tool.name),
|
||||
name: tool.name.to_string(),
|
||||
arguments,
|
||||
};
|
||||
let result = execute_tool(&call, &ctx).await;
|
||||
assert!(
|
||||
!result.content.to_lowercase().contains("confirm"),
|
||||
"read tool {} unexpectedly raised something confirmation-shaped: {}",
|
||||
tool.name,
|
||||
result.content
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// S-13 / D-05: the loop is bounded two ways — `MAX_TURNS` overall,
|
||||
/// and an early abort when the same tool name fails validation 3
|
||||
/// times in a row (rather than burning the whole `MAX_TURNS` budget on
|
||||
/// a model that keeps sending malformed args).
|
||||
#[tokio::test]
|
||||
async fn loop_is_bounded() {
|
||||
let (handler, _tmp) = test_rpc_handler().await;
|
||||
grant_all(&handler).await;
|
||||
|
||||
// MAX_TURNS: a backend that always proposes another tool call must
|
||||
// not run forever.
|
||||
let ctx = local_operator_ctx(handler.clone());
|
||||
let good_call = ToolCall {
|
||||
id: "1".to_string(),
|
||||
name: "system_disk_status".to_string(),
|
||||
arguments: json!({}),
|
||||
};
|
||||
let turns: Vec<BackendTurn> = (0..MAX_TURNS + 2)
|
||||
.map(|_| BackendTurn::ToolCalls(vec![good_call.clone()]))
|
||||
.collect();
|
||||
let backend = ScriptedBackend::new(turns);
|
||||
let tools_list = vec![system_disk_status_tool()];
|
||||
let result = run_loop(&backend, "sys", &tools_list, vec![], &ctx).await;
|
||||
assert!(
|
||||
result.is_err(),
|
||||
"run_loop must stop after MAX_TURNS rather than looping forever"
|
||||
);
|
||||
|
||||
// 3 consecutive malformed calls for the SAME tool name abort the
|
||||
// turn with an apology, before a 4th (correctly-shaped) scripted
|
||||
// turn is ever reached.
|
||||
let ctx2 = local_operator_ctx(handler);
|
||||
let bad_call = ToolCall {
|
||||
id: "x".to_string(),
|
||||
name: "app_logs".to_string(),
|
||||
arguments: json!({ "not_app_id": 1 }),
|
||||
};
|
||||
let turns2 = vec![
|
||||
BackendTurn::ToolCalls(vec![bad_call.clone()]),
|
||||
BackendTurn::ToolCalls(vec![bad_call.clone()]),
|
||||
BackendTurn::ToolCalls(vec![bad_call.clone()]),
|
||||
BackendTurn::Text("should never be reached".to_string()),
|
||||
];
|
||||
let backend2 = ScriptedBackend::new(turns2);
|
||||
let tools_list2 = vec![app_logs_tool()];
|
||||
let answer = run_loop(&backend2, "sys", &tools_list2, vec![], &ctx2)
|
||||
.await
|
||||
.expect("run_loop should abort gracefully with an apology, not error");
|
||||
assert_ne!(
|
||||
answer, "should never be reached",
|
||||
"the loop must abort before the 4th scripted turn is ever polled"
|
||||
);
|
||||
}
|
||||
|
||||
/// `every_tool_has_explicit_category_and_destructive`: Rust's type
|
||||
/// system already forbids a partially-constructed `ToolDef` literal —
|
||||
/// there is no `Default` impl for it, so struct-update syntax is not
|
||||
/// even available as an escape hatch (the acceptance criterion's grep
|
||||
/// asserts this directly against the source). This test is the
|
||||
/// runtime sanity check that every hand-written constructor above
|
||||
/// actually made it into the registry, so a silently-dropped tool
|
||||
/// doesn't slip through unnoticed.
|
||||
#[test]
|
||||
fn every_tool_has_explicit_category_and_destructive() {
|
||||
let reg = registry();
|
||||
let all = reg.all();
|
||||
assert_eq!(
|
||||
all.len(),
|
||||
13,
|
||||
"expected exactly 13 hand-written tools in the curated registry"
|
||||
);
|
||||
let destructive_count = all.iter().filter(|t| t.destructive).count();
|
||||
assert_eq!(
|
||||
destructive_count, 4,
|
||||
"expected exactly 4 destructive tools: app_start, app_stop, app_restart, settings_set"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user