From 60d558e9a7eb1944fc58ee88e121a5db5248a332 Mon Sep 17 00:00:00 2001 From: tanaka takashi Date: Mon, 10 Aug 2026 01:39:40 +0900 Subject: fix(agent): fail fast on sandbox policy denial Treat Policy Engine denials and UnauthorizedAccessException results as terminal tool failures by adding RetryAllowed metadata to ToolExecutionResult. Stop the Agent loop immediately after recording the denied tool result, transition the Task to failed, and run normal Task resource cleanup instead of returning the same failure to the LLM for another iteration. Correctable validation and process failures retain the existing bounded retry behavior. Expose RetryAllowed in llm.jsonl and document the fail-fast security invariant in Specifications, Design, and ImplementationPlan. Tests: Agent.Core.Tests 50/50 passed with skip 0; Release build passed with 0 warnings and 0 errors. The full Windows suite remained environment-blocked outside an elevated token: 14 tests failed while creating/deriving the fixed AppContainer or creating symlinks; 11 passed and none were skipped. Release: refreshed the win-x64 self-contained folder binaries, publish manifest, ZIP, and checksum while preserving the local agent.yaml. The clean ZIP contains no runtime logs or Task state. --- doc/Design.md | 2 +- doc/Specifications.md | 2 +- plans/ImplementationPlan.md | 1 + src/Agent.App/Logging/JsonLinesRuntimeObserver.cs | 2 +- src/Agent.Core/Runtime/AgentRuntime.cs | 8 +++- src/Agent.Core/Tools/InternalToolDispatcher.cs | 2 +- src/Agent.Core/Tools/ToolPolicy.cs | 7 +++- tests/Agent.Core.Tests/InternalToolTests.cs | 1 + tests/Agent.Core.Tests/PromptAndRuntimeTests.cs | 49 +++++++++++++++++++++++ 9 files changed, 68 insertions(+), 6 deletions(-) diff --git a/doc/Design.md b/doc/Design.md index 1bcbc99..693cdeb 100644 --- a/doc/Design.md +++ b/doc/Design.md @@ -317,7 +317,7 @@ Ollamaに公開するInternal Toolは次の5群に固定する。個々のOS com ``` PlannerがDAG、不変条件、completed immutable、max_tasksを再検査してから適用する。 ### 7.6 ToolUseGuard -Taskには `requires_external_effect` をPlanner内部属性として持たせる。trueのTaskで、当該iteration中のtool callが0かつ完了証拠がない場合、LLMへ「未実行」を示すsystem-generated observationを追加して再試行する。同一失敗fingerprintが5回に達したらTaskをfailedにする。 +Taskには `requires_external_effect` をPlanner内部属性として持たせる。trueのTaskで、当該iteration中のtool callが0かつ完了証拠がない場合、LLMへ「未実行」を示すsystem-generated observationを追加して再試行する。同一失敗fingerprintが5回に達したらTaskをfailedにする。ただし、Policy EngineのdenyおよびSandbox backendの`UnauthorizedAccessException`は`RetryAllowed=false`のterminal Tool resultへ変換する。Runtimeはobserverへ結果を1回記録した直後にTaskをfailedへ遷移させ、LLMへ再投入しない。 --- ## 8. Token Planner設計 ### 8.1 Task model diff --git a/doc/Specifications.md b/doc/Specifications.md index e723d03..cc04762 100644 --- a/doc/Specifications.md +++ b/doc/Specifications.md @@ -231,7 +231,7 @@ Vision入力は設定可能な画像数、1画像あたりのsize、request合 ### 7.2 Tool未使用検出 -AgentはTaskの期待副作用と観測結果を比較し、実行が必要なのにtool callが0件の場合はTaskをcompletedにしない。再試行はAgent Loopの失敗回数へ算入する。純粋な説明・分析Taskにはtool callを強制しない。 +AgentはTaskの期待副作用と観測結果を比較し、実行が必要なのにtool callが0件の場合はTaskをcompletedにしない。再試行はAgent Loopの失敗回数へ算入する。純粋な説明・分析Taskにはtool callを強制しない。Sandbox Policyによる未許可操作は修正可能なTool失敗としてLLMへ再試行させず、初回のdenyでTaskをfailedにしてcleanupを実行する。 Internal Toolが返す読取・検索・tail結果は設定可能な上限を持ち、contextを無制限に消費してはならない。上限値は設計および設定schemaで定義する。 diff --git a/plans/ImplementationPlan.md b/plans/ImplementationPlan.md index d87a357..8127701 100644 --- a/plans/ImplementationPlan.md +++ b/plans/ImplementationPlan.md @@ -346,6 +346,7 @@ Planner deterministic test全PASS。 - [ ] Ollamaには5群のみ公開 - [ ] malformed tool args拒否 - [ ] Policy deny時に副作用なし +- [ ] Policy denyは初回でfailedとなり、LLMへ再試行されない - [ ] external effect必須Taskでtool未使用ならcompletedにならない - [ ] 同一失敗5回でfailed - [ ] pure analysis TaskにはToolを強制しない diff --git a/src/Agent.App/Logging/JsonLinesRuntimeObserver.cs b/src/Agent.App/Logging/JsonLinesRuntimeObserver.cs index b7cbd26..20ef88d 100644 --- a/src/Agent.App/Logging/JsonLinesRuntimeObserver.cs +++ b/src/Agent.App/Logging/JsonLinesRuntimeObserver.cs @@ -26,7 +26,7 @@ public sealed class JsonLinesRuntimeObserver(string logDirectory) : IAgentRuntim public void ToolResult(Guid taskId, string toolName, ToolExecutionResult result) => Write(new { timestamp = DateTimeOffset.UtcNow, task_id = taskId, event_name = "llm.tool_result", tool_name = toolName, - result.Success, result.ExternalEffect, result.Output, result.FailureFingerprint + result.Success, result.ExternalEffect, result.RetryAllowed, result.Output, result.FailureFingerprint }); private void Write(object value) diff --git a/src/Agent.Core/Runtime/AgentRuntime.cs b/src/Agent.Core/Runtime/AgentRuntime.cs index c2fa568..0fcf7d0 100644 --- a/src/Agent.Core/Runtime/AgentRuntime.cs +++ b/src/Agent.Core/Runtime/AgentRuntime.cs @@ -63,7 +63,11 @@ public sealed class AgentRuntime(TaskRegistry registry, IOllamaClient ollama, Ag { toolResult = await _tools.DispatchAsync(taskId, call.Function.Name, call.Function.Arguments, cancellationToken).ConfigureAwait(false); } - catch (Exception ex) when (ex is ToolValidationException or ArgumentException or UnauthorizedAccessException) + catch (UnauthorizedAccessException ex) + { + toolResult = new(false, ex.Message, false, $"{call.Function.Name}:{ex.GetType().Name}:{ex.Message}", RetryAllowed: false); + } + catch (Exception ex) when (ex is ToolValidationException or ArgumentException) { toolResult = new(false, ex.Message, false, $"{call.Function.Name}:{ex.GetType().Name}:{ex.Message}"); } @@ -72,6 +76,8 @@ public sealed class AgentRuntime(TaskRegistry registry, IOllamaClient ollama, Ag else unresolvedToolFailures[call.Function.Name] = toolResult.FailureFingerprint ?? toolResult.Output; if (toolResult.Success) successfulToolCalls[call.Function.Name] = successfulToolCalls.GetValueOrDefault(call.Function.Name) + 1; _observer.ToolResult(taskId, call.Function.Name, toolResult); + if (!toolResult.Success && !toolResult.RetryAllowed) + throw new InvalidOperationException($"Tool '{call.Function.Name}' was denied by sandbox policy and will not be retried: {toolResult.Output}"); failureFingerprint = toolResult.FailureFingerprint ?? failureFingerprint; messages.Add(new OllamaMessage("tool", toolResult.Output, ToolName: call.Function.Name)); } diff --git a/src/Agent.Core/Tools/InternalToolDispatcher.cs b/src/Agent.Core/Tools/InternalToolDispatcher.cs index 7302c1f..0dca5df 100644 --- a/src/Agent.Core/Tools/InternalToolDispatcher.cs +++ b/src/Agent.Core/Tools/InternalToolDispatcher.cs @@ -11,7 +11,7 @@ public sealed class InternalToolDispatcher(IToolPolicyEngine policy, IInternalTo { InternalToolValidator.Validate(toolName, arguments); ToolPolicyDecision decision = policy.Evaluate(toolName, arguments); - if (!decision.Allowed) return new(false, decision.Reason, false, $"policy:{toolName}:{decision.Reason}"); + if (!decision.Allowed) return new(false, decision.Reason, false, $"policy:{toolName}:{decision.Reason}", RetryAllowed: false); return await backend.ExecuteAsync(taskId, toolName, arguments, cancellationToken).ConfigureAwait(false); } } diff --git a/src/Agent.Core/Tools/ToolPolicy.cs b/src/Agent.Core/Tools/ToolPolicy.cs index c9f216a..d478a2c 100644 --- a/src/Agent.Core/Tools/ToolPolicy.cs +++ b/src/Agent.Core/Tools/ToolPolicy.cs @@ -20,7 +20,12 @@ public interface IInternalToolBackend Task EndTaskAsync(Guid taskId, bool success, CancellationToken cancellationToken) => Task.CompletedTask; } -public sealed record ToolExecutionResult(bool Success, string Output, bool ExternalEffect, string? FailureFingerprint = null); +public sealed record ToolExecutionResult( + bool Success, + string Output, + bool ExternalEffect, + string? FailureFingerprint = null, + bool RetryAllowed = true); public sealed class DenyByDefaultPolicy(IReadOnlySet? allowedTools = null) : IToolPolicyEngine { diff --git a/tests/Agent.Core.Tests/InternalToolTests.cs b/tests/Agent.Core.Tests/InternalToolTests.cs index 1e241ce..e452cab 100644 --- a/tests/Agent.Core.Tests/InternalToolTests.cs +++ b/tests/Agent.Core.Tests/InternalToolTests.cs @@ -36,6 +36,7 @@ public sealed class InternalToolTests Assert.False(result.Success); Assert.False(result.ExternalEffect); + Assert.False(result.RetryAllowed); Assert.Empty(backend.Calls); } diff --git a/tests/Agent.Core.Tests/PromptAndRuntimeTests.cs b/tests/Agent.Core.Tests/PromptAndRuntimeTests.cs index 23ad2d2..c3c634d 100644 --- a/tests/Agent.Core.Tests/PromptAndRuntimeTests.cs +++ b/tests/Agent.Core.Tests/PromptAndRuntimeTests.cs @@ -144,6 +144,29 @@ public sealed class PromptAndRuntimeTests Assert.Contains("iteration limit", failed.Result, StringComparison.OrdinalIgnoreCase); } + [Fact] + public async Task Runtime_does_not_retry_a_sandbox_policy_denial() + { + TaskRegistry registry = new(); + var backend = new UnauthorizedBackend(); + var dispatcher = new InternalToolDispatcher( + new DenyByDefaultPolicy(new HashSet(["file_search"], StringComparer.Ordinal)), backend); + var ollama = new FileSearchOllamaClient(); + var runtime = new AgentRuntime(registry, ollama, new AgentRuntimeSettings("", "en", [], "sandbox"), dispatcher); + TaskSnapshot task = registry.Enqueue("research regional curry", null, null); + registry.Transition(task.TaskId, AgentTaskState.Planning); + registry.Transition(task.TaskId, AgentTaskState.Running); + + await runtime.ExecuteAsync(task.TaskId, TestContext.Current.CancellationToken); + + Assert.True(registry.TryGet(task.TaskId, null, out TaskSnapshot? failed)); + Assert.Equal(AgentTaskState.Failed, failed!.State); + Assert.Contains("denied by sandbox policy", failed.Result, StringComparison.OrdinalIgnoreCase); + Assert.Contains("not approved for Read", failed.Result, StringComparison.OrdinalIgnoreCase); + Assert.Equal(1, ollama.CallCount); + Assert.Equal(1, backend.CallCount); + } + private sealed class RecordingOllamaClient : IOllamaClient { public List> Calls { get; } = []; @@ -217,4 +240,30 @@ public sealed class PromptAndRuntimeTests return Task.FromResult(new OllamaResult("", 1, 1, [new OllamaToolCall(new OllamaFunctionCall("process_run", arguments.RootElement.Clone()))])); } } + + private sealed class UnauthorizedBackend : IInternalToolBackend + { + public int CallCount { get; private set; } + + public Task ExecuteAsync(Guid taskId, string toolName, JsonElement arguments, CancellationToken cancellationToken) + { + CallCount++; + throw new UnauthorizedAccessException("'C:\\Users' is not approved for Read."); + } + } + + private sealed class FileSearchOllamaClient : IOllamaClient + { + public int CallCount { get; private set; } + + public Task ChatAsync(IReadOnlyList messages, double? temperatureOverride, CancellationToken cancellationToken) => + Task.FromResult(new OllamaResult("unexpected", 1, 1)); + + public Task ChatWithToolsAsync(IReadOnlyList messages, double? temperatureOverride, IReadOnlyList tools, CancellationToken cancellationToken) + { + CallCount++; + using JsonDocument arguments = JsonDocument.Parse("""{"root":"C:\\Users","mode":"name","query":"*.txt"}"""); + return Task.FromResult(new OllamaResult("", 1, 1, [new OllamaToolCall(new OllamaFunctionCall("file_search", arguments.RootElement.Clone()))])); + } + } } -- cgit v1.2.3