From 8f8e82e00e5d0e9347feef10111faac788ad7d88 Mon Sep 17 00:00:00 2001 From: tanaka takashi Date: Tue, 11 Aug 2026 22:36:21 +0900 Subject: Require task-specific tool evidence --- README.md | 4 +- config/agent.yaml | 2 +- doc/Design.md | 8 ++-- scripts/Operational-E2E.ps1 | 2 +- src/Agent.Core/Runtime/AgentRuntime.cs | 25 +++++++----- src/Agent.Core/Runtime/PlanResult.cs | 6 +-- src/Agent.Core/Runtime/PlannerService.cs | 24 ++++++------ src/Agent.Core/Tools/InternalToolCatalog.cs | 2 +- src/Agent.Core/Tools/TaskGraphToolHandler.cs | 4 +- src/Agent.Core/Tools/ToolUseGuard.cs | 4 +- src/Agent.Planner/PlannerTask.cs | 2 +- src/Agent.Planner/TaskGraph.cs | 15 ++++--- tests/Agent.Core.Tests/PromptAndRuntimeTests.cs | 52 ++++++++++++++++++++++--- 13 files changed, 97 insertions(+), 53 deletions(-) diff --git a/README.md b/README.md index 33fb1df..3dede12 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ Windows 11上でlocal Ollamaを利用し、明示的に承認したFilesystem、 - 管理者権限（TaskごとのWFP filter追加・削除に必要） - Base Filtering Engine (BFE)が実行中 - Ollama。tested versionは実測した `0.32.6`。minimum supported versionは未確定です。 -- Release既定local model: `hf.co/TrevorJS/gemma-4-E4B-it-uncensored-GGUF:Q4_K_M`（native Tool calling対応） +- Release既定model: `deepseek-v4-flash:cloud`（native Tool calling対応） サービス、常駐helper、恒久Firewall ruleは使用しません。WFP objectはdynamic sessionだけに作成されます。AppContainerからlocalhostへ接続するTaskでは、固定AppContainer SIDのloopback exemptionをTask中だけ追加し、終了時または次回起動Recoveryで削除します。 @@ -18,7 +18,7 @@ Windows 11上でlocal Ollamaを利用し、明示的に承認したFilesystem、 ```powershell ollama --version -ollama pull hf.co/TrevorJS/gemma-4-E4B-it-uncensored-GGUF:Q4_K_M +ollama pull deepseek-v4-flash:cloud ollama serve ``` diff --git a/config/agent.yaml b/config/agent.yaml index a39bdec..64c5f9d 100644 --- a/config/agent.yaml +++ b/config/agent.yaml @@ -19,7 +19,7 @@ agent: same_failure_limit: 5 ollama: endpoint: http://127.0.0.1:11434 - model: hf.co/TrevorJS/gemma-4-E4B-it-uncensored-GGUF:Q4_K_M + model: deepseek-v4-flash:cloud think: true keep_alive: 5m options: diff --git a/doc/Design.md b/doc/Design.md index 2d7ba7e..3623cb4 100644 --- a/doc/Design.md +++ b/doc/Design.md @@ -321,7 +321,7 @@ Ollamaに公開する現行Internal Tool catalogは次のとおりとする。 "most_likely_tokens":0, "pessimistic_tokens":0, "requires_external_effect":"boolean|null", - "required_effect_tools":["file_edit|process_run|dir"] + "required_tools":["file_search|file_read|file_edit|process_run|task_update|dir"] } ``` Task状態遷移はAgent Runtimeだけが行う。`task_update` はTask追加、dependency追加、estimate変更に限定し、LLMから状態を変更するschemaを公開しない。 @@ -356,9 +356,9 @@ Task ActualOutputTokens: long ActualTotalTokens: long RequiresExternalEffect: bool - RequiredEffectTools: set + RequiredTools: set ``` -`RequiresExternalEffect=true`のTaskは、Tool catalogに存在し副作用を持つTool名を`RequiredEffectTools`へ1件以上持つ。Planner structured outputの`required_effect_tools`から構築し、Runtimeは成功した副作用Toolがこの集合に含まれる場合だけ期待副作用の証跡として扱う。例えば`process_run`が必要なTask中の`file_edit`成功は完了根拠にしない。current Taskの集合に含まれない副作用ToolはBackendへdispatchせず、Task scope外のTool resultとしてLLMへ返してqueued Taskの先取りを決定的に防ぐ。`RequiresExternalEffect=false`では集合を空にする。 +Taskは、完了に成功証跡が必要なTool名を`RequiredTools`として持つ。Planner structured outputの`required_tools`から構築し、Runtimeは集合内の全Toolが成功した場合だけTaskの操作を検証済みとする。read/search/list Taskは副作用不要でも対応Toolを持ち、純粋な説明Taskだけが空集合となる。`RequiresExternalEffect=true`では集合内の副作用Toolから外部効果も確認する。例えば`process_run`が必要なTask中の`file_edit`成功は完了根拠にしない。current Taskの集合に含まれない副作用ToolはBackendへdispatchせず、Task scope外のTool resultとしてLLMへ返してqueued Taskの先取りを決定的に防ぐ。scope外結果はAudit/LLM相関には残すが、次Taskのactive Execution Result contextへは引き継がない。 ### 8.2 PERT計算 ```text ExpectedTokens = (O + 4*M + P) / 6.0 @@ -442,7 +442,7 @@ agent: same_failure_limit: 5 ollama: endpoint: http://localhost:11434 - model: hf.co/TrevorJS/gemma-4-E4B-it-uncensored-GGUF:Q4_K_M + model: deepseek-v4-flash:cloud think: true keep_alive: 5m options: diff --git a/scripts/Operational-E2E.ps1 b/scripts/Operational-E2E.ps1 index 6e3e082..4b964f1 100644 --- a/scripts/Operational-E2E.ps1 +++ b/scripts/Operational-E2E.ps1 @@ -138,7 +138,7 @@ agent: same_failure_limit: 5 ollama: endpoint: http://127.0.0.1:11434 - model: hf.co/TrevorJS/gemma-4-E4B-it-uncensored-GGUF:Q4_K_M + model: deepseek-v4-flash:cloud think: false keep_alive: 5m options: diff --git a/src/Agent.Core/Runtime/AgentRuntime.cs b/src/Agent.Core/Runtime/AgentRuntime.cs index 9c051e9..4aaf646 100644 --- a/src/Agent.Core/Runtime/AgentRuntime.cs +++ b/src/Agent.Core/Runtime/AgentRuntime.cs @@ -228,6 +228,7 @@ public sealed class AgentRuntime(TaskRegistry registry, IOllamaClient ollama, Ag var guard = new ToolUseGuard(settings.SameFailureLimit); bool requiresExternalEffect = plannerTask.RequiresExternalEffect; bool externalEffectEvidence = false; + var successfulRequiredTools = new HashSet(StringComparer.Ordinal); var unresolvedToolFailures = new Dictionary(StringComparer.Ordinal); string? finalContent = null; @@ -266,11 +267,11 @@ public sealed class AgentRuntime(TaskRegistry registry, IOllamaClient ollama, Ag foreach (OllamaToolCall call in calls) { ToolExecutionResult toolResult; - bool outOfScopeEffect = IsOutOfScopeEffectCall(call, plannerTask.RequiredEffectTools); + bool outOfScopeEffect = IsOutOfScopeEffectCall(call, requiresExternalEffect, plannerTask.RequiredTools); if (outOfScopeEffect) { toolResult = new(false, - $"The '{call.Function.Name}' external effect belongs to another Planner Task. Perform only the current Task; required effect Tool(s): {string.Join(", ", plannerTask.RequiredEffectTools.Order(StringComparer.Ordinal))}.", + $"The '{call.Function.Name}' external effect belongs to another Planner Task. Perform only the current Task; required Tool(s): {string.Join(", ", plannerTask.RequiredTools.Order(StringComparer.Ordinal))}.", false, $"task_scope:{plannerTask.Id}:{call.Function.Name}"); } else try @@ -286,14 +287,16 @@ public sealed class AgentRuntime(TaskRegistry registry, IOllamaClient ollama, Ag toolResult = new(false, ex.Message, false, $"{call.Function.Name}:{ex.GetType().Name}:{ex.Message}"); } - externalEffectEvidence |= toolResult.Success && toolResult.ExternalEffect && plannerTask.RequiredEffectTools.Contains(call.Function.Name); + if (toolResult.Success && plannerTask.RequiredTools.Contains(call.Function.Name)) successfulRequiredTools.Add(call.Function.Name); + externalEffectEvidence |= toolResult.Success && toolResult.ExternalEffect && plannerTask.RequiredTools.Contains(call.Function.Name); // task_update is a metadata tool for graph bookkeeping; its failures should not block completion. bool isMetadataTool = call.Function.Name == "task_update"; if (toolResult.Success) unresolvedToolFailures.Remove(call.Function.Name); else if (!isMetadataTool && !outOfScopeEffect) unresolvedToolFailures[call.Function.Name] = toolResult.FailureFingerprint ?? toolResult.Output; // Record ExecutionResult in session for context summary. - session.AppendExecutionResult(new ExecutionResult(taskId, call.Function.Name, toolResult.Success, toolResult.Output)); + if (!outOfScopeEffect) + session.AppendExecutionResult(new ExecutionResult(taskId, call.Function.Name, toolResult.Success, toolResult.Output)); _observer.ToolResult(callContext, call.Function.Name, toolResult); if (!toolResult.Success && !toolResult.RetryAllowed && !isMetadataTool) @@ -303,11 +306,12 @@ public sealed class AgentRuntime(TaskRegistry registry, IOllamaClient ollama, Ag failureFingerprint = toolResult.FailureFingerprint ?? failureFingerprint; messages.Add(new OllamaMessage("tool", toolResult.Output, ToolName: call.Function.Name)); } - ToolUseGuardResult callGuard = guard.Evaluate(requiresExternalEffect, true, externalEffectEvidence, failureFingerprint); + bool verified = plannerTask.RequiredTools.All(successfulRequiredTools.Contains) && (!requiresExternalEffect || externalEffectEvidence); + ToolUseGuardResult callGuard = guard.Evaluate(plannerTask.RequiredTools.Count > 0 || requiresExternalEffect, true, verified, failureFingerprint); if (callGuard.Decision == ToolUseGuardDecision.Fail) throw new InvalidOperationException(callGuard.Observation); if (callGuard.Decision == ToolUseGuardDecision.Retry) - messages.Add(new OllamaMessage("system", callGuard.Observation + " Required effect Tool(s) for the current Task: " + - string.Join(", ", plannerTask.RequiredEffectTools.Order(StringComparer.Ordinal)) + ".")); + messages.Add(new OllamaMessage("system", callGuard.Observation + " Required Tool(s) for the current Task: " + + string.Join(", ", plannerTask.RequiredTools.Order(StringComparer.Ordinal)) + ".")); continue; } @@ -322,7 +326,8 @@ public sealed class AgentRuntime(TaskRegistry registry, IOllamaClient ollama, Ag continue; } - ToolUseGuardResult completion = guard.Evaluate(requiresExternalEffect, false, externalEffectEvidence); + bool completionVerified = plannerTask.RequiredTools.All(successfulRequiredTools.Contains) && (!requiresExternalEffect || externalEffectEvidence); + ToolUseGuardResult completion = guard.Evaluate(plannerTask.RequiredTools.Count > 0 || requiresExternalEffect, false, completionVerified); if (completion.Decision == ToolUseGuardDecision.Retry) { messages.Add(new OllamaMessage("system", completion.Observation)); @@ -337,12 +342,12 @@ public sealed class AgentRuntime(TaskRegistry registry, IOllamaClient ollama, Ag return finalContent; } - private static bool IsOutOfScopeEffectCall(OllamaToolCall call, IReadOnlySet requiredEffectTools) + private static bool IsOutOfScopeEffectCall(OllamaToolCall call, bool requiresExternalEffect, IReadOnlySet requiredTools) { bool createsExternalEffect = call.Function.Name is "file_edit" or "process_run" || call.Function.Name == "dir" && call.Function.Arguments.TryGetProperty("operation", out JsonElement operation) && operation.GetString() is "mkdir" or "rmdir"; - return createsExternalEffect && !requiredEffectTools.Contains(call.Function.Name); + return createsExternalEffect && (!requiresExternalEffect || !requiredTools.Contains(call.Function.Name)); } /// diff --git a/src/Agent.Core/Runtime/PlanResult.cs b/src/Agent.Core/Runtime/PlanResult.cs index 2e8dbae..cf4830f 100644 --- a/src/Agent.Core/Runtime/PlanResult.cs +++ b/src/Agent.Core/Runtime/PlanResult.cs @@ -37,8 +37,8 @@ public sealed class PlannedTask [JsonPropertyName("requires_external_effect")] public bool RequiresExternalEffect { get; init; } - [JsonPropertyName("required_effect_tools")] - public List RequiredEffectTools { get; init; } = []; + [JsonPropertyName("required_tools")] + public List RequiredTools { get; init; } = []; } /// @@ -46,7 +46,7 @@ public sealed class PlannedTask /// public static class PlanSchema { - public static string Json => """{"type":"object","properties":{"tasks":{"type":"array","items":{"type":"object","properties":{"description":{"type":"string"},"dependencies":{"type":"array","items":{"type":"integer"}},"budget_class":{"type":"string","enum":["1K","2K","4K","8K","16K","32K","64K","128K"]},"optimistic_tokens":{"type":"integer","minimum":1},"most_likely_tokens":{"type":"integer","minimum":1},"pessimistic_tokens":{"type":"integer","minimum":1},"requires_external_effect":{"type":"boolean"},"required_effect_tools":{"type":"array","items":{"type":"string","enum":["file_edit","process_run","dir"]}}},"required":["description","dependencies","budget_class","optimistic_tokens","most_likely_tokens","pessimistic_tokens","requires_external_effect","required_effect_tools"]}}},"required":["tasks"]}"""; + public static string Json => """{"type":"object","properties":{"tasks":{"type":"array","items":{"type":"object","properties":{"description":{"type":"string"},"dependencies":{"type":"array","items":{"type":"integer"}},"budget_class":{"type":"string","enum":["1K","2K","4K","8K","16K","32K","64K","128K"]},"optimistic_tokens":{"type":"integer","minimum":1},"most_likely_tokens":{"type":"integer","minimum":1},"pessimistic_tokens":{"type":"integer","minimum":1},"requires_external_effect":{"type":"boolean"},"required_tools":{"type":"array","items":{"type":"string","enum":["file_search","file_read","file_edit","process_run","task_update","dir"]}}},"required":["description","dependencies","budget_class","optimistic_tokens","most_likely_tokens","pessimistic_tokens","requires_external_effect","required_tools"]}}},"required":["tasks"]}"""; private static readonly JsonElement _element = JsonDocument.Parse(Json).RootElement.Clone(); diff --git a/src/Agent.Core/Runtime/PlannerService.cs b/src/Agent.Core/Runtime/PlannerService.cs index 18e4043..b5ac80e 100644 --- a/src/Agent.Core/Runtime/PlannerService.cs +++ b/src/Agent.Core/Runtime/PlannerService.cs @@ -94,7 +94,7 @@ public sealed class PlannerService(IOllamaClient ollama, AgentRuntimeSettings se string description = string.IsNullOrWhiteSpace(pt.Description) ? originalRequest : pt.Description; PlannerTask task = graph.AddTask(description, budgetClass, estimate, dependencies, pt.RequiresExternalEffect, - requiredEffectTools: ValidateRequiredEffectTools(pt)); + requiredTools: ValidateRequiredTools(pt)); idMap.Add(task.Id); } @@ -112,7 +112,7 @@ public sealed class PlannerService(IOllamaClient ollama, AgentRuntimeSettings se foreach (PlannerTask item in completed) { Guid[] dependencies = item.Dependencies.Where(id => completed.Any(candidate => candidate.Id == id)).ToArray(); - PlannerTask copy = graph.AddTask(item.Description, item.BudgetClass, item.Estimate, dependencies, item.RequiresExternalEffect, item.Id, item.RequiredEffectTools); + PlannerTask copy = graph.AddTask(item.Description, item.BudgetClass, item.Estimate, dependencies, item.RequiresExternalEffect, item.Id, item.RequiredTools); graph.AddUsage(copy.Id, item.ActualInputTokens, item.ActualOutputTokens); graph.SetState(copy.Id, PlannerTaskState.Completed); } @@ -142,7 +142,7 @@ public sealed class PlannerService(IOllamaClient ollama, AgentRuntimeSettings se string description = string.IsNullOrWhiteSpace(pt.Description) ? originalRequest : pt.Description; PlannerTask task = graph.AddTask(description, bc, est, dependencies, pt.RequiresExternalEffect, - requiredEffectTools: ValidateRequiredEffectTools(pt)); + requiredTools: ValidateRequiredTools(pt)); idMap.Add(task.Id); } @@ -169,16 +169,16 @@ public sealed class PlannerService(IOllamaClient ollama, AgentRuntimeSettings se return new TokenEstimate(o, m, p); } - private static IReadOnlySet ValidateRequiredEffectTools(PlannedTask task) + private static IReadOnlySet ValidateRequiredTools(PlannedTask task) { - HashSet tools = task.RequiredEffectTools.ToHashSet(StringComparer.Ordinal); - string[] allowed = ["file_edit", "process_run", "dir"]; + HashSet tools = task.RequiredTools.ToHashSet(StringComparer.Ordinal); + string[] allowed = ["file_search", "file_read", "file_edit", "process_run", "task_update", "dir"]; if (tools.Any(tool => !allowed.Contains(tool, StringComparer.Ordinal))) - throw new PlannerValidationException("required_effect_tools contains a Tool that cannot provide external-effect evidence."); + throw new PlannerValidationException("required_tools contains an unknown Internal Tool."); if (task.RequiresExternalEffect && tools.Count == 0) - throw new PlannerValidationException("An external-effect Task must provide required_effect_tools."); - if (!task.RequiresExternalEffect && tools.Count != 0) - throw new PlannerValidationException("A non-effect Task must have empty required_effect_tools."); + throw new PlannerValidationException("An external-effect Task must provide required_tools."); + if (!task.RequiresExternalEffect && tools.Any(tool => tool is "file_edit" or "process_run")) + throw new PlannerValidationException("A non-effect Task cannot require an effect-only Tool."); return tools; } @@ -300,12 +300,12 @@ public sealed class PlannerService(IOllamaClient ollama, AgentRuntimeSettings se - budget_class: one of 1K, 2K, 4K, 8K, 16K, 32K, 64K, 128K (token budget for this task) - optimistic_tokens, most_likely_tokens, pessimistic_tokens: PERT estimates (0 < O <= M <= P <= budget) - requires_external_effect: true if this task requires a verifiable external effect (file edit, process run, etc.) - - required_effect_tools: the exact effect-producing Internal Tool names whose successful result can complete this task; use only file_edit, process_run, or dir + - required_tools: the exact Internal Tool names whose successful result is required to complete this task; use names from the supplied Tool catalog, or [] for a pure response task For pure analysis or informational tasks, requires_external_effect should be false. For tasks that modify files, run processes, or produce verifiable side effects, requires_external_effect should be true. Read-only operations (file_read, file_search, and dir list) do not create an external effect and must use requires_external_effect=false. - When requires_external_effect is false, required_effect_tools must be empty. When true, it must contain at least one Tool and only the Tool(s) that prove this task's own expected effect. + A read/search/list task must name file_read, file_search, or dir in required_tools even though requires_external_effect is false. A pure response task uses an empty array. When requires_external_effect is true, required_tools must include the effect Tool that proves this task's own expected effect. Preserve every user-supplied absolute path, executable, cwd, argument array, operation name, and literal value exactly in the task that uses it; never abbreviate these values into a shell-like command. Keep each task limited to its own described operation. Do not put later dependent operations into an earlier task. A single task is valid for simple requests. diff --git a/src/Agent.Core/Tools/InternalToolCatalog.cs b/src/Agent.Core/Tools/InternalToolCatalog.cs index e28673d..fbb7423 100644 --- a/src/Agent.Core/Tools/InternalToolCatalog.cs +++ b/src/Agent.Core/Tools/InternalToolCatalog.cs @@ -19,7 +19,7 @@ public static class InternalToolCatalog Define("file_read", "Read an approved file.", """{"type":"object","additionalProperties":false,"required":["path","mode"],"properties":{"path":{"type":"string"},"mode":{"type":"string","enum":["whole","range","metadata"]},"start":{"type":"integer","minimum":0},"length":{"type":"integer","minimum":0}}}"""), Define("file_edit", "Edit an approved file transactionally.", """{"type":"object","additionalProperties":false,"required":["path","operation"],"properties":{"path":{"type":"string"},"operation":{"type":"string","enum":["overwrite","patch","atomic_replace","delete","rename"]},"content":{"type":["string","null"]},"patch":{"type":["string","null"]},"expected_sha256":{"type":["string","null"]},"new_path":{"type":["string","null"]}}}"""), Define("process_run", "Run one approved executable with separated arguments.", """{"type":"object","additionalProperties":false,"required":["executable","args","cwd"],"properties":{"executable":{"type":"string"},"args":{"type":"array","items":{"type":"string"}},"cwd":{"type":"string"},"timeout_seconds":{"type":"integer","minimum":1},"stdin":{"type":["string","null"]}}}"""), - Define("task_update", "Update the deterministic task graph structure or estimates.", """{"type":"object","additionalProperties":false,"required":["operation"],"properties":{"operation":{"type":"string","enum":["add","add_dependency","set_estimate"]},"task_id":{"type":["string","null"]},"description":{"type":["string","null"]},"depends_on":{"type":"array","items":{"type":"string"}},"budget_class":{"type":["string","null"],"enum":["1K","2K","4K","8K","16K","32K","64K","128K",null]},"optimistic_tokens":{"type":"integer","minimum":0},"most_likely_tokens":{"type":"integer","minimum":0},"pessimistic_tokens":{"type":"integer","minimum":0},"requires_external_effect":{"type":["boolean","null"]},"required_effect_tools":{"type":"array","items":{"type":"string","enum":["file_edit","process_run","dir"]}}}}"""), + Define("task_update", "Update the deterministic task graph structure or estimates.", """{"type":"object","additionalProperties":false,"required":["operation"],"properties":{"operation":{"type":"string","enum":["add","add_dependency","set_estimate"]},"task_id":{"type":["string","null"]},"description":{"type":["string","null"]},"depends_on":{"type":"array","items":{"type":"string"}},"budget_class":{"type":["string","null"],"enum":["1K","2K","4K","8K","16K","32K","64K","128K",null]},"optimistic_tokens":{"type":"integer","minimum":0},"most_likely_tokens":{"type":"integer","minimum":0},"pessimistic_tokens":{"type":"integer","minimum":0},"requires_external_effect":{"type":["boolean","null"]},"required_tools":{"type":"array","items":{"type":"string","enum":["file_search","file_read","file_edit","process_run","task_update","dir"]}}}}"""), Define("dir", "Create, remove, or list directories under an approved root. Only empty directories can be removed.", """{"type":"object","additionalProperties":false,"required":["path","operation"],"properties":{"path":{"type":"string"},"operation":{"type":"string","enum":["mkdir","rmdir","list"]}}}""") ]; return definitions.ToDictionary(static definition => definition.Name, StringComparer.Ordinal); diff --git a/src/Agent.Core/Tools/TaskGraphToolHandler.cs b/src/Agent.Core/Tools/TaskGraphToolHandler.cs index fb3979b..de460d6 100644 --- a/src/Agent.Core/Tools/TaskGraphToolHandler.cs +++ b/src/Agent.Core/Tools/TaskGraphToolHandler.cs @@ -53,7 +53,7 @@ public sealed class TaskGraphToolHandler(int maxTasks = 32, SessionStore? sessio Ids(arguments, "depends_on"), requiresExternalEffect: OptionalBool(arguments, "requires_external_effect"), id: OptionalId(arguments, "task_id"), - requiredEffectTools: Strings(arguments, "required_effect_tools")); + requiredTools: Strings(arguments, "required_tools")); break; case "add_dependency": Guid taskIdValue = RequiredId(arguments, "task_id"); @@ -81,7 +81,7 @@ public sealed class TaskGraphToolHandler(int maxTasks = 32, SessionStore? sessio state = task.State.ToString().ToLowerInvariant(), expected_tokens = task.ExpectedTokens, requires_external_effect = task.RequiresExternalEffect, - required_effect_tools = task.RequiredEffectTools + required_tools = task.RequiredTools }), critical_path = graph.CriticalPath.TaskIds, critical_path_tokens = graph.CriticalPath.CumulativeTokenCost, diff --git a/src/Agent.Core/Tools/ToolUseGuard.cs b/src/Agent.Core/Tools/ToolUseGuard.cs index 633d34c..eeb815b 100644 --- a/src/Agent.Core/Tools/ToolUseGuard.cs +++ b/src/Agent.Core/Tools/ToolUseGuard.cs @@ -16,7 +16,7 @@ public sealed class ToolUseGuard(int sameFailureLimit = 5) string fingerprint = string.IsNullOrWhiteSpace(failureFingerprint) ? "missing-external-effect" : failureFingerprint; int count = _failures.TryGetValue(fingerprint, out int prior) ? prior + 1 : 1; _failures[fingerprint] = count; - if (count >= SameFailureLimit) return new(ToolUseGuardDecision.Fail, "Required external effect was not completed after repeated identical failures.", count); - return new(ToolUseGuardDecision.Retry, "No verified external effect occurred; perform the required operation using an internal tool.", count); + if (count >= SameFailureLimit) return new(ToolUseGuardDecision.Fail, "The required operation was not verified after repeated identical failures.", count); + return new(ToolUseGuardDecision.Retry, "Required Tool evidence is missing; perform the current operation using the required Internal Tool.", count); } } diff --git a/src/Agent.Planner/PlannerTask.cs b/src/Agent.Planner/PlannerTask.cs index afba0ef..abd4b08 100644 --- a/src/Agent.Planner/PlannerTask.cs +++ b/src/Agent.Planner/PlannerTask.cs @@ -10,7 +10,7 @@ public sealed record PlannerTask( long ActualInputTokens, long ActualOutputTokens, bool RequiresExternalEffect, - IReadOnlySet RequiredEffectTools) + IReadOnlySet RequiredTools) { public double ExpectedTokens => Estimate.ExpectedTokens; public long ActualTotalTokens => checked(ActualInputTokens + ActualOutputTokens); diff --git a/src/Agent.Planner/TaskGraph.cs b/src/Agent.Planner/TaskGraph.cs index 729f646..a4cdc5f 100644 --- a/src/Agent.Planner/TaskGraph.cs +++ b/src/Agent.Planner/TaskGraph.cs @@ -19,7 +19,7 @@ public sealed class TaskGraph(int maxTasks = 32) IEnumerable? dependencies = null, bool requiresExternalEffect = false, Guid? id = null, - IEnumerable? requiredEffectTools = null) + IEnumerable? requiredTools = null) { if (string.IsNullOrWhiteSpace(description)) throw new PlannerValidationException("Task description is required."); if (_tasks.Count >= MaxTasks) throw new PlannerValidationException($"Task limit of {MaxTasks} exceeded."); @@ -27,13 +27,12 @@ public sealed class TaskGraph(int maxTasks = 32) Guid taskId = id ?? Guid.NewGuid(); if (_tasks.ContainsKey(taskId)) throw new PlannerValidationException($"Task '{taskId}' already exists."); HashSet dependencySet = dependencies?.ToHashSet() ?? []; - HashSet effectTools = requiredEffectTools?.ToHashSet(StringComparer.Ordinal) ?? []; - if (requiresExternalEffect && effectTools.Count == 0) throw new PlannerValidationException("An external-effect Task must name at least one required effect tool."); - if (!requiresExternalEffect && effectTools.Count != 0) throw new PlannerValidationException("A non-effect Task cannot require an effect tool."); + HashSet tools = requiredTools?.ToHashSet(StringComparer.Ordinal) ?? []; + if (requiresExternalEffect && tools.Count == 0) throw new PlannerValidationException("An external-effect Task must name at least one required tool."); EnsureDependenciesExist(dependencySet); if (dependencySet.Contains(taskId)) throw new PlannerValidationException("A task cannot depend on itself."); - _tasks.Add(taskId, new MutableTask(taskId, description, dependencySet, budgetClass, estimate, requiresExternalEffect, effectTools)); + _tasks.Add(taskId, new MutableTask(taskId, description, dependencySet, budgetClass, estimate, requiresExternalEffect, tools)); _insertionOrder.Add(taskId); try { @@ -167,7 +166,7 @@ public sealed class TaskGraph(int maxTasks = 32) if (task.State == PlannerTaskState.Completed) throw new PlannerValidationException("Completed tasks are immutable."); } - private sealed class MutableTask(Guid id, string description, HashSet dependencies, TokenBudgetClass budgetClass, TokenEstimate estimate, bool requiresExternalEffect, HashSet requiredEffectTools) + private sealed class MutableTask(Guid id, string description, HashSet dependencies, TokenBudgetClass budgetClass, TokenEstimate estimate, bool requiresExternalEffect, HashSet requiredTools) { public Guid Id { get; } = id; public string Description { get; } = description; @@ -178,7 +177,7 @@ public sealed class TaskGraph(int maxTasks = 32) public long ActualInputTokens { get; set; } public long ActualOutputTokens { get; set; } public bool RequiresExternalEffect { get; } = requiresExternalEffect; - public HashSet RequiredEffectTools { get; } = requiredEffectTools; - public PlannerTask Snapshot() => new(Id, Description, Dependencies.ToHashSet(), State, BudgetClass, Estimate, ActualInputTokens, ActualOutputTokens, RequiresExternalEffect, RequiredEffectTools.ToHashSet(StringComparer.Ordinal)); + public HashSet RequiredTools { get; } = requiredTools; + public PlannerTask Snapshot() => new(Id, Description, Dependencies.ToHashSet(), State, BudgetClass, Estimate, ActualInputTokens, ActualOutputTokens, RequiresExternalEffect, RequiredTools.ToHashSet(StringComparer.Ordinal)); } } diff --git a/tests/Agent.Core.Tests/PromptAndRuntimeTests.cs b/tests/Agent.Core.Tests/PromptAndRuntimeTests.cs index c13f03c..3ef5bde 100644 --- a/tests/Agent.Core.Tests/PromptAndRuntimeTests.cs +++ b/tests/Agent.Core.Tests/PromptAndRuntimeTests.cs @@ -145,6 +145,26 @@ public sealed class PromptAndRuntimeTests Assert.Equal(["process_run"], backend.Calls.Select(static call => call.Name)); } + [Fact] + public async Task Runtime_requires_read_tool_evidence_for_a_read_task() + { + TaskRegistry registry = new(); + FakeInternalToolBackend backend = new(); + var dispatcher = new InternalToolDispatcher( + new DenyByDefaultPolicy(InternalToolCatalog.All.Select(static item => item.Name).ToHashSet(StringComparer.Ordinal)), backend); + var client = new RequiredReadOllamaClient(); + var runtime = new AgentRuntime(registry, client, new AgentRuntimeSettings("", "en", [], "sandbox"), dispatcher); + TaskSnapshot task = registry.Enqueue("read file", null, null); + registry.Transition(task.TaskId, AgentTaskState.Planning); + + await runtime.ExecuteAsync(task.TaskId, TestContext.Current.CancellationToken); + + Assert.True(registry.TryGet(task.TaskId, null, out TaskSnapshot? completed)); + Assert.Equal(AgentTaskState.Completed, completed!.State); + Assert.Equal(3, client.RunCalls); + Assert.Equal(["file_read"], backend.Calls.Select(static call => call.Name)); + } + [Fact] public async Task Runtime_reaches_terminal_failed_state_when_resource_cleanup_fails() { @@ -180,7 +200,7 @@ public sealed class PromptAndRuntimeTests Assert.True(registry.TryGet(task.TaskId, null, out TaskSnapshot? failed)); Assert.Equal(AgentTaskState.Failed, failed!.State); - Assert.Contains("external effect", failed.Result, StringComparison.OrdinalIgnoreCase); + Assert.Contains("required operation", failed.Result, StringComparison.OrdinalIgnoreCase); } [Fact] @@ -432,7 +452,7 @@ public sealed class PromptAndRuntimeTests public Task ChatStructuredAsync(IReadOnlyList messages, double? temperatureOverride, JsonElement format, CancellationToken cancellationToken) { // Return a plan with requires_external_effect=true for file edit task. - string planJson = """{"tasks":[{"description":"fileを修正","dependencies":[],"budget_class":"4K","optimistic_tokens":512,"most_likely_tokens":2048,"pessimistic_tokens":4096,"requires_external_effect":true,"required_effect_tools":["file_edit"]}]}"""; + string planJson = """{"tasks":[{"description":"fileを修正","dependencies":[],"budget_class":"4K","optimistic_tokens":512,"most_likely_tokens":2048,"pessimistic_tokens":4096,"requires_external_effect":true,"required_tools":["file_edit"]}]}"""; return Task.FromResult(new OllamaResult(planJson, 1, 1)); } public Task ChatWithToolsAsync(IReadOnlyList messages, double? temperatureOverride, IReadOnlyList tools, CancellationToken cancellationToken) @@ -454,7 +474,7 @@ public sealed class PromptAndRuntimeTests public Task ChatAsync(IReadOnlyList messages, double? temperatureOverride, CancellationToken cancellationToken) => Task.FromResult(new OllamaResult("done", 1, 1)); public Task ChatStructuredAsync(IReadOnlyList messages, double? temperatureOverride, JsonElement format, CancellationToken cancellationToken) => - Task.FromResult(new OllamaResult("""{"tasks":[{"description":"run build","dependencies":[],"budget_class":"4K","optimistic_tokens":10,"most_likely_tokens":20,"pessimistic_tokens":30,"requires_external_effect":true,"required_effect_tools":["process_run"]}]}""", 1, 1)); + Task.FromResult(new OllamaResult("""{"tasks":[{"description":"run build","dependencies":[],"budget_class":"4K","optimistic_tokens":10,"most_likely_tokens":20,"pessimistic_tokens":30,"requires_external_effect":true,"required_tools":["process_run"]}]}""", 1, 1)); public Task ChatWithToolsAsync(IReadOnlyList messages, double? temperatureOverride, IReadOnlyList tools, CancellationToken cancellationToken) { int call = Interlocked.Increment(ref _runCalls); @@ -473,6 +493,26 @@ public sealed class PromptAndRuntimeTests } } + private sealed class RequiredReadOllamaClient : IOllamaClient + { + public int RunCalls { get; private set; } + public Task ChatAsync(IReadOnlyList messages, double? temperatureOverride, CancellationToken cancellationToken) => + Task.FromResult(new OllamaResult("done", 1, 1)); + public Task ChatStructuredAsync(IReadOnlyList messages, double? temperatureOverride, JsonElement format, CancellationToken cancellationToken) => + Task.FromResult(new OllamaResult("""{"tasks":[{"description":"read file","dependencies":[],"budget_class":"1K","optimistic_tokens":10,"most_likely_tokens":20,"pessimistic_tokens":30,"requires_external_effect":false,"required_tools":["file_read"]}]}""", 1, 1)); + public Task ChatWithToolsAsync(IReadOnlyList messages, double? temperatureOverride, IReadOnlyList tools, CancellationToken cancellationToken) + { + RunCalls++; + if (RunCalls == 1) return Task.FromResult(new OllamaResult("I will read it.", 1, 1)); + if (RunCalls == 2) + { + using JsonDocument arguments = JsonDocument.Parse("""{"path":"x","mode":"whole"}"""); + return Task.FromResult(new OllamaResult("", 1, 1, [new OllamaToolCall(new OllamaFunctionCall("file_read", arguments.RootElement.Clone()))])); + } + return Task.FromResult(new OllamaResult("read complete", 1, 1)); + } + } + private sealed class NonVisionOllamaClient : IOllamaClient { public int ChatCalls { get; private set; } @@ -514,7 +554,7 @@ public sealed class PromptAndRuntimeTests Task.FromResult(new OllamaResult("done", 1, 1)); public Task ChatStructuredAsync(IReadOnlyList messages, double? temperatureOverride, JsonElement format, CancellationToken cancellationToken) { - string planJson = """{"tasks":[{"description":"run build","dependencies":[],"budget_class":"4K","optimistic_tokens":512,"most_likely_tokens":2048,"pessimistic_tokens":4096,"requires_external_effect":true,"required_effect_tools":["process_run"]}]}"""; + string planJson = """{"tasks":[{"description":"run build","dependencies":[],"budget_class":"4K","optimistic_tokens":512,"most_likely_tokens":2048,"pessimistic_tokens":4096,"requires_external_effect":true,"required_tools":["process_run"]}]}"""; return Task.FromResult(new OllamaResult(planJson, 1, 1)); } public Task ChatWithToolsAsync(IReadOnlyList messages, double? temperatureOverride, IReadOnlyList tools, CancellationToken cancellationToken) @@ -547,7 +587,7 @@ public sealed class PromptAndRuntimeTests public Task ChatStructuredAsync(IReadOnlyList messages, double? temperatureOverride, JsonElement format, CancellationToken cancellationToken) { - string planJson = """{"tasks":[{"description":"research regional curry","dependencies":[],"budget_class":"4K","optimistic_tokens":512,"most_likely_tokens":2048,"pessimistic_tokens":4096,"requires_external_effect":true,"required_effect_tools":["file_edit"]}]}"""; + string planJson = """{"tasks":[{"description":"research regional curry","dependencies":[],"budget_class":"4K","optimistic_tokens":512,"most_likely_tokens":2048,"pessimistic_tokens":4096,"requires_external_effect":true,"required_tools":["file_edit"]}]}"""; return Task.FromResult(new OllamaResult(planJson, 1, 1)); } @@ -592,7 +632,7 @@ public sealed class PromptAndRuntimeTests public Task ChatStructuredAsync(IReadOnlyList messages, double? temperatureOverride, JsonElement format, bool? thinkOverride, CancellationToken cancellationToken) { int plan = Interlocked.Increment(ref _plans); - string json = $$"""{"tasks":[{"description":"search-{{plan}}","dependencies":[],"budget_class":"4K","optimistic_tokens":10,"most_likely_tokens":20,"pessimistic_tokens":30,"requires_external_effect":true,"required_effect_tools":["file_edit"]}]}"""; + string json = $$"""{"tasks":[{"description":"search-{{plan}}","dependencies":[],"budget_class":"4K","optimistic_tokens":10,"most_likely_tokens":20,"pessimistic_tokens":30,"requires_external_effect":true,"required_tools":["file_edit"]}]}"""; return Task.FromResult(new OllamaResult(json, 1, 1)); } public Task ChatWithToolsAsync(IReadOnlyList messages, double? temperatureOverride, IReadOnlyList tools, CancellationToken cancellationToken) -- cgit v1.2.3