From 8f8e82e00e5d0e9347feef10111faac788ad7d88 Mon Sep 17 00:00:00 2001 From: tanaka takashi Date: Tue, 11 Aug 2026 22:36:21 +0900 Subject: Require task-specific tool evidence --- tests/Agent.Core.Tests/PromptAndRuntimeTests.cs | 52 ++++++++++++++++++++++--- 1 file changed, 46 insertions(+), 6 deletions(-) (limited to 'tests/Agent.Core.Tests/PromptAndRuntimeTests.cs') diff --git a/tests/Agent.Core.Tests/PromptAndRuntimeTests.cs b/tests/Agent.Core.Tests/PromptAndRuntimeTests.cs index c13f03c..3ef5bde 100644 --- a/tests/Agent.Core.Tests/PromptAndRuntimeTests.cs +++ b/tests/Agent.Core.Tests/PromptAndRuntimeTests.cs @@ -145,6 +145,26 @@ public sealed class PromptAndRuntimeTests Assert.Equal(["process_run"], backend.Calls.Select(static call => call.Name)); } + [Fact] + public async Task Runtime_requires_read_tool_evidence_for_a_read_task() + { + TaskRegistry registry = new(); + FakeInternalToolBackend backend = new(); + var dispatcher = new InternalToolDispatcher( + new DenyByDefaultPolicy(InternalToolCatalog.All.Select(static item => item.Name).ToHashSet(StringComparer.Ordinal)), backend); + var client = new RequiredReadOllamaClient(); + var runtime = new AgentRuntime(registry, client, new AgentRuntimeSettings("", "en", [], "sandbox"), dispatcher); + TaskSnapshot task = registry.Enqueue("read file", null, null); + registry.Transition(task.TaskId, AgentTaskState.Planning); + + await runtime.ExecuteAsync(task.TaskId, TestContext.Current.CancellationToken); + + Assert.True(registry.TryGet(task.TaskId, null, out TaskSnapshot? completed)); + Assert.Equal(AgentTaskState.Completed, completed!.State); + Assert.Equal(3, client.RunCalls); + Assert.Equal(["file_read"], backend.Calls.Select(static call => call.Name)); + } + [Fact] public async Task Runtime_reaches_terminal_failed_state_when_resource_cleanup_fails() { @@ -180,7 +200,7 @@ public sealed class PromptAndRuntimeTests Assert.True(registry.TryGet(task.TaskId, null, out TaskSnapshot? failed)); Assert.Equal(AgentTaskState.Failed, failed!.State); - Assert.Contains("external effect", failed.Result, StringComparison.OrdinalIgnoreCase); + Assert.Contains("required operation", failed.Result, StringComparison.OrdinalIgnoreCase); } [Fact] @@ -432,7 +452,7 @@ public sealed class PromptAndRuntimeTests public Task ChatStructuredAsync(IReadOnlyList messages, double? temperatureOverride, JsonElement format, CancellationToken cancellationToken) { // Return a plan with requires_external_effect=true for file edit task. - string planJson = """{"tasks":[{"description":"fileを修正","dependencies":[],"budget_class":"4K","optimistic_tokens":512,"most_likely_tokens":2048,"pessimistic_tokens":4096,"requires_external_effect":true,"required_effect_tools":["file_edit"]}]}"""; + string planJson = """{"tasks":[{"description":"fileを修正","dependencies":[],"budget_class":"4K","optimistic_tokens":512,"most_likely_tokens":2048,"pessimistic_tokens":4096,"requires_external_effect":true,"required_tools":["file_edit"]}]}"""; return Task.FromResult(new OllamaResult(planJson, 1, 1)); } public Task ChatWithToolsAsync(IReadOnlyList messages, double? temperatureOverride, IReadOnlyList tools, CancellationToken cancellationToken) @@ -454,7 +474,7 @@ public sealed class PromptAndRuntimeTests public Task ChatAsync(IReadOnlyList messages, double? temperatureOverride, CancellationToken cancellationToken) => Task.FromResult(new OllamaResult("done", 1, 1)); public Task ChatStructuredAsync(IReadOnlyList messages, double? temperatureOverride, JsonElement format, CancellationToken cancellationToken) => - Task.FromResult(new OllamaResult("""{"tasks":[{"description":"run build","dependencies":[],"budget_class":"4K","optimistic_tokens":10,"most_likely_tokens":20,"pessimistic_tokens":30,"requires_external_effect":true,"required_effect_tools":["process_run"]}]}""", 1, 1)); + Task.FromResult(new OllamaResult("""{"tasks":[{"description":"run build","dependencies":[],"budget_class":"4K","optimistic_tokens":10,"most_likely_tokens":20,"pessimistic_tokens":30,"requires_external_effect":true,"required_tools":["process_run"]}]}""", 1, 1)); public Task ChatWithToolsAsync(IReadOnlyList messages, double? temperatureOverride, IReadOnlyList tools, CancellationToken cancellationToken) { int call = Interlocked.Increment(ref _runCalls); @@ -473,6 +493,26 @@ public sealed class PromptAndRuntimeTests } } + private sealed class RequiredReadOllamaClient : IOllamaClient + { + public int RunCalls { get; private set; } + public Task ChatAsync(IReadOnlyList messages, double? temperatureOverride, CancellationToken cancellationToken) => + Task.FromResult(new OllamaResult("done", 1, 1)); + public Task ChatStructuredAsync(IReadOnlyList messages, double? temperatureOverride, JsonElement format, CancellationToken cancellationToken) => + Task.FromResult(new OllamaResult("""{"tasks":[{"description":"read file","dependencies":[],"budget_class":"1K","optimistic_tokens":10,"most_likely_tokens":20,"pessimistic_tokens":30,"requires_external_effect":false,"required_tools":["file_read"]}]}""", 1, 1)); + public Task ChatWithToolsAsync(IReadOnlyList messages, double? temperatureOverride, IReadOnlyList tools, CancellationToken cancellationToken) + { + RunCalls++; + if (RunCalls == 1) return Task.FromResult(new OllamaResult("I will read it.", 1, 1)); + if (RunCalls == 2) + { + using JsonDocument arguments = JsonDocument.Parse("""{"path":"x","mode":"whole"}"""); + return Task.FromResult(new OllamaResult("", 1, 1, [new OllamaToolCall(new OllamaFunctionCall("file_read", arguments.RootElement.Clone()))])); + } + return Task.FromResult(new OllamaResult("read complete", 1, 1)); + } + } + private sealed class NonVisionOllamaClient : IOllamaClient { public int ChatCalls { get; private set; } @@ -514,7 +554,7 @@ public sealed class PromptAndRuntimeTests Task.FromResult(new OllamaResult("done", 1, 1)); public Task ChatStructuredAsync(IReadOnlyList messages, double? temperatureOverride, JsonElement format, CancellationToken cancellationToken) { - string planJson = """{"tasks":[{"description":"run build","dependencies":[],"budget_class":"4K","optimistic_tokens":512,"most_likely_tokens":2048,"pessimistic_tokens":4096,"requires_external_effect":true,"required_effect_tools":["process_run"]}]}"""; + string planJson = """{"tasks":[{"description":"run build","dependencies":[],"budget_class":"4K","optimistic_tokens":512,"most_likely_tokens":2048,"pessimistic_tokens":4096,"requires_external_effect":true,"required_tools":["process_run"]}]}"""; return Task.FromResult(new OllamaResult(planJson, 1, 1)); } public Task ChatWithToolsAsync(IReadOnlyList messages, double? temperatureOverride, IReadOnlyList tools, CancellationToken cancellationToken) @@ -547,7 +587,7 @@ public sealed class PromptAndRuntimeTests public Task ChatStructuredAsync(IReadOnlyList messages, double? temperatureOverride, JsonElement format, CancellationToken cancellationToken) { - string planJson = """{"tasks":[{"description":"research regional curry","dependencies":[],"budget_class":"4K","optimistic_tokens":512,"most_likely_tokens":2048,"pessimistic_tokens":4096,"requires_external_effect":true,"required_effect_tools":["file_edit"]}]}"""; + string planJson = """{"tasks":[{"description":"research regional curry","dependencies":[],"budget_class":"4K","optimistic_tokens":512,"most_likely_tokens":2048,"pessimistic_tokens":4096,"requires_external_effect":true,"required_tools":["file_edit"]}]}"""; return Task.FromResult(new OllamaResult(planJson, 1, 1)); } @@ -592,7 +632,7 @@ public sealed class PromptAndRuntimeTests public Task ChatStructuredAsync(IReadOnlyList messages, double? temperatureOverride, JsonElement format, bool? thinkOverride, CancellationToken cancellationToken) { int plan = Interlocked.Increment(ref _plans); - string json = $$"""{"tasks":[{"description":"search-{{plan}}","dependencies":[],"budget_class":"4K","optimistic_tokens":10,"most_likely_tokens":20,"pessimistic_tokens":30,"requires_external_effect":true,"required_effect_tools":["file_edit"]}]}"""; + string json = $$"""{"tasks":[{"description":"search-{{plan}}","dependencies":[],"budget_class":"4K","optimistic_tokens":10,"most_likely_tokens":20,"pessimistic_tokens":30,"requires_external_effect":true,"required_tools":["file_edit"]}]}"""; return Task.FromResult(new OllamaResult(json, 1, 1)); } public Task ChatWithToolsAsync(IReadOnlyList messages, double? temperatureOverride, IReadOnlyList tools, CancellationToken cancellationToken) -- cgit v1.2.3