From 842e55654db060c84448f3a0deddb24f77fad5a7 Mon Sep 17 00:00:00 2001 From: tanaka takashi Date: Tue, 11 Aug 2026 22:20:25 +0900 Subject: Verify task-specific external effect evidence --- tests/Agent.Core.Tests/PromptAndRuntimeTests.cs | 52 +++++++++++++++++++++++-- 1 file changed, 48 insertions(+), 4 deletions(-) (limited to 'tests') diff --git a/tests/Agent.Core.Tests/PromptAndRuntimeTests.cs b/tests/Agent.Core.Tests/PromptAndRuntimeTests.cs index dce2b28..d72e02a 100644 --- a/tests/Agent.Core.Tests/PromptAndRuntimeTests.cs +++ b/tests/Agent.Core.Tests/PromptAndRuntimeTests.cs @@ -126,6 +126,25 @@ public sealed class PromptAndRuntimeTests Assert.Equal(InternalToolCatalog.All.Count, ollama.ToolCount); } + [Fact] + public async Task Runtime_requires_the_planned_effect_tool_not_an_unrelated_effect() + { + TaskRegistry registry = new(); + FakeInternalToolBackend backend = new(); + var dispatcher = new InternalToolDispatcher( + new DenyByDefaultPolicy(InternalToolCatalog.All.Select(static item => item.Name).ToHashSet(StringComparer.Ordinal)), backend); + var runtime = new AgentRuntime(registry, new MismatchedThenRequiredEffectOllamaClient(), + new AgentRuntimeSettings("", "en", [], "sandbox"), dispatcher); + TaskSnapshot task = registry.Enqueue("run build", null, null); + registry.Transition(task.TaskId, AgentTaskState.Planning); + + await runtime.ExecuteAsync(task.TaskId, TestContext.Current.CancellationToken); + + Assert.True(registry.TryGet(task.TaskId, null, out TaskSnapshot? completed)); + Assert.Equal(AgentTaskState.Completed, completed!.State); + Assert.Equal(["file_edit", "process_run"], backend.Calls.Select(static call => call.Name)); + } + [Fact] public async Task Runtime_reaches_terminal_failed_state_when_resource_cleanup_fails() { @@ -413,7 +432,7 @@ public sealed class PromptAndRuntimeTests public Task ChatStructuredAsync(IReadOnlyList messages, double? temperatureOverride, JsonElement format, CancellationToken cancellationToken) { // Return a plan with requires_external_effect=true for file edit task. - string planJson = """{"tasks":[{"description":"fileを修正","dependencies":[],"budget_class":"4K","optimistic_tokens":512,"most_likely_tokens":2048,"pessimistic_tokens":4096,"requires_external_effect":true}]}"""; + string planJson = """{"tasks":[{"description":"fileを修正","dependencies":[],"budget_class":"4K","optimistic_tokens":512,"most_likely_tokens":2048,"pessimistic_tokens":4096,"requires_external_effect":true,"required_effect_tools":["file_edit"]}]}"""; return Task.FromResult(new OllamaResult(planJson, 1, 1)); } public Task ChatWithToolsAsync(IReadOnlyList messages, double? temperatureOverride, IReadOnlyList tools, CancellationToken cancellationToken) @@ -429,6 +448,31 @@ public sealed class PromptAndRuntimeTests } } + private sealed class MismatchedThenRequiredEffectOllamaClient : IOllamaClient + { + private int _runCalls; + public Task ChatAsync(IReadOnlyList messages, double? temperatureOverride, CancellationToken cancellationToken) => + Task.FromResult(new OllamaResult("done", 1, 1)); + public Task ChatStructuredAsync(IReadOnlyList messages, double? temperatureOverride, JsonElement format, CancellationToken cancellationToken) => + Task.FromResult(new OllamaResult("""{"tasks":[{"description":"run build","dependencies":[],"budget_class":"4K","optimistic_tokens":10,"most_likely_tokens":20,"pessimistic_tokens":30,"requires_external_effect":true,"required_effect_tools":["process_run"]}]}""", 1, 1)); + public Task ChatWithToolsAsync(IReadOnlyList messages, double? temperatureOverride, IReadOnlyList tools, CancellationToken cancellationToken) + { + int call = Interlocked.Increment(ref _runCalls); + if (call == 1) + { + using JsonDocument arguments = JsonDocument.Parse("""{"path":"x","operation":"overwrite","content":"new"}"""); + return Task.FromResult(new OllamaResult("", 1, 1, [new OllamaToolCall(new OllamaFunctionCall("file_edit", arguments.RootElement.Clone()))])); + } + if (call == 2) + { + Assert.Contains(messages, message => message.Role == "system" && message.Content.Contains("process_run", StringComparison.Ordinal)); + using JsonDocument arguments = JsonDocument.Parse("""{"executable":"x","args":[],"cwd":"."}"""); + return Task.FromResult(new OllamaResult("", 1, 1, [new OllamaToolCall(new OllamaFunctionCall("process_run", arguments.RootElement.Clone()))])); + } + return Task.FromResult(new OllamaResult("done", 1, 1)); + } + } + private sealed class NonVisionOllamaClient : IOllamaClient { public int ChatCalls { get; private set; } @@ -470,7 +514,7 @@ public sealed class PromptAndRuntimeTests Task.FromResult(new OllamaResult("done", 1, 1)); public Task ChatStructuredAsync(IReadOnlyList messages, double? temperatureOverride, JsonElement format, CancellationToken cancellationToken) { - string planJson = """{"tasks":[{"description":"run build","dependencies":[],"budget_class":"4K","optimistic_tokens":512,"most_likely_tokens":2048,"pessimistic_tokens":4096,"requires_external_effect":true}]}"""; + string planJson = """{"tasks":[{"description":"run build","dependencies":[],"budget_class":"4K","optimistic_tokens":512,"most_likely_tokens":2048,"pessimistic_tokens":4096,"requires_external_effect":true,"required_effect_tools":["process_run"]}]}"""; return Task.FromResult(new OllamaResult(planJson, 1, 1)); } public Task ChatWithToolsAsync(IReadOnlyList messages, double? temperatureOverride, IReadOnlyList tools, CancellationToken cancellationToken) @@ -503,7 +547,7 @@ public sealed class PromptAndRuntimeTests public Task ChatStructuredAsync(IReadOnlyList messages, double? temperatureOverride, JsonElement format, CancellationToken cancellationToken) { - string planJson = """{"tasks":[{"description":"research regional curry","dependencies":[],"budget_class":"4K","optimistic_tokens":512,"most_likely_tokens":2048,"pessimistic_tokens":4096,"requires_external_effect":true}]}"""; + string planJson = """{"tasks":[{"description":"research regional curry","dependencies":[],"budget_class":"4K","optimistic_tokens":512,"most_likely_tokens":2048,"pessimistic_tokens":4096,"requires_external_effect":true,"required_effect_tools":["file_edit"]}]}"""; return Task.FromResult(new OllamaResult(planJson, 1, 1)); } @@ -548,7 +592,7 @@ public sealed class PromptAndRuntimeTests public Task ChatStructuredAsync(IReadOnlyList messages, double? temperatureOverride, JsonElement format, bool? thinkOverride, CancellationToken cancellationToken) { int plan = Interlocked.Increment(ref _plans); - string json = $$"""{"tasks":[{"description":"search-{{plan}}","dependencies":[],"budget_class":"4K","optimistic_tokens":10,"most_likely_tokens":20,"pessimistic_tokens":30,"requires_external_effect":true}]}"""; + string json = $$"""{"tasks":[{"description":"search-{{plan}}","dependencies":[],"budget_class":"4K","optimistic_tokens":10,"most_likely_tokens":20,"pessimistic_tokens":30,"requires_external_effect":true,"required_effect_tools":["file_edit"]}]}"""; return Task.FromResult(new OllamaResult(json, 1, 1)); } public Task ChatWithToolsAsync(IReadOnlyList messages, double? temperatureOverride, IReadOnlyList tools, CancellationToken cancellationToken) -- cgit v1.2.3