diff options
Diffstat (limited to 'tests')
| -rw-r--r-- | tests/Agent.Core.Tests/PromptAndRuntimeTests.cs | 52 |
1 files changed, 46 insertions, 6 deletions
diff --git a/tests/Agent.Core.Tests/PromptAndRuntimeTests.cs b/tests/Agent.Core.Tests/PromptAndRuntimeTests.cs index c13f03c..3ef5bde 100644 --- a/tests/Agent.Core.Tests/PromptAndRuntimeTests.cs +++ b/tests/Agent.Core.Tests/PromptAndRuntimeTests.cs @@ -146,6 +146,26 @@ public sealed class PromptAndRuntimeTests } [Fact] + public async Task Runtime_requires_read_tool_evidence_for_a_read_task() + { + TaskRegistry registry = new(); + FakeInternalToolBackend backend = new(); + var dispatcher = new InternalToolDispatcher( + new DenyByDefaultPolicy(InternalToolCatalog.All.Select(static item => item.Name).ToHashSet(StringComparer.Ordinal)), backend); + var client = new RequiredReadOllamaClient(); + var runtime = new AgentRuntime(registry, client, new AgentRuntimeSettings("", "en", [], "sandbox"), dispatcher); + TaskSnapshot task = registry.Enqueue("read file", null, null); + registry.Transition(task.TaskId, AgentTaskState.Planning); + + await runtime.ExecuteAsync(task.TaskId, TestContext.Current.CancellationToken); + + Assert.True(registry.TryGet(task.TaskId, null, out TaskSnapshot? completed)); + Assert.Equal(AgentTaskState.Completed, completed!.State); + Assert.Equal(3, client.RunCalls); + Assert.Equal(["file_read"], backend.Calls.Select(static call => call.Name)); + } + + [Fact] public async Task Runtime_reaches_terminal_failed_state_when_resource_cleanup_fails() { TaskRegistry registry = new(); @@ -180,7 +200,7 @@ public sealed class PromptAndRuntimeTests Assert.True(registry.TryGet(task.TaskId, null, out TaskSnapshot? failed)); Assert.Equal(AgentTaskState.Failed, failed!.State); - Assert.Contains("external effect", failed.Result, StringComparison.OrdinalIgnoreCase); + Assert.Contains("required operation", failed.Result, StringComparison.OrdinalIgnoreCase); } [Fact] @@ -432,7 +452,7 @@ public sealed class PromptAndRuntimeTests public Task<OllamaResult> ChatStructuredAsync(IReadOnlyList<OllamaMessage> messages, double? temperatureOverride, JsonElement format, CancellationToken cancellationToken) { // Return a plan with requires_external_effect=true for file edit task. - string planJson = """{"tasks":[{"description":"fileを修正","dependencies":[],"budget_class":"4K","optimistic_tokens":512,"most_likely_tokens":2048,"pessimistic_tokens":4096,"requires_external_effect":true,"required_effect_tools":["file_edit"]}]}"""; + string planJson = """{"tasks":[{"description":"fileを修正","dependencies":[],"budget_class":"4K","optimistic_tokens":512,"most_likely_tokens":2048,"pessimistic_tokens":4096,"requires_external_effect":true,"required_tools":["file_edit"]}]}"""; return Task.FromResult(new OllamaResult(planJson, 1, 1)); } public Task<OllamaResult> ChatWithToolsAsync(IReadOnlyList<OllamaMessage> messages, double? temperatureOverride, IReadOnlyList<InternalToolDefinition> tools, CancellationToken cancellationToken) @@ -454,7 +474,7 @@ public sealed class PromptAndRuntimeTests public Task<OllamaResult> ChatAsync(IReadOnlyList<OllamaMessage> messages, double? temperatureOverride, CancellationToken cancellationToken) => Task.FromResult(new OllamaResult("done", 1, 1)); public Task<OllamaResult> ChatStructuredAsync(IReadOnlyList<OllamaMessage> messages, double? temperatureOverride, JsonElement format, CancellationToken cancellationToken) => - Task.FromResult(new OllamaResult("""{"tasks":[{"description":"run build","dependencies":[],"budget_class":"4K","optimistic_tokens":10,"most_likely_tokens":20,"pessimistic_tokens":30,"requires_external_effect":true,"required_effect_tools":["process_run"]}]}""", 1, 1)); + Task.FromResult(new OllamaResult("""{"tasks":[{"description":"run build","dependencies":[],"budget_class":"4K","optimistic_tokens":10,"most_likely_tokens":20,"pessimistic_tokens":30,"requires_external_effect":true,"required_tools":["process_run"]}]}""", 1, 1)); public Task<OllamaResult> ChatWithToolsAsync(IReadOnlyList<OllamaMessage> messages, double? temperatureOverride, IReadOnlyList<InternalToolDefinition> tools, CancellationToken cancellationToken) { int call = Interlocked.Increment(ref _runCalls); @@ -473,6 +493,26 @@ public sealed class PromptAndRuntimeTests } } + private sealed class RequiredReadOllamaClient : IOllamaClient + { + public int RunCalls { get; private set; } + public Task<OllamaResult> ChatAsync(IReadOnlyList<OllamaMessage> messages, double? temperatureOverride, CancellationToken cancellationToken) => + Task.FromResult(new OllamaResult("done", 1, 1)); + public Task<OllamaResult> ChatStructuredAsync(IReadOnlyList<OllamaMessage> messages, double? temperatureOverride, JsonElement format, CancellationToken cancellationToken) => + Task.FromResult(new OllamaResult("""{"tasks":[{"description":"read file","dependencies":[],"budget_class":"1K","optimistic_tokens":10,"most_likely_tokens":20,"pessimistic_tokens":30,"requires_external_effect":false,"required_tools":["file_read"]}]}""", 1, 1)); + public Task<OllamaResult> ChatWithToolsAsync(IReadOnlyList<OllamaMessage> messages, double? temperatureOverride, IReadOnlyList<InternalToolDefinition> tools, CancellationToken cancellationToken) + { + RunCalls++; + if (RunCalls == 1) return Task.FromResult(new OllamaResult("I will read it.", 1, 1)); + if (RunCalls == 2) + { + using JsonDocument arguments = JsonDocument.Parse("""{"path":"x","mode":"whole"}"""); + return Task.FromResult(new OllamaResult("", 1, 1, [new OllamaToolCall(new OllamaFunctionCall("file_read", arguments.RootElement.Clone()))])); + } + return Task.FromResult(new OllamaResult("read complete", 1, 1)); + } + } + private sealed class NonVisionOllamaClient : IOllamaClient { public int ChatCalls { get; private set; } @@ -514,7 +554,7 @@ public sealed class PromptAndRuntimeTests Task.FromResult(new OllamaResult("done", 1, 1)); public Task<OllamaResult> ChatStructuredAsync(IReadOnlyList<OllamaMessage> messages, double? temperatureOverride, JsonElement format, CancellationToken cancellationToken) { - string planJson = """{"tasks":[{"description":"run build","dependencies":[],"budget_class":"4K","optimistic_tokens":512,"most_likely_tokens":2048,"pessimistic_tokens":4096,"requires_external_effect":true,"required_effect_tools":["process_run"]}]}"""; + string planJson = """{"tasks":[{"description":"run build","dependencies":[],"budget_class":"4K","optimistic_tokens":512,"most_likely_tokens":2048,"pessimistic_tokens":4096,"requires_external_effect":true,"required_tools":["process_run"]}]}"""; return Task.FromResult(new OllamaResult(planJson, 1, 1)); } public Task<OllamaResult> ChatWithToolsAsync(IReadOnlyList<OllamaMessage> messages, double? temperatureOverride, IReadOnlyList<InternalToolDefinition> tools, CancellationToken cancellationToken) @@ -547,7 +587,7 @@ public sealed class PromptAndRuntimeTests public Task<OllamaResult> ChatStructuredAsync(IReadOnlyList<OllamaMessage> messages, double? temperatureOverride, JsonElement format, CancellationToken cancellationToken) { - string planJson = """{"tasks":[{"description":"research regional curry","dependencies":[],"budget_class":"4K","optimistic_tokens":512,"most_likely_tokens":2048,"pessimistic_tokens":4096,"requires_external_effect":true,"required_effect_tools":["file_edit"]}]}"""; + string planJson = """{"tasks":[{"description":"research regional curry","dependencies":[],"budget_class":"4K","optimistic_tokens":512,"most_likely_tokens":2048,"pessimistic_tokens":4096,"requires_external_effect":true,"required_tools":["file_edit"]}]}"""; return Task.FromResult(new OllamaResult(planJson, 1, 1)); } @@ -592,7 +632,7 @@ public sealed class PromptAndRuntimeTests public Task<OllamaResult> ChatStructuredAsync(IReadOnlyList<OllamaMessage> messages, double? temperatureOverride, JsonElement format, bool? thinkOverride, CancellationToken cancellationToken) { int plan = Interlocked.Increment(ref _plans); - string json = $$"""{"tasks":[{"description":"search-{{plan}}","dependencies":[],"budget_class":"4K","optimistic_tokens":10,"most_likely_tokens":20,"pessimistic_tokens":30,"requires_external_effect":true,"required_effect_tools":["file_edit"]}]}"""; + string json = $$"""{"tasks":[{"description":"search-{{plan}}","dependencies":[],"budget_class":"4K","optimistic_tokens":10,"most_likely_tokens":20,"pessimistic_tokens":30,"requires_external_effect":true,"required_tools":["file_edit"]}]}"""; return Task.FromResult(new OllamaResult(json, 1, 1)); } public Task<OllamaResult> ChatWithToolsAsync(IReadOnlyList<OllamaMessage> messages, double? temperatureOverride, IReadOnlyList<InternalToolDefinition> tools, CancellationToken cancellationToken) |