From 6c3458a5ec37a5b69957ad17c84de58984698f25 Mon Sep 17 00:00:00 2001 From: tanaka takashi Date: Tue, 11 Aug 2026 21:44:53 +0900 Subject: Audit and align v1 runtime design --- tests/Agent.Core.Tests/PromptAndRuntimeTests.cs | 333 +++++++++++++++++++++++- 1 file changed, 323 insertions(+), 10 deletions(-) (limited to 'tests/Agent.Core.Tests/PromptAndRuntimeTests.cs') diff --git a/tests/Agent.Core.Tests/PromptAndRuntimeTests.cs b/tests/Agent.Core.Tests/PromptAndRuntimeTests.cs index 130097e..85aba56 100644 --- a/tests/Agent.Core.Tests/PromptAndRuntimeTests.cs +++ b/tests/Agent.Core.Tests/PromptAndRuntimeTests.cs @@ -45,9 +45,8 @@ public sealed class PromptAndRuntimeTests await runtime.ExecuteAsync(first.TaskId, TestContext.Current.CancellationToken); Assert.True(registry.TryGet(first.TaskId, null, out TaskSnapshot? completed)); Assert.True(completed!.State == AgentTaskState.Completed, completed.Result); - // Only Run-phase tokens are counted via registry.AddUsage (Plan tokens are not added to registry) - Assert.Equal(3, completed.InputTokens); - Assert.Equal(2, completed.OutputTokens); + Assert.Equal(6, completed.InputTokens); + Assert.Equal(4, completed.OutputTokens); TaskSnapshot second = registry.Enqueue("second", first.SessionId, null); Assert.True(registry.Transition(second.TaskId, AgentTaskState.Planning)); @@ -60,8 +59,8 @@ public sealed class PromptAndRuntimeTests Assert.Contains(ollama.Calls[3], message => message.Role == "user" && message.Content == "first"); // First Run returns answer-2 (Plan=answer-1, Run=answer-2 due to call ordering) Assert.Contains(ollama.Calls[3], message => message.Role == "assistant" && message.Content == "answer-2"); - // Plan temperature = 0.1, Run temperature = task.Temperature (0.5 for first, null for second) - Assert.Equal(0.1, ollama.Temperatures[0]); + // The task-scoped temperature is propagated to Planning and Run calls. + Assert.Equal(0.5, ollama.Temperatures[0]); Assert.Equal(0.5, ollama.Temperatures[1]); } @@ -116,7 +115,7 @@ public sealed class PromptAndRuntimeTests Assert.Single(backend.Calls); Assert.Equal("file_edit", backend.Calls[0].Name); Assert.Equal(2, ollama.CallCount); - Assert.Equal(6, ollama.ToolCount); + Assert.Equal(InternalToolCatalog.All.Count, ollama.ToolCount); } [Fact] @@ -173,14 +172,200 @@ public sealed class PromptAndRuntimeTests Assert.True(registry.TryGet(task.TaskId, null, out TaskSnapshot? failed)); Assert.Equal(AgentTaskState.Failed, failed!.State); - Assert.Contains("denied by sandbox policy", failed.Result, StringComparison.OrdinalIgnoreCase); - Assert.Contains("not approved for Read", failed.Result, StringComparison.OrdinalIgnoreCase); - // After replan (max 3 attempts), the task fails. Each replan cycle = 1 ChatWithTools call. - // Initial attempt (1) + 3 replan attempts = 4 ChatWithTools calls. + Assert.Contains("Replan did not change", failed.Result, StringComparison.OrdinalIgnoreCase); + Assert.Equal("policy_denied", failed.LastErrorCode); + Assert.Contains("not approved for Read", failed.LastError, StringComparison.OrdinalIgnoreCase); Assert.True(ollama.CallCount >= 1, $"Expected at least 1 call, got {ollama.CallCount}"); Assert.True(backend.CallCount >= 1, $"Expected at least 1 backend call, got {backend.CallCount}"); } + [Fact] + public async Task Policy_denial_replanning_is_limited_to_three_changed_plans() + { + TaskRegistry registry = new(); + var ollama = new ChangingPolicyPlanOllamaClient(); + var backend = new ChangingPolicyBackend(); + var dispatcher = new InternalToolDispatcher( + new DenyByDefaultPolicy(new HashSet(["file_search"], StringComparer.Ordinal)), backend); + var runtime = new AgentRuntime(registry, ollama, new AgentRuntimeSettings("", "en", [], "sandbox"), dispatcher); + TaskSnapshot task = registry.Enqueue("search", null, null); + registry.Transition(task.TaskId, AgentTaskState.Planning); + + await runtime.ExecuteAsync(task.TaskId, TestContext.Current.CancellationToken); + + Assert.True(registry.TryGet(task.TaskId, null, out TaskSnapshot? failed)); + Assert.Equal(AgentTaskState.Failed, failed!.State); + Assert.Equal(3, failed.Replans); + Assert.Equal(4, backend.CallCount); + Assert.Contains("3 replans", failed.Result, StringComparison.Ordinal); + } + + [Fact] + public async Task Repeated_identical_policy_denial_stops_after_one_replan() + { + TaskRegistry registry = new(); + var ollama = new ChangingPolicyPlanOllamaClient(sameDeny: true); + var backend = new ChangingPolicyBackend(); + var dispatcher = new InternalToolDispatcher( + new DenyByDefaultPolicy(new HashSet(["file_search"], StringComparer.Ordinal)), backend); + var runtime = new AgentRuntime(registry, ollama, new AgentRuntimeSettings("", "en", [], "sandbox"), dispatcher); + TaskSnapshot task = registry.Enqueue("search", null, null); + registry.Transition(task.TaskId, AgentTaskState.Planning); + + await runtime.ExecuteAsync(task.TaskId, TestContext.Current.CancellationToken); + + Assert.True(registry.TryGet(task.TaskId, null, out TaskSnapshot? failed)); + Assert.Equal(AgentTaskState.Failed, failed!.State); + Assert.Equal(1, failed.Replans); + Assert.Equal(2, backend.CallCount); + Assert.Contains("same sandbox policy denial", failed.Result, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public async Task Invalid_structured_plan_retries_once_with_think_disabled() + { + TaskRegistry registry = new(); + var ollama = new ThinkRetryOllamaClient(succeedOnRetry: true); + var runtime = new AgentRuntime(registry, ollama, new AgentRuntimeSettings("", "en", [], "sandbox")); + TaskSnapshot task = registry.Enqueue("analyze", null, 0.7); + registry.Transition(task.TaskId, AgentTaskState.Planning); + + await runtime.ExecuteAsync(task.TaskId, TestContext.Current.CancellationToken); + + Assert.True(registry.TryGet(task.TaskId, null, out TaskSnapshot? completed)); + Assert.Equal(AgentTaskState.Completed, completed!.State); + Assert.Equal([null, false], ollama.ThinkOverrides); + Assert.All(ollama.Temperatures, value => Assert.Equal(0.7, value)); + } + + [Fact] + public async Task Invalid_structured_plan_after_retry_fails_without_running_tools() + { + TaskRegistry registry = new(); + var ollama = new ThinkRetryOllamaClient(succeedOnRetry: false); + var runtime = new AgentRuntime(registry, ollama, new AgentRuntimeSettings("", "en", [], "sandbox")); + TaskSnapshot task = registry.Enqueue("change a file", null, null); + registry.Transition(task.TaskId, AgentTaskState.Planning); + + await runtime.ExecuteAsync(task.TaskId, TestContext.Current.CancellationToken); + + Assert.True(registry.TryGet(task.TaskId, null, out TaskSnapshot? failed)); + Assert.Equal(AgentTaskState.Failed, failed!.State); + Assert.Contains("think=false retry", failed.Result, StringComparison.OrdinalIgnoreCase); + Assert.Equal(0, ollama.RunCalls); + } + + [Fact] + public async Task Failed_context_summary_does_not_delete_original_context() + { + var sessions = new SessionStore(); + AgentSession session = sessions.Create(); + for (int index = 0; index < 10; index++) session.Append(new ConversationMessage("user", $"constraint-{index}")); + var registry = new TaskRegistry(sessions); + var runtime = new AgentRuntime(registry, new SummaryOllamaClient(summarySucceeds: false), + new AgentRuntimeSettings("", "en", [], "sandbox", NumCtx: 100, ContextSummaryThreshold: 0.8)); + TaskSnapshot task = registry.Enqueue("analyze", session.SessionId, null); + registry.Transition(task.TaskId, AgentTaskState.Planning); + + await runtime.ExecuteAsync(task.TaskId, TestContext.Current.CancellationToken); + + Assert.Null(session.Summary); + Assert.Contains(session.Conversation, message => message.Content == "constraint-0"); + } + + [Fact] + public async Task Successful_context_summary_preserves_required_categories_before_compaction() + { + TaskRegistry registry = new(); + var ollama = new SummaryOllamaClient(summarySucceeds: true); + var runtime = new AgentRuntime(registry, ollama, + new AgentRuntimeSettings("", "en", [], "sandbox", NumCtx: 100, ContextSummaryThreshold: 0.8, ConversationKeepLastN: 2)); + TaskSnapshot task = registry.Enqueue("analyze", null, null); + Assert.True(registry.Sessions.TryGet(task.SessionId, out AgentSession? session)); + for (int i = 0; i < 10; i++) session!.Append(new ConversationMessage("user", $"constraint-{i}")); + registry.Transition(task.TaskId, AgentTaskState.Planning); + + await runtime.ExecuteAsync(task.TaskId, TestContext.Current.CancellationToken); + + Assert.Equal("preserved constraints", session!.Summary); + Assert.True(session.Conversation.Count <= 4); + string summaryPrompt = string.Join("\n", ollama.SummaryRequest!.Select(static message => message.Content)); + Assert.Contains("incomplete Tasks", summaryPrompt, StringComparison.Ordinal); + Assert.Contains("security denials", summaryPrompt, StringComparison.Ordinal); + Assert.Contains("verification results", summaryPrompt, StringComparison.Ordinal); + Assert.Contains("constraint-0", summaryPrompt, StringComparison.Ordinal); + } + + [Fact] + public async Task Running_task_becomes_cancelled_only_after_cleanup_finishes() + { + TaskRegistry registry = new(); + var backend = new BlockingCleanupBackend(); + var dispatcher = new InternalToolDispatcher(new DenyByDefaultPolicy(InternalToolCatalog.All.Select(static item => item.Name).ToHashSet(StringComparer.Ordinal)), backend); + var runtime = new AgentRuntime(registry, new CancellableOllamaClient(), new AgentRuntimeSettings("", "en", [], "sandbox"), dispatcher); + TaskSnapshot task = registry.Enqueue("wait", null, null); + registry.Transition(task.TaskId, AgentTaskState.Planning); + + Task execution = runtime.ExecuteAsync(task.TaskId, registry.GetCancellationToken(task.TaskId)); + await backend.Begun.Task.WaitAsync(TestContext.Current.CancellationToken); + Assert.True(registry.Cancel(task.TaskId)); + await backend.CleanupStarted.Task.WaitAsync(TestContext.Current.CancellationToken); + Assert.True(registry.TryGet(task.TaskId, null, out TaskSnapshot? stopping)); + Assert.NotEqual(AgentTaskState.Cancelled, stopping!.State); + + backend.AllowCleanup.TrySetResult(); + await execution; + Assert.True(registry.TryGet(task.TaskId, null, out TaskSnapshot? cancelled)); + Assert.Equal(AgentTaskState.Cancelled, cancelled!.State); + } + + [Fact] + public async Task Cancellation_cleanup_failure_becomes_failed() + { + TaskRegistry registry = new(); + var backend = new BlockingCleanupBackend(failCleanup: true); + var dispatcher = new InternalToolDispatcher(new DenyByDefaultPolicy(InternalToolCatalog.All.Select(static item => item.Name).ToHashSet(StringComparer.Ordinal)), backend); + var runtime = new AgentRuntime(registry, new CancellableOllamaClient(), new AgentRuntimeSettings("", "en", [], "sandbox"), dispatcher); + TaskSnapshot task = registry.Enqueue("wait", null, null); + registry.Transition(task.TaskId, AgentTaskState.Planning); + + Task execution = runtime.ExecuteAsync(task.TaskId, registry.GetCancellationToken(task.TaskId)); + await backend.Begun.Task.WaitAsync(TestContext.Current.CancellationToken); + Assert.True(registry.Cancel(task.TaskId)); + await backend.CleanupStarted.Task.WaitAsync(TestContext.Current.CancellationToken); + backend.AllowCleanup.TrySetResult(); + await execution; + + Assert.True(registry.TryGet(task.TaskId, null, out TaskSnapshot? failed)); + Assert.Equal(AgentTaskState.Failed, failed!.State); + Assert.Equal("cleanup_failed", failed.LastErrorCode); + } + + [Fact] + public async Task Task_progress_current_task_and_critical_path_follow_the_graph() + { + TaskRegistry registry = new(); + var ollama = new TwoTaskProgressOllamaClient(); + var runtime = new AgentRuntime(registry, ollama, new AgentRuntimeSettings("", "en", [], "sandbox")); + TaskSnapshot task = registry.Enqueue("two steps", null, null); + registry.Transition(task.TaskId, AgentTaskState.Planning); + + Task execution = runtime.ExecuteAsync(task.TaskId, registry.GetCancellationToken(task.TaskId)); + await ollama.SecondTaskStarted.Task.WaitAsync(TestContext.Current.CancellationToken); + Assert.True(registry.TryGet(task.TaskId, null, out TaskSnapshot? progress)); + Assert.Equal(1, progress!.CompletedTasks); + Assert.Equal(2, progress.TotalTasks); + Assert.Equal("second", progress.CurrentTask); + Assert.True(progress.CriticalPathTokens > 0); + + ollama.AllowSecondTask.TrySetResult(); + await execution; + Assert.True(registry.TryGet(task.TaskId, null, out TaskSnapshot? completed)); + Assert.Equal(2, completed!.CompletedTasks); + Assert.Equal(2, completed.TotalTasks); + Assert.Null(completed.CurrentTask); + } + private sealed class RecordingOllamaClient : IOllamaClient { public List> Calls { get; } = []; @@ -320,4 +505,132 @@ public sealed class PromptAndRuntimeTests return Task.FromResult(new OllamaResult("", 1, 1, [new OllamaToolCall(new OllamaFunctionCall("file_search", arguments.RootElement.Clone()))])); } } + + private sealed class ThinkRetryOllamaClient(bool succeedOnRetry) : IOllamaClient + { + public List ThinkOverrides { get; } = []; + public List Temperatures { get; } = []; + public int RunCalls { get; private set; } + public Task ChatAsync(IReadOnlyList messages, double? temperatureOverride, CancellationToken cancellationToken) => + Task.FromResult(new OllamaResult("done", 1, 1)); + public Task ChatStructuredAsync(IReadOnlyList messages, double? temperatureOverride, JsonElement format, bool? thinkOverride, CancellationToken cancellationToken) + { + ThinkOverrides.Add(thinkOverride); + Temperatures.Add(temperatureOverride); + string content = ThinkOverrides.Count == 2 && succeedOnRetry + ? """{"tasks":[{"description":"analyze","dependencies":[],"budget_class":"4K","optimistic_tokens":10,"most_likely_tokens":20,"pessimistic_tokens":30,"requires_external_effect":false}]}""" + : "not-json"; + return Task.FromResult(new OllamaResult(content, 1, 1)); + } + public Task ChatWithToolsAsync(IReadOnlyList messages, double? temperatureOverride, IReadOnlyList tools, CancellationToken cancellationToken) + { + RunCalls++; + Temperatures.Add(temperatureOverride); + return Task.FromResult(new OllamaResult("done", 1, 1)); + } + } + + private sealed class ChangingPolicyPlanOllamaClient(bool sameDeny = false) : IOllamaClient + { + private int _plans; + private int _runs; + public Task ChatAsync(IReadOnlyList messages, double? temperatureOverride, CancellationToken cancellationToken) => + Task.FromResult(new OllamaResult("unused", 1, 1)); + public Task ChatStructuredAsync(IReadOnlyList messages, double? temperatureOverride, JsonElement format, bool? thinkOverride, CancellationToken cancellationToken) + { + int plan = Interlocked.Increment(ref _plans); + string json = $$"""{"tasks":[{"description":"search-{{plan}}","dependencies":[],"budget_class":"4K","optimistic_tokens":10,"most_likely_tokens":20,"pessimistic_tokens":30,"requires_external_effect":true}]}"""; + return Task.FromResult(new OllamaResult(json, 1, 1)); + } + public Task ChatWithToolsAsync(IReadOnlyList messages, double? temperatureOverride, IReadOnlyList tools, CancellationToken cancellationToken) + { + int run = Interlocked.Increment(ref _runs); + string root = sameDeny ? "C:/denied/same" : $"C:/denied/{run}"; + using JsonDocument arguments = JsonDocument.Parse($$"""{"root":{{JsonSerializer.Serialize(root)}},"mode":"name","query":"*.txt"}"""); + return Task.FromResult(new OllamaResult("", 1, 1, [new OllamaToolCall(new OllamaFunctionCall("file_search", arguments.RootElement.Clone()))])); + } + } + + private sealed class ChangingPolicyBackend : IInternalToolBackend + { + public int CallCount { get; private set; } + public Task ExecuteAsync(Guid taskId, string toolName, JsonElement arguments, CancellationToken cancellationToken) + { + CallCount++; + string root = arguments.GetProperty("root").GetString()!; + return Task.FromResult(new ToolExecutionResult(false, $"denied: {root}", false, root, RetryAllowed: false)); + } + } + + private sealed class SummaryOllamaClient(bool summarySucceeds) : IOllamaClient + { + public IReadOnlyList? SummaryRequest { get; private set; } + public Task ChatAsync(IReadOnlyList messages, double? temperatureOverride, CancellationToken cancellationToken) => + Task.FromResult(new OllamaResult("done", 1, 1)); + public Task ChatStructuredAsync(IReadOnlyList messages, double? temperatureOverride, JsonElement format, bool? thinkOverride, CancellationToken cancellationToken) + { + if (format.GetProperty("properties").TryGetProperty("summary", out _)) + { + SummaryRequest = messages.ToArray(); + if (!summarySucceeds) throw new InvalidOperationException("summary failed"); + return Task.FromResult(new OllamaResult("""{"summary":"preserved constraints"}""", 1, 1)); + } + return Task.FromResult(new OllamaResult("""{"tasks":[{"description":"analyze","dependencies":[],"budget_class":"4K","optimistic_tokens":10,"most_likely_tokens":20,"pessimistic_tokens":30,"requires_external_effect":false}]}""", 1, 1)); + } + public Task ChatWithToolsAsync(IReadOnlyList messages, double? temperatureOverride, IReadOnlyList tools, CancellationToken cancellationToken) => + Task.FromResult(new OllamaResult("done", 1, 1, PromptEvalCount: 90)); + } + + private sealed class CancellableOllamaClient : IOllamaClient + { + public Task ChatAsync(IReadOnlyList messages, double? temperatureOverride, CancellationToken cancellationToken) => + Task.FromResult(new OllamaResult("done", 1, 1)); + public Task ChatStructuredAsync(IReadOnlyList messages, double? temperatureOverride, JsonElement format, bool? thinkOverride, CancellationToken cancellationToken) => + Task.FromResult(new OllamaResult("""{"tasks":[{"description":"wait","dependencies":[],"budget_class":"4K","optimistic_tokens":10,"most_likely_tokens":20,"pessimistic_tokens":30,"requires_external_effect":false}]}""", 1, 1)); + public async Task ChatWithToolsAsync(IReadOnlyList messages, double? temperatureOverride, IReadOnlyList tools, CancellationToken cancellationToken) + { + await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken); + throw new InvalidOperationException("unreachable"); + } + } + + private sealed class TwoTaskProgressOllamaClient : IOllamaClient + { + private int _runCalls; + public TaskCompletionSource SecondTaskStarted { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + public TaskCompletionSource AllowSecondTask { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + public Task ChatAsync(IReadOnlyList messages, double? temperatureOverride, CancellationToken cancellationToken) => + Task.FromResult(new OllamaResult("done", 1, 1)); + public Task ChatStructuredAsync(IReadOnlyList messages, double? temperatureOverride, JsonElement format, bool? thinkOverride, CancellationToken cancellationToken) => + Task.FromResult(new OllamaResult("""{"tasks":[{"description":"first","dependencies":[],"budget_class":"4K","optimistic_tokens":10,"most_likely_tokens":20,"pessimistic_tokens":30,"requires_external_effect":false},{"description":"second","dependencies":[0],"budget_class":"4K","optimistic_tokens":10,"most_likely_tokens":20,"pessimistic_tokens":30,"requires_external_effect":false}]}""", 1, 1)); + public async Task ChatWithToolsAsync(IReadOnlyList messages, double? temperatureOverride, IReadOnlyList tools, CancellationToken cancellationToken) + { + if (Interlocked.Increment(ref _runCalls) == 2) + { + SecondTaskStarted.TrySetResult(); + await AllowSecondTask.Task.WaitAsync(cancellationToken); + } + return new OllamaResult("done", 1, 1); + } + } + + private sealed class BlockingCleanupBackend(bool failCleanup = false) : IInternalToolBackend + { + public TaskCompletionSource Begun { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + public TaskCompletionSource CleanupStarted { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + public TaskCompletionSource AllowCleanup { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + public Task BeginTaskAsync(Guid taskId, Guid sessionId, CancellationToken cancellationToken) + { + Begun.TrySetResult(); + return Task.CompletedTask; + } + public Task ExecuteAsync(Guid taskId, string toolName, JsonElement arguments, CancellationToken cancellationToken) => + Task.FromResult(new ToolExecutionResult(true, "unused", false)); + public async Task EndTaskAsync(Guid taskId, bool success, CancellationToken cancellationToken) + { + CleanupStarted.TrySetResult(); + await AllowCleanup.Task.WaitAsync(cancellationToken); + if (failCleanup) throw new InvalidOperationException("cleanup failed"); + } + } } -- cgit v1.2.3