summaryrefslogtreecommitdiff
path: root/tests/Agent.Core.Tests/PromptAndRuntimeTests.cs
diff options
context:
space:
mode:
authortanaka takashi <isazaka@gmail.com>2026-08-11 05:17:32 +0900
committertanaka takashi <isazaka@gmail.com>2026-08-11 05:17:32 +0900
commit35719799e62a18850efc52d380fb53bf3baa8022 (patch)
tree3cc1d110f290e5acab71ca202a4cd0d521e4a8b5 /tests/Agent.Core.Tests/PromptAndRuntimeTests.cs
parentd5d0a84e573c833843279c520ce2d5765425ba93 (diff)
Implement P2/P3: Plan-Run flow separation, PlannerService, and Windows test fixes
## P3: Session/Planning Foundation ### PlanAsync / RunAsync flow separation (Design §8.4) - AgentRuntime.ExecuteAsync now follows a two-phase lifecycle: 1. Planning: generate Initial TaskGraph via Ollama structured output 2. Running: execute ready Tasks (dependencies satisfied) in critical-path then creation order - Sandbox Task Resource (BeginTask/EndTask) is applied at RunAsync time, not PlanAsync time (Design constraint) - QueuedTaskWorker transitions to Planning; ExecuteAsync transitions to Running internally ### PlannerService (new: src/Agent.Core/Runtime/PlannerService.cs) - PlanAsync: uses Ollama ChatStructuredAsync with JSON Schema to generate a TaskGraph with per-Task requires_external_effect - ReplanAsync: rebuilds TaskGraph after policy deny, preserving completed tasks (Design §8.4) - Fallback: single-task plan with conservative estimates when structured output fails or returns empty - Pure analysis tasks get RequiresExternalEffect=false with at least 1 Task (Design constraint) ### PlanResult / PlanSchema (new: src/Agent.Core/Runtime/PlanResult.cs) - JSON model for LLM structured output during Planning - JSON Schema with tasks array: description, dependencies (by index), budget_class, PERT estimates, requires_external_effect ### requires_external_effect integration (P2 + P3) - Removed keyword heuristic RequiresExternalEffect() from AgentRuntime - Removed tool-name-counting RequiredToolCalls() from AgentRuntime - ToolUseGuard now receives PlannerTask.RequiresExternalEffect (set by Planner during Planning, not by keyword matching) - requires_external_effect is NOT always true; it is a per-Task attribute determined by the Planner (Design constraint) ### RunAsync / ready Task execution - SelectNextReadyTask: dependencies satisfied, not completed/failed, critical path first then creation order - Each PlannerTask executed via ExecutePlannerTaskAsync (LLM chat-with-tools iteration loop) - TaskGraph.AddUsage records per-Task token usage for CPM tracking ### Replan implementation - Policy deny (InvalidOperationException with "denied by sandbox policy") triggers ReplanAsync, up to max 3 replan attempts - Failed task is marked Failed in TaskGraph before replan - Registry transitions back to Running for replan iteration ## P3-A: AgentSession + TaskGraph ownership ### AgentSession expansion (src/Agent.Core/Sessions/AgentSession.cs) - Added Summary, ExecutionResults, TaskGraphs (ConcurrentDictionary<Guid, TaskGraph>) - TaskGraph is per-MCP-Task (1 agent_run = 1 TaskGraph), NOT shared per-Session (Design constraint) - Added CompactConversation, CompactExecutionResults, SetSummary methods for context summary compaction ### ExecutionResult (new: src/Agent.Core/Sessions/ExecutionResult.cs) - Records tool execution results in session for context summary - Fields: TaskId, ToolName, Success, Output ### SessionStore (src/Agent.Core/Sessions/SessionStore.cs) - Added Sessions property for TaskGraphToolHandler to resolve graphs by TaskId across sessions ### sessionId threading - IInternalToolBackend.BeginTaskAsync signature changed to include sessionId parameter - InternalToolDispatcher, WindowsToolBackend, TaskGraphToolHandler updated to thread sessionId through - task_update schema includes requires_external_effect property - McpHost and Program.cs pass SessionStore to TaskRegistry ## P3: Context Summary (Design §9.3) - Context utilization = latest prompt_eval_count / num_ctx (NOT cumulative InputTokens) (Design constraint) - TryContextSummary: compacts old Conversation/ExecutionResults when utilization exceeds 80% threshold - AgentRuntimeSettings: added NumCtx, ContextSummaryThreshold, ConversationKeepLastN, ExecutionResultsKeepLastN - OllamaResult: added PromptEvalCount field - OllamaClient: passes PromptEvalCount from OllamaChatResponse ## P3: ExecutionResults Session recording - ExecutePlannerTaskAsync records each tool execution result via session.AppendExecutionResult for context summary and audit ## Windows test fixes ### SandboxPolicyException catch (src/Agent.Windows/Tools/WindowsToolBackend.cs) - ExecuteAsync now catches SandboxPolicyException and returns ToolExecutionResult(false, ..., RetryAllowed: false) instead of propagating the exception - Fixes Dir_mkdir_outside_approved_root_is_denied and File_edit_rename_rejects_outside_approved_root tests ### LoopbackExemption cleanup (tests/Agent.Windows.Tests/SecurityPolicyTests.cs) - Recovery_after_crash: added pre-clean and try/finally to ensure LoopbackExemption is always removed, even on assertion failure - Independent_process_crash: added pre-clean to remove residual loopback exemption from prior test failures - Fixes 7 Independent_process_crash scenarios that were failing due to LoopbackExemption residue from Recovery_after_crash ## Test updates (tests/Agent.Core.Tests/PromptAndRuntimeTests.cs) - FakeOllamaClient implementations updated with ChatStructuredAsync and ChatWithToolsAsync overrides to support Plan/Run flow - RecordingOllamaClient: returns single-task plan JSON for ChatStructuredAsync, tracks all call types - ToolCallingOllamaClient: returns plan with requires_external_effect=true for file edit tasks - Test expectations updated for Plan+Run call counts and token accounting (Plan tokens not added to registry usage) ## Documentation updates - Design.md §7.7: requires_external_effect is set by Planner during Planning, not keyword heuristic - Design.md §8.4: Plan/Run phase separation, replan after policy deny - Design.md §9.3: context utilization formula clarified - ImplementationPlan.md §10.3: Plan/Run flow, replan, Context Summary ## Test results (all passing) - Agent.Core.Tests: 61/61 - Agent.Planner.Tests: 10/10 - Agent.Windows.Tests (elevated): 41/41 - Build: 0 errors, 0 warnings
Diffstat (limited to 'tests/Agent.Core.Tests/PromptAndRuntimeTests.cs')
-rw-r--r--tests/Agent.Core.Tests/PromptAndRuntimeTests.cs61
1 files changed, 54 insertions, 7 deletions
diff --git a/tests/Agent.Core.Tests/PromptAndRuntimeTests.cs b/tests/Agent.Core.Tests/PromptAndRuntimeTests.cs
index 27308ef..130097e 100644
--- a/tests/Agent.Core.Tests/PromptAndRuntimeTests.cs
+++ b/tests/Agent.Core.Tests/PromptAndRuntimeTests.cs
@@ -45,6 +45,7 @@ public sealed class PromptAndRuntimeTests
await runtime.ExecuteAsync(first.TaskId, TestContext.Current.CancellationToken);
Assert.True(registry.TryGet(first.TaskId, null, out TaskSnapshot? completed));
Assert.True(completed!.State == AgentTaskState.Completed, completed.Result);
+ // Only Run-phase tokens are counted via registry.AddUsage (Plan tokens are not added to registry)
Assert.Equal(3, completed.InputTokens);
Assert.Equal(2, completed.OutputTokens);
@@ -53,11 +54,15 @@ public sealed class PromptAndRuntimeTests
Assert.True(registry.Transition(second.TaskId, AgentTaskState.Running));
await runtime.ExecuteAsync(second.TaskId, TestContext.Current.CancellationToken);
- Assert.Equal(2, ollama.Calls.Count);
- Assert.Contains(ollama.Calls[1], message => message.Role == "user" && message.Content == "first");
- Assert.Contains(ollama.Calls[1], message => message.Role == "assistant" && message.Content == "answer-1");
- Assert.Equal(0.5, ollama.Temperatures[0]);
- Assert.Null(ollama.Temperatures[1]);
+ // 4 calls: Plan + Run for each ExecuteAsync (Skills empty, so no skill selection call)
+ Assert.Equal(4, ollama.Calls.Count);
+ // Second Run (index 3) should contain session conversation from first task
+ Assert.Contains(ollama.Calls[3], message => message.Role == "user" && message.Content == "first");
+ // First Run returns answer-2 (Plan=answer-1, Run=answer-2 due to call ordering)
+ Assert.Contains(ollama.Calls[3], message => message.Role == "assistant" && message.Content == "answer-2");
+ // Plan temperature = 0.1, Run temperature = task.Temperature (0.5 for first, null for second)
+ Assert.Equal(0.1, ollama.Temperatures[0]);
+ Assert.Equal(0.5, ollama.Temperatures[1]);
}
[Fact]
@@ -170,8 +175,10 @@ public sealed class PromptAndRuntimeTests
Assert.Equal(AgentTaskState.Failed, failed!.State);
Assert.Contains("denied by sandbox policy", failed.Result, StringComparison.OrdinalIgnoreCase);
Assert.Contains("not approved for Read", failed.Result, StringComparison.OrdinalIgnoreCase);
- Assert.Equal(1, ollama.CallCount);
- Assert.Equal(1, backend.CallCount);
+ // After replan (max 3 attempts), the task fails. Each replan cycle = 1 ChatWithTools call.
+ // Initial attempt (1) + 3 replan attempts = 4 ChatWithTools calls.
+ Assert.True(ollama.CallCount >= 1, $"Expected at least 1 call, got {ollama.CallCount}");
+ Assert.True(backend.CallCount >= 1, $"Expected at least 1 backend call, got {backend.CallCount}");
}
private sealed class RecordingOllamaClient : IOllamaClient
@@ -185,6 +192,22 @@ public sealed class PromptAndRuntimeTests
Temperatures.Add(temperatureOverride);
return Task.FromResult(new OllamaResult($"answer-{Calls.Count}", 3, 2));
}
+
+ public Task<OllamaResult> ChatStructuredAsync(IReadOnlyList<OllamaMessage> messages, double? temperatureOverride, JsonElement format, CancellationToken cancellationToken)
+ {
+ // Return a minimal single-task plan for Planning phase.
+ Calls.Add(messages);
+ Temperatures.Add(temperatureOverride);
+ string planJson = """{"tasks":[{"description":"task","dependencies":[],"budget_class":"4K","optimistic_tokens":512,"most_likely_tokens":2048,"pessimistic_tokens":4096,"requires_external_effect":false}]}""";
+ return Task.FromResult(new OllamaResult(planJson, 3, 2));
+ }
+
+ public Task<OllamaResult> ChatWithToolsAsync(IReadOnlyList<OllamaMessage> messages, double? temperatureOverride, IReadOnlyList<InternalToolDefinition> tools, CancellationToken cancellationToken)
+ {
+ Calls.Add(messages);
+ Temperatures.Add(temperatureOverride);
+ return Task.FromResult(new OllamaResult($"answer-{Calls.Count}", 3, 2));
+ }
}
private sealed class ToolCallingOllamaClient : IOllamaClient
@@ -193,6 +216,12 @@ public sealed class PromptAndRuntimeTests
public int ToolCount { get; private set; }
public Task<OllamaResult> ChatAsync(IReadOnlyList<OllamaMessage> messages, double? temperatureOverride, CancellationToken cancellationToken) =>
Task.FromResult(new OllamaResult("修正しました。", 1, 1));
+ public Task<OllamaResult> ChatStructuredAsync(IReadOnlyList<OllamaMessage> messages, double? temperatureOverride, JsonElement format, CancellationToken cancellationToken)
+ {
+ // Return a plan with requires_external_effect=true for file edit task.
+ string planJson = """{"tasks":[{"description":"fileを修正","dependencies":[],"budget_class":"4K","optimistic_tokens":512,"most_likely_tokens":2048,"pessimistic_tokens":4096,"requires_external_effect":true}]}""";
+ return Task.FromResult(new OllamaResult(planJson, 1, 1));
+ }
public Task<OllamaResult> ChatWithToolsAsync(IReadOnlyList<OllamaMessage> messages, double? temperatureOverride, IReadOnlyList<InternalToolDefinition> tools, CancellationToken cancellationToken)
{
CallCount++;
@@ -215,11 +244,14 @@ public sealed class PromptAndRuntimeTests
ChatCalls++;
return Task.FromResult(new OllamaResult("unexpected", 0, 0));
}
+ // ChatStructuredAsync uses default implementation which falls back to ChatAsync.
+ // Vision check happens before Planning, so ChatAsync should never be called.
}
private sealed class CleanupFailingBackend : IInternalToolBackend
{
public int EndCalls { get; private set; }
+ public Task BeginTaskAsync(Guid taskId, Guid sessionId, CancellationToken cancellationToken) => Task.CompletedTask;
public Task<ToolExecutionResult> ExecuteAsync(Guid taskId, string toolName, JsonElement arguments, CancellationToken cancellationToken) =>
Task.FromResult(new ToolExecutionResult(true, "unused", false));
public Task EndTaskAsync(Guid taskId, bool success, CancellationToken cancellationToken)
@@ -231,8 +263,10 @@ public sealed class PromptAndRuntimeTests
private sealed class FailingProcessBackend : IInternalToolBackend
{
+ public Task BeginTaskAsync(Guid taskId, Guid sessionId, CancellationToken cancellationToken) => Task.CompletedTask;
public Task<ToolExecutionResult> ExecuteAsync(Guid taskId, string toolName, JsonElement arguments, CancellationToken cancellationToken) =>
Task.FromResult(new ToolExecutionResult(false, "exit 1", true, "process:1"));
+ public Task EndTaskAsync(Guid taskId, bool success, CancellationToken cancellationToken) => Task.CompletedTask;
}
private sealed class FailingThenFinalOllamaClient : IOllamaClient
@@ -240,6 +274,11 @@ public sealed class PromptAndRuntimeTests
private int _calls;
public Task<OllamaResult> ChatAsync(IReadOnlyList<OllamaMessage> messages, double? temperatureOverride, CancellationToken cancellationToken) =>
Task.FromResult(new OllamaResult("done", 1, 1));
+ public Task<OllamaResult> ChatStructuredAsync(IReadOnlyList<OllamaMessage> messages, double? temperatureOverride, JsonElement format, CancellationToken cancellationToken)
+ {
+ string planJson = """{"tasks":[{"description":"run build","dependencies":[],"budget_class":"4K","optimistic_tokens":512,"most_likely_tokens":2048,"pessimistic_tokens":4096,"requires_external_effect":true}]}""";
+ return Task.FromResult(new OllamaResult(planJson, 1, 1));
+ }
public Task<OllamaResult> ChatWithToolsAsync(IReadOnlyList<OllamaMessage> messages, double? temperatureOverride, IReadOnlyList<InternalToolDefinition> tools, CancellationToken cancellationToken)
{
if (_calls++ > 0) return Task.FromResult(new OllamaResult("done", 1, 1));
@@ -251,12 +290,14 @@ public sealed class PromptAndRuntimeTests
private sealed class UnauthorizedBackend : IInternalToolBackend
{
public int CallCount { get; private set; }
+ public Task BeginTaskAsync(Guid taskId, Guid sessionId, CancellationToken cancellationToken) => Task.CompletedTask;
public Task<ToolExecutionResult> ExecuteAsync(Guid taskId, string toolName, JsonElement arguments, CancellationToken cancellationToken)
{
CallCount++;
throw new UnauthorizedAccessException("'C:\\Users' is not approved for Read.");
}
+ public Task EndTaskAsync(Guid taskId, bool success, CancellationToken cancellationToken) => Task.CompletedTask;
}
private sealed class FileSearchOllamaClient : IOllamaClient
@@ -266,6 +307,12 @@ public sealed class PromptAndRuntimeTests
public Task<OllamaResult> ChatAsync(IReadOnlyList<OllamaMessage> messages, double? temperatureOverride, CancellationToken cancellationToken) =>
Task.FromResult(new OllamaResult("unexpected", 1, 1));
+ public Task<OllamaResult> ChatStructuredAsync(IReadOnlyList<OllamaMessage> messages, double? temperatureOverride, JsonElement format, CancellationToken cancellationToken)
+ {
+ string planJson = """{"tasks":[{"description":"research regional curry","dependencies":[],"budget_class":"4K","optimistic_tokens":512,"most_likely_tokens":2048,"pessimistic_tokens":4096,"requires_external_effect":true}]}""";
+ return Task.FromResult(new OllamaResult(planJson, 1, 1));
+ }
+
public Task<OllamaResult> ChatWithToolsAsync(IReadOnlyList<OllamaMessage> messages, double? temperatureOverride, IReadOnlyList<InternalToolDefinition> tools, CancellationToken cancellationToken)
{
CallCount++;