summaryrefslogtreecommitdiff
path: root/tests
diff options
context:
space:
mode:
authortanaka takashi <isazaka@gmail.com>2026-08-11 21:44:53 +0900
committertanaka takashi <isazaka@gmail.com>2026-08-11 21:44:53 +0900
commit6c3458a5ec37a5b69957ad17c84de58984698f25 (patch)
treebe5c5716e7488fcf1bc1a58de0ded0b9238f34cb /tests
parentb5ccecca01007f2a2459265df6d749fbfd31aa84 (diff)
Audit and align v1 runtime design
Diffstat (limited to 'tests')
-rw-r--r--tests/Agent.Core.Tests/ConfigurationTests.cs57
-rw-r--r--tests/Agent.Core.Tests/InternalToolTests.cs3
-rw-r--r--tests/Agent.Core.Tests/ObservabilityTests.cs56
-rw-r--r--tests/Agent.Core.Tests/PromptAndRuntimeTests.cs333
-rw-r--r--tests/Agent.Core.Tests/TaskRegistryTests.cs17
-rw-r--r--tests/Agent.Windows.Tests/CrashScenarioBootstrap.cs2
-rw-r--r--tests/Agent.Windows.Tests/SecurityPolicyTests.cs13
7 files changed, 465 insertions, 16 deletions
diff --git a/tests/Agent.Core.Tests/ConfigurationTests.cs b/tests/Agent.Core.Tests/ConfigurationTests.cs
index 79c746d..e84d132 100644
--- a/tests/Agent.Core.Tests/ConfigurationTests.cs
+++ b/tests/Agent.Core.Tests/ConfigurationTests.cs
@@ -44,6 +44,63 @@ public sealed class ConfigurationTests
}
[Fact]
+ public void CommandLine_defaults_to_serve_when_command_is_omitted()
+ {
+ Assert.Equal(AppCommand.Serve, CommandLine.Parse([]).Command);
+ CommandLine withOption = CommandLine.Parse(["--lang", "ja"]);
+ Assert.Equal(AppCommand.Serve, withOption.Command);
+ Assert.Equal("ja", withOption.Language);
+ }
+
+ [Fact]
+ public void Local_configuration_recursively_merges_maps_and_replaces_sequences()
+ {
+ using var directory = new TestDirectory();
+ string config = CreateValidConfiguration(directory);
+ directory.File("agent.local.yaml", """
+ application:
+ language: ja
+ sandbox:
+ filesystem:
+ read: [C:/local-only]
+ """);
+
+ AgentConfiguration result = ConfigurationLoader.Load(config);
+
+ Assert.Equal("ja", result.Application.Language);
+ Assert.Equal(["C:/local-only"], result.Sandbox.Filesystem.Read);
+ Assert.Equal("model", result.Ollama.Model);
+ }
+
+ [Fact]
+ public void Unknown_local_configuration_property_is_fatal()
+ {
+ using var directory = new TestDirectory();
+ string config = CreateValidConfiguration(directory);
+ directory.File("agent.local.yaml", "unknown_local: true\n");
+
+ Assert.Throws<ConfigurationException>(() => ConfigurationLoader.Load(config));
+ }
+
+ [Fact]
+ public void Execute_file_without_process_approval_reports_canonical_keys()
+ {
+ using var directory = new TestDirectory();
+ string config = CreateValidConfiguration(directory);
+ string executable = directory.File("tools/tool.exe", "fixture").Replace('\\', '/');
+ directory.File("agent.local.yaml", $$"""
+ sandbox:
+ filesystem:
+ execute: [{{executable}}]
+ """);
+
+ ConfigurationException error = Assert.Throws<ConfigurationException>(() => ConfigurationLoader.Load(config));
+ Assert.Contains("sandbox.filesystem.execute", error.Message, StringComparison.Ordinal);
+ Assert.Contains("sandbox.process.approvals", error.Message, StringComparison.Ordinal);
+ Assert.Contains(Path.GetFullPath(executable), error.Message, StringComparison.OrdinalIgnoreCase);
+ }
+
+ [Fact]
public void SkillLoader_loads_standard_skill_directories_only()
{
using var directory = new TestDirectory();
diff --git a/tests/Agent.Core.Tests/InternalToolTests.cs b/tests/Agent.Core.Tests/InternalToolTests.cs
index 94bdde9..4128863 100644
--- a/tests/Agent.Core.Tests/InternalToolTests.cs
+++ b/tests/Agent.Core.Tests/InternalToolTests.cs
@@ -6,7 +6,7 @@ namespace Agent.Core.Tests;
public sealed class InternalToolTests
{
[Fact]
- public void Catalog_exposes_six_strict_tool_groups()
+ public void Catalog_matches_the_design_and_uses_strict_schemas()
{
Assert.Equal(["file_search", "file_read", "file_edit", "process_run", "task_update", "dir"], InternalToolCatalog.All.Select(x => x.Name));
foreach (InternalToolDefinition tool in InternalToolCatalog.All) Assert.False(tool.Parameters.GetProperty("additionalProperties").GetBoolean());
@@ -19,6 +19,7 @@ public sealed class InternalToolTests
[InlineData("file_search", "{\"root\":\".\",\"mode\":\"text\",\"query\":\"x\",\"max_results\":1001}")]
[InlineData("process_run", "{\"executable\":\"x\",\"args\":\"bad\",\"cwd\":\".\"}")]
[InlineData("task_update", "{\"operation\":\"add\",\"description\":\"missing estimate\"}")]
+ [InlineData("task_update", "{\"operation\":\"set_state\",\"task_id\":null}")]
[InlineData("file_edit", "{\"path\":\"a\",\"operation\":\"rename\"}")]
[InlineData("file_edit", "{\"path\":\"a\",\"operation\":\"rename\",\"new_path\":\"\"}")]
public void Malformed_arguments_are_rejected(string tool, string json)
diff --git a/tests/Agent.Core.Tests/ObservabilityTests.cs b/tests/Agent.Core.Tests/ObservabilityTests.cs
index c259eb0..8231ab9 100644
--- a/tests/Agent.Core.Tests/ObservabilityTests.cs
+++ b/tests/Agent.Core.Tests/ObservabilityTests.cs
@@ -1,14 +1,70 @@
using Agent.App.Logging;
using Agent.Core.Localization;
+using Agent.Core.Ollama;
+using Agent.Core.Runtime;
+using Agent.Core.Tools;
using Agent.Mcp;
using System.Diagnostics;
using System.Text;
+using System.Text.Json;
namespace Agent.Core.Tests;
public sealed class ObservabilityTests
{
[Fact]
+ public void Llm_log_carries_correlation_usage_and_tool_iteration_fields()
+ {
+ using var directory = new TestDirectory();
+ Guid sessionId = Guid.NewGuid();
+ Guid taskId = Guid.NewGuid();
+ Guid plannerTaskId = Guid.NewGuid();
+ Guid callId = Guid.NewGuid();
+ var context = new AgentLlmCallContext(callId, sessionId, taskId, plannerTaskId, "replan", 7, 2, "model-a");
+ using (var observer = new JsonLinesRuntimeObserver(directory.Path, 1024 * 1024, 2))
+ {
+ observer.LlmRequest(context, [new OllamaMessage("user", "request")]);
+ observer.LlmResponse(context, new OllamaResult("response", 11, 13));
+ observer.ToolResult(context, "file_read", new ToolExecutionResult(true, "ok", false));
+ observer.LlmFailure(context, "planning_failed", "invalid plan");
+ }
+
+ JsonElement[] entries = File.ReadLines(Path.Combine(directory.Path, "llm.jsonl"))
+ .Select(static line => JsonDocument.Parse(line).RootElement.Clone()).ToArray();
+ Assert.Equal(4, entries.Length);
+ Assert.All(entries, entry =>
+ {
+ Assert.Equal(callId, entry.GetProperty("call_id").GetGuid());
+ Assert.Equal(taskId, entry.GetProperty("task_id").GetGuid());
+ Assert.Equal(sessionId, entry.GetProperty("session_id").GetGuid());
+ Assert.Equal(plannerTaskId, entry.GetProperty("planner_task_id").GetGuid());
+ Assert.Equal(7, entry.GetProperty("iteration").GetInt32());
+ Assert.Equal(2, entry.GetProperty("replan").GetInt32());
+ });
+ Assert.Equal(11, entries[1].GetProperty("input_tokens").GetInt64());
+ Assert.Equal("file_read", entries[2].GetProperty("tool_name").GetString());
+ Assert.Equal("llm.failure", entries[3].GetProperty("event_name").GetString());
+ Assert.Equal("planning_failed", entries[3].GetProperty("error_code").GetString());
+ }
+
+ [Fact]
+ public void Read_agent_log_script_exposes_read_only_filters()
+ {
+ DirectoryInfo? root = new(AppContext.BaseDirectory);
+ while (root is not null && !File.Exists(Path.Combine(root.FullName, "scripts", "Read-AgentLog.ps1"))) root = root.Parent;
+ Assert.NotNull(root);
+ string scriptPath = Path.Combine(root!.FullName, "scripts", "Read-AgentLog.ps1");
+ string script = File.ReadAllText(scriptPath);
+ Assert.Contains("[string]$TaskId", script, StringComparison.Ordinal);
+ Assert.Contains("[string]$SessionId", script, StringComparison.Ordinal);
+ Assert.Contains("[string]$Event", script, StringComparison.Ordinal);
+ Assert.Contains("[string]$CallId", script, StringComparison.Ordinal);
+ Assert.Contains("[int]$Tail", script, StringComparison.Ordinal);
+ Assert.DoesNotContain("Set-Content", script, StringComparison.OrdinalIgnoreCase);
+ Assert.DoesNotContain("Remove-Item", script, StringComparison.OrdinalIgnoreCase);
+ }
+
+ [Fact]
public void Ready_is_always_visible_even_when_log_threshold_is_fatal()
{
using var directory = new TestDirectory();
diff --git a/tests/Agent.Core.Tests/PromptAndRuntimeTests.cs b/tests/Agent.Core.Tests/PromptAndRuntimeTests.cs
index 130097e..85aba56 100644
--- a/tests/Agent.Core.Tests/PromptAndRuntimeTests.cs
+++ b/tests/Agent.Core.Tests/PromptAndRuntimeTests.cs
@@ -45,9 +45,8 @@ public sealed class PromptAndRuntimeTests
await runtime.ExecuteAsync(first.TaskId, TestContext.Current.CancellationToken);
Assert.True(registry.TryGet(first.TaskId, null, out TaskSnapshot? completed));
Assert.True(completed!.State == AgentTaskState.Completed, completed.Result);
- // Only Run-phase tokens are counted via registry.AddUsage (Plan tokens are not added to registry)
- Assert.Equal(3, completed.InputTokens);
- Assert.Equal(2, completed.OutputTokens);
+ Assert.Equal(6, completed.InputTokens);
+ Assert.Equal(4, completed.OutputTokens);
TaskSnapshot second = registry.Enqueue("second", first.SessionId, null);
Assert.True(registry.Transition(second.TaskId, AgentTaskState.Planning));
@@ -60,8 +59,8 @@ public sealed class PromptAndRuntimeTests
Assert.Contains(ollama.Calls[3], message => message.Role == "user" && message.Content == "first");
// First Run returns answer-2 (Plan=answer-1, Run=answer-2 due to call ordering)
Assert.Contains(ollama.Calls[3], message => message.Role == "assistant" && message.Content == "answer-2");
- // Plan temperature = 0.1, Run temperature = task.Temperature (0.5 for first, null for second)
- Assert.Equal(0.1, ollama.Temperatures[0]);
+ // The task-scoped temperature is propagated to Planning and Run calls.
+ Assert.Equal(0.5, ollama.Temperatures[0]);
Assert.Equal(0.5, ollama.Temperatures[1]);
}
@@ -116,7 +115,7 @@ public sealed class PromptAndRuntimeTests
Assert.Single(backend.Calls);
Assert.Equal("file_edit", backend.Calls[0].Name);
Assert.Equal(2, ollama.CallCount);
- Assert.Equal(6, ollama.ToolCount);
+ Assert.Equal(InternalToolCatalog.All.Count, ollama.ToolCount);
}
[Fact]
@@ -173,14 +172,200 @@ public sealed class PromptAndRuntimeTests
Assert.True(registry.TryGet(task.TaskId, null, out TaskSnapshot? failed));
Assert.Equal(AgentTaskState.Failed, failed!.State);
- Assert.Contains("denied by sandbox policy", failed.Result, StringComparison.OrdinalIgnoreCase);
- Assert.Contains("not approved for Read", failed.Result, StringComparison.OrdinalIgnoreCase);
- // After replan (max 3 attempts), the task fails. Each replan cycle = 1 ChatWithTools call.
- // Initial attempt (1) + 3 replan attempts = 4 ChatWithTools calls.
+ Assert.Contains("Replan did not change", failed.Result, StringComparison.OrdinalIgnoreCase);
+ Assert.Equal("policy_denied", failed.LastErrorCode);
+ Assert.Contains("not approved for Read", failed.LastError, StringComparison.OrdinalIgnoreCase);
Assert.True(ollama.CallCount >= 1, $"Expected at least 1 call, got {ollama.CallCount}");
Assert.True(backend.CallCount >= 1, $"Expected at least 1 backend call, got {backend.CallCount}");
}
+ [Fact]
+ public async Task Policy_denial_replanning_is_limited_to_three_changed_plans()
+ {
+ TaskRegistry registry = new();
+ var ollama = new ChangingPolicyPlanOllamaClient();
+ var backend = new ChangingPolicyBackend();
+ var dispatcher = new InternalToolDispatcher(
+ new DenyByDefaultPolicy(new HashSet<string>(["file_search"], StringComparer.Ordinal)), backend);
+ var runtime = new AgentRuntime(registry, ollama, new AgentRuntimeSettings("", "en", [], "sandbox"), dispatcher);
+ TaskSnapshot task = registry.Enqueue("search", null, null);
+ registry.Transition(task.TaskId, AgentTaskState.Planning);
+
+ await runtime.ExecuteAsync(task.TaskId, TestContext.Current.CancellationToken);
+
+ Assert.True(registry.TryGet(task.TaskId, null, out TaskSnapshot? failed));
+ Assert.Equal(AgentTaskState.Failed, failed!.State);
+ Assert.Equal(3, failed.Replans);
+ Assert.Equal(4, backend.CallCount);
+ Assert.Contains("3 replans", failed.Result, StringComparison.Ordinal);
+ }
+
+ [Fact]
+ public async Task Repeated_identical_policy_denial_stops_after_one_replan()
+ {
+ TaskRegistry registry = new();
+ var ollama = new ChangingPolicyPlanOllamaClient(sameDeny: true);
+ var backend = new ChangingPolicyBackend();
+ var dispatcher = new InternalToolDispatcher(
+ new DenyByDefaultPolicy(new HashSet<string>(["file_search"], StringComparer.Ordinal)), backend);
+ var runtime = new AgentRuntime(registry, ollama, new AgentRuntimeSettings("", "en", [], "sandbox"), dispatcher);
+ TaskSnapshot task = registry.Enqueue("search", null, null);
+ registry.Transition(task.TaskId, AgentTaskState.Planning);
+
+ await runtime.ExecuteAsync(task.TaskId, TestContext.Current.CancellationToken);
+
+ Assert.True(registry.TryGet(task.TaskId, null, out TaskSnapshot? failed));
+ Assert.Equal(AgentTaskState.Failed, failed!.State);
+ Assert.Equal(1, failed.Replans);
+ Assert.Equal(2, backend.CallCount);
+ Assert.Contains("same sandbox policy denial", failed.Result, StringComparison.OrdinalIgnoreCase);
+ }
+
+ [Fact]
+ public async Task Invalid_structured_plan_retries_once_with_think_disabled()
+ {
+ TaskRegistry registry = new();
+ var ollama = new ThinkRetryOllamaClient(succeedOnRetry: true);
+ var runtime = new AgentRuntime(registry, ollama, new AgentRuntimeSettings("", "en", [], "sandbox"));
+ TaskSnapshot task = registry.Enqueue("analyze", null, 0.7);
+ registry.Transition(task.TaskId, AgentTaskState.Planning);
+
+ await runtime.ExecuteAsync(task.TaskId, TestContext.Current.CancellationToken);
+
+ Assert.True(registry.TryGet(task.TaskId, null, out TaskSnapshot? completed));
+ Assert.Equal(AgentTaskState.Completed, completed!.State);
+ Assert.Equal([null, false], ollama.ThinkOverrides);
+ Assert.All(ollama.Temperatures, value => Assert.Equal(0.7, value));
+ }
+
+ [Fact]
+ public async Task Invalid_structured_plan_after_retry_fails_without_running_tools()
+ {
+ TaskRegistry registry = new();
+ var ollama = new ThinkRetryOllamaClient(succeedOnRetry: false);
+ var runtime = new AgentRuntime(registry, ollama, new AgentRuntimeSettings("", "en", [], "sandbox"));
+ TaskSnapshot task = registry.Enqueue("change a file", null, null);
+ registry.Transition(task.TaskId, AgentTaskState.Planning);
+
+ await runtime.ExecuteAsync(task.TaskId, TestContext.Current.CancellationToken);
+
+ Assert.True(registry.TryGet(task.TaskId, null, out TaskSnapshot? failed));
+ Assert.Equal(AgentTaskState.Failed, failed!.State);
+ Assert.Contains("think=false retry", failed.Result, StringComparison.OrdinalIgnoreCase);
+ Assert.Equal(0, ollama.RunCalls);
+ }
+
+ [Fact]
+ public async Task Failed_context_summary_does_not_delete_original_context()
+ {
+ var sessions = new SessionStore();
+ AgentSession session = sessions.Create();
+ for (int index = 0; index < 10; index++) session.Append(new ConversationMessage("user", $"constraint-{index}"));
+ var registry = new TaskRegistry(sessions);
+ var runtime = new AgentRuntime(registry, new SummaryOllamaClient(summarySucceeds: false),
+ new AgentRuntimeSettings("", "en", [], "sandbox", NumCtx: 100, ContextSummaryThreshold: 0.8));
+ TaskSnapshot task = registry.Enqueue("analyze", session.SessionId, null);
+ registry.Transition(task.TaskId, AgentTaskState.Planning);
+
+ await runtime.ExecuteAsync(task.TaskId, TestContext.Current.CancellationToken);
+
+ Assert.Null(session.Summary);
+ Assert.Contains(session.Conversation, message => message.Content == "constraint-0");
+ }
+
+ [Fact]
+ public async Task Successful_context_summary_preserves_required_categories_before_compaction()
+ {
+ TaskRegistry registry = new();
+ var ollama = new SummaryOllamaClient(summarySucceeds: true);
+ var runtime = new AgentRuntime(registry, ollama,
+ new AgentRuntimeSettings("", "en", [], "sandbox", NumCtx: 100, ContextSummaryThreshold: 0.8, ConversationKeepLastN: 2));
+ TaskSnapshot task = registry.Enqueue("analyze", null, null);
+ Assert.True(registry.Sessions.TryGet(task.SessionId, out AgentSession? session));
+ for (int i = 0; i < 10; i++) session!.Append(new ConversationMessage("user", $"constraint-{i}"));
+ registry.Transition(task.TaskId, AgentTaskState.Planning);
+
+ await runtime.ExecuteAsync(task.TaskId, TestContext.Current.CancellationToken);
+
+ Assert.Equal("preserved constraints", session!.Summary);
+ Assert.True(session.Conversation.Count <= 4);
+ string summaryPrompt = string.Join("\n", ollama.SummaryRequest!.Select(static message => message.Content));
+ Assert.Contains("incomplete Tasks", summaryPrompt, StringComparison.Ordinal);
+ Assert.Contains("security denials", summaryPrompt, StringComparison.Ordinal);
+ Assert.Contains("verification results", summaryPrompt, StringComparison.Ordinal);
+ Assert.Contains("constraint-0", summaryPrompt, StringComparison.Ordinal);
+ }
+
+ [Fact]
+ public async Task Running_task_becomes_cancelled_only_after_cleanup_finishes()
+ {
+ TaskRegistry registry = new();
+ var backend = new BlockingCleanupBackend();
+ var dispatcher = new InternalToolDispatcher(new DenyByDefaultPolicy(InternalToolCatalog.All.Select(static item => item.Name).ToHashSet(StringComparer.Ordinal)), backend);
+ var runtime = new AgentRuntime(registry, new CancellableOllamaClient(), new AgentRuntimeSettings("", "en", [], "sandbox"), dispatcher);
+ TaskSnapshot task = registry.Enqueue("wait", null, null);
+ registry.Transition(task.TaskId, AgentTaskState.Planning);
+
+ Task execution = runtime.ExecuteAsync(task.TaskId, registry.GetCancellationToken(task.TaskId));
+ await backend.Begun.Task.WaitAsync(TestContext.Current.CancellationToken);
+ Assert.True(registry.Cancel(task.TaskId));
+ await backend.CleanupStarted.Task.WaitAsync(TestContext.Current.CancellationToken);
+ Assert.True(registry.TryGet(task.TaskId, null, out TaskSnapshot? stopping));
+ Assert.NotEqual(AgentTaskState.Cancelled, stopping!.State);
+
+ backend.AllowCleanup.TrySetResult();
+ await execution;
+ Assert.True(registry.TryGet(task.TaskId, null, out TaskSnapshot? cancelled));
+ Assert.Equal(AgentTaskState.Cancelled, cancelled!.State);
+ }
+
+ [Fact]
+ public async Task Cancellation_cleanup_failure_becomes_failed()
+ {
+ TaskRegistry registry = new();
+ var backend = new BlockingCleanupBackend(failCleanup: true);
+ var dispatcher = new InternalToolDispatcher(new DenyByDefaultPolicy(InternalToolCatalog.All.Select(static item => item.Name).ToHashSet(StringComparer.Ordinal)), backend);
+ var runtime = new AgentRuntime(registry, new CancellableOllamaClient(), new AgentRuntimeSettings("", "en", [], "sandbox"), dispatcher);
+ TaskSnapshot task = registry.Enqueue("wait", null, null);
+ registry.Transition(task.TaskId, AgentTaskState.Planning);
+
+ Task execution = runtime.ExecuteAsync(task.TaskId, registry.GetCancellationToken(task.TaskId));
+ await backend.Begun.Task.WaitAsync(TestContext.Current.CancellationToken);
+ Assert.True(registry.Cancel(task.TaskId));
+ await backend.CleanupStarted.Task.WaitAsync(TestContext.Current.CancellationToken);
+ backend.AllowCleanup.TrySetResult();
+ await execution;
+
+ Assert.True(registry.TryGet(task.TaskId, null, out TaskSnapshot? failed));
+ Assert.Equal(AgentTaskState.Failed, failed!.State);
+ Assert.Equal("cleanup_failed", failed.LastErrorCode);
+ }
+
+ [Fact]
+ public async Task Task_progress_current_task_and_critical_path_follow_the_graph()
+ {
+ TaskRegistry registry = new();
+ var ollama = new TwoTaskProgressOllamaClient();
+ var runtime = new AgentRuntime(registry, ollama, new AgentRuntimeSettings("", "en", [], "sandbox"));
+ TaskSnapshot task = registry.Enqueue("two steps", null, null);
+ registry.Transition(task.TaskId, AgentTaskState.Planning);
+
+ Task execution = runtime.ExecuteAsync(task.TaskId, registry.GetCancellationToken(task.TaskId));
+ await ollama.SecondTaskStarted.Task.WaitAsync(TestContext.Current.CancellationToken);
+ Assert.True(registry.TryGet(task.TaskId, null, out TaskSnapshot? progress));
+ Assert.Equal(1, progress!.CompletedTasks);
+ Assert.Equal(2, progress.TotalTasks);
+ Assert.Equal("second", progress.CurrentTask);
+ Assert.True(progress.CriticalPathTokens > 0);
+
+ ollama.AllowSecondTask.TrySetResult();
+ await execution;
+ Assert.True(registry.TryGet(task.TaskId, null, out TaskSnapshot? completed));
+ Assert.Equal(2, completed!.CompletedTasks);
+ Assert.Equal(2, completed.TotalTasks);
+ Assert.Null(completed.CurrentTask);
+ }
+
private sealed class RecordingOllamaClient : IOllamaClient
{
public List<IReadOnlyList<OllamaMessage>> Calls { get; } = [];
@@ -320,4 +505,132 @@ public sealed class PromptAndRuntimeTests
return Task.FromResult(new OllamaResult("", 1, 1, [new OllamaToolCall(new OllamaFunctionCall("file_search", arguments.RootElement.Clone()))]));
}
}
+
+ private sealed class ThinkRetryOllamaClient(bool succeedOnRetry) : IOllamaClient
+ {
+ public List<bool?> ThinkOverrides { get; } = [];
+ public List<double?> Temperatures { get; } = [];
+ public int RunCalls { get; private set; }
+ public Task<OllamaResult> ChatAsync(IReadOnlyList<OllamaMessage> messages, double? temperatureOverride, CancellationToken cancellationToken) =>
+ Task.FromResult(new OllamaResult("done", 1, 1));
+ public Task<OllamaResult> ChatStructuredAsync(IReadOnlyList<OllamaMessage> messages, double? temperatureOverride, JsonElement format, bool? thinkOverride, CancellationToken cancellationToken)
+ {
+ ThinkOverrides.Add(thinkOverride);
+ Temperatures.Add(temperatureOverride);
+ string content = ThinkOverrides.Count == 2 && succeedOnRetry
+ ? """{"tasks":[{"description":"analyze","dependencies":[],"budget_class":"4K","optimistic_tokens":10,"most_likely_tokens":20,"pessimistic_tokens":30,"requires_external_effect":false}]}"""
+ : "not-json";
+ return Task.FromResult(new OllamaResult(content, 1, 1));
+ }
+ public Task<OllamaResult> ChatWithToolsAsync(IReadOnlyList<OllamaMessage> messages, double? temperatureOverride, IReadOnlyList<InternalToolDefinition> tools, CancellationToken cancellationToken)
+ {
+ RunCalls++;
+ Temperatures.Add(temperatureOverride);
+ return Task.FromResult(new OllamaResult("done", 1, 1));
+ }
+ }
+
+ private sealed class ChangingPolicyPlanOllamaClient(bool sameDeny = false) : IOllamaClient
+ {
+ private int _plans;
+ private int _runs;
+ public Task<OllamaResult> ChatAsync(IReadOnlyList<OllamaMessage> messages, double? temperatureOverride, CancellationToken cancellationToken) =>
+ Task.FromResult(new OllamaResult("unused", 1, 1));
+ public Task<OllamaResult> ChatStructuredAsync(IReadOnlyList<OllamaMessage> messages, double? temperatureOverride, JsonElement format, bool? thinkOverride, CancellationToken cancellationToken)
+ {
+ int plan = Interlocked.Increment(ref _plans);
+ string json = $$"""{"tasks":[{"description":"search-{{plan}}","dependencies":[],"budget_class":"4K","optimistic_tokens":10,"most_likely_tokens":20,"pessimistic_tokens":30,"requires_external_effect":true}]}""";
+ return Task.FromResult(new OllamaResult(json, 1, 1));
+ }
+ public Task<OllamaResult> ChatWithToolsAsync(IReadOnlyList<OllamaMessage> messages, double? temperatureOverride, IReadOnlyList<InternalToolDefinition> tools, CancellationToken cancellationToken)
+ {
+ int run = Interlocked.Increment(ref _runs);
+ string root = sameDeny ? "C:/denied/same" : $"C:/denied/{run}";
+ using JsonDocument arguments = JsonDocument.Parse($$"""{"root":{{JsonSerializer.Serialize(root)}},"mode":"name","query":"*.txt"}""");
+ return Task.FromResult(new OllamaResult("", 1, 1, [new OllamaToolCall(new OllamaFunctionCall("file_search", arguments.RootElement.Clone()))]));
+ }
+ }
+
+ private sealed class ChangingPolicyBackend : IInternalToolBackend
+ {
+ public int CallCount { get; private set; }
+ public Task<ToolExecutionResult> ExecuteAsync(Guid taskId, string toolName, JsonElement arguments, CancellationToken cancellationToken)
+ {
+ CallCount++;
+ string root = arguments.GetProperty("root").GetString()!;
+ return Task.FromResult(new ToolExecutionResult(false, $"denied: {root}", false, root, RetryAllowed: false));
+ }
+ }
+
+ private sealed class SummaryOllamaClient(bool summarySucceeds) : IOllamaClient
+ {
+ public IReadOnlyList<OllamaMessage>? SummaryRequest { get; private set; }
+ public Task<OllamaResult> ChatAsync(IReadOnlyList<OllamaMessage> messages, double? temperatureOverride, CancellationToken cancellationToken) =>
+ Task.FromResult(new OllamaResult("done", 1, 1));
+ public Task<OllamaResult> ChatStructuredAsync(IReadOnlyList<OllamaMessage> messages, double? temperatureOverride, JsonElement format, bool? thinkOverride, CancellationToken cancellationToken)
+ {
+ if (format.GetProperty("properties").TryGetProperty("summary", out _))
+ {
+ SummaryRequest = messages.ToArray();
+ if (!summarySucceeds) throw new InvalidOperationException("summary failed");
+ return Task.FromResult(new OllamaResult("""{"summary":"preserved constraints"}""", 1, 1));
+ }
+ return Task.FromResult(new OllamaResult("""{"tasks":[{"description":"analyze","dependencies":[],"budget_class":"4K","optimistic_tokens":10,"most_likely_tokens":20,"pessimistic_tokens":30,"requires_external_effect":false}]}""", 1, 1));
+ }
+ public Task<OllamaResult> ChatWithToolsAsync(IReadOnlyList<OllamaMessage> messages, double? temperatureOverride, IReadOnlyList<InternalToolDefinition> tools, CancellationToken cancellationToken) =>
+ Task.FromResult(new OllamaResult("done", 1, 1, PromptEvalCount: 90));
+ }
+
+ private sealed class CancellableOllamaClient : IOllamaClient
+ {
+ public Task<OllamaResult> ChatAsync(IReadOnlyList<OllamaMessage> messages, double? temperatureOverride, CancellationToken cancellationToken) =>
+ Task.FromResult(new OllamaResult("done", 1, 1));
+ public Task<OllamaResult> ChatStructuredAsync(IReadOnlyList<OllamaMessage> messages, double? temperatureOverride, JsonElement format, bool? thinkOverride, CancellationToken cancellationToken) =>
+ Task.FromResult(new OllamaResult("""{"tasks":[{"description":"wait","dependencies":[],"budget_class":"4K","optimistic_tokens":10,"most_likely_tokens":20,"pessimistic_tokens":30,"requires_external_effect":false}]}""", 1, 1));
+ public async Task<OllamaResult> ChatWithToolsAsync(IReadOnlyList<OllamaMessage> messages, double? temperatureOverride, IReadOnlyList<InternalToolDefinition> tools, CancellationToken cancellationToken)
+ {
+ await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken);
+ throw new InvalidOperationException("unreachable");
+ }
+ }
+
+ private sealed class TwoTaskProgressOllamaClient : IOllamaClient
+ {
+ private int _runCalls;
+ public TaskCompletionSource SecondTaskStarted { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
+ public TaskCompletionSource AllowSecondTask { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
+ public Task<OllamaResult> ChatAsync(IReadOnlyList<OllamaMessage> messages, double? temperatureOverride, CancellationToken cancellationToken) =>
+ Task.FromResult(new OllamaResult("done", 1, 1));
+ public Task<OllamaResult> ChatStructuredAsync(IReadOnlyList<OllamaMessage> messages, double? temperatureOverride, JsonElement format, bool? thinkOverride, CancellationToken cancellationToken) =>
+ Task.FromResult(new OllamaResult("""{"tasks":[{"description":"first","dependencies":[],"budget_class":"4K","optimistic_tokens":10,"most_likely_tokens":20,"pessimistic_tokens":30,"requires_external_effect":false},{"description":"second","dependencies":[0],"budget_class":"4K","optimistic_tokens":10,"most_likely_tokens":20,"pessimistic_tokens":30,"requires_external_effect":false}]}""", 1, 1));
+ public async Task<OllamaResult> ChatWithToolsAsync(IReadOnlyList<OllamaMessage> messages, double? temperatureOverride, IReadOnlyList<InternalToolDefinition> tools, CancellationToken cancellationToken)
+ {
+ if (Interlocked.Increment(ref _runCalls) == 2)
+ {
+ SecondTaskStarted.TrySetResult();
+ await AllowSecondTask.Task.WaitAsync(cancellationToken);
+ }
+ return new OllamaResult("done", 1, 1);
+ }
+ }
+
+ private sealed class BlockingCleanupBackend(bool failCleanup = false) : IInternalToolBackend
+ {
+ public TaskCompletionSource Begun { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
+ public TaskCompletionSource CleanupStarted { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
+ public TaskCompletionSource AllowCleanup { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
+ public Task BeginTaskAsync(Guid taskId, Guid sessionId, CancellationToken cancellationToken)
+ {
+ Begun.TrySetResult();
+ return Task.CompletedTask;
+ }
+ public Task<ToolExecutionResult> ExecuteAsync(Guid taskId, string toolName, JsonElement arguments, CancellationToken cancellationToken) =>
+ Task.FromResult(new ToolExecutionResult(true, "unused", false));
+ public async Task EndTaskAsync(Guid taskId, bool success, CancellationToken cancellationToken)
+ {
+ CleanupStarted.TrySetResult();
+ await AllowCleanup.Task.WaitAsync(cancellationToken);
+ if (failCleanup) throw new InvalidOperationException("cleanup failed");
+ }
+ }
}
diff --git a/tests/Agent.Core.Tests/TaskRegistryTests.cs b/tests/Agent.Core.Tests/TaskRegistryTests.cs
index 28a2844..7e84dfd 100644
--- a/tests/Agent.Core.Tests/TaskRegistryTests.cs
+++ b/tests/Agent.Core.Tests/TaskRegistryTests.cs
@@ -18,6 +18,23 @@ public sealed class TaskRegistryTests
}
[Fact]
+ public async Task Running_cancel_is_terminal_only_after_runtime_cleanup_transition()
+ {
+ var registry = new TaskRegistry();
+ TaskSnapshot task = registry.Enqueue("work", null, null);
+ Assert.True(registry.Transition(task.TaskId, AgentTaskState.Planning));
+ Assert.True(registry.Transition(task.TaskId, AgentTaskState.Running));
+
+ Assert.True(registry.Cancel(task.TaskId));
+ Assert.True(registry.GetCancellationToken(task.TaskId).IsCancellationRequested);
+ Task wait = registry.WaitForTerminalAsync(task.TaskId, CancellationToken.None);
+ Assert.False(wait.IsCompleted);
+
+ Assert.True(registry.Transition(task.TaskId, AgentTaskState.Cancelled, result: "clean"));
+ Assert.Equal(AgentTaskState.Cancelled, (await registry.WaitForTerminalAsync(task.TaskId, TestContext.Current.CancellationToken)).State);
+ }
+
+ [Fact]
public void NewTaskGetsUniqueTaskAndSessionIds()
{
var registry = new TaskRegistry();
diff --git a/tests/Agent.Windows.Tests/CrashScenarioBootstrap.cs b/tests/Agent.Windows.Tests/CrashScenarioBootstrap.cs
index dd170ab..d025eac 100644
--- a/tests/Agent.Windows.Tests/CrashScenarioBootstrap.cs
+++ b/tests/Agent.Windows.Tests/CrashScenarioBootstrap.cs
@@ -62,7 +62,7 @@ internal static class CrashScenarioBootstrap
new ExecutablePolicy([new ExecutableApproval(powershell, ExecutablePolicy.ComputeSha256(powershell))], true), processAudit);
string escapedAudit = auditPath.Replace("'", "''", StringComparison.Ordinal);
string crashError = Path.Combine(root, "crash-error.txt").Replace("'", "''", StringComparison.Ordinal);
- string watchdogScript = $"$d=[DateTime]::UtcNow.AddSeconds(10); while([DateTime]::UtcNow -lt $d){{ if((Test-Path -LiteralPath '{escapedAudit}') -and (Select-String -LiteralPath '{escapedAudit}' -SimpleMatch '\"Event\":\"process.start\"' -Quiet)){{ Stop-Process -Id {Environment.ProcessId} -Force; exit 0 }}; Start-Sleep -Milliseconds 25 }}; Set-Content -LiteralPath '{crashError}' -Value 'Sandbox process did not start before crash'; Stop-Process -Id {Environment.ProcessId} -Force";
+ string watchdogScript = $"$d=[DateTime]::UtcNow.AddSeconds(10); while([DateTime]::UtcNow -lt $d){{ if((Test-Path -LiteralPath '{escapedAudit}') -and (Select-String -LiteralPath '{escapedAudit}' -SimpleMatch '\"event\":\"process.start\"' -Quiet)){{ Stop-Process -Id {Environment.ProcessId} -Force; exit 0 }}; Start-Sleep -Milliseconds 25 }}; Set-Content -LiteralPath '{crashError}' -Value 'Sandbox process did not start before crash'; Stop-Process -Id {Environment.ProcessId} -Force";
_ = Process.Start(new ProcessStartInfo
{
FileName = powershell, UseShellExecute = false, CreateNoWindow = true,
diff --git a/tests/Agent.Windows.Tests/SecurityPolicyTests.cs b/tests/Agent.Windows.Tests/SecurityPolicyTests.cs
index 10415ce..7489dad 100644
--- a/tests/Agent.Windows.Tests/SecurityPolicyTests.cs
+++ b/tests/Agent.Windows.Tests/SecurityPolicyTests.cs
@@ -457,7 +457,7 @@ public sealed class SecurityPolicyTests
{
string auditPath = Path.Combine(directory.Path, "crash-audit.jsonl");
using JsonDocument audit = JsonDocument.Parse(File.ReadLines(auditPath).Single(line => line.Contains("process.start", StringComparison.Ordinal)));
- int sandboxPid = int.Parse(audit.RootElement.GetProperty("Data").GetProperty("pid").GetString()!,
+ int sandboxPid = int.Parse(audit.RootElement.GetProperty("data").GetProperty("pid").GetString()!,
System.Globalization.CultureInfo.InvariantCulture);
Assert.True(SpinWait.SpinUntil(() => !ProcessExists(sandboxPid), TimeSpan.FromSeconds(5)), $"Orphan sandbox PID {sandboxPid} survived broker crash.");
}
@@ -562,7 +562,8 @@ public sealed class SecurityPolicyTests
16 * 1024 * 1024, 1024 * 1024, 1000, 10000);
var backend = new WindowsToolBackend(settings, audit);
Guid taskId = Guid.NewGuid();
- await backend.BeginTaskAsync(taskId, Guid.NewGuid(), TestContext.Current.CancellationToken);
+ Guid sessionId = Guid.NewGuid();
+ await backend.BeginTaskAsync(taskId, sessionId, TestContext.Current.CancellationToken);
using JsonDocument edit = JsonDocument.Parse($$"""{"path":{{JsonSerializer.Serialize(target)}},"operation":"overwrite","content":"after"}""");
ToolExecutionResult result = await backend.ExecuteAsync(taskId, "file_edit", edit.RootElement, TestContext.Current.CancellationToken);
@@ -576,7 +577,11 @@ public sealed class SecurityPolicyTests
Assert.Empty(Directory.EnumerateFiles(Path.Combine(directory.Path, "journals")));
string backups = Path.Combine(directory.Path, "backups");
Assert.True(!Directory.Exists(backups) || !Directory.EnumerateFileSystemEntries(backups).Any());
- Assert.Contains(audit.Events, item => item.Event == "wfp.remove" && item.Data["residual_filters"] == "0");
+ Assert.Contains(audit.Events, item => item.Event == "network.rule.remove" && item.Data["residual_filters"] == "0");
+ Assert.Contains(audit.Events, item => item.Event == "task.start" && item.SessionId == sessionId);
+ Assert.Contains(audit.Events, item => item.Event == "tool.call" && item.SessionId == sessionId);
+ Assert.Contains(audit.Events, item => item.Event == "file.write" && item.SessionId == sessionId);
+ Assert.Contains(audit.Events, item => item.Event == "task.end" && item.SessionId == sessionId);
Assert.Single(audit.Events, item => item.Event == "acl.apply");
Assert.Single(audit.Events, item => item.Event == "acl.restore");
}
@@ -925,7 +930,7 @@ public sealed class SecurityPolicyTests
Assert.False(deniedPort.Accepted);
}
allowJournal.CompleteAndDelete();
- Assert.Contains(audit.Events, item => item.Event == "wfp.remove" && item.Data["residual_filters"] == "0");
+ Assert.Contains(audit.Events, item => item.Event == "network.rule.remove" && item.Data["residual_filters"] == "0");
}
private static RestrictedProcessResult Curl(RestrictedProcessLauncher launcher, Guid taskId, string curl, string url, string cwd) =>