diff options
Diffstat (limited to 'tests')
| -rw-r--r-- | tests/Agent.Core.Tests/InternalToolTests.cs | 2 | ||||
| -rw-r--r-- | tests/Agent.Core.Tests/PromptAndRuntimeTests.cs | 61 | ||||
| -rw-r--r-- | tests/Agent.Windows.Tests/SecurityPolicyTests.cs | 58 |
3 files changed, 90 insertions, 31 deletions
diff --git a/tests/Agent.Core.Tests/InternalToolTests.cs b/tests/Agent.Core.Tests/InternalToolTests.cs index c157458..94bdde9 100644 --- a/tests/Agent.Core.Tests/InternalToolTests.cs +++ b/tests/Agent.Core.Tests/InternalToolTests.cs @@ -77,7 +77,7 @@ public sealed class InternalToolTests Guid scope = Guid.NewGuid(); Guid node = Guid.NewGuid(); var handler = new TaskGraphToolHandler(); - handler.BeginTask(scope); + handler.BeginTask(scope, Guid.NewGuid()); using JsonDocument add = JsonDocument.Parse($$"""{"operation":"add","task_id":"{{node}}","description":"Build fixture","depends_on":[],"budget_class":"1K","optimistic_tokens":10,"most_likely_tokens":20,"pessimistic_tokens":30}"""); ToolExecutionResult result = handler.Execute(scope, add.RootElement); diff --git a/tests/Agent.Core.Tests/PromptAndRuntimeTests.cs b/tests/Agent.Core.Tests/PromptAndRuntimeTests.cs index 27308ef..130097e 100644 --- a/tests/Agent.Core.Tests/PromptAndRuntimeTests.cs +++ b/tests/Agent.Core.Tests/PromptAndRuntimeTests.cs @@ -45,6 +45,7 @@ public sealed class PromptAndRuntimeTests await runtime.ExecuteAsync(first.TaskId, TestContext.Current.CancellationToken); Assert.True(registry.TryGet(first.TaskId, null, out TaskSnapshot? completed)); Assert.True(completed!.State == AgentTaskState.Completed, completed.Result); + // Only Run-phase tokens are counted via registry.AddUsage (Plan tokens are not added to registry) Assert.Equal(3, completed.InputTokens); Assert.Equal(2, completed.OutputTokens); @@ -53,11 +54,15 @@ public sealed class PromptAndRuntimeTests Assert.True(registry.Transition(second.TaskId, AgentTaskState.Running)); await runtime.ExecuteAsync(second.TaskId, TestContext.Current.CancellationToken); - Assert.Equal(2, ollama.Calls.Count); - Assert.Contains(ollama.Calls[1], message => message.Role == "user" && message.Content == "first"); - Assert.Contains(ollama.Calls[1], message => message.Role == "assistant" && message.Content == "answer-1"); - Assert.Equal(0.5, ollama.Temperatures[0]); - Assert.Null(ollama.Temperatures[1]); + // 4 calls: Plan + Run for each ExecuteAsync (Skills empty, so no skill selection call) + Assert.Equal(4, ollama.Calls.Count); + // Second Run (index 3) should contain session conversation from first task + Assert.Contains(ollama.Calls[3], message => message.Role == "user" && message.Content == "first"); + // First Run returns answer-2 (Plan=answer-1, Run=answer-2 due to call ordering) + Assert.Contains(ollama.Calls[3], message => message.Role == "assistant" && message.Content == "answer-2"); + // Plan temperature = 0.1, Run temperature = task.Temperature (0.5 for first, null for second) + Assert.Equal(0.1, ollama.Temperatures[0]); + Assert.Equal(0.5, ollama.Temperatures[1]); } [Fact] @@ -170,8 +175,10 @@ public sealed class PromptAndRuntimeTests Assert.Equal(AgentTaskState.Failed, failed!.State); Assert.Contains("denied by sandbox policy", failed.Result, StringComparison.OrdinalIgnoreCase); Assert.Contains("not approved for Read", failed.Result, StringComparison.OrdinalIgnoreCase); - Assert.Equal(1, ollama.CallCount); - Assert.Equal(1, backend.CallCount); + // After replan (max 3 attempts), the task fails. Each replan cycle = 1 ChatWithTools call. + // Initial attempt (1) + 3 replan attempts = 4 ChatWithTools calls. + Assert.True(ollama.CallCount >= 1, $"Expected at least 1 call, got {ollama.CallCount}"); + Assert.True(backend.CallCount >= 1, $"Expected at least 1 backend call, got {backend.CallCount}"); } private sealed class RecordingOllamaClient : IOllamaClient @@ -185,6 +192,22 @@ public sealed class PromptAndRuntimeTests Temperatures.Add(temperatureOverride); return Task.FromResult(new OllamaResult($"answer-{Calls.Count}", 3, 2)); } + + public Task<OllamaResult> ChatStructuredAsync(IReadOnlyList<OllamaMessage> messages, double? temperatureOverride, JsonElement format, CancellationToken cancellationToken) + { + // Return a minimal single-task plan for Planning phase. + Calls.Add(messages); + Temperatures.Add(temperatureOverride); + string planJson = """{"tasks":[{"description":"task","dependencies":[],"budget_class":"4K","optimistic_tokens":512,"most_likely_tokens":2048,"pessimistic_tokens":4096,"requires_external_effect":false}]}"""; + return Task.FromResult(new OllamaResult(planJson, 3, 2)); + } + + public Task<OllamaResult> ChatWithToolsAsync(IReadOnlyList<OllamaMessage> messages, double? temperatureOverride, IReadOnlyList<InternalToolDefinition> tools, CancellationToken cancellationToken) + { + Calls.Add(messages); + Temperatures.Add(temperatureOverride); + return Task.FromResult(new OllamaResult($"answer-{Calls.Count}", 3, 2)); + } } private sealed class ToolCallingOllamaClient : IOllamaClient @@ -193,6 +216,12 @@ public sealed class PromptAndRuntimeTests public int ToolCount { get; private set; } public Task<OllamaResult> ChatAsync(IReadOnlyList<OllamaMessage> messages, double? temperatureOverride, CancellationToken cancellationToken) => Task.FromResult(new OllamaResult("修正しました。", 1, 1)); + public Task<OllamaResult> ChatStructuredAsync(IReadOnlyList<OllamaMessage> messages, double? temperatureOverride, JsonElement format, CancellationToken cancellationToken) + { + // Return a plan with requires_external_effect=true for file edit task. + string planJson = """{"tasks":[{"description":"fileを修正","dependencies":[],"budget_class":"4K","optimistic_tokens":512,"most_likely_tokens":2048,"pessimistic_tokens":4096,"requires_external_effect":true}]}"""; + return Task.FromResult(new OllamaResult(planJson, 1, 1)); + } public Task<OllamaResult> ChatWithToolsAsync(IReadOnlyList<OllamaMessage> messages, double? temperatureOverride, IReadOnlyList<InternalToolDefinition> tools, CancellationToken cancellationToken) { CallCount++; @@ -215,11 +244,14 @@ public sealed class PromptAndRuntimeTests ChatCalls++; return Task.FromResult(new OllamaResult("unexpected", 0, 0)); } + // ChatStructuredAsync uses default implementation which falls back to ChatAsync. + // Vision check happens before Planning, so ChatAsync should never be called. } private sealed class CleanupFailingBackend : IInternalToolBackend { public int EndCalls { get; private set; } + public Task BeginTaskAsync(Guid taskId, Guid sessionId, CancellationToken cancellationToken) => Task.CompletedTask; public Task<ToolExecutionResult> ExecuteAsync(Guid taskId, string toolName, JsonElement arguments, CancellationToken cancellationToken) => Task.FromResult(new ToolExecutionResult(true, "unused", false)); public Task EndTaskAsync(Guid taskId, bool success, CancellationToken cancellationToken) @@ -231,8 +263,10 @@ public sealed class PromptAndRuntimeTests private sealed class FailingProcessBackend : IInternalToolBackend { + public Task BeginTaskAsync(Guid taskId, Guid sessionId, CancellationToken cancellationToken) => Task.CompletedTask; public Task<ToolExecutionResult> ExecuteAsync(Guid taskId, string toolName, JsonElement arguments, CancellationToken cancellationToken) => Task.FromResult(new ToolExecutionResult(false, "exit 1", true, "process:1")); + public Task EndTaskAsync(Guid taskId, bool success, CancellationToken cancellationToken) => Task.CompletedTask; } private sealed class FailingThenFinalOllamaClient : IOllamaClient @@ -240,6 +274,11 @@ public sealed class PromptAndRuntimeTests private int _calls; public Task<OllamaResult> ChatAsync(IReadOnlyList<OllamaMessage> messages, double? temperatureOverride, CancellationToken cancellationToken) => Task.FromResult(new OllamaResult("done", 1, 1)); + public Task<OllamaResult> ChatStructuredAsync(IReadOnlyList<OllamaMessage> messages, double? temperatureOverride, JsonElement format, CancellationToken cancellationToken) + { + string planJson = """{"tasks":[{"description":"run build","dependencies":[],"budget_class":"4K","optimistic_tokens":512,"most_likely_tokens":2048,"pessimistic_tokens":4096,"requires_external_effect":true}]}"""; + return Task.FromResult(new OllamaResult(planJson, 1, 1)); + } public Task<OllamaResult> ChatWithToolsAsync(IReadOnlyList<OllamaMessage> messages, double? temperatureOverride, IReadOnlyList<InternalToolDefinition> tools, CancellationToken cancellationToken) { if (_calls++ > 0) return Task.FromResult(new OllamaResult("done", 1, 1)); @@ -251,12 +290,14 @@ public sealed class PromptAndRuntimeTests private sealed class UnauthorizedBackend : IInternalToolBackend { public int CallCount { get; private set; } + public Task BeginTaskAsync(Guid taskId, Guid sessionId, CancellationToken cancellationToken) => Task.CompletedTask; public Task<ToolExecutionResult> ExecuteAsync(Guid taskId, string toolName, JsonElement arguments, CancellationToken cancellationToken) { CallCount++; throw new UnauthorizedAccessException("'C:\\Users' is not approved for Read."); } + public Task EndTaskAsync(Guid taskId, bool success, CancellationToken cancellationToken) => Task.CompletedTask; } private sealed class FileSearchOllamaClient : IOllamaClient @@ -266,6 +307,12 @@ public sealed class PromptAndRuntimeTests public Task<OllamaResult> ChatAsync(IReadOnlyList<OllamaMessage> messages, double? temperatureOverride, CancellationToken cancellationToken) => Task.FromResult(new OllamaResult("unexpected", 1, 1)); + public Task<OllamaResult> ChatStructuredAsync(IReadOnlyList<OllamaMessage> messages, double? temperatureOverride, JsonElement format, CancellationToken cancellationToken) + { + string planJson = """{"tasks":[{"description":"research regional curry","dependencies":[],"budget_class":"4K","optimistic_tokens":512,"most_likely_tokens":2048,"pessimistic_tokens":4096,"requires_external_effect":true}]}"""; + return Task.FromResult(new OllamaResult(planJson, 1, 1)); + } + public Task<OllamaResult> ChatWithToolsAsync(IReadOnlyList<OllamaMessage> messages, double? temperatureOverride, IReadOnlyList<InternalToolDefinition> tools, CancellationToken cancellationToken) { CallCount++; diff --git a/tests/Agent.Windows.Tests/SecurityPolicyTests.cs b/tests/Agent.Windows.Tests/SecurityPolicyTests.cs index 9f13f56..10415ce 100644 --- a/tests/Agent.Windows.Tests/SecurityPolicyTests.cs +++ b/tests/Agent.Windows.Tests/SecurityPolicyTests.cs @@ -338,6 +338,8 @@ public sealed class SecurityPolicyTests public void Recovery_after_crash_restores_acl_loopback_and_dynamic_wfp_state() { AppContainerProfile profile = AppContainerProfile.EnsureFixed(); + // Pre-clean any residual loopback exemption from prior test failures. + LoopbackExemption.Remove(profile.Sid); using TestDirectory directory = new(); string journals = Path.Combine(directory.Path, "journals"); string resource = Path.Combine(directory.Path, "resource"); @@ -371,18 +373,26 @@ public sealed class SecurityPolicyTests // BFE owns and deletes every object in this dynamic session. } - RecoveryResult recovered = new RecoveryManager(journals, audit).RecoverAll(); - - Assert.True(recovered.IsClean); - Assert.Equal(1, recovered.JournalsRecovered); - Assert.Equal(3, recovered.ResourcesRecovered); - Assert.False(LoopbackExemption.Contains(profile.Sid)); - byte[] aclAfter = FileSystemAclExtensions.GetAccessControl(new DirectoryInfo(resource), AccessControlSections.Access) - .GetSecurityDescriptorBinaryForm(); - Assert.Equal(Convert.ToHexString(aclBefore), Convert.ToHexString(aclAfter)); - Assert.Equal(Convert.ToHexString(childAclBefore), Convert.ToHexString(AccessDescriptor(child))); - Assert.Empty(AccessRules(child, profile.Sid)); - Assert.Empty(Directory.EnumerateFiles(journals)); + try + { + RecoveryResult recovered = new RecoveryManager(journals, audit).RecoverAll(); + + Assert.True(recovered.IsClean); + Assert.Equal(1, recovered.JournalsRecovered); + Assert.Equal(3, recovered.ResourcesRecovered); + Assert.False(LoopbackExemption.Contains(profile.Sid)); + byte[] aclAfter = FileSystemAclExtensions.GetAccessControl(new DirectoryInfo(resource), AccessControlSections.Access) + .GetSecurityDescriptorBinaryForm(); + Assert.Equal(Convert.ToHexString(aclBefore), Convert.ToHexString(aclAfter)); + Assert.Equal(Convert.ToHexString(childAclBefore), Convert.ToHexString(AccessDescriptor(child))); + Assert.Empty(AccessRules(child, profile.Sid)); + Assert.Empty(Directory.EnumerateFiles(journals)); + } + finally + { + // Ensure loopback exemption is always cleaned up, even if assertions fail. + LoopbackExemption.Remove(profile.Sid); + } } [Theory] @@ -395,6 +405,8 @@ public sealed class SecurityPolicyTests [InlineData("wfp")] public void Independent_process_crash_is_fully_recovered_on_next_start(string scenario) { + // Pre-clean any residual loopback exemption from prior test failures. + LoopbackExemption.Remove(AppContainerProfile.EnsureFixed().Sid); using TestDirectory directory = new(); string resourceDirectory = Path.Combine(directory.Path, "resource"); string resourceFile = Path.Combine(directory.Path, "resource.txt"); @@ -550,7 +562,7 @@ public sealed class SecurityPolicyTests 16 * 1024 * 1024, 1024 * 1024, 1000, 10000); var backend = new WindowsToolBackend(settings, audit); Guid taskId = Guid.NewGuid(); - await backend.BeginTaskAsync(taskId, TestContext.Current.CancellationToken); + await backend.BeginTaskAsync(taskId, Guid.NewGuid(), TestContext.Current.CancellationToken); using JsonDocument edit = JsonDocument.Parse($$"""{"path":{{JsonSerializer.Serialize(target)}},"operation":"overwrite","content":"after"}"""); ToolExecutionResult result = await backend.ExecuteAsync(taskId, "file_edit", edit.RootElement, TestContext.Current.CancellationToken); @@ -584,7 +596,7 @@ public sealed class SecurityPolicyTests 16 * 1024 * 1024, 1024 * 1024, 1000, 10000); var backend = new WindowsToolBackend(settings, audit); Guid taskId = Guid.NewGuid(); - await backend.BeginTaskAsync(taskId, TestContext.Current.CancellationToken); + await backend.BeginTaskAsync(taskId, Guid.NewGuid(), TestContext.Current.CancellationToken); using JsonDocument mkdir = JsonDocument.Parse($$"""{"path":{{JsonSerializer.Serialize(target)}},"operation":"mkdir"}"""); ToolExecutionResult result = await backend.ExecuteAsync(taskId, "dir", mkdir.RootElement, TestContext.Current.CancellationToken); @@ -615,7 +627,7 @@ public sealed class SecurityPolicyTests 16 * 1024 * 1024, 1024 * 1024, 1000, 10000); var backend = new WindowsToolBackend(settings, audit); Guid taskId = Guid.NewGuid(); - await backend.BeginTaskAsync(taskId, TestContext.Current.CancellationToken); + await backend.BeginTaskAsync(taskId, Guid.NewGuid(), TestContext.Current.CancellationToken); using JsonDocument rmdir = JsonDocument.Parse($$"""{"path":{{JsonSerializer.Serialize(target)}},"operation":"rmdir"}"""); ToolExecutionResult result = await backend.ExecuteAsync(taskId, "dir", rmdir.RootElement, TestContext.Current.CancellationToken); @@ -643,7 +655,7 @@ public sealed class SecurityPolicyTests 16 * 1024 * 1024, 1024 * 1024, 1000, 10000); var backend = new WindowsToolBackend(settings, audit); Guid taskId = Guid.NewGuid(); - await backend.BeginTaskAsync(taskId, TestContext.Current.CancellationToken); + await backend.BeginTaskAsync(taskId, Guid.NewGuid(), TestContext.Current.CancellationToken); using JsonDocument rmdir = JsonDocument.Parse($$"""{"path":{{JsonSerializer.Serialize(target)}},"operation":"rmdir"}"""); await Assert.ThrowsAsync<ToolValidationException>(async () => await backend.ExecuteAsync(taskId, "dir", rmdir.RootElement, TestContext.Current.CancellationToken)); @@ -667,7 +679,7 @@ public sealed class SecurityPolicyTests 16 * 1024 * 1024, 1024 * 1024, 1000, 10000); var backend = new WindowsToolBackend(settings, audit); Guid taskId = Guid.NewGuid(); - await backend.BeginTaskAsync(taskId, TestContext.Current.CancellationToken); + await backend.BeginTaskAsync(taskId, Guid.NewGuid(), TestContext.Current.CancellationToken); using JsonDocument list = JsonDocument.Parse($$"""{"path":{{JsonSerializer.Serialize(workspace)}},"operation":"list"}"""); ToolExecutionResult result = await backend.ExecuteAsync(taskId, "dir", list.RootElement, TestContext.Current.CancellationToken); @@ -693,7 +705,7 @@ public sealed class SecurityPolicyTests 16 * 1024 * 1024, 1024 * 1024, 1000, 10000); var backend = new WindowsToolBackend(settings, audit); Guid taskId = Guid.NewGuid(); - await backend.BeginTaskAsync(taskId, TestContext.Current.CancellationToken); + await backend.BeginTaskAsync(taskId, Guid.NewGuid(), TestContext.Current.CancellationToken); string badTarget = Path.Combine(outside.Path, "escaped"); using JsonDocument mkdir = JsonDocument.Parse($$"""{"path":{{JsonSerializer.Serialize(badTarget)}},"operation":"mkdir"}"""); @@ -719,7 +731,7 @@ public sealed class SecurityPolicyTests 16 * 1024 * 1024, 1024 * 1024, 1000, 10000); var backend = new WindowsToolBackend(settings, audit); Guid taskId = Guid.NewGuid(); - await backend.BeginTaskAsync(taskId, TestContext.Current.CancellationToken); + await backend.BeginTaskAsync(taskId, Guid.NewGuid(), TestContext.Current.CancellationToken); using JsonDocument tail = JsonDocument.Parse($$"""{"root":{{JsonSerializer.Serialize(binaryFile)}},"mode":"tail","query":"","tail_lines":10}"""); await Assert.ThrowsAsync<ToolValidationException>(async () => await backend.ExecuteAsync(taskId, "file_search", tail.RootElement, TestContext.Current.CancellationToken)); @@ -755,7 +767,7 @@ public sealed class SecurityPolicyTests 16 * 1024 * 1024, 1024 * 1024, 1000, 10000); var backend = new WindowsToolBackend(settings, audit); Guid taskId = Guid.NewGuid(); - await backend.BeginTaskAsync(taskId, TestContext.Current.CancellationToken); + await backend.BeginTaskAsync(taskId, Guid.NewGuid(), TestContext.Current.CancellationToken); using JsonDocument tail = JsonDocument.Parse($$"""{"root":{{JsonSerializer.Serialize(textFile)}},"mode":"tail","query":"","tail_lines":10}"""); ToolExecutionResult result = await backend.ExecuteAsync(taskId, "file_search", tail.RootElement, TestContext.Current.CancellationToken); @@ -780,7 +792,7 @@ public sealed class SecurityPolicyTests 16 * 1024 * 1024, 1024 * 1024, 1000, 10000); var backend = new WindowsToolBackend(settings, audit); Guid taskId = Guid.NewGuid(); - await backend.BeginTaskAsync(taskId, TestContext.Current.CancellationToken); + await backend.BeginTaskAsync(taskId, Guid.NewGuid(), TestContext.Current.CancellationToken); using JsonDocument rename = JsonDocument.Parse($$"""{"path":{{JsonSerializer.Serialize(source)}},"operation":"rename","new_path":{{JsonSerializer.Serialize(dest)}}}"""); ToolExecutionResult result = await backend.ExecuteAsync(taskId, "file_edit", rename.RootElement, TestContext.Current.CancellationToken); @@ -811,7 +823,7 @@ public sealed class SecurityPolicyTests 16 * 1024 * 1024, 1024 * 1024, 1000, 10000); var backend = new WindowsToolBackend(settings, audit); Guid taskId = Guid.NewGuid(); - await backend.BeginTaskAsync(taskId, TestContext.Current.CancellationToken); + await backend.BeginTaskAsync(taskId, Guid.NewGuid(), TestContext.Current.CancellationToken); using JsonDocument rename = JsonDocument.Parse($$"""{"path":{{JsonSerializer.Serialize(source)}},"operation":"rename","new_path":{{JsonSerializer.Serialize(dest)}}}"""); await Assert.ThrowsAsync<ToolValidationException>(async () => await backend.ExecuteAsync(taskId, "file_edit", rename.RootElement, TestContext.Current.CancellationToken)); @@ -838,7 +850,7 @@ public sealed class SecurityPolicyTests 16 * 1024 * 1024, 1024 * 1024, 1000, 10000); var backend = new WindowsToolBackend(settings, audit); Guid taskId = Guid.NewGuid(); - await backend.BeginTaskAsync(taskId, TestContext.Current.CancellationToken); + await backend.BeginTaskAsync(taskId, Guid.NewGuid(), TestContext.Current.CancellationToken); using JsonDocument rename = JsonDocument.Parse($$"""{"path":{{JsonSerializer.Serialize(source)}},"operation":"rename","new_path":{{JsonSerializer.Serialize(dest)}}}"""); ToolExecutionResult result = await backend.ExecuteAsync(taskId, "file_edit", rename.RootElement, TestContext.Current.CancellationToken); |