diff options
Diffstat (limited to 'tests')
22 files changed, 1712 insertions, 0 deletions
diff --git a/tests/Agent.Core.Tests/Agent.Core.Tests.csproj b/tests/Agent.Core.Tests/Agent.Core.Tests.csproj new file mode 100644 index 0000000..1326733 --- /dev/null +++ b/tests/Agent.Core.Tests/Agent.Core.Tests.csproj @@ -0,0 +1,16 @@ +<Project Sdk="Microsoft.NET.Sdk"> + <PropertyGroup> + <TargetFramework>net10.0-windows10.0.26100.0</TargetFramework> + <IsPackable>false</IsPackable> + <IsTestProject>true</IsTestProject> + <TestingPlatformDotnetTestSupport>true</TestingPlatformDotnetTestSupport> + </PropertyGroup> + <ItemGroup> + <ProjectReference Include="..\..\src\Agent.Core\Agent.Core.csproj" /> + <ProjectReference Include="..\..\src\Agent.App\Agent.App.csproj" /> + </ItemGroup> + <ItemGroup> + <PackageReference Include="Microsoft.NET.Test.Sdk" /> + <PackageReference Include="xunit.v3" /> + </ItemGroup> +</Project> diff --git a/tests/Agent.Core.Tests/BaselineTests.cs b/tests/Agent.Core.Tests/BaselineTests.cs new file mode 100644 index 0000000..a9b8fb7 --- /dev/null +++ b/tests/Agent.Core.Tests/BaselineTests.cs @@ -0,0 +1,10 @@ +namespace Agent.Core.Tests; + +public sealed class BaselineTests +{ + [Fact] + public void CoreAssemblyLoads() + { + Assert.Equal("Agent.Core", typeof(Core.AssemblyMarker).Assembly.GetName().Name); + } +} diff --git a/tests/Agent.Core.Tests/ConfigurationTests.cs b/tests/Agent.Core.Tests/ConfigurationTests.cs new file mode 100644 index 0000000..7ef4afb --- /dev/null +++ b/tests/Agent.Core.Tests/ConfigurationTests.cs @@ -0,0 +1,66 @@ +using Agent.App.Cli; +using Agent.App.Configuration; + +namespace Agent.Core.Tests; + +public sealed class ConfigurationTests +{ + [Fact] + public void ValidYamlLoadsTypedConfiguration() + { + using var directory = new TestDirectory(); + string config = CreateValidConfiguration(directory); + AgentConfiguration result = ConfigurationLoader.Load(config); + Assert.Equal(1, result.SchemaVersion); + Assert.Equal("model", result.Ollama.Model); + Assert.Equal("deny", result.Sandbox.Network.Mode); + } + + [Fact] + public void UnknownYamlPropertyIsFatal() + { + using var directory = new TestDirectory(); + string config = CreateValidConfiguration(directory); + System.IO.File.AppendAllText(config, "unknown_root: true\n"); + Assert.Throws<ConfigurationException>(() => ConfigurationLoader.Load(config)); + } + + [Fact] + public void InvalidSchemaIsFatal() + { + using var directory = new TestDirectory(); + string config = CreateValidConfiguration(directory); + System.IO.File.WriteAllText(config, System.IO.File.ReadAllText(config).Replace("schema_version: 1", "schema_version: 2", StringComparison.Ordinal)); + Assert.Throws<ConfigurationException>(() => ConfigurationLoader.Load(config)); + } + + [Fact] + public void CommandLineLanguageOverrideIsCaptured() + { + CommandLine result = CommandLine.Parse(["serve", "--lang", "ja", "--transport", "http"]); + Assert.Equal("ja", result.Language); + Assert.Equal("http", result.Transport); + } + + private static string CreateValidConfiguration(TestDirectory directory) + { + directory.File("prompts/user-system.md", string.Empty); + Directory.CreateDirectory(System.IO.Path.Combine(directory.Path, "config/locales")); + Directory.CreateDirectory(System.IO.Path.Combine(directory.Path, "skills")); + return directory.File("agent.yaml", """ + schema_version: 1 + application: + language: en + agent: + user_system_prompt: prompts/user-system.md + ollama: + endpoint: http://127.0.0.1:11434 + model: model + localization: + directory: config/locales + fallback: en + skills: + directory: skills + """); + } +} diff --git a/tests/Agent.Core.Tests/FixedPromptTests.cs b/tests/Agent.Core.Tests/FixedPromptTests.cs new file mode 100644 index 0000000..93ec7be --- /dev/null +++ b/tests/Agent.Core.Tests/FixedPromptTests.cs @@ -0,0 +1,14 @@ +using Agent.Core.Prompts; + +namespace Agent.Core.Tests; + +public sealed class FixedPromptTests +{ + [Fact] + public void FixedPromptIsEmbeddedAndNonEmpty() + { + string prompt = FixedPrompt.Load(); + Assert.Contains("SandboxPolicy", prompt, StringComparison.Ordinal); + Assert.DoesNotContain("user-system.md", prompt, StringComparison.Ordinal); + } +} diff --git a/tests/Agent.Core.Tests/InternalToolTests.cs b/tests/Agent.Core.Tests/InternalToolTests.cs new file mode 100644 index 0000000..1e241ce --- /dev/null +++ b/tests/Agent.Core.Tests/InternalToolTests.cs @@ -0,0 +1,88 @@ +using System.Text.Json; +using Agent.Core.Tools; + +namespace Agent.Core.Tests; + +public sealed class InternalToolTests +{ + [Fact] + public void Catalog_exposes_exactly_five_strict_tool_groups() + { + Assert.Equal(["file_search", "file_read", "file_edit", "process_run", "task_update"], InternalToolCatalog.All.Select(x => x.Name)); + foreach (InternalToolDefinition tool in InternalToolCatalog.All) Assert.False(tool.Parameters.GetProperty("additionalProperties").GetBoolean()); + } + + [Theory] + [InlineData("file_read", "{}")] + [InlineData("file_read", "{\"path\":\"a\",\"mode\":\"whole\",\"unknown\":1}")] + [InlineData("file_read", "{\"path\":\"a\",\"mode\":\"invalid\"}")] + [InlineData("file_search", "{\"root\":\".\",\"mode\":\"text\",\"query\":\"x\",\"max_results\":1001}")] + [InlineData("process_run", "{\"executable\":\"x\",\"args\":\"bad\",\"cwd\":\".\"}")] + [InlineData("task_update", "{\"operation\":\"add\",\"description\":\"missing estimate\"}")] + public void Malformed_arguments_are_rejected(string tool, string json) + { + using JsonDocument document = JsonDocument.Parse(json); + Assert.Throws<ToolValidationException>(() => InternalToolValidator.Validate(tool, document.RootElement)); + } + + [Fact] + public async Task Policy_deny_has_no_backend_side_effect() + { + FakeInternalToolBackend backend = new(); + InternalToolDispatcher dispatcher = new(new DenyByDefaultPolicy(), backend); + using JsonDocument document = JsonDocument.Parse("""{"path":"x","operation":"overwrite","content":"changed"}"""); + + ToolExecutionResult result = await dispatcher.DispatchAsync(Guid.NewGuid(), "file_edit", document.RootElement, TestContext.Current.CancellationToken); + + Assert.False(result.Success); + Assert.False(result.ExternalEffect); + Assert.Empty(backend.Calls); + } + + [Fact] + public async Task Allowed_call_reaches_fake_backend_only_after_validation_and_policy() + { + FakeInternalToolBackend backend = new(); + InternalToolDispatcher dispatcher = new(new DenyByDefaultPolicy(new HashSet<string>(["file_read"], StringComparer.Ordinal)), backend); + using JsonDocument document = JsonDocument.Parse("""{"path":"x","mode":"whole"}"""); + + ToolExecutionResult result = await dispatcher.DispatchAsync(Guid.NewGuid(), "file_read", document.RootElement, TestContext.Current.CancellationToken); + + Assert.True(result.Success); + Assert.Single(backend.Calls); + } + + [Fact] + public void Guard_requires_effect_and_fails_on_fifth_identical_failure() + { + ToolUseGuard guard = new(); + for (int attempt = 1; attempt < 5; attempt++) Assert.Equal(ToolUseGuardDecision.Retry, guard.Evaluate(true, false, false, "same").Decision); + ToolUseGuardResult fifth = guard.Evaluate(true, false, false, "same"); + Assert.Equal(ToolUseGuardDecision.Fail, fifth.Decision); + Assert.Equal(5, fifth.IdenticalFailureCount); + } + + [Fact] + public void Guard_does_not_force_tools_for_pure_analysis() + { + Assert.Equal(ToolUseGuardDecision.Complete, new ToolUseGuard().Evaluate(false, false, false).Decision); + } + + [Fact] + public void Task_update_mutates_the_real_deterministic_graph_without_external_effect_evidence() + { + Guid scope = Guid.NewGuid(); + Guid node = Guid.NewGuid(); + var handler = new TaskGraphToolHandler(); + handler.BeginTask(scope); + using JsonDocument add = JsonDocument.Parse($$"""{"operation":"add","task_id":"{{node}}","description":"Build fixture","depends_on":[],"budget_class":"1K","optimistic_tokens":10,"most_likely_tokens":20,"pessimistic_tokens":30}"""); + + ToolExecutionResult result = handler.Execute(scope, add.RootElement); + + Assert.True(result.Success); + Assert.False(result.ExternalEffect); + Assert.Contains(node.ToString(), result.Output, StringComparison.OrdinalIgnoreCase); + handler.EndTask(scope); + Assert.Throws<InvalidOperationException>(() => handler.Execute(scope, add.RootElement)); + } +} diff --git a/tests/Agent.Core.Tests/LocaleCatalogTests.cs b/tests/Agent.Core.Tests/LocaleCatalogTests.cs new file mode 100644 index 0000000..e5b6085 --- /dev/null +++ b/tests/Agent.Core.Tests/LocaleCatalogTests.cs @@ -0,0 +1,54 @@ +using Agent.Core.Localization; + +namespace Agent.Core.Tests; + +public sealed class LocaleCatalogTests +{ + [Fact] + public void MissingSelectedKeyFallsBackToEnglish() + { + using var directory = new TestDirectory(); + directory.File("en.json", """{"hello":"Hello {name}","only_en":"fallback"}"""); + directory.File("ja.json", """{"hello":"こんにちは {name}"}"""); + LocaleCatalog catalog = LocaleCatalog.Load(directory.Path, "ja"); + Assert.Equal("fallback", catalog.Get("only_en")); + Assert.Equal("こんにちは Codex", catalog.Format("hello", new Dictionary<string, object?> { ["name"] = "Codex" })); + } + + [Fact] + public void AdditionalLocaleLoadsWithoutCodeChange() + { + using var directory = new TestDirectory(); + directory.File("en.json", """{"hello":"Hello"}"""); + directory.File("zh-CN.json", """{"hello":"你好"}"""); + Assert.Equal("你好", LocaleCatalog.Load(directory.Path, "zh-CN").Get("hello")); + } + + [Theory] + [InlineData("{broken")] + [InlineData("[]")] + public void MalformedLocaleIsFatal(string json) + { + using var directory = new TestDirectory(); + directory.File("en.json", json); + Assert.Throws<LocaleException>(() => LocaleCatalog.Load(directory.Path, "en")); + } + + [Fact] + public void UnknownKeyIsFatal() + { + using var directory = new TestDirectory(); + directory.File("en.json", """{"known":"value"}"""); + directory.File("ja.json", """{"known":"値","unknown":"x"}"""); + Assert.Throws<LocaleException>(() => LocaleCatalog.Load(directory.Path, "ja")); + } + + [Fact] + public void PlaceholderMismatchIsFatal() + { + using var directory = new TestDirectory(); + directory.File("en.json", """{"hello":"Hello {name}"}"""); + directory.File("ja.json", """{"hello":"こんにちは {user}"}"""); + Assert.Throws<LocaleException>(() => LocaleCatalog.Load(directory.Path, "ja")); + } +} diff --git a/tests/Agent.Core.Tests/McpHttpTests.cs b/tests/Agent.Core.Tests/McpHttpTests.cs new file mode 100644 index 0000000..7e3f1cf --- /dev/null +++ b/tests/Agent.Core.Tests/McpHttpTests.cs @@ -0,0 +1,88 @@ +using System.Net; +using System.Net.Sockets; +using Agent.Core.Localization; +using Agent.Core.Ollama; +using Agent.Core.Runtime; +using Agent.Mcp; +using ModelContextProtocol; +using ModelContextProtocol.Client; + +namespace Agent.Core.Tests; + +public sealed class McpHttpTests +{ + [Fact(Timeout = 30_000)] + public async Task Loopback_streamable_http_works_and_get_sse_legacy_sse_are_rejected() + { + int port = FreePort(); + using var stop = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); + LocaleCatalog locale = LocaleCatalog.Load(Path.GetFullPath("config/locales"), "en"); + var settings = new OllamaSettings(new Uri("http://127.0.0.1:1"), "unused", false, "0", new Dictionary<string, System.Text.Json.JsonElement>()); + Task server = McpHost.RunHttpAsync(locale, settings, new AgentRuntimeSettings("", "en", [], "sandbox"), new VisionLimits(), "127.0.0.1", port, stop.Token); + using var http = new HttpClient(); + Uri endpoint = new($"http://127.0.0.1:{port}/mcp"); + await WaitUntilListening(http, endpoint, TestContext.Current.CancellationToken); + + var transport = new HttpClientTransport(new HttpClientTransportOptions + { + Name = "http-test", + Endpoint = endpoint, + TransportMode = HttpTransportMode.StreamableHttp + }); + await using (McpClient client = await McpClient.CreateAsync(transport, cancellationToken: TestContext.Current.CancellationToken)) + { + IList<McpClientTool> tools = await client.ListToolsAsync(cancellationToken: TestContext.Current.CancellationToken); + Assert.Equal(["agent_run", "agent_status"], tools.Select(static tool => tool.Name).Order(StringComparer.Ordinal)); + } + + using HttpResponseMessage get = await http.GetAsync(endpoint, TestContext.Current.CancellationToken); + Assert.Equal(HttpStatusCode.MethodNotAllowed, get.StatusCode); + using var hostileOrigin = new HttpRequestMessage(HttpMethod.Post, endpoint); + hostileOrigin.Headers.Add("Origin", "https://example.invalid"); + using HttpResponseMessage forbidden = await http.SendAsync(hostileOrigin, TestContext.Current.CancellationToken); + Assert.Equal(HttpStatusCode.Forbidden, forbidden.StatusCode); + using var loopbackOrigin = new HttpRequestMessage(HttpMethod.Get, endpoint); + loopbackOrigin.Headers.Add("Origin", $"http://127.0.0.1:{port}"); + using HttpResponseMessage loopbackAllowed = await http.SendAsync(loopbackOrigin, TestContext.Current.CancellationToken); + Assert.Equal(HttpStatusCode.MethodNotAllowed, loopbackAllowed.StatusCode); + Assert.Equal(HttpStatusCode.NotFound, (await http.GetAsync(new Uri($"http://127.0.0.1:{port}/sse"), TestContext.Current.CancellationToken)).StatusCode); + Assert.Equal(HttpStatusCode.NotFound, (await http.PostAsync(new Uri($"http://127.0.0.1:{port}/message"), null, TestContext.Current.CancellationToken)).StatusCode); + + await stop.CancelAsync(); + await server; + } + + [Fact] + public async Task Non_loopback_bind_is_rejected_before_server_start() + { + LocaleCatalog locale = LocaleCatalog.Load(Path.GetFullPath("config/locales"), "en"); + var settings = new OllamaSettings(new Uri("http://127.0.0.1:1"), "unused", false, "0", new Dictionary<string, System.Text.Json.JsonElement>()); + await Assert.ThrowsAsync<ArgumentException>(() => McpHost.RunHttpAsync(locale, settings, + new AgentRuntimeSettings("", "en", [], "sandbox"), new VisionLimits(), "0.0.0.0", 8080, TestContext.Current.CancellationToken)); + } + + private static int FreePort() + { + TcpListener listener = new(IPAddress.Loopback, 0); + listener.Start(); + int port = ((IPEndPoint)listener.LocalEndpoint).Port; + listener.Stop(); + return port; + } + + private static async Task WaitUntilListening(HttpClient client, Uri endpoint, CancellationToken cancellationToken) + { + for (int attempt = 0; attempt < 100; attempt++) + { + try + { + using HttpResponseMessage response = await client.GetAsync(endpoint, cancellationToken); + return; + } + catch (HttpRequestException) when (attempt < 99) + { + await Task.Delay(25, cancellationToken); + } + } + } +} diff --git a/tests/Agent.Core.Tests/McpStdioTests.cs b/tests/Agent.Core.Tests/McpStdioTests.cs new file mode 100644 index 0000000..66f4851 --- /dev/null +++ b/tests/Agent.Core.Tests/McpStdioTests.cs @@ -0,0 +1,157 @@ +using Agent.App.Configuration; +using ModelContextProtocol; +using ModelContextProtocol.Client; +using ModelContextProtocol.Protocol; +using System.Net; +using System.Net.Sockets; +using System.Text; + +namespace Agent.Core.Tests; + +public sealed class McpStdioTests +{ + [Fact(Timeout = 30_000)] + public async Task InitializeListAndCallExposeOnlyTwoStrictTools() + { + using var directory = new TestDirectory(); + await using var ollama = new FakeOllamaServer(); + string configPath = CreateBundleConfiguration(directory, ollama.Endpoint); + string application = typeof(ConfigurationLoader).Assembly.Location; + var transport = new StdioClientTransport(new StdioClientTransportOptions + { + Name = "agent-mcp-test", + Command = "dotnet", + Arguments = [application, "serve", "--config", configPath, "--lang", "en"], + WorkingDirectory = directory.Path + }); + + await using McpClient client = await McpClient.CreateAsync(transport, cancellationToken: TestContext.Current.CancellationToken); + IList<McpClientTool> tools = await client.ListToolsAsync(cancellationToken: TestContext.Current.CancellationToken); + Assert.Equal(["agent_run", "agent_status"], tools.Select(static tool => tool.Name).Order(StringComparer.Ordinal)); + foreach (McpClientTool tool in tools) + { + Assert.False(tool.JsonSchema.GetProperty("additionalProperties").GetBoolean()); + } + + CallToolResult run = await client.CallToolAsync( + "agent_run", + new Dictionary<string, object?> { ["request"] = "deterministic test" }, + cancellationToken: TestContext.Current.CancellationToken); + string text = Assert.IsType<TextContentBlock>(Assert.Single(run.Content)).Text; + Assert.Contains("status: queued", text, StringComparison.Ordinal); + Guid taskId = ReadGuid(text, "task_id"); + Guid sessionId = ReadGuid(text, "session_id"); + + CallToolResult status = await client.CallToolAsync( + "agent_status", + new Dictionary<string, object?> { ["task_id"] = taskId, ["session_id"] = sessionId }, + cancellationToken: TestContext.Current.CancellationToken); + Assert.Contains("task_id:", Assert.IsType<TextContentBlock>(Assert.Single(status.Content)).Text, StringComparison.Ordinal); + + CallToolResult invalid = await client.CallToolAsync( + "agent_run", + new Dictionary<string, object?> { ["request"] = "x", ["model"] = "forbidden" }, + cancellationToken: TestContext.Current.CancellationToken); + Assert.True(invalid.IsError); + + CallToolResult invalidImage = await client.CallToolAsync( + "agent_run", + new Dictionary<string, object?> + { + ["request"] = "x", + ["images"] = new[] { new Dictionary<string, object?> { ["mime_type"] = "image/gif", ["data_base64"] = "AA==" } } + }, + cancellationToken: TestContext.Current.CancellationToken); + Assert.True(invalidImage.IsError); + } + + private static Guid ReadGuid(string text, string key) + { + string line = text.Split('\n', StringSplitOptions.TrimEntries).Single(value => value.StartsWith(key + ":", StringComparison.Ordinal)); + return Guid.Parse(line[(key.Length + 1)..].Trim()); + } + + private static string CreateBundleConfiguration(TestDirectory directory, Uri endpoint) + { + directory.File("prompts/user-system.md", string.Empty); + directory.File("config/locales/en.json", """{"app.starting":"Starting","check.success":"OK","check.failure":"Failed {message}","check.wfp_admin_required":"Administrator required","config.invalid":"Invalid {message}","locale.invalid":"Locale {message}","mcp.task_queued":"Task {task} queued in {session}","mcp.task_not_found":"Task {task} not found","sandbox.access_denied":"Denied {path}","ollama.vision_unsupported":"No vision"}"""); + Directory.CreateDirectory(System.IO.Path.Combine(directory.Path, "skills")); + return directory.File("agent.yaml", $$""" + schema_version: 1 + application: + language: en + mcp: + transport: stdio + agent: + user_system_prompt: prompts/user-system.md + ollama: + endpoint: {{endpoint}} + model: model + localization: + directory: config/locales + fallback: en + skills: + directory: skills + logging: + directory: logs + """); + } + + private sealed class FakeOllamaServer : IAsyncDisposable + { + private readonly TcpListener _listener = new(IPAddress.Loopback, 0); + private readonly CancellationTokenSource _stop = new(); + private readonly Task _loop; + + public FakeOllamaServer() + { + _listener.Start(); + Endpoint = new Uri($"http://127.0.0.1:{((IPEndPoint)_listener.LocalEndpoint).Port}/"); + _loop = RunAsync(); + } + + public Uri Endpoint { get; } + + private async Task RunAsync() + { + try + { + while (!_stop.IsCancellationRequested) + { + using TcpClient client = await _listener.AcceptTcpClientAsync(_stop.Token); + await HandleAsync(client, _stop.Token); + } + } + catch (OperationCanceledException) when (_stop.IsCancellationRequested) { } + } + + private static async Task HandleAsync(TcpClient client, CancellationToken cancellationToken) + { + NetworkStream stream = client.GetStream(); + using var reader = new StreamReader(stream, Encoding.ASCII, false, 1024, true); + int contentLength = 0; + while (await reader.ReadLineAsync(cancellationToken) is { } line && line.Length > 0) + { + if (line.StartsWith("Content-Length:", StringComparison.OrdinalIgnoreCase)) + contentLength = int.Parse(line[15..].Trim(), System.Globalization.CultureInfo.InvariantCulture); + } + if (contentLength > 0) + { + char[] body = new char[contentLength]; + await reader.ReadBlockAsync(body.AsMemory(), cancellationToken); + } + byte[] payload = Encoding.UTF8.GetBytes("""{"message":{"role":"assistant","content":"done"},"prompt_eval_count":3,"eval_count":2}"""); + byte[] header = Encoding.ASCII.GetBytes($"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {payload.Length}\r\nConnection: close\r\n\r\n"); + await stream.WriteAsync(header, cancellationToken); + await stream.WriteAsync(payload, cancellationToken); + } + + public async ValueTask DisposeAsync() + { + await _stop.CancelAsync(); + _listener.Stop(); + await _loop; + _stop.Dispose(); + } + } +} diff --git a/tests/Agent.Core.Tests/OllamaClientTests.cs b/tests/Agent.Core.Tests/OllamaClientTests.cs new file mode 100644 index 0000000..aa98764 --- /dev/null +++ b/tests/Agent.Core.Tests/OllamaClientTests.cs @@ -0,0 +1,143 @@ +using System.Net; +using System.Text; +using System.Text.Json; +using Agent.App.Configuration; +using Agent.Core.Ollama; +using Agent.Core.Tools; + +namespace Agent.Core.Tests; + +public sealed class OllamaClientTests +{ + [Fact] + public async Task Chat_sends_expected_request_and_returns_usage() + { + string? body = null; + var handler = new StubHandler(async (request, _) => + { + body = await request.Content!.ReadAsStringAsync(); + return Json(HttpStatusCode.OK, """{"message":{"role":"assistant","content":"done"},"prompt_eval_count":12,"eval_count":7}"""); + }); + OllamaClient client = Create(handler, options: new Dictionary<string, JsonElement> + { + ["temperature"] = JsonSerializer.SerializeToElement(0.2), + ["seed"] = JsonSerializer.SerializeToElement(42) + }); + + OllamaResult result = await client.ChatAsync([new OllamaMessage("user", "hello")], 0.7, TestContext.Current.CancellationToken); + + Assert.Equal("done", result.Content); + Assert.Equal(12, result.InputTokens); + Assert.Equal(7, result.OutputTokens); + using JsonDocument document = JsonDocument.Parse(body!); + JsonElement root = document.RootElement; + Assert.Equal("test-model", root.GetProperty("model").GetString()); + Assert.False(root.GetProperty("stream").GetBoolean()); + Assert.Equal(0.7, root.GetProperty("options").GetProperty("temperature").GetDouble()); + Assert.Equal(42, root.GetProperty("options").GetProperty("seed").GetInt32()); + } + + [Fact] + public async Task Transient_failure_retries_three_times_with_configured_backoff() + { + int attempts = 0; + var delays = new List<TimeSpan>(); + var handler = new StubHandler((_, _) => + { + attempts++; + return Task.FromResult(attempts < 4 + ? Json(HttpStatusCode.ServiceUnavailable, "busy") + : Json(HttpStatusCode.OK, """{"message":{"role":"assistant","content":"ok"}}""")); + }); + var client = new OllamaClient(new HttpClient(handler), Settings(), (delay, _) => + { + delays.Add(delay); + return Task.CompletedTask; + }); + + OllamaResult result = await client.ChatAsync([new OllamaMessage("user", "hello")], null, TestContext.Current.CancellationToken); + + Assert.Equal("ok", result.Content); + Assert.Equal(4, attempts); + Assert.Equal([TimeSpan.FromMilliseconds(250), TimeSpan.FromMilliseconds(500), TimeSpan.FromMilliseconds(1000)], delays); + } + + [Fact] + public async Task Permanent_failure_is_not_retried() + { + int attempts = 0; + var handler = new StubHandler((_, _) => + { + attempts++; + return Task.FromResult(Json(HttpStatusCode.BadRequest, "invalid")); + }); + OllamaClient client = Create(handler); + + OllamaException error = await Assert.ThrowsAsync<OllamaException>(() => + client.ChatAsync([new OllamaMessage("user", "hello")], null, TestContext.Current.CancellationToken)); + + Assert.False(error.IsTransient); + Assert.Equal(1, attempts); + } + + [Fact] + public void Typed_options_override_extra_options() + { + var configuration = new OllamaConfiguration + { + Endpoint = "http://127.0.0.1:11434", + Model = "model", + Options = new OllamaGenerationOptions { Temperature = 0.4, NumCtx = 8192 }, + ExtraOptions = new Dictionary<string, object?> + { + ["temperature"] = 1.9, + ["num_ctx"] = 256, + ["custom"] = "preserved" + } + }; + + OllamaSettings settings = OllamaSettingsFactory.Create(configuration); + + Assert.Equal(0.4, settings.Options["temperature"].GetDouble()); + Assert.Equal(8192, settings.Options["num_ctx"].GetInt32()); + Assert.Equal("preserved", settings.Options["custom"].GetString()); + } + + [Fact] + public async Task Native_tool_catalog_is_serialized_and_tool_call_is_parsed() + { + string? body = null; + var handler = new StubHandler(async (request, _) => + { + body = await request.Content!.ReadAsStringAsync(); + return Json(HttpStatusCode.OK, """{"message":{"role":"assistant","content":"","tool_calls":[{"function":{"name":"file_read","arguments":{"path":"x","mode":"whole"}}}]},"prompt_eval_count":2,"eval_count":1}"""); + }); + OllamaClient client = Create(handler); + + OllamaResult result = await client.ChatWithToolsAsync([new OllamaMessage("user", "read")], null, InternalToolCatalog.All, TestContext.Current.CancellationToken); + + using JsonDocument document = JsonDocument.Parse(body!); + Assert.Equal(5, document.RootElement.GetProperty("tools").GetArrayLength()); + Assert.All(document.RootElement.GetProperty("tools").EnumerateArray(), tool => + Assert.False(tool.GetProperty("function").GetProperty("parameters").GetProperty("additionalProperties").GetBoolean())); + OllamaToolCall call = Assert.Single(result.ToolCalls!); + Assert.Equal("file_read", call.Function.Name); + Assert.Equal("x", call.Function.Arguments.GetProperty("path").GetString()); + } + + private static OllamaClient Create(HttpMessageHandler handler, IReadOnlyDictionary<string, JsonElement>? options = null) => + new(new HttpClient(handler), Settings(options), static (_, _) => Task.CompletedTask); + + private static OllamaSettings Settings(IReadOnlyDictionary<string, JsonElement>? options = null) => + new(new Uri("http://127.0.0.1:11434"), "test-model", false, "0", options ?? new Dictionary<string, JsonElement>()); + + private static HttpResponseMessage Json(HttpStatusCode statusCode, string content) => new(statusCode) + { + Content = new StringContent(content, Encoding.UTF8, "application/json") + }; + + private sealed class StubHandler(Func<HttpRequestMessage, CancellationToken, Task<HttpResponseMessage>> callback) : HttpMessageHandler + { + protected override Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) => callback(request, cancellationToken); + } +} diff --git a/tests/Agent.Core.Tests/PromptAndRuntimeTests.cs b/tests/Agent.Core.Tests/PromptAndRuntimeTests.cs new file mode 100644 index 0000000..23ad2d2 --- /dev/null +++ b/tests/Agent.Core.Tests/PromptAndRuntimeTests.cs @@ -0,0 +1,220 @@ +using Agent.Core.Ollama; +using Agent.Core.Prompts; +using Agent.Core.Runtime; +using Agent.Core.Sessions; +using Agent.Core.Tasks; +using Agent.Core.Tools; +using System.Text.Json; + +namespace Agent.Core.Tests; + +public sealed class PromptAndRuntimeTests +{ + [Fact] + public void Prompt_order_is_fixed_and_request_is_last() + { + IReadOnlyList<OllamaMessage> messages = PromptComposer.Compose(new PromptContext( + "fixed", "user system", "ja", ["skill"], "capability", "graph", "summary", + [new ConversationMessage("user", "old"), new ConversationMessage("assistant", "reply")], + "new request", ["image"])); + + Assert.Equal(["fixed", "user system", "Respond in locale: ja.", "skill", "capability", "graph", "summary", "old", "reply", "new request"], messages.Select(x => x.Content)); + Assert.Equal("system", messages[0].Role); + Assert.Equal("user", messages[^1].Role); + Assert.Equal(["image"], messages[^1].Images); + } + + [Fact] + public async Task Runtime_continues_session_and_accumulates_usage() + { + var sessions = new SessionStore(); + var registry = new TaskRegistry(sessions); + var ollama = new RecordingOllamaClient(); + var runtime = new AgentRuntime(registry, ollama, new AgentRuntimeSettings("custom", "ja", [], "sandbox")); + TaskSnapshot first = registry.Enqueue("first", null, 0.5); + Assert.True(registry.Transition(first.TaskId, AgentTaskState.Planning)); + Assert.True(registry.Transition(first.TaskId, AgentTaskState.Running)); + + await runtime.ExecuteAsync(first.TaskId, TestContext.Current.CancellationToken); + Assert.True(registry.TryGet(first.TaskId, null, out TaskSnapshot? completed)); + Assert.True(completed!.State == AgentTaskState.Completed, completed.Result); + Assert.Equal(3, completed.InputTokens); + Assert.Equal(2, completed.OutputTokens); + + TaskSnapshot second = registry.Enqueue("second", first.SessionId, null); + Assert.True(registry.Transition(second.TaskId, AgentTaskState.Planning)); + Assert.True(registry.Transition(second.TaskId, AgentTaskState.Running)); + await runtime.ExecuteAsync(second.TaskId, TestContext.Current.CancellationToken); + + Assert.Equal(2, ollama.Calls.Count); + Assert.Contains(ollama.Calls[1], message => message.Role == "user" && message.Content == "first"); + Assert.Contains(ollama.Calls[1], message => message.Role == "assistant" && message.Content == "answer-1"); + Assert.Equal(0.5, ollama.Temperatures[0]); + Assert.Null(ollama.Temperatures[1]); + } + + [Fact] + public void Session_store_is_process_local() + { + SessionStore first = new(); + Guid id = first.Create().SessionId; + SessionStore restarted = new(); + + Assert.False(restarted.TryGet(id, out _)); + } + + [Fact] + public async Task Non_vision_model_rejects_images_with_selected_locale_message_before_chat() + { + TaskRegistry registry = new(); + var ollama = new NonVisionOllamaClient(); + const string localized = "選択したモデルは画像入力に対応していません。"; + var runtime = new AgentRuntime(registry, ollama, + new AgentRuntimeSettings("", "ja", [], "sandbox", VisionUnsupportedMessage: localized)); + TaskSnapshot task = registry.Enqueue("画像を確認", null, null, ["base64-image"]); + registry.Transition(task.TaskId, AgentTaskState.Planning); + registry.Transition(task.TaskId, AgentTaskState.Running); + + await runtime.ExecuteAsync(task.TaskId, TestContext.Current.CancellationToken); + + Assert.True(registry.TryGet(task.TaskId, null, out TaskSnapshot? failed)); + Assert.Equal(AgentTaskState.Failed, failed!.State); + Assert.Equal(localized, failed.Result); + Assert.Equal(0, ollama.ChatCalls); + } + + [Fact] + public async Task Runtime_dispatches_native_tool_call_then_completes_only_after_effect_evidence() + { + TaskRegistry registry = new(); + FakeInternalToolBackend backend = new(); + var dispatcher = new InternalToolDispatcher( + new DenyByDefaultPolicy(InternalToolCatalog.All.Select(static item => item.Name).ToHashSet(StringComparer.Ordinal)), backend); + var ollama = new ToolCallingOllamaClient(); + var runtime = new AgentRuntime(registry, ollama, new AgentRuntimeSettings("", "ja", [], "sandbox"), dispatcher); + TaskSnapshot task = registry.Enqueue("fileを修正", null, null); + registry.Transition(task.TaskId, AgentTaskState.Planning); + registry.Transition(task.TaskId, AgentTaskState.Running); + + await runtime.ExecuteAsync(task.TaskId, TestContext.Current.CancellationToken); + + Assert.True(registry.TryGet(task.TaskId, null, out TaskSnapshot? completed)); + Assert.True(completed!.State == AgentTaskState.Completed, $"{completed.Result}; backend={backend.Calls.Count}:{string.Join(',', backend.Calls.Select(x => x.Name))}; ollama={ollama.CallCount}"); + Assert.Equal("修正しました。", completed.Result); + Assert.Single(backend.Calls); + Assert.Equal("file_edit", backend.Calls[0].Name); + Assert.Equal(2, ollama.CallCount); + Assert.Equal(5, ollama.ToolCount); + } + + [Fact] + public async Task Runtime_reaches_terminal_failed_state_when_resource_cleanup_fails() + { + TaskRegistry registry = new(); + var backend = new CleanupFailingBackend(); + var dispatcher = new InternalToolDispatcher(new DenyByDefaultPolicy(), backend); + var runtime = new AgentRuntime(registry, new RecordingOllamaClient(), new AgentRuntimeSettings("", "en", [], "sandbox"), dispatcher); + TaskSnapshot task = registry.Enqueue("analyze", null, null); + registry.Transition(task.TaskId, AgentTaskState.Planning); + registry.Transition(task.TaskId, AgentTaskState.Running); + + await runtime.ExecuteAsync(task.TaskId, TestContext.Current.CancellationToken); + + Assert.True(registry.TryGet(task.TaskId, null, out TaskSnapshot? failed)); + Assert.Equal(AgentTaskState.Failed, failed!.State); + Assert.Contains("cleanup failed", failed.Result, StringComparison.OrdinalIgnoreCase); + Assert.Equal(2, backend.EndCalls); + } + + [Fact] + public async Task Runtime_cannot_complete_with_an_unresolved_tool_failure() + { + TaskRegistry registry = new(); + var dispatcher = new InternalToolDispatcher( + new DenyByDefaultPolicy(new HashSet<string>(["process_run"], StringComparer.Ordinal)), new FailingProcessBackend()); + var runtime = new AgentRuntime(registry, new FailingThenFinalOllamaClient(), new AgentRuntimeSettings("", "en", [], "sandbox", MaxIterations: 2), dispatcher); + TaskSnapshot task = registry.Enqueue("run build", null, null); + registry.Transition(task.TaskId, AgentTaskState.Planning); + registry.Transition(task.TaskId, AgentTaskState.Running); + + await runtime.ExecuteAsync(task.TaskId, TestContext.Current.CancellationToken); + + Assert.True(registry.TryGet(task.TaskId, null, out TaskSnapshot? failed)); + Assert.Equal(AgentTaskState.Failed, failed!.State); + Assert.Contains("iteration limit", failed.Result, StringComparison.OrdinalIgnoreCase); + } + + private sealed class RecordingOllamaClient : IOllamaClient + { + public List<IReadOnlyList<OllamaMessage>> Calls { get; } = []; + public List<double?> Temperatures { get; } = []; + + public Task<OllamaResult> ChatAsync(IReadOnlyList<OllamaMessage> messages, double? temperatureOverride, CancellationToken cancellationToken) + { + Calls.Add(messages); + Temperatures.Add(temperatureOverride); + return Task.FromResult(new OllamaResult($"answer-{Calls.Count}", 3, 2)); + } + } + + private sealed class ToolCallingOllamaClient : IOllamaClient + { + public int CallCount { get; private set; } + public int ToolCount { get; private set; } + public Task<OllamaResult> ChatAsync(IReadOnlyList<OllamaMessage> messages, double? temperatureOverride, CancellationToken cancellationToken) => + Task.FromResult(new OllamaResult("修正しました。", 1, 1)); + public Task<OllamaResult> ChatWithToolsAsync(IReadOnlyList<OllamaMessage> messages, double? temperatureOverride, IReadOnlyList<InternalToolDefinition> tools, CancellationToken cancellationToken) + { + CallCount++; + ToolCount = tools.Count; + if (CallCount == 1) + { + using JsonDocument document = JsonDocument.Parse("""{"path":"x","operation":"overwrite","content":"new"}"""); + return Task.FromResult(new OllamaResult("", 2, 1, [new OllamaToolCall(new OllamaFunctionCall("file_edit", document.RootElement.Clone()))])); + } + return Task.FromResult(new OllamaResult("修正しました。", 2, 2)); + } + } + + private sealed class NonVisionOllamaClient : IOllamaClient + { + public int ChatCalls { get; private set; } + public Task<bool?> SupportsVisionAsync(CancellationToken cancellationToken) => Task.FromResult<bool?>(false); + public Task<OllamaResult> ChatAsync(IReadOnlyList<OllamaMessage> messages, double? temperatureOverride, CancellationToken cancellationToken) + { + ChatCalls++; + return Task.FromResult(new OllamaResult("unexpected", 0, 0)); + } + } + + private sealed class CleanupFailingBackend : IInternalToolBackend + { + public int EndCalls { get; private set; } + public Task<ToolExecutionResult> ExecuteAsync(Guid taskId, string toolName, JsonElement arguments, CancellationToken cancellationToken) => + Task.FromResult(new ToolExecutionResult(true, "unused", false)); + public Task EndTaskAsync(Guid taskId, bool success, CancellationToken cancellationToken) + { + EndCalls++; + throw new IOException("cleanup failed"); + } + } + + private sealed class FailingProcessBackend : IInternalToolBackend + { + public Task<ToolExecutionResult> ExecuteAsync(Guid taskId, string toolName, JsonElement arguments, CancellationToken cancellationToken) => + Task.FromResult(new ToolExecutionResult(false, "exit 1", true, "process:1")); + } + + private sealed class FailingThenFinalOllamaClient : IOllamaClient + { + private int _calls; + public Task<OllamaResult> ChatAsync(IReadOnlyList<OllamaMessage> messages, double? temperatureOverride, CancellationToken cancellationToken) => + Task.FromResult(new OllamaResult("done", 1, 1)); + public Task<OllamaResult> ChatWithToolsAsync(IReadOnlyList<OllamaMessage> messages, double? temperatureOverride, IReadOnlyList<InternalToolDefinition> tools, CancellationToken cancellationToken) + { + if (_calls++ > 0) return Task.FromResult(new OllamaResult("done", 1, 1)); + using JsonDocument arguments = JsonDocument.Parse("""{"executable":"x","args":[],"cwd":"."}"""); + return Task.FromResult(new OllamaResult("", 1, 1, [new OllamaToolCall(new OllamaFunctionCall("process_run", arguments.RootElement.Clone()))])); + } + } +} diff --git a/tests/Agent.Core.Tests/TaskRegistryTests.cs b/tests/Agent.Core.Tests/TaskRegistryTests.cs new file mode 100644 index 0000000..33e16ab --- /dev/null +++ b/tests/Agent.Core.Tests/TaskRegistryTests.cs @@ -0,0 +1,35 @@ +using Agent.Core.Tasks; + +namespace Agent.Core.Tests; + +public sealed class TaskRegistryTests +{ + [Fact] + public void NewTaskGetsUniqueTaskAndSessionIds() + { + var registry = new TaskRegistry(); + TaskSnapshot first = registry.Enqueue("one", null, null); + TaskSnapshot second = registry.Enqueue("two", null, 0.2); + Assert.NotEqual(first.TaskId, second.TaskId); + Assert.NotEqual(first.SessionId, second.SessionId); + Assert.Equal(AgentTaskState.Queued, first.State); + } + + [Fact] + public void UnknownSessionIsRejected() + { + var registry = new TaskRegistry(); + Assert.Throws<KeyNotFoundException>(() => registry.Enqueue("request", Guid.NewGuid(), null)); + } + + [Fact] + public void TerminalTaskCannotTransition() + { + var registry = new TaskRegistry(); + TaskSnapshot task = registry.Enqueue("request", null, null); + Assert.True(registry.Transition(task.TaskId, AgentTaskState.Planning)); + Assert.True(registry.Transition(task.TaskId, AgentTaskState.Running)); + Assert.True(registry.Transition(task.TaskId, AgentTaskState.Completed)); + Assert.False(registry.Transition(task.TaskId, AgentTaskState.Running)); + } +} diff --git a/tests/Agent.Core.Tests/TestDirectory.cs b/tests/Agent.Core.Tests/TestDirectory.cs new file mode 100644 index 0000000..075d050 --- /dev/null +++ b/tests/Agent.Core.Tests/TestDirectory.cs @@ -0,0 +1,21 @@ +namespace Agent.Core.Tests; + +internal sealed class TestDirectory : IDisposable +{ + public TestDirectory() + { + Path = System.IO.Path.Combine(System.IO.Path.GetTempPath(), "agent-mcp-tests", Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(Path); + } + + public string Path { get; } + public string File(string relativePath, string content) + { + string path = System.IO.Path.Combine(Path, relativePath); + Directory.CreateDirectory(System.IO.Path.GetDirectoryName(path)!); + System.IO.File.WriteAllText(path, content); + return path; + } + + public void Dispose() => Directory.Delete(Path, true); +} diff --git a/tests/Agent.Core.Tests/Usings.cs b/tests/Agent.Core.Tests/Usings.cs new file mode 100644 index 0000000..c802f44 --- /dev/null +++ b/tests/Agent.Core.Tests/Usings.cs @@ -0,0 +1 @@ +global using Xunit; diff --git a/tests/Agent.Planner.Tests/Agent.Planner.Tests.csproj b/tests/Agent.Planner.Tests/Agent.Planner.Tests.csproj new file mode 100644 index 0000000..16264cb --- /dev/null +++ b/tests/Agent.Planner.Tests/Agent.Planner.Tests.csproj @@ -0,0 +1,14 @@ +<Project Sdk="Microsoft.NET.Sdk"> + <PropertyGroup> + <IsPackable>false</IsPackable> + <IsTestProject>true</IsTestProject> + <TestingPlatformDotnetTestSupport>true</TestingPlatformDotnetTestSupport> + </PropertyGroup> + <ItemGroup> + <ProjectReference Include="..\..\src\Agent.Planner\Agent.Planner.csproj" /> + </ItemGroup> + <ItemGroup> + <PackageReference Include="Microsoft.NET.Test.Sdk" /> + <PackageReference Include="xunit.v3" /> + </ItemGroup> +</Project> diff --git a/tests/Agent.Planner.Tests/BaselineTests.cs b/tests/Agent.Planner.Tests/BaselineTests.cs new file mode 100644 index 0000000..a459b18 --- /dev/null +++ b/tests/Agent.Planner.Tests/BaselineTests.cs @@ -0,0 +1,10 @@ +namespace Agent.Planner.Tests; + +public sealed class BaselineTests +{ + [Fact] + public void PlannerAssemblyLoads() + { + Assert.Equal("Agent.Planner", typeof(Planner.AssemblyMarker).Assembly.GetName().Name); + } +} diff --git a/tests/Agent.Planner.Tests/TaskGraphTests.cs b/tests/Agent.Planner.Tests/TaskGraphTests.cs new file mode 100644 index 0000000..b09cd56 --- /dev/null +++ b/tests/Agent.Planner.Tests/TaskGraphTests.cs @@ -0,0 +1,80 @@ +using Agent.Planner; + +namespace Agent.Planner.Tests; + +public sealed class TaskGraphTests +{ + private static readonly TokenEstimate Small = new(60, 60, 60); + + [Fact] + public void Known_dag_has_deterministic_critical_path() + { + TaskGraph graph = new(); + PlannerTask a = graph.AddTask("A", TokenBudgetClass.K1, new TokenEstimate(60, 60, 60)); + PlannerTask b = graph.AddTask("B", TokenBudgetClass.K1, new TokenEstimate(120, 120, 120), [a.Id]); + _ = graph.AddTask("C", TokenBudgetClass.K1, new TokenEstimate(30, 30, 30), [a.Id]); + PlannerTask d = graph.AddTask("D", TokenBudgetClass.K1, new TokenEstimate(60, 60, 60), [b.Id]); + + Assert.Equal([a.Id, b.Id, d.Id], graph.CriticalPath.TaskIds); + Assert.Equal(240, graph.CriticalPath.CumulativeTokenCost); + } + + [Fact] + public void Cycle_is_rejected_without_mutating_graph() + { + TaskGraph graph = new(); + PlannerTask a = graph.AddTask("A", TokenBudgetClass.K1, Small); + PlannerTask b = graph.AddTask("B", TokenBudgetClass.K1, Small, [a.Id]); + + Assert.Throws<PlannerValidationException>(() => graph.AddDependency(a.Id, b.Id)); + Assert.Empty(graph.Get(a.Id).Dependencies); + } + + [Fact] + public void Max_tasks_is_enforced() + { + TaskGraph graph = new(1); + _ = graph.AddTask("A", TokenBudgetClass.K1, Small); + Assert.Throws<PlannerValidationException>(() => graph.AddTask("B", TokenBudgetClass.K1, Small)); + } + + [Theory] + [InlineData(0, 1, 1)] + [InlineData(2, 1, 2)] + [InlineData(1, 3, 2)] + [InlineData(1, 2, 2048)] + public void Invalid_estimate_is_rejected(long optimistic, long likely, long pessimistic) + { + TaskGraph graph = new(); + Assert.Throws<PlannerValidationException>(() => graph.AddTask("bad", TokenBudgetClass.K1, new TokenEstimate(optimistic, likely, pessimistic))); + } + + [Fact] + public void Completed_task_is_immutable() + { + TaskGraph graph = new(); + PlannerTask task = graph.AddTask("A", TokenBudgetClass.K1, Small); + graph.SetState(task.Id, PlannerTaskState.Completed); + + Assert.Throws<PlannerValidationException>(() => graph.SetEstimate(task.Id, TokenBudgetClass.K2, new TokenEstimate(1, 2, 3))); + Assert.Throws<PlannerValidationException>(() => graph.SetState(task.Id, PlannerTaskState.Failed)); + Assert.Throws<PlannerValidationException>(() => graph.AddUsage(task.Id, 1, 1)); + } + + [Fact] + public void Graph_change_recalculates_cpm_and_usage_accumulates() + { + TaskGraph graph = new(); + PlannerTask a = graph.AddTask("A", TokenBudgetClass.K1, Small); + PlannerTask b = graph.AddTask("B", TokenBudgetClass.K1, new TokenEstimate(30, 30, 30)); + Assert.Equal([a.Id], graph.CriticalPath.TaskIds); + + graph.AddDependency(b.Id, a.Id); + graph.AddUsage(b.Id, 10, 5); + graph.AddUsage(b.Id, 2, 3); + + Assert.Equal([a.Id, b.Id], graph.CriticalPath.TaskIds); + Assert.Equal(20, graph.Get(b.Id).ActualTotalTokens); + Assert.Equal(2_048, graph.TotalTokenBudget); + } +} diff --git a/tests/Agent.Planner.Tests/Usings.cs b/tests/Agent.Planner.Tests/Usings.cs new file mode 100644 index 0000000..c802f44 --- /dev/null +++ b/tests/Agent.Planner.Tests/Usings.cs @@ -0,0 +1 @@ +global using Xunit; diff --git a/tests/Agent.Windows.Tests/Agent.Windows.Tests.csproj b/tests/Agent.Windows.Tests/Agent.Windows.Tests.csproj new file mode 100644 index 0000000..0b45401 --- /dev/null +++ b/tests/Agent.Windows.Tests/Agent.Windows.Tests.csproj @@ -0,0 +1,15 @@ +<Project Sdk="Microsoft.NET.Sdk"> + <PropertyGroup> + <TargetFramework>net10.0-windows10.0.26100.0</TargetFramework> + <IsPackable>false</IsPackable> + <IsTestProject>true</IsTestProject> + <TestingPlatformDotnetTestSupport>true</TestingPlatformDotnetTestSupport> + </PropertyGroup> + <ItemGroup> + <ProjectReference Include="..\..\src\Agent.Windows\Agent.Windows.csproj" /> + </ItemGroup> + <ItemGroup> + <PackageReference Include="Microsoft.NET.Test.Sdk" /> + <PackageReference Include="xunit.v3" /> + </ItemGroup> +</Project> diff --git a/tests/Agent.Windows.Tests/BaselineTests.cs b/tests/Agent.Windows.Tests/BaselineTests.cs new file mode 100644 index 0000000..a4c1cfd --- /dev/null +++ b/tests/Agent.Windows.Tests/BaselineTests.cs @@ -0,0 +1,10 @@ +namespace Agent.Windows.Tests; + +public sealed class BaselineTests +{ + [Fact] + public void WindowsAssemblyLoads() + { + Assert.Equal("Agent.Windows", typeof(Windows.AssemblyMarker).Assembly.GetName().Name); + } +} diff --git a/tests/Agent.Windows.Tests/CrashScenarioBootstrap.cs b/tests/Agent.Windows.Tests/CrashScenarioBootstrap.cs new file mode 100644 index 0000000..7dd3629 --- /dev/null +++ b/tests/Agent.Windows.Tests/CrashScenarioBootstrap.cs @@ -0,0 +1,89 @@ +using System.Runtime.CompilerServices; +using System.Security.AccessControl; +using System.Diagnostics; +using Agent.Windows.Network; +using Agent.Windows.Recovery; +using Agent.Windows.Security; + +namespace Agent.Windows.Tests; + +internal static class CrashScenarioBootstrap +{ + [ModuleInitializer] + internal static void RunIfRequested() + { + string? scenario = Environment.GetEnvironmentVariable("AGENT_MCP_CRASH_SCENARIO"); + if (string.IsNullOrEmpty(scenario)) return; + string root = Environment.GetEnvironmentVariable("AGENT_MCP_CRASH_ROOT") + ?? throw new InvalidOperationException("Crash root is required."); + Guid taskId = Guid.Parse(Environment.GetEnvironmentVariable("AGENT_MCP_CRASH_TASK")!); + try + { + TaskResourceJournal journal = TaskResourceJournal.Create(Path.Combine(root, "journals"), taskId); + switch (scenario) + { + case "acl": + var acl = new TaskAclManager(AppContainerProfile.EnsureFixed().Sid, new MemorySecurityAuditSink(), journal); + _ = acl.Grant(taskId, Path.Combine(root, "resource"), FileCapability.Read | FileCapability.Write); + break; + case "acl-before": + byte[] descriptor = FileSystemAclExtensions.GetAccessControl(new DirectoryInfo(Path.Combine(root, "resource")), AccessControlSections.Access) + .GetSecurityDescriptorBinaryForm(); + _ = journal.Prepare("acl", Path.Combine(root, "resource"), + new Dictionary<string, string> { ["security_descriptor"] = Convert.ToBase64String(descriptor) }); + break; + case "file-before": + string temporary = Path.Combine(root, "resource.txt.agent-mcp-prepared.tmp"); + string backup = Path.Combine(root, "backups", taskId.ToString("N"), "prepared.backup"); + Directory.CreateDirectory(Path.GetDirectoryName(backup)!); + File.WriteAllText(temporary, "temporary"); + File.WriteAllText(backup, "before-crash"); + _ = journal.Prepare("file.replace", Path.Combine(root, "resource.txt"), new Dictionary<string, string> + { + ["existed"] = bool.TrueString, ["original_sha256"] = string.Empty, ["original_base64"] = string.Empty, + ["backup_path"] = backup, ["temporary_path"] = temporary + }); + break; + case "file-replace": + _ = new TransactionalFileEditor(journal, Path.Combine(root, "backups"), 1) + .Replace(Path.Combine(root, "resource.txt"), "after-crash"); + break; + case "process-after": + string powershell = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.System), "WindowsPowerShell", "v1.0", "powershell.exe"); + string auditPath = Path.Combine(root, "crash-audit.jsonl"); + var processAudit = new JsonLinesSecurityAuditSink(auditPath); + var launcher = new RestrictedProcessLauncher( + new ExecutablePolicy([new ExecutableApproval(powershell, ExecutablePolicy.ComputeSha256(powershell))], true), processAudit); + string escapedAudit = auditPath.Replace("'", "''", StringComparison.Ordinal); + string crashError = Path.Combine(root, "crash-error.txt").Replace("'", "''", StringComparison.Ordinal); + string watchdogScript = $"$d=[DateTime]::UtcNow.AddSeconds(10); while([DateTime]::UtcNow -lt $d){{ if((Test-Path -LiteralPath '{escapedAudit}') -and (Select-String -LiteralPath '{escapedAudit}' -SimpleMatch '\"Event\":\"process.start\"' -Quiet)){{ Stop-Process -Id {Environment.ProcessId} -Force; exit 0 }}; Start-Sleep -Milliseconds 25 }}; Set-Content -LiteralPath '{crashError}' -Value 'Sandbox process did not start before crash'; Stop-Process -Id {Environment.ProcessId} -Force"; + _ = Process.Start(new ProcessStartInfo + { + FileName = powershell, UseShellExecute = false, CreateNoWindow = true, + ArgumentList = { "-NoLogo", "-NoProfile", "-NonInteractive", "-Command", watchdogScript } + }); + _ = launcher.Run(taskId, powershell, + ["-NoLogo", "-NoProfile", "-NonInteractive", "-Command", "Start-Sleep -Seconds 30"], + Environment.SystemDirectory, TimeSpan.FromMinutes(1)); + break; + case "cleanup-mid": + var cleanupAcl = new TaskAclManager(AppContainerProfile.EnsureFixed().Sid, new MemorySecurityAuditSink(), journal); + _ = cleanupAcl.Grant(taskId, Path.Combine(root, "resource"), FileCapability.Read | FileCapability.Write); + FileEditTransaction edit = new TransactionalFileEditor(journal, Path.Combine(root, "backups"), 1) + .Replace(Path.Combine(root, "resource.txt"), "during-cleanup"); + edit.Rollback(); + break; + case "wfp": + _ = TaskNetworkPolicy.Apply(journal, AppContainerProfile.EnsureFixed().Sid, [], new MemorySecurityAuditSink()); + break; + default: + throw new InvalidOperationException($"Unknown crash scenario '{scenario}'."); + } + } + catch (Exception ex) + { + File.WriteAllText(Path.Combine(root, "crash-error.txt"), ex.ToString()); + } + Environment.FailFast($"Intentional security recovery crash point: {scenario}"); + } +} diff --git a/tests/Agent.Windows.Tests/SecurityPolicyTests.cs b/tests/Agent.Windows.Tests/SecurityPolicyTests.cs new file mode 100644 index 0000000..d54ab13 --- /dev/null +++ b/tests/Agent.Windows.Tests/SecurityPolicyTests.cs @@ -0,0 +1,579 @@ +using System.Diagnostics; +using System.Net; +using System.Net.Sockets; +using System.Security.AccessControl; +using System.Text; +using Agent.Windows.Recovery; +using Agent.Windows.Security; +using Agent.Windows.Network; +using Agent.Windows.Tools; +using Agent.Core.Tools; +using System.Text.Json; + +namespace Agent.Windows.Tests; + +public sealed class SecurityPolicyTests +{ + [Fact] + public void Read_write_and_delete_capabilities_are_independent() + { + using TestDirectory directory = new(); + string file = Path.Combine(directory.Path, "file.txt"); + File.WriteAllText(file, "data"); + PathPolicy policy = new([new FilePolicyEntry(directory.Path, FileCapability.Read | FileCapability.Write)]); + + Assert.True(policy.IsAllowed(file, FileCapability.Read)); + Assert.True(policy.IsAllowed(Path.Combine(directory.Path, "new.txt"), FileCapability.Write)); + Assert.False(policy.IsAllowed(file, FileCapability.Delete)); + Assert.Throws<SandboxPolicyException>(() => policy.Demand(file, FileCapability.Delete)); + } + + [Fact] + public void Path_outside_approved_root_is_denied() + { + using TestDirectory approved = new(); + using TestDirectory outside = new(); + string secret = Path.Combine(outside.Path, "secret.txt"); + File.WriteAllText(secret, "secret"); + PathPolicy policy = new([new FilePolicyEntry(approved.Path, FileCapability.Read)]); + + Assert.False(policy.IsAllowed(secret, FileCapability.Read)); + } + + [Fact] + public void Reparse_escape_is_rejected() + { + using TestDirectory approved = new(); + using TestDirectory outside = new(); + string link = Path.Combine(approved.Path, "escape"); + using Process junction = Process.Start(new ProcessStartInfo + { + FileName = Environment.GetEnvironmentVariable("ComSpec")!, + UseShellExecute = false, + CreateNoWindow = true, + ArgumentList = { "/d", "/c", "mklink", "/J", link, outside.Path } + })!; + junction.WaitForExit(); + Assert.Equal(0, junction.ExitCode); + PathPolicy policy = new([new FilePolicyEntry(approved.Path, FileCapability.Read)]); + + try + { + SandboxPolicyException error = Assert.Throws<SandboxPolicyException>(() => policy.IsAllowed(Path.Combine(link, "secret.txt"), FileCapability.Read)); + Assert.Equal("path.reparse", error.Code); + } + finally + { + Directory.Delete(link); + } + } + + [Fact] + public void Symlink_chain_escape_is_rejected() + { + using TestDirectory approved = new(); + using TestDirectory outside = new(); + string second = Path.Combine(approved.Path, "second-link"); + string first = Path.Combine(approved.Path, "first-link"); + Directory.CreateSymbolicLink(second, outside.Path); + Directory.CreateSymbolicLink(first, second); + PathPolicy policy = new([new FilePolicyEntry(approved.Path, FileCapability.Read)]); + + SandboxPolicyException error = Assert.Throws<SandboxPolicyException>(() => + policy.IsAllowed(Path.Combine(first, "secret.txt"), FileCapability.Read)); + + Assert.Equal("path.reparse", error.Code); + Directory.Delete(first); + Directory.Delete(second); + } + + [Fact] + public void Executable_requires_approved_path_and_hash_and_shell_toggle() + { + string executable = Environment.ProcessPath!; + string hash = ExecutablePolicy.ComputeSha256(executable); + Assert.Equal(hash, new ExecutablePolicy([new ExecutableApproval(executable, hash)], true).Verify(executable).Sha256); + Assert.Throws<SandboxPolicyException>(() => new ExecutablePolicy([], true).Verify(executable)); + string badHash = new string(hash[0] == '0' ? '1' : '0', 1) + hash[1..]; + Assert.Throws<SandboxPolicyException>(() => new ExecutablePolicy([new ExecutableApproval(executable, badHash)], true).Verify(executable)); + + string powershell = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.System), "WindowsPowerShell", "v1.0", "powershell.exe"); + Assert.Throws<SandboxPolicyException>(() => new ExecutablePolicy([new ExecutableApproval(powershell, ExecutablePolicy.ComputeSha256(powershell))], false).Verify(powershell)); + } + + [Fact] + public void Restricted_parent_cannot_start_unbrokered_descendant() + { + using TestDirectory directory = new(); + string powershell = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.System), "WindowsPowerShell", "v1.0", "powershell.exe"); + var audit = new MemorySecurityAuditSink(); + var launcher = new RestrictedProcessLauncher( + new ExecutablePolicy([new ExecutableApproval(powershell, ExecutablePolicy.ComputeSha256(powershell))], true), audit); + string copied = Path.Combine(directory.Path, "renamed-child.exe"); + File.Copy(Environment.GetEnvironmentVariable("ComSpec")!, copied); + string[] markers = Enumerable.Range(1, 4).Select(index => Path.Combine(directory.Path, $"descendant-{index}.txt")).ToArray(); + string[] commands = + [ + $"$ErrorActionPreference='Stop'; Start-Process -FilePath $env:ComSpec -ArgumentList '/d','/c','echo compromised > \"{markers[0]}\"' -Wait", + $"$ErrorActionPreference='Stop'; [Diagnostics.Process]::Start($env:ComSpec, '/d /c echo compromised ^> \"{markers[1]}\"').WaitForExit()", + $"$ErrorActionPreference='Stop'; & $env:ComSpec /d /c 'echo compromised > \"{markers[2]}\"'", + $"$ErrorActionPreference='Stop'; Start-Process -FilePath '{copied}' -ArgumentList '/d','/c','echo compromised > \"{markers[3]}\"' -Wait" + ]; + + foreach (string command in commands) + { + RestrictedProcessResult result = launcher.Run(Guid.NewGuid(), powershell, + ["-NoLogo", "-NoProfile", "-NonInteractive", "-Command", command], directory.Path, TimeSpan.FromSeconds(30)); + Assert.NotEqual(0u, result.ExitCode); + } + + Assert.All(markers, marker => Assert.False(File.Exists(marker))); + Assert.Contains(audit.Events, item => item.Event == "process.start" && item.Data["child_process_policy"] == "restricted"); + Assert.Contains(audit.Events, item => item.Event == "process.exit" && item.Data["exit_code"] != "0"); + Assert.Equal(commands.Length, audit.Events.Count(item => item.Event == "policy.deny" && item.Data["child_process_policy"] == "restricted")); + } + + [Fact] + public void Job_timeout_terminates_the_entire_sandbox_job_without_orphan() + { + string powershell = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.System), "WindowsPowerShell", "v1.0", "powershell.exe"); + var audit = new MemorySecurityAuditSink(); + var launcher = new RestrictedProcessLauncher( + new ExecutablePolicy([new ExecutableApproval(powershell, ExecutablePolicy.ComputeSha256(powershell))], true), audit); + + Assert.Throws<TimeoutException>(() => launcher.Run(Guid.NewGuid(), powershell, + ["-NoLogo", "-NoProfile", "-NonInteractive", "-Command", "Start-Sleep -Seconds 30"], + Environment.SystemDirectory, TimeSpan.FromMilliseconds(250))); + + SecurityAuditEvent timeout = Assert.Single(audit.Events, item => item.Event == "process.timeout"); + int pid = int.Parse(timeout.Data["pid"], System.Globalization.CultureInfo.InvariantCulture); + Assert.Throws<ArgumentException>(() => Process.GetProcessById(pid)); + } + + [Fact] + public void Dotnet_managed_entry_requires_its_own_path_and_hash_approval() + { + string dotnet = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.ProgramFiles), "dotnet", "dotnet.exe"); + string managedAssembly = typeof(SecurityPolicyTests).Assembly.Location; + var audit = new MemorySecurityAuditSink(); + var launcher = new RestrictedProcessLauncher( + new ExecutablePolicy([new ExecutableApproval(dotnet, ExecutablePolicy.ComputeSha256(dotnet))], true), audit); + + SandboxPolicyException denied = Assert.Throws<SandboxPolicyException>(() => launcher.Run( + Guid.NewGuid(), dotnet, [managedAssembly], Path.GetDirectoryName(managedAssembly)!, TimeSpan.FromSeconds(5))); + + Assert.Equal("process.path_denied", denied.Code); + Assert.Contains(audit.Events, item => item.Event == "policy.deny" && item.Data["code"] == "process.path_denied"); + } + + [Fact] + public void Fixed_app_container_is_stable_and_acl_is_restored_exactly() + { + AppContainerProfile first = AppContainerProfile.EnsureFixed(); + AppContainerProfile second = AppContainerProfile.EnsureFixed(); + Assert.Equal(AppContainerProfile.FixedName, first.Name); + Assert.Equal(first.Sid.Value, second.Sid.Value); + + using TestDirectory directory = new(); + byte[] before = FileSystemAclExtensions.GetAccessControl(new DirectoryInfo(directory.Path), AccessControlSections.Access).GetSecurityDescriptorBinaryForm(); + var audit = new MemorySecurityAuditSink(); + TaskAclManager manager = new(first.Sid, audit); + Guid taskId = Guid.NewGuid(); + AclSnapshot snapshot = manager.Grant(taskId, directory.Path, FileCapability.Read | FileCapability.Write); + byte[] during = FileSystemAclExtensions.GetAccessControl(new DirectoryInfo(directory.Path), AccessControlSections.Access).GetSecurityDescriptorBinaryForm(); + Assert.NotEqual(Convert.ToHexString(before), Convert.ToHexString(during)); + + manager.Restore(taskId, snapshot); + + byte[] after = FileSystemAclExtensions.GetAccessControl(new DirectoryInfo(directory.Path), AccessControlSections.Access).GetSecurityDescriptorBinaryForm(); + Assert.Equal(Convert.ToHexString(before), Convert.ToHexString(after)); + Assert.Contains(audit.Events, item => item.Event == "acl.apply"); + Assert.Contains(audit.Events, item => item.Event == "acl.restore"); + } + + [Fact] + public void Brokered_process_runs_in_fixed_app_container_with_task_acl() + { + AppContainerProfile profile = AppContainerProfile.EnsureFixed(); + using TestDirectory directory = new(); + string marker = Path.Combine(directory.Path, "appcontainer-write.txt"); + string commandProcessor = Environment.GetEnvironmentVariable("ComSpec")!; + var audit = new MemorySecurityAuditSink(); + TaskAclManager acl = new(profile.Sid, audit); + Guid taskId = Guid.NewGuid(); + AclSnapshot snapshot = acl.Grant(taskId, directory.Path, FileCapability.Read | FileCapability.Write); + try + { + var launcher = new RestrictedProcessLauncher( + new ExecutablePolicy([new ExecutableApproval(commandProcessor, ExecutablePolicy.ComputeSha256(commandProcessor))], true), + audit, profile.Sid, new SandboxJobLimits(2, 256UL * 1024 * 1024)); + + RestrictedProcessResult result = launcher.Run(taskId, commandProcessor, ["/d", "/c", $"echo APP_OK>{marker} & echo CAPTURE_OK"], directory.Path, TimeSpan.FromSeconds(30)); + + Assert.Equal(0u, result.ExitCode); + Assert.Equal("APP_OK", File.ReadAllText(marker).Trim()); + Assert.Contains("CAPTURE_OK", result.StandardOutput, StringComparison.Ordinal); + Assert.Contains(audit.Events, item => item.Event == "process.start" && item.Data["app_container_sid"] == profile.Sid.Value); + } + finally + { + acl.Restore(taskId, snapshot); + } + } + + [Fact] + public void File_transaction_commit_rollback_and_hash_guard_leave_no_journal() + { + using TestDirectory directory = new(); + string journals = Path.Combine(directory.Path, "journals"); + string backups = Path.Combine(directory.Path, "backups"); + string target = Path.Combine(directory.Path, "target.txt"); + File.WriteAllText(target, "original"); + TaskResourceJournal journal = TaskResourceJournal.Create(journals, Guid.NewGuid()); + TransactionalFileEditor editor = new(journal, backups, memoryBackupLimitBytes: 1); + + FileEditTransaction rollback = editor.Replace(target, "replacement"); + Assert.Equal("replacement", File.ReadAllText(target)); + rollback.Rollback(); + Assert.Equal("original", File.ReadAllText(target)); + + FileEditTransaction commit = editor.Replace(target, "committed"); + commit.Commit(); + journal.CompleteAndDelete(); + + Assert.Equal("committed", File.ReadAllText(target)); + Assert.Empty(Directory.EnumerateFiles(journals)); + Assert.False(Directory.Exists(Path.Combine(backups, journal.TaskId.ToString("N")))); + } + + [Fact] + public void Recovery_rolls_back_applied_file_in_reverse_cleanup_path() + { + using TestDirectory directory = new(); + string journals = Path.Combine(directory.Path, "journals"); + string backups = Path.Combine(directory.Path, "backups"); + string target = Path.Combine(directory.Path, "target.txt"); + File.WriteAllText(target, "before-crash"); + TaskResourceJournal journal = TaskResourceJournal.Create(journals, Guid.NewGuid()); + _ = new TransactionalFileEditor(journal, backups, memoryBackupLimitBytes: 1).Replace(target, "after-crash"); + + RecoveryResult result = new RecoveryManager(journals, new MemorySecurityAuditSink()).RecoverAll(); + + Assert.True(result.IsClean); + Assert.Equal("before-crash", File.ReadAllText(target)); + Assert.Empty(Directory.EnumerateFiles(journals)); + Assert.False(Directory.Exists(Path.Combine(backups, journal.TaskId.ToString("N")))); + } + + [Fact] + public void Recovery_after_crash_restores_acl_loopback_and_dynamic_wfp_state() + { + AppContainerProfile profile = AppContainerProfile.EnsureFixed(); + using TestDirectory directory = new(); + string journals = Path.Combine(directory.Path, "journals"); + string resource = Path.Combine(directory.Path, "resource"); + Directory.CreateDirectory(resource); + byte[] aclBefore = FileSystemAclExtensions.GetAccessControl(new DirectoryInfo(resource), AccessControlSections.Access) + .GetSecurityDescriptorBinaryForm(); + var audit = new MemorySecurityAuditSink(); + Guid taskId = Guid.NewGuid(); + TaskResourceJournal journal = TaskResourceJournal.Create(journals, taskId); + + var acl = new TaskAclManager(profile.Sid, audit, journal); + _ = acl.Grant(taskId, resource, FileCapability.Read | FileCapability.Write); + + Assert.False(LoopbackExemption.Contains(profile.Sid)); + Guid loopbackEntry = journal.Prepare("network.loopback_exemption", profile.Sid.Value, + new Dictionary<string, string> { ["originally_present"] = bool.FalseString }); + Assert.True(LoopbackExemption.Ensure(profile.Sid)); + journal.MarkApplied(loopbackEntry, new Dictionary<string, string> { ["added"] = bool.TrueString }); + + Guid wfpEntry = journal.Prepare("wfp.dynamic", profile.Sid.Value, + new Dictionary<string, string> { ["session"] = "dynamic", ["allowances"] = string.Empty }); + using (WfpDynamicSession crashedSession = WfpDynamicSession.Open()) + { + IReadOnlyList<ulong> filterIds = crashedSession.ApplyDefaultBlock(profile.Sid); + journal.MarkApplied(wfpEntry, new Dictionary<string, string> { ["filter_ids"] = string.Join(',', filterIds) }); + // Closing the engine without RemovePolicy models abrupt process exit: + // BFE owns and deletes every object in this dynamic session. + } + + RecoveryResult recovered = new RecoveryManager(journals, audit).RecoverAll(); + + Assert.True(recovered.IsClean); + Assert.Equal(1, recovered.JournalsRecovered); + Assert.Equal(3, recovered.ResourcesRecovered); + Assert.False(LoopbackExemption.Contains(profile.Sid)); + byte[] aclAfter = FileSystemAclExtensions.GetAccessControl(new DirectoryInfo(resource), AccessControlSections.Access) + .GetSecurityDescriptorBinaryForm(); + Assert.Equal(Convert.ToHexString(aclBefore), Convert.ToHexString(aclAfter)); + Assert.Empty(Directory.EnumerateFiles(journals)); + } + + [Theory] + [InlineData("acl")] + [InlineData("acl-before")] + [InlineData("file-before")] + [InlineData("file-replace")] + [InlineData("process-after")] + [InlineData("cleanup-mid")] + [InlineData("wfp")] + public void Independent_process_crash_is_fully_recovered_on_next_start(string scenario) + { + using TestDirectory directory = new(); + string resourceDirectory = Path.Combine(directory.Path, "resource"); + string resourceFile = Path.Combine(directory.Path, "resource.txt"); + Directory.CreateDirectory(resourceDirectory); + File.WriteAllText(resourceFile, "before-crash"); + byte[] aclBefore = FileSystemAclExtensions.GetAccessControl(new DirectoryInfo(resourceDirectory), AccessControlSections.Access) + .GetSecurityDescriptorBinaryForm(); + Guid taskId = Guid.NewGuid(); + using var child = new Process + { + StartInfo = new ProcessStartInfo + { + FileName = Environment.ProcessPath!, + UseShellExecute = false, + CreateNoWindow = true, + Environment = + { + ["AGENT_MCP_CRASH_SCENARIO"] = scenario, + ["AGENT_MCP_CRASH_ROOT"] = directory.Path, + ["AGENT_MCP_CRASH_TASK"] = taskId.ToString("D") + } + } + }; + Assert.True(child.Start()); + Assert.True(child.WaitForExit(60_000)); + Assert.NotEqual(0, child.ExitCode); + Assert.False(File.Exists(Path.Combine(directory.Path, "crash-error.txt")), + File.Exists(Path.Combine(directory.Path, "crash-error.txt")) ? File.ReadAllText(Path.Combine(directory.Path, "crash-error.txt")) : string.Empty); + Assert.Single(Directory.EnumerateFiles(Path.Combine(directory.Path, "journals"))); + + RecoveryResult recovered = new RecoveryManager(Path.Combine(directory.Path, "journals"), new MemorySecurityAuditSink()).RecoverAll(); + + Assert.True(recovered.IsClean, string.Join(Environment.NewLine, recovered.Errors)); + Assert.Empty(Directory.EnumerateFiles(Path.Combine(directory.Path, "journals"))); + Assert.Equal("before-crash", File.ReadAllText(resourceFile)); + byte[] aclAfter = FileSystemAclExtensions.GetAccessControl(new DirectoryInfo(resourceDirectory), AccessControlSections.Access) + .GetSecurityDescriptorBinaryForm(); + Assert.Equal(Convert.ToHexString(aclBefore), Convert.ToHexString(aclAfter)); + Assert.False(LoopbackExemption.Contains(AppContainerProfile.EnsureFixed().Sid)); + string backups = Path.Combine(directory.Path, "backups"); + Assert.True(!Directory.Exists(backups) || !Directory.EnumerateFileSystemEntries(backups).Any()); + if (scenario == "process-after") + { + string auditPath = Path.Combine(directory.Path, "crash-audit.jsonl"); + using JsonDocument audit = JsonDocument.Parse(File.ReadLines(auditPath).Single(line => line.Contains("process.start", StringComparison.Ordinal))); + int sandboxPid = int.Parse(audit.RootElement.GetProperty("Data").GetProperty("pid").GetString()!, + System.Globalization.CultureInfo.InvariantCulture); + Assert.True(SpinWait.SpinUntil(() => !ProcessExists(sandboxPid), TimeSpan.FromSeconds(5)), $"Orphan sandbox PID {sandboxPid} survived broker crash."); + } + } + + private static bool ProcessExists(int pid) + { + try { using Process process = Process.GetProcessById(pid); return !process.HasExited; } + catch (ArgumentException) { return false; } + } + + [Fact] + public void Rollback_refuses_to_overwrite_post_transaction_external_change() + { + using TestDirectory directory = new(); + string target = Path.Combine(directory.Path, "target.txt"); + File.WriteAllText(target, "before"); + TaskResourceJournal journal = TaskResourceJournal.Create(Path.Combine(directory.Path, "journals"), Guid.NewGuid()); + FileEditTransaction transaction = new TransactionalFileEditor(journal, Path.Combine(directory.Path, "backups")).Replace(target, "replacement"); + File.WriteAllText(target, "external-change"); + + Assert.Throws<InvalidOperationException>(() => transaction.Rollback()); + + File.WriteAllText(target, "replacement"); + transaction.Rollback(); + journal.CompleteAndDelete(); + } + + [Fact] + public void Wfp_dynamic_session_adds_and_explicitly_removes_ipv4_ipv6_package_filters() + { + AppContainerProfile profile = AppContainerProfile.EnsureFixed(); + using WfpDynamicSession session = WfpDynamicSession.Open(); + session.ProbeWriteTransaction(); + + IReadOnlyList<ulong> ids = session.ApplyDefaultBlock(profile.Sid, + [new NetworkEndpoint(IPAddress.Loopback, 11434), new NetworkEndpoint(IPAddress.IPv6Loopback, 11434)]); + + Assert.Equal(4, ids.Count); + Assert.All(ids, id => Assert.NotEqual(0UL, id)); + session.RemovePolicy(); + } + + [Fact] + public async Task Windows_backend_applies_file_effect_and_cleans_acl_wfp_temp_and_journal() + { + using TestDirectory directory = new(); + string workspace = Path.Combine(directory.Path, "workspace"); + Directory.CreateDirectory(workspace); + string target = Path.Combine(workspace, "target.txt"); + File.WriteAllText(target, "before"); + byte[] aclBefore = FileSystemAclExtensions.GetAccessControl(new DirectoryInfo(workspace), AccessControlSections.Access).GetSecurityDescriptorBinaryForm(); + var audit = new MemorySecurityAuditSink(); + var settings = new WindowsSandboxSettings( + [new FilePolicyEntry(workspace, FileCapability.Read | FileCapability.Write | FileCapability.Delete)], [], false, + new SandboxJobLimits(), "deny", [], Path.Combine(directory.Path, "journals"), Path.Combine(directory.Path, "backups"), + 16 * 1024 * 1024, 1024 * 1024, 1000, 10000); + var backend = new WindowsToolBackend(settings, audit); + Guid taskId = Guid.NewGuid(); + await backend.BeginTaskAsync(taskId, TestContext.Current.CancellationToken); + using JsonDocument edit = JsonDocument.Parse($$"""{"path":{{JsonSerializer.Serialize(target)}},"operation":"overwrite","content":"after"}"""); + + ToolExecutionResult result = await backend.ExecuteAsync(taskId, "file_edit", edit.RootElement, TestContext.Current.CancellationToken); + await backend.EndTaskAsync(taskId, true, TestContext.Current.CancellationToken); + + Assert.True(result.Success); + Assert.True(result.ExternalEffect); + Assert.Equal("after", File.ReadAllText(target)); + byte[] aclAfter = FileSystemAclExtensions.GetAccessControl(new DirectoryInfo(workspace), AccessControlSections.Access).GetSecurityDescriptorBinaryForm(); + Assert.Equal(Convert.ToHexString(aclBefore), Convert.ToHexString(aclAfter)); + Assert.Empty(Directory.EnumerateFiles(Path.Combine(directory.Path, "journals"))); + string backups = Path.Combine(directory.Path, "backups"); + Assert.True(!Directory.Exists(backups) || !Directory.EnumerateFileSystemEntries(backups).Any()); + Assert.Contains(audit.Events, item => item.Event == "wfp.remove" && item.Data["residual_filters"] == "0"); + Assert.Contains(audit.Events, item => item.Event == "acl.restore"); + } + + [Fact(Timeout = 60_000)] + public async Task Wfp_enforces_actual_appcontainer_loopback_deny_allow_and_port_scope() + { + AppContainerProfile profile = AppContainerProfile.EnsureFixed(); + string curl = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.System), "curl.exe"); + var audit = new MemorySecurityAuditSink(); + var launcher = new RestrictedProcessLauncher( + new ExecutablePolicy([new ExecutableApproval(curl, ExecutablePolicy.ComputeSha256(curl))], true), + audit, profile.Sid, new SandboxJobLimits(2, 256UL * 1024 * 1024)); + using TestDirectory directory = new(); + + bool diagnosticExemptionAdded = LoopbackExemption.Ensure(profile.Sid); + try + { + using var isolationControl = new OneShotHttpServer(IPAddress.Loopback); + RestrictedProcessResult isolationResult = Curl(launcher, Guid.NewGuid(), curl, isolationControl.Url, Environment.SystemDirectory); + Assert.True(isolationResult.ExitCode == 0u, + $"Loopback exemption did not enable AppContainer traffic before WFP was applied. exit {isolationResult.ExitCode}, stderr: {isolationResult.StandardError}"); + Assert.True(isolationControl.Accepted); + } + finally + { + if (diagnosticExemptionAdded) LoopbackExemption.Remove(profile.Sid); + } + + using (var deniedServer = new OneShotHttpServer(IPAddress.Loopback)) + { + Guid taskId = Guid.NewGuid(); + TaskResourceJournal journal = TaskResourceJournal.Create(Path.Combine(directory.Path, "deny-journal"), taskId); + using (TaskNetworkPolicy.Apply(journal, profile.Sid, [], audit)) + { + RestrictedProcessResult denied = Curl(launcher, taskId, curl, deniedServer.Url, Environment.SystemDirectory); + Assert.NotEqual(0u, denied.ExitCode); + Assert.False(deniedServer.Accepted); + } + journal.CompleteAndDelete(); + } + + IPAddress allowedAddress = IPAddress.Loopback; + using (var controlServer = new OneShotHttpServer(allowedAddress)) + using (var controlClient = new HttpClient { Timeout = TimeSpan.FromSeconds(2) }) + { + string controlBody = await controlClient.GetStringAsync(controlServer.Url, TestContext.Current.CancellationToken); + Assert.Equal("WFP_OK", controlBody); + Assert.True(controlServer.Accepted); + } + using var allowedV4 = new OneShotHttpServer(allowedAddress); + using var deniedPort = new OneShotHttpServer(allowedAddress); + Guid allowTaskId = Guid.NewGuid(); + TaskResourceJournal allowJournal = TaskResourceJournal.Create(Path.Combine(directory.Path, "allow-journal"), allowTaskId); + IReadOnlyList<NetworkEndpoint> allowances = + [ + new(allowedAddress, checked((ushort)allowedV4.Port)) + ]; + using (TaskNetworkPolicy.Apply(allowJournal, profile.Sid, allowances, audit)) + { + RestrictedProcessResult v4 = Curl(launcher, allowTaskId, curl, allowedV4.Url, Environment.SystemDirectory); + RestrictedProcessResult wrongPort = Curl(launcher, allowTaskId, curl, deniedPort.Url, Environment.SystemDirectory); + Assert.True(v4.ExitCode == 0u, $"Allowed AppContainer curl failed with exit {v4.ExitCode}. stderr: {v4.StandardError}"); + Assert.Contains("WFP_OK", v4.StandardOutput, StringComparison.Ordinal); + Assert.True(allowedV4.Accepted); + Assert.NotEqual(0u, wrongPort.ExitCode); + Assert.False(deniedPort.Accepted); + } + allowJournal.CompleteAndDelete(); + Assert.Contains(audit.Events, item => item.Event == "wfp.remove" && item.Data["residual_filters"] == "0"); + } + + private static RestrictedProcessResult Curl(RestrictedProcessLauncher launcher, Guid taskId, string curl, string url, string cwd) => + launcher.Run(taskId, curl, ["--noproxy", "*", "--silent", "--show-error", "--max-time", "2", url], cwd, TimeSpan.FromSeconds(10)); + + private sealed class TestDirectory : IDisposable + { + public TestDirectory() + { + Path = System.IO.Path.Combine(System.IO.Path.GetTempPath(), "agent-mcp-tests", Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(Path); + } + public string Path { get; } + public void Dispose() => Directory.Delete(Path, recursive: true); + } + + private sealed class OneShotHttpServer : IDisposable + { + private readonly TcpListener _listener; + private readonly CancellationTokenSource _stop = new(); + private readonly Task _serve; + private int _accepted; + + public OneShotHttpServer(IPAddress address) + { + _listener = new TcpListener(address, 0); + _listener.Start(); + Port = ((IPEndPoint)_listener.LocalEndpoint).Port; + string host = address.AddressFamily == AddressFamily.InterNetworkV6 ? $"[{address}]" : address.ToString(); + Url = $"http://{host}:{Port}/"; + _serve = ServeAsync(); + } + + public int Port { get; } + public string Url { get; } + public bool Accepted => Volatile.Read(ref _accepted) != 0; + + private async Task ServeAsync() + { + try + { + using TcpClient client = await _listener.AcceptTcpClientAsync(_stop.Token); + Interlocked.Exchange(ref _accepted, 1); + using NetworkStream stream = client.GetStream(); + byte[] request = new byte[4096]; + int received = 0; + while (received < request.Length) + { + int count = await stream.ReadAsync(request.AsMemory(received), _stop.Token); + if (count == 0) break; + received += count; + if (received >= 4 && request.AsSpan(0, received).IndexOf("\r\n\r\n"u8) >= 0) break; + } + byte[] response = Encoding.ASCII.GetBytes("HTTP/1.1 200 OK\r\nContent-Length: 6\r\nConnection: close\r\n\r\nWFP_OK"); + await stream.WriteAsync(response, _stop.Token); + } + catch (Exception ex) when (ex is OperationCanceledException or SocketException or ObjectDisposedException) { } + } + + public void Dispose() + { + _stop.Cancel(); + _listener.Stop(); + try { _serve.Wait(TimeSpan.FromSeconds(2)); } catch (AggregateException) { } + _stop.Dispose(); + } + } +} diff --git a/tests/Agent.Windows.Tests/Usings.cs b/tests/Agent.Windows.Tests/Usings.cs new file mode 100644 index 0000000..c802f44 --- /dev/null +++ b/tests/Agent.Windows.Tests/Usings.cs @@ -0,0 +1 @@ +global using Xunit; |