From b07b41cd435991cd254957af79055080c85b346e Mon Sep 17 00:00:00 2001 From: tanaka takashi Date: Sun, 9 Aug 2026 21:08:20 +0900 Subject: feat: AI Agent Sandbox MCP Server v1を実装 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Specifications、Design、ImplementationPlanの優先順位を整理し、単一Task実行モデル、typed option優先、設定可能なresource上限、local Ollamaを必須とするRelease Gateを文書化。 .NET 10/C# 14の5 production project、3 test project、E2E Clientを構築。typed YAML、en/ja locale、fixed/user prompt、stdioおよびloopback限定Streamable HTTP、agent_run/agent_status、Session、Ollama text/tool/Vision、TaskGraphとtoken accountingを実装。 Windows sandboxとしてfixed AppContainer、canonical/reparse検査、能力別ACL、path+SHA-256 executable approval、child-process restriction、Job Object、task journal、atomic file transaction、crash recoveryを実装。WFPのIPv4/IPv6 default denyとexact allow、dynamic session、temporary loopback exemption、管理者/BFE診断を追加。 Security/Recovery test、MCP/Ollama integration test、stdio/HTTP Operational E2E、Release Bundle生成・checksum・manifest・fresh extraction検査を追加。Requirement TraceabilityとPhase 0〜11の進捗、初回セットアップ可能なREADMEを整備。 Debug/Release/publish/objをrepository rootのbuild/へ集約し、旧project-local bin/objへの依存を除去。artifacts/とbuild/をGit管理外に設定し、clean checkoutでもsolution buildからE2E Clientまで生成可能にした。 --- tests/Agent.Core.Tests/Agent.Core.Tests.csproj | 16 ++ tests/Agent.Core.Tests/BaselineTests.cs | 10 ++ tests/Agent.Core.Tests/ConfigurationTests.cs | 66 +++++++ tests/Agent.Core.Tests/FixedPromptTests.cs | 14 ++ tests/Agent.Core.Tests/InternalToolTests.cs | 88 ++++++++++ tests/Agent.Core.Tests/LocaleCatalogTests.cs | 54 ++++++ tests/Agent.Core.Tests/McpHttpTests.cs | 88 ++++++++++ tests/Agent.Core.Tests/McpStdioTests.cs | 157 +++++++++++++++++ tests/Agent.Core.Tests/OllamaClientTests.cs | 143 +++++++++++++++ tests/Agent.Core.Tests/PromptAndRuntimeTests.cs | 220 ++++++++++++++++++++++++ tests/Agent.Core.Tests/TaskRegistryTests.cs | 35 ++++ tests/Agent.Core.Tests/TestDirectory.cs | 21 +++ tests/Agent.Core.Tests/Usings.cs | 1 + 13 files changed, 913 insertions(+) create mode 100644 tests/Agent.Core.Tests/Agent.Core.Tests.csproj create mode 100644 tests/Agent.Core.Tests/BaselineTests.cs create mode 100644 tests/Agent.Core.Tests/ConfigurationTests.cs create mode 100644 tests/Agent.Core.Tests/FixedPromptTests.cs create mode 100644 tests/Agent.Core.Tests/InternalToolTests.cs create mode 100644 tests/Agent.Core.Tests/LocaleCatalogTests.cs create mode 100644 tests/Agent.Core.Tests/McpHttpTests.cs create mode 100644 tests/Agent.Core.Tests/McpStdioTests.cs create mode 100644 tests/Agent.Core.Tests/OllamaClientTests.cs create mode 100644 tests/Agent.Core.Tests/PromptAndRuntimeTests.cs create mode 100644 tests/Agent.Core.Tests/TaskRegistryTests.cs create mode 100644 tests/Agent.Core.Tests/TestDirectory.cs create mode 100644 tests/Agent.Core.Tests/Usings.cs (limited to 'tests/Agent.Core.Tests') diff --git a/tests/Agent.Core.Tests/Agent.Core.Tests.csproj b/tests/Agent.Core.Tests/Agent.Core.Tests.csproj new file mode 100644 index 0000000..1326733 --- /dev/null +++ b/tests/Agent.Core.Tests/Agent.Core.Tests.csproj @@ -0,0 +1,16 @@ + + + net10.0-windows10.0.26100.0 + false + true + true + + + + + + + + + + diff --git a/tests/Agent.Core.Tests/BaselineTests.cs b/tests/Agent.Core.Tests/BaselineTests.cs new file mode 100644 index 0000000..a9b8fb7 --- /dev/null +++ b/tests/Agent.Core.Tests/BaselineTests.cs @@ -0,0 +1,10 @@ +namespace Agent.Core.Tests; + +public sealed class BaselineTests +{ + [Fact] + public void CoreAssemblyLoads() + { + Assert.Equal("Agent.Core", typeof(Core.AssemblyMarker).Assembly.GetName().Name); + } +} diff --git a/tests/Agent.Core.Tests/ConfigurationTests.cs b/tests/Agent.Core.Tests/ConfigurationTests.cs new file mode 100644 index 0000000..7ef4afb --- /dev/null +++ b/tests/Agent.Core.Tests/ConfigurationTests.cs @@ -0,0 +1,66 @@ +using Agent.App.Cli; +using Agent.App.Configuration; + +namespace Agent.Core.Tests; + +public sealed class ConfigurationTests +{ + [Fact] + public void ValidYamlLoadsTypedConfiguration() + { + using var directory = new TestDirectory(); + string config = CreateValidConfiguration(directory); + AgentConfiguration result = ConfigurationLoader.Load(config); + Assert.Equal(1, result.SchemaVersion); + Assert.Equal("model", result.Ollama.Model); + Assert.Equal("deny", result.Sandbox.Network.Mode); + } + + [Fact] + public void UnknownYamlPropertyIsFatal() + { + using var directory = new TestDirectory(); + string config = CreateValidConfiguration(directory); + System.IO.File.AppendAllText(config, "unknown_root: true\n"); + Assert.Throws(() => ConfigurationLoader.Load(config)); + } + + [Fact] + public void InvalidSchemaIsFatal() + { + using var directory = new TestDirectory(); + string config = CreateValidConfiguration(directory); + System.IO.File.WriteAllText(config, System.IO.File.ReadAllText(config).Replace("schema_version: 1", "schema_version: 2", StringComparison.Ordinal)); + Assert.Throws(() => ConfigurationLoader.Load(config)); + } + + [Fact] + public void CommandLineLanguageOverrideIsCaptured() + { + CommandLine result = CommandLine.Parse(["serve", "--lang", "ja", "--transport", "http"]); + Assert.Equal("ja", result.Language); + Assert.Equal("http", result.Transport); + } + + private static string CreateValidConfiguration(TestDirectory directory) + { + directory.File("prompts/user-system.md", string.Empty); + Directory.CreateDirectory(System.IO.Path.Combine(directory.Path, "config/locales")); + Directory.CreateDirectory(System.IO.Path.Combine(directory.Path, "skills")); + return directory.File("agent.yaml", """ + schema_version: 1 + application: + language: en + agent: + user_system_prompt: prompts/user-system.md + ollama: + endpoint: http://127.0.0.1:11434 + model: model + localization: + directory: config/locales + fallback: en + skills: + directory: skills + """); + } +} diff --git a/tests/Agent.Core.Tests/FixedPromptTests.cs b/tests/Agent.Core.Tests/FixedPromptTests.cs new file mode 100644 index 0000000..93ec7be --- /dev/null +++ b/tests/Agent.Core.Tests/FixedPromptTests.cs @@ -0,0 +1,14 @@ +using Agent.Core.Prompts; + +namespace Agent.Core.Tests; + +public sealed class FixedPromptTests +{ + [Fact] + public void FixedPromptIsEmbeddedAndNonEmpty() + { + string prompt = FixedPrompt.Load(); + Assert.Contains("SandboxPolicy", prompt, StringComparison.Ordinal); + Assert.DoesNotContain("user-system.md", prompt, StringComparison.Ordinal); + } +} diff --git a/tests/Agent.Core.Tests/InternalToolTests.cs b/tests/Agent.Core.Tests/InternalToolTests.cs new file mode 100644 index 0000000..1e241ce --- /dev/null +++ b/tests/Agent.Core.Tests/InternalToolTests.cs @@ -0,0 +1,88 @@ +using System.Text.Json; +using Agent.Core.Tools; + +namespace Agent.Core.Tests; + +public sealed class InternalToolTests +{ + [Fact] + public void Catalog_exposes_exactly_five_strict_tool_groups() + { + Assert.Equal(["file_search", "file_read", "file_edit", "process_run", "task_update"], InternalToolCatalog.All.Select(x => x.Name)); + foreach (InternalToolDefinition tool in InternalToolCatalog.All) Assert.False(tool.Parameters.GetProperty("additionalProperties").GetBoolean()); + } + + [Theory] + [InlineData("file_read", "{}")] + [InlineData("file_read", "{\"path\":\"a\",\"mode\":\"whole\",\"unknown\":1}")] + [InlineData("file_read", "{\"path\":\"a\",\"mode\":\"invalid\"}")] + [InlineData("file_search", "{\"root\":\".\",\"mode\":\"text\",\"query\":\"x\",\"max_results\":1001}")] + [InlineData("process_run", "{\"executable\":\"x\",\"args\":\"bad\",\"cwd\":\".\"}")] + [InlineData("task_update", "{\"operation\":\"add\",\"description\":\"missing estimate\"}")] + public void Malformed_arguments_are_rejected(string tool, string json) + { + using JsonDocument document = JsonDocument.Parse(json); + Assert.Throws(() => InternalToolValidator.Validate(tool, document.RootElement)); + } + + [Fact] + public async Task Policy_deny_has_no_backend_side_effect() + { + FakeInternalToolBackend backend = new(); + InternalToolDispatcher dispatcher = new(new DenyByDefaultPolicy(), backend); + using JsonDocument document = JsonDocument.Parse("""{"path":"x","operation":"overwrite","content":"changed"}"""); + + ToolExecutionResult result = await dispatcher.DispatchAsync(Guid.NewGuid(), "file_edit", document.RootElement, TestContext.Current.CancellationToken); + + Assert.False(result.Success); + Assert.False(result.ExternalEffect); + Assert.Empty(backend.Calls); + } + + [Fact] + public async Task Allowed_call_reaches_fake_backend_only_after_validation_and_policy() + { + FakeInternalToolBackend backend = new(); + InternalToolDispatcher dispatcher = new(new DenyByDefaultPolicy(new HashSet(["file_read"], StringComparer.Ordinal)), backend); + using JsonDocument document = JsonDocument.Parse("""{"path":"x","mode":"whole"}"""); + + ToolExecutionResult result = await dispatcher.DispatchAsync(Guid.NewGuid(), "file_read", document.RootElement, TestContext.Current.CancellationToken); + + Assert.True(result.Success); + Assert.Single(backend.Calls); + } + + [Fact] + public void Guard_requires_effect_and_fails_on_fifth_identical_failure() + { + ToolUseGuard guard = new(); + for (int attempt = 1; attempt < 5; attempt++) Assert.Equal(ToolUseGuardDecision.Retry, guard.Evaluate(true, false, false, "same").Decision); + ToolUseGuardResult fifth = guard.Evaluate(true, false, false, "same"); + Assert.Equal(ToolUseGuardDecision.Fail, fifth.Decision); + Assert.Equal(5, fifth.IdenticalFailureCount); + } + + [Fact] + public void Guard_does_not_force_tools_for_pure_analysis() + { + Assert.Equal(ToolUseGuardDecision.Complete, new ToolUseGuard().Evaluate(false, false, false).Decision); + } + + [Fact] + public void Task_update_mutates_the_real_deterministic_graph_without_external_effect_evidence() + { + Guid scope = Guid.NewGuid(); + Guid node = Guid.NewGuid(); + var handler = new TaskGraphToolHandler(); + handler.BeginTask(scope); + using JsonDocument add = JsonDocument.Parse($$"""{"operation":"add","task_id":"{{node}}","description":"Build fixture","depends_on":[],"budget_class":"1K","optimistic_tokens":10,"most_likely_tokens":20,"pessimistic_tokens":30}"""); + + ToolExecutionResult result = handler.Execute(scope, add.RootElement); + + Assert.True(result.Success); + Assert.False(result.ExternalEffect); + Assert.Contains(node.ToString(), result.Output, StringComparison.OrdinalIgnoreCase); + handler.EndTask(scope); + Assert.Throws(() => handler.Execute(scope, add.RootElement)); + } +} diff --git a/tests/Agent.Core.Tests/LocaleCatalogTests.cs b/tests/Agent.Core.Tests/LocaleCatalogTests.cs new file mode 100644 index 0000000..e5b6085 --- /dev/null +++ b/tests/Agent.Core.Tests/LocaleCatalogTests.cs @@ -0,0 +1,54 @@ +using Agent.Core.Localization; + +namespace Agent.Core.Tests; + +public sealed class LocaleCatalogTests +{ + [Fact] + public void MissingSelectedKeyFallsBackToEnglish() + { + using var directory = new TestDirectory(); + directory.File("en.json", """{"hello":"Hello {name}","only_en":"fallback"}"""); + directory.File("ja.json", """{"hello":"こんにちは {name}"}"""); + LocaleCatalog catalog = LocaleCatalog.Load(directory.Path, "ja"); + Assert.Equal("fallback", catalog.Get("only_en")); + Assert.Equal("こんにちは Codex", catalog.Format("hello", new Dictionary { ["name"] = "Codex" })); + } + + [Fact] + public void AdditionalLocaleLoadsWithoutCodeChange() + { + using var directory = new TestDirectory(); + directory.File("en.json", """{"hello":"Hello"}"""); + directory.File("zh-CN.json", """{"hello":"你好"}"""); + Assert.Equal("你好", LocaleCatalog.Load(directory.Path, "zh-CN").Get("hello")); + } + + [Theory] + [InlineData("{broken")] + [InlineData("[]")] + public void MalformedLocaleIsFatal(string json) + { + using var directory = new TestDirectory(); + directory.File("en.json", json); + Assert.Throws(() => LocaleCatalog.Load(directory.Path, "en")); + } + + [Fact] + public void UnknownKeyIsFatal() + { + using var directory = new TestDirectory(); + directory.File("en.json", """{"known":"value"}"""); + directory.File("ja.json", """{"known":"値","unknown":"x"}"""); + Assert.Throws(() => LocaleCatalog.Load(directory.Path, "ja")); + } + + [Fact] + public void PlaceholderMismatchIsFatal() + { + using var directory = new TestDirectory(); + directory.File("en.json", """{"hello":"Hello {name}"}"""); + directory.File("ja.json", """{"hello":"こんにちは {user}"}"""); + Assert.Throws(() => LocaleCatalog.Load(directory.Path, "ja")); + } +} diff --git a/tests/Agent.Core.Tests/McpHttpTests.cs b/tests/Agent.Core.Tests/McpHttpTests.cs new file mode 100644 index 0000000..7e3f1cf --- /dev/null +++ b/tests/Agent.Core.Tests/McpHttpTests.cs @@ -0,0 +1,88 @@ +using System.Net; +using System.Net.Sockets; +using Agent.Core.Localization; +using Agent.Core.Ollama; +using Agent.Core.Runtime; +using Agent.Mcp; +using ModelContextProtocol; +using ModelContextProtocol.Client; + +namespace Agent.Core.Tests; + +public sealed class McpHttpTests +{ + [Fact(Timeout = 30_000)] + public async Task Loopback_streamable_http_works_and_get_sse_legacy_sse_are_rejected() + { + int port = FreePort(); + using var stop = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); + LocaleCatalog locale = LocaleCatalog.Load(Path.GetFullPath("config/locales"), "en"); + var settings = new OllamaSettings(new Uri("http://127.0.0.1:1"), "unused", false, "0", new Dictionary()); + Task server = McpHost.RunHttpAsync(locale, settings, new AgentRuntimeSettings("", "en", [], "sandbox"), new VisionLimits(), "127.0.0.1", port, stop.Token); + using var http = new HttpClient(); + Uri endpoint = new($"http://127.0.0.1:{port}/mcp"); + await WaitUntilListening(http, endpoint, TestContext.Current.CancellationToken); + + var transport = new HttpClientTransport(new HttpClientTransportOptions + { + Name = "http-test", + Endpoint = endpoint, + TransportMode = HttpTransportMode.StreamableHttp + }); + await using (McpClient client = await McpClient.CreateAsync(transport, cancellationToken: TestContext.Current.CancellationToken)) + { + IList tools = await client.ListToolsAsync(cancellationToken: TestContext.Current.CancellationToken); + Assert.Equal(["agent_run", "agent_status"], tools.Select(static tool => tool.Name).Order(StringComparer.Ordinal)); + } + + using HttpResponseMessage get = await http.GetAsync(endpoint, TestContext.Current.CancellationToken); + Assert.Equal(HttpStatusCode.MethodNotAllowed, get.StatusCode); + using var hostileOrigin = new HttpRequestMessage(HttpMethod.Post, endpoint); + hostileOrigin.Headers.Add("Origin", "https://example.invalid"); + using HttpResponseMessage forbidden = await http.SendAsync(hostileOrigin, TestContext.Current.CancellationToken); + Assert.Equal(HttpStatusCode.Forbidden, forbidden.StatusCode); + using var loopbackOrigin = new HttpRequestMessage(HttpMethod.Get, endpoint); + loopbackOrigin.Headers.Add("Origin", $"http://127.0.0.1:{port}"); + using HttpResponseMessage loopbackAllowed = await http.SendAsync(loopbackOrigin, TestContext.Current.CancellationToken); + Assert.Equal(HttpStatusCode.MethodNotAllowed, loopbackAllowed.StatusCode); + Assert.Equal(HttpStatusCode.NotFound, (await http.GetAsync(new Uri($"http://127.0.0.1:{port}/sse"), TestContext.Current.CancellationToken)).StatusCode); + Assert.Equal(HttpStatusCode.NotFound, (await http.PostAsync(new Uri($"http://127.0.0.1:{port}/message"), null, TestContext.Current.CancellationToken)).StatusCode); + + await stop.CancelAsync(); + await server; + } + + [Fact] + public async Task Non_loopback_bind_is_rejected_before_server_start() + { + LocaleCatalog locale = LocaleCatalog.Load(Path.GetFullPath("config/locales"), "en"); + var settings = new OllamaSettings(new Uri("http://127.0.0.1:1"), "unused", false, "0", new Dictionary()); + await Assert.ThrowsAsync(() => McpHost.RunHttpAsync(locale, settings, + new AgentRuntimeSettings("", "en", [], "sandbox"), new VisionLimits(), "0.0.0.0", 8080, TestContext.Current.CancellationToken)); + } + + private static int FreePort() + { + TcpListener listener = new(IPAddress.Loopback, 0); + listener.Start(); + int port = ((IPEndPoint)listener.LocalEndpoint).Port; + listener.Stop(); + return port; + } + + private static async Task WaitUntilListening(HttpClient client, Uri endpoint, CancellationToken cancellationToken) + { + for (int attempt = 0; attempt < 100; attempt++) + { + try + { + using HttpResponseMessage response = await client.GetAsync(endpoint, cancellationToken); + return; + } + catch (HttpRequestException) when (attempt < 99) + { + await Task.Delay(25, cancellationToken); + } + } + } +} diff --git a/tests/Agent.Core.Tests/McpStdioTests.cs b/tests/Agent.Core.Tests/McpStdioTests.cs new file mode 100644 index 0000000..66f4851 --- /dev/null +++ b/tests/Agent.Core.Tests/McpStdioTests.cs @@ -0,0 +1,157 @@ +using Agent.App.Configuration; +using ModelContextProtocol; +using ModelContextProtocol.Client; +using ModelContextProtocol.Protocol; +using System.Net; +using System.Net.Sockets; +using System.Text; + +namespace Agent.Core.Tests; + +public sealed class McpStdioTests +{ + [Fact(Timeout = 30_000)] + public async Task InitializeListAndCallExposeOnlyTwoStrictTools() + { + using var directory = new TestDirectory(); + await using var ollama = new FakeOllamaServer(); + string configPath = CreateBundleConfiguration(directory, ollama.Endpoint); + string application = typeof(ConfigurationLoader).Assembly.Location; + var transport = new StdioClientTransport(new StdioClientTransportOptions + { + Name = "agent-mcp-test", + Command = "dotnet", + Arguments = [application, "serve", "--config", configPath, "--lang", "en"], + WorkingDirectory = directory.Path + }); + + await using McpClient client = await McpClient.CreateAsync(transport, cancellationToken: TestContext.Current.CancellationToken); + IList tools = await client.ListToolsAsync(cancellationToken: TestContext.Current.CancellationToken); + Assert.Equal(["agent_run", "agent_status"], tools.Select(static tool => tool.Name).Order(StringComparer.Ordinal)); + foreach (McpClientTool tool in tools) + { + Assert.False(tool.JsonSchema.GetProperty("additionalProperties").GetBoolean()); + } + + CallToolResult run = await client.CallToolAsync( + "agent_run", + new Dictionary { ["request"] = "deterministic test" }, + cancellationToken: TestContext.Current.CancellationToken); + string text = Assert.IsType(Assert.Single(run.Content)).Text; + Assert.Contains("status: queued", text, StringComparison.Ordinal); + Guid taskId = ReadGuid(text, "task_id"); + Guid sessionId = ReadGuid(text, "session_id"); + + CallToolResult status = await client.CallToolAsync( + "agent_status", + new Dictionary { ["task_id"] = taskId, ["session_id"] = sessionId }, + cancellationToken: TestContext.Current.CancellationToken); + Assert.Contains("task_id:", Assert.IsType(Assert.Single(status.Content)).Text, StringComparison.Ordinal); + + CallToolResult invalid = await client.CallToolAsync( + "agent_run", + new Dictionary { ["request"] = "x", ["model"] = "forbidden" }, + cancellationToken: TestContext.Current.CancellationToken); + Assert.True(invalid.IsError); + + CallToolResult invalidImage = await client.CallToolAsync( + "agent_run", + new Dictionary + { + ["request"] = "x", + ["images"] = new[] { new Dictionary { ["mime_type"] = "image/gif", ["data_base64"] = "AA==" } } + }, + cancellationToken: TestContext.Current.CancellationToken); + Assert.True(invalidImage.IsError); + } + + private static Guid ReadGuid(string text, string key) + { + string line = text.Split('\n', StringSplitOptions.TrimEntries).Single(value => value.StartsWith(key + ":", StringComparison.Ordinal)); + return Guid.Parse(line[(key.Length + 1)..].Trim()); + } + + private static string CreateBundleConfiguration(TestDirectory directory, Uri endpoint) + { + directory.File("prompts/user-system.md", string.Empty); + directory.File("config/locales/en.json", """{"app.starting":"Starting","check.success":"OK","check.failure":"Failed {message}","check.wfp_admin_required":"Administrator required","config.invalid":"Invalid {message}","locale.invalid":"Locale {message}","mcp.task_queued":"Task {task} queued in {session}","mcp.task_not_found":"Task {task} not found","sandbox.access_denied":"Denied {path}","ollama.vision_unsupported":"No vision"}"""); + Directory.CreateDirectory(System.IO.Path.Combine(directory.Path, "skills")); + return directory.File("agent.yaml", $$""" + schema_version: 1 + application: + language: en + mcp: + transport: stdio + agent: + user_system_prompt: prompts/user-system.md + ollama: + endpoint: {{endpoint}} + model: model + localization: + directory: config/locales + fallback: en + skills: + directory: skills + logging: + directory: logs + """); + } + + private sealed class FakeOllamaServer : IAsyncDisposable + { + private readonly TcpListener _listener = new(IPAddress.Loopback, 0); + private readonly CancellationTokenSource _stop = new(); + private readonly Task _loop; + + public FakeOllamaServer() + { + _listener.Start(); + Endpoint = new Uri($"http://127.0.0.1:{((IPEndPoint)_listener.LocalEndpoint).Port}/"); + _loop = RunAsync(); + } + + public Uri Endpoint { get; } + + private async Task RunAsync() + { + try + { + while (!_stop.IsCancellationRequested) + { + using TcpClient client = await _listener.AcceptTcpClientAsync(_stop.Token); + await HandleAsync(client, _stop.Token); + } + } + catch (OperationCanceledException) when (_stop.IsCancellationRequested) { } + } + + private static async Task HandleAsync(TcpClient client, CancellationToken cancellationToken) + { + NetworkStream stream = client.GetStream(); + using var reader = new StreamReader(stream, Encoding.ASCII, false, 1024, true); + int contentLength = 0; + while (await reader.ReadLineAsync(cancellationToken) is { } line && line.Length > 0) + { + if (line.StartsWith("Content-Length:", StringComparison.OrdinalIgnoreCase)) + contentLength = int.Parse(line[15..].Trim(), System.Globalization.CultureInfo.InvariantCulture); + } + if (contentLength > 0) + { + char[] body = new char[contentLength]; + await reader.ReadBlockAsync(body.AsMemory(), cancellationToken); + } + byte[] payload = Encoding.UTF8.GetBytes("""{"message":{"role":"assistant","content":"done"},"prompt_eval_count":3,"eval_count":2}"""); + byte[] header = Encoding.ASCII.GetBytes($"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {payload.Length}\r\nConnection: close\r\n\r\n"); + await stream.WriteAsync(header, cancellationToken); + await stream.WriteAsync(payload, cancellationToken); + } + + public async ValueTask DisposeAsync() + { + await _stop.CancelAsync(); + _listener.Stop(); + await _loop; + _stop.Dispose(); + } + } +} diff --git a/tests/Agent.Core.Tests/OllamaClientTests.cs b/tests/Agent.Core.Tests/OllamaClientTests.cs new file mode 100644 index 0000000..aa98764 --- /dev/null +++ b/tests/Agent.Core.Tests/OllamaClientTests.cs @@ -0,0 +1,143 @@ +using System.Net; +using System.Text; +using System.Text.Json; +using Agent.App.Configuration; +using Agent.Core.Ollama; +using Agent.Core.Tools; + +namespace Agent.Core.Tests; + +public sealed class OllamaClientTests +{ + [Fact] + public async Task Chat_sends_expected_request_and_returns_usage() + { + string? body = null; + var handler = new StubHandler(async (request, _) => + { + body = await request.Content!.ReadAsStringAsync(); + return Json(HttpStatusCode.OK, """{"message":{"role":"assistant","content":"done"},"prompt_eval_count":12,"eval_count":7}"""); + }); + OllamaClient client = Create(handler, options: new Dictionary + { + ["temperature"] = JsonSerializer.SerializeToElement(0.2), + ["seed"] = JsonSerializer.SerializeToElement(42) + }); + + OllamaResult result = await client.ChatAsync([new OllamaMessage("user", "hello")], 0.7, TestContext.Current.CancellationToken); + + Assert.Equal("done", result.Content); + Assert.Equal(12, result.InputTokens); + Assert.Equal(7, result.OutputTokens); + using JsonDocument document = JsonDocument.Parse(body!); + JsonElement root = document.RootElement; + Assert.Equal("test-model", root.GetProperty("model").GetString()); + Assert.False(root.GetProperty("stream").GetBoolean()); + Assert.Equal(0.7, root.GetProperty("options").GetProperty("temperature").GetDouble()); + Assert.Equal(42, root.GetProperty("options").GetProperty("seed").GetInt32()); + } + + [Fact] + public async Task Transient_failure_retries_three_times_with_configured_backoff() + { + int attempts = 0; + var delays = new List(); + var handler = new StubHandler((_, _) => + { + attempts++; + return Task.FromResult(attempts < 4 + ? Json(HttpStatusCode.ServiceUnavailable, "busy") + : Json(HttpStatusCode.OK, """{"message":{"role":"assistant","content":"ok"}}""")); + }); + var client = new OllamaClient(new HttpClient(handler), Settings(), (delay, _) => + { + delays.Add(delay); + return Task.CompletedTask; + }); + + OllamaResult result = await client.ChatAsync([new OllamaMessage("user", "hello")], null, TestContext.Current.CancellationToken); + + Assert.Equal("ok", result.Content); + Assert.Equal(4, attempts); + Assert.Equal([TimeSpan.FromMilliseconds(250), TimeSpan.FromMilliseconds(500), TimeSpan.FromMilliseconds(1000)], delays); + } + + [Fact] + public async Task Permanent_failure_is_not_retried() + { + int attempts = 0; + var handler = new StubHandler((_, _) => + { + attempts++; + return Task.FromResult(Json(HttpStatusCode.BadRequest, "invalid")); + }); + OllamaClient client = Create(handler); + + OllamaException error = await Assert.ThrowsAsync(() => + client.ChatAsync([new OllamaMessage("user", "hello")], null, TestContext.Current.CancellationToken)); + + Assert.False(error.IsTransient); + Assert.Equal(1, attempts); + } + + [Fact] + public void Typed_options_override_extra_options() + { + var configuration = new OllamaConfiguration + { + Endpoint = "http://127.0.0.1:11434", + Model = "model", + Options = new OllamaGenerationOptions { Temperature = 0.4, NumCtx = 8192 }, + ExtraOptions = new Dictionary + { + ["temperature"] = 1.9, + ["num_ctx"] = 256, + ["custom"] = "preserved" + } + }; + + OllamaSettings settings = OllamaSettingsFactory.Create(configuration); + + Assert.Equal(0.4, settings.Options["temperature"].GetDouble()); + Assert.Equal(8192, settings.Options["num_ctx"].GetInt32()); + Assert.Equal("preserved", settings.Options["custom"].GetString()); + } + + [Fact] + public async Task Native_tool_catalog_is_serialized_and_tool_call_is_parsed() + { + string? body = null; + var handler = new StubHandler(async (request, _) => + { + body = await request.Content!.ReadAsStringAsync(); + return Json(HttpStatusCode.OK, """{"message":{"role":"assistant","content":"","tool_calls":[{"function":{"name":"file_read","arguments":{"path":"x","mode":"whole"}}}]},"prompt_eval_count":2,"eval_count":1}"""); + }); + OllamaClient client = Create(handler); + + OllamaResult result = await client.ChatWithToolsAsync([new OllamaMessage("user", "read")], null, InternalToolCatalog.All, TestContext.Current.CancellationToken); + + using JsonDocument document = JsonDocument.Parse(body!); + Assert.Equal(5, document.RootElement.GetProperty("tools").GetArrayLength()); + Assert.All(document.RootElement.GetProperty("tools").EnumerateArray(), tool => + Assert.False(tool.GetProperty("function").GetProperty("parameters").GetProperty("additionalProperties").GetBoolean())); + OllamaToolCall call = Assert.Single(result.ToolCalls!); + Assert.Equal("file_read", call.Function.Name); + Assert.Equal("x", call.Function.Arguments.GetProperty("path").GetString()); + } + + private static OllamaClient Create(HttpMessageHandler handler, IReadOnlyDictionary? options = null) => + new(new HttpClient(handler), Settings(options), static (_, _) => Task.CompletedTask); + + private static OllamaSettings Settings(IReadOnlyDictionary? options = null) => + new(new Uri("http://127.0.0.1:11434"), "test-model", false, "0", options ?? new Dictionary()); + + private static HttpResponseMessage Json(HttpStatusCode statusCode, string content) => new(statusCode) + { + Content = new StringContent(content, Encoding.UTF8, "application/json") + }; + + private sealed class StubHandler(Func> callback) : HttpMessageHandler + { + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) => callback(request, cancellationToken); + } +} diff --git a/tests/Agent.Core.Tests/PromptAndRuntimeTests.cs b/tests/Agent.Core.Tests/PromptAndRuntimeTests.cs new file mode 100644 index 0000000..23ad2d2 --- /dev/null +++ b/tests/Agent.Core.Tests/PromptAndRuntimeTests.cs @@ -0,0 +1,220 @@ +using Agent.Core.Ollama; +using Agent.Core.Prompts; +using Agent.Core.Runtime; +using Agent.Core.Sessions; +using Agent.Core.Tasks; +using Agent.Core.Tools; +using System.Text.Json; + +namespace Agent.Core.Tests; + +public sealed class PromptAndRuntimeTests +{ + [Fact] + public void Prompt_order_is_fixed_and_request_is_last() + { + IReadOnlyList messages = PromptComposer.Compose(new PromptContext( + "fixed", "user system", "ja", ["skill"], "capability", "graph", "summary", + [new ConversationMessage("user", "old"), new ConversationMessage("assistant", "reply")], + "new request", ["image"])); + + Assert.Equal(["fixed", "user system", "Respond in locale: ja.", "skill", "capability", "graph", "summary", "old", "reply", "new request"], messages.Select(x => x.Content)); + Assert.Equal("system", messages[0].Role); + Assert.Equal("user", messages[^1].Role); + Assert.Equal(["image"], messages[^1].Images); + } + + [Fact] + public async Task Runtime_continues_session_and_accumulates_usage() + { + var sessions = new SessionStore(); + var registry = new TaskRegistry(sessions); + var ollama = new RecordingOllamaClient(); + var runtime = new AgentRuntime(registry, ollama, new AgentRuntimeSettings("custom", "ja", [], "sandbox")); + TaskSnapshot first = registry.Enqueue("first", null, 0.5); + Assert.True(registry.Transition(first.TaskId, AgentTaskState.Planning)); + Assert.True(registry.Transition(first.TaskId, AgentTaskState.Running)); + + await runtime.ExecuteAsync(first.TaskId, TestContext.Current.CancellationToken); + Assert.True(registry.TryGet(first.TaskId, null, out TaskSnapshot? completed)); + Assert.True(completed!.State == AgentTaskState.Completed, completed.Result); + Assert.Equal(3, completed.InputTokens); + Assert.Equal(2, completed.OutputTokens); + + TaskSnapshot second = registry.Enqueue("second", first.SessionId, null); + Assert.True(registry.Transition(second.TaskId, AgentTaskState.Planning)); + Assert.True(registry.Transition(second.TaskId, AgentTaskState.Running)); + await runtime.ExecuteAsync(second.TaskId, TestContext.Current.CancellationToken); + + Assert.Equal(2, ollama.Calls.Count); + Assert.Contains(ollama.Calls[1], message => message.Role == "user" && message.Content == "first"); + Assert.Contains(ollama.Calls[1], message => message.Role == "assistant" && message.Content == "answer-1"); + Assert.Equal(0.5, ollama.Temperatures[0]); + Assert.Null(ollama.Temperatures[1]); + } + + [Fact] + public void Session_store_is_process_local() + { + SessionStore first = new(); + Guid id = first.Create().SessionId; + SessionStore restarted = new(); + + Assert.False(restarted.TryGet(id, out _)); + } + + [Fact] + public async Task Non_vision_model_rejects_images_with_selected_locale_message_before_chat() + { + TaskRegistry registry = new(); + var ollama = new NonVisionOllamaClient(); + const string localized = "選択したモデルは画像入力に対応していません。"; + var runtime = new AgentRuntime(registry, ollama, + new AgentRuntimeSettings("", "ja", [], "sandbox", VisionUnsupportedMessage: localized)); + TaskSnapshot task = registry.Enqueue("画像を確認", null, null, ["base64-image"]); + registry.Transition(task.TaskId, AgentTaskState.Planning); + registry.Transition(task.TaskId, AgentTaskState.Running); + + await runtime.ExecuteAsync(task.TaskId, TestContext.Current.CancellationToken); + + Assert.True(registry.TryGet(task.TaskId, null, out TaskSnapshot? failed)); + Assert.Equal(AgentTaskState.Failed, failed!.State); + Assert.Equal(localized, failed.Result); + Assert.Equal(0, ollama.ChatCalls); + } + + [Fact] + public async Task Runtime_dispatches_native_tool_call_then_completes_only_after_effect_evidence() + { + TaskRegistry registry = new(); + FakeInternalToolBackend backend = new(); + var dispatcher = new InternalToolDispatcher( + new DenyByDefaultPolicy(InternalToolCatalog.All.Select(static item => item.Name).ToHashSet(StringComparer.Ordinal)), backend); + var ollama = new ToolCallingOllamaClient(); + var runtime = new AgentRuntime(registry, ollama, new AgentRuntimeSettings("", "ja", [], "sandbox"), dispatcher); + TaskSnapshot task = registry.Enqueue("fileを修正", null, null); + registry.Transition(task.TaskId, AgentTaskState.Planning); + registry.Transition(task.TaskId, AgentTaskState.Running); + + await runtime.ExecuteAsync(task.TaskId, TestContext.Current.CancellationToken); + + Assert.True(registry.TryGet(task.TaskId, null, out TaskSnapshot? completed)); + Assert.True(completed!.State == AgentTaskState.Completed, $"{completed.Result}; backend={backend.Calls.Count}:{string.Join(',', backend.Calls.Select(x => x.Name))}; ollama={ollama.CallCount}"); + Assert.Equal("修正しました。", completed.Result); + Assert.Single(backend.Calls); + Assert.Equal("file_edit", backend.Calls[0].Name); + Assert.Equal(2, ollama.CallCount); + Assert.Equal(5, ollama.ToolCount); + } + + [Fact] + public async Task Runtime_reaches_terminal_failed_state_when_resource_cleanup_fails() + { + TaskRegistry registry = new(); + var backend = new CleanupFailingBackend(); + var dispatcher = new InternalToolDispatcher(new DenyByDefaultPolicy(), backend); + var runtime = new AgentRuntime(registry, new RecordingOllamaClient(), new AgentRuntimeSettings("", "en", [], "sandbox"), dispatcher); + TaskSnapshot task = registry.Enqueue("analyze", null, null); + registry.Transition(task.TaskId, AgentTaskState.Planning); + registry.Transition(task.TaskId, AgentTaskState.Running); + + await runtime.ExecuteAsync(task.TaskId, TestContext.Current.CancellationToken); + + Assert.True(registry.TryGet(task.TaskId, null, out TaskSnapshot? failed)); + Assert.Equal(AgentTaskState.Failed, failed!.State); + Assert.Contains("cleanup failed", failed.Result, StringComparison.OrdinalIgnoreCase); + Assert.Equal(2, backend.EndCalls); + } + + [Fact] + public async Task Runtime_cannot_complete_with_an_unresolved_tool_failure() + { + TaskRegistry registry = new(); + var dispatcher = new InternalToolDispatcher( + new DenyByDefaultPolicy(new HashSet(["process_run"], StringComparer.Ordinal)), new FailingProcessBackend()); + var runtime = new AgentRuntime(registry, new FailingThenFinalOllamaClient(), new AgentRuntimeSettings("", "en", [], "sandbox", MaxIterations: 2), dispatcher); + TaskSnapshot task = registry.Enqueue("run build", null, null); + registry.Transition(task.TaskId, AgentTaskState.Planning); + registry.Transition(task.TaskId, AgentTaskState.Running); + + await runtime.ExecuteAsync(task.TaskId, TestContext.Current.CancellationToken); + + Assert.True(registry.TryGet(task.TaskId, null, out TaskSnapshot? failed)); + Assert.Equal(AgentTaskState.Failed, failed!.State); + Assert.Contains("iteration limit", failed.Result, StringComparison.OrdinalIgnoreCase); + } + + private sealed class RecordingOllamaClient : IOllamaClient + { + public List> Calls { get; } = []; + public List Temperatures { get; } = []; + + public Task ChatAsync(IReadOnlyList messages, double? temperatureOverride, CancellationToken cancellationToken) + { + Calls.Add(messages); + Temperatures.Add(temperatureOverride); + return Task.FromResult(new OllamaResult($"answer-{Calls.Count}", 3, 2)); + } + } + + private sealed class ToolCallingOllamaClient : IOllamaClient + { + public int CallCount { get; private set; } + public int ToolCount { get; private set; } + public Task ChatAsync(IReadOnlyList messages, double? temperatureOverride, CancellationToken cancellationToken) => + Task.FromResult(new OllamaResult("修正しました。", 1, 1)); + public Task ChatWithToolsAsync(IReadOnlyList messages, double? temperatureOverride, IReadOnlyList tools, CancellationToken cancellationToken) + { + CallCount++; + ToolCount = tools.Count; + if (CallCount == 1) + { + using JsonDocument document = JsonDocument.Parse("""{"path":"x","operation":"overwrite","content":"new"}"""); + return Task.FromResult(new OllamaResult("", 2, 1, [new OllamaToolCall(new OllamaFunctionCall("file_edit", document.RootElement.Clone()))])); + } + return Task.FromResult(new OllamaResult("修正しました。", 2, 2)); + } + } + + private sealed class NonVisionOllamaClient : IOllamaClient + { + public int ChatCalls { get; private set; } + public Task SupportsVisionAsync(CancellationToken cancellationToken) => Task.FromResult(false); + public Task ChatAsync(IReadOnlyList messages, double? temperatureOverride, CancellationToken cancellationToken) + { + ChatCalls++; + return Task.FromResult(new OllamaResult("unexpected", 0, 0)); + } + } + + private sealed class CleanupFailingBackend : IInternalToolBackend + { + public int EndCalls { get; private set; } + public Task ExecuteAsync(Guid taskId, string toolName, JsonElement arguments, CancellationToken cancellationToken) => + Task.FromResult(new ToolExecutionResult(true, "unused", false)); + public Task EndTaskAsync(Guid taskId, bool success, CancellationToken cancellationToken) + { + EndCalls++; + throw new IOException("cleanup failed"); + } + } + + private sealed class FailingProcessBackend : IInternalToolBackend + { + public Task ExecuteAsync(Guid taskId, string toolName, JsonElement arguments, CancellationToken cancellationToken) => + Task.FromResult(new ToolExecutionResult(false, "exit 1", true, "process:1")); + } + + private sealed class FailingThenFinalOllamaClient : IOllamaClient + { + private int _calls; + public Task ChatAsync(IReadOnlyList messages, double? temperatureOverride, CancellationToken cancellationToken) => + Task.FromResult(new OllamaResult("done", 1, 1)); + public Task ChatWithToolsAsync(IReadOnlyList messages, double? temperatureOverride, IReadOnlyList tools, CancellationToken cancellationToken) + { + if (_calls++ > 0) return Task.FromResult(new OllamaResult("done", 1, 1)); + using JsonDocument arguments = JsonDocument.Parse("""{"executable":"x","args":[],"cwd":"."}"""); + return Task.FromResult(new OllamaResult("", 1, 1, [new OllamaToolCall(new OllamaFunctionCall("process_run", arguments.RootElement.Clone()))])); + } + } +} diff --git a/tests/Agent.Core.Tests/TaskRegistryTests.cs b/tests/Agent.Core.Tests/TaskRegistryTests.cs new file mode 100644 index 0000000..33e16ab --- /dev/null +++ b/tests/Agent.Core.Tests/TaskRegistryTests.cs @@ -0,0 +1,35 @@ +using Agent.Core.Tasks; + +namespace Agent.Core.Tests; + +public sealed class TaskRegistryTests +{ + [Fact] + public void NewTaskGetsUniqueTaskAndSessionIds() + { + var registry = new TaskRegistry(); + TaskSnapshot first = registry.Enqueue("one", null, null); + TaskSnapshot second = registry.Enqueue("two", null, 0.2); + Assert.NotEqual(first.TaskId, second.TaskId); + Assert.NotEqual(first.SessionId, second.SessionId); + Assert.Equal(AgentTaskState.Queued, first.State); + } + + [Fact] + public void UnknownSessionIsRejected() + { + var registry = new TaskRegistry(); + Assert.Throws(() => registry.Enqueue("request", Guid.NewGuid(), null)); + } + + [Fact] + public void TerminalTaskCannotTransition() + { + var registry = new TaskRegistry(); + TaskSnapshot task = registry.Enqueue("request", null, null); + Assert.True(registry.Transition(task.TaskId, AgentTaskState.Planning)); + Assert.True(registry.Transition(task.TaskId, AgentTaskState.Running)); + Assert.True(registry.Transition(task.TaskId, AgentTaskState.Completed)); + Assert.False(registry.Transition(task.TaskId, AgentTaskState.Running)); + } +} diff --git a/tests/Agent.Core.Tests/TestDirectory.cs b/tests/Agent.Core.Tests/TestDirectory.cs new file mode 100644 index 0000000..075d050 --- /dev/null +++ b/tests/Agent.Core.Tests/TestDirectory.cs @@ -0,0 +1,21 @@ +namespace Agent.Core.Tests; + +internal sealed class TestDirectory : IDisposable +{ + public TestDirectory() + { + Path = System.IO.Path.Combine(System.IO.Path.GetTempPath(), "agent-mcp-tests", Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(Path); + } + + public string Path { get; } + public string File(string relativePath, string content) + { + string path = System.IO.Path.Combine(Path, relativePath); + Directory.CreateDirectory(System.IO.Path.GetDirectoryName(path)!); + System.IO.File.WriteAllText(path, content); + return path; + } + + public void Dispose() => Directory.Delete(Path, true); +} diff --git a/tests/Agent.Core.Tests/Usings.cs b/tests/Agent.Core.Tests/Usings.cs new file mode 100644 index 0000000..c802f44 --- /dev/null +++ b/tests/Agent.Core.Tests/Usings.cs @@ -0,0 +1 @@ +global using Xunit; -- cgit v1.2.3