diff options
| author | tanaka takashi <isazaka@gmail.com> | 2026-08-12 00:21:23 +0900 |
|---|---|---|
| committer | tanaka takashi <isazaka@gmail.com> | 2026-08-12 00:21:23 +0900 |
| commit | 98b6a2aeedd591058c1d81e401d24f3ded162926 (patch) | |
| tree | 74cbb38af825526e4fc90b150feed14426d727b4 /tests/Agent.Core.Tests | |
| parent | ce550a127b7dc972651bd15b9f09979a2c38688f (diff) | |
fix: implement native Ollama thinking semantics
Model Ollama chat thinking separately from final content, preserve it only across assistant tool-call continuation, and keep private reasoning out of user results, session context, and logs.
Accept native boolean and low/medium/high/max think values, normalize YAML booleans, and serialize explicit overrides using the native top-level chat field.
Stop forcing think=false for structured planning, skill selection, and context summaries; corrected retries now retain the configured thinking mode.
Update the specification, design, audit report, and requirement traceability for OLL-14.
Add request/response, configuration, observability, retry, and tool-continuation coverage. Verified with 92 passing Agent.Core tests and a deepseek-v4-flash:cloud think:true MCP operational run.
Diffstat (limited to 'tests/Agent.Core.Tests')
| -rw-r--r-- | tests/Agent.Core.Tests/ConfigurationTests.cs | 30 | ||||
| -rw-r--r-- | tests/Agent.Core.Tests/ObservabilityTests.cs | 7 | ||||
| -rw-r--r-- | tests/Agent.Core.Tests/OllamaClientTests.cs | 54 | ||||
| -rw-r--r-- | tests/Agent.Core.Tests/PromptAndRuntimeTests.cs | 13 |
4 files changed, 98 insertions, 6 deletions
diff --git a/tests/Agent.Core.Tests/ConfigurationTests.cs b/tests/Agent.Core.Tests/ConfigurationTests.cs index 64b2d85..e588825 100644 --- a/tests/Agent.Core.Tests/ConfigurationTests.cs +++ b/tests/Agent.Core.Tests/ConfigurationTests.cs @@ -37,6 +37,36 @@ public sealed class ConfigurationTests Assert.Throws<ConfigurationException>(() => ConfigurationLoader.Load(config)); } + [Theory] + [InlineData("true")] + [InlineData("false")] + [InlineData("low")] + [InlineData("medium")] + [InlineData("high")] + [InlineData("max")] + public void Ollama_think_accepts_native_boolean_and_level_values(string value) + { + using var directory = new TestDirectory(); + string config = CreateValidConfiguration(directory); + System.IO.File.WriteAllText(config, System.IO.File.ReadAllText(config).Replace(" model: model", $" model: model\n think: {value}", StringComparison.Ordinal)); + + AgentConfiguration result = ConfigurationLoader.Load(config); + + Assert.Equal(value is "true" or "false" ? bool.Parse(value) : value, result.Ollama.Think); + } + + [Fact] + public void Ollama_think_rejects_unknown_level() + { + using var directory = new TestDirectory(); + string config = CreateValidConfiguration(directory); + System.IO.File.WriteAllText(config, System.IO.File.ReadAllText(config).Replace(" model: model", " model: model\n think: extreme", StringComparison.Ordinal)); + + ConfigurationException error = Assert.Throws<ConfigurationException>(() => ConfigurationLoader.Load(config)); + + Assert.Contains("ollama.think", error.Message, StringComparison.Ordinal); + } + [Fact] public void CommandLineLanguageOverrideIsCaptured() { diff --git a/tests/Agent.Core.Tests/ObservabilityTests.cs b/tests/Agent.Core.Tests/ObservabilityTests.cs index 8231ab9..3dcfdf6 100644 --- a/tests/Agent.Core.Tests/ObservabilityTests.cs +++ b/tests/Agent.Core.Tests/ObservabilityTests.cs @@ -23,8 +23,8 @@ public sealed class ObservabilityTests var context = new AgentLlmCallContext(callId, sessionId, taskId, plannerTaskId, "replan", 7, 2, "model-a"); using (var observer = new JsonLinesRuntimeObserver(directory.Path, 1024 * 1024, 2)) { - observer.LlmRequest(context, [new OllamaMessage("user", "request")]); - observer.LlmResponse(context, new OllamaResult("response", 11, 13)); + observer.LlmRequest(context, [new OllamaMessage("assistant", "request", Thinking: "private request reasoning")]); + observer.LlmResponse(context, new OllamaResult("response", 11, 13, Thinking: "private response reasoning")); observer.ToolResult(context, "file_read", new ToolExecutionResult(true, "ok", false)); observer.LlmFailure(context, "planning_failed", "invalid plan"); } @@ -45,6 +45,9 @@ public sealed class ObservabilityTests Assert.Equal("file_read", entries[2].GetProperty("tool_name").GetString()); Assert.Equal("llm.failure", entries[3].GetProperty("event_name").GetString()); Assert.Equal("planning_failed", entries[3].GetProperty("error_code").GetString()); + string log = File.ReadAllText(Path.Combine(directory.Path, "llm.jsonl")); + Assert.DoesNotContain("private request reasoning", log, StringComparison.Ordinal); + Assert.DoesNotContain("private response reasoning", log, StringComparison.Ordinal); } [Fact] diff --git a/tests/Agent.Core.Tests/OllamaClientTests.cs b/tests/Agent.Core.Tests/OllamaClientTests.cs index 94ce8cd..87a3e20 100644 --- a/tests/Agent.Core.Tests/OllamaClientTests.cs +++ b/tests/Agent.Core.Tests/OllamaClientTests.cs @@ -38,6 +38,60 @@ public sealed class OllamaClientTests } [Fact] + public async Task Thinking_is_separated_from_final_content_and_not_discarded() + { + var handler = new StubHandler((_, _) => Task.FromResult(Json(HttpStatusCode.OK, + """{"message":{"role":"assistant","thinking":"private reasoning","content":"final answer"},"prompt_eval_count":3,"eval_count":5}"""))); + var client = new OllamaClient(new HttpClient(handler), + new OllamaSettings(new Uri("http://127.0.0.1:11434"), "test-model", true, "0", new Dictionary<string, JsonElement>())); + + OllamaResult result = await client.ChatAsync([new OllamaMessage("user", "hello")], null, TestContext.Current.CancellationToken); + + Assert.Equal("final answer", result.Content); + Assert.Equal("private reasoning", result.Thinking); + } + + [Fact] + public async Task Thinking_level_and_assistant_thinking_history_follow_native_chat_schema() + { + string? body = null; + var handler = new StubHandler(async (request, _) => + { + body = await request.Content!.ReadAsStringAsync(); + return Json(HttpStatusCode.OK, """{"message":{"role":"assistant","content":"done"}}"""); + }); + var client = new OllamaClient(new HttpClient(handler), + new OllamaSettings(new Uri("http://127.0.0.1:11434"), "test-model", "high", "0", new Dictionary<string, JsonElement>())); + + await client.ChatAsync([new OllamaMessage("assistant", "", ToolCalls: [], Thinking: "prior reasoning")], null, + TestContext.Current.CancellationToken); + + using JsonDocument document = JsonDocument.Parse(body!); + Assert.Equal("high", document.RootElement.GetProperty("think").GetString()); + Assert.Equal("prior reasoning", document.RootElement.GetProperty("messages")[0].GetProperty("thinking").GetString()); + } + + [Fact] + public async Task Explicit_false_override_is_serialized_as_boolean_false() + { + string? body = null; + var handler = new StubHandler(async (request, _) => + { + body = await request.Content!.ReadAsStringAsync(); + return Json(HttpStatusCode.OK, """{"message":{"role":"assistant","content":"{}"}}"""); + }); + var client = new OllamaClient(new HttpClient(handler), + new OllamaSettings(new Uri("http://127.0.0.1:11434"), "test-model", "high", "0", new Dictionary<string, JsonElement>())); + using JsonDocument schema = JsonDocument.Parse("""{"type":"object"}"""); + + await client.ChatStructuredAsync([new OllamaMessage("user", "json")], null, schema.RootElement.Clone(), false, + TestContext.Current.CancellationToken); + + using JsonDocument document = JsonDocument.Parse(body!); + Assert.False(document.RootElement.GetProperty("think").GetBoolean()); + } + + [Fact] public async Task Transient_failure_retries_three_times_with_configured_backoff() { int attempts = 0; diff --git a/tests/Agent.Core.Tests/PromptAndRuntimeTests.cs b/tests/Agent.Core.Tests/PromptAndRuntimeTests.cs index 6dad528..082ea9b 100644 --- a/tests/Agent.Core.Tests/PromptAndRuntimeTests.cs +++ b/tests/Agent.Core.Tests/PromptAndRuntimeTests.cs @@ -124,6 +124,7 @@ public sealed class PromptAndRuntimeTests Assert.Equal("file_edit", backend.Calls[0].Name); Assert.Equal(2, ollama.CallCount); Assert.Equal(InternalToolCatalog.All.Count, ollama.ToolCount); + Assert.True(ollama.ThinkingPreservedForToolContinuation); } [Fact] @@ -270,7 +271,7 @@ public sealed class PromptAndRuntimeTests } [Fact] - public async Task Invalid_structured_plan_retries_once_with_think_disabled() + public async Task Invalid_structured_plan_retries_once_without_disabling_configured_thinking() { TaskRegistry registry = new(); var ollama = new ThinkRetryOllamaClient(succeedOnRetry: true); @@ -282,7 +283,7 @@ public sealed class PromptAndRuntimeTests Assert.True(registry.TryGet(task.TaskId, null, out TaskSnapshot? completed)); Assert.Equal(AgentTaskState.Completed, completed!.State); - Assert.Equal([null, false], ollama.ThinkOverrides); + Assert.Equal([null, null], ollama.ThinkOverrides); Assert.All(ollama.Temperatures, value => Assert.Equal(0.7, value)); Assert.Contains(ollama.PlanningMessages[0], message => message.Role == "system" && message.Content.Contains("There is no file_write Tool", StringComparison.Ordinal)); @@ -305,7 +306,7 @@ public sealed class PromptAndRuntimeTests Assert.True(registry.TryGet(task.TaskId, null, out TaskSnapshot? failed)); Assert.Equal(AgentTaskState.Failed, failed!.State); - Assert.Contains("think=false retry", failed.Result, StringComparison.OrdinalIgnoreCase); + Assert.Contains("one corrected retry", failed.Result, StringComparison.OrdinalIgnoreCase); Assert.Equal(0, ollama.RunCalls); } @@ -453,6 +454,7 @@ public sealed class PromptAndRuntimeTests { public int CallCount { get; private set; } public int ToolCount { get; private set; } + public bool ThinkingPreservedForToolContinuation { get; private set; } public Task<OllamaResult> ChatAsync(IReadOnlyList<OllamaMessage> messages, double? temperatureOverride, CancellationToken cancellationToken) => Task.FromResult(new OllamaResult("修正しました。", 1, 1)); public Task<OllamaResult> ChatStructuredAsync(IReadOnlyList<OllamaMessage> messages, double? temperatureOverride, JsonElement format, CancellationToken cancellationToken) @@ -468,8 +470,11 @@ public sealed class PromptAndRuntimeTests if (CallCount == 1) { using JsonDocument document = JsonDocument.Parse("""{"path":"x","operation":"overwrite","content":"new"}"""); - return Task.FromResult(new OllamaResult("", 2, 1, [new OllamaToolCall(new OllamaFunctionCall("file_edit", document.RootElement.Clone()))])); + return Task.FromResult(new OllamaResult("", 2, 1, + [new OllamaToolCall(new OllamaFunctionCall("file_edit", document.RootElement.Clone()))], Thinking: "private tool reasoning")); } + ThinkingPreservedForToolContinuation = messages.Any(message => + message.Role == "assistant" && message.Thinking == "private tool reasoning" && message.ToolCalls?.Count == 1); return Task.FromResult(new OllamaResult("修正しました。", 2, 2)); } } |
