1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
|
param(
[Parameter(Mandatory)][string]$TestRoot,
[Parameter(Mandatory)][string]$VenvPython,
[string]$EvidenceRoot = 'artifacts\evidence\readme-verification',
[switch]$CliOnly
)
$ErrorActionPreference = 'Stop'
$repository = (Resolve-Path (Join-Path $PSScriptRoot '..')).Path
$testRootPath = (Resolve-Path -LiteralPath $TestRoot).Path
$venvPythonPath = (Resolve-Path -LiteralPath $VenvPython).Path
$evidence = [IO.Path]::GetFullPath((Join-Path $repository $EvidenceRoot))
if (-not $evidence.StartsWith($repository + [IO.Path]::DirectorySeparatorChar, [StringComparison]::OrdinalIgnoreCase)) { throw 'Evidence output must remain inside the repository.' }
$pwshPath = (Get-Command pwsh.exe -ErrorAction Stop).Source
$cmdPath = $env:ComSpec
$venvRoot = Split-Path (Split-Path $venvPythonPath)
$pythonRuntime = (((Get-Content -LiteralPath (Join-Path $venvRoot 'pyvenv.cfg')) | Where-Object { $_ -match '^home\s*=' }) -split '=', 2)[1].Trim()
$basePythonPath = Join-Path $pythonRuntime 'python.exe'
if (-not (Test-Path -LiteralPath $basePythonPath -PathType Leaf)) { throw "Base Python does not exist: $basePythonPath" }
New-Item -ItemType Directory -Force -Path $evidence | Out-Null
$runRoot = Join-Path $testRootPath ('agent-mcp-readme-' + [Guid]::NewGuid().ToString('N'))
New-Item -ItemType Directory -Path $runRoot | Out-Null
$agent = Join-Path $repository 'build\Release\Agent.App\net10.0-windows10.0.26100.0\agent-mcp.exe'
$windowsTests = Join-Path $repository 'build\Release\Agent.Windows.Tests\net10.0-windows10.0.26100.0\Agent.Windows.Tests.exe'
$configPath = Join-Path $runRoot 'agent.yaml'
$logPath = Join-Path $evidence 'verification.log'
$server = $null
$failed = $false
function Invoke-Step([string]$Name, [scriptblock]$Action) {
"START $Name $(Get-Date -Format o)" | Tee-Object -FilePath $logPath -Append
try {
$global:LASTEXITCODE = 0
& $Action *>&1 | Tee-Object -FilePath $logPath -Append
if ($LASTEXITCODE -ne 0) { throw "$Name exited with $LASTEXITCODE" }
"PASS $Name" | Tee-Object -FilePath $logPath -Append
} catch { $script:failed = $true; "FAIL $Name $($_.Exception.Message)" | Tee-Object -FilePath $logPath -Append }
}
function Invoke-ExpectedFailure([string]$Name, [string[]]$Arguments) {
& $agent @Arguments *>&1 | Tee-Object -FilePath $logPath -Append
if ($LASTEXITCODE -ne 2) { $script:failed = $true; "FAIL $Name expected=2 actual=$LASTEXITCODE" | Tee-Object -FilePath $logPath -Append }
else { "PASS $Name" | Tee-Object -FilePath $logPath -Append }
}
try {
$listener = [Net.Sockets.TcpListener]::new([Net.IPAddress]::Loopback, 0); $listener.Start()
$httpPort = ([Net.IPEndPoint]$listener.LocalEndpoint).Port; $listener.Stop()
$forwardRoot = $testRootPath.Replace('\','/')
$forwardPrompt = (Join-Path $repository 'prompts\user-system.md').Replace('\','/')
$forwardSkills = (Join-Path $repository 'skills').Replace('\','/')
$forwardLocales = (Join-Path $repository 'config\locales').Replace('\','/')
$forwardLogs = (Join-Path $runRoot 'logs').Replace('\','/')
$pythonYaml = $basePythonPath.Replace('\','/'); $runtimeYaml = $pythonRuntime.Replace('\','/')
$pwshYaml = $pwshPath.Replace('\','/'); $cmdYaml = $cmdPath.Replace('\','/')
@"
schema_version: 2
application: { language: en }
mcp:
transport: http
http: { host: 127.0.0.1, port: $httpPort }
agent: { user_system_prompt: $forwardPrompt, max_iterations: 8, same_failure_limit: 2 }
ollama: { endpoint: 'http://127.0.0.1:11434', model: 'deepseek-v4-flash:cloud', think: true }
planner: { max_tasks: 8 }
sandbox:
profile: development
workspaces: [$forwardRoot]
mounts: []
execution:
entries:
- { alias: python, path: $pythonYaml, runtime_root: $runtimeYaml }
- { alias: powershell, path: '$pwshYaml' }
- { alias: cmd, path: $cmdYaml }
network: { access: none, local_endpoints: [], restricted_endpoints: [] }
resources: { profile: constrained }
localization: { directory: $forwardLocales, fallback: en }
skills: { directory: $forwardSkills }
logging: { directory: $forwardLogs, level: information, max_file_bytes: 10485760, retained_files: 5 }
"@ | Set-Content -LiteralPath $configPath -Encoding utf8NoBOM
$normalizedConfig = [IO.Path]::GetFullPath($configPath).ToUpperInvariant()
$hash = [Convert]::ToHexString([Security.Cryptography.SHA256]::HashData([Text.Encoding]::UTF8.GetBytes($normalizedConfig)))
$appliedStorePath = Join-Path $env:LOCALAPPDATA "AgentSandboxMcp\applied-configurations\$hash.json"
Invoke-Step 'cli-show-before-apply' { & $agent show $configPath }
Invoke-Step 'cli-apply-yes' { & $agent apply $configPath --yes }
Invoke-Step 'cli-show-applied' { & $agent show $configPath }
Invoke-ExpectedFailure 'removed-serve' @('serve')
Invoke-ExpectedFailure 'removed-check' @('check')
Invoke-ExpectedFailure 'removed-config-option' @('--config', $configPath)
Invoke-ExpectedFailure 'removed-language-option' @('--lang', 'ja')
Invoke-ExpectedFailure 'removed-transport-option' @('--transport', 'http')
Invoke-ExpectedFailure 'removed-executable-command' @('executable', 'list')
Invoke-ExpectedFailure 'removed-sandbox-command' @('sandbox', 'show')
if (-not $CliOnly) {
Invoke-Step 'implicit-http-start' {
$script:server = Start-Process -FilePath $agent -ArgumentList @($configPath) -WorkingDirectory (Split-Path $agent) -WindowStyle Hidden -PassThru
$endpoint = "http://127.0.0.1:$httpPort/mcp"
for ($attempt=0; $attempt -lt 100; $attempt++) {
if ($script:server.HasExited) { throw "Server exited: $($script:server.ExitCode)" }
try { $response = Invoke-WebRequest -Uri $endpoint -Method Get -SkipHttpErrorCheck; if ($response.StatusCode -eq 405) { break } } catch { }
Start-Sleep -Milliseconds 100
}
if ($attempt -eq 100) { throw 'Server did not become ready.' }
$script:server.Kill($true); $script:server.WaitForExit(); $script:server = $null
}
$env:AGENT_MCP_MONKEY_ROOT = $testRootPath
$env:AGENT_MCP_TEST_VENV_PYTHON = $venvPythonPath
$env:AGENT_MCP_TEST_POWERSHELL = $pwshPath
$env:AGENT_MCP_TEST_POWERSHELL_RUNTIME = Split-Path $pwshPath
$env:AGENT_MCP_TEST_CMD = $cmdPath
Invoke-Step 'readme-operational-windows-tests' { & $windowsTests -class Agent.Windows.Tests.ReadmeOperationalTests -parallel none -failSkips }
}
} finally {
if ($server -and -not $server.HasExited) { $server.Kill($true); $server.WaitForExit() }
if ($appliedStorePath -and (Test-Path -LiteralPath $appliedStorePath)) { Remove-Item -LiteralPath $appliedStorePath -Force }
if (Test-Path -LiteralPath $runRoot) {
$resolved = (Resolve-Path -LiteralPath $runRoot).Path
if (-not $resolved.StartsWith($testRootPath.TrimEnd('\') + '\', [StringComparison]::OrdinalIgnoreCase)) { throw 'Refusing to clean outside TestRoot.' }
Remove-Item -LiteralPath $resolved -Recurse -Force
}
Remove-Item Env:AGENT_MCP_MONKEY_ROOT,Env:AGENT_MCP_TEST_VENV_PYTHON,Env:AGENT_MCP_TEST_POWERSHELL,Env:AGENT_MCP_TEST_POWERSHELL_RUNTIME,Env:AGENT_MCP_TEST_CMD -ErrorAction SilentlyContinue
}
if ($failed) { throw "README verification failed. Evidence: $evidence" }
Write-Output "README verification PASS. Evidence: $evidence"
|
