param( [Parameter(Mandatory)] [string]$TestRoot, [Parameter(Mandatory)] [string]$VenvPython, [string]$EvidenceRoot = 'artifacts\evidence\readme-verification', [switch]$CliOnly ) $ErrorActionPreference = 'Stop' $OutputEncoding = [Text.UTF8Encoding]::new($false) [Console]::OutputEncoding = $OutputEncoding $repository = (Resolve-Path (Join-Path $PSScriptRoot '..')).Path $testRootPath = (Resolve-Path -LiteralPath $TestRoot).Path $venvPythonPath = (Resolve-Path -LiteralPath $VenvPython).Path $evidence = [IO.Path]::GetFullPath((Join-Path $repository $EvidenceRoot)) if (-not $evidence.StartsWith($repository + [IO.Path]::DirectorySeparatorChar, [StringComparison]::OrdinalIgnoreCase)) { throw 'Evidence output must remain inside the repository.' } $administrator = ([Security.Principal.WindowsPrincipal][Security.Principal.WindowsIdentity]::GetCurrent()).IsInRole( [Security.Principal.WindowsBuiltInRole]::Administrator) if (-not $administrator -and -not $CliOnly) { throw 'README verification must run as Windows Administrator unless -CliOnly is specified.' } if (-not (Test-Path -LiteralPath $venvPythonPath -PathType Leaf)) { throw 'VenvPython must be an existing file.' } $pwshPath = (Get-Command pwsh.exe -ErrorAction Stop).Source $pwshRuntime = Split-Path $pwshPath $cmdPath = $env:ComSpec $venvRoot = Split-Path (Split-Path $venvPythonPath) $pyvenv = Get-Content -LiteralPath (Join-Path $venvRoot 'pyvenv.cfg') $pythonRuntime = (($pyvenv | Where-Object { $_ -match '^home\s*=' }) -split '=', 2)[1].Trim() if (-not (Test-Path -LiteralPath $pythonRuntime -PathType Container)) { throw "Python Runtime Root does not exist: $pythonRuntime" } $basePythonPath = Join-Path $pythonRuntime 'python.exe' if (-not (Test-Path -LiteralPath $basePythonPath -PathType Leaf)) { throw "Base Python does not exist: $basePythonPath" } New-Item -ItemType Directory -Force -Path $evidence | Out-Null $runRoot = Join-Path $testRootPath ('agent-mcp-readme-cli-' + [Guid]::NewGuid().ToString('N')) New-Item -ItemType Directory -Path $runRoot | Out-Null $agent = Join-Path $repository 'build\Release\Agent.App\net10.0-windows10.0.26100.0\agent-mcp.exe' $windowsTests = Join-Path $repository 'build\Release\Agent.Windows.Tests\net10.0-windows10.0.26100.0\Agent.Windows.Tests.exe' $configPath = Join-Path $runRoot 'agent.yaml' $logPath = Join-Path $evidence 'verification.log' $failed = $false $aliases = @( ('readme-python-' + [Guid]::NewGuid().ToString('N').Substring(0, 8)) ('readme-powershell-' + [Guid]::NewGuid().ToString('N').Substring(0, 8)) ('readme-cmd-' + [Guid]::NewGuid().ToString('N').Substring(0, 8)) ) $pythonAlias, $powershellAlias, $cmdAlias = $aliases $listener = [Net.Sockets.TcpListener]::new([Net.IPAddress]::Loopback, 0) $listener.Start() $httpPort = ([Net.IPEndPoint]$listener.LocalEndpoint).Port $listener.Stop() $resolutionStorePath = $null $server = $null function Invoke-Step { param([string]$Name, [scriptblock]$Action) "START $Name $(Get-Date -Format o)" | Tee-Object -FilePath $logPath -Append try { $global:LASTEXITCODE = 0 & $Action *>&1 | Tee-Object -FilePath $logPath -Append if ($LASTEXITCODE -ne 0) { throw "$Name exited with $LASTEXITCODE" } "PASS $Name $(Get-Date -Format o)" | Tee-Object -FilePath $logPath -Append } catch { $_ | Out-String | Tee-Object -FilePath $logPath -Append $script:failed = $true "FAIL $Name $(Get-Date -Format o)" | Tee-Object -FilePath $logPath -Append } } function Invoke-ExpectedFailure { param([string]$Name, [string[]]$Arguments) "START $Name $(Get-Date -Format o)" | Tee-Object -FilePath $logPath -Append & $agent @Arguments *>&1 | Tee-Object -FilePath $logPath -Append if ($LASTEXITCODE -ne 2) { $script:failed = $true "FAIL $Name expected_exit=2 actual_exit=$LASTEXITCODE" | Tee-Object -FilePath $logPath -Append } else { "PASS $Name expected_exit=2" | Tee-Object -FilePath $logPath -Append } } function Start-And-ProbeServer { param([bool]$ExplicitServe) $arguments = if ($ExplicitServe) { @('serve', '--config', $configPath, '--lang', 'ja', '--transport', 'http') } else { @('--config', $configPath, '--lang', 'ja', '--transport', 'http') } $script:server = Start-Process -FilePath $agent -ArgumentList $arguments -WorkingDirectory (Split-Path $agent) -WindowStyle Hidden -PassThru $endpoint = "http://127.0.0.1:$httpPort/mcp" for ($attempt = 0; $attempt -lt 100; $attempt++) { if ($script:server.HasExited) { throw "Server exited before becoming ready: $($script:server.ExitCode)" } try { $response = Invoke-WebRequest -Uri $endpoint -Method Get -SkipHttpErrorCheck if ($response.StatusCode -eq 405) { return } } catch { } Start-Sleep -Milliseconds 100 } throw 'Server did not become ready.' } try { $forwardRoot = $testRootPath.Replace('\', '/') $forwardPrompt = (Join-Path $repository 'prompts\user-system.md').Replace('\', '/') $forwardSkills = (Join-Path $repository 'skills').Replace('\', '/') $forwardLocales = (Join-Path $repository 'config\locales').Replace('\', '/') $forwardLogs = (Join-Path $runRoot 'logs').Replace('\', '/') @" schema_version: 1 application: language: en mcp: transport: http http: host: 127.0.0.1 port: $httpPort images: max_count: 4 max_bytes_each: 10485760 max_total_bytes: 20971520 tasks: time_to_live_hours: 24 poll_interval_ms: 1000 agent: user_system_prompt: $forwardPrompt max_iterations: 8 same_failure_limit: 2 ollama: endpoint: http://127.0.0.1:11434 model: deepseek-v4-flash:cloud think: true keep_alive: 5m options: num_ctx: 32768 temperature: 0 extra_options: {} planner: max_tasks: 8 sandbox: profile: development workspaces: - $forwardRoot mounts: [] execution: executables: - $pythonAlias shells: - $powershellAlias - $cmdAlias network: access: none local_endpoints: [] restricted_endpoints: [] resources: profile: constrained file_edit: memory_backup_limit_mb: 16 limits: whole_read_bytes: 1048576 search_max_results: 1000 tail_max_lines: 10000 context_summary_ratio: 0.8 localization: directory: $forwardLocales fallback: en skills: directory: $forwardSkills logging: directory: $forwardLogs level: information http_requests: true max_file_bytes: 10485760 retained_files: 5 "@ | Set-Content -LiteralPath $configPath -Encoding utf8NoBOM @( "timestamp=$(Get-Date -Format o)", "commit=$(git -C $repository rev-parse HEAD)", "administrator=$administrator", "os=$([Environment]::OSVersion.VersionString)", "venv_python=$venvPythonPath", "venv_version=$(& $venvPythonPath --version)", "powershell=$pwshPath", "cmd=$cmdPath" ) | Set-Content -LiteralPath (Join-Path $evidence 'metadata.txt') -Encoding utf8NoBOM Invoke-Step 'cli-sandbox-resolve' { & $agent sandbox resolve --config $configPath --lang ja } Invoke-Step 'cli-sandbox-confirm-yes' { & $agent sandbox confirm --yes --config $configPath --lang ja } $normalizedConfig = [IO.Path]::GetFullPath($configPath).ToUpperInvariant() $hash = [Convert]::ToHexString([Security.Cryptography.SHA256]::HashData([Text.Encoding]::UTF8.GetBytes($normalizedConfig))) $resolutionStorePath = Join-Path $env:LOCALAPPDATA "AgentSandboxMcp\environment-resolutions\$hash.json" Invoke-Step 'cli-executable-resolve-path' { & $agent executable resolve $pythonAlias --path $basePythonPath --config $configPath --lang ja } Invoke-Step 'cli-executable-approve-runtime-root-yes' { & $agent executable approve $pythonAlias --path $basePythonPath --runtime-root $pythonRuntime --yes --config $configPath --lang ja } Invoke-Step 'cli-shell-resolve-powershell' { & $agent executable resolve $powershellAlias --path $pwshPath --config $configPath --lang en } Invoke-Step 'cli-shell-approve-runtime-root-yes' { & $agent executable approve $powershellAlias --path $pwshPath --runtime-root $pwshRuntime --yes --config $configPath --lang en } Invoke-Step 'cli-shell-resolve-cmd' { & $agent executable resolve $cmdAlias --path $cmdPath --config $configPath --lang en } Invoke-Step 'cli-shell-approve-cmd-no-runtime' { & $agent executable approve $cmdAlias --path $cmdPath --no-runtime --yes --config $configPath --lang en } Invoke-Step 'cli-executable-list' { & $agent executable list --config $configPath --lang ja } Invoke-Step 'cli-sandbox-show-text-default-format' { & $agent sandbox show --config $configPath --lang en } Invoke-Step 'cli-sandbox-show-json' { & $agent sandbox show --format json --config $configPath --lang ja } if ($CliOnly) { Invoke-ExpectedFailure 'cli-check-detects-non-admin' @('check', '--config', $configPath, '--lang', 'ja') } else { Invoke-Step 'cli-check' { & $agent check --config $configPath --lang ja } } Invoke-ExpectedFailure 'cli-invalid-transport' @('serve', '--config', $configPath, '--transport', 'invalid') Invoke-ExpectedFailure 'cli-invalid-format' @('sandbox', 'show', '--format', 'xml', '--config', $configPath) Invoke-ExpectedFailure 'cli-management-missing-config' @('sandbox', 'resolve') Invoke-ExpectedFailure 'cli-mutually-exclusive-runtime-options' @('executable', 'approve', $pythonAlias, '--path', $basePythonPath, '--runtime-root', $pythonRuntime, '--no-runtime', '--config', $configPath) if (-not $CliOnly) { Invoke-Step 'cli-serve-explicit-http' { Start-And-ProbeServer $true $script:server.Kill($true) $script:server.WaitForExit() $script:server = $null } Invoke-Step 'cli-serve-implicit-option-first' { Start-And-ProbeServer $false $script:server.Kill($true) $script:server.WaitForExit() $script:server = $null } $env:AGENT_MCP_MONKEY_ROOT = $testRootPath $env:AGENT_MCP_TEST_VENV_PYTHON = $venvPythonPath $env:AGENT_MCP_TEST_POWERSHELL = $pwshPath $env:AGENT_MCP_TEST_POWERSHELL_RUNTIME = $pwshRuntime $env:AGENT_MCP_TEST_CMD = $cmdPath Invoke-Step 'readme-operational-windows-tests' { & $windowsTests -class Agent.Windows.Tests.ReadmeOperationalTests -parallel none -failSkips } } Invoke-Step 'cli-executable-revoke-python' { & $agent executable revoke $pythonAlias --yes --config $configPath --lang ja } Invoke-Step 'cli-executable-revoke-powershell' { & $agent executable revoke $powershellAlias --yes --config $configPath --lang en } Invoke-Step 'cli-executable-revoke-cmd' { & $agent executable revoke $cmdAlias --yes --config $configPath --lang en } } finally { if ($server -and -not $server.HasExited) { $server.Kill($true); $server.WaitForExit() } foreach ($alias in $aliases) { & $agent executable revoke $alias --yes --config $configPath --lang en 2>$null | Out-Null } if ($resolutionStorePath -and (Test-Path -LiteralPath $resolutionStorePath)) { Remove-Item -LiteralPath $resolutionStorePath -Force } if (Test-Path -LiteralPath $runRoot) { $resolvedRunRoot = (Resolve-Path -LiteralPath $runRoot).Path if (-not $resolvedRunRoot.StartsWith($testRootPath.TrimEnd('\') + '\', [StringComparison]::OrdinalIgnoreCase)) { throw 'Refusing to clean a test directory outside TestRoot.' } Remove-Item -LiteralPath $resolvedRunRoot -Recurse -Force } Remove-Item Env:AGENT_MCP_MONKEY_ROOT,Env:AGENT_MCP_TEST_VENV_PYTHON,Env:AGENT_MCP_TEST_POWERSHELL,Env:AGENT_MCP_TEST_POWERSHELL_RUNTIME,Env:AGENT_MCP_TEST_CMD -ErrorAction SilentlyContinue } if ($failed) { throw "README verification failed. Evidence: $evidence" } Write-Output "README verification PASS. Evidence: $evidence"