diff options
Diffstat (limited to 'scripts/Operational-E2E.ps1')
| -rw-r--r-- | scripts/Operational-E2E.ps1 | 271 |
1 files changed, 95 insertions, 176 deletions
diff --git a/scripts/Operational-E2E.ps1 b/scripts/Operational-E2E.ps1 index 4b964f1..b253198 100644 --- a/scripts/Operational-E2E.ps1 +++ b/scripts/Operational-E2E.ps1 @@ -3,122 +3,52 @@ param( [string]$Transport = 'stdio', [ValidateSet('127.0.0.1', '::1')] [string]$HttpHost = '127.0.0.1', - [string]$ReleaseZip = 'build\Release\publish\agent-sandbox-mcp-1.0.0-win-x64.zip' + [string]$ReleaseZip = 'build\Release\publish\agent-sandbox-mcp-1.0.0-win-x64.zip', + [string]$PythonPath ) $ErrorActionPreference = 'Stop' $OutputEncoding = [Text.UTF8Encoding]::new($false) [Console]::OutputEncoding = $OutputEncoding -$originalTemp = $env:TEMP -$originalTmp = $env:TMP -$originalDotnetCliHome = $env:DOTNET_CLI_HOME -$originalFirstTimeExperience = $env:DOTNET_SKIP_FIRST_TIME_EXPERIENCE -$originalTelemetry = $env:DOTNET_CLI_TELEMETRY_OPTOUT -$originalAddGlobalToolsToPath = $env:DOTNET_ADD_GLOBAL_TOOLS_TO_PATH -$originalUserPath = [Environment]::GetEnvironmentVariable('Path', 'User') $repository = (Resolve-Path (Join-Path $PSScriptRoot '..')).Path $zip = (Resolve-Path (Join-Path $repository $ReleaseZip)).Path $root = Join-Path ([IO.Path]::GetTempPath()) ("agent-mcp-operational-e2e-" + [Guid]::NewGuid().ToString('N')) $bundleRoot = Join-Path $root 'bundle' $fixture = Join-Path $root 'fixture' -New-Item -ItemType Directory -Path $bundleRoot,$fixture | Out-Null +$alias = 'python-e2e-' + [Guid]::NewGuid().ToString('N').Substring(0, 12) $passed = $false +$approved = $false +$resolutionStorePath = $null +$server = $null +New-Item -ItemType Directory -Path $bundleRoot,$fixture | Out-Null $fixtureAclBefore = (Get-Acl -LiteralPath $fixture).Sddl try { Expand-Archive -LiteralPath $zip -DestinationPath $bundleRoot $bundle = (Get-ChildItem -LiteralPath $bundleRoot -Directory | Select-Object -First 1).FullName - $e2eSkills = Join-Path $bundle 'e2e-skills' - New-Item -ItemType Directory -Path $e2eSkills | Out-Null - $program = @' -namespace OperationalFixture; - -public static class Program -{ - private const string Value = "BUG"; - public static int Main(string[] args) - { - if (args.Length == 1 && args[0] == "--self-test") return Value == "FIXED" ? 0 : 1; - System.Console.WriteLine(Value); - return 0; + $agent = Join-Path $bundle 'agent-mcp.exe' + $configPath = Join-Path $bundle 'agent.yaml' + if (-not $PythonPath) { + $PythonPath = (& uv python find 3.14).Trim() + if ($LASTEXITCODE -ne 0 -or -not $PythonPath) { throw 'Python 3.14 could not be resolved with uv.' } } -} -'@ - Set-Content -LiteralPath (Join-Path $fixture 'Program.cs') -Value $program -Encoding utf8NoBOM - Set-Content -LiteralPath (Join-Path $fixture 'OperationalFixture.csproj') -Encoding utf8NoBOM -Value @' -<Project Sdk="Microsoft.NET.Sdk"> - <PropertyGroup> - <OutputType>Exe</OutputType> - <TargetFramework>net10.0</TargetFramework> - <UseAppHost>false</UseAppHost> - <UseSharedCompilation>false</UseSharedCompilation> - </PropertyGroup> -</Project> -'@ - $sandboxTemp = Join-Path $fixture '.sandbox-temp' - $sandboxCliHome = Join-Path $fixture '.dotnet-home' - New-Item -ItemType Directory -Path $sandboxTemp,$sandboxCliHome | Out-Null - $env:TEMP = $sandboxTemp - $env:TMP = $sandboxTemp - $env:DOTNET_CLI_HOME = $sandboxCliHome - $env:DOTNET_SKIP_FIRST_TIME_EXPERIENCE = '1' - $env:DOTNET_CLI_TELEMETRY_OPTOUT = '1' - $env:DOTNET_ADD_GLOBAL_TOOLS_TO_PATH = 'false' + $pythonLogical = [IO.Path]::GetFullPath($PythonPath) + $python = (Get-Item -LiteralPath $pythonLogical).Target + if (-not $python) { $python = $pythonLogical } + elseif (-not [IO.Path]::IsPathFullyQualified($python)) { $python = [IO.Path]::GetFullPath((Join-Path (Split-Path $pythonLogical) $python)) } + $python = (Resolve-Path -LiteralPath $python).Path + $pythonVersion = (& $python --version 2>&1 | Out-String).Trim() + if ($pythonVersion -notmatch '^Python 3\.14\.') { throw "Operational E2E requires Python 3.14; resolved '$pythonVersion'." } + $runtimeRoot = Split-Path $python - $dotnet = (Get-Command dotnet).Source - $dotnetRoot = Split-Path $dotnet - $sdkVersion = (& $dotnet --version).Trim() - $csc = Join-Path $dotnetRoot "sdk\$sdkVersion\Roslyn\bincore\csc.dll" - $referenceDirectory = (Get-ChildItem (Join-Path $dotnetRoot 'packs\Microsoft.NETCore.App.Ref') -Directory | - Sort-Object { [Version]$_.Name } -Descending | Select-Object -First 1).FullName + '\ref\net10.0' - $outputDirectory = Join-Path $fixture 'build' - New-Item -ItemType Directory -Path $outputDirectory | Out-Null - $dll = Join-Path $outputDirectory 'OperationalFixture.dll' - Set-Content -LiteralPath (Join-Path $outputDirectory 'OperationalFixture.runtimeconfig.json') -Encoding utf8NoBOM -Value @' -{"runtimeOptions":{"tfm":"net10.0","framework":{"name":"Microsoft.NETCore.App","version":"10.0.0"},"rollForward":"LatestPatch"}} -'@ - $rsp = Join-Path $fixture 'build.rsp' - Set-Content -LiteralPath $rsp -Encoding utf8NoBOM -Value @" --nostdlib+ --target:exe --deterministic+ --debug- --out:"$dll" --r:"$(Join-Path $referenceDirectory 'System.Runtime.dll')" --r:"$(Join-Path $referenceDirectory 'System.Console.dll')" -"$(Join-Path $fixture 'Program.cs')" -"@ - & $dotnet exec $csc "@$rsp" - if ($LASTEXITCODE -ne 0) { throw 'Initial in-process compiler fixture build failed.' } + Set-Content -LiteralPath (Join-Path $fixture 'README.md') -Encoding utf8NoBOM -Value @' +# Graph metrics fixture - $precompute = Join-Path $root 'precompute' - New-Item -ItemType Directory -Path $precompute | Out-Null - $expectedSource = Join-Path $precompute 'Program.cs' - $expectedDll = Join-Path $precompute 'OperationalFixture.dll' - $expectedRsp = Join-Path $precompute 'expected.rsp' - Set-Content -LiteralPath $expectedSource -Value $program.Replace('"BUG"','"FIXED"') -Encoding utf8NoBOM - Set-Content -LiteralPath $expectedRsp -Encoding utf8NoBOM -Value @" --nostdlib+ --target:exe --deterministic+ --debug- --out:"$expectedDll" --r:"$(Join-Path $referenceDirectory 'System.Runtime.dll')" --r:"$(Join-Path $referenceDirectory 'System.Console.dll')" -"$expectedSource" -"@ - & $dotnet exec $csc "@$expectedRsp" - if ($LASTEXITCODE -ne 0) { throw 'Expected fixture hash build failed.' } - $expectedDllHash = (Get-FileHash -LiteralPath $expectedDll -Algorithm SHA256).Hash - $fixtureAclBaseline = @{} - @($fixture) + @(Get-ChildItem -LiteralPath $fixture -Recurse -Force | Select-Object -ExpandProperty FullName) | ForEach-Object { - $fixtureAclBaseline[$_] = (Get-Acl -LiteralPath $_).Sddl - } +Implement `graph_metrics.py` and `test_graph_metrics.py`. The implementation must validate dependencies, +produce a deterministic topological order, detect cycles, and calculate the weighted critical path. +'@ - $dotnetHash = (Get-FileHash -LiteralPath $dotnet -Algorithm SHA256).Hash - $cscHash = (Get-FileHash -LiteralPath $csc -Algorithm SHA256).Hash $fixtureYaml = $fixture.Replace('\','/') - $dotnetYaml = $dotnet.Replace('\','/') $config = @" schema_version: 1 application: @@ -139,7 +69,7 @@ agent: ollama: endpoint: http://127.0.0.1:11434 model: deepseek-v4-flash:cloud - think: false + think: true keep_alive: 5m options: num_ctx: 32768 @@ -148,27 +78,21 @@ ollama: planner: max_tasks: 32 sandbox: - filesystem: - read: [$fixtureYaml] - write: [$fixtureYaml] - execute: [$dotnetYaml, $($csc.Replace('\','/')), $($dll.Replace('\','/'))] - delete: [$fixtureYaml] - process: - allow_shell: false - approvals: - - path: $dotnetYaml - sha256: $dotnetHash - - path: $($csc.Replace('\','/')) - sha256: $cscHash - - path: $($dll.Replace('\','/')) - sha256: $expectedDllHash - max_processes: 8 + profile: development + workspaces: + - $fixtureYaml + mounts: [] + execution: + executables: + - $alias + shells: [] network: - mode: deny - allow: [] + access: none + local_endpoints: [] + restricted_endpoints: [] resources: - memory_mb: 2048 - process_timeout_seconds: 300 + profile: constrained + execution_timeout_seconds: 300 file_edit: memory_backup_limit_mb: 16 limits: @@ -180,95 +104,90 @@ localization: directory: config/locales fallback: en skills: - directory: e2e-skills + directory: skills logging: directory: logs max_file_bytes: 10485760 retained_files: 5 "@ - Set-Content -LiteralPath (Join-Path $bundle 'agent.yaml') -Value $config -Encoding utf8NoBOM - $programPath = (Join-Path $fixture 'Program.cs').Replace('\','/') - $cscPath = $csc.Replace('\','/') - $rspPath = $rsp.Replace('\','/') - $dllPath = $dll.Replace('\','/') - $buildRequest = @" -実作業を行ってください。最終応答は日本語にしてください。 -1. file_readで $programPath を読む。 -2. file_editのoverwriteで、他の内容を維持したまま文字列 \"BUG\" を \"FIXED\" に変更する。 -3. process_runで executable=$dotnetYaml, cwd=$fixtureYaml, args=[\"exec\",\"$cscPath\",\"@$rspPath\"] を実行し、descendantを生成しないin-process C# buildのexit code 0を確認する。 -すべてのtool結果を確認してから完了すること。 -"@ - $testRequest = @" -直前のbuild成果物を独立Taskで実テストしてください。最終応答は日本語にしてください。 -process_runで executable=$dotnetYaml, cwd=$fixtureYaml, args=[\"$dllPath\",\"--self-test\"] を1回実行し、exit code 0を確認してから完了すること。 -"@ + Set-Content -LiteralPath $configPath -Value $config -Encoding utf8NoBOM + + & $agent executable approve $alias --path $pythonLogical --runtime-root $runtimeRoot --yes --config $configPath --lang en + if ($LASTEXITCODE -ne 0) { throw 'Executable approval failed.' } + $approved = $true + & $agent sandbox confirm --yes --config $configPath --lang en + if ($LASTEXITCODE -ne 0) { throw 'Sandbox Environment Resolution confirmation failed.' } + $normalizedConfig = [IO.Path]::GetFullPath($configPath).ToUpperInvariant() + $hash = [Convert]::ToHexString([Security.Cryptography.SHA256]::HashData([Text.Encoding]::UTF8.GetBytes($normalizedConfig))) + $resolutionStorePath = Join-Path $env:LOCALAPPDATA "AgentSandboxMcp\environment-resolutions\$hash.json" + & $agent sandbox show --format json --config $configPath --lang en | Out-Null + if ($LASTEXITCODE -ne 0) { throw 'Effective Sandbox Policy show failed.' } + $request = @" +実作業を行ってください。作業ディレクトリは $fixtureYaml です。 +README.md の仕様に従い、外部パッケージを使わず graph_metrics.py と test_graph_metrics.py を作成してください。 +テストには、正常なDAG、同順位時の決定的な順序、未知dependency、cycle、weighted critical pathを含めてください。 +さらに test_graph_metrics.py 自身が subprocess.run([sys.executable, ...]) で小さな子Pythonを起動して成功を検証するテストを含めてください。 +README.md の読取後、2ファイルの作成、最初のテスト実行、失敗時の修正と再実行は、分割せず単一の実装・検証Taskとして計画してください。 +最後に process_run で executable=$alias, cwd=$fixtureYaml, args=["-m","unittest","discover","-v"] を実行し、全テスト成功を確認してください。 +最終応答は日本語にし、作成ファイルとテスト結果を簡潔に報告してください。 +"@ $clientProject = Join-Path $repository 'scripts\E2EClient\E2EClient.csproj' if ($Transport -eq 'stdio') { - dotnet run --project $clientProject -c Release --no-build -- stdio (Join-Path $bundle 'agent-mcp.exe') (Join-Path $bundle 'agent.yaml') $buildRequest $testRequest - if ($LASTEXITCODE -ne 0) { throw 'E2E-01 stdio client failed.' } + dotnet run --project $clientProject -c Release --no-build -- stdio $agent $configPath $request + if ($LASTEXITCODE -ne 0) { throw 'E2E stdio client failed.' } } else { - $server = Start-Process -FilePath (Join-Path $bundle 'agent-mcp.exe') -ArgumentList @('serve','--config',(Join-Path $bundle 'agent.yaml'),'--lang','ja','--transport','http') -WorkingDirectory $bundle -WindowStyle Hidden -PassThru - try { - $httpAuthority = if ($HttpHost -eq '::1') { '[::1]' } else { $HttpHost } - $endpoint = "http://${httpAuthority}:18080/mcp" - $ready = $false - for ($attempt = 0; $attempt -lt 100; $attempt++) { - try { $response = Invoke-WebRequest -Uri $endpoint -Method Get -SkipHttpErrorCheck; if ($response.StatusCode -eq 405) { $ready=$true; break } } catch { } - Start-Sleep -Milliseconds 100 - } - if (-not $ready) { throw 'HTTP server did not become ready.' } - dotnet run --project $clientProject -c Release --no-build -- http $endpoint unused $buildRequest $testRequest - if ($LASTEXITCODE -ne 0) { throw 'E2E-02 HTTP client failed.' } + $server = Start-Process -FilePath $agent -ArgumentList @('serve','--config',$configPath,'--lang','ja','--transport','http') -WorkingDirectory $bundle -WindowStyle Hidden -PassThru + $httpAuthority = if ($HttpHost -eq '::1') { '[::1]' } else { $HttpHost } + $endpoint = "http://${httpAuthority}:18080/mcp" + $ready = $false + for ($attempt = 0; $attempt -lt 100; $attempt++) { + try { $response = Invoke-WebRequest -Uri $endpoint -Method Get -SkipHttpErrorCheck; if ($response.StatusCode -eq 405) { $ready=$true; break } } catch { } + Start-Sleep -Milliseconds 100 } - finally { if (-not $server.HasExited) { $server.Kill($true); $server.WaitForExit() } } + if (-not $ready) { throw 'HTTP server did not become ready.' } + dotnet run --project $clientProject -c Release --no-build -- http $endpoint unused $request + if ($LASTEXITCODE -ne 0) { throw 'E2E HTTP client failed.' } + if (-not $server.HasExited) { $server.Kill($true); $server.WaitForExit() } + $server = $null } - & $dotnet exec $csc "@$rsp" - if ($LASTEXITCODE -ne 0) { throw 'Post-task fixture build failed.' } - & $dotnet $dll --self-test - if ($LASTEXITCODE -ne 0) { throw 'Post-task fixture self-test failed.' } + & $python -m unittest discover -v -s $fixture + if ($LASTEXITCODE -ne 0) { throw 'Independent Python verification failed.' } + foreach ($required in 'graph_metrics.py','test_graph_metrics.py') { + if (-not (Test-Path -LiteralPath (Join-Path $fixture $required))) { throw "Expected artifact is missing: $required" } + } if ((Get-ChildItem -LiteralPath (Join-Path $bundle 'state\journals') -File -ErrorAction SilentlyContinue | Measure-Object).Count -ne 0) { throw 'Journal residue detected.' } if ((Get-ChildItem -LiteralPath (Join-Path $bundle 'state\backups') -Recurse -File -ErrorAction SilentlyContinue | Measure-Object).Count -ne 0) { throw 'Temporary backup residue detected.' } if ((Get-Acl -LiteralPath $fixture).Sddl -ne $fixtureAclBefore) { throw 'Fixture ACL was not restored byte-for-byte.' } - $bundleExe = (Join-Path $bundle 'agent-mcp.exe') - $orphan = Get-Process -Name 'agent-mcp' -ErrorAction SilentlyContinue | Where-Object { try { $_.Path -eq $bundleExe } catch { $false } } - if ($orphan) { throw 'Orphan agent-mcp process detected.' } - foreach ($entry in $fixtureAclBaseline.GetEnumerator()) { - if ((Get-Acl -LiteralPath $entry.Key).Sddl -ne $entry.Value) { throw "Existing resource ACL was not restored byte-for-byte: $($entry.Key)" } - } - $audit = Get-Content -LiteralPath (Join-Path $bundle 'logs\audit.jsonl') - $auditEvents = $audit | ForEach-Object { $_ | ConvertFrom-Json } + $auditEvents = Get-Content -LiteralPath (Join-Path $bundle 'logs\audit.jsonl') | ForEach-Object { $_ | ConvertFrom-Json } $appContainerSid = ($auditEvents | Where-Object Event -eq 'acl.apply' | Select-Object -First 1).Data.sid - if (-not $appContainerSid) { throw 'Audit did not contain the AppContainer SID.' } - $processStarts = @($auditEvents | Where-Object Event -eq 'process.start') - if (-not ($processStarts | Where-Object { $_.Data.managed_entry_path -eq $csc })) { throw 'Audit is missing the approved in-process compiler execution.' } - if (-not ($processStarts | Where-Object { $_.Data.managed_entry_path -eq $dll })) { throw 'Audit is missing the approved fixture self-test execution.' } + if (-not $appContainerSid) { throw 'Audit is missing the AppContainer SID.' } foreach ($path in @($fixture) + @(Get-ChildItem -LiteralPath $fixture -Recurse -Force | Select-Object -ExpandProperty FullName)) { if ((Get-Acl -LiteralPath $path).Sddl.Contains($appContainerSid, [StringComparison]::OrdinalIgnoreCase)) { throw "AppContainer ACL residue detected: $path" } } - $auditText = $audit -join "`n" - if ($auditText -notmatch 'file.write' -or $auditText -notmatch 'process.start' -or $auditText -notmatch 'network.rule.remove' -or $auditText -notmatch 'acl.restore') { throw 'Audit trace is incomplete.' } + if (-not ($auditEvents | Where-Object Event -eq 'process.start')) { throw 'Audit is missing process.start.' } + if (-not ($auditEvents | Where-Object Event -eq 'process.exit')) { throw 'Audit is missing process.exit.' } + if (-not ($auditEvents | Where-Object { $_.Event -eq 'network.rule.remove' -and $_.Data.residual_filters -eq '0' })) { throw 'Audit is missing zero-residual network.rule.remove.' } + $orphan = Get-Process -Name agent-mcp -ErrorAction SilentlyContinue | Where-Object { try { $_.Path -eq $agent } catch { $false } } + if ($orphan) { throw 'Operational Bundle left an orphan agent-mcp process.' } $passed = $true - Write-Output "Operational E2E $Transport/$HttpHost PASS: $root" + Write-Output "Operational E2E $Transport/$HttpHost PASS with $pythonVersion" } catch { Write-Error "Operational E2E $Transport FAILED. Evidence retained at $root. $($_.Exception.Message)" throw } finally { - $env:TEMP = $originalTemp - $env:TMP = $originalTmp - $env:DOTNET_CLI_HOME = $originalDotnetCliHome - $env:DOTNET_SKIP_FIRST_TIME_EXPERIENCE = $originalFirstTimeExperience - $env:DOTNET_CLI_TELEMETRY_OPTOUT = $originalTelemetry - $env:DOTNET_ADD_GLOBAL_TOOLS_TO_PATH = $originalAddGlobalToolsToPath - $currentUserPath = [Environment]::GetEnvironmentVariable('Path', 'User') - if ($currentUserPath -cne $originalUserPath) { - Write-Error 'Operational E2E changed the persistent user PATH.' + if ($server -and -not $server.HasExited) { $server.Kill($true); $server.WaitForExit() } + if ($approved -and (Test-Path -LiteralPath $agent)) { + & $agent executable revoke $alias --yes --config $configPath --lang en | Out-Null + } + if ($resolutionStorePath -and (Test-Path -LiteralPath $resolutionStorePath)) { + Remove-Item -LiteralPath $resolutionStorePath -Force } - if ($passed) { Remove-Item -LiteralPath $root -Recurse -Force } + if ($passed -and (Test-Path -LiteralPath $root)) { Remove-Item -LiteralPath $root -Recurse -Force } } |